def exception_handler(exception, context):
"""Handle Django ValidationError as an accepted exception
Must be set in settings:
>>> REST_FRAMEWORK = {
... # ...
... 'EXCEPTION_HANDLER': 'Authentication.exception_handler.exception_handler',
... # ...
... }
For the parameters, see ``exception_handler``
"""
if isinstance(exception, DjangoValidationError):
if isinstance(exception, DjangoValidationError):
if hasattr(exception, 'message_dict'):
detail = exception.message_dict
elif hasattr(exception, 'message'):
detail = exception.message
elif hasattr(exception, 'messages'):
detail = exception.messages
else:
logging.error("BAD VALIDATION MESSAGE: %s" % exception)
exception = DRFValidationError(detail={"detail": detail})
return drf_exception_handler(exception, context)
def create_presigned_post(s3_client, bucket_name, object_name, fields,
conditions, expiration):
"""Generate a presigned URL S3 POST request to upload a file
:param s3_client: S3 client object
:param bucket_name: string
:param object_name: string
:param fields: Dictionary of prefilled form fields
:param conditions: List of conditions to include in the policy
:param expiration: Time in seconds for the presigned URL to remain valid
:return: Dictionary with the following keys:
url: URL to post to
fields: Dictionary of form fields and values to submit with the POST
:return: None if error.
"""
# Generate a presigned S3 POST URL
try:
response = s3_client.generate_presigned_post(bucket_name,
object_name,
Fields=fields,
Conditions=conditions,
ExpiresIn=expiration)
except ClientError as e:
logging.error(e)
return None
# The response contains the presigned URL and required fields
return response
def find_elements(self, *args, message=''):
try:
WebDriverWait(self.driver, 10).until(
lambda driver: driver.find_element(*args).is_displayed())
return self.driver.find_elements(*args)
except NoSuchElementException:
logging.error(message)
raise
except TimeoutException:
logging.error(message)
raise
def create_cmk(desc='Customer Master Key'):
"""Create a KMS Customer Master Key
The created CMK is a Customer-managed key stored in AWS KMS.
:param desc: key description
:return Tuple(KeyId, KeyArn) where:
KeyId: AWS globally-unique string ID
KeyArn: Amazon Resource Name of the CMK
:return Tuple(None, None) if error
"""
# Create CMK
kms_client = boto3.client('kms')
try:
response = kms_client.create_key(Description=desc)
except ClientError as e:
logging.error(e)
return None, None
# Return the key ID and ARN
return response['KeyMetadata']['KeyId'], response['KeyMetadata']['Arn']
def create_presigned_url(s3_client, bucket_name, object_name, expiration):
"""Generate a presigned URL to share an S3 object
:param:s3_client : s3 client object
:param bucket_name: string
:param object_name: string
:param expiration: Time in seconds for the presigned URL to remain valid
:return: Presigned URL as string. If error, returns None.
"""
# Generate a presigned URL for the S3 object
try:
response = s3_client.generate_presigned_url('get_object',
Params={
'Bucket': bucket_name,
'Key': object_name
},
ExpiresIn=expiration)
except ClientError as e:
logging.error(e)
return None
# The response contains the presigned URL
return response
def retrieve_cmk(desc):
"""Retrieve an existing KMS CMK based on its description
:param desc: Description of CMK specified when the CMK was created
:return Tuple(KeyId, KeyArn) where:
KeyId: CMK ID
KeyArn: Amazon Resource Name of CMK
:return Tuple(None, None) if a CMK with the specified description was
not found
"""
# Retrieve a list of existing CMKs
# If more than 100 keys exist, retrieve and process them in batches
kms_client = boto3.client('kms')
try:
response = kms_client.list_keys()
except ClientError as e:
logging.error(e)
return None, None
done = False
while not done:
for cmk in response['Keys']:
# Get info about the key, including its description
try:
key_info = kms_client.describe_key(KeyId=cmk['KeyArn'])
except ClientError as e:
logging.error(e)
return None, None
# Is this the key we're looking for?
if key_info['KeyMetadata']['Description'] == desc:
return cmk['KeyId'], cmk['KeyArn']
# Are there more keys to retrieve?
if not response['Truncated']:
# No, the CMK was not found
logging.debug('A CMK with the specified description was not found')
done = True
else:
# Yes, retrieve another batch
try:
response = kms_client.list_keys(Marker=response['NextMarker'])
except ClientError as e:
logging.error(e)
return None, None
# All existing CMKs were checked and the desired key was not found
return None, None
print(i);
file.write(user.screen_name.encode(encoding="utf-8", errors="strict")
+";"+ '%1s' % user.location.encode(encoding="utf-8", errors="strict")
+";"+ '%1s' % str(user.created_at)
+";"+ '%1s' % str(user.followers_count) + '\n')
time.sleep(0.5)
#if(i==20):
# break
except tweepy.TweepError:
time.sleep(60 * 15)
continue
except (Timeout, SSLError, ReadTimeoutError, ConnectionError) as e:
logging.warning("Network error occurred. Keep calm and carry on.", str(e))
except Exception as e:
logging.error("Unexpected error!", e)
except TweepError as e:
if 'Failed to send request:' in e.reason:
print("Time out error caught.")
time.sleep(180)
continue
except:
print("blat")
time.sleep(60)
continue
file.close()
# for follower in c:
# #ids.append(follower)
# file.write(str(follower))
# print(follower)
# file.write('\n')
def parse_email(pathname, orig=True):
if path.isdir(pathname):
print(pathname)
emails = []
for child in listdir(pathname):
# only parse visible files
if child[0] != ".":
parse_email(path.join(pathname, child), False)
else:
print("file %s" % pathname)
with open(pathname, encoding="utf8", errors='ignore') as TextFile:
text = TextFile.read().replace("\r", "")
try:
time = time_pattern.search(text).group("data").replace("\n", "")
subject = subject_pattern.search(text).group("data").replace("\n", "")
sender = sender_pattern.search(text).group("data").replace("\n", "")
recipient = recipient_pattern.search(text).group("data").split(", ")
cc = cc_pattern.search(text).group("data").split(", ")
bcc = bcc_pattern.search(text).group("data").split(", ")
msg_start_iter = msg_start_pattern.search(text).end()
try:
msg_end_iter = msg_end_pattern.search(text).start()
message = text[msg_start_iter:msg_end_iter]
except AttributeError: # not a reply
message = text[msg_start_iter:]
message = re.sub("[\n\r]", " ", message)
message = re.sub(" +", " ", message)
except AttributeError:
logging.error("Failed to parse %s" % pathname)
return None
# get user and thread ids
sender_id = get_or_allocate_uid(sender)
recipient_id = [get_or_allocate_uid(u.replace("\n", "")) for u in recipient if u != ""]
cc_ids = [get_or_allocate_uid(u.replace("\n", "")) for u in cc if u != ""]
bcc_ids = [get_or_allocate_uid(u.replace("\n", "")) for u in bcc if u != ""]
thread_id = get_or_allocate_tid(subject)
if thread_id not in thread_users:
thread_users[thread_id] = set()
# maintain list of users involved in thread
users_involved = []
users_involved.append(sender_id)
users_involved.extend(recipient_id)
users_involved.extend(cc_ids)
users_involved.extend(bcc_ids)
thread_users[thread_id] |= set(users_involved)
# maintain list of threads where user is involved
for user in set(users_involved):
if user not in user_threads:
user_threads[user] = set()
user_threads[user].add(thread_id)
entry = {"time": time, "thread": thread_id, "sender": sender_id, "recipient": recipient_id, "cc": cc_ids,
"bcc": bcc_ids, "message": message}
feeds.append(entry)
if orig:
try:
with open(pathname + "/" + 'messages.json', 'w') as f:
json.dump(feeds, f, indent=4)
with open(pathname + "/" + 'users.json', 'w') as f:
json.dump(users, f, indent=4)
with open(pathname + "/" + 'threads.json', 'w') as f:
json.dump(threads, f, indent=4)
with open(pathname + "/" + 'thread-users.json', 'w') as f:
for thread in thread_users:
thread_users[thread] = list(thread_users[thread])
json.dump(thread_users, f, indent=4)
with open(pathname + "/" + 'user-threads.json', 'w') as f:
for user in user_threads:
user_threads[user] = list(user_threads[user])
json.dump(user_threads, f, indent=4)
except IOError:
print("Unable to write to output files, aborting")
exit(1)