def _connect_to_s3():
try:
s3 = wait_for_response('Opening connection to s3', boto3.resource,
's3')
print('Successfully connected to s3.')
return s3
except ClientError as ex:
message = 'Unable to connect to s3.'
log_unsuccessful('connection')(message, ex)
def _parse_html_file(path_info):
'''Open the html file with the given path then parse the file.'''
file, file_dir, url = path_info
with open(file_dir + file, 'r') as html:
try:
document = Document(url, file_dir, html).export()
return document
except Exception as ex:
message = 'Problem parsing file ' + file
log_unsuccessful('parse')(message=message, exception=ex)
def restore(self):
'''Attempt to reupload the previous version of the file in s3.'''
try:
with open(self.backup_path, 'r') as backup:
wait_for_response('Attempting to restore backup to s3',
boto3.resource('s3').Bucket(
self.bucket).put_object,
Key=self.key,
Body=backup.read(),
ContentType='application/json')
print('Successfully restored backup to s3.')
except ClientError as ex:
message = [
'Unable to restore backup to s3.',
'Search is definitely out of sync.'
]
message = ' '.join(message)
log_unsuccessful('backup restore')(message, ex)
def create(self):
'''Creates the backup file.'''
try:
bucket = boto3.resource('s3').Bucket(self.bucket)
if self.key in [obj.key for obj in bucket.objects.all()]:
wait_for_response('Backing up current manifest from s3',
bucket.download_file, self.key,
self.backup_path)
print('Successfully backed up current manifest from s3.')
else:
print(' '.join([
'No object named', self.key, 'found in s3. No backup made.'
]))
return True
except ClientError as ex:
message = 'Unable to backup current manifest from s3.'
log_unsuccessful('backup')(message, ex, exit=False)
return False
def _upload(s3, bucket, key, manifest):
try:
wait_for_response('Attempting to upload to s3 with key: ' + key,
s3.Bucket(bucket).put_object,
Key=key,
Body=manifest,
ContentType='application/json')
success_message = ('Successfully uploaded manifest '
'to {0} as {1}').format(bucket, key)
print(success_message)
except ParamValidationError as ex:
message = ' '.join([
'Unable to upload to s3.'
'This is likely due to a bad manifest file.'
'Check the file type and syntax.'
])
log_unsuccessful('upload')(message, ex)
except ClientError as ex:
message = 'Unable to upload to s3.'
log_unsuccessful('upload')(message, ex)
def __init__(self, exception, message):
super(FailedRefreshError, self).__init__()
log_unsuccessful('refresh')(message=message,
exception=exception,
exit=False)
def __init__(self):
message = 'No documents were found.'
log_unsuccessful('index')(message=message, exception=None)