def _parse_html_file(path_info: FileInfo):
'''Open the html file with the given path then parse the file.'''
root_dir, path, url = path_info
with open(path, 'r') as html:
try:
document = Document(url, root_dir, html).export()
return document
except Exception as ex:
message = 'Problem parsing file ' + path
log_unsuccessful('parse')(message=message, exception=ex)
def parse_html_file(path_info: FileInfo) -> Optional[Dict[str, Any]]:
'''Open the html file with the given path then parse the file.'''
root_dir, path, url = path_info
with open(path, 'r') as html:
try:
document = Document(url, root_dir, html).export()
return document
except Exception as ex:
message = 'Problem parsing file ' + path
log_unsuccessful('parse', message=message, exception=ex)
return None
def _connect_to_s3() -> Any:
authentication_info = AuthenticationInfo.load()
session = boto3.session.Session(
aws_access_key_id=authentication_info.access_key,
aws_secret_access_key=authentication_info.secret_key)
try:
s3 = wait_for_response('Opening connection to s3',
lambda: session.resource('s3'))
return s3
except ClientError as ex:
message = 'Unable to connect to s3.'
log_unsuccessful('connection', message, ex)
def _connect_to_s3() -> Any:
authentication_info = AuthenticationInfo.load()
session = boto3.session.Session(
aws_access_key_id=authentication_info.access_key,
aws_secret_access_key=authentication_info.secret_key)
try:
s3 = wait_for_response(
'Opening connection to s3',
lambda: session.resource('s3')
)
return s3
except ClientError as ex:
message = 'Unable to connect to s3.'
log_unsuccessful('connection', message, ex)
def restore(self):
'''Attempt to reupload the previous version of the file in s3.'''
try:
with open(self.backup_path, 'r') as backup:
wait_for_response(
'Attempting to restore backup to s3',
self.s3.Bucket(self.bucket).put_object,
Key=self.key,
Body=backup.read(),
ContentType='application/json'
)
print('Successfully restored backup to s3.')
except ClientError as ex:
message = ['Unable to restore backup to s3.',
'Search is definitely out of sync.']
message = ' '.join(message)
log_unsuccessful('backup restore')(message, ex)
def _upload(s3, bucket: str, key: str, manifest: str) -> None:
try:
wait_for_response(
'Attempting to upload to s3 with key: ' + key,
lambda: s3.Bucket(bucket).put_object(
Key=key, Body=manifest, ContentType='application/json')
)
success_message = ('Successfully uploaded manifest '
'to {0} as {1}').format(bucket, key)
print(success_message)
except ParamValidationError as ex:
message = ' '.join(['Unable to upload to s3.'
'This is likely due to a bad manifest file.'
'Check the file type and syntax.'])
log_unsuccessful('upload', message, ex)
except ClientError as ex:
message = 'Unable to upload to s3.'
log_unsuccessful('upload', message, ex)
def _upload(s3, bucket: str, key: str, manifest: str) -> None:
try:
wait_for_response(
'Attempting to upload to s3 with key: ' + key,
lambda: s3.Bucket(bucket).put_object(
Key=key, Body=manifest, ContentType='application/json'))
success_message = ('Successfully uploaded manifest '
'to {0} as {1}').format(bucket, key)
print(success_message)
except ParamValidationError as ex:
message = ' '.join([
'Unable to upload to s3.'
'This is likely due to a bad manifest file.'
'Check the file type and syntax.'
])
log_unsuccessful('upload', message, ex)
except ClientError as ex:
message = 'Unable to upload to s3.'
log_unsuccessful('upload', message, ex)
def create(self):
'''Creates the backup file.'''
try:
bucket = self.s3.Bucket(self.bucket)
if self.key in [obj.key for obj in bucket.objects.all()]:
wait_for_response(
'Backing up current manifest from s3',
bucket.download_file,
self.key,
self.backup_path
)
print('Successfully backed up current manifest from s3.')
else:
print(' '.join([
'No object named',
self.key,
'found in s3. No backup made.'
]))
return True
except ClientError as ex:
message = 'Unable to backup current manifest from s3.'
log_unsuccessful('backup')(message, ex, exit=False)
return False
def __init__(self, exception, message) -> None:
super(FailedRefreshError, self).__init__()
log_unsuccessful('refresh')(message=message,
exception=exception,
exit=False)
def __init__(self) -> None:
message = 'No documents were found.'
log_unsuccessful('index')(message=message, exception=None)
def __init__(self) -> None:
message = 'No documents were found.'
log_unsuccessful('index', message=message, exception=None)