def get_pure_org(pure_org_uuid, pure_api_config):
pure_org = None
try:
r = client.get(pure_api_record_type + '/' + pure_org_uuid,
config=pure_api_config)
pure_org = response.transform(pure_api_record_type,
r.json(),
version=pure_api_config.version)
except Exception:
pass
return pure_org
def run(
# Do we need other default functions here?
#extract_api_changes=extract_api_changes,
db_name=db_name,
transaction_record_limit=transaction_record_limit,
experts_etl_logger=None,
pure_api_config=None):
if experts_etl_logger is None:
experts_etl_logger = loggers.experts_etl_logger()
experts_etl_logger.info(
'starting: extracting/loading',
extra={'pure_api_record_type': pure_api_record_type})
if pure_api_config is None:
pure_api_config = Config()
# Capture the current record for each iteration, so we can log it in case of an exception:
latest_change = None
try:
with db.session(db_name) as session:
processed_changes = []
for changes in changes_for_family_ordered_by_uuid_version(
session, 'ExternalPerson'):
latest_change = changes[0]
db_person = get_db_person(session, latest_change.uuid)
# We delete here and continue, because there will be no record
# to download from the Pure API when it has been deleted.
if latest_change.change_type == 'DELETE':
if db_person:
delete_db_person(session, db_person)
processed_changes.extend(changes)
if len(processed_changes) >= transaction_record_limit:
record_changes_as_processed(session, processed_changes)
processed_changes = []
session.commit()
continue
r = None
try:
r = client.get(pure_api_record_type + '/' +
latest_change.uuid,
config=pure_api_config)
except PureAPIHTTPError as e:
if e.response.status_code == 404:
if db_person:
# This record has been deleted from Pure but still exists in our local db:
delete_db_person(session, db_person)
processed_changes.extend(changes)
if len(processed_changes) >= transaction_record_limit:
record_changes_as_processed(
session, processed_changes)
processed_changes = []
session.commit()
else:
experts_etl_logger.error(
f'HTTP error {e.response.status_code} returned during record extraction',
extra={
'pure_uuid': latest_change.uuid,
'pure_api_record_type': pure_api_record_type
})
continue
except PureAPIRequestException as e:
formatted_exception = loggers.format_exception(e)
experts_etl_logger.error(
f'mysterious client request exception encountered during record extraction: {formatted_exception}',
extra={
'pure_uuid': latest_change.uuid,
'pure_api_record_type': pure_api_record_type
})
continue
except Exception:
raise
api_external_person = response.transform(
pure_api_record_type,
r.json(),
version=pure_api_config.version)
delete_merged_records(session, api_external_person)
if db_person and db_person_same_or_newer_than_api_external_person(
session, db_person, api_external_person):
continue
if already_loaded_same_api_external_person(
session, api_external_person):
continue
load_api_external_person(session, api_external_person, r.text)
processed_changes.extend(changes)
if len(processed_changes) >= transaction_record_limit:
record_changes_as_processed(session, processed_changes)
processed_changes = []
session.commit()
record_changes_as_processed(session, processed_changes)
session.commit()
except Exception as e:
formatted_exception = loggers.format_exception(e)
experts_etl_logger.error(
f'exception encountered during record extraction: {formatted_exception}',
extra={
'pure_uuid': latest_change.uuid,
'pure_api_record_type': pure_api_record_type
})
experts_etl_logger.info(
'ending: extracting/loading',
extra={'pure_api_record_type': pure_api_record_type})