def run(event, _context): """ Run the process to retrieve and process Aleph metadata. """ _supplement_event(event) config = setup_pipeline_config(event) if config: time_to_break = datetime.now() + timedelta( seconds=config['seconds-to-allow-for-processing']) print("Will break after ", time_to_break) subject_terms = get_subject_terms_needing_expanded( config.get('website-metadata-tablename'), 1) if 'countToProcess' not in event: event['countToProcess'] = len(subject_terms) print('count of subject terms we need to expand = ', len(subject_terms)) while len(subject_terms): subject = subject_terms.pop(0) expand_subject_term(subject, config.get('website-metadata-tablename')) if datetime.now() > time_to_break: break event['countRemaining'] = len(subject_terms) if len(subject_terms) == 0: event['expandSubjectTermsComplete'] = True if event["expandSubjectTermsExecutionCount"] >= event[ "maximumExpandSubjectTermsExecutions"]: event['expandSubjectTermsComplete'] = True return event
def setUp(self): self.event = {"local": True} self.event['local-path'] = str( Path(__file__).parent.absolute()) + "/../example/" self.config = setup_pipeline_config(self.event) self.config['curate-token'] = 'some_token' time_to_break = datetime.now() + timedelta( seconds=self.config['seconds-to-allow-for-processing']) self.get_curate_metadata_class = GetCurateMetadata( self.config, self.event, time_to_break)
def setUp(self): self.event = {"local": True} self.event['local-path'] = str( Path(__file__).parent.absolute()) + "/../example/" self.config = setup_pipeline_config(self.event) self.create_standard_json_class = CreateStandardJson(self.config) filename = local_folder + 'test/zp38w953h0s_curate.json' with open(filename, 'r') as input_source: self.curate_json = json.load(input_source) filename = local_folder + 'test/zp38w953h0s_preliminary_standard.json' with open(filename, 'r') as input_source: self.preliminary_standard_json = json.load(input_source)
def run(event: dict, context: dict) -> dict: """ Run the process to retrieve and process Aleph metadata. """ _supplement_event(event) config = setup_pipeline_config(event) if config: time_to_break = datetime.now() + timedelta( seconds=config['seconds-to-allow-for-processing']) print("Will break after ", time_to_break) curate_config = load_config_ssm(config['curate_keys_ssm_base']) config.update(curate_config) unique_bendo_items = _get_unique_list_of_bendo_items(config) # print("unique_bendo_items = ", unique_bendo_items) unique_bendo_items = ['zp38w953h0s' ] # , 'pv63fx74g23'] # overwrite for testing _get_detail_for_all_items(config, unique_bendo_items) return event
def run(event, context): """ run the process to retrieve and process web kiosk metadata """ _suplement_event(event) config = setup_pipeline_config(event) google_config = load_config_ssm(config['google_keys_ssm_base']) config.update(google_config) google_credentials = json.loads(config["museum-google-credentials"]) harvest_metadata_rules_class = HarvestMetadataRules(google_credentials) local_folder = os.path.dirname(os.path.realpath(__file__)) + '/' for site_name in event['sites']: harvest_metadata_rules_class.harvest_google_spreadsheet_info(site_name) s3_sync(config["process-bucket"], "sites", local_folder + "sites") try: copy_tree(local_folder + "sites/", local_folder + "../process_manifest/sites/") except EnvironmentError as e: print('Unable to sync sites files to process_manifest ') capture_exception(e)
def run(event: dict, context: dict) -> dict: """ Run the process to retrieve and process Aleph metadata. """ _supplement_event(event) config = setup_pipeline_config(event) if config: time_to_break = datetime.now() + timedelta( seconds=config['seconds-to-allow-for-processing']) print("Will break after ", time_to_break) if event.get('curateExecutionCount', 0) == 1 and not event.get('local', True): save_source_system_record(config.get('website-metadata-tablename'), 'Curate') _save_seed_files_to_s3(config['process-bucket'], 'save') if not config.get('local', True): curate_config = load_config_ssm(config['curate_keys_ssm_base']) config.update(curate_config) save_file_system_record(config.get('website-metadata-tablename'), 'Curate', 'Curate') if not event.get("ids", False): string_list_to_save = _read_harvest_ids_from_json( './source_system_export_ids.json', 'Curate') save_harvest_ids(config, 'Curate', string_list_to_save, config.get('website-metadata-tablename')) event['ids'] = _read_harvest_ids_from_dynamo( config.get('website-metadata-tablename'), 'Curate') event['countToProcess'] = len(event['ids']) if "ids" in event: print("ids to process: ", event["ids"]) curate_api_class = CurateApi(config, event, time_to_break) event[ "curateHarvestComplete"] = curate_api_class.process_curate_items( event["ids"]) event['countRemaining'] = len(event['ids']) if event["curateExecutionCount"] >= event[ "maxCurateExecutions"] and not event["curateHarvestComplete"]: event["curateHarvestComplete"] = True sentry_sdk.capture_message( 'Curate did not complete harvest after maximum executions threshold of ' + str(event["maxCurateExecutions"])) return event
def run(event: dict, _context: dict): """ Run the process to retrieve and process ArchivesSpace metadata. Information on the API can be found here: http://archivesspace.github.io/archivesspace/api/ """ _supplement_event(event) _init_sentry() config = setup_pipeline_config(event) if not event.get("ids", False): string_list_to_save = _read_harvest_ids_from_json('./source_system_export_ids.json') save_harvest_ids(config, 'ArchivesSpace', string_list_to_save, config.get('website-metadata-tablename')) event['ids'] = _read_harvest_ids_from_dynamo(config.get('website-metadata-tablename'), 'ArchivesSpace') event['countToProcess'] = len(event['ids']) start_time = time.time() time_to_break = datetime.now() + timedelta(seconds=config['seconds-to-allow-for-processing']) print("Will break after ", time_to_break) if event.get('archivesSpaceExecutionCount', 0) == 1 and not event.get('local'): save_source_system_record(config.get('website-metadata-tablename'), 'ArchivesSpace') harvest_oai_eads_class = HarvestOaiEads(config) standard_json_helpers_class = StandardJsonHelpers(config) save_standard_json_to_dynamo_class = SaveStandardJsonToDynamo(config) ids = event.get("ids", []) while len(ids) > 0 and datetime.now() < time_to_break: standard_json = harvest_oai_eads_class.get_standard_json_from_archives_space_url(ids[0]) if standard_json: print("ArchivesSpace ead_id = ", standard_json.get("id", ""), " source_system_url = ", ids[0], int(time.time() - start_time), 'seconds.') standard_json = standard_json_helpers_class.enhance_standard_json(standard_json) save_standard_json(config, standard_json) save_standard_json_to_dynamo_class.save_standard_json(standard_json) del ids[0] event['countRemaining'] = len(event['ids']) event['archivesSpaceHarvestComplete'] = (len(ids) == 0) event['eadsSavedToS3'] = os.path.join(config['process-bucket'], config['process-bucket-data-basepath']) if event["archivesSpaceExecutionCount"] >= event["maximumArchivesSpaceExecutions"]: event['archivesSpaceHarvestComplete'] = True return event
def run(event, _context): """ run the process to retrieve and process web kiosk metadata """ _suplement_event(event) config = setup_pipeline_config(event) google_config = load_config_ssm(config['google_keys_ssm_base']) config.update(google_config) museum_config = load_config_ssm(config['museum_keys_ssm_base']) config.update(museum_config) time_to_break = datetime.now() + timedelta( seconds=config['seconds-to-allow-for-processing']) print("Will break after ", time_to_break) mode = event.get("mode", "full") if mode not in ["full", "incremental", "ids"]: mode = "full" json_web_kiosk_class = ProcessWebKioskJsonMetadata(config, event, time_to_break) if event["museumExecutionCount"] == 1: if not event.get('local'): save_file_system_record(config.get('website-metadata-tablename'), 'Google', 'Museum') save_source_system_record(config.get('website-metadata-tablename'), 'EmbARK') composite_json = json_web_kiosk_class.get_composite_json_metadata(mode) museum_image_metadata = json_web_kiosk_class.find_images_for_composite_json_metadata( composite_json) composite_json = CleanUpCompositeJson( composite_json).cleaned_up_content event['countToProcess'] = len(composite_json.get('objects')) write_s3_json(config['process-bucket'], 'museum_composite_metadata.json', composite_json) write_s3_json(config['process-bucket'], 'museum_image_metadata.json', museum_image_metadata) else: composite_json = read_s3_json(config['process-bucket'], 'museum_composite_metadata.json') museum_image_metadata = read_s3_json(config['process-bucket'], 'museum_image_metadata.json') if composite_json: objects_processed = json_web_kiosk_class.process_composite_json_metadata( composite_json, museum_image_metadata) event['museumHarvestComplete'] = _done_processing(composite_json) else: print('No JSON to process') if event["museumExecutionCount"] >= event["maximumMuseumExecutions"]: event['museumHarvestComplete'] = True if event['museumHarvestComplete']: if s3_file_exists(config['process-bucket'], 'museum_composite_metadata.json'): delete_s3_key(config['process-bucket'], 'museum_composite_metadata.json') if s3_file_exists(config['process-bucket'], 'museum_image_metadata.json'): delete_s3_key(config['process-bucket'], 'museum_image_metadata.json') elif composite_json: write_s3_json(config['process-bucket'], 'museum_composite_metadata.json', composite_json) key = 'countHarvestedLoop' + str(event["museumExecutionCount"]) event[key] = objects_processed event['countRemaining'] = len(composite_json.get('objects')) return event
def setUp(self): self.event = {"local": True} self.event['local-path'] = str( Path(__file__).parent.absolute()) + "/../example/" self.config = setup_pipeline_config(self.event)