def harvest_by_set(corpus, target, target_set):
logging.info("harvest_by_set: {}".format(target_set))
date_begin = datetime.datetime.now()
# harvest
metajson_list = oaipmh_harvester.list_records(target, None, None, target_set)
date_harvest = datetime.datetime.now()
chrono.chrono_trace("harvest spire and convert to metajson", date_begin, date_harvest, len(ids))
# import
result_import = corpus_service.import_metajson_list(corpus, metajson_list, True, None)
date_import = datetime.datetime.now()
chrono.chrono_trace("harvest spire, convert metadata and save to MongoDB", date_harvest, date_import, len(result_import[0]))
def harvest_by_ids(corpus, target, ids):
logging.info("harvest_by_ids: {}".format(ids))
date_begin = datetime.datetime.now()
# harvest
metajson_list = []
for identifier in ids:
metajson_list.append(oaipmh_harvester.get_record(target, identifier))
date_harvest = datetime.datetime.now()
chrono.chrono_trace("harvest spire and convert to metajson", date_begin, date_harvest, len(ids))
# import
result_import = corpus_service.import_metajson_list(corpus, metajson_list, True, None)
date_import = datetime.datetime.now()
chrono.chrono_trace("import", date_harvest, date_import, len(result_import))
def harvest_by_set(corpus, target, target_set):
logging.info("harvest_by_set: {}".format(target_set))
date_begin = datetime.datetime.now()
# harvest
metajson_list = oaipmh_harvester.list_records(target, None, None,
target_set)
date_harvest = datetime.datetime.now()
chrono.chrono_trace("harvest spire and convert to metajson", date_begin,
date_harvest, len(ids))
# import
result_import = corpus_service.import_metajson_list(
corpus, metajson_list, True, None)
date_import = datetime.datetime.now()
chrono.chrono_trace("harvest spire, convert metadata and save to MongoDB",
date_harvest, date_import, len(result_import[0]))
def harvest_by_ids(corpus, target, ids):
logging.info("harvest_by_ids: {}".format(ids))
date_begin = datetime.datetime.now()
# harvest
metajson_list = []
for identifier in ids:
metajson_list.append(oaipmh_harvester.get_record(target, identifier))
date_harvest = datetime.datetime.now()
chrono.chrono_trace("harvest spire and convert to metajson", date_begin,
date_harvest, len(ids))
# import
result_import = corpus_service.import_metajson_list(
corpus, metajson_list, True, None)
date_import = datetime.datetime.now()
chrono.chrono_trace("import", date_harvest, date_import,
len(result_import))
