def test_map_records(self):
self.count = 0
def f(r):
self.count += 1
with open('test/test.dat', 'rb') as fh:
pymarc.map_records(f, fh)
self.assertEqual(self.count, 10, 'map_records appears to work')
def test_map_records(self):
self.count = 0
def f(r):
self.count += 1
with open('test/test.dat', 'rb') as fh:
pymarc.map_records(f, fh)
self.assertEqual(self.count, 10, 'map_records appears to work')
def test_multi_map_records(self):
self.count = 0
def f(r):
self.count += 1
pymarc.map_records(f, file('test/test.dat'), file('test/test.dat'))
self.assertEquals(self.count, 20, 'map_records appears to work')
def main():
'''Main method. Magic starts here.'''
# TODO: räknare på allt!
parser = argparse.ArgumentParser()
parser.add_argument("records_file", help="path to marc records folder")
parser.add_argument("result_path", help="path to Instance results file")
parser.add_argument("okapi_url", help=("OKAPI base url"))
parser.add_argument("tenant_id", help=("id of the FOLIO tenant."))
parser.add_argument("username", help=("the api user"))
parser.add_argument("password", help=("the api users password"))
parser.add_argument("-holdings_id_dict_path", "-ih", help=(""))
parser.add_argument("-instance_id_dict_path", "-i", help=(""))
parser.add_argument("-postgres_dump",
"-p",
help=("results will be written out for Postgres"
"ingestion. Default is JSON"),
action="store_true")
parser.add_argument("-marcxml",
"-x",
help=("DATA is in MARCXML format"),
action="store_true")
parser.add_argument("-validate",
"-v",
help=("Validate JSON data against JSON Schema"),
action="store_true")
args = parser.parse_args()
print('\tresults file:\t', args.result_path)
print("\tOkapi URL:\t", args.okapi_url)
print("\tTenanti Id:\t", args.tenant_id)
print("\tUsername: \t", args.username)
print("\tPassword: \tSecret")
print("\tinstance idMap will get stored at:\t", args.instance_id_dict_path)
print("\thold idMap will get stored at:\t", args.holdings_id_dict_path)
print("File to process: {}".format(args.records_file))
folio_client = FolioClient(args.okapi_url, args.tenant_id, args.username,
args.password)
instance_id_map = {}
with open(args.instance_id_dict_path, 'r') as json_file:
instance_id_map = json.load(json_file)
print("Number of instances in ID map: {}".format(len(instance_id_map)))
default_mapper = HoldingsDefaultMapper(folio_client, instance_id_map)
print("Starting")
print("Rec./s\t\tTot. recs\t\t")
with open(args.result_path + '/folio_holdings.json', 'w+') as results_file:
processor = HoldingsMarcProcessor(default_mapper, folio_client,
results_file, args)
if args.marcxml:
pymarc.map_xml(processor.process_record, args.records_file)
else:
with open(args.records_file, 'rb') as marc_file:
pymarc.map_records(processor.process_record, marc_file)
# wrap up
print("Done. Wrapping up...")
processor.wrap_up()
print("done")
def test_map_records(self):
self.count = 0
def f(r):
self.count += 1
with open("test/test.dat", "rb") as fh:
pymarc.map_records(f, fh)
self.assertEqual(self.count, 10, "map_records appears to work")
def test_multi_map_records(self):
self.count = 0
def f(r):
self.count += 1
fh1 = open('test/test.dat', 'rb')
fh2 = open('test/test.dat', 'rb')
pymarc.map_records(f, fh1, fh2)
self.assertEqual(self.count, 20, 'map_records appears to work')
fh1.close()
fh2.close()
def test_multi_map_records(self):
self.count = 0
def f(r):
self.count += 1
fh1 = open('test/test.dat', 'rb')
fh2 = open('test/test.dat', 'rb')
pymarc.map_records(f, fh1, fh2)
self.assertEqual(self.count, 20, 'map_records appears to work')
fh1.close()
fh2.close()
def test_multi_map_records(self):
self.count = 0
def f(r):
self.count += 1
fh1 = open("test/test.dat", "rb")
fh2 = open("test/test.dat", "rb")
pymarc.map_records(f, fh1, fh2)
self.assertEqual(self.count, 20, "map_records appears to work")
fh1.close()
fh2.close()
def test_multi_map_records(self):
self.count = 0
def f(r):
self.count += 1
pymarc.map_records(f, file('test/test.dat'), file('test/test.dat'))
self.assertEquals(self.count, 20, 'map_records appears to work')
logging.warning("#Reading %s as MARCXML" % (arg))
if (opt.verbose):
print "#Reading %s as MARCXML" % (arg)
fh = open(arg,'rb')
reader = pymarc.MARCReader(fh)
pymarc.map_xml(mg.grep, fh)
else:
if (re.search(r'\.gz$',arg)):
logging.warning("#Reading %s as gzipped MARC21" % (arg))
if (opt.verbose):
print "#Reading %s as gzipped MARC21" % (arg)
fh = gzip.open(arg,'rb')
else:
logging.warning("#Reading %s as MARC21" % (arg))
if (opt.verbose):
print "#Reading %s as MARC21" % (arg)
fh = open(arg,'rb')
reader = pymarc.MARCReader(fh,to_unicode=True)
pymarc.map_records(mg.grep, fh)
except Exception as e:
# Catch any error, log it and move on to the next file.
# At least this way we get to look at every file even if
# only parts of some are examined
logging.warning("ERROR READING FILE %s, SKIPPING TO NEXT: %s" % (arg,str(e)))
if (len(args)>1):
print "# %d files" % files
print "# %d records seen, %d matched, %d multi-valued" % (mg.records_seen,mg.records_matched,mg.records_multi)
print "# %d field matches, %d duplicate entries, %d bad entries, %d e-suffixed (ignored)" % (mg.fields_matched,mg.fields_duped,mg.fields_bad,mg.fields_esuffix)
logging.warning("FINISHED at %s" % (datetime.datetime.now()))
