Beispiel #1
0
 def test_map_records(self):
     self.count = 0
     def f(r):
         self.count += 1
     with open('test/test.dat', 'rb') as fh:
         pymarc.map_records(f, fh)
         self.assertEqual(self.count, 10, 'map_records appears to work')
Beispiel #2
0
 def test_map_records(self):
     self.count = 0
     def f(r):
         self.count += 1
     with open('test/test.dat', 'rb') as fh:
         pymarc.map_records(f, fh)
         self.assertEqual(self.count, 10, 'map_records appears to work')
Beispiel #3
0
    def test_multi_map_records(self):
        self.count = 0

        def f(r):
            self.count += 1

        pymarc.map_records(f, file('test/test.dat'), file('test/test.dat'))
        self.assertEquals(self.count, 20, 'map_records appears to work')
def main():
    '''Main method. Magic starts here.'''
    # TODO: räknare på allt!
    parser = argparse.ArgumentParser()
    parser.add_argument("records_file", help="path to marc records folder")
    parser.add_argument("result_path", help="path to Instance results file")
    parser.add_argument("okapi_url", help=("OKAPI base url"))
    parser.add_argument("tenant_id", help=("id of the FOLIO tenant."))
    parser.add_argument("username", help=("the api user"))
    parser.add_argument("password", help=("the api users password"))
    parser.add_argument("-holdings_id_dict_path", "-ih", help=(""))
    parser.add_argument("-instance_id_dict_path", "-i", help=(""))
    parser.add_argument("-postgres_dump",
                        "-p",
                        help=("results will be written out for Postgres"
                              "ingestion. Default is JSON"),
                        action="store_true")
    parser.add_argument("-marcxml",
                        "-x",
                        help=("DATA is in MARCXML format"),
                        action="store_true")
    parser.add_argument("-validate",
                        "-v",
                        help=("Validate JSON data against JSON Schema"),
                        action="store_true")
    args = parser.parse_args()
    print('\tresults file:\t', args.result_path)
    print("\tOkapi URL:\t", args.okapi_url)
    print("\tTenanti Id:\t", args.tenant_id)
    print("\tUsername:   \t", args.username)
    print("\tPassword:   \tSecret")
    print("\tinstance idMap will get stored at:\t", args.instance_id_dict_path)
    print("\thold idMap will get stored at:\t", args.holdings_id_dict_path)

    print("File to process: {}".format(args.records_file))
    folio_client = FolioClient(args.okapi_url, args.tenant_id, args.username,
                               args.password)
    instance_id_map = {}
    with open(args.instance_id_dict_path, 'r') as json_file:
        instance_id_map = json.load(json_file)
    print("Number of instances in ID map: {}".format(len(instance_id_map)))
    default_mapper = HoldingsDefaultMapper(folio_client, instance_id_map)
    print("Starting")
    print("Rec./s\t\tTot. recs\t\t")

    with open(args.result_path + '/folio_holdings.json', 'w+') as results_file:
        processor = HoldingsMarcProcessor(default_mapper, folio_client,
                                          results_file, args)
        if args.marcxml:
            pymarc.map_xml(processor.process_record, args.records_file)
        else:
            with open(args.records_file, 'rb') as marc_file:
                pymarc.map_records(processor.process_record, marc_file)
    # wrap up
    print("Done. Wrapping up...")
    processor.wrap_up()
    print("done")
Beispiel #5
0
    def test_map_records(self):
        self.count = 0

        def f(r):
            self.count += 1

        with open("test/test.dat", "rb") as fh:
            pymarc.map_records(f, fh)
            self.assertEqual(self.count, 10, "map_records appears to work")
Beispiel #6
0
 def test_multi_map_records(self):
     self.count = 0
     def f(r):
         self.count += 1
     fh1 = open('test/test.dat', 'rb')
     fh2 = open('test/test.dat', 'rb')
     pymarc.map_records(f, fh1, fh2)
     self.assertEqual(self.count, 20, 'map_records appears to work')
     fh1.close()
     fh2.close()
Beispiel #7
0
 def test_multi_map_records(self):
     self.count = 0
     def f(r):
         self.count += 1
     fh1 = open('test/test.dat', 'rb')
     fh2 = open('test/test.dat', 'rb')
     pymarc.map_records(f, fh1, fh2)
     self.assertEqual(self.count, 20, 'map_records appears to work')
     fh1.close()
     fh2.close()
Beispiel #8
0
    def test_multi_map_records(self):
        self.count = 0

        def f(r):
            self.count += 1

        fh1 = open("test/test.dat", "rb")
        fh2 = open("test/test.dat", "rb")
        pymarc.map_records(f, fh1, fh2)
        self.assertEqual(self.count, 20, "map_records appears to work")
        fh1.close()
        fh2.close()
Beispiel #9
0
 def test_multi_map_records(self):
     self.count = 0
     def f(r):
         self.count += 1
     pymarc.map_records(f, file('test/test.dat'), file('test/test.dat'))
     self.assertEquals(self.count, 20, 'map_records appears to work')
Beispiel #10
0
                logging.warning("#Reading %s as MARCXML" % (arg))
                if (opt.verbose):
                    print "#Reading %s as MARCXML" % (arg)
                fh = open(arg,'rb')
            reader = pymarc.MARCReader(fh)
            pymarc.map_xml(mg.grep, fh)
        else:
            if (re.search(r'\.gz$',arg)):
                logging.warning("#Reading %s as gzipped MARC21" % (arg))
                if (opt.verbose):
                    print "#Reading %s as gzipped MARC21" % (arg)
                fh = gzip.open(arg,'rb')
            else:
                logging.warning("#Reading %s as MARC21" % (arg))
                if (opt.verbose):
                    print "#Reading %s as MARC21" % (arg)
                fh = open(arg,'rb')
            reader = pymarc.MARCReader(fh,to_unicode=True)
            pymarc.map_records(mg.grep, fh)
    except Exception as e:
        # Catch any error, log it and move on to the next file.
        # At least this way we get to look at every file even if
        # only parts of some are examined
        logging.warning("ERROR READING FILE %s, SKIPPING TO NEXT: %s" % (arg,str(e)))
if (len(args)>1):
    print "# %d files" % files
print "# %d records seen, %d matched, %d multi-valued" % (mg.records_seen,mg.records_matched,mg.records_multi)
print "# %d field matches, %d duplicate entries, %d bad entries, %d e-suffixed (ignored)" % (mg.fields_matched,mg.fields_duped,mg.fields_bad,mg.fields_esuffix)

logging.warning("FINISHED at %s" % (datetime.datetime.now()))