def main():
#Commnd line arguments
parser = argparse.ArgumentParser()
parser.add_argument('-apikey', '--isbndbkey', help="Insert ISBNDB apikey")
args = parser.parse_args()
if args.isbndbkey:
print(args.isbndbkey)
else:
print('NO MANUAL API KEY')
for x in range(len(GCISPAR)):
try:
#Extracts book identifier from GCIS#
IDEN = GCISPAR[x]["identifier"]
match = re.search(r'.*/(.*?)\..*?$', GCIS)
if match:
FILETYPE = match.groups()[0]
#HREF = url that leads to book.json in GCIS-DEV
HREF = 'https://gcis-search-stage.jpl.net:3000/{}/{}.json'.format(
FILETYPE, IDEN)
HREFPAR = parse(HREF)
#Extracts book title and isbn from GCIS-DEV
d = dict(HREFPAR)
TITLE = d['title']
ISBNS = d['isbn']
#Cleans ISBNS to only conatian valid characters
CISBN = clean(ISBNS)
#V13 = validated canonical ISBN-13
V13 = EAN13(CISBN)
if V13 is None:
V13 = canonical(CISBN)
M = parse(HREF)
apikey = args.isbndbkey
if args.isbndbkey is None:
apikey = 'XOATAY1G'
data = 'http://isbndb.com/api/v2/json/{}/book/{}'.format(
apikey, V13)
v = parse(data)
GCISDATA = "GCIS-DEV\n\n\t{}\n\n\tisbn_original:{}\n\n\tisbn_mod:{}\n\n".format(
M, ISBNS, V13)
APIDATA = "ISBNDB\n\n\t{}\n\n------------\n\n".format(v)
print("GCIS-DEV\n\n\t", M, '\n\n\t', "isbn_original:", ISBNS,
'\n\n\t', "isbn_mod:", V13, "\n\n")
print("ISBNDB\n\n\t", v, '\n\n')
if v['error']:
file.write(v['error'] + "\n")
else:
pass
#Writing Metadata onto file2
file2.write(GCISDATA)
file2.write(APIDATA)
except:
Error = '\n\t######## PROBLEM #######\n\tTitle:{}\n\tGCIS-ISBN:{}\n\tIdentifier:{}\n\n'.format(
TITLE, ISBNS, IDEN)
print(Error)
file.write(Error)
def marc21_to_identifier_isbn(self, key, value):
"""Get identifier isbn.
identifiers_isbn: 020 $a
"""
if isbn13 := EAN13(value.get('a')):
identifiers = self.get('identifiedBy', [])
identifier = {'type': 'bf:Isbn', 'value': isbn13}
identifiers.append(identifier)
return identifiers
def marc21_to_identifier_isbn(self, key, value):
"""Get identifier isbn.
identifiers:isbn: 020$a
"""
isbn13 = EAN13(value.get('a'))
if isbn13:
identifiers = self.get('identifiers', {})
identifiers['isbn'] = isbn13
return identifiers
else:
return None
def unimarc_identifier_isbn(self, key, value):
"""Get identifier isbn.
identifiers:isbn: 010$a
"""
from isbnlib import EAN13
identifiers = self.get('identifiedBy', [])
if value.get('a'):
ean = {"type": "bf:Ean", "value": value.get('a')}
check_ean = EAN13(value.get('a'))
# Do we have to check also cancelled status?
if not check_ean:
ean['status'] = 'invalid'
identifiers.append(ean)
return identifiers
def unimarc_identifier_isbn_tag073(self, key, value):
"""Get identifier isbn.
identifiers:isbn: 010$a
identified_by.type = bf:Ean = UNIMARC 073
* value = 073$a
* qualifier = 073$b
* ""status"":""invalid or cancelled"" = 073$z
"""
from isbnlib import EAN13
identifiers = self.get('identifiedBy', [])
if value.get('a'):
ean = {"type": "bf:Ean", "value": value.get('a')}
check_ean = EAN13(value.get('a'))
# Do we have to check also cancelled status?
if not check_ean:
ean['status'] = 'invalid'
identifiers.append(ean)
return identifiers
def main():
#Commnd line arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'-path',
'--GCIS',
help=
"Insert url path to GCIS book in JSON format [ex.'https://gcis-search-stage.jpl.net:3000/book.json?all=1'] "
)
args = parser.parse_args()
GCIS = args.GCIS
if GCIS is None:
GCIS = 'https://gcis-search-stage.jpl.net:3000/book.json?all=1'
print(
'NO MANUAL GCIS PATH\n ALL GCIS BOOK JSON FORMATS WILL BE USED AS DEFAULT'
)
GCISPAR = parse(GCIS)
for x in range(len(GCISPAR)):
try:
#Extracts book identifier from GCIS#
IDEN = GCISPAR[x]["identifier"]
match = re.search(r'.*/(.*?)\..*?$', GCIS)
if match:
FILETYPE = match.groups()[0]
#HREF = url that leads to book.json in GCIS-DEV
HREF = 'https://gcis-search-stage.jpl.net:3000/{}/{}.json'.format(
FILETYPE, IDEN)
HREFPAR = parse(HREF)
#Extracts book title and isbn from GCIS-DEV
d = dict(HREFPAR)
TITLE = d['title']
ISBNS = d['isbn']
#Cleans ISBNS to only conatian valid characters
CISBN = clean(ISBNS)
#V13 = validated canonical ISBN-13
V13 = EAN13(CISBN)
if V13 is None:
V13 = canonical(CISBN)
M = parse(HREF)
print("GCIS-DEV\n\n\t", M, '\n\n\t', "isbn_original:", ISBNS,
'\n\n\t', "isbn_mod:", V13, "\n\n")
#DBpedia ISBN formats
a = ISBNS
b = canonical(CISBN)
c = to_isbn10(CISBN)
d = hyphenate(to_isbn10(CISBN))
e = to_isbn13(CISBN)
f = hyphenate(to_isbn13(CISBN))
g = V13
h = "ISBN {}".format(CISBN)
i = "ISBN {}".format(canonical(CISBN))
j = "ISBN {}".format(hyphenate(to_isbn13(CISBN)))
k = "ISBN {}".format(V13)
l = "ISBN {}".format(to_isbn10(CISBN))
m = "ISBN {}".format(hyphenate(to_isbn10(CISBN)))
tests = [a, b, c, d, e, f, g, h, i, j, k, l, m]
for indie in tests:
r = QUERY % indie
RQUERY(r)
if len(RQUERY(r)) != 0:
print(RQUERY(r))
break
except:
Error = '\n\t######## PROBLEM #######\n\tTitle:{}\n\tGCIS-ISBN:{}\n\tIdentifier:{}\n\n'.format(
TITLE, ISBNS, IDEN)
print(Error)
file.write(Error)
def main():
#Commnd line arguments
parser = argparse.ArgumentParser()
parser.add_argument('-log',
'--login',
help="Route path to Gcis.conf YAML file")
parser.add_argument(
'-url',
'--gcis',
help=
'INSERT EITHER: https://data.globalchange.gov OR https://gcis-search-stage.jpl.net:3000'
)
parser.add_argument('-name', '--username', help="Insert GCIS username")
parser.add_argument('-pw',
'--apikey',
help="Insert GCIS username's api key")
args = parser.parse_args()
gcis = 'https://data.globalchange.gov'
gcisdev = 'https://gcis-search-stage.jpl.net:3000'
#Extracts login info from Gcis.conf
if args.login:
a = open(args.login, "r")
list = (yaml.load(a))
diction = list[0]
path = diction['url']
user = diction['userinfo']
key = diction['key']
print(path + '\n' + user + '\n' + key)
else:
pass
if args.gcis == gcis:
print(args.gcis)
elif args.gcis == gcisdev:
print(args.gcis)
else:
print('NO MANUAL ENDPOINT (Ignore if using Config file)')
if args.username:
print(args.username)
else:
print('NO MANUAL USERNAME (Ignore if using Config file)')
if args.apikey:
print(args.apikey)
else:
print('NO MANUAL API KEY (Ignore if using Config file)')
#Credentials
path = diction['url']
if diction['url'] is None:
path = args.gcis
else:
path = gcisdev
user = diction['userinfo']
if diction['userinfo'] is None:
user = args.username
key = diction['key']
if diction['key'] is None:
key = args.apikey
#Parses url.json#
def parse(url):
import requests
r = requests.get(url, verify=False)
JSONdict = r.json()
return JSONdict
GCIS = 'https://gcis-search-stage.jpl.net:3000/book.json?all=1'
GCISPAR = parse(GCIS)
for x in range(len(GCISPAR)):
#Extracts book identifier from GCIS#
IDEN = GCISPAR[x]["identifier"]
match = re.search(r'.*/(.*?)\..*?$', GCIS)
if match:
FILETYPE = match.groups()[0]
#HREF = url that leads to book.json in GCIS-DEV
HREF = 'https://gcis-search-stage.jpl.net:3000/{}/{}.json'.format(
FILETYPE, IDEN)
#HREF for either GCIS or GCIS-DEV
#HREF = '{}//{}/{}.json' .format(path, FILETYPE, IDEN)
#test
#HREF = 'https://gcis-search-stage.jpl.net:3000/book/305e4144-39d2-4d84-8843-3f502ab890e0.json'
HREFPAR = parse(HREF)
print(HREFPAR)
#Extracts book title and isbn from GCIS-DEV
d = dict(HREFPAR)
TITLE = d['title']
ISBNS = d['isbn']
#Cleans ISBNS to only conatian valid characters
CISBN = clean(ISBNS)
#V13 = validated canonical ISBN-13
V13 = EAN13(CISBN)
if V13 is None:
V13 = canonical(CISBN)
M = parse(HREF)
#For possible future implementation of adding original isbn into the JSON dictionary.
"""M["isbn"] = V13
M["org_isbn"] = ISBNS"""
print(M, '\n\t', "isbn_original:", ISBNS)
#Posts updated JSON dictionary back into GCIS-DEV using credentials from command line arguments.
s = requests.Session()
s.auth = (user, key)
s.headers.update({'Accept': 'application/json'})
r = s.post(HREF, data=M, verify=False)
r.raise_for_status()
sys.exit()
def main():
#Commnd line arguments
parser = argparse.ArgumentParser()
parser.add_argument('-awsid', '--AWSAccessKeyID', help = "Insert AWS Access Key ID")
parser.add_argument('-astag', '--AssociateTag', help = "Insert Amazon Associate Tag")
parser.add_argument('-path', '--GCIS', help = "Insert url path to GCIS book in JSON format [ex.'https://gcis-search-stage.jpl.net:3000/book.json?all=1'] ")
args = parser.parse_args()
GCIS = args.GCIS
if args.AWSAccessKeyID:
print(args.AWSAccessKeyID)
else:
print('NO AWS Access Key ID')
if args.AssociateTag:
print(args.AssociateTag)
else:
print('NO Amazon Associate Tag')
if GCIS is None:
GCIS = 'https://gcis-search-stage.jpl.net:3000/book.json?all=1'
print('NO MANUAL GCIS PATH\n ALL GCIS BOOK JSON FORMATS WILL BE USED AS DEFAULT')
GCISPAR = parse(GCIS)
for x in range(len(GCISPAR)):
try:
#Extracts book identifier from GCIS#
IDEN = GCISPAR[x]["identifier"]
match = re.search(r'.*/(.*?)\..*?$', GCIS)
if match:
FILETYPE = match.groups()[0]
#HREF = url that leads to book.json in GCIS-DEV
HREF = 'https://gcis-search-stage.jpl.net:3000/{}/{}.json' .format(FILETYPE,IDEN)
HREFPAR = parse(HREF)
#Extracts book title and isbn from GCIS-DEV
d = dict(HREFPAR)
TITLE = d['title']
ISBNS = d['isbn']
#Cleans ISBNS to only conatian valid characters
CISBN = clean(ISBNS)
#V13 = validated canonical ISBN-13
V13 = EAN13(CISBN)
if V13 is None:
V13 = canonical(CISBN)
M = parse(HREF)
#MV13 = M["isbn"] = V13
#ORGISBN = M["org_isbn"] = ISBNS
locapi = 'http://lx2.loc.gov:210/lcdb?version=1.1&operation=searchRetrieve&query=bath.isbn={}&maximumRecords=1&recordSchema=mods' .format(V13)
results = xmlparse(locapi)
GCISDATA = "GCIS-DEV\n\n\t{}\n\n\tisbn_original:{}\n\n\tisbn_mod:{}\n\n" .format(M, ISBNS, V13)
APIDATA = "AMAZON\n\n\t{}\n\n------------\n\n" .format(results)
print("GCIS-DEV\n\t", M, '\n\n\t', "isbn_original:", ISBNS, '\n\n\t', "isbn_mod:", V13, "\n\n")
print('AMAZON\n\t',results)
file2.write(GCISDATA)
file2.write(APIDATA)
except:
Error = '\n\t######## PROBLEM #######\n\tTitle:{}\n\tGCIS-ISBN:{}\n\tIdentifier:{}\n\n'.format(TITLE, ISBNS, IDEN)
print(Error)
file.write(Error)
match = re.search(r'.*/(.*?)\..*?$', GCIS)
if match:
FILETYPE = match.groups()[0]
#HREF = url that leads to book.json in GCIS-DEV
try:
HREF = 'https://gcis-search-stage.jpl.net:3000/{}/{}.json' .format(FILETYPE,IDEN)
#HREF = 'https://gcis-search-stage.jpl.net:3000/book/13b8b4fc-3de1-4bd8-82aa-7d3a6aa54ad5.json'
HREFPAR = parse(HREF)
#Extracts book title and isbn from GCIS-DEV
d = dict(HREFPAR)
TITLE = d['title']
ISBNS = d['isbn']
#Cleans ISBNS to only conatian valid characters
CISBN = clean(ISBNS)
#V13 = validated canonical ISBN-13
V13 = EAN13(CISBN)
if V13 is None:
V13 = canonical(CISBN)
M = parse(HREF)
v = meta(V13, service = 'wcat', cache ='default')
GCISDATA = "GCIS-DEV\n\n\t{}\n\n\tisbn_original:{}\n\n\tisbn_mod:{}\n\n" .format(M, ISBNS, V13)
APIDATA = "WorldCat\n\n\t{}\n\n------------\n\n" .format(v)
print("GCIS-DEV\n\n\t", M, '\n\n\t', "isbn_original:", ISBNS, '\n\n\t', "isbn_mod:", V13, "\n\n")
print ("WorldCat\n\n\t", v, '\n\n')
file2.write(GCISDATA)
file2.write(APIDATA)
except:
Error = '\n\t######## PROBLEM #######\n\tTitle:{}\n\tGCIS-ISBN:{}\n\tIdentifier:{}\n\n'.format(TITLE, ISBNS, IDEN)
print(Error)
file.write(Error)
#print "hi"
json_item = json.load(item)
book_isbn = json_item['isbn']
if book_isbn is not None:
if book_isbn == "None":
with open("problem_book/" + str(f), 'w') as jsonFile:
jsonFile.write(
json.dumps(json_item,
sort_keys=True,
indent=4,
separators=(',', ': ')))
else:
book_isbn = clean(book_isbn)
#book_isbn = book_isbn.replace("-", "")
#book_isbn = EAN13(book_isbn)
if EAN13(book_isbn) != None:
book_isbn = EAN13(book_isbn)
json_item['isbn'] = book_isbn
print json_item['isbn']
with open("isbn13_book/" + str(f), 'w') as jsonFile:
jsonFile.write(
json.dumps(json_item,
sort_keys=True,
indent=4,
separators=(',', ': ')))
else:
with open("non13_book/" + str(f), 'w') as jsonFile:
jsonFile.write(
json.dumps(json_item,
sort_keys=True,