def test_issue12(t, c_name):
src = '''[
{"id": "1", "c1": 1, "c2": 2, "c3": 3 },
{"id": "2", "c1": 2, "c2": 2, "c3": 3 },
{"id": "3", "c1": 3, "c2": 3, "c3": 3 },
{"id": "4", "c1": 1, "c2": 1, "c3": 1 },
{"id": "5", "c1": 6, "c2": 6, "c3": 6 }
]'''
#dataset.verbose_on() # DEBUG
#dataset.use_strict_dotpath(True) # DEBUG
if dataset.status(c_name) == False:
if not dataset.init(c_name):
err = dataset.error_message()
t.error(f'failed to create {c_name}')
return
objects = json.loads(src)
for obj in objects:
key = obj['id']
if dataset.has_key(c_name, key):
dataset.update(c_name, key, obj)
else:
dataset.create(c_name, key, obj)
f_names = dataset.frames(c_name)
for f_name in f_names:
ok = dataset.delete_frame(c_name, f_name)
if ok == False:
err = dataset.error_message()
t.error(f'Failed to delete {f_name} from {c_name} -> "{err}"')
return
if dataset.has_frame(c_name, f_name) == True:
t.error(
f'Failed to delete frame {c_name} from {c_name}, frame still exists'
)
return
f_name = 'issue12'
dot_paths = [".c1", "c3"]
labels = [".col1", ".col3"]
keys = dataset.keys(c_name)
if not dataset.frame_create(c_name, f_name, keys, dot_paths, labels):
err = dataset.error_message()
t.error(f'failed to create {f_name} from {c_name}, {err}')
if not dataset.has_frame(c_name, f_name):
err = dataset.error_message()
t.error(f'expected frame {f_name} to exists, {err}')
return
f_keys = dataset.frame_keys(c_name, f_name)
if len(f_keys) == 0:
err = dataset.error_message()
t.error(f'expected keys in {f_name}, got zero, {err}')
return
f_objects = dataset.frame_objects(c_name, f_name)
if len(f_objects) == 0:
err = dataset.error_message()
t.error(f'expected objects in {f_name}, got zero, {err}')
return
if not dataset.delete_frame(c_name, f_name):
err = dataset.error_message()
t.error(f'expected to delete {f_name} in {c_name}, {err}')
def get_records(dot_paths, f_name, d_name, keys, labels=None):
if dataset.has_frame(d_name, f_name):
dataset.delete_frame(d_name, f_name)
if labels:
f, err = dataset.frame(d_name, f_name, keys, dot_paths, labels)
if err != "":
print(f"ERROR: Can't create {f_name} in {d_name}, {err}")
else:
# If labels arn't provided, just base on dot path
labels = []
for d in dot_paths:
labels.append(d.split(".")[-1])
f, err = dataset.frame(d_name, f_name, keys, dot_paths, labels)
if err != "":
print(f"ERROR: Can't create {f_name} in {d_name}, {err}")
return dataset.frame_objects(d_name, f_name)
def get_records(dot_paths, f_name, d_name, keys, labels=None, clear=True):
if dataset.has_frame(d_name, f_name):
if clear:
dataset.delete_frame(d_name, f_name)
else:
dataset.frame_refresh(d_name, f_name)
return dataset.frame_objects(d_name, f_name)
if labels:
if not dataset.frame_create(d_name, f_name, keys, dot_paths, labels):
err = dataset.error_message()
print(f"ERROR: Can't create {f_name} in {d_name}, {err}")
else:
# If labels arn't provided, just base on dot path
labels = []
for d in dot_paths:
labels.append(d.split(".")[-1])
if not dataset.frame_create(d_name, f_name, keys, dot_paths, labels):
err = dataset.error_message()
print(f"ERROR: Can't create {f_name} in {d_name}, {err}")
return dataset.frame_objects(d_name, f_name)
def test_frame(t, c_name):
if os.path.exists(c_name):
shutil.rmtree(c_name)
if dataset.init(c_name) == False:
err = dataset.error_message()
t.error(err)
return
data = [{
"id": "A",
"one": "one",
"two": 22,
"three": 3.0,
"four": ["one", "two", "three"]
}, {
"id": "B",
"two": 2000,
"three": 3000.1
}, {
"id": "C"
}, {
"id": "D",
"one": "ONE",
"two": 20,
"three": 334.1,
"four": []
}]
keys = []
dot_paths = ["._Key", ".one", ".two", ".three", ".four"]
labels = ["_Key", "one", "two", "three", "four"]
for row in data:
key = row['id']
keys.append(key)
dataset.create(c_name, key, row)
f_name = 'f1'
if dataset.frame_create(c_name, f_name, keys, dot_paths, labels) == False:
err = dataset.error_message()
t.error(err)
if dataset.frame_reframe(c_name, f_name) == False:
err = dataset.error_message()
t.error(err)
l = dataset.frames(c_name)
if len(l) != 1 or l[0] != 'f1':
t.error(f"expected one frame name, f1, got {l}")
if dataset.delete_frame(c_name, f_name) == False:
err = dataset.error_message()
t.error(f'delete_frame({c_name}, {f_name}), {err}')
if identifiers['type'] == 'xref_doi':
link = 'https://doi.org/' + idv['value']
elif identifiers['type'] == 'doi':
link = 'https://doi.org/' + idv['value']
else:
link = idv['value']
record =\
{'id':uid,'title':title,'journal':journal,'authors':author_list,'identifiers':identifier_list,'affiliations':affiliation_list,'link':link,'year':publication_date.year}
dataset.create(collection, link, record)
#Export to Google Sheet
os.environ['GOOGLE_CLIENT_SECRET_JSON'] = "/etc/client_secret.json"
#Google sheet ID for output
sheet_name = "Sheet1"
sheet_range = "A1:CZ"
f_name = 'f_name'
export_list = [".link", ".title", ".journal", ".year"]
title_list = ["link", "title", "journal", "year"]
keys = dataset.keys(collection)
if dataset.has_frame(collection, f_name):
dataset.delete_frame(collection, f_name)
frame, err = dataset.frame(collection, f_name, keys, export_list, title_list)
if err != '':
print(err)
err = dataset.export_gsheet(collection, f_name, sheet, sheet_name, sheet_range)
if err != '':
print(err)
if subject.links not in dupe.links:
dupe.links += subject.links
print("Total collaborators: ", len(deduped))
collab = 'collaborators.ds'
subprocess.run(['rm', '-rf', collab])
dataset.init(collab)
for d in deduped:
dataset.create(collab, d.ca_id, d.write())
#Export to Google Sheet
os.environ['GOOGLE_CLIENT_SECRET_JSON'] = "/etc/client_secret.json"
#Google sheet ID for output
f_name = 'frm'
sheet_name = "Sheet1"
sheet_range = "A1:CZ"
export_list = [".names", ".years", ".affiliations", ".links"]
title_list = ["name", "years", "affiliations", "links"]
keys = dataset.keys(collab)
if dataset.has_frame(collab, f_name):
dataset.delete_frame(collab, f_name)
frame, err = dataset.frame(collab, f_name, keys, export_list, title_list)
if err != '':
print(err)
err = dataset.export_gsheet(collab, f_name, output_sheet, sheet_name,
sheet_range)
if err != '':
print(err)
def test_frame_objects(t, c_name):
if dataset.status(c_name) == True:
dataset.close(c_name)
if os.path.exists(c_name):
shutil.rmtree(c_name)
if dataset.init(c_name) == False:
err = dataset.error_message()
t.error(f'init({c_name}), {err}')
return
data = [{
"id":
"A",
"nameIdentifiers": [{
"nameIdentifier": "0000-000X-XXXX-XXXX",
"nameIdentifierScheme": "ORCID",
"schemeURI": "http://orcid.org/"
}, {
"nameIdentifier": "H-XXXX-XXXX",
"nameIdentifierScheme": "ResearcherID",
"schemeURI": "http://www.researcherid.com/rid/"
}],
"two":
22,
"three":
3.0,
"four": ["one", "two", "three"]
}, {
"id": "B",
"two": 2000,
"three": 3000.1
}, {
"id": "C"
}, {
"id":
"D",
"nameIdentifiers": [{
"nameIdentifier": "0000-000X-XXXX-XXXX",
"nameIdentifierScheme": "ORCID",
"schemeURI": "http://orcid.org/"
}],
"two":
20,
"three":
334.1,
"four": []
}]
keys = []
dot_paths = [
"._Key", ".nameIdentifiers", ".nameIdentifiers[:].nameIdentifier",
".two", ".three", ".four"
]
labels = [
"id", "nameIdentifiers", "nameIdentifier", "two", "three", "four"
]
for row in data:
key = row['id']
keys.append(key)
err = dataset.create(c_name, key, row)
f_name = 'f1'
if dataset.frame_create(c_name, f_name, keys, dot_paths, labels) == False:
err = dataset.error_message()
t.error(
f'frame_create({c_name}, {f_name}, {keys}, {dot_paths}, {labels}), {err}'
)
return
f_keys = dataset.frame_keys(c_name, f_name)
if len(f_keys) != len(keys):
t.error(f'expected {len(keys)}, got {len(f_keys)}')
if dataset.frame_refresh(c_name, f_name) == False:
err = dataset.error_message()
t.error(f'frame_reframe({c_name}, {f_name}), {err}')
l = dataset.frames(c_name)
if len(l) != 1 or l[0] != 'f1':
t.error(f"expected one frame name, f1, got {l}")
object_result = dataset.frame_objects(c_name, f_name)
if len(object_result) != 4:
t.error(
f'Did not get correct number of objects back, expected 4 got {len(object_result)}, {object_result}'
)
count_nameId = 0
count_nameIdObj = 0
for obj in object_result:
if 'id' not in obj:
t.error('Did not get id in object')
if 'nameIdentifiers' in obj:
count_nameId += 1
for idv in obj['nameIdentifiers']:
if 'nameIdentifier' not in idv:
t.error('Missing part of object')
if 'nameIdentifier' in obj:
count_nameIdObj += 1
if "0000-000X-XXXX-XXXX" not in obj['nameIdentifier']:
t.error('Missing object in complex dot path')
if count_nameId != 2:
t.error(
f"Incorrect number of nameIdentifiers elements, expected 2, got {count_nameId}"
)
if count_nameIdObj != 2:
t.error(
f"Incorrect number of nameIdentifier elements, expected 2, got {count_nameIdObj}"
)
if dataset.delete_frame(c_name, f_name) == False:
err = dataset.error_message()
t.error(f'delete_frame({c_name}, {f_name}), {err}')