def test_check_repair(t, collection_name):
t.print("Testing status on", collection_name)
# Make sure we have a left over collection to check and repair
if os.path.exists(collection_name) == True:
shutil.rmtree(collection_name)
if dataset.status(collection_name) == True:
dataset.close(collection_name)
if dataset.init(collection_name) == False:
err = dataset.error_message()
t.error(f'init({collection_name}) failed, {err}')
return
if dataset.status(collection_name) == False:
t.error(
f"Failed, expected dataset.status() == True, got False for {collection_name}"
)
return
if dataset.has_key(collection_name, 'one') == False:
if dataset.create(collection_name, 'one', {"one": 1}) == False:
err = dataset.error_message()
t.error(
f'create({collection_name}, "one", {"one": 1}) failed, {err}')
t.print(f"Testing check on {collection_name}")
# Check our collection
if not (dataset.check(collection_name) == True):
err = dataset.error_message()
t.error(
"Failed, (before break) expected check True, got False for {collection_name} (err: {err})"
)
return
# Break and recheck our collection
print(f"Removing {collection_name}/collection.json to cause a fail")
if os.path.exists(collection_name + "/collection.json"):
os.remove(collection_name + "/collection.json")
print(f"Testing check on (broken) {collection_name}")
if not (dataset.check(collection_name) == False):
err = dataset.error_message()
t.error(
f"Failed, (after break) expected check False got True for {collection_name} (err: {err})"
)
else:
t.print(f"Should have see error output for broken {collection_name}")
# Repair our collection
t.print("Testing repair on", collection_name)
if dataset.repair(collection_name) == False:
err = dataset.error_message()
t.error("Failed, expected repair to return True, got, ", err)
if os.path.exists(os.path.join(collection_name,
"collection.json")) == False:
t.error(
f"Failed, expected recreated {collection_name}/collection.json")
def test_issue12(t, c_name):
src = '''[
{"id": "1", "c1": 1, "c2": 2, "c3": 3 },
{"id": "2", "c1": 2, "c2": 2, "c3": 3 },
{"id": "3", "c1": 3, "c2": 3, "c3": 3 },
{"id": "4", "c1": 1, "c2": 1, "c3": 1 },
{"id": "5", "c1": 6, "c2": 6, "c3": 6 }
]'''
#dataset.verbose_on() # DEBUG
#dataset.use_strict_dotpath(True) # DEBUG
if dataset.status(c_name) == False:
if not dataset.init(c_name):
err = dataset.error_message()
t.error(f'failed to create {c_name}')
return
objects = json.loads(src)
for obj in objects:
key = obj['id']
if dataset.has_key(c_name, key):
dataset.update(c_name, key, obj)
else:
dataset.create(c_name, key, obj)
f_names = dataset.frames(c_name)
for f_name in f_names:
ok = dataset.delete_frame(c_name, f_name)
if ok == False:
err = dataset.error_message()
t.error(f'Failed to delete {f_name} from {c_name} -> "{err}"')
return
if dataset.has_frame(c_name, f_name) == True:
t.error(
f'Failed to delete frame {c_name} from {c_name}, frame still exists'
)
return
f_name = 'issue12'
dot_paths = [".c1", "c3"]
labels = [".col1", ".col3"]
keys = dataset.keys(c_name)
if not dataset.frame_create(c_name, f_name, keys, dot_paths, labels):
err = dataset.error_message()
t.error(f'failed to create {f_name} from {c_name}, {err}')
if not dataset.has_frame(c_name, f_name):
err = dataset.error_message()
t.error(f'expected frame {f_name} to exists, {err}')
return
f_keys = dataset.frame_keys(c_name, f_name)
if len(f_keys) == 0:
err = dataset.error_message()
t.error(f'expected keys in {f_name}, got zero, {err}')
return
f_objects = dataset.frame_objects(c_name, f_name)
if len(f_objects) == 0:
err = dataset.error_message()
t.error(f'expected objects in {f_name}, got zero, {err}')
return
if not dataset.delete_frame(c_name, f_name):
err = dataset.error_message()
t.error(f'expected to delete {f_name} in {c_name}, {err}')
def test_join(t, collection_name):
key = "test_join1"
obj1 = {"one": 1}
obj2 = {"two": 2}
if dataset.status(collection_name) == False:
t.error("Failed, collection status is False,", collection_name)
return
ok = dataset.has_key(collection_name, key)
err = ''
if ok == True:
ok = dataset.update(collection_nane, key, obj1)
else:
ok = dataset.create(collection_name, key, obj1)
if ok == False:
err = dataset.error_message()
t.error(
f'Failed, could not add record for test ({collection_name}, {key}, {obj1}), {err}'
)
return
if dataset.join(collection_name, key, obj2, overwrite=False) == False:
err = dataset.error_message()
t.error(
f'Failed, join for {collection_name}, {key}, {obj2}, overwrite = False -> {err}'
)
obj_result, err = dataset.read(collection_name, key)
if err != '':
t.error(f'Unexpected error for {key} in {collection_name}, {err}')
if obj_result.get('one') != 1:
t.error(f'Failed to join append key {key}, {obj_result}')
if obj_result.get("two") != 2:
t.error(f'Failed to join append key {key}, {obj_result}')
obj2['one'] = 3
obj2['two'] = 3
obj2['three'] = 3
if dataset.join(collection_name, key, obj2, overwrite=True) == False:
err = dataset.error_message()
t.error(
f'Failed to join overwrite {collection_name}, {key}, {obj2}, overwrite = True -> {err}'
)
obj_result, err = dataset.read(collection_name, key)
if err != '':
t.error(f'Unexpected error for {key} in {collection_name}, {err}')
for k in obj_result:
if k != '_Key' and obj_result[k] != 3:
t.error('Failed to update value in join overwrite', k, obj_result)
cmd.append("-username")
cmd.append(eprint_username)
cmd.append("-password")
cmd.append(eprint_password)
cmd.append("-export")
cmd.append("all")
p = run(cmd)
exit_code = p.returncode
if exit_code != 0:
print(f"ERROR: {' '.join(cmd)}, exit code {exit_code}")
sys.exit(1)
c_name = "campuspubs.ds"
ok = dataset.status(c_name)
if ok == False:
err = dataset.init(c_name)
if err != "":
print(f"{c_name}, {err}")
harvest = False
if harvest == True:
username = os.environ["USER"]
password = os.environ["PW"]
returnc = ep_full(
c_name, "https://caltechcampuspubs.library.caltech.edu/", username, password
)
print(returnc)
def test_frame_objects(t, c_name):
if dataset.status(c_name) == True:
dataset.close(c_name)
if os.path.exists(c_name):
shutil.rmtree(c_name)
if dataset.init(c_name) == False:
err = dataset.error_message()
t.error(f'init({c_name}), {err}')
return
data = [{
"id":
"A",
"nameIdentifiers": [{
"nameIdentifier": "0000-000X-XXXX-XXXX",
"nameIdentifierScheme": "ORCID",
"schemeURI": "http://orcid.org/"
}, {
"nameIdentifier": "H-XXXX-XXXX",
"nameIdentifierScheme": "ResearcherID",
"schemeURI": "http://www.researcherid.com/rid/"
}],
"two":
22,
"three":
3.0,
"four": ["one", "two", "three"]
}, {
"id": "B",
"two": 2000,
"three": 3000.1
}, {
"id": "C"
}, {
"id":
"D",
"nameIdentifiers": [{
"nameIdentifier": "0000-000X-XXXX-XXXX",
"nameIdentifierScheme": "ORCID",
"schemeURI": "http://orcid.org/"
}],
"two":
20,
"three":
334.1,
"four": []
}]
keys = []
dot_paths = [
"._Key", ".nameIdentifiers", ".nameIdentifiers[:].nameIdentifier",
".two", ".three", ".four"
]
labels = [
"id", "nameIdentifiers", "nameIdentifier", "two", "three", "four"
]
for row in data:
key = row['id']
keys.append(key)
err = dataset.create(c_name, key, row)
f_name = 'f1'
if dataset.frame_create(c_name, f_name, keys, dot_paths, labels) == False:
err = dataset.error_message()
t.error(
f'frame_create({c_name}, {f_name}, {keys}, {dot_paths}, {labels}), {err}'
)
return
f_keys = dataset.frame_keys(c_name, f_name)
if len(f_keys) != len(keys):
t.error(f'expected {len(keys)}, got {len(f_keys)}')
if dataset.frame_refresh(c_name, f_name) == False:
err = dataset.error_message()
t.error(f'frame_reframe({c_name}, {f_name}), {err}')
l = dataset.frames(c_name)
if len(l) != 1 or l[0] != 'f1':
t.error(f"expected one frame name, f1, got {l}")
object_result = dataset.frame_objects(c_name, f_name)
if len(object_result) != 4:
t.error(
f'Did not get correct number of objects back, expected 4 got {len(object_result)}, {object_result}'
)
count_nameId = 0
count_nameIdObj = 0
for obj in object_result:
if 'id' not in obj:
t.error('Did not get id in object')
if 'nameIdentifiers' in obj:
count_nameId += 1
for idv in obj['nameIdentifiers']:
if 'nameIdentifier' not in idv:
t.error('Missing part of object')
if 'nameIdentifier' in obj:
count_nameIdObj += 1
if "0000-000X-XXXX-XXXX" not in obj['nameIdentifier']:
t.error('Missing object in complex dot path')
if count_nameId != 2:
t.error(
f"Incorrect number of nameIdentifiers elements, expected 2, got {count_nameId}"
)
if count_nameIdObj != 2:
t.error(
f"Incorrect number of nameIdentifier elements, expected 2, got {count_nameIdObj}"
)
if dataset.delete_frame(c_name, f_name) == False:
err = dataset.error_message()
t.error(f'delete_frame({c_name}, {f_name}), {err}')
def test_attachments(t, collection_name):
t.print("Testing attach, attachments, detach and prune")
# Generate two files to attach.
with open('a1.txt', 'w') as text_file:
text_file.write('This is file a1')
with open('a2.txt', 'w') as text_file:
text_file.write('This is file a2')
filenames = ['a1.txt', 'a2.txt']
if dataset.status(collection_name) == False:
t.error("Failed,", collection_name, "missing")
return
keys = dataset.keys(collection_name)
if len(keys) < 1:
t.error("Failed,", collection_name, "should have keys")
return
key = keys[0]
if dataset.attach(collection_name, key, filenames) == False:
err = dataset.error_message()
t.error("Failed, to attach files for", collection_name, key, filenames,
', ', err)
return
l = dataset.attachments(collection_name, key)
if len(l) != 2:
t.error("Failed, expected two attachments for", collection_name, key,
"got", l)
return
#Check that attachments arn't impacted by update
if dataset.update(collection_name, key, {"testing": "update"}) == False:
err = dataset.error_message()
t.error("Failed, to update record", collection_name, key, err)
return
l = dataset.attachments(collection_name, key)
if len(l) != 2:
t.error("Failed, expected two attachments after update for",
collection_name, key, "got", l)
return
if os.path.exists(filenames[0]):
os.remove(filenames[0])
if os.path.exists(filenames[1]):
os.remove(filenames[1])
# First try detaching one file.
if dataset.detach(collection_name, key, [filenames[1]]) == False:
err = dataset.error_message()
t.error("Failed, expected True for", collection_name, key,
filenames[1], ', ', err)
if os.path.exists(filenames[1]):
os.remove(filenames[1])
else:
t.error("Failed to detch", filenames[1], "from", collection_name, key)
# Test explicit filenames detch
if dataset.detach(collection_name, key, filenames) == False:
err = dataset.error_message()
t.error("Failed, expected True for", collection_name, key, filenames,
', ', err)
for fname in filenames:
if os.path.exists(fname):
os.remove(fname)
else:
t.error("Failed, expected", fname, "to be detached from",
collection_name, key)
# Test detaching all files
if dataset.detach(collection_name, key, []) == False:
err = dataset.error_message()
t.error("Failed, expected True for (detaching all)", collection_name,
key, ', ', err)
for fname in filenames:
if os.path.exists(fname):
os.remove(fname)
else:
t.error("Failed, expected", fname, "for detaching all from",
collection_name, key)
if dataset.prune(collection_name, key, [filenames[0]]) == False:
err = dataset.error_messag()
t.error("Failed, expected True for prune", collection_name, key,
[filenames[0]], ', ', err)
l = dataset.attachments(collection_name, key)
if len(l) != 1:
t.error("Failed, expected one file after prune for", collection_name,
key, [filenames[0]], "got", l)
if dataset.prune(collection_name, key, []) == False:
err = dataset.error_message()
t.error("Failed, expected True for prune (all)", collection_name, key,
', ', err)
l = dataset.attachments(collection_name, key)
if len(l) != 0:
t.error("Failed, expected zero files after prune for", collection_name,
key, "got", l)