def collection(tmpdir_factory):
fn = tmpdir_factory.mktemp('repo').join(COLLECTION_ID)
collection_path = str(fn)
collection_json = os.path.join(collection_path, 'collection.json')
repo = dvcs.initialize_repository(collection_path, GIT_USER, GIT_MAIL)
ci = identifier.Identifier(collection_path)
collection = models.collection.Collection.new(ci)
collection.public = True
collection.status = 'completed'
collection.write_json()
for oid, public, status in ENTITY_IDS:
oi = identifier.Identifier(id=oid,
base_path=collection.identifier.basepath)
o = models.entity.Entity.new(oi)
o.public = public
o.status = status
o.write_json()
for oid, public, status in FILE_IDS:
oi = identifier.Identifier(id=oid,
base_path=collection.identifier.basepath)
o = models.files.File.new(oi)
o.public = public
o.status = status
o.sha1 = oi.idparts['sha1']
o.write_json()
return collection
def test_is_iaobject():
class Thing():
identifier = None
format = None
thing0 = Thing()
thing0.identifier = identifier.Identifier('ddr-densho-1000-1-1')
thing0.format = 'vh'
out0 = archivedotorg.is_iaobject(thing0)
assert out0 == True
thing1 = Thing()
thing1.identifier = identifier.Identifier('ddr-densho-1000')
thing1.format = 'vh'
out1 = archivedotorg.is_iaobject(thing1)
assert out1 == True
thing2 = Thing()
thing2.identifier = identifier.Identifier('ddr-densho-1000-1-1')
thing2.format = 'img'
out2 = archivedotorg.is_iaobject(thing2)
assert out2 == True
thing3 = Thing()
thing3.identifier = identifier.Identifier('ddr-densho-1000')
thing3.format = 'img'
out3 = archivedotorg.is_iaobject(thing3)
assert out3 == False
def test_fidentifier_parent(self):
ei = identifier.Identifier('ddr-testing-123-4')
si = identifier.Identifier('ddr-testing-123-4-5')
efi = identifier.Identifier('ddr-testing-123-4-master-abc123')
sfi = identifier.Identifier('ddr-testing-123-4-5-master-abc123')
out0 = batch.Importer._fidentifier_parent(efi)
out1 = batch.Importer._fidentifier_parent(sfi)
assert out0.id == ei.id
assert out1.id == si.id
def collection(tmpdir_factory):
fn = tmpdir_factory.mktemp('repo').join(COLLECTION_ID)
collection_path = str(fn)
collection_json = os.path.join(collection_path, 'collection.json')
if os.path.exists(collection_json):
return identifier.Identifier(collection_path).object()
else:
repo = dvcs.initialize_repository(collection_path, GIT_USER, GIT_MAIL)
ci = identifier.Identifier(collection_path)
collection = Collection.new(ci)
collection.save(GIT_USER, GIT_MAIL, AGENT)
return collection
def update_inheritables(parent_object, objecttype, inheritables, cleaned_data):
"""Update specified inheritable fields of child objects using form data.
@param parent_object: Collection or Entity with values to be inherited.
@param cleaned_data: Form cleaned_data from POST.
@returns: tuple List of changed object Ids, list of changed objects' JSON files.
"""
child_ids = []
changed_files = []
# values of selected inheritable fields from parent
field_values = _selected_field_values(parent_object, inheritables)
# load child objects and apply the change
if field_values:
for json_path in _child_jsons(parent_object.path):
child = None
oid = identifier.Identifier(path=json_path)
child = oid.object()
if child:
# set field if exists in child and doesn't already match parent value
changed = False
for field, value in field_values:
if hasattr(child, field):
existing_value = getattr(child, field)
if existing_value != value:
setattr(child, field, value)
changed = True
# write json and add to list of changed IDs/files
if changed:
child.write_json()
if hasattr(child, 'id'): child_ids.append(child.id)
elif hasattr(child, 'basename'):
child_ids.append(child.basename)
changed_files.append(json_path)
return child_ids, changed_files
def test_00_pick_signatures(tmpdir, collection):
paths = util.find_meta_files(collection.path_abs,
recursive=True,
force_read=True)
print('paths')
for x in paths:
print(x)
parents = signatures.choose(paths)
print('parents')
for x in parents:
print(x)
updates = signatures.find_updates(parents)
print('updates')
for x in updates:
print(x)
files_written = signatures.write_updates(updates)
print('files_written')
for x in files_written:
print(x)
for oid, expected in SIGNATURES:
oi = identifier.Identifier(id=oid,
base_path=collection.identifier.basepath)
o = oi.object()
print('expected ', expected)
print('o.signature_id ', o.signature_id)
assert o.signature_id == expected
def file_info(repo):
"""Info from looking at set of metadata files
@param repo: GitPython.Repository object
@returns: dict
"""
data = {}
paths = util.find_meta_files(repo.working_dir,
recursive=True,
force_read=True)
identifiers = [identifier.Identifier(path) for path in paths]
data['total objects'] = len(identifiers)
# model totals
for i in identifiers:
key = '%s objects' % i.model
if not data.get(key):
data[key] = 0
data[key] = data[key] + 1
# role totals
roles = identifier.VALID_COMPONENTS['role']
for role in roles:
key = '%s files' % role
data[key] = 0
for i in identifiers:
if i.model == 'file':
for role in roles:
if role in i.path_abs():
key = '%s files' % role
data[key] = data[key] + 1
return data
def astatus(collection, log):
"""Report git annex status of collection repository.
"""
set_logging(log)
msg = commands.annex_status(identifier.Identifier(collection).object())
exit = 0
click.echo(msg)
def save(path, user, mail, log):
"""Stage and commit specified object(s).
NOTE: Only adds .json files - does not add children!
TODO Let user specify multiple files in path e.g. ddr-testing-123-(1,2,4)
"""
set_logging(log)
oi = identifier.Identifier(path)
# TODO hard-coded models!!
if oi.model == 'collection':
exit, status, updated_files = oi.object().save(user,
mail,
agent=AGENT,
commit=True)
elif oi.model == 'entity':
exit, status, updated_files = oi.object().save(
user,
mail,
agent=AGENT,
collection=oi.collection().object(),
commit=True)
elif oi.model == 'file':
exit, status, updated_files = oi.object().save(
user,
mail,
agent=AGENT,
collection=oi.collection().object(),
parent=oi.parent().object(),
commit=True)
click.echo(status)
sys.exit(exit)
def validate_id(text: str) -> Union[identifier.Identifier, bool]:
try:
i = identifier.Identifier(id=text)
return i
except:
pass
return False
def check(path):
"""Validate JSON correctness of specified object.
TODO better error reporting
TODO Let user specify multiple files in path e.g. ddr-testing-123-(1,2,4)
"""
oi = identifier.Identifier(path)
json.loads(oi.path_abs('json'))
def clone(collection, user, mail, log):
"""Clone existing collection repository from hub server.
"""
set_logging(log)
exit, msg = commands.clone(user, mail, identifier.Identifier(collection),
collection)
click.echo(msg)
sys.exit(exit)
def sync(collection, user, mail, log):
"""Sync collection repository with hub server
"""
set_logging(log)
exit, msg = commands.sync(user, mail,
identifier.Identifier(collection).object())
click.echo(msg)
sys.exit(exit)
def add(role, src, entity, user, mail, log):
"""Add file to entity.
"""
entity = identifier.Identifier(entity).object()
file_, repo, log = entity.add_local_file(src_path=src,
role=role,
data={},
git_name=user,
git_mail=mail,
agent=AGENT)
def next(objectid, model, username, password):
"""Check ID server for next object ID in series.
"""
ic = _idservice_login(username, password)
http_status, reason, new_entity_id = ic.next_object_id(
identifier.Identifier(objectid), model, register=False)
if http_status != 200:
click.echo('Error: %s %s' % (http_status, reason))
sys.exit(1)
click.echo('%s %s: %s' % (http_status, reason, new_entity_id))
def register(objectid, username, password):
"""Register object ID with ID server.
"""
oi = identifier.Identifier(objectid)
ic = _idservice_login(username, password)
http_status, reason, added_ids = ic.register_eids(
cidentifier=oi.collection(), entity_ids=[objectid])
if http_status not in [200, 201]:
click.echo('Error: %s %s' % (http_status, reason))
sys.exit(1)
click.echo('%s %s: %s' % (http_status, reason, ','.join(added_ids)))
def create(collection, user, mail, log):
"""Create a new collection repository.
Technically, this clones a blank collection object from the hub server,
adds initial collection files, and commits.
"""
set_logging(log)
exit, msg = models.Collection.create(identifier.Identifier(collection),
user,
mail,
agent=AGENT)
click.echo(msg)
sys.exit(exit)