def create(collection, entity_id, git_name, git_mail, agent=settings.AGENT):
"""create new entity given an entity ID
"""
repo,org,cid,eid = entity_id.split('-')
entity_path = Entity.entity_path(None, repo, org, cid, eid)
# write entity.json template to entity location and commit
Entity(entity_path).dump_json(path=settings.TEMPLATE_EJSON, template=True)
exit,status = commands.entity_create(
git_name, git_mail,
collection.path, entity_id,
[collection.json_path_rel, collection.ead_path_rel],
[settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
agent=agent)
# load new entity, inherit values from parent, write and commit
entity = Entity.from_json(entity_path)
entity.inherit(collection)
entity.dump_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection.path, entity.id,
updated_files,
agent=agent)
# delete cache, update search index
collection.cache_delete()
with open(entity.json_path, 'r') as f:
document = json.loads(f.read())
docstore.post(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, document)
return entity
def create(identifier, git_name, git_mail, agent='cmdln'):
"""Creates new Entity, writes files, performs initial commit
@param identifier: Identifier
@param git_name: str
@param git_mail: str
@param agent: str
@returns: exit,status int,str
"""
collection = identifier.collection().object()
if not collection:
raise Exception('Parent collection for %s does not exist.' %
identifier)
exit, status = commands.entity_create(
user_name=git_name,
user_mail=git_mail,
collection=collection,
eidentifier=identifier,
updated_files=[collection.json_path_rel, collection.ead_path_rel],
agent=agent)
if exit:
raise Exception('Could not create new Entity: %s, %s' %
(exit, status))
# load Entity object, inherit values from parent, write back to file
entity = Entity.from_identifier(identifier)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit, status = commands.entity_update(git_name,
git_mail,
collection,
entity,
updated_files,
agent=agent)
return exit, status
def test_10_entity_create(tmpdir, test_paths):
"""Create new entity in the collection
collection = identifier.Identifier(test_paths['TEST_COLLECTION']).object()
collection.notes = 'testing testing'
collection.write_json()
updated_files = [
collection.identifier.path_abs('json')
]
"""
collection = identifier.Identifier(test_paths['TEST_COLLECTION']).object()
print(collection)
for eid in TEST_EIDS:
ei = identifier.Identifier(eid, collection.identifier.basepath)
entity = models.entity.Entity.new(ei)
print(entity)
exit,status = commands.entity_create(
GIT_USER, GIT_MAIL,
collection,
entity.identifier,
[
collection.json_path_rel,
collection.ead_path_rel,
],
agent=AGENT
)
print(exit,status)
# confirm entity files exist
assert os.path.exists(test_paths['COLLECTION_FILES'])
for eid in TEST_EIDS:
entity_path = os.path.join(test_paths['COLLECTION_FILES'],eid)
assert os.path.exists(entity_path)
assert os.path.exists(os.path.join(entity_path,'changelog'))
assert os.path.exists(os.path.join(entity_path,'control'))
assert os.path.exists(os.path.join(entity_path,'mets.xml'))
# TODO test contents of entity files
print('entity files exist')
# confirm entities in changelog
changelog_entries = []
for eid in TEST_EIDS:
changelog_entries.append('* Initialized entity {}'.format(eid))
changelog = None
with open(test_paths['COLLECTION_CHANGELOG'],'r') as cl:
changelog = cl.read()
for entry in changelog_entries:
assert entry in changelog
print('entities in changelog')
# confirm entities in ead.xml
assert os.path.exists(test_paths['COLLECTION_EAD'])
def new(identifier, git_name, git_mail, agent='cmdln'):
"""Creates new File (metadata only!), writes to filesystem, performs initial commit
@param identifier: Identifier
@param git_name: str
@param git_mail: str
@param agent: str
@returns: exit,status int,str
"""
parent = identifier.parent().object()
if not parent:
raise Exception('Parent for %s does not exist.' % identifier)
file_ = File.create(identifier.path_abs(), identifier)
file_.write_json()
entity_file_edit(request, collection, file_, git_name, git_mail)
exit, status = commands.entity_create(
git_name,
git_mail,
collection,
entity.identifier,
[collection.json_path_rel, collection.ead_path_rel],
[config.TEMPLATE_EJSON, config.TEMPLATE_METS],
agent=agent)
if exit:
raise Exception('Could not create new Entity: %s, %s' %
(exit, status))
# load Entity object, inherit values from parent, write back to file
entity = Identifier(identifier).object()
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit, status = commands.entity_update(git_name,
git_mail,
collection,
entity,
updated_files,
agent=agent)
return exit, status
def new(identifier, git_name, git_mail, agent='cmdln'):
"""Creates new File (metadata only!), writes to filesystem, performs initial commit
@param identifier: Identifier
@param git_name: str
@param git_mail: str
@param agent: str
@returns: exit,status int,str
"""
parent = identifier.parent().object()
if not parent:
raise Exception('Parent for %s does not exist.' % identifier)
file_ = File.create(identifier)
file_.write_json()
entity_file_edit(request, collection, file_, git_name, git_mail)
exit,status = commands.entity_create(
git_name, git_mail,
collection, entity.identifier,
[collection.json_path_rel, collection.ead_path_rel],
[config.TEMPLATE_EJSON, config.TEMPLATE_METS],
agent=agent
)
if exit:
raise Exception('Could not create new Entity: %s, %s' % (exit, status))
# load Entity object, inherit values from parent, write back to file
entity = Identifier(identifier).object()
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=agent
)
return exit,status
def new(identifier, git_name, git_mail, agent='cmdln'):
"""Creates new Entity, writes to filesystem, does initial commit.
@param identifier: Identifier
@param git_name: str
@param git_mail: str
@param agent: str
@returns: exit,status int,str
"""
collection = identifier.collection().object()
if not collection:
raise Exception('Parent collection for %s does not exist.' % identifier)
entity = Entity.create(identifier.path_abs(), identifier)
fileio.write_text(
entity.dump_json(template=True),
config.TEMPLATE_EJSON
)
exit,status = commands.entity_create(
git_name, git_mail,
collection, entity.identifier,
[collection.json_path_rel, collection.ead_path_rel],
[config.TEMPLATE_EJSON, config.TEMPLATE_METS],
agent=agent
)
if exit:
raise Exception('Could not create new Entity: %s, %s' % (exit, status))
# load Entity object, inherit values from parent, write back to file
entity = Entity.from_identifier(identifier)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=agent
)
return exit,status
def create(collection, entity_id, git_name, git_mail, agent=settings.AGENT):
"""create new entity given an entity ID
TODO remove write and commit, just create object
"""
eidentifier = Identifier(id=entity_id)
entity_path = eidentifier.path_abs()
# write entity.json template to entity location and commit
fileio.write_text(Entity(entity_path).dump_json(template=True),
settings.TEMPLATE_EJSON)
exit,status = commands.entity_create(
git_name, git_mail,
collection, eidentifier,
[collection.json_path_rel, collection.ead_path_rel],
[settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
agent=agent)
# load new entity, inherit values from parent, write and commit
entity = Entity.from_json(entity_path)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=agent)
# delete cache, update search index
collection.cache_delete()
with open(entity.json_path, 'r') as f:
document = json.loads(f.read())
try:
docstore.post(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, document)
except ConnectionError:
logger.error('Could not post to Elasticsearch.')
return entity
def new(identifier, git_name, git_mail, agent='cmdln'):
"""Creates new Entity, writes to filesystem, does initial commit.
@param identifier: Identifier
@param git_name: str
@param git_mail: str
@param agent: str
@returns: exit,status int,str
"""
collection = identifier.collection().object()
if not collection:
raise Exception('Parent collection for %s does not exist.' % identifier)
entity = Entity.create(identifier)
fileio.write_text(
entity.dump_json(template=True),
config.TEMPLATE_EJSON
)
exit,status = commands.entity_create(
git_name, git_mail,
collection, entity.identifier,
[collection.json_path_rel, collection.ead_path_rel],
[config.TEMPLATE_EJSON, config.TEMPLATE_METS],
agent=agent
)
if exit:
raise Exception('Could not create new Entity: %s, %s' % (exit, status))
# load Entity object, inherit values from parent, write back to file
entity = Entity.from_identifier(identifier)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=agent
)
return exit,status
def import_entities(csv_path, collection_path, git_name, git_mail):
"""
@param csv_path: Absolute path to CSV data file.
@param collection_path: Absolute path to collection repo.
@param git_name: Username for use in changelog, git log
@param git_mail: User email address for use in changelog, git log
"""
rows = read_csv(csv_path)
headers = rows[0]
rows = rows[1:]
headers = replace_variant_headers("entity", headers)
field_names = FIELD_NAMES["entity"]
nonrequired_fields = REQUIRED_FIELDS_EXCEPTIONS["entity"]
required_fields = get_required_fields(ENTITY_FIELDS, nonrequired_fields)
rows = replace_variant_cv_field_values("entity", headers, rows)
invalid_headers("entity", headers, field_names, nonrequired_fields)
all_rows_valid("entity", headers, required_fields, rows)
if True:
collection = Collection.from_json(collection_path)
print(collection)
# --------------------------------------------------
def prep_creators(data):
return [x.strip() for x in data.strip().split(";") if x]
def prep_language(data):
"""language can be 'eng', 'eng;jpn', 'eng:English', 'jpn:Japanese'
"""
y = []
for x in data.strip().split(";"):
if ":" in x:
y.append(x.strip().split(":")[0])
else:
y.append(x.strip())
return y
def prep_topics(data):
return [x.strip() for x in data.strip().split(";") if x]
def prep_persons(data):
return [x.strip() for x in data.strip().split(";") if x]
def prep_facility(data):
return [x.strip() for x in data.strip().split(";") if x]
# --------------------------------------------------
print("Data file looks ok")
started = datetime.now()
print("%s starting import" % dtfmt(started))
print("")
for n, row in enumerate(rows):
rowstarted = datetime.now()
rowd = make_row_dict(headers, row)
# create new entity
entity_id = rowd["id"]
entity_path = os.path.join(collection_path, COLLECTION_FILES_PREFIX, entity_id)
# write entity.json template to entity location
fileio.write_text(Entity(entity_path).dump_json(template=True), TEMPLATE_EJSON)
# commit files
exit, status = commands.entity_create(
git_name,
git_mail,
collection.path,
entity_id,
[collection.json_path_rel, collection.ead_path_rel],
[TEMPLATE_EJSON, TEMPLATE_METS],
agent=AGENT,
)
# reload newly-created Entity object
entity = Entity.from_json(entity_path)
# preppers
rowd["creators"] = prep_creators(rowd["creators"])
rowd["language"] = prep_language(rowd["language"])
rowd["topics"] = prep_topics(rowd["topics"])
rowd["persons"] = prep_persons(rowd["persons"])
rowd["facility"] = prep_facility(rowd["facility"])
# insert values from CSV
for key in rowd.keys():
setattr(entity, key, rowd[key])
entity.record_created = datetime.now()
entity.record_lastmod = datetime.now()
# write back to file
entity.write_json()
updated_files = [entity.json_path]
exit, status = commands.entity_update(
git_name, git_mail, entity.parent_path, entity.id, updated_files, agent=AGENT
)
rowfinished = datetime.now()
rowelapsed = rowfinished - rowstarted
print("%s %s/%s %s (%s)" % (dtfmt(rowfinished), n + 1, len(rows), entity.id, rowelapsed))
finished = datetime.now()
elapsed = finished - started
print("")
print("%s done (%s rows)" % (dtfmt(finished), len(rows)))
print("%s elapsed" % elapsed)
print("")
def new( request, repo, org, cid ):
"""Gets new EID from workbench, creates new entity record.
If it messes up, goes back to collection.
"""
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not (git_name and git_mail):
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_request(request)
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect(collection.absolute_url())
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect(collection.absolute_url())
ic = idservice.IDServiceClient()
# resume session
auth_status,auth_reason = ic.resume(request.session['idservice_token'])
if auth_status != 200:
request.session['idservice_username'] = None
request.session['idservice_token'] = None
messages.warning(
request,
'Session resume failed: %s %s (%s)' % (
auth_status,auth_reason,settings.IDSERVICE_API_BASE
)
)
return HttpResponseRedirect(collection.absolute_url())
# get new entity ID
http_status,http_reason,new_entity_id = ic.next_object_id(
collection.identifier,
'entity'
)
if http_status not in [200,201]:
err = '%s %s' % (http_status, http_reason)
msg = WEBUI_MESSAGES['VIEWS_ENT_ERR_NO_IDS'] % (settings.IDSERVICE_API_BASE, err)
logger.error(msg)
messages.error(request, msg)
return HttpResponseRedirect(collection.absolute_url())
eidentifier = Identifier(id=new_entity_id)
# create new entity
entity_path = eidentifier.path_abs()
# write entity.json template to entity location
fileio.write_text(
Entity(entity_path).dump_json(template=True),
settings.TEMPLATE_EJSON
)
# commit files
exit,status = commands.entity_create(
git_name, git_mail,
collection, eidentifier,
[collection.json_path_rel, collection.ead_path_rel],
[settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
agent=settings.AGENT
)
# load Entity object, inherit values from parent, write back to file
entity = Entity.from_identifier(eidentifier)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=settings.AGENT
)
collection.cache_delete()
if exit:
logger.error(exit)
logger.error(status)
messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
else:
# update search index
try:
entity.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
except ConnectionError:
logger.error('Could not post to Elasticsearch.')
gitstatus_update.apply_async((collection.path,), countdown=2)
# positive feedback
return HttpResponseRedirect(reverse('webui-entity-edit', args=entity.idparts))
# something happened...
logger.error('Could not create new entity!')
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_ERR_CREATE'])
return HttpResponseRedirect(collection.absolute_url())