def create(collection_path, git_name, git_mail):
"""create new entity given an entity ID
TODO remove write and commit, just create object
"""
# write collection.json template to collection location and commit
fileio.write_text(Collection(collection_path).dump_json(template=True),
settings.TEMPLATE_CJSON)
templates = [settings.TEMPLATE_CJSON, settings.TEMPLATE_EAD]
agent = settings.AGENT
cidentifier = Identifier(collection_path)
exit,status = commands.create(
git_name, git_mail, cidentifier, templates, agent
)
collection = Collection.from_identifier(cidentifier)
# [delete cache], update search index
#collection.cache_delete()
if settings.DOCSTORE_ENABLED:
try:
docstore.Docstore().post(self)
except ConnectionError:
logger.error('Could not post to Elasticsearch.')
return collection
def write_object_metadata(obj, tmp_dir, log):
tmp_json = os.path.join(tmp_dir, os.path.basename(obj.json_path))
log.ok('| %s' % tmp_json)
fileio.write_text(obj.dump_json(), tmp_json)
if not os.path.exists(tmp_json):
log.crash('Could not write file metadata %s' % tmp_json)
return tmp_json
def create( path, collection_id, entity_id ):
logging.debug(' EntityControlFile.create({})'.format(path))
t = load_template(ENTITY_CONTROL_TEMPLATE)
fileio.write_text(
t.format(cid=collection_id, eid=entity_id),
path
)
def write_object_metadata(obj, tmp_dir, log):
tmp_json = os.path.join(tmp_dir, os.path.basename(obj.json_path))
log.ok('| %s' % tmp_json)
fileio.write_text(obj.dump_json(), tmp_json)
if not os.path.exists(tmp_json):
log.crash('Could not write file metadata %s' % tmp_json)
return tmp_json
def write_xml(self, path=None):
"""Write METS XML file to disk.
@param path: str Alternate absolute file path
"""
if not path:
path = self.mets_path
fileio.write_text(self.dump_xml(), path)
def write_json(self, obj_metadata={}):
"""Write Collection JSON file to disk.
@param obj_metadata: dict Cached results of object_metadata.
"""
if not os.path.exists(self.identifier.path_abs()):
os.makedirs(self.identifier.path_abs())
fileio.write_text(
self.dump_json(doc_metadata=True, obj_metadata=obj_metadata),
self.json_path)
def test_write_text(tmpdir):
# TODO test bad chars
path = str(tmpdir / 'write_text.json')
# regular
fileio.write_text(TEXT, path)
with open(path, 'r') as f:
written = f.read()
assert written == TEXT
# clean up
os.remove(path)
def write(self, path: str):
"""Write to the specified file (.json or .csv).
@param path: Absolute path to file; must be .json or .csv.
"""
extension = os.path.splitext(path)[1]
if not extension in ['.json', '.csv']:
raise Exception('Index.read only writes .json and .csv files.')
if extension.lower() == '.json':
fileio.write_text(self.dump_json(), path)
elif extension.lower() == '.csv':
fileio.write_text(self.dump_csv(), path)
def edit_json( request, repo, org, cid, eid ):
"""
NOTE: will permit editing even if entity is locked!
(which you need to do sometimes).
"""
entity = Entity.from_request(request)
collection = entity.collection()
#if collection.locked():
# messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
# return HttpResponseRedirect(entity.absolute_url())
#collection.repo_fetch()
#if collection.repo_behind():
# messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
# return HttpResponseRedirect(entity.absolute_url())
#if entity.locked():
# messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
# return HttpResponseRedirect(entity.absolute_url())
#
if request.method == 'POST':
form = JSONForm(request.POST)
if form.is_valid():
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if git_name and git_mail:
json_text = form.cleaned_data['json']
fileio.write_text(json_text, entity.json_path)
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
[entity.json_path],
agent=settings.AGENT
)
collection.cache_delete()
if exit:
messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
else:
gitstatus_update.apply_async((collection.path,), countdown=2)
messages.success(request, WEBUI_MESSAGES['VIEWS_ENT_UPDATED'])
return HttpResponseRedirect(entity.absolute_url())
else:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
else:
form = JSONForm({'json': entity.dump_json(),})
return render_to_response(
'webui/entities/edit-raw.html',
{'entity': entity,
'form': form,},
context_instance=RequestContext(request, processors=[])
)
def test_write_text():
# TODO test bad chars
path = os.path.join(TESTING_BASE_DIR, 'write_text.json')
# regular
fileio.write_text(TEXT, path)
with open(path, 'r') as f:
written = f.read()
assert written == TEXT
# utf8_strict
fileio.write_text(TEXT, path, utf8_strict=True)
with open(path, 'r') as f:
written = f.read()
assert written == TEXT
# clean up
os.remove(path)
def write_json(self, doc_metadata=True, obj_metadata={}, force=False):
"""Write Collection/Entity JSON file to disk.
@param doc_metadata: boolean
@param obj_metadata: dict Cached results of object_metadata.
@param force: boolean Write even nothing looks changed.
"""
if force or self.is_modified():
if not os.path.exists(os.path.dirname(self.json_path)):
os.makedirs(os.path.dirname(self.json_path))
fileio.write_text(
self.dump_json(
doc_metadata=doc_metadata, obj_metadata=obj_metadata
),
self.json_path
)
def new(identifier, git_name, git_mail, agent='cmdln'):
"""Creates new Collection, writes to filesystem, performs initial commit
@param identifier: Identifier
@param git_name: str
@param git_mail: str
@param agent: str
@returns: exit,status int,str
"""
collection = Collection.create(identifier.path_abs(), identifier)
fileio.write_text(collection.dump_json(template=True),
config.TEMPLATE_CJSON)
exit, status = commands.create(
git_name,
git_mail,
identifier, [config.TEMPLATE_CJSON, config.TEMPLATE_EAD],
agent=agent)
return exit, status
def write_json(self, doc_metadata=True, obj_metadata={}, force=False, path=None):
"""Write Collection/Entity JSON file to disk.
@param doc_metadata: boolean
@param obj_metadata: dict Cached results of object_metadata.
@param force: boolean Write even nothing looks changed.
@param path: str Alternate absolute file path
"""
if force or self.is_modified():
if not path:
path = self.json_path
if not os.path.exists(os.path.dirname(path)):
os.makedirs(os.path.dirname(path))
fileio.write_text(
self.dump_json(
doc_metadata=doc_metadata, obj_metadata=obj_metadata
),
path
)
def new(identifier, git_name, git_mail, agent='cmdln'):
"""Creates new Entity, writes to filesystem, does initial commit.
@param identifier: Identifier
@param git_name: str
@param git_mail: str
@param agent: str
@returns: exit,status int,str
"""
collection = identifier.collection().object()
if not collection:
raise Exception('Parent collection for %s does not exist.' % identifier)
entity = Entity.create(identifier.path_abs(), identifier)
fileio.write_text(
entity.dump_json(template=True),
config.TEMPLATE_EJSON
)
exit,status = commands.entity_create(
git_name, git_mail,
collection, entity.identifier,
[collection.json_path_rel, collection.ead_path_rel],
[config.TEMPLATE_EJSON, config.TEMPLATE_METS],
agent=agent
)
if exit:
raise Exception('Could not create new Entity: %s, %s' % (exit, status))
# load Entity object, inherit values from parent, write back to file
entity = Entity.from_identifier(identifier)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=agent
)
return exit,status
def new(identifier, git_name, git_mail, agent='cmdln'):
"""Creates new Entity, writes to filesystem, does initial commit.
@param identifier: Identifier
@param git_name: str
@param git_mail: str
@param agent: str
@returns: exit,status int,str
"""
collection = identifier.collection().object()
if not collection:
raise Exception('Parent collection for %s does not exist.' % identifier)
entity = Entity.create(identifier)
fileio.write_text(
entity.dump_json(template=True),
config.TEMPLATE_EJSON
)
exit,status = commands.entity_create(
git_name, git_mail,
collection, entity.identifier,
[collection.json_path_rel, collection.ead_path_rel],
[config.TEMPLATE_EJSON, config.TEMPLATE_METS],
agent=agent
)
if exit:
raise Exception('Could not create new Entity: %s, %s' % (exit, status))
# load Entity object, inherit values from parent, write back to file
entity = Entity.from_identifier(identifier)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=agent
)
return exit,status
def create(collection, entity_id, git_name, git_mail, agent=settings.AGENT):
"""create new entity given an entity ID
TODO remove write and commit, just create object
"""
eidentifier = Identifier(id=entity_id)
entity_path = eidentifier.path_abs()
# write entity.json template to entity location and commit
fileio.write_text(Entity(entity_path).dump_json(template=True),
settings.TEMPLATE_EJSON)
exit,status = commands.entity_create(
git_name, git_mail,
collection, eidentifier,
[collection.json_path_rel, collection.ead_path_rel],
[settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
agent=agent)
# load new entity, inherit values from parent, write and commit
entity = Entity.from_json(entity_path)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=agent)
# delete cache, update search index
collection.cache_delete()
with open(entity.json_path, 'r') as f:
document = json.loads(f.read())
try:
docstore.post(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, document)
except ConnectionError:
logger.error('Could not post to Elasticsearch.')
return entity
def gitignore(self):
if not os.path.exists(self.gitignore_path):
fileio.write_text(fileio.read_text(GITIGNORE_TEMPLATE),
self.gitignore_path)
return fileio.read_text(self.gitignore_path)
def new( request, repo, org, cid ):
"""Gets new EID from workbench, creates new entity record.
If it messes up, goes back to collection.
"""
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not (git_name and git_mail):
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_request(request)
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect(collection.absolute_url())
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect(collection.absolute_url())
ic = idservice.IDServiceClient()
# resume session
auth_status,auth_reason = ic.resume(request.session['idservice_token'])
if auth_status != 200:
request.session['idservice_username'] = None
request.session['idservice_token'] = None
messages.warning(
request,
'Session resume failed: %s %s (%s)' % (
auth_status,auth_reason,settings.IDSERVICE_API_BASE
)
)
return HttpResponseRedirect(collection.absolute_url())
# get new entity ID
http_status,http_reason,new_entity_id = ic.next_object_id(
collection.identifier,
'entity'
)
if http_status not in [200,201]:
err = '%s %s' % (http_status, http_reason)
msg = WEBUI_MESSAGES['VIEWS_ENT_ERR_NO_IDS'] % (settings.IDSERVICE_API_BASE, err)
logger.error(msg)
messages.error(request, msg)
return HttpResponseRedirect(collection.absolute_url())
eidentifier = Identifier(id=new_entity_id)
# create new entity
entity_path = eidentifier.path_abs()
# write entity.json template to entity location
fileio.write_text(
Entity(entity_path).dump_json(template=True),
settings.TEMPLATE_EJSON
)
# commit files
exit,status = commands.entity_create(
git_name, git_mail,
collection, eidentifier,
[collection.json_path_rel, collection.ead_path_rel],
[settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
agent=settings.AGENT
)
# load Entity object, inherit values from parent, write back to file
entity = Entity.from_identifier(eidentifier)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=settings.AGENT
)
collection.cache_delete()
if exit:
logger.error(exit)
logger.error(status)
messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
else:
# update search index
try:
entity.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
except ConnectionError:
logger.error('Could not post to Elasticsearch.')
gitstatus_update.apply_async((collection.path,), countdown=2)
# positive feedback
return HttpResponseRedirect(reverse('webui-entity-edit', args=entity.idparts))
# something happened...
logger.error('Could not create new entity!')
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_ERR_CREATE'])
return HttpResponseRedirect(collection.absolute_url())
def write_xml(self):
"""Write METS XML file to disk.
"""
fileio.write_text(self.dump_xml(), self.mets_path)
def import_entities(csv_path, collection_path, git_name, git_mail):
"""
@param csv_path: Absolute path to CSV data file.
@param collection_path: Absolute path to collection repo.
@param git_name: Username for use in changelog, git log
@param git_mail: User email address for use in changelog, git log
"""
rows = read_csv(csv_path)
headers = rows[0]
rows = rows[1:]
headers = replace_variant_headers("entity", headers)
field_names = FIELD_NAMES["entity"]
nonrequired_fields = REQUIRED_FIELDS_EXCEPTIONS["entity"]
required_fields = get_required_fields(ENTITY_FIELDS, nonrequired_fields)
rows = replace_variant_cv_field_values("entity", headers, rows)
invalid_headers("entity", headers, field_names, nonrequired_fields)
all_rows_valid("entity", headers, required_fields, rows)
if True:
collection = Collection.from_json(collection_path)
print(collection)
# --------------------------------------------------
def prep_creators(data):
return [x.strip() for x in data.strip().split(";") if x]
def prep_language(data):
"""language can be 'eng', 'eng;jpn', 'eng:English', 'jpn:Japanese'
"""
y = []
for x in data.strip().split(";"):
if ":" in x:
y.append(x.strip().split(":")[0])
else:
y.append(x.strip())
return y
def prep_topics(data):
return [x.strip() for x in data.strip().split(";") if x]
def prep_persons(data):
return [x.strip() for x in data.strip().split(";") if x]
def prep_facility(data):
return [x.strip() for x in data.strip().split(";") if x]
# --------------------------------------------------
print("Data file looks ok")
started = datetime.now()
print("%s starting import" % dtfmt(started))
print("")
for n, row in enumerate(rows):
rowstarted = datetime.now()
rowd = make_row_dict(headers, row)
# create new entity
entity_id = rowd["id"]
entity_path = os.path.join(collection_path, COLLECTION_FILES_PREFIX, entity_id)
# write entity.json template to entity location
fileio.write_text(Entity(entity_path).dump_json(template=True), TEMPLATE_EJSON)
# commit files
exit, status = commands.entity_create(
git_name,
git_mail,
collection.path,
entity_id,
[collection.json_path_rel, collection.ead_path_rel],
[TEMPLATE_EJSON, TEMPLATE_METS],
agent=AGENT,
)
# reload newly-created Entity object
entity = Entity.from_json(entity_path)
# preppers
rowd["creators"] = prep_creators(rowd["creators"])
rowd["language"] = prep_language(rowd["language"])
rowd["topics"] = prep_topics(rowd["topics"])
rowd["persons"] = prep_persons(rowd["persons"])
rowd["facility"] = prep_facility(rowd["facility"])
# insert values from CSV
for key in rowd.keys():
setattr(entity, key, rowd[key])
entity.record_created = datetime.now()
entity.record_lastmod = datetime.now()
# write back to file
entity.write_json()
updated_files = [entity.json_path]
exit, status = commands.entity_update(
git_name, git_mail, entity.parent_path, entity.id, updated_files, agent=AGENT
)
rowfinished = datetime.now()
rowelapsed = rowfinished - rowstarted
print("%s %s/%s %s (%s)" % (dtfmt(rowfinished), n + 1, len(rows), entity.id, rowelapsed))
finished = datetime.now()
elapsed = finished - started
print("")
print("%s done (%s rows)" % (dtfmt(finished), len(rows)))
print("%s elapsed" % elapsed)
print("")
def new( request, repo, org ):
"""Gets new CID from workbench, creates new collection record.
If it messes up, goes back to collection list.
"""
oidentifier = Identifier(request)
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not (git_name and git_mail):
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
ic = idservice.IDServiceClient()
# resume session
auth_status,auth_reason = ic.resume(request.session['idservice_token'])
if auth_status != 200:
request.session['idservice_username'] = None
request.session['idservice_token'] = None
messages.warning(
request,
'Session resume failed: %s %s (%s)' % (
auth_status,auth_reason,settings.IDSERVICE_API_BASE
)
)
return HttpResponseRedirect(reverse('webui-collections'))
# get new collection ID
http_status,http_reason,collection_id = ic.next_object_id(
oidentifier,
'collection'
)
if http_status not in [200,201]:
err = '%s %s' % (http_status, http_reason)
msg = WEBUI_MESSAGES['VIEWS_COLL_ERR_NO_IDS'] % (settings.IDSERVICE_API_BASE, err)
logger.error(msg)
messages.error(request, msg)
return HttpResponseRedirect(reverse('webui-collections'))
identifier = Identifier(id=collection_id, base_path=settings.MEDIA_BASE)
# create the new collection repo
collection_path = identifier.path_abs()
# collection.json template
fileio.write_text(
Collection(collection_path).dump_json(template=True),
settings.TEMPLATE_CJSON
)
exit,status = commands.create(
git_name, git_mail,
identifier,
[settings.TEMPLATE_CJSON, settings.TEMPLATE_EAD],
agent=settings.AGENT
)
if exit:
logger.error(exit)
logger.error(status)
messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
else:
# update search index
collection = Collection.from_identifier(identifier)
try:
collection.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
except ConnectionError:
logger.error('Could not post to Elasticsearch.')
gitstatus_update.apply_async((collection_path,), countdown=2)
# positive feedback
return HttpResponseRedirect( reverse('webui-collection-edit', args=collection.idparts) )
# something happened...
logger.error('Could not create new collecion!')
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_ERR_CREATE'])
return HttpResponseRedirect(reverse('webui-collections'))
def create(path: str, collection_id: str, entity_id: str):
logging.debug(' EntityControlFile.create({})'.format(path))
t = load_template(ENTITY_CONTROL_TEMPLATE)
fileio.write_text(t.format(cid=collection_id, eid=entity_id), path)
def create(path: str, collection_id: str):
logging.debug(' CollectionControlFile.create({})'.format(path))
t = load_template(COLLECTION_CONTROL_TEMPLATE)
fileio.write_text(t.format(cid=collection_id), path)
def write_xml(self):
"""Write EAD XML file to disk.
"""
fileio.write_text(self.dump_xml(), self.ead_path)
def create( path, collection_id ):
logging.debug(' CollectionControlFile.create({})'.format(path))
t = load_template(COLLECTION_CONTROL_TEMPLATE)
fileio.write_text(t.format(cid=collection_id), path)
def write_xml(self):
"""Write METS XML file to disk.
"""
fileio.write_text(self.dump_xml(), self.mets_path)