def after_return(self, status, retval, task_id, args, kwargs, einfo):
logger.debug('DeleteFileTask.after_return(%s, %s, %s, %s, %s)' % (status, retval, task_id, args, kwargs))
collection_path = args[2]
collection = Collection.from_json(collection_path)
lockstatus = collection.unlock(task_id)
gitstatus.update(settings.MEDIA_BASE, collection_path)
gitstatus.unlock(settings.MEDIA_BASE, 'delete_file')
def csv_export( request, repo, org, cid, model=None ):
"""
"""
if (not model) or (not (model in ['entity','file'])):
raise Http404
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
things = {'entity':'objects', 'file':'files'}
csv_path = export_csv_path(collection.path, model)
csv_filename = os.path.basename(csv_path)
if model == 'entity':
file_url = reverse('webui-collection-csv-entities', args=[repo,org,cid])
elif model == 'file':
file_url = reverse('webui-collection-csv-files', args=[repo,org,cid])
# do it
result = csv_export_model.apply_async( (collection.path,model), countdown=2)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-csv-export-model',
'collection_id': collection.id,
'collection_url': collection.url(),
'things': things[model],
'file_name': csv_filename,
'file_url': file_url,
'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),}
celery_tasks[result.task_id] = task
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
def after_return(self, status, retval, task_id, args, kwargs, einfo):
logger.debug('EntityEditTask.after_return(%s, %s, %s, %s, %s)' % (status, retval, task_id, args, kwargs))
collection_path = args[0]
collection = Collection.from_json(collection_path)
lockstatus = collection.unlock(task_id)
gitstatus.update(settings.MEDIA_BASE, collection_path)
gitstatus.unlock(settings.MEDIA_BASE, 'entity_edit')
def csv_download( request, repo, org, cid, model=None ):
"""Offers CSV file in settings.CSV_TMPDIR for download.
File must actually exist in settings.CSV_TMPDIR and be readable.
File must be readable by Python csv module.
If all that is true then it must be a legal CSV file.
"""
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
path = export_csv_path(collection.path, model)
filename = os.path.basename(path)
if not os.path.exists(path):
raise Http404
import csv
# TODO use vars from migrations.densho or put them in settings.
CSV_DELIMITER = ','
CSV_QUOTECHAR = '"'
CSV_QUOTING = csv.QUOTE_ALL
response = HttpResponse(content_type='text/csv')
response['Content-Disposition'] = 'attachment; filename="%s"' % filename
writer = csv.writer(response, delimiter=CSV_DELIMITER, quotechar=CSV_QUOTECHAR, quoting=CSV_QUOTING)
with open(path, 'rb') as f:
reader = csv.reader(f, delimiter=CSV_DELIMITER, quotechar=CSV_QUOTECHAR, quoting=CSV_QUOTING)
for row in reader:
writer.writerow(row)
return response
def collections( request ):
"""
We are displaying collection status vis-a-vis the project Gitolite server.
It takes too long to run git-status on every repo so, if repo statuses are not
cached they will be updated by jQuery after page load has finished.
"""
collections = []
collection_status_urls = []
for o in gitolite.get_repos_orgs():
repo,org = o.split('-')
colls = []
for coll in commands.collections_local(settings.MEDIA_BASE, repo, org):
if coll:
coll = os.path.basename(coll)
c = coll.split('-')
repo,org,cid = c[0],c[1],c[2]
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
colls.append(collection)
gitstatus = collection.gitstatus()
if gitstatus and gitstatus.get('sync_status'):
collection.sync_status = gitstatus['sync_status']
else:
collection_status_urls.append( "'%s'" % collection.sync_status_url())
collections.append( (o,repo,org,colls) )
# load statuses in random order
random.shuffle(collection_status_urls)
return render_to_response(
'webui/collections/index.html',
{'collections': collections,
'collection_status_urls': ', '.join(collection_status_urls),},
context_instance=RequestContext(request, processors=[])
)
def merge( request, repo, org, cid ):
"""
Decides how to merge the various files in a merge conflict.
Sends user around to different editors and things until everything is merged.
"""
collection_path = Collection.collection_path(request,repo,org,cid)
repository = dvcs.repository(collection_path)
collection = Collection.from_json(collection_path)
task_id = collection.locked()
status = commands.status(collection_path)
ahead = collection.repo_ahead()
behind = collection.repo_behind()
diverged = collection.repo_diverged()
conflicted = collection.repo_conflicted()
unmerged = dvcs.list_conflicted(repository)
staged = dvcs.list_staged(repository)
if request.method == 'POST':
form = MergeCommitForm(request.POST)
if form.is_valid():
which = form.cleaned_data['which']
if which == 'merge':
dvcs.merge_commit(repository)
committed = 1
elif which == 'commit':
dvcs.diverge_commit(repository)
committed = 1
else:
committed = 0
if committed:
if task_id:
collection.unlock(task_id)
messages.error(request, 'Merge conflict has been resolved. Please sync to make your changes available to other users.')
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
return HttpResponseRedirect( reverse('webui-merge', args=[repo,org,cid]) )
else:
which = 'unknown'
if conflicted and not unmerged:
which = 'merge'
elif diverged and staged:
which = 'commit'
form = MergeCommitForm({'path':collection_path, 'which':which,})
return render_to_response(
'webui/merge/index.html',
{'repo': repo,
'org': org,
'cid': cid,
'collection_path': collection_path,
'collection': collection,
'status': status,
'conflicted': conflicted,
'ahead': ahead,
'behind': behind,
'unmerged': unmerged,
'diverged': diverged,
'staged': staged,
'form': form,},
context_instance=RequestContext(request, processors=[])
)
def sync_status_ajax( request, repo, org, cid ):
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
gitstatus = collection.gitstatus()
if gitstatus:
sync_status = gitstatus['sync_status']
if sync_status.get('timestamp',None):
sync_status['timestamp'] = sync_status['timestamp'].strftime(settings.TIMESTAMP_FORMAT)
return HttpResponse(json.dumps(sync_status), mimetype="application/json")
raise Http404
def new_access( request, repo, org, cid, eid, role, sha1 ):
"""Generate a new access file for the specified file.
NOTE: There is no GET for this view. GET requests will redirect to entity.
"""
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
file_ = entity.file(repo, org, cid, eid, role, sha1)
#
if request.method == 'POST':
form = NewAccessFileForm(request.POST)
if form.is_valid():
src_path = form.cleaned_data['path']
# start tasks
result = entity_add_access.apply_async(
(git_name, git_mail, entity, file_),
countdown=2)
result_dict = result.__dict__
entity.files_log(1,'START task_id %s' % result.task_id)
entity.files_log(1,'ddrlocal.webui.file.new_access')
entity.files_log(1,'Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
entity.files_log(1, 'locked')
else:
entity.files_log(0, lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-file-new-access',
'filename': os.path.basename(src_path),
'file_url': file_.url(),
'entity_id': entity.id,
'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# feedback
messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_NEWACCESS'] % os.path.basename(src_path))
# redirect to entity
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
def after_return(self, status, retval, task_id, args, kwargs, cinfo):
collection_path = args[2]
collection = Collection.from_json(collection_path)
# NOTE: collection is locked immediately after collection_sync task
# starts in webui.views.collections.sync
collection.unlock(task_id)
collection.cache_delete()
gitstatus.update(settings.MEDIA_BASE, collection_path)
gitstatus.unlock(settings.MEDIA_BASE, 'collection_sync')
def changelog( request, repo, org, cid ):
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
alert_if_conflicted(request, collection)
return render_to_response(
'webui/collections/changelog.html',
{'repo': repo,
'org': org,
'cid': cid,
'collection': collection,},
context_instance=RequestContext(request, processors=[])
)
def edit_ead( request, repo, org, cid ):
"""
on GET
- reads contents of EAD.xml
- puts in form, in textarea
- user edits XML
on POST
- write contents of field to EAD.xml
- commands.update
"""
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
ead_path_rel = 'ead.xml'
ead_path_abs = os.path.join(collection.path, ead_path_rel)
#
if request.method == 'POST':
form = UpdateForm(request.POST)
if form.is_valid():
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if git_name and git_mail:
xml = form.cleaned_data['xml']
# TODO validate XML
with open(ead_path_abs, 'w') as f:
f.write(xml)
exit,status = commands.update(git_name, git_mail,
collection.path, [ead_path_rel],
agent=settings.AGENT)
if exit:
messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
else:
messages.success(request, 'Collection metadata updated')
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
else:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
else:
with open(ead_path_abs, 'r') as f:
xml = f.read()
form = UpdateForm({'xml':xml,})
return render_to_response(
'webui/collections/edit-ead.html',
{'repo': repo,
'org': org,
'cid': cid,
'collection_uid': collection.id,
'collection': collection,
'form': form,},
context_instance=RequestContext(request, processors=[])
)
def unlock( request, repo, org, cid, task_id ):
"""Provides a way to remove collection lockfile through the web UI.
"""
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
if task_id and collection.locked() and (task_id == collection.locked()):
collection.unlock(task_id)
messages.success(request, 'Collection <b>%s</b> unlocked.' % collection.id)
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
def detail( request, repo, org, cid ):
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
alert_if_conflicted(request, collection)
return render_to_response(
'webui/collections/detail.html',
{'repo': repo,
'org': org,
'cid': cid,
'collection': collection,
'unlock_task_id': collection.locked(),},
context_instance=RequestContext(request, processors=[])
)
def edit( request, repo, org, cid, eid, role, sha1 ):
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
file_ = entity.file(repo, org, cid, eid, role, sha1)
#
if request.method == 'POST':
form = DDRForm(request.POST, fields=FILE_FIELDS)
if form.is_valid():
file_.form_post(form)
file_.dump_json()
# commit files, delete cache, update search index, update git status
entity_file_edit(request, collection, file_, git_name, git_mail)
return HttpResponseRedirect( file_.url() )
else:
form = DDRForm(file_.form_prep(), fields=FILE_FIELDS)
return render_to_response(
'webui/files/edit-json.html',
{'repo': file_.repo,
'org': file_.org,
'cid': file_.cid,
'eid': file_.eid,
'role': file_.role,
'sha1': file_.sha1,
'collection': collection,
'entity': entity,
'file': file_,
'form': form,
},
context_instance=RequestContext(request, processors=[])
)
def git_status( request, repo, org, cid ):
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
alert_if_conflicted(request, collection)
gitstatus = collection.gitstatus()
return render_to_response(
'webui/collections/git-status.html',
{'repo': repo,
'org': org,
'cid': cid,
'collection': collection,
'status': gitstatus['status'],
'astatus': gitstatus['annex_status'],
'timestamp': gitstatus['timestamp'],
},
context_instance=RequestContext(request, processors=[])
)
def collection_save(collection_path, updated_files, git_name, git_mail):
"""The time-consuming parts of collection-edit.
@param collection_path: str Absolute path to collection
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
"""
logger.debug('collection_save(%s,%s,%s,%s)' % (
git_name, git_mail, collection_path, updated_files))
collection = Collection.from_json(collection_path)
gitstatus.lock(settings.MEDIA_BASE, 'collection_edit')
exit,status = collection.save(updated_files, git_name, git_mail)
gitstatus_update.apply_async((collection_path,), countdown=2)
return status,collection_path
def browse( request, repo, org, cid, eid, role='master' ):
"""Browse for a file in vbox shared folder.
"""
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
path = request.GET.get('path')
home = None
parent = None
if path:
path_abs = os.path.join(settings.VIRTUALBOX_SHARED_FOLDER, path)
parent = os.path.dirname(path)
home = settings.VIRTUALBOX_SHARED_FOLDER
else:
path_abs = settings.VIRTUALBOX_SHARED_FOLDER
listdir = []
if os.path.exists(path_abs):
for x in os.listdir(path_abs):
xabs = os.path.join(path_abs, x)
rel = xabs.replace(settings.VIRTUALBOX_SHARED_FOLDER, '')
if rel and rel[0] == '/':
rel = rel[1:]
isdir = os.path.isdir(xabs)
if isdir:
x = '%s/' % x
mtime = datetime.fromtimestamp(os.path.getmtime(xabs))
size = None
if not isdir:
size = os.path.getsize(xabs)
attribs = {'basename':x, 'rel':rel, 'path':xabs, 'isdir':isdir, 'size':size, 'mtime':mtime}
if os.path.exists(xabs):
listdir.append(attribs)
return render_to_response(
'webui/files/browse.html',
{'repo': repo,
'org': org,
'cid': cid,
'eid': eid,
'collection_uid': collection.id,
'collection': collection,
'entity': entity,
'role': role,
'listdir': listdir,
'parent': parent,
'home': home,},
context_instance=RequestContext(request, processors=[])
)
def edit( request, repo, org, cid ):
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
if request.method == 'POST':
form = DDRForm(request.POST, fields=COLLECTION_FIELDS)
if form.is_valid():
collection.form_post(form)
collection.dump_json()
collection.dump_ead()
updated_files = [collection.json_path, collection.ead_path,]
# if inheritable fields selected, propagate changes to child objects
inheritables = collection.selected_inheritables(form.cleaned_data)
modified_ids,modified_files = collection.update_inheritables(inheritables, form.cleaned_data)
if modified_files:
updated_files = updated_files + modified_files
# commit files, delete cache, update search index, update git status
collection_edit(request, collection, updated_files, git_name, git_mail)
return HttpResponseRedirect(collection.url())
else:
form = DDRForm(collection.form_prep(), fields=COLLECTION_FIELDS)
return render_to_response(
'webui/collections/edit-json.html',
{'repo': repo,
'org': org,
'cid': cid,
'collection': collection,
'form': form,
},
context_instance=RequestContext(request, processors=[])
)
def entity_newexpert(collection_path, entity_id, git_name, git_mail):
"""Create new entity using known entity ID.
@param collection_path: str Absolute path to collection
@param entity_id: str
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
"""
logger.debug('collection_entity_newexpert(%s,%s,%s,%s)' % (
collection_path, entity_id, git_name, git_mail))
collection = Collection.from_json(collection_path)
gitstatus.lock(settings.MEDIA_BASE, 'entity_newexpert')
entity = Entity.create(collection, entity_id, git_name, git_mail)
gitstatus_update.apply_async((collection.path,), countdown=2)
return 'status',collection_path,entity.id
def after_return(self, status, retval, task_id, args, kwargs, einfo):
entity = args[2]
entity.files_log(1,'DDRTask.AFTER_RETURN')
entity.files_log(1,'task_id: %s' % task_id)
entity.files_log(1,'status: %s' % status)
entity.files_log(1,'retval: %s' % retval)
entity.files_log(1,'Unlocking %s' % entity.id)
lockstatus = entity.unlock(task_id)
if lockstatus == 'ok':
entity.files_log(1,'unlocked')
else:
entity.files_log(0,lockstatus)
entity.files_log(1, 'END task_id %s\n' % task_id)
collection_path = Collection.collection_path(None,entity.repo,entity.org,entity.cid)
collection = Collection.from_json(collection_path)
collection.cache_delete()
gitstatus.update(settings.MEDIA_BASE, collection_path)
gitstatus.unlock(settings.MEDIA_BASE, 'entity_add_file')
def entities( request, repo, org, cid ):
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
alert_if_conflicted(request, collection)
entities = collection.entities(quick=True)
# paginate
thispage = request.GET.get('page', 1)
paginator = Paginator(entities, settings.RESULTS_PER_PAGE)
page = paginator.page(thispage)
return render_to_response(
'webui/collections/entities.html',
{'repo': repo,
'org': org,
'cid': cid,
'collection': collection,
'paginator': paginator,
'page': page,
'thispage': thispage,},
context_instance=RequestContext(request, processors=[])
)
def sync( request, repo, org, cid ):
try:
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
except:
raise Http404
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
alert_if_conflicted(request, collection)
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
if request.method == 'POST':
form = SyncConfirmForm(request.POST)
form_is_valid = form.is_valid()
if form.is_valid() and form.cleaned_data['confirmed']:
result = collection_sync.apply_async( (git_name,git_mail,collection.path), countdown=2)
lockstatus = collection.lock(result.task_id)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-collection-sync',
'collection_id': collection.id,
'collection_url': collection.url(),
'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),}
celery_tasks[result.task_id] = task
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
#else:
# assert False
else:
form = SyncConfirmForm()
return render_to_response(
'webui/collections/sync-confirm.html',
{'repo': collection.repo,
'org': collection.org,
'cid': collection.cid,
'collection': collection,
'form': form,},
context_instance=RequestContext(request, processors=[])
)
def batch( request, repo, org, cid, eid, role='master' ):
"""Add multiple files to entity.
"""
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
return render_to_response(
'webui/files/new.html',
{'collection': collection,
'entity': entity,},
context_instance=RequestContext(request, processors=[])
)
def entity_edit(collection_path, entity_id, updated_files, git_name, git_mail, agent=''):
"""The time-consuming parts of entity-edit.
@param collection_path: str Absolute path to collection
@param entity_id: str
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
logger.debug('collection_entity_edit(%s,%s,%s,%s,%s)' % (
git_name, git_mail, collection_path, entity_id, agent))
collection = Collection.from_json(collection_path)
repo,org,cid,eid = entity_id.split('-')
entity_path = Entity.entity_path(None,repo,org,cid,eid)
entity = Entity.from_json(entity_path)
gitstatus.lock(settings.MEDIA_BASE, 'entity_edit')
exit,status = entity.save_part2(updated_files, collection, git_name, git_mail)
gitstatus_update.apply_async((collection.path,), countdown=2)
return status,collection_path,entity_id
def detail( request, repo, org, cid, eid, role, sha1 ):
"""Add file to entity.
"""
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
file_ = entity.file(repo, org, cid, eid, role, sha1)
formdata = {'path':file_.path_rel}
return render_to_response(
'webui/files/detail.html',
{'repo': file_.repo,
'org': file_.org,
'cid': file_.cid,
'eid': file_.eid,
'role': file_.role,
'sha1': file_.sha1,
'collection_uid': collection.id,
'collection': collection,
'entity': entity,
'file': file_,
'new_access_form': NewAccessFileForm(formdata),},
context_instance=RequestContext(request, processors=[])
)
def delete( request, repo, org, cid, eid, role, sha1 ):
try:
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
file_ = entity.file(repo, org, cid, eid, role, sha1)
except:
raise Http404
collection = Collection.from_json(file_.collection_path)
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
#
if request.method == 'POST':
form = DeleteFileForm(request.POST)
if form.is_valid() and form.cleaned_data['confirmed']:
entity_delete_file(request, git_name, git_mail, collection, entity, file_, settings.AGENT)
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
else:
form = DeleteFileForm()
return render_to_response(
'webui/files/delete.html',
{'repo': file_.repo,
'org': file_.org,
'cid': file_.cid,
'eid': file_.eid,
'role': file_.role,
'sha1': file_.sha1,
'file': file_,
'form': form,
},
context_instance=RequestContext(request, processors=[])
)
def import_files(csv_path, collection_path, git_name, git_mail):
"""
@param csv_path: Absolute path to CSV data file.
@param collection_path: Absolute path to collection repo.
@param git_name: Username for use in changelog, git log
@param git_mail: User email address for use in changelog, git log
"""
csv_dir = os.path.dirname(csv_path)
rows = read_csv(csv_path)
headers = rows[0]
rows = rows[1:]
headers = replace_variant_headers("file", headers)
nonrequired_fields = REQUIRED_FIELDS_EXCEPTIONS["file"]
required_fields = get_required_fields(FILE_FIELDS, nonrequired_fields)
rows = replace_variant_cv_field_values("file", headers, rows)
# crash if bad headers or rows
invalid_headers("file", headers, field_names, nonrequired_fields)
all_rows_valid("file", headers, required_fields, rows)
if True:
collection = Collection.from_json(collection_path)
print(collection)
# def prep_creators( data ): return [x.strip() for x in data.strip().split(';') if x]
# load entities - if any Entities are missing this will error out
bad_entities = test_entities(headers, rows)
if bad_entities:
print("ONE OR MORE OBJECTS ARE COULD NOT BE LOADED! - IMPORT CANCELLED!")
for f in bad_entities:
print(" %s" % f)
# check for missing files
missing_files = find_missing_files(csv_dir, headers, rows)
if missing_files:
print("ONE OR MORE SOURCE FILES ARE MISSING! - IMPORT CANCELLED!")
for f in missing_files:
print(" %s" % f)
else:
print("Source files present")
# check for unreadable files
unreadable_files = find_unreadable_files(csv_dir, headers, rows)
if unreadable_files:
print("ONE OR MORE SOURCE FILES COULD NOT BE OPENED! - IMPORT CANCELLED!")
for f in unreadable_files:
print(" %s" % f)
print("Files must be readable to the user running this script (probably ddr).")
else:
print("Source files readable")
# files are all accounted for, let's import
if not (bad_entities or missing_files or unreadable_files):
print("Data file looks ok and files are present")
print('"$ tail -f /var/log/ddr/local.log" in a separate console for more details')
started = datetime.now()
print("%s starting import" % started)
print("")
for n, row in enumerate(rows):
rowd = make_row_dict(headers, row)
entity_id = rowd.pop("entity_id")
repo, org, cid, eid = entity_id.split("-")
entity_path = Entity.entity_path(None, repo, org, cid, eid)
entity = Entity.from_json(entity_path)
src_path = os.path.join(csv_dir, rowd.pop("basename_orig"))
role = rowd.pop("role")
rowstarted = datetime.now()
print(
"%s %s/%s %s %s (%s)"
% (
dtfmt(rowstarted),
n + 1,
len(rows),
entity.id,
src_path,
humanize_bytes(os.path.getsize(src_path)),
)
)
# print('add_file(%s, %s, %s, %s, %s, %s)' % (git_name, git_mail, entity, src_path, role, rowd))
entity.add_file(git_name, git_mail, src_path, role, rowd, agent=AGENT)
rowfinished = datetime.now()
rowelapsed = rowfinished - rowstarted
print("%s done (%s)" % (dtfmt(rowfinished), rowelapsed))
finished = datetime.now()
elapsed = finished - started
print("")
print("%s done (%s rows)" % (dtfmt(finished), len(rows)))
print("%s elapsed" % elapsed)
print("")
def import_entities(csv_path, collection_path, git_name, git_mail):
"""
@param csv_path: Absolute path to CSV data file.
@param collection_path: Absolute path to collection repo.
@param git_name: Username for use in changelog, git log
@param git_mail: User email address for use in changelog, git log
"""
rows = read_csv(csv_path)
headers = rows[0]
rows = rows[1:]
headers = replace_variant_headers("entity", headers)
field_names = FIELD_NAMES["entity"]
nonrequired_fields = REQUIRED_FIELDS_EXCEPTIONS["entity"]
required_fields = get_required_fields(ENTITY_FIELDS, nonrequired_fields)
rows = replace_variant_cv_field_values("entity", headers, rows)
invalid_headers("entity", headers, field_names, nonrequired_fields)
all_rows_valid("entity", headers, required_fields, rows)
if True:
collection = Collection.from_json(collection_path)
print(collection)
# --------------------------------------------------
def prep_creators(data):
return [x.strip() for x in data.strip().split(";") if x]
def prep_language(data):
"""language can be 'eng', 'eng;jpn', 'eng:English', 'jpn:Japanese'
"""
y = []
for x in data.strip().split(";"):
if ":" in x:
y.append(x.strip().split(":")[0])
else:
y.append(x.strip())
return y
def prep_topics(data):
return [x.strip() for x in data.strip().split(";") if x]
def prep_persons(data):
return [x.strip() for x in data.strip().split(";") if x]
def prep_facility(data):
return [x.strip() for x in data.strip().split(";") if x]
# --------------------------------------------------
print("Data file looks ok")
started = datetime.now()
print("%s starting import" % dtfmt(started))
print("")
for n, row in enumerate(rows):
rowstarted = datetime.now()
rowd = make_row_dict(headers, row)
# create new entity
entity_id = rowd["id"]
entity_path = os.path.join(collection_path, COLLECTION_FILES_PREFIX, entity_id)
# write entity.json template to entity location
fileio.write_text(Entity(entity_path).dump_json(template=True), TEMPLATE_EJSON)
# commit files
exit, status = commands.entity_create(
git_name,
git_mail,
collection.path,
entity_id,
[collection.json_path_rel, collection.ead_path_rel],
[TEMPLATE_EJSON, TEMPLATE_METS],
agent=AGENT,
)
# reload newly-created Entity object
entity = Entity.from_json(entity_path)
# preppers
rowd["creators"] = prep_creators(rowd["creators"])
rowd["language"] = prep_language(rowd["language"])
rowd["topics"] = prep_topics(rowd["topics"])
rowd["persons"] = prep_persons(rowd["persons"])
rowd["facility"] = prep_facility(rowd["facility"])
# insert values from CSV
for key in rowd.keys():
setattr(entity, key, rowd[key])
entity.record_created = datetime.now()
entity.record_lastmod = datetime.now()
# write back to file
entity.write_json()
updated_files = [entity.json_path]
exit, status = commands.entity_update(
git_name, git_mail, entity.parent_path, entity.id, updated_files, agent=AGENT
)
rowfinished = datetime.now()
rowelapsed = rowfinished - rowstarted
print("%s %s/%s %s (%s)" % (dtfmt(rowfinished), n + 1, len(rows), entity.id, rowelapsed))
finished = datetime.now()
elapsed = finished - started
print("")
print("%s done (%s rows)" % (dtfmt(finished), len(rows)))
print("%s elapsed" % elapsed)
print("")
def ead_xml( request, repo, org, cid ):
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
alert_if_conflicted(request, collection)
soup = BeautifulSoup(collection.ead().xml, 'xml')
return HttpResponse(soup.prettify(), mimetype="application/xml")
def collection_json( request, repo, org, cid ):
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
alert_if_conflicted(request, collection)
return HttpResponse(json.dumps(collection.json().data), mimetype="application/json")
