def delete_file( git_name, git_mail, collection_path, entity_id, file_basename, agent='' ):
"""
@param collection_path: string
@param entity_id: string
@param file_basename: string
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
logger.debug('delete_file(%s,%s,%s,%s,%s,%s)' % (git_name, git_mail, collection_path, entity_id, file_basename, agent))
gitstatus.lock(settings.MEDIA_BASE, 'delete_file')
# TODO rm_files list should come from the File model
file_id = os.path.splitext(file_basename)[0]
repo,org,cid,eid,role,sha1 = file_id.split('-')
entity = Entity.from_json(Entity.entity_path(None,repo,org,cid,eid))
file_ = entity.file(repo, org, cid, eid, role, sha1)
rm_files = file_.files_rel(collection_path)
logger.debug('rm_files: %s' % rm_files)
# remove file from entity.json
# TODO move this to commands.file_destroy or models.Entity
for f in entity.files:
if f.basename == file_basename:
entity.files.remove(f)
entity.dump_json()
updated_files = ['entity.json']
logger.debug('updated_files: %s' % updated_files)
status,message = file_destroy(git_name, git_mail, collection_path, entity_id, rm_files, updated_files, agent)
return status,message,collection_path,file_basename
def file_json( request, repo, org, cid, eid, role, sha1 ):
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
file_ = entity.file(repo, org, cid, eid, role, sha1)
if file_.json_path and os.path.exists(file_.json_path):
with open(file_.json_path, 'r') as f:
json = f.read()
return HttpResponse(json, mimetype="application/json")
messages.success(request, 'no JSON file. sorry.')
return HttpResponseRedirect( reverse('webui-file', args=[repo,org,cid,eid,role,sha1]) )
def new_access( request, repo, org, cid, eid, role, sha1 ):
"""Generate a new access file for the specified file.
NOTE: There is no GET for this view. GET requests will redirect to entity.
"""
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
file_ = entity.file(repo, org, cid, eid, role, sha1)
#
if request.method == 'POST':
form = NewAccessFileForm(request.POST)
if form.is_valid():
src_path = form.cleaned_data['path']
# start tasks
result = entity_add_access.apply_async(
(git_name, git_mail, entity, file_),
countdown=2)
result_dict = result.__dict__
entity.files_log(1,'START task_id %s' % result.task_id)
entity.files_log(1,'ddrlocal.webui.file.new_access')
entity.files_log(1,'Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
entity.files_log(1, 'locked')
else:
entity.files_log(0, lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-file-new-access',
'filename': os.path.basename(src_path),
'file_url': file_.url(),
'entity_id': entity.id,
'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# feedback
messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_NEWACCESS'] % os.path.basename(src_path))
# redirect to entity
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
def edit( request, repo, org, cid, eid, role, sha1 ):
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
file_ = entity.file(repo, org, cid, eid, role, sha1)
#
if request.method == 'POST':
form = DDRForm(request.POST, fields=FILE_FIELDS)
if form.is_valid():
file_.form_post(form)
file_.dump_json()
# commit files, delete cache, update search index, update git status
entity_file_edit(request, collection, file_, git_name, git_mail)
return HttpResponseRedirect( file_.url() )
else:
form = DDRForm(file_.form_prep(), fields=FILE_FIELDS)
return render_to_response(
'webui/files/edit-json.html',
{'repo': file_.repo,
'org': file_.org,
'cid': file_.cid,
'eid': file_.eid,
'role': file_.role,
'sha1': file_.sha1,
'collection': collection,
'entity': entity,
'file': file_,
'form': form,
},
context_instance=RequestContext(request, processors=[])
)
def browse( request, repo, org, cid, eid, role='master' ):
"""Browse for a file in vbox shared folder.
"""
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
path = request.GET.get('path')
home = None
parent = None
if path:
path_abs = os.path.join(settings.VIRTUALBOX_SHARED_FOLDER, path)
parent = os.path.dirname(path)
home = settings.VIRTUALBOX_SHARED_FOLDER
else:
path_abs = settings.VIRTUALBOX_SHARED_FOLDER
listdir = []
if os.path.exists(path_abs):
for x in os.listdir(path_abs):
xabs = os.path.join(path_abs, x)
rel = xabs.replace(settings.VIRTUALBOX_SHARED_FOLDER, '')
if rel and rel[0] == '/':
rel = rel[1:]
isdir = os.path.isdir(xabs)
if isdir:
x = '%s/' % x
mtime = datetime.fromtimestamp(os.path.getmtime(xabs))
size = None
if not isdir:
size = os.path.getsize(xabs)
attribs = {'basename':x, 'rel':rel, 'path':xabs, 'isdir':isdir, 'size':size, 'mtime':mtime}
if os.path.exists(xabs):
listdir.append(attribs)
return render_to_response(
'webui/files/browse.html',
{'repo': repo,
'org': org,
'cid': cid,
'eid': eid,
'collection_uid': collection.id,
'collection': collection,
'entity': entity,
'role': role,
'listdir': listdir,
'parent': parent,
'home': home,},
context_instance=RequestContext(request, processors=[])
)
def file_edit(collection_path, file_id, git_name, git_mail):
"""The time-consuming parts of file-edit.
@param collection_path: str Absolute path to collection
@param file_id: str
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
"""
logger.debug('file_edit(%s,%s,%s,%s)' % (git_name, git_mail, collection_path, file_id))
model,repo,org,cid,eid,role,sha1 = models.split_object_id(file_id)
entity = Entity.from_json(Entity.entity_path(None,repo,org,cid,eid))
file_ = entity.file(repo, org, cid, eid, role, sha1)
gitstatus.lock(settings.MEDIA_BASE, 'file_edit')
exit,status = file_.save(git_name, git_mail)
gitstatus_update.apply_async((collection_path,), countdown=2)
return status,collection_path,file_id
def test_entities(headers, rows):
"""Test-loads Entities mentioned in rows; crashes if any are missing.
@param headers: List of field names
@param rows: List of rows (each with list of fields, not dict)
@returns list of invalid entities
"""
bad_entities = []
for row in rows:
rowd = make_row_dict(headers, row)
entity_id = rowd.pop("entity_id")
repo, org, cid, eid = entity_id.split("-")
entity_path = Entity.entity_path(None, repo, org, cid, eid)
try:
entity = Entity.from_json(entity_path)
except:
entity = None
if not entity:
bad_entities.append(entity_id)
return bad_entities
def batch( request, repo, org, cid, eid, role='master' ):
"""Add multiple files to entity.
"""
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
return render_to_response(
'webui/files/new.html',
{'collection': collection,
'entity': entity,},
context_instance=RequestContext(request, processors=[])
)
def detail( request, repo, org, cid, eid, role, sha1 ):
"""Add file to entity.
"""
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
file_ = entity.file(repo, org, cid, eid, role, sha1)
formdata = {'path':file_.path_rel}
return render_to_response(
'webui/files/detail.html',
{'repo': file_.repo,
'org': file_.org,
'cid': file_.cid,
'eid': file_.eid,
'role': file_.role,
'sha1': file_.sha1,
'collection_uid': collection.id,
'collection': collection,
'entity': entity,
'file': file_,
'new_access_form': NewAccessFileForm(formdata),},
context_instance=RequestContext(request, processors=[])
)
def entity_edit(collection_path, entity_id, updated_files, git_name, git_mail, agent=''):
"""The time-consuming parts of entity-edit.
@param collection_path: str Absolute path to collection
@param entity_id: str
@param git_name: Username of git committer.
@param git_mail: Email of git committer.
@param agent: (optional) Name of software making the change.
"""
logger.debug('collection_entity_edit(%s,%s,%s,%s,%s)' % (
git_name, git_mail, collection_path, entity_id, agent))
collection = Collection.from_json(collection_path)
repo,org,cid,eid = entity_id.split('-')
entity_path = Entity.entity_path(None,repo,org,cid,eid)
entity = Entity.from_json(entity_path)
gitstatus.lock(settings.MEDIA_BASE, 'entity_edit')
exit,status = entity.save_part2(updated_files, collection, git_name, git_mail)
gitstatus_update.apply_async((collection.path,), countdown=2)
return status,collection_path,entity_id
def delete( request, repo, org, cid, eid, role, sha1 ):
try:
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
file_ = entity.file(repo, org, cid, eid, role, sha1)
except:
raise Http404
collection = Collection.from_json(file_.collection_path)
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
#
if request.method == 'POST':
form = DeleteFileForm(request.POST)
if form.is_valid() and form.cleaned_data['confirmed']:
entity_delete_file(request, git_name, git_mail, collection, entity, file_, settings.AGENT)
return HttpResponseRedirect( reverse('webui-collection', args=[repo,org,cid]) )
else:
form = DeleteFileForm()
return render_to_response(
'webui/files/delete.html',
{'repo': file_.repo,
'org': file_.org,
'cid': file_.cid,
'eid': file_.eid,
'role': file_.role,
'sha1': file_.sha1,
'file': file_,
'form': form,
},
context_instance=RequestContext(request, processors=[])
)
def import_files(csv_path, collection_path, git_name, git_mail):
"""
@param csv_path: Absolute path to CSV data file.
@param collection_path: Absolute path to collection repo.
@param git_name: Username for use in changelog, git log
@param git_mail: User email address for use in changelog, git log
"""
csv_dir = os.path.dirname(csv_path)
rows = read_csv(csv_path)
headers = rows[0]
rows = rows[1:]
headers = replace_variant_headers("file", headers)
nonrequired_fields = REQUIRED_FIELDS_EXCEPTIONS["file"]
required_fields = get_required_fields(FILE_FIELDS, nonrequired_fields)
rows = replace_variant_cv_field_values("file", headers, rows)
# crash if bad headers or rows
invalid_headers("file", headers, field_names, nonrequired_fields)
all_rows_valid("file", headers, required_fields, rows)
if True:
collection = Collection.from_json(collection_path)
print(collection)
# def prep_creators( data ): return [x.strip() for x in data.strip().split(';') if x]
# load entities - if any Entities are missing this will error out
bad_entities = test_entities(headers, rows)
if bad_entities:
print("ONE OR MORE OBJECTS ARE COULD NOT BE LOADED! - IMPORT CANCELLED!")
for f in bad_entities:
print(" %s" % f)
# check for missing files
missing_files = find_missing_files(csv_dir, headers, rows)
if missing_files:
print("ONE OR MORE SOURCE FILES ARE MISSING! - IMPORT CANCELLED!")
for f in missing_files:
print(" %s" % f)
else:
print("Source files present")
# check for unreadable files
unreadable_files = find_unreadable_files(csv_dir, headers, rows)
if unreadable_files:
print("ONE OR MORE SOURCE FILES COULD NOT BE OPENED! - IMPORT CANCELLED!")
for f in unreadable_files:
print(" %s" % f)
print("Files must be readable to the user running this script (probably ddr).")
else:
print("Source files readable")
# files are all accounted for, let's import
if not (bad_entities or missing_files or unreadable_files):
print("Data file looks ok and files are present")
print('"$ tail -f /var/log/ddr/local.log" in a separate console for more details')
started = datetime.now()
print("%s starting import" % started)
print("")
for n, row in enumerate(rows):
rowd = make_row_dict(headers, row)
entity_id = rowd.pop("entity_id")
repo, org, cid, eid = entity_id.split("-")
entity_path = Entity.entity_path(None, repo, org, cid, eid)
entity = Entity.from_json(entity_path)
src_path = os.path.join(csv_dir, rowd.pop("basename_orig"))
role = rowd.pop("role")
rowstarted = datetime.now()
print(
"%s %s/%s %s %s (%s)"
% (
dtfmt(rowstarted),
n + 1,
len(rows),
entity.id,
src_path,
humanize_bytes(os.path.getsize(src_path)),
)
)
# print('add_file(%s, %s, %s, %s, %s, %s)' % (git_name, git_mail, entity, src_path, role, rowd))
entity.add_file(git_name, git_mail, src_path, role, rowd, agent=AGENT)
rowfinished = datetime.now()
rowelapsed = rowfinished - rowstarted
print("%s done (%s)" % (dtfmt(rowfinished), rowelapsed))
finished = datetime.now()
elapsed = finished - started
print("")
print("%s done (%s rows)" % (dtfmt(finished), len(rows)))
print("%s elapsed" % elapsed)
print("")
def edit_old( request, repo, org, cid, eid, role, sha1 ):
"""Edit file metadata
"""
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
f = entity.file(repo, org, cid, eid, role, sha1)
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( f.url() )
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_FILES_PARENT_LOCKED'])
return HttpResponseRedirect( f.url() )
if request.method == 'POST':
form = EditFileForm(request.POST, request.FILES)
if form.is_valid():
#f.status = form.cleaned_data['status']
#f.public = form.cleaned_data['public']
f.sort = form.cleaned_data['sort']
f.label = form.cleaned_data['label']
f.xmp = form.cleaned_data['xmp']
result = entity.file(repo, org, cid, eid, role, sha1, f)
if result in ['added','updated']:
entity.dump_json()
entity.dump_mets()
exit,status = commands.entity_update(git_name, git_mail,
entity.parent_path, entity.id,
[entity.json_path, entity.mets_path,],
agent=settings.AGENT)
if exit:
messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
else:
messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_UPDATED'])
return HttpResponseRedirect( reverse('webui-file', args=[repo,org,cid,eid,role,sha1]) )
# something went wrong
assert False
else:
data = {
#'status': f.status,
#'public': f.public,
'sort': f.sort,
'label': f.label,
'xmp': f.xmp,
}
form = EditFileForm(data)
return render_to_response(
'webui/files/edit.html',
{'repo': entity.repo,
'org': entity.org,
'cid': entity.cid,
'eid': entity.eid,
'role': file_.role,
'sha1': file_.sha1,
'collection_uid': collection.id,
'collection': collection,
'entity': entity,
'file': f,
'form': form,},
context_instance=RequestContext(request, processors=[])
)
def new( request, repo, org, cid, eid, role='master' ):
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_json(Collection.collection_path(request,repo,org,cid))
entity = Entity.from_json(Entity.entity_path(request,repo,org,cid,eid))
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
#
path = request.GET.get('path', None)
if request.method == 'POST':
form = NewFileForm(request.POST, path_choices=shared_folder_files())
if form.is_valid():
data = form.cleaned_data
src_path = data.pop('path')
role = data.pop('role')
# inheritable fields
inherited = []
for field in entity.inheritable_fields():
inherited.append( (field,getattr(entity,field)) )
# start tasks
result = entity_add_file.apply_async(
(git_name, git_mail, entity, src_path, role, data, settings.AGENT),
countdown=2)
entity.files_log(1,'START task_id %s' % result.task_id)
entity.files_log(1,'ddrlocal.webui.file.new')
entity.files_log(1,'Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
entity.files_log(1, 'locked')
else:
entity.files_log(0, lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-file-new-%s' % role,
'filename': os.path.basename(src_path),
'entity_id': entity.id,
'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# feedback
# messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_UPLOADING'] % (os.path.basename(src_path), result))
# redirect to entity
return HttpResponseRedirect( reverse('webui-entity', args=[repo,org,cid,eid]) )
else:
if not path:
messages.error(request, 'specify a path')
data = {'path': path,
'role':role,
'sort': 1,
'label': '',}
# inheritable fields
for field in entity.inheritable_fields():
data[field] = getattr(entity, field)
form = NewFileForm(data, path_choices=shared_folder_files())
return render_to_response(
'webui/files/new.html',
{'repo': entity.repo,
'org': entity.org,
'cid': entity.cid,
'eid': entity.eid,
'role': role,
'collection_uid': collection.id,
'collection': collection,
'entity': entity,
'form': form,
'path': path,},
context_instance=RequestContext(request, processors=[])
)