def next_repo( queue, local=False ):
"""Gets next collection_path or time til next ready to be updated
@param queue:
@param local: Boolean Use local per-collection locks or global lock.
@returns: collection_path or (msg,timedelta)
"""
collection_path = None
message = None
next_available = None
# sorts collections in ascending order by timestamp
collections = sorted(queue['collections'])
# now choose
if local:
# choose first collection that is not locked
for timestamp,cid in collections:
if datetime.now() > timestamp:
ci = Identifier(id=cid)
if not Collection.from_identifier(ci).locked():
return ci.path_abs()
if (not next_available) or (timestamp < next_available):
next_available = timestamp
else:
# global lock - just take the first collection
for timestamp,cid in collections:
if datetime.now() > timestamp:
ci = Identifier(id=cid)
return ci.path_abs()
if (not next_available) or (timestamp < next_available):
next_available = timestamp
return ('notready',next_available)
def new_manual( request, oid ):
"""Ask for Entity ID, then create new Entity.
"""
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
oidentifier = Identifier(oid).object()
idparts = oidentifier.idparts
collection_ids = sorted([
os.path.basename(cpath)
for cpath in Collection.collection_paths(
settings.MEDIA_BASE,
idparts['repo'],
idparts['org']
)
])
collection_ids.reverse()
if request.method == 'POST':
form = NewCollectionForm(request.POST)
if form.is_valid():
# TODO get this from Entity class or something
idparts['model'] = 'collection'
idparts['cid'] = str(form.cleaned_data['cid'])
cidentifier = Identifier(parts=idparts)
if not cidentifier:
messages.error(request, "Could not generate a legal ID from your input. Try again.")
elif cidentifier.parent_id(stubs=True) != oidentifier.id:
messages.error(request, "Can only create collections for this organization. Try again.")
elif cidentifier.id in collection_ids:
messages.error(request, "Object ID %s already exists. Try again." % cidentifier.id)
else:
# Create collection and redirect to edit page
collection = _create_collection(request, cidentifier, git_name, git_mail)
if collection:
messages.warning(request, 'IMPORTANT: Register this ID with the ID service as soon as possible!')
return HttpResponseRedirect(reverse('webui-collection-edit', args=[collection.id]))
else:
data = {
'repo': idparts['repo'],
'org': idparts['org'],
}
form = NewCollectionForm(data)
return render(request, 'webui/collections/new-manual.html', {
'form': form,
'collection_ids': collection_ids,
})
def new_manual( request, oid ):
"""Ask for Entity ID, then create new Entity.
"""
git_name,git_mail = enforce_git_credentials(request)
# note: oid could be either a Collection or an Entity
parent = Identifier(oid).object()
collection = Collection.from_identifier(
Identifier(oid).collection()
)
check_parent(collection)
oidentifier = Identifier(oid)
model = request.GET.get('model', 'entity')
if request.method == 'POST':
form = ObjectIDForm(request.POST)
if form.is_valid():
eid = form.cleaned_data['object_id']
eidentifier = Identifier(id=eid)
# Create entity and redirect to edit page
entity = _create_entity(
request,
eidentifier, collection,
git_name, git_mail
)
if entity:
messages.warning(request, 'IMPORTANT: Register this ID with the ID service as soon as possible!')
return HttpResponseRedirect(
reverse('webui-entity-edit', args=[entity.id])
)
else:
form = ObjectIDForm(initial={
'model': model,
'parent_id': oidentifier.id,
})
if isinstance(parent, Collection):
existing_ids = sorted([entity.id for entity in parent.children(quick=True)])
elif isinstance(parent, Entity):
existing_ids = sorted([e['id'] for e in parent.children_meta])
existing_ids.reverse()
return render(request, 'webui/entities/new-manual.html', {
'collection': collection,
'parent': parent,
'model': model,
'form': form,
'existing_ids': existing_ids,
})
def browse( request, repo, org, cid, eid, role='master' ):
"""Browse for a file in vbox shared folder.
"""
identifier = Identifier(request)
file_role = identifier.object()
entity = file_role.parent(stubs=True)
collection = entity.collection()
path = request.GET.get('path')
home = None
parent = None
if path:
path_abs = os.path.join(settings.VIRTUALBOX_SHARED_FOLDER, path)
parent = os.path.dirname(path)
home = settings.VIRTUALBOX_SHARED_FOLDER
else:
path_abs = settings.VIRTUALBOX_SHARED_FOLDER
listdir = []
if os.path.exists(path_abs):
for x in os.listdir(path_abs):
xabs = os.path.join(path_abs, x)
rel = xabs.replace(settings.VIRTUALBOX_SHARED_FOLDER, '')
if rel and rel[0] == '/':
rel = rel[1:]
isdir = os.path.isdir(xabs)
if isdir:
x = '%s/' % x
mtime = datetime.fromtimestamp(os.path.getmtime(xabs))
size = None
if not isdir:
size = os.path.getsize(xabs)
attribs = {'basename':x, 'rel':rel, 'path':xabs, 'isdir':isdir, 'size':size, 'mtime':mtime}
if os.path.exists(xabs):
listdir.append(attribs)
return render_to_response(
'webui/files/browse.html',
{'collection': collection,
'entity': entity,
'file_role': file_role,
'new_file_url': entity.new_file_url(role),
'shared_folder': settings.VIRTUALBOX_SHARED_FOLDER,
'listdir': listdir,
'parent': parent,
'home': home,},
context_instance=RequestContext(request, processors=[])
)
def check(request, cid):
ci = Identifier(cid)
result = collection_tasks.check.apply_async(
[ci.path_abs()],
countdown=2
)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {
'task_id': result.task_id,
'action': 'collection-check',
'collection_id': ci.id,
'collection_url': ci.urlpath('editor'),
'start': converters.datetime_to_text(datetime.now(settings.TZ)),
}
celery_tasks[result.task_id] = task
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
return HttpResponseRedirect(ci.urlpath('editor'))
def create(collection, entity_id, git_name, git_mail, agent=settings.AGENT):
"""create new entity given an entity ID
TODO remove write and commit, just create object
"""
eidentifier = Identifier(id=entity_id)
entity_path = eidentifier.path_abs()
# write entity.json template to entity location and commit
fileio.write_text(Entity(entity_path).dump_json(template=True),
settings.TEMPLATE_EJSON)
exit,status = commands.entity_create(
git_name, git_mail,
collection, eidentifier,
[collection.json_path_rel, collection.ead_path_rel],
[settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
agent=agent)
# load new entity, inherit values from parent, write and commit
entity = Entity.from_json(entity_path)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=agent)
# delete cache, update search index
collection.cache_delete()
with open(entity.json_path, 'r') as f:
document = json.loads(f.read())
try:
docstore.post(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, document)
except ConnectionError:
logger.error('Could not post to Elasticsearch.')
return entity
def new( request, repo, org, cid, eid, role='master' ):
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
fidentifier = Identifier(request)
file_role = fidentifier.object()
module = MODULES[CHILDREN_ALL[fidentifier.model]]
entity = file_role.parent(stubs=True)
collection = entity.collection()
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect(entity.absolute_url())
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect(entity.absolute_url())
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect(entity.absolute_url())
#
path = request.GET.get('path', None)
FIELDS = prep_newfile_form_fields(module.FIELDS_NEW)
if request.method == 'POST':
form = NewFileDDRForm(request.POST, fields=FIELDS, path_choices=shared_folder_files())
if form.is_valid():
data = form.cleaned_data
src_path = path
# inheritable fields
inherited = []
for field in entity.inheritable_fields():
inherited.append( (field,getattr(entity,field)) )
# start tasks
result = entity_add_file.apply_async(
(git_name, git_mail, entity, src_path, role, data, settings.AGENT),
countdown=2)
result_dict = result.__dict__
log = addfile_logger(entity.identifier)
log.ok('START task_id %s' % result.task_id)
log.ok('ddrlocal.webui.file.new')
log.ok('Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
log.ok('locked')
else:
log.not_ok(lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-file-new-%s' % role,
'filename': os.path.basename(src_path),
'entity_id': entity.id,
'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# feedback
# messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_UPLOADING'] % (os.path.basename(src_path), result))
# redirect to entity
return HttpResponseRedirect(entity.absolute_url())
else:
if not path:
messages.error(request, 'specify a path')
data = {'path': path,
'role':role,
'sort': 1,
'label': '',}
# inheritable fields
for field in entity.inheritable_fields():
data[field] = getattr(entity, field)
form = NewFileDDRForm(data, fields=FIELDS, path_choices=shared_folder_files())
return render_to_response(
'webui/files/new.html',
{'collection': collection,
'entity': entity,
'file_role': file_role,
'form': form,
'path': path,},
context_instance=RequestContext(request, processors=[])
)
def new( request, repo, org ):
"""Gets new CID from workbench, creates new collection record.
If it messes up, goes back to collection list.
"""
oidentifier = Identifier(request)
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not (git_name and git_mail):
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
ic = idservice.IDServiceClient()
# resume session
auth_status,auth_reason = ic.resume(request.session['idservice_token'])
if auth_status != 200:
request.session['idservice_username'] = None
request.session['idservice_token'] = None
messages.warning(
request,
'Session resume failed: %s %s (%s)' % (
auth_status,auth_reason,settings.IDSERVICE_API_BASE
)
)
return HttpResponseRedirect(reverse('webui-collections'))
# get new collection ID
http_status,http_reason,collection_id = ic.next_object_id(
oidentifier,
'collection'
)
if http_status not in [200,201]:
err = '%s %s' % (http_status, http_reason)
msg = WEBUI_MESSAGES['VIEWS_COLL_ERR_NO_IDS'] % (settings.IDSERVICE_API_BASE, err)
logger.error(msg)
messages.error(request, msg)
return HttpResponseRedirect(reverse('webui-collections'))
identifier = Identifier(id=collection_id, base_path=settings.MEDIA_BASE)
# create the new collection repo
collection_path = identifier.path_abs()
# collection.json template
fileio.write_text(
Collection(collection_path).dump_json(template=True),
settings.TEMPLATE_CJSON
)
exit,status = commands.create(
git_name, git_mail,
identifier,
[settings.TEMPLATE_CJSON, settings.TEMPLATE_EAD],
agent=settings.AGENT
)
if exit:
logger.error(exit)
logger.error(status)
messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
else:
# update search index
collection = Collection.from_identifier(identifier)
try:
collection.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
except ConnectionError:
logger.error('Could not post to Elasticsearch.')
gitstatus_update.apply_async((collection_path,), countdown=2)
# positive feedback
return HttpResponseRedirect( reverse('webui-collection-edit', args=collection.idparts) )
# something happened...
logger.error('Could not create new collecion!')
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_ERR_CREATE'])
return HttpResponseRedirect(reverse('webui-collections'))
def new( request, repo, org, cid ):
"""Gets new EID from workbench, creates new entity record.
If it messes up, goes back to collection.
"""
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not (git_name and git_mail):
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
collection = Collection.from_request(request)
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect(collection.absolute_url())
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect(collection.absolute_url())
ic = idservice.IDServiceClient()
# resume session
auth_status,auth_reason = ic.resume(request.session['idservice_token'])
if auth_status != 200:
request.session['idservice_username'] = None
request.session['idservice_token'] = None
messages.warning(
request,
'Session resume failed: %s %s (%s)' % (
auth_status,auth_reason,settings.IDSERVICE_API_BASE
)
)
return HttpResponseRedirect(collection.absolute_url())
# get new entity ID
http_status,http_reason,new_entity_id = ic.next_object_id(
collection.identifier,
'entity'
)
if http_status not in [200,201]:
err = '%s %s' % (http_status, http_reason)
msg = WEBUI_MESSAGES['VIEWS_ENT_ERR_NO_IDS'] % (settings.IDSERVICE_API_BASE, err)
logger.error(msg)
messages.error(request, msg)
return HttpResponseRedirect(collection.absolute_url())
eidentifier = Identifier(id=new_entity_id)
# create new entity
entity_path = eidentifier.path_abs()
# write entity.json template to entity location
fileio.write_text(
Entity(entity_path).dump_json(template=True),
settings.TEMPLATE_EJSON
)
# commit files
exit,status = commands.entity_create(
git_name, git_mail,
collection, eidentifier,
[collection.json_path_rel, collection.ead_path_rel],
[settings.TEMPLATE_EJSON, settings.TEMPLATE_METS],
agent=settings.AGENT
)
# load Entity object, inherit values from parent, write back to file
entity = Entity.from_identifier(eidentifier)
entity.inherit(collection)
entity.write_json()
updated_files = [entity.json_path]
exit,status = commands.entity_update(
git_name, git_mail,
collection, entity,
updated_files,
agent=settings.AGENT
)
collection.cache_delete()
if exit:
logger.error(exit)
logger.error(status)
messages.error(request, WEBUI_MESSAGES['ERROR'].format(status))
else:
# update search index
try:
entity.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX)
except ConnectionError:
logger.error('Could not post to Elasticsearch.')
gitstatus_update.apply_async((collection.path,), countdown=2)
# positive feedback
return HttpResponseRedirect(reverse('webui-entity-edit', args=entity.idparts))
# something happened...
logger.error('Could not create new entity!')
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_ERR_CREATE'])
return HttpResponseRedirect(collection.absolute_url())
