def next_repo( queue, local=False ): """Gets next collection_path or time til next ready to be updated @param queue: @param local: Boolean Use local per-collection locks or global lock. @returns: collection_path or (msg,timedelta) """ collection_path = None message = None next_available = None # sorts collections in ascending order by timestamp collections = sorted(queue['collections']) # now choose if local: # choose first collection that is not locked for timestamp,cid in collections: if datetime.now() > timestamp: ci = Identifier(id=cid) if not Collection.from_identifier(ci).locked(): return ci.path_abs() if (not next_available) or (timestamp < next_available): next_available = timestamp else: # global lock - just take the first collection for timestamp,cid in collections: if datetime.now() > timestamp: ci = Identifier(id=cid) return ci.path_abs() if (not next_available) or (timestamp < next_available): next_available = timestamp return ('notready',next_available)
def new_manual( request, oid ): """Ask for Entity ID, then create new Entity. """ git_name = request.session.get('git_name') git_mail = request.session.get('git_mail') if not git_name and git_mail: messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED']) oidentifier = Identifier(oid).object() idparts = oidentifier.idparts collection_ids = sorted([ os.path.basename(cpath) for cpath in Collection.collection_paths( settings.MEDIA_BASE, idparts['repo'], idparts['org'] ) ]) collection_ids.reverse() if request.method == 'POST': form = NewCollectionForm(request.POST) if form.is_valid(): # TODO get this from Entity class or something idparts['model'] = 'collection' idparts['cid'] = str(form.cleaned_data['cid']) cidentifier = Identifier(parts=idparts) if not cidentifier: messages.error(request, "Could not generate a legal ID from your input. Try again.") elif cidentifier.parent_id(stubs=True) != oidentifier.id: messages.error(request, "Can only create collections for this organization. Try again.") elif cidentifier.id in collection_ids: messages.error(request, "Object ID %s already exists. Try again." % cidentifier.id) else: # Create collection and redirect to edit page collection = _create_collection(request, cidentifier, git_name, git_mail) if collection: messages.warning(request, 'IMPORTANT: Register this ID with the ID service as soon as possible!') return HttpResponseRedirect(reverse('webui-collection-edit', args=[collection.id])) else: data = { 'repo': idparts['repo'], 'org': idparts['org'], } form = NewCollectionForm(data) return render(request, 'webui/collections/new-manual.html', { 'form': form, 'collection_ids': collection_ids, })
def new_manual( request, oid ): """Ask for Entity ID, then create new Entity. """ git_name,git_mail = enforce_git_credentials(request) # note: oid could be either a Collection or an Entity parent = Identifier(oid).object() collection = Collection.from_identifier( Identifier(oid).collection() ) check_parent(collection) oidentifier = Identifier(oid) model = request.GET.get('model', 'entity') if request.method == 'POST': form = ObjectIDForm(request.POST) if form.is_valid(): eid = form.cleaned_data['object_id'] eidentifier = Identifier(id=eid) # Create entity and redirect to edit page entity = _create_entity( request, eidentifier, collection, git_name, git_mail ) if entity: messages.warning(request, 'IMPORTANT: Register this ID with the ID service as soon as possible!') return HttpResponseRedirect( reverse('webui-entity-edit', args=[entity.id]) ) else: form = ObjectIDForm(initial={ 'model': model, 'parent_id': oidentifier.id, }) if isinstance(parent, Collection): existing_ids = sorted([entity.id for entity in parent.children(quick=True)]) elif isinstance(parent, Entity): existing_ids = sorted([e['id'] for e in parent.children_meta]) existing_ids.reverse() return render(request, 'webui/entities/new-manual.html', { 'collection': collection, 'parent': parent, 'model': model, 'form': form, 'existing_ids': existing_ids, })
def browse( request, repo, org, cid, eid, role='master' ): """Browse for a file in vbox shared folder. """ identifier = Identifier(request) file_role = identifier.object() entity = file_role.parent(stubs=True) collection = entity.collection() path = request.GET.get('path') home = None parent = None if path: path_abs = os.path.join(settings.VIRTUALBOX_SHARED_FOLDER, path) parent = os.path.dirname(path) home = settings.VIRTUALBOX_SHARED_FOLDER else: path_abs = settings.VIRTUALBOX_SHARED_FOLDER listdir = [] if os.path.exists(path_abs): for x in os.listdir(path_abs): xabs = os.path.join(path_abs, x) rel = xabs.replace(settings.VIRTUALBOX_SHARED_FOLDER, '') if rel and rel[0] == '/': rel = rel[1:] isdir = os.path.isdir(xabs) if isdir: x = '%s/' % x mtime = datetime.fromtimestamp(os.path.getmtime(xabs)) size = None if not isdir: size = os.path.getsize(xabs) attribs = {'basename':x, 'rel':rel, 'path':xabs, 'isdir':isdir, 'size':size, 'mtime':mtime} if os.path.exists(xabs): listdir.append(attribs) return render_to_response( 'webui/files/browse.html', {'collection': collection, 'entity': entity, 'file_role': file_role, 'new_file_url': entity.new_file_url(role), 'shared_folder': settings.VIRTUALBOX_SHARED_FOLDER, 'listdir': listdir, 'parent': parent, 'home': home,}, context_instance=RequestContext(request, processors=[]) )
def check(request, cid): ci = Identifier(cid) result = collection_tasks.check.apply_async( [ci.path_abs()], countdown=2 ) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = { 'task_id': result.task_id, 'action': 'collection-check', 'collection_id': ci.id, 'collection_url': ci.urlpath('editor'), 'start': converters.datetime_to_text(datetime.now(settings.TZ)), } celery_tasks[result.task_id] = task request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks return HttpResponseRedirect(ci.urlpath('editor'))
def create(collection, entity_id, git_name, git_mail, agent=settings.AGENT): """create new entity given an entity ID TODO remove write and commit, just create object """ eidentifier = Identifier(id=entity_id) entity_path = eidentifier.path_abs() # write entity.json template to entity location and commit fileio.write_text(Entity(entity_path).dump_json(template=True), settings.TEMPLATE_EJSON) exit,status = commands.entity_create( git_name, git_mail, collection, eidentifier, [collection.json_path_rel, collection.ead_path_rel], [settings.TEMPLATE_EJSON, settings.TEMPLATE_METS], agent=agent) # load new entity, inherit values from parent, write and commit entity = Entity.from_json(entity_path) entity.inherit(collection) entity.write_json() updated_files = [entity.json_path] exit,status = commands.entity_update( git_name, git_mail, collection, entity, updated_files, agent=agent) # delete cache, update search index collection.cache_delete() with open(entity.json_path, 'r') as f: document = json.loads(f.read()) try: docstore.post(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX, document) except ConnectionError: logger.error('Could not post to Elasticsearch.') return entity
def new( request, repo, org, cid, eid, role='master' ): git_name = request.session.get('git_name') git_mail = request.session.get('git_mail') if not git_name and git_mail: messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED']) fidentifier = Identifier(request) file_role = fidentifier.object() module = MODULES[CHILDREN_ALL[fidentifier.model]] entity = file_role.parent(stubs=True) collection = entity.collection() if collection.locked(): messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id)) return HttpResponseRedirect(entity.absolute_url()) if collection.repo_behind(): messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id)) return HttpResponseRedirect(entity.absolute_url()) if entity.locked(): messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED']) return HttpResponseRedirect(entity.absolute_url()) # path = request.GET.get('path', None) FIELDS = prep_newfile_form_fields(module.FIELDS_NEW) if request.method == 'POST': form = NewFileDDRForm(request.POST, fields=FIELDS, path_choices=shared_folder_files()) if form.is_valid(): data = form.cleaned_data src_path = path # inheritable fields inherited = [] for field in entity.inheritable_fields(): inherited.append( (field,getattr(entity,field)) ) # start tasks result = entity_add_file.apply_async( (git_name, git_mail, entity, src_path, role, data, settings.AGENT), countdown=2) result_dict = result.__dict__ log = addfile_logger(entity.identifier) log.ok('START task_id %s' % result.task_id) log.ok('ddrlocal.webui.file.new') log.ok('Locking %s' % entity.id) # lock entity lockstatus = entity.lock(result.task_id) if lockstatus == 'ok': log.ok('locked') else: log.not_ok(lockstatus) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = {'task_id': result.task_id, 'action': 'webui-file-new-%s' % role, 'filename': os.path.basename(src_path), 'entity_id': entity.id, 'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),} celery_tasks[result.task_id] = task #del request.session[settings.CELERY_TASKS_SESSION_KEY] request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks # feedback # messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_UPLOADING'] % (os.path.basename(src_path), result)) # redirect to entity return HttpResponseRedirect(entity.absolute_url()) else: if not path: messages.error(request, 'specify a path') data = {'path': path, 'role':role, 'sort': 1, 'label': '',} # inheritable fields for field in entity.inheritable_fields(): data[field] = getattr(entity, field) form = NewFileDDRForm(data, fields=FIELDS, path_choices=shared_folder_files()) return render_to_response( 'webui/files/new.html', {'collection': collection, 'entity': entity, 'file_role': file_role, 'form': form, 'path': path,}, context_instance=RequestContext(request, processors=[]) )
def new( request, repo, org ): """Gets new CID from workbench, creates new collection record. If it messes up, goes back to collection list. """ oidentifier = Identifier(request) git_name = request.session.get('git_name') git_mail = request.session.get('git_mail') if not (git_name and git_mail): messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED']) ic = idservice.IDServiceClient() # resume session auth_status,auth_reason = ic.resume(request.session['idservice_token']) if auth_status != 200: request.session['idservice_username'] = None request.session['idservice_token'] = None messages.warning( request, 'Session resume failed: %s %s (%s)' % ( auth_status,auth_reason,settings.IDSERVICE_API_BASE ) ) return HttpResponseRedirect(reverse('webui-collections')) # get new collection ID http_status,http_reason,collection_id = ic.next_object_id( oidentifier, 'collection' ) if http_status not in [200,201]: err = '%s %s' % (http_status, http_reason) msg = WEBUI_MESSAGES['VIEWS_COLL_ERR_NO_IDS'] % (settings.IDSERVICE_API_BASE, err) logger.error(msg) messages.error(request, msg) return HttpResponseRedirect(reverse('webui-collections')) identifier = Identifier(id=collection_id, base_path=settings.MEDIA_BASE) # create the new collection repo collection_path = identifier.path_abs() # collection.json template fileio.write_text( Collection(collection_path).dump_json(template=True), settings.TEMPLATE_CJSON ) exit,status = commands.create( git_name, git_mail, identifier, [settings.TEMPLATE_CJSON, settings.TEMPLATE_EAD], agent=settings.AGENT ) if exit: logger.error(exit) logger.error(status) messages.error(request, WEBUI_MESSAGES['ERROR'].format(status)) else: # update search index collection = Collection.from_identifier(identifier) try: collection.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX) except ConnectionError: logger.error('Could not post to Elasticsearch.') gitstatus_update.apply_async((collection_path,), countdown=2) # positive feedback return HttpResponseRedirect( reverse('webui-collection-edit', args=collection.idparts) ) # something happened... logger.error('Could not create new collecion!') messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_ERR_CREATE']) return HttpResponseRedirect(reverse('webui-collections'))
def new( request, repo, org, cid ): """Gets new EID from workbench, creates new entity record. If it messes up, goes back to collection. """ git_name = request.session.get('git_name') git_mail = request.session.get('git_mail') if not (git_name and git_mail): messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED']) collection = Collection.from_request(request) if collection.locked(): messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id)) return HttpResponseRedirect(collection.absolute_url()) collection.repo_fetch() if collection.repo_behind(): messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id)) return HttpResponseRedirect(collection.absolute_url()) ic = idservice.IDServiceClient() # resume session auth_status,auth_reason = ic.resume(request.session['idservice_token']) if auth_status != 200: request.session['idservice_username'] = None request.session['idservice_token'] = None messages.warning( request, 'Session resume failed: %s %s (%s)' % ( auth_status,auth_reason,settings.IDSERVICE_API_BASE ) ) return HttpResponseRedirect(collection.absolute_url()) # get new entity ID http_status,http_reason,new_entity_id = ic.next_object_id( collection.identifier, 'entity' ) if http_status not in [200,201]: err = '%s %s' % (http_status, http_reason) msg = WEBUI_MESSAGES['VIEWS_ENT_ERR_NO_IDS'] % (settings.IDSERVICE_API_BASE, err) logger.error(msg) messages.error(request, msg) return HttpResponseRedirect(collection.absolute_url()) eidentifier = Identifier(id=new_entity_id) # create new entity entity_path = eidentifier.path_abs() # write entity.json template to entity location fileio.write_text( Entity(entity_path).dump_json(template=True), settings.TEMPLATE_EJSON ) # commit files exit,status = commands.entity_create( git_name, git_mail, collection, eidentifier, [collection.json_path_rel, collection.ead_path_rel], [settings.TEMPLATE_EJSON, settings.TEMPLATE_METS], agent=settings.AGENT ) # load Entity object, inherit values from parent, write back to file entity = Entity.from_identifier(eidentifier) entity.inherit(collection) entity.write_json() updated_files = [entity.json_path] exit,status = commands.entity_update( git_name, git_mail, collection, entity, updated_files, agent=settings.AGENT ) collection.cache_delete() if exit: logger.error(exit) logger.error(status) messages.error(request, WEBUI_MESSAGES['ERROR'].format(status)) else: # update search index try: entity.post_json(settings.DOCSTORE_HOSTS, settings.DOCSTORE_INDEX) except ConnectionError: logger.error('Could not post to Elasticsearch.') gitstatus_update.apply_async((collection.path,), countdown=2) # positive feedback return HttpResponseRedirect(reverse('webui-entity-edit', args=entity.idparts)) # something happened... logger.error('Could not create new entity!') messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_ERR_CREATE']) return HttpResponseRedirect(collection.absolute_url())