def dumps(info, source): """Dump raw gitolite info and source to cache value with timestamp Response from Gitolite looks like this: hello ddr, this is git@mits running gitolite3 v3.2-19-gb9bbb78 on git 1.7.2.5 R W C ddr-test-[0-9]+ R W C ddr-test-[0-9]+-[0-9]+ R W ddr-test R W ddr-test-1 ... Timestamp and source are prepended to the cached value: 2014-08-20T09:39:28:386187 [email protected] hello ddr, this is git@mits running gitolite3 v3.2-19-gb9bbb78 on git 1.7.2.5 R W C ddr-test-[0-9]+ ... @param info: str @param source: str @returns: str """ timestamp = converters.datetime_to_text(datetime.now(settings.TZ)) text = '%s %s\n%s' % (timestamp, source, info) return text
def csv_export( request, cid, model=None ): """ """ if (not model) or (not (model in ['entity','file'])): raise Http404 collection = Collection.from_identifier(Identifier(cid)) things = {'entity':'objects', 'file':'files'} csv_path = settings.CSV_EXPORT_PATH[model] % collection.id csv_filename = os.path.basename(csv_path) if model == 'entity': file_url = reverse('webui-collection-csv-entities', args=[collection.id]) elif model == 'file': file_url = reverse('webui-collection-csv-files', args=[collection.id]) # do it result = collection_tasks.csv_export_model.apply_async( (collection.path,model), countdown=2 ) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = {'task_id': result.task_id, 'action': 'csv-export-model', 'collection_id': collection.id, 'collection_url': collection.absolute_url(), 'things': things[model], 'file_name': csv_filename, 'file_url': file_url, 'start': converters.datetime_to_text(datetime.now(settings.TZ)),} celery_tasks[result.task_id] = task request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks return HttpResponseRedirect(collection.absolute_url())
def write_changelog_entry(path, messages, user, email, timestamp=None): logging.debug(' write_changelog_entry({})'.format(path)) template = load_template(CHANGELOG_TEMPLATE) date_format = load_template(CHANGELOG_DATE_FORMAT) # one line per message lines = [] [lines.append('* {}'.format(m)) for m in messages] changes = '\n'.join(lines) if not timestamp: timestamp = datetime.now(converters.config.TZ) # render entry = template.format(changes=changes, user=user, email=email, date=converters.datetime_to_text( timestamp, converters.config.PRETTY_DATETIME_FORMAT)) try: preexisting = os.path.getsize(path) except: preexisting = False with open(path, 'a') as f: if preexisting: f.write('\n') f.write(entry)
def dtfmt(dt, fmt='%Y-%m-%dT%H:%M:%S.%f'): """Format dates in consistent format. @param dt: datetime @param fmt: str Format string (default: '%Y-%m-%dT%H:%M:%S.%f') @returns: str """ return converters.datetime_to_text(fmt)
def sync_status_ajax( request, cid ): collection = Collection.from_identifier(Identifier(cid)) gitstatus = collection.gitstatus() if gitstatus: sync_status = gitstatus['sync_status'] if sync_status.get('timestamp',None): sync_status['timestamp'] = converters.datetime_to_text(sync_status['timestamp']) return HttpResponse(json.dumps(sync_status), content_type="application/json") raise Http404
def jsondump_record_lastmod(data): return converters.datetime_to_text(data) # display_* --- Display functions -------------------------------------- # # These functions take Python data from the corresponding Entity field # and format it for display. # # id def display_record_created(data):
def new_access( request, fid ): """Generate a new access file for the specified file. NOTE: There is no GET for this view. GET requests will redirect to entity. """ enforce_git_credentials(request) file_ = DDRFile.from_identifier(Identifier(fid)) check_file(file_) entity = file_.parent() collection = file_.collection() check_parents(entity, collection) # if request.method == 'POST': form = NewAccessFileForm(request.POST) if form.is_valid(): src_path = form.cleaned_data['path'] # start tasks result = file_tasks.add_access.apply_async( ( request.session['git_name'], request.session['git_mail'], entity, file_ ), countdown=2 ) result_dict = result.__dict__ log = addfile_logger(entity.identifier) log.ok('START task_id %s' % result.task_id) log.ok('ddrlocal.webui.file.new_access') log.ok('Locking %s' % entity.id) # lock entity lockstatus = entity.lock(result.task_id) if lockstatus == 'ok': log.ok( 'locked') else: log.not_ok( lockstatus) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = {'task_id': result.task_id, 'action': 'webui-file-new-access', 'filename': os.path.basename(src_path), 'file_url': file_.absolute_url(), 'entity_id': entity.id, 'start': converters.datetime_to_text(datetime.now(settings.TZ)),} celery_tasks[result.task_id] = task #del request.session[settings.CELERY_TASKS_SESSION_KEY] request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks # feedback #messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_NEWACCESS'] % os.path.basename(src_path)) # redirect to entity return HttpResponseRedirect(entity.absolute_url())
def make_entry(messages, user, mail, timestamp=None): """Makes a (new-style) changelog entry. @param messages: List of strings. @param user: Person's name. @param mail: A valid email address. @param timestamp: datetime (optional). @returns string """ if not timestamp: timestamp = datetime.now(converters.config.TZ) stamp = '%s -- %s <%s>' % (converters.datetime_to_text(timestamp), user, mail) lines = [stamp] + ['* %s' % m for m in messages] return '\n'.join(lines)
def reindex( request ): if request.method == 'POST': form = IndexConfirmForm(request.POST, request=request) if form.is_valid(): index = form.cleaned_data['index'] if index: result = docstore_tasks.reindex.apply_async( [index], countdown=2) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = {'task_id': result.task_id, 'action': 'webui-search-reindex', 'index': index, 'start': converters.datetime_to_text(datetime.now(settings.TZ)),} celery_tasks[result.task_id] = task request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks return HttpResponseRedirect( reverse('webui-search-admin') )
def reindex_and_notify( index ): """Drop existing index and build another from scratch; hand off to Celery. This function is intended for use in a view. """ result = reindex(index).apply_async( countdown=2 ) celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = { 'task_id': result.task_id, 'action': 'search-reindex', 'index': index, 'start': converters.datetime_to_text(datetime.now(settings.TZ)), } celery_tasks[result.task_id] = task request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
def write_changelog_entry(path, messages, user, email, timestamp=None): logging.debug(' write_changelog_entry({})'.format(path)) template = load_template(CHANGELOG_TEMPLATE) date_format = load_template(CHANGELOG_DATE_FORMAT) # one line per message lines = [] [lines.append('* {}'.format(m)) for m in messages] changes = '\n'.join(lines) if not timestamp: timestamp = datetime.now(converters.config.TZ) # render entry = template.format(changes=changes, user=user, email=email, date=converters.datetime_to_text( timestamp, converters.config.PRETTY_DATETIME_FORMAT)) fileio.append_text(entry, path)
def signatures( request, cid ): try: collection = Collection.from_identifier(Identifier(cid)) except: raise Http404 git_name = request.session.get('git_name') git_mail = request.session.get('git_mail') if not git_name and git_mail: messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED']) alert_if_conflicted(request, collection) if collection.locked(): messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id)) return HttpResponseRedirect(collection.absolute_url()) if request.method == 'POST': form = SignaturesConfirmForm(request.POST) form_is_valid = form.is_valid() if form.is_valid() and form.cleaned_data['confirmed']: result = collection_tasks.signatures.apply_async( (collection.path,git_name,git_mail), countdown=2 ) lockstatus = collection.lock(result.task_id) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = { 'task_id': result.task_id, 'action': 'collection-signatures', 'collection_id': collection.id, 'collection_url': collection.absolute_url(), 'start': converters.datetime_to_text(datetime.now(settings.TZ)), } celery_tasks[result.task_id] = task request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks return HttpResponseRedirect(collection.absolute_url()) else: form = SignaturesConfirmForm() return render(request, 'webui/collections/signatures-confirm.html', { 'collection': collection, 'form': form, })
def refresh(): """ Check the cached value of DDR.dvcs.gitolite_info(). If it is stale (e.g. timestamp is older than cutoff) then hit the Gitolite server for an update and re-cache. """ logger.debug('gitolite_info_check') gitolite = dvcs.Gitolite() feedback = [] needs_update = None cached = cache.get(GITOLITE_INFO_CACHE_KEY) if cached: feedback.append('cached') try: timestamp,source,info = loads(cached) feedback.append(converters.datetime_to_text(timestamp)) feedback.append(source) except ValueError: timestamp,source,info = None,None,None feedback.append('malformed') needs_update = True if timestamp: elapsed = datetime.now(settings.TZ) - timestamp cutoff = timedelta(seconds=settings.GITOLITE_INFO_CACHE_CUTOFF) if elapsed > cutoff: needs_update = True feedback.append('stale') else: needs_update = True feedback.append('missing') if needs_update: gitolite.initialize() if gitolite.info: cached = dumps(gitolite.info, settings.GITOLITE) cache.set( GITOLITE_INFO_CACHE_KEY, cached, settings.GITOLITE_INFO_CACHE_TIMEOUT ) feedback.append('refreshed') else: feedback.append('ok') return ' '.join(feedback)
def to_dict(document, module, json_safe=False): """Returns an OrderedDict containing the object fields and values. @param document: Collection, Entity, File document object @param module: collection, entity, files model definitions module @param json_safe: bool Serialize Python objects e.g. datetime to text @returns: OrderedDict """ data = OrderedDict() for f in module.FIELDS: fieldname = f['name'] field_data = getattr(document, f['name']) if json_safe: if isinstance(field_data, Identifier): field_data = str(field_data) elif isinstance(field_data, datetime): field_data = converters.datetime_to_text(field_data) data[fieldname] = field_data return data
def check(request, cid): ci = Identifier(cid) result = collection_tasks.check.apply_async( [ci.path_abs()], countdown=2 ) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = { 'task_id': result.task_id, 'action': 'collection-check', 'collection_id': ci.id, 'collection_url': ci.urlpath('editor'), 'start': converters.datetime_to_text(datetime.now(settings.TZ)), } celery_tasks[result.task_id] = task request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks return HttpResponseRedirect(ci.urlpath('editor'))
def entity_reload_files(request, collection, entity, git_name, git_mail, agent): # start tasks result = reload_files.apply_async( (collection.path, entity.id, git_name, git_mail, agent), countdown=2 ) # lock collection lockstatus = entity.lock(result.task_id) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. celery_tasks[result.task_id] = { 'task_id': result.task_id, 'action': 'entity-reload-files', 'entity_url': entity.absolute_url(), 'entity_id': entity.id, 'start': converters.datetime_to_text(datetime.now(settings.TZ)),} request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
def _format_datetimes(data): """force datetimes into ES mapping format # TODO refactor this once we get it working """ DATETIME_FIELDS = [ 'record_created', 'record_lastmod', ] for field, value in data.iteritems(): if field in DATETIME_FIELDS: dt = converters.text_to_datetime(value) # Use default timezone unless... if data['org'] in config.ALT_TIMEZONES.keys(): timezone = config.ALT_TIMEZONES[data['org']] else: timezone = config.TZ if not dt.tzinfo: timezone.localize(dt) data[field] = converters.datetime_to_text( dt, config.ELASTICSEARCH_DATETIME_FORMAT)
def edit(request, collection, cleaned_data, git_name, git_mail): # start tasks result = save.apply_async( (collection.path, cleaned_data, git_name, git_mail), countdown=2 ) # lock collection lockstatus = collection.lock(result.task_id) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. celery_tasks[result.task_id] = { 'task_id': result.task_id, 'action': 'collection-edit', 'collection_url': collection.absolute_url(), 'collection_id': collection.id, 'start': converters.datetime_to_text(datetime.now(settings.TZ)),} request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
def _flatten_json(self) -> Dict[str, str]: """Converts Term into a dict suitable for writing to JSON. """ data = {} for key, val in self.__dict__.items(): if (key in ['parent_id']) and val: val = getattr(self, key) elif (key in ['siblings', 'children']) and val: val = [] for t in getattr(self, key): val.append(t.id) elif (key in ['created', 'modified']): val_before = val if isinstance(val, datetime): val = converters.datetime_to_text(val) elif isinstance(val, str): pass else: val = None data[key] = val return data
def _format_datetimes(data): """force datetimes into ES mapping format # TODO refactor this once we get it working """ DATETIME_FIELDS = [ 'record_created', 'record_lastmod', ] for field,value in data.iteritems(): if field in DATETIME_FIELDS: dt = converters.text_to_datetime(value) # Use default timezone unless... if data['org'] in config.ALT_TIMEZONES.keys(): timezone = config.ALT_TIMEZONES[data['org']] else: timezone = config.TZ if not dt.tzinfo: timezone.localize(dt) data[field] = converters.datetime_to_text( dt, config.ELASTICSEARCH_DATETIME_FORMAT )
def to_dict(document, module, json_safe=False): """Returns an OrderedDict containing the object fields and values. All fields of the object *definition* are included. Fields missing from the provided object are included as blank strings (''). @param document: Collection, Entity, File document object @param module: collection, entity, files model definitions module @param json_safe: bool Serialize Python objects e.g. datetime to text @returns: OrderedDict """ data = OrderedDict() for f in module.FIELDS: fieldname = f['name'] field_data = getattr(document, f['name'], '') if json_safe: if isinstance(field_data, Identifier): field_data = str(field_data) elif isinstance(field_data, datetime): field_data = converters.datetime_to_text(field_data) data[fieldname] = field_data return data
def new_external(request, rid): """Enter initial data for external file An external file is one that is external to the DDR collection. The hashes are known but the binary file itself is not present within the collection. """ file_role = Stub.from_identifier(Identifier(rid)) role = file_role.identifier.parts['role'] entity = file_role.parent(stubs=True) collection = entity.collection() check_parents(entity, collection, check_locks=0, fetch=0) if request.method == 'POST': form = NewExternalFileForm(request.POST) if form.is_valid(): idparts = file_role.identifier.idparts idparts['model'] = 'file' idparts['sha1'] = form.cleaned_data['sha1'] fi = Identifier(parts=idparts) basename_orig = form.cleaned_data['filename'] data = { 'id': fi.id, 'external': 1, 'role': role, 'basename_orig': basename_orig, 'sha1': form.cleaned_data['sha1'], 'sha256': form.cleaned_data['sha256'], 'md5': form.cleaned_data['md5'], 'size': form.cleaned_data['size'], 'mimetype': form.cleaned_data['mimetype'], } # start tasks result = file_tasks.add_external.apply_async( ( request.session['git_name'], request.session['git_mail'], entity, data, settings.AGENT ), countdown=2) result_dict = result.__dict__ log = addfile_logger(entity.identifier) log.ok('START task_id %s' % result.task_id) log.ok('ddrlocal.webui.file.new_external') log.ok('Locking %s' % entity.id) # lock entity lockstatus = entity.lock(result.task_id) if lockstatus == 'ok': log.ok('locked') else: log.not_ok(lockstatus) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = { 'task_id': result.task_id, 'action': 'webui-file-new-external', 'filename': os.path.basename(basename_orig), 'entity_id': entity.id, 'start': converters.datetime_to_text(datetime.now(settings.TZ)), } celery_tasks[result.task_id] = task #del request.session[settings.CELERY_TASKS_SESSION_KEY] request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks # redirect to entity return HttpResponseRedirect(entity.absolute_url()) else: form = NewExternalFileForm() return render(request, 'webui/files/new-external.html', { 'collection': collection, 'entity': entity, 'file_role': file_role, 'form': form, })
def index_record_lastmod(data): return converters.datetime_to_text( data, converters.config.ELASTICSEARCH_DATETIME_FORMAT)
def csvdump_record_lastmod(data): return converters.datetime_to_text(data) def csvdump_creators(data): return converters.rolepeople_to_text(data)
def new( request, rid ): enforce_git_credentials(request) file_role = Stub.from_identifier(Identifier(rid)) entity = file_role.parent(stubs=True) collection = entity.collection() check_parents(entity, collection, fetch=0) role = file_role.identifier.parts['role'] #child_models = CHILDREN_ALL[file_role.identifier.model] FILE_MODEL = 'file' module = MODULES[FILE_MODEL] # path = request.GET.get('path', None) FIELDS = prep_newfile_form_fields(module.FIELDS_NEW) if request.method == 'POST': form = NewFileDDRForm(request.POST, fields=FIELDS, path_choices=shared_folder_files()) if form.is_valid(): data = form.cleaned_data src_path = path # start tasks result = file_tasks.add_file.apply_async( ( request.session['git_name'], request.session['git_mail'], entity, src_path, role, data, settings.AGENT ), countdown=2) result_dict = result.__dict__ log = addfile_logger(entity.identifier) log.ok('START task_id %s' % result.task_id) log.ok('ddrlocal.webui.file.new') log.ok('Locking %s' % entity.id) # lock entity lockstatus = entity.lock(result.task_id) if lockstatus == 'ok': log.ok('locked') else: log.not_ok(lockstatus) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = {'task_id': result.task_id, 'action': 'webui-file-new-%s' % role, 'filename': os.path.basename(src_path), 'entity_id': entity.id, 'start': converters.datetime_to_text(datetime.now(settings.TZ)),} celery_tasks[result.task_id] = task #del request.session[settings.CELERY_TASKS_SESSION_KEY] request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks # feedback # messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_UPLOADING'] % (os.path.basename(src_path), result)) # redirect to entity return HttpResponseRedirect(entity.absolute_url()) else: if not path: messages.error(request, 'specify a path') data = {'path': path, 'role':role, 'sort': 1, 'label': '',} # inheritable fields for field in entity.inheritable_fields(): data[field] = getattr(entity, field) form = NewFileDDRForm(data, fields=FIELDS, path_choices=shared_folder_files()) return render(request, 'webui/files/new.html', { 'collection': collection, 'entity': entity, 'file_role': file_role, 'form': form, 'path': path, })
def index_record_lastmod(data): return converters.datetime_to_text( data, converters.config.ELASTICSEARCH_DATETIME_FORMAT )
def csvdump_record_created(data): return converters.datetime_to_text(data) def csvdump_record_lastmod(data): return converters.datetime_to_text(data)
def test_datetime_to_text(): assert converters.datetime_to_text(TEXT_DATETIME_DATA_NOTZ) == TEXT_DATETIME_TEXT0
def display_record_lastmod(data): return converters.datetime_to_text( data, converters.config.PRETTY_DATETIME_FORMAT )
def ead_record_lastmod(tree, namespaces, field, value): return _set_attr(tree, namespaces, "/ead/eadheader/eadid", "record_lastmod", converters.datetime_to_text(value))
def jsondump_record_created(data): return converters.datetime_to_text(data)
def jsondump_record_lastmod(data): return converters.datetime_to_text(data)
def test_datetime_to_text(): assert converters.datetime_to_text( TEXT_DATETIME_DATA_NOTZ) == TEXT_DATETIME_TEXT0
def display_record_lastmod(data): return converters.datetime_to_text( data, converters.config.PRETTY_DATETIME_FORMAT)