def dumps(info, source):
"""Dump raw gitolite info and source to cache value with timestamp
Response from Gitolite looks like this:
hello ddr, this is git@mits running gitolite3 v3.2-19-gb9bbb78 on git 1.7.2.5
R W C ddr-test-[0-9]+
R W C ddr-test-[0-9]+-[0-9]+
R W ddr-test
R W ddr-test-1
...
Timestamp and source are prepended to the cached value:
2014-08-20T09:39:28:386187 [email protected]
hello ddr, this is git@mits running gitolite3 v3.2-19-gb9bbb78 on git 1.7.2.5
R W C ddr-test-[0-9]+
...
@param info: str
@param source: str
@returns: str
"""
timestamp = converters.datetime_to_text(datetime.now(settings.TZ))
text = '%s %s\n%s' % (timestamp, source, info)
return text
def csv_export( request, cid, model=None ):
"""
"""
if (not model) or (not (model in ['entity','file'])):
raise Http404
collection = Collection.from_identifier(Identifier(cid))
things = {'entity':'objects', 'file':'files'}
csv_path = settings.CSV_EXPORT_PATH[model] % collection.id
csv_filename = os.path.basename(csv_path)
if model == 'entity':
file_url = reverse('webui-collection-csv-entities', args=[collection.id])
elif model == 'file':
file_url = reverse('webui-collection-csv-files', args=[collection.id])
# do it
result = collection_tasks.csv_export_model.apply_async(
(collection.path,model),
countdown=2
)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'csv-export-model',
'collection_id': collection.id,
'collection_url': collection.absolute_url(),
'things': things[model],
'file_name': csv_filename,
'file_url': file_url,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),}
celery_tasks[result.task_id] = task
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
return HttpResponseRedirect(collection.absolute_url())
def write_changelog_entry(path, messages, user, email, timestamp=None):
logging.debug(' write_changelog_entry({})'.format(path))
template = load_template(CHANGELOG_TEMPLATE)
date_format = load_template(CHANGELOG_DATE_FORMAT)
# one line per message
lines = []
[lines.append('* {}'.format(m)) for m in messages]
changes = '\n'.join(lines)
if not timestamp:
timestamp = datetime.now(converters.config.TZ)
# render
entry = template.format(changes=changes,
user=user,
email=email,
date=converters.datetime_to_text(
timestamp,
converters.config.PRETTY_DATETIME_FORMAT))
try:
preexisting = os.path.getsize(path)
except:
preexisting = False
with open(path, 'a') as f:
if preexisting:
f.write('\n')
f.write(entry)
def dtfmt(dt, fmt='%Y-%m-%dT%H:%M:%S.%f'):
"""Format dates in consistent format.
@param dt: datetime
@param fmt: str Format string (default: '%Y-%m-%dT%H:%M:%S.%f')
@returns: str
"""
return converters.datetime_to_text(fmt)
def sync_status_ajax( request, cid ):
collection = Collection.from_identifier(Identifier(cid))
gitstatus = collection.gitstatus()
if gitstatus:
sync_status = gitstatus['sync_status']
if sync_status.get('timestamp',None):
sync_status['timestamp'] = converters.datetime_to_text(sync_status['timestamp'])
return HttpResponse(json.dumps(sync_status), content_type="application/json")
raise Http404
def jsondump_record_lastmod(data): return converters.datetime_to_text(data) # display_* --- Display functions -------------------------------------- # # These functions take Python data from the corresponding Entity field # and format it for display. # # id def display_record_created(data):
def new_access( request, fid ):
"""Generate a new access file for the specified file.
NOTE: There is no GET for this view. GET requests will redirect to entity.
"""
enforce_git_credentials(request)
file_ = DDRFile.from_identifier(Identifier(fid))
check_file(file_)
entity = file_.parent()
collection = file_.collection()
check_parents(entity, collection)
#
if request.method == 'POST':
form = NewAccessFileForm(request.POST)
if form.is_valid():
src_path = form.cleaned_data['path']
# start tasks
result = file_tasks.add_access.apply_async(
(
request.session['git_name'], request.session['git_mail'],
entity, file_
),
countdown=2
)
result_dict = result.__dict__
log = addfile_logger(entity.identifier)
log.ok('START task_id %s' % result.task_id)
log.ok('ddrlocal.webui.file.new_access')
log.ok('Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
log.ok( 'locked')
else:
log.not_ok( lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-file-new-access',
'filename': os.path.basename(src_path),
'file_url': file_.absolute_url(),
'entity_id': entity.id,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# feedback
#messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_NEWACCESS'] % os.path.basename(src_path))
# redirect to entity
return HttpResponseRedirect(entity.absolute_url())
def make_entry(messages, user, mail, timestamp=None):
"""Makes a (new-style) changelog entry.
@param messages: List of strings.
@param user: Person's name.
@param mail: A valid email address.
@param timestamp: datetime (optional).
@returns string
"""
if not timestamp:
timestamp = datetime.now(converters.config.TZ)
stamp = '%s -- %s <%s>' % (converters.datetime_to_text(timestamp), user,
mail)
lines = [stamp] + ['* %s' % m for m in messages]
return '\n'.join(lines)
def reindex( request ):
if request.method == 'POST':
form = IndexConfirmForm(request.POST, request=request)
if form.is_valid():
index = form.cleaned_data['index']
if index:
result = docstore_tasks.reindex.apply_async( [index], countdown=2)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-search-reindex',
'index': index,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),}
celery_tasks[result.task_id] = task
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
return HttpResponseRedirect( reverse('webui-search-admin') )
def reindex_and_notify( index ):
"""Drop existing index and build another from scratch; hand off to Celery.
This function is intended for use in a view.
"""
result = reindex(index).apply_async(
countdown=2
)
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {
'task_id': result.task_id,
'action': 'search-reindex',
'index': index,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),
}
celery_tasks[result.task_id] = task
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
def write_changelog_entry(path, messages, user, email, timestamp=None):
logging.debug(' write_changelog_entry({})'.format(path))
template = load_template(CHANGELOG_TEMPLATE)
date_format = load_template(CHANGELOG_DATE_FORMAT)
# one line per message
lines = []
[lines.append('* {}'.format(m)) for m in messages]
changes = '\n'.join(lines)
if not timestamp:
timestamp = datetime.now(converters.config.TZ)
# render
entry = template.format(changes=changes,
user=user,
email=email,
date=converters.datetime_to_text(
timestamp,
converters.config.PRETTY_DATETIME_FORMAT))
fileio.append_text(entry, path)
def signatures( request, cid ):
try:
collection = Collection.from_identifier(Identifier(cid))
except:
raise Http404
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
alert_if_conflicted(request, collection)
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect(collection.absolute_url())
if request.method == 'POST':
form = SignaturesConfirmForm(request.POST)
form_is_valid = form.is_valid()
if form.is_valid() and form.cleaned_data['confirmed']:
result = collection_tasks.signatures.apply_async(
(collection.path,git_name,git_mail),
countdown=2
)
lockstatus = collection.lock(result.task_id)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {
'task_id': result.task_id,
'action': 'collection-signatures',
'collection_id': collection.id,
'collection_url': collection.absolute_url(),
'start': converters.datetime_to_text(datetime.now(settings.TZ)),
}
celery_tasks[result.task_id] = task
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
return HttpResponseRedirect(collection.absolute_url())
else:
form = SignaturesConfirmForm()
return render(request, 'webui/collections/signatures-confirm.html', {
'collection': collection,
'form': form,
})
def refresh():
"""
Check the cached value of DDR.dvcs.gitolite_info().
If it is stale (e.g. timestamp is older than cutoff)
then hit the Gitolite server for an update and re-cache.
"""
logger.debug('gitolite_info_check')
gitolite = dvcs.Gitolite()
feedback = []
needs_update = None
cached = cache.get(GITOLITE_INFO_CACHE_KEY)
if cached:
feedback.append('cached')
try:
timestamp,source,info = loads(cached)
feedback.append(converters.datetime_to_text(timestamp))
feedback.append(source)
except ValueError:
timestamp,source,info = None,None,None
feedback.append('malformed')
needs_update = True
if timestamp:
elapsed = datetime.now(settings.TZ) - timestamp
cutoff = timedelta(seconds=settings.GITOLITE_INFO_CACHE_CUTOFF)
if elapsed > cutoff:
needs_update = True
feedback.append('stale')
else:
needs_update = True
feedback.append('missing')
if needs_update:
gitolite.initialize()
if gitolite.info:
cached = dumps(gitolite.info, settings.GITOLITE)
cache.set(
GITOLITE_INFO_CACHE_KEY,
cached,
settings.GITOLITE_INFO_CACHE_TIMEOUT
)
feedback.append('refreshed')
else:
feedback.append('ok')
return ' '.join(feedback)
def to_dict(document, module, json_safe=False):
"""Returns an OrderedDict containing the object fields and values.
@param document: Collection, Entity, File document object
@param module: collection, entity, files model definitions module
@param json_safe: bool Serialize Python objects e.g. datetime to text
@returns: OrderedDict
"""
data = OrderedDict()
for f in module.FIELDS:
fieldname = f['name']
field_data = getattr(document, f['name'])
if json_safe:
if isinstance(field_data, Identifier):
field_data = str(field_data)
elif isinstance(field_data, datetime):
field_data = converters.datetime_to_text(field_data)
data[fieldname] = field_data
return data
def check(request, cid):
ci = Identifier(cid)
result = collection_tasks.check.apply_async(
[ci.path_abs()],
countdown=2
)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {
'task_id': result.task_id,
'action': 'collection-check',
'collection_id': ci.id,
'collection_url': ci.urlpath('editor'),
'start': converters.datetime_to_text(datetime.now(settings.TZ)),
}
celery_tasks[result.task_id] = task
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
return HttpResponseRedirect(ci.urlpath('editor'))
def entity_reload_files(request, collection, entity, git_name, git_mail, agent):
# start tasks
result = reload_files.apply_async(
(collection.path, entity.id, git_name, git_mail, agent),
countdown=2
)
# lock collection
lockstatus = entity.lock(result.task_id)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
celery_tasks[result.task_id] = {
'task_id': result.task_id,
'action': 'entity-reload-files',
'entity_url': entity.absolute_url(),
'entity_id': entity.id,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),}
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
def _format_datetimes(data):
"""force datetimes into ES mapping format
# TODO refactor this once we get it working
"""
DATETIME_FIELDS = [
'record_created',
'record_lastmod',
]
for field, value in data.iteritems():
if field in DATETIME_FIELDS:
dt = converters.text_to_datetime(value)
# Use default timezone unless...
if data['org'] in config.ALT_TIMEZONES.keys():
timezone = config.ALT_TIMEZONES[data['org']]
else:
timezone = config.TZ
if not dt.tzinfo:
timezone.localize(dt)
data[field] = converters.datetime_to_text(
dt, config.ELASTICSEARCH_DATETIME_FORMAT)
def edit(request, collection, cleaned_data, git_name, git_mail):
# start tasks
result = save.apply_async(
(collection.path, cleaned_data, git_name, git_mail),
countdown=2
)
# lock collection
lockstatus = collection.lock(result.task_id)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
celery_tasks[result.task_id] = {
'task_id': result.task_id,
'action': 'collection-edit',
'collection_url': collection.absolute_url(),
'collection_id': collection.id,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),}
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
def _flatten_json(self) -> Dict[str, str]:
"""Converts Term into a dict suitable for writing to JSON.
"""
data = {}
for key, val in self.__dict__.items():
if (key in ['parent_id']) and val:
val = getattr(self, key)
elif (key in ['siblings', 'children']) and val:
val = []
for t in getattr(self, key):
val.append(t.id)
elif (key in ['created', 'modified']):
val_before = val
if isinstance(val, datetime):
val = converters.datetime_to_text(val)
elif isinstance(val, str):
pass
else:
val = None
data[key] = val
return data
def _format_datetimes(data):
"""force datetimes into ES mapping format
# TODO refactor this once we get it working
"""
DATETIME_FIELDS = [
'record_created',
'record_lastmod',
]
for field,value in data.iteritems():
if field in DATETIME_FIELDS:
dt = converters.text_to_datetime(value)
# Use default timezone unless...
if data['org'] in config.ALT_TIMEZONES.keys():
timezone = config.ALT_TIMEZONES[data['org']]
else:
timezone = config.TZ
if not dt.tzinfo:
timezone.localize(dt)
data[field] = converters.datetime_to_text(
dt, config.ELASTICSEARCH_DATETIME_FORMAT
)
def to_dict(document, module, json_safe=False):
"""Returns an OrderedDict containing the object fields and values.
All fields of the object *definition* are included. Fields missing
from the provided object are included as blank strings ('').
@param document: Collection, Entity, File document object
@param module: collection, entity, files model definitions module
@param json_safe: bool Serialize Python objects e.g. datetime to text
@returns: OrderedDict
"""
data = OrderedDict()
for f in module.FIELDS:
fieldname = f['name']
field_data = getattr(document, f['name'], '')
if json_safe:
if isinstance(field_data, Identifier):
field_data = str(field_data)
elif isinstance(field_data, datetime):
field_data = converters.datetime_to_text(field_data)
data[fieldname] = field_data
return data
def new_external(request, rid):
"""Enter initial data for external file
An external file is one that is external to the DDR collection.
The hashes are known but the binary file itself is not present
within the collection.
"""
file_role = Stub.from_identifier(Identifier(rid))
role = file_role.identifier.parts['role']
entity = file_role.parent(stubs=True)
collection = entity.collection()
check_parents(entity, collection, check_locks=0, fetch=0)
if request.method == 'POST':
form = NewExternalFileForm(request.POST)
if form.is_valid():
idparts = file_role.identifier.idparts
idparts['model'] = 'file'
idparts['sha1'] = form.cleaned_data['sha1']
fi = Identifier(parts=idparts)
basename_orig = form.cleaned_data['filename']
data = {
'id': fi.id,
'external': 1,
'role': role,
'basename_orig': basename_orig,
'sha1': form.cleaned_data['sha1'],
'sha256': form.cleaned_data['sha256'],
'md5': form.cleaned_data['md5'],
'size': form.cleaned_data['size'],
'mimetype': form.cleaned_data['mimetype'],
}
# start tasks
result = file_tasks.add_external.apply_async(
(
request.session['git_name'], request.session['git_mail'],
entity, data, settings.AGENT
),
countdown=2)
result_dict = result.__dict__
log = addfile_logger(entity.identifier)
log.ok('START task_id %s' % result.task_id)
log.ok('ddrlocal.webui.file.new_external')
log.ok('Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
log.ok('locked')
else:
log.not_ok(lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {
'task_id': result.task_id,
'action': 'webui-file-new-external',
'filename': os.path.basename(basename_orig),
'entity_id': entity.id,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),
}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# redirect to entity
return HttpResponseRedirect(entity.absolute_url())
else:
form = NewExternalFileForm()
return render(request, 'webui/files/new-external.html', {
'collection': collection,
'entity': entity,
'file_role': file_role,
'form': form,
})
def index_record_lastmod(data):
return converters.datetime_to_text(
data, converters.config.ELASTICSEARCH_DATETIME_FORMAT)
def csvdump_record_lastmod(data): return converters.datetime_to_text(data) def csvdump_creators(data): return converters.rolepeople_to_text(data)
def new( request, rid ):
enforce_git_credentials(request)
file_role = Stub.from_identifier(Identifier(rid))
entity = file_role.parent(stubs=True)
collection = entity.collection()
check_parents(entity, collection, fetch=0)
role = file_role.identifier.parts['role']
#child_models = CHILDREN_ALL[file_role.identifier.model]
FILE_MODEL = 'file'
module = MODULES[FILE_MODEL]
#
path = request.GET.get('path', None)
FIELDS = prep_newfile_form_fields(module.FIELDS_NEW)
if request.method == 'POST':
form = NewFileDDRForm(request.POST, fields=FIELDS, path_choices=shared_folder_files())
if form.is_valid():
data = form.cleaned_data
src_path = path
# start tasks
result = file_tasks.add_file.apply_async(
(
request.session['git_name'], request.session['git_mail'],
entity, src_path, role, data, settings.AGENT
),
countdown=2)
result_dict = result.__dict__
log = addfile_logger(entity.identifier)
log.ok('START task_id %s' % result.task_id)
log.ok('ddrlocal.webui.file.new')
log.ok('Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
log.ok('locked')
else:
log.not_ok(lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-file-new-%s' % role,
'filename': os.path.basename(src_path),
'entity_id': entity.id,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# feedback
# messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_UPLOADING'] % (os.path.basename(src_path), result))
# redirect to entity
return HttpResponseRedirect(entity.absolute_url())
else:
if not path:
messages.error(request, 'specify a path')
data = {'path': path,
'role':role,
'sort': 1,
'label': '',}
# inheritable fields
for field in entity.inheritable_fields():
data[field] = getattr(entity, field)
form = NewFileDDRForm(data, fields=FIELDS, path_choices=shared_folder_files())
return render(request, 'webui/files/new.html', {
'collection': collection,
'entity': entity,
'file_role': file_role,
'form': form,
'path': path,
})
def index_record_lastmod(data):
return converters.datetime_to_text(
data, converters.config.ELASTICSEARCH_DATETIME_FORMAT
)
def csvdump_record_created(data): return converters.datetime_to_text(data) def csvdump_record_lastmod(data): return converters.datetime_to_text(data)
def test_datetime_to_text():
assert converters.datetime_to_text(TEXT_DATETIME_DATA_NOTZ) == TEXT_DATETIME_TEXT0
def display_record_lastmod(data):
return converters.datetime_to_text(
data, converters.config.PRETTY_DATETIME_FORMAT
)
def ead_record_lastmod(tree, namespaces, field, value):
return _set_attr(tree, namespaces, "/ead/eadheader/eadid",
"record_lastmod", converters.datetime_to_text(value))
def jsondump_record_created(data):
return converters.datetime_to_text(data)
def jsondump_record_lastmod(data):
return converters.datetime_to_text(data)
def test_datetime_to_text():
assert converters.datetime_to_text(
TEXT_DATETIME_DATA_NOTZ) == TEXT_DATETIME_TEXT0
def display_record_lastmod(data):
return converters.datetime_to_text(
data, converters.config.PRETTY_DATETIME_FORMAT)