def test_addfile_logger():
out0 = ingest.addfile_logger(
identifier=ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR
).logpath
print(out0)
assert out0 == LOGPATH
out1 = ingest.addfile_logger(
log_path=LOGPATH, base_dir=TESTING_BASE_DIR
).logpath
print(out1)
assert out1 == LOGPATH
def test_move_files(test_base_dir, entity_identifier):
# this seems way too complicated
# inputs
files = [
(os.path.join(test_base_dir, 'src', 'file1.txt'),
os.path.join(test_base_dir, 'dest', 'file1.txt')),
(os.path.join(test_base_dir, 'src', 'file2.txt'),
os.path.join(test_base_dir, 'dest', 'file2.txt')),
]
log = ingest.addfile_logger(entity_identifier, base_dir=test_base_dir)
# fresh start
for tmp, dest in files:
shutil.rmtree(os.path.dirname(tmp), ignore_errors=True)
shutil.rmtree(os.path.dirname(dest), ignore_errors=True)
for tmp, dest in files:
tmp_dir = os.path.dirname(tmp)
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
with open(tmp, 'w') as f:
f.write('test file 1')
# test
for tmp, dest in files:
assert os.path.exists(tmp)
assert not os.path.exists(dest)
ingest.move_files(files, log)
for tmp, dest in files:
assert not os.path.exists(tmp)
assert os.path.exists(dest)
# clean up
for tmp, dest in files:
shutil.rmtree(os.path.dirname(tmp), ignore_errors=True)
shutil.rmtree(os.path.dirname(dest), ignore_errors=True)
def test_repr(self):
log = ingest.addfile_logger(
identifier=ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR
)
out = log.__repr__()
expected = "<DDR.ingest.AddFileLogger '%s'>" % LOGPATH
assert out == expected
def test_move_files():
# this seems way too complicated
# inputs
files = [
(os.path.join(TESTING_BASE_DIR, 'src', 'file1.txt'), os.path.join(TESTING_BASE_DIR, 'dest', 'file1.txt')),
(os.path.join(TESTING_BASE_DIR, 'src', 'file2.txt'), os.path.join(TESTING_BASE_DIR, 'dest', 'file2.txt')),
]
log = ingest.addfile_logger(ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR)
# fresh start
for tmp,dest in files:
shutil.rmtree(os.path.dirname(tmp), ignore_errors=True)
shutil.rmtree(os.path.dirname(dest), ignore_errors=True)
for tmp,dest in files:
tmp_dir = os.path.dirname(tmp)
if not os.path.exists(tmp_dir):
os.makedirs(tmp_dir)
dest_dir = os.path.dirname(dest)
if not os.path.exists(dest_dir):
os.makedirs(dest_dir)
with open(tmp, 'w') as f:
f.write('test file 1')
# test
for tmp,dest in files:
assert os.path.exists(tmp)
assert not os.path.exists(dest)
ingest.move_files(files, log)
for tmp,dest in files:
assert not os.path.exists(tmp)
assert os.path.exists(dest)
# clean up
for tmp,dest in files:
shutil.rmtree(os.path.dirname(tmp), ignore_errors=True)
shutil.rmtree(os.path.dirname(dest), ignore_errors=True)
def test_copy_to_workdir(test_base_dir, entity_identifier):
log = ingest.addfile_logger(entity_identifier, base_dir=test_base_dir)
# prep
src_path = os.path.join(test_base_dir, 'src', 'somefile.tif')
tmp_path = os.path.join(test_base_dir, 'tmp', 'somefile.tif')
tmp_path_renamed = os.path.join(test_base_dir,
'ddr-test-123-456-master-abc123.tif')
src_dir = os.path.dirname(src_path)
tmp_dir = os.path.dirname(tmp_path)
# clean slate
if os.path.exists(src_dir):
shutil.rmtree(src_dir, ignore_errors=True)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir, ignore_errors=True)
os.makedirs(src_dir)
os.makedirs(tmp_dir)
with open(src_path, 'w') as f:
f.write('test_copy_to_workdir')
# tests
assert os.path.exists(src_path)
ingest.copy_to_workdir(src_path, tmp_path, tmp_path_renamed, log)
assert os.path.exists(src_path)
assert not os.path.exists(tmp_path)
assert os.path.exists(tmp_path_renamed)
# clean up
if os.path.exists(src_dir):
shutil.rmtree(src_dir, ignore_errors=True)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir, ignore_errors=True)
def test_copy_to_workdir():
log = ingest.addfile_logger(ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR)
# prep
src_path = os.path.join(TESTING_BASE_DIR, 'src', 'somefile.tif')
tmp_path = os.path.join(TESTING_BASE_DIR, 'tmp', 'somefile.tif')
tmp_path_renamed = os.path.join(TESTING_BASE_DIR, 'ddr-test-123-456-master-abc123.tif')
src_dir = os.path.dirname(src_path)
tmp_dir = os.path.dirname(tmp_path)
# clean slate
if os.path.exists(src_dir):
shutil.rmtree(src_dir, ignore_errors=True)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir, ignore_errors=True)
os.makedirs(src_dir)
os.makedirs(tmp_dir)
with open(src_path, 'w') as f:
f.write('test_copy_to_workdir')
# tests
assert os.path.exists(src_path)
ingest.copy_to_workdir(src_path, tmp_path, tmp_path_renamed, log)
assert os.path.exists(src_path)
assert not os.path.exists(tmp_path)
assert os.path.exists(tmp_path_renamed)
# clean up
if os.path.exists(src_dir):
shutil.rmtree(src_dir, ignore_errors=True)
if os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir, ignore_errors=True)
def test_check_dir():
log = ingest.addfile_logger(ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR)
label = 'testing'
assert ingest.check_dir('tmp', '/tmp', log)
assert_raises(
Exception,
ingest.check_dir, 'var', '/var', log
)
def new_access( request, repo, org, cid, eid, role, sha1 ):
"""Generate a new access file for the specified file.
NOTE: There is no GET for this view. GET requests will redirect to entity.
"""
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
file_ = DDRFile.from_request(request)
entity = file_.parent()
collection = file_.collection()
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect(entity.absolute_url())
collection.repo_fetch()
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect(entity.absolute_url())
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect(entity.absolute_url())
#
if request.method == 'POST':
form = NewAccessFileForm(request.POST)
if form.is_valid():
src_path = form.cleaned_data['path']
# start tasks
result = entity_add_access.apply_async(
(git_name, git_mail, entity, file_),
countdown=2)
result_dict = result.__dict__
log = addfile_logger(entity.identifier)
log.ok('START task_id %s' % result.task_id)
log.ok('ddrlocal.webui.file.new_access')
log.ok('Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
log.ok( 'locked')
else:
log.not_ok( lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-file-new-access',
'filename': os.path.basename(src_path),
'file_url': file_.absolute_url(),
'entity_id': entity.id,
'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# feedback
#messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_NEWACCESS'] % os.path.basename(src_path))
# redirect to entity
return HttpResponseRedirect(entity.absolute_url())
def test_checksums():
md5,sha1,sha256 = ingest.checksums(
IMG_PATH,
ingest.addfile_logger(
identifier=ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR
)
)
assert md5 == IMG_MD5
assert sha1 == IMG_SHA1
assert sha256 == IMG_SHA256
def test_make_access_file(test_base_dir, entity_identifier, test_image):
src_path = test_image
access_path = os.path.join(test_base_dir, '%s-a.jpg' % FILE_ID)
# inputs
log = ingest.addfile_logger(entity_identifier, base_dir=test_base_dir)
# no src_path so fails
missing_file = os.path.join(test_base_dir, 'src', 'somefile.png')
assert ingest.make_access_file(missing_file, access_path, log) == None
# get test jpg
assert ingest.make_access_file(src_path, access_path, log) == access_path
def new_access( request, fid ):
"""Generate a new access file for the specified file.
NOTE: There is no GET for this view. GET requests will redirect to entity.
"""
enforce_git_credentials(request)
file_ = DDRFile.from_identifier(Identifier(fid))
check_file(file_)
entity = file_.parent()
collection = file_.collection()
check_parents(entity, collection)
#
if request.method == 'POST':
form = NewAccessFileForm(request.POST)
if form.is_valid():
src_path = form.cleaned_data['path']
# start tasks
result = file_tasks.add_access.apply_async(
(
request.session['git_name'], request.session['git_mail'],
entity, file_
),
countdown=2
)
result_dict = result.__dict__
log = addfile_logger(entity.identifier)
log.ok('START task_id %s' % result.task_id)
log.ok('ddrlocal.webui.file.new_access')
log.ok('Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
log.ok( 'locked')
else:
log.not_ok( lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-file-new-access',
'filename': os.path.basename(src_path),
'file_url': file_.absolute_url(),
'entity_id': entity.id,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# feedback
#messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_NEWACCESS'] % os.path.basename(src_path))
# redirect to entity
return HttpResponseRedirect(entity.absolute_url())
def after_return(self, status, retval, task_id, args, kwargs, einfo):
entity = args[2]
collection = entity.collection()
log = addfile_logger(entity.identifier)
log.ok('FileAddDebugTask.AFTER_RETURN')
log.ok('task_id: %s' % task_id)
log.ok('status: %s' % status)
log.ok('retval: %s' % retval)
log.ok('Unlocking %s' % entity.id)
lockstatus = entity.unlock(task_id)
if lockstatus == 'ok':
log.ok('unlocked')
else:
log.not_ok(lockstatus)
log.ok( 'END task_id %s\n' % task_id)
collection.cache_delete()
gitstatus.update(settings.MEDIA_BASE, collection.path)
gitstatus.unlock(settings.MEDIA_BASE, 'entity_add_file')
def test_make_access_file():
# inputs
log = ingest.addfile_logger(ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR)
# prep
if os.path.exists(ACCESS_PATH):
os.remove(ACCESS_PATH)
# no src_path so fails
missing_file = os.path.join(TESTING_BASE_DIR, 'src', 'somefile.png')
assert ingest.make_access_file(missing_file, ACCESS_PATH, log) == None
# get test jpg
src_path = IMG_PATH
parent_dir = os.path.dirname(src_path)
if not os.path.exists(parent_dir):
os.makedirs(parent_dir)
if not os.path.exists(src_path):
urllib.urlretrieve(IMG_URL, IMG_PATH)
assert ingest.make_access_file(src_path, ACCESS_PATH, log) == ACCESS_PATH
# clean up
if os.path.exists(ACCESS_PATH):
os.remove(ACCESS_PATH)
def addfile_logger(self):
return ingest.addfile_logger(self)
def new_external(request, rid):
"""Enter initial data for external file
An external file is one that is external to the DDR collection.
The hashes are known but the binary file itself is not present
within the collection.
"""
file_role = Stub.from_identifier(Identifier(rid))
role = file_role.identifier.parts['role']
entity = file_role.parent(stubs=True)
collection = entity.collection()
check_parents(entity, collection, check_locks=0, fetch=0)
if request.method == 'POST':
form = NewExternalFileForm(request.POST)
if form.is_valid():
idparts = file_role.identifier.idparts
idparts['model'] = 'file'
idparts['sha1'] = form.cleaned_data['sha1']
fi = Identifier(parts=idparts)
basename_orig = form.cleaned_data['filename']
data = {
'id': fi.id,
'external': 1,
'role': role,
'basename_orig': basename_orig,
'sha1': form.cleaned_data['sha1'],
'sha256': form.cleaned_data['sha256'],
'md5': form.cleaned_data['md5'],
'size': form.cleaned_data['size'],
'mimetype': form.cleaned_data['mimetype'],
}
# start tasks
result = file_tasks.add_external.apply_async(
(
request.session['git_name'], request.session['git_mail'],
entity, data, settings.AGENT
),
countdown=2)
result_dict = result.__dict__
log = addfile_logger(entity.identifier)
log.ok('START task_id %s' % result.task_id)
log.ok('ddrlocal.webui.file.new_external')
log.ok('Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
log.ok('locked')
else:
log.not_ok(lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {
'task_id': result.task_id,
'action': 'webui-file-new-external',
'filename': os.path.basename(basename_orig),
'entity_id': entity.id,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),
}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# redirect to entity
return HttpResponseRedirect(entity.absolute_url())
else:
form = NewExternalFileForm()
return render(request, 'webui/files/new-external.html', {
'collection': collection,
'entity': entity,
'file_role': file_role,
'form': form,
})
def new( request, rid ):
enforce_git_credentials(request)
file_role = Stub.from_identifier(Identifier(rid))
entity = file_role.parent(stubs=True)
collection = entity.collection()
check_parents(entity, collection, fetch=0)
role = file_role.identifier.parts['role']
#child_models = CHILDREN_ALL[file_role.identifier.model]
FILE_MODEL = 'file'
module = MODULES[FILE_MODEL]
#
path = request.GET.get('path', None)
FIELDS = prep_newfile_form_fields(module.FIELDS_NEW)
if request.method == 'POST':
form = NewFileDDRForm(request.POST, fields=FIELDS, path_choices=shared_folder_files())
if form.is_valid():
data = form.cleaned_data
src_path = path
# start tasks
result = file_tasks.add_file.apply_async(
(
request.session['git_name'], request.session['git_mail'],
entity, src_path, role, data, settings.AGENT
),
countdown=2)
result_dict = result.__dict__
log = addfile_logger(entity.identifier)
log.ok('START task_id %s' % result.task_id)
log.ok('ddrlocal.webui.file.new')
log.ok('Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
log.ok('locked')
else:
log.not_ok(lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-file-new-%s' % role,
'filename': os.path.basename(src_path),
'entity_id': entity.id,
'start': converters.datetime_to_text(datetime.now(settings.TZ)),}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# feedback
# messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_UPLOADING'] % (os.path.basename(src_path), result))
# redirect to entity
return HttpResponseRedirect(entity.absolute_url())
else:
if not path:
messages.error(request, 'specify a path')
data = {'path': path,
'role':role,
'sort': 1,
'label': '',}
# inheritable fields
for field in entity.inheritable_fields():
data[field] = getattr(entity, field)
form = NewFileDDRForm(data, fields=FIELDS, path_choices=shared_folder_files())
return render(request, 'webui/files/new.html', {
'collection': collection,
'entity': entity,
'file_role': file_role,
'form': form,
'path': path,
})
def on_success(self, retval, task_id, args, kwargs):
entity = args[2]
log = addfile_logger(entity.identifier)
log.ok('DDRTask.ON_SUCCESS')
def on_failure(self, exc, task_id, args, kwargs, einfo):
entity = args[2]
log = addfile_logger(entity.identifier)
log.not_ok('DDRTask.ON_FAILURE')
def addfile_logger(self):
return ingest.addfile_logger(self)
def new( request, repo, org, cid, eid, role='master' ):
git_name = request.session.get('git_name')
git_mail = request.session.get('git_mail')
if not git_name and git_mail:
messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED'])
fidentifier = Identifier(request)
file_role = fidentifier.object()
module = MODULES[CHILDREN_ALL[fidentifier.model]]
entity = file_role.parent(stubs=True)
collection = entity.collection()
if collection.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id))
return HttpResponseRedirect(entity.absolute_url())
if collection.repo_behind():
messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id))
return HttpResponseRedirect(entity.absolute_url())
if entity.locked():
messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED'])
return HttpResponseRedirect(entity.absolute_url())
#
path = request.GET.get('path', None)
FIELDS = prep_newfile_form_fields(module.FIELDS_NEW)
if request.method == 'POST':
form = NewFileDDRForm(request.POST, fields=FIELDS, path_choices=shared_folder_files())
if form.is_valid():
data = form.cleaned_data
src_path = path
# inheritable fields
inherited = []
for field in entity.inheritable_fields():
inherited.append( (field,getattr(entity,field)) )
# start tasks
result = entity_add_file.apply_async(
(git_name, git_mail, entity, src_path, role, data, settings.AGENT),
countdown=2)
result_dict = result.__dict__
log = addfile_logger(entity.identifier)
log.ok('START task_id %s' % result.task_id)
log.ok('ddrlocal.webui.file.new')
log.ok('Locking %s' % entity.id)
# lock entity
lockstatus = entity.lock(result.task_id)
if lockstatus == 'ok':
log.ok('locked')
else:
log.not_ok(lockstatus)
# add celery task_id to session
celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {})
# IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES.
task = {'task_id': result.task_id,
'action': 'webui-file-new-%s' % role,
'filename': os.path.basename(src_path),
'entity_id': entity.id,
'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),}
celery_tasks[result.task_id] = task
#del request.session[settings.CELERY_TASKS_SESSION_KEY]
request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks
# feedback
# messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_UPLOADING'] % (os.path.basename(src_path), result))
# redirect to entity
return HttpResponseRedirect(entity.absolute_url())
else:
if not path:
messages.error(request, 'specify a path')
data = {'path': path,
'role':role,
'sort': 1,
'label': '',}
# inheritable fields
for field in entity.inheritable_fields():
data[field] = getattr(entity, field)
form = NewFileDDRForm(data, fields=FIELDS, path_choices=shared_folder_files())
return render_to_response(
'webui/files/new.html',
{'collection': collection,
'entity': entity,
'file_role': file_role,
'form': form,
'path': path,},
context_instance=RequestContext(request, processors=[])
)
def test_repr(self, tmpdir, entity_identifier):
log = ingest.addfile_logger(identifier=entity_identifier,
base_dir=str(tmpdir))
out = log.__repr__()
assert LOGPATH_REL in out
def test_addfile_logger(tmpdir, logpath, entity_identifier):
out0 = ingest.addfile_logger(identifier=entity_identifier,
base_dir=str(tmpdir)).logpath
print('out0 %s' % out0)
assert LOGPATH_REL in out0
def test_check_dir(tmpdir, entity_identifier):
log = ingest.addfile_logger(entity_identifier, base_dir=str(tmpdir))
label = 'testing'
assert ingest.check_dir('tmp', '/tmp', log)
assert_raises(Exception, ingest.check_dir, 'var', '/var', log)
def test_checksums(tmpdir, test_base_dir, entity_identifier, test_image):
log = ingest.addfile_logger(entity_identifier, base_dir=test_base_dir)
md5, sha1, sha256 = ingest.checksums(test_image, log)
assert md5 == IMG_MD5
assert sha1 == IMG_SHA1
assert sha256 == IMG_SHA256