def test_addfile_logger(): out0 = ingest.addfile_logger( identifier=ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR ).logpath print(out0) assert out0 == LOGPATH out1 = ingest.addfile_logger( log_path=LOGPATH, base_dir=TESTING_BASE_DIR ).logpath print(out1) assert out1 == LOGPATH
def test_move_files(test_base_dir, entity_identifier): # this seems way too complicated # inputs files = [ (os.path.join(test_base_dir, 'src', 'file1.txt'), os.path.join(test_base_dir, 'dest', 'file1.txt')), (os.path.join(test_base_dir, 'src', 'file2.txt'), os.path.join(test_base_dir, 'dest', 'file2.txt')), ] log = ingest.addfile_logger(entity_identifier, base_dir=test_base_dir) # fresh start for tmp, dest in files: shutil.rmtree(os.path.dirname(tmp), ignore_errors=True) shutil.rmtree(os.path.dirname(dest), ignore_errors=True) for tmp, dest in files: tmp_dir = os.path.dirname(tmp) if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): os.makedirs(dest_dir) with open(tmp, 'w') as f: f.write('test file 1') # test for tmp, dest in files: assert os.path.exists(tmp) assert not os.path.exists(dest) ingest.move_files(files, log) for tmp, dest in files: assert not os.path.exists(tmp) assert os.path.exists(dest) # clean up for tmp, dest in files: shutil.rmtree(os.path.dirname(tmp), ignore_errors=True) shutil.rmtree(os.path.dirname(dest), ignore_errors=True)
def test_repr(self): log = ingest.addfile_logger( identifier=ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR ) out = log.__repr__() expected = "<DDR.ingest.AddFileLogger '%s'>" % LOGPATH assert out == expected
def test_move_files(): # this seems way too complicated # inputs files = [ (os.path.join(TESTING_BASE_DIR, 'src', 'file1.txt'), os.path.join(TESTING_BASE_DIR, 'dest', 'file1.txt')), (os.path.join(TESTING_BASE_DIR, 'src', 'file2.txt'), os.path.join(TESTING_BASE_DIR, 'dest', 'file2.txt')), ] log = ingest.addfile_logger(ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR) # fresh start for tmp,dest in files: shutil.rmtree(os.path.dirname(tmp), ignore_errors=True) shutil.rmtree(os.path.dirname(dest), ignore_errors=True) for tmp,dest in files: tmp_dir = os.path.dirname(tmp) if not os.path.exists(tmp_dir): os.makedirs(tmp_dir) dest_dir = os.path.dirname(dest) if not os.path.exists(dest_dir): os.makedirs(dest_dir) with open(tmp, 'w') as f: f.write('test file 1') # test for tmp,dest in files: assert os.path.exists(tmp) assert not os.path.exists(dest) ingest.move_files(files, log) for tmp,dest in files: assert not os.path.exists(tmp) assert os.path.exists(dest) # clean up for tmp,dest in files: shutil.rmtree(os.path.dirname(tmp), ignore_errors=True) shutil.rmtree(os.path.dirname(dest), ignore_errors=True)
def test_copy_to_workdir(test_base_dir, entity_identifier): log = ingest.addfile_logger(entity_identifier, base_dir=test_base_dir) # prep src_path = os.path.join(test_base_dir, 'src', 'somefile.tif') tmp_path = os.path.join(test_base_dir, 'tmp', 'somefile.tif') tmp_path_renamed = os.path.join(test_base_dir, 'ddr-test-123-456-master-abc123.tif') src_dir = os.path.dirname(src_path) tmp_dir = os.path.dirname(tmp_path) # clean slate if os.path.exists(src_dir): shutil.rmtree(src_dir, ignore_errors=True) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir, ignore_errors=True) os.makedirs(src_dir) os.makedirs(tmp_dir) with open(src_path, 'w') as f: f.write('test_copy_to_workdir') # tests assert os.path.exists(src_path) ingest.copy_to_workdir(src_path, tmp_path, tmp_path_renamed, log) assert os.path.exists(src_path) assert not os.path.exists(tmp_path) assert os.path.exists(tmp_path_renamed) # clean up if os.path.exists(src_dir): shutil.rmtree(src_dir, ignore_errors=True) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir, ignore_errors=True)
def test_copy_to_workdir(): log = ingest.addfile_logger(ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR) # prep src_path = os.path.join(TESTING_BASE_DIR, 'src', 'somefile.tif') tmp_path = os.path.join(TESTING_BASE_DIR, 'tmp', 'somefile.tif') tmp_path_renamed = os.path.join(TESTING_BASE_DIR, 'ddr-test-123-456-master-abc123.tif') src_dir = os.path.dirname(src_path) tmp_dir = os.path.dirname(tmp_path) # clean slate if os.path.exists(src_dir): shutil.rmtree(src_dir, ignore_errors=True) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir, ignore_errors=True) os.makedirs(src_dir) os.makedirs(tmp_dir) with open(src_path, 'w') as f: f.write('test_copy_to_workdir') # tests assert os.path.exists(src_path) ingest.copy_to_workdir(src_path, tmp_path, tmp_path_renamed, log) assert os.path.exists(src_path) assert not os.path.exists(tmp_path) assert os.path.exists(tmp_path_renamed) # clean up if os.path.exists(src_dir): shutil.rmtree(src_dir, ignore_errors=True) if os.path.exists(tmp_dir): shutil.rmtree(tmp_dir, ignore_errors=True)
def test_check_dir(): log = ingest.addfile_logger(ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR) label = 'testing' assert ingest.check_dir('tmp', '/tmp', log) assert_raises( Exception, ingest.check_dir, 'var', '/var', log )
def new_access( request, repo, org, cid, eid, role, sha1 ): """Generate a new access file for the specified file. NOTE: There is no GET for this view. GET requests will redirect to entity. """ git_name = request.session.get('git_name') git_mail = request.session.get('git_mail') if not git_name and git_mail: messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED']) file_ = DDRFile.from_request(request) entity = file_.parent() collection = file_.collection() if collection.locked(): messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id)) return HttpResponseRedirect(entity.absolute_url()) collection.repo_fetch() if collection.repo_behind(): messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id)) return HttpResponseRedirect(entity.absolute_url()) if entity.locked(): messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED']) return HttpResponseRedirect(entity.absolute_url()) # if request.method == 'POST': form = NewAccessFileForm(request.POST) if form.is_valid(): src_path = form.cleaned_data['path'] # start tasks result = entity_add_access.apply_async( (git_name, git_mail, entity, file_), countdown=2) result_dict = result.__dict__ log = addfile_logger(entity.identifier) log.ok('START task_id %s' % result.task_id) log.ok('ddrlocal.webui.file.new_access') log.ok('Locking %s' % entity.id) # lock entity lockstatus = entity.lock(result.task_id) if lockstatus == 'ok': log.ok( 'locked') else: log.not_ok( lockstatus) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = {'task_id': result.task_id, 'action': 'webui-file-new-access', 'filename': os.path.basename(src_path), 'file_url': file_.absolute_url(), 'entity_id': entity.id, 'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),} celery_tasks[result.task_id] = task #del request.session[settings.CELERY_TASKS_SESSION_KEY] request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks # feedback #messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_NEWACCESS'] % os.path.basename(src_path)) # redirect to entity return HttpResponseRedirect(entity.absolute_url())
def test_checksums(): md5,sha1,sha256 = ingest.checksums( IMG_PATH, ingest.addfile_logger( identifier=ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR ) ) assert md5 == IMG_MD5 assert sha1 == IMG_SHA1 assert sha256 == IMG_SHA256
def test_make_access_file(test_base_dir, entity_identifier, test_image): src_path = test_image access_path = os.path.join(test_base_dir, '%s-a.jpg' % FILE_ID) # inputs log = ingest.addfile_logger(entity_identifier, base_dir=test_base_dir) # no src_path so fails missing_file = os.path.join(test_base_dir, 'src', 'somefile.png') assert ingest.make_access_file(missing_file, access_path, log) == None # get test jpg assert ingest.make_access_file(src_path, access_path, log) == access_path
def new_access( request, fid ): """Generate a new access file for the specified file. NOTE: There is no GET for this view. GET requests will redirect to entity. """ enforce_git_credentials(request) file_ = DDRFile.from_identifier(Identifier(fid)) check_file(file_) entity = file_.parent() collection = file_.collection() check_parents(entity, collection) # if request.method == 'POST': form = NewAccessFileForm(request.POST) if form.is_valid(): src_path = form.cleaned_data['path'] # start tasks result = file_tasks.add_access.apply_async( ( request.session['git_name'], request.session['git_mail'], entity, file_ ), countdown=2 ) result_dict = result.__dict__ log = addfile_logger(entity.identifier) log.ok('START task_id %s' % result.task_id) log.ok('ddrlocal.webui.file.new_access') log.ok('Locking %s' % entity.id) # lock entity lockstatus = entity.lock(result.task_id) if lockstatus == 'ok': log.ok( 'locked') else: log.not_ok( lockstatus) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = {'task_id': result.task_id, 'action': 'webui-file-new-access', 'filename': os.path.basename(src_path), 'file_url': file_.absolute_url(), 'entity_id': entity.id, 'start': converters.datetime_to_text(datetime.now(settings.TZ)),} celery_tasks[result.task_id] = task #del request.session[settings.CELERY_TASKS_SESSION_KEY] request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks # feedback #messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_NEWACCESS'] % os.path.basename(src_path)) # redirect to entity return HttpResponseRedirect(entity.absolute_url())
def after_return(self, status, retval, task_id, args, kwargs, einfo): entity = args[2] collection = entity.collection() log = addfile_logger(entity.identifier) log.ok('FileAddDebugTask.AFTER_RETURN') log.ok('task_id: %s' % task_id) log.ok('status: %s' % status) log.ok('retval: %s' % retval) log.ok('Unlocking %s' % entity.id) lockstatus = entity.unlock(task_id) if lockstatus == 'ok': log.ok('unlocked') else: log.not_ok(lockstatus) log.ok( 'END task_id %s\n' % task_id) collection.cache_delete() gitstatus.update(settings.MEDIA_BASE, collection.path) gitstatus.unlock(settings.MEDIA_BASE, 'entity_add_file')
def test_make_access_file(): # inputs log = ingest.addfile_logger(ENTITY_IDENTIFIER, base_dir=TESTING_BASE_DIR) # prep if os.path.exists(ACCESS_PATH): os.remove(ACCESS_PATH) # no src_path so fails missing_file = os.path.join(TESTING_BASE_DIR, 'src', 'somefile.png') assert ingest.make_access_file(missing_file, ACCESS_PATH, log) == None # get test jpg src_path = IMG_PATH parent_dir = os.path.dirname(src_path) if not os.path.exists(parent_dir): os.makedirs(parent_dir) if not os.path.exists(src_path): urllib.urlretrieve(IMG_URL, IMG_PATH) assert ingest.make_access_file(src_path, ACCESS_PATH, log) == ACCESS_PATH # clean up if os.path.exists(ACCESS_PATH): os.remove(ACCESS_PATH)
def addfile_logger(self): return ingest.addfile_logger(self)
def new_external(request, rid): """Enter initial data for external file An external file is one that is external to the DDR collection. The hashes are known but the binary file itself is not present within the collection. """ file_role = Stub.from_identifier(Identifier(rid)) role = file_role.identifier.parts['role'] entity = file_role.parent(stubs=True) collection = entity.collection() check_parents(entity, collection, check_locks=0, fetch=0) if request.method == 'POST': form = NewExternalFileForm(request.POST) if form.is_valid(): idparts = file_role.identifier.idparts idparts['model'] = 'file' idparts['sha1'] = form.cleaned_data['sha1'] fi = Identifier(parts=idparts) basename_orig = form.cleaned_data['filename'] data = { 'id': fi.id, 'external': 1, 'role': role, 'basename_orig': basename_orig, 'sha1': form.cleaned_data['sha1'], 'sha256': form.cleaned_data['sha256'], 'md5': form.cleaned_data['md5'], 'size': form.cleaned_data['size'], 'mimetype': form.cleaned_data['mimetype'], } # start tasks result = file_tasks.add_external.apply_async( ( request.session['git_name'], request.session['git_mail'], entity, data, settings.AGENT ), countdown=2) result_dict = result.__dict__ log = addfile_logger(entity.identifier) log.ok('START task_id %s' % result.task_id) log.ok('ddrlocal.webui.file.new_external') log.ok('Locking %s' % entity.id) # lock entity lockstatus = entity.lock(result.task_id) if lockstatus == 'ok': log.ok('locked') else: log.not_ok(lockstatus) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = { 'task_id': result.task_id, 'action': 'webui-file-new-external', 'filename': os.path.basename(basename_orig), 'entity_id': entity.id, 'start': converters.datetime_to_text(datetime.now(settings.TZ)), } celery_tasks[result.task_id] = task #del request.session[settings.CELERY_TASKS_SESSION_KEY] request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks # redirect to entity return HttpResponseRedirect(entity.absolute_url()) else: form = NewExternalFileForm() return render(request, 'webui/files/new-external.html', { 'collection': collection, 'entity': entity, 'file_role': file_role, 'form': form, })
def new( request, rid ): enforce_git_credentials(request) file_role = Stub.from_identifier(Identifier(rid)) entity = file_role.parent(stubs=True) collection = entity.collection() check_parents(entity, collection, fetch=0) role = file_role.identifier.parts['role'] #child_models = CHILDREN_ALL[file_role.identifier.model] FILE_MODEL = 'file' module = MODULES[FILE_MODEL] # path = request.GET.get('path', None) FIELDS = prep_newfile_form_fields(module.FIELDS_NEW) if request.method == 'POST': form = NewFileDDRForm(request.POST, fields=FIELDS, path_choices=shared_folder_files()) if form.is_valid(): data = form.cleaned_data src_path = path # start tasks result = file_tasks.add_file.apply_async( ( request.session['git_name'], request.session['git_mail'], entity, src_path, role, data, settings.AGENT ), countdown=2) result_dict = result.__dict__ log = addfile_logger(entity.identifier) log.ok('START task_id %s' % result.task_id) log.ok('ddrlocal.webui.file.new') log.ok('Locking %s' % entity.id) # lock entity lockstatus = entity.lock(result.task_id) if lockstatus == 'ok': log.ok('locked') else: log.not_ok(lockstatus) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = {'task_id': result.task_id, 'action': 'webui-file-new-%s' % role, 'filename': os.path.basename(src_path), 'entity_id': entity.id, 'start': converters.datetime_to_text(datetime.now(settings.TZ)),} celery_tasks[result.task_id] = task #del request.session[settings.CELERY_TASKS_SESSION_KEY] request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks # feedback # messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_UPLOADING'] % (os.path.basename(src_path), result)) # redirect to entity return HttpResponseRedirect(entity.absolute_url()) else: if not path: messages.error(request, 'specify a path') data = {'path': path, 'role':role, 'sort': 1, 'label': '',} # inheritable fields for field in entity.inheritable_fields(): data[field] = getattr(entity, field) form = NewFileDDRForm(data, fields=FIELDS, path_choices=shared_folder_files()) return render(request, 'webui/files/new.html', { 'collection': collection, 'entity': entity, 'file_role': file_role, 'form': form, 'path': path, })
def on_success(self, retval, task_id, args, kwargs): entity = args[2] log = addfile_logger(entity.identifier) log.ok('DDRTask.ON_SUCCESS')
def on_failure(self, exc, task_id, args, kwargs, einfo): entity = args[2] log = addfile_logger(entity.identifier) log.not_ok('DDRTask.ON_FAILURE')
def new( request, repo, org, cid, eid, role='master' ): git_name = request.session.get('git_name') git_mail = request.session.get('git_mail') if not git_name and git_mail: messages.error(request, WEBUI_MESSAGES['LOGIN_REQUIRED']) fidentifier = Identifier(request) file_role = fidentifier.object() module = MODULES[CHILDREN_ALL[fidentifier.model]] entity = file_role.parent(stubs=True) collection = entity.collection() if collection.locked(): messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_LOCKED'].format(collection.id)) return HttpResponseRedirect(entity.absolute_url()) if collection.repo_behind(): messages.error(request, WEBUI_MESSAGES['VIEWS_COLL_BEHIND'].format(collection.id)) return HttpResponseRedirect(entity.absolute_url()) if entity.locked(): messages.error(request, WEBUI_MESSAGES['VIEWS_ENT_LOCKED']) return HttpResponseRedirect(entity.absolute_url()) # path = request.GET.get('path', None) FIELDS = prep_newfile_form_fields(module.FIELDS_NEW) if request.method == 'POST': form = NewFileDDRForm(request.POST, fields=FIELDS, path_choices=shared_folder_files()) if form.is_valid(): data = form.cleaned_data src_path = path # inheritable fields inherited = [] for field in entity.inheritable_fields(): inherited.append( (field,getattr(entity,field)) ) # start tasks result = entity_add_file.apply_async( (git_name, git_mail, entity, src_path, role, data, settings.AGENT), countdown=2) result_dict = result.__dict__ log = addfile_logger(entity.identifier) log.ok('START task_id %s' % result.task_id) log.ok('ddrlocal.webui.file.new') log.ok('Locking %s' % entity.id) # lock entity lockstatus = entity.lock(result.task_id) if lockstatus == 'ok': log.ok('locked') else: log.not_ok(lockstatus) # add celery task_id to session celery_tasks = request.session.get(settings.CELERY_TASKS_SESSION_KEY, {}) # IMPORTANT: 'action' *must* match a message in webui.tasks.TASK_STATUS_MESSAGES. task = {'task_id': result.task_id, 'action': 'webui-file-new-%s' % role, 'filename': os.path.basename(src_path), 'entity_id': entity.id, 'start': datetime.now().strftime(settings.TIMESTAMP_FORMAT),} celery_tasks[result.task_id] = task #del request.session[settings.CELERY_TASKS_SESSION_KEY] request.session[settings.CELERY_TASKS_SESSION_KEY] = celery_tasks # feedback # messages.success(request, WEBUI_MESSAGES['VIEWS_FILES_UPLOADING'] % (os.path.basename(src_path), result)) # redirect to entity return HttpResponseRedirect(entity.absolute_url()) else: if not path: messages.error(request, 'specify a path') data = {'path': path, 'role':role, 'sort': 1, 'label': '',} # inheritable fields for field in entity.inheritable_fields(): data[field] = getattr(entity, field) form = NewFileDDRForm(data, fields=FIELDS, path_choices=shared_folder_files()) return render_to_response( 'webui/files/new.html', {'collection': collection, 'entity': entity, 'file_role': file_role, 'form': form, 'path': path,}, context_instance=RequestContext(request, processors=[]) )
def test_repr(self, tmpdir, entity_identifier): log = ingest.addfile_logger(identifier=entity_identifier, base_dir=str(tmpdir)) out = log.__repr__() assert LOGPATH_REL in out
def test_addfile_logger(tmpdir, logpath, entity_identifier): out0 = ingest.addfile_logger(identifier=entity_identifier, base_dir=str(tmpdir)).logpath print('out0 %s' % out0) assert LOGPATH_REL in out0
def test_check_dir(tmpdir, entity_identifier): log = ingest.addfile_logger(entity_identifier, base_dir=str(tmpdir)) label = 'testing' assert ingest.check_dir('tmp', '/tmp', log) assert_raises(Exception, ingest.check_dir, 'var', '/var', log)
def test_checksums(tmpdir, test_base_dir, entity_identifier, test_image): log = ingest.addfile_logger(entity_identifier, base_dir=test_base_dir) md5, sha1, sha256 = ingest.checksums(test_image, log) assert md5 == IMG_MD5 assert sha1 == IMG_SHA1 assert sha256 == IMG_SHA256