def set_context_props(self, request, **kwargs): super(MetadataMixin, self).set_context_props(request, **kwargs) mgr = FileManager(self.agave_client) return mgr.get(system_id = self.filesystem, path = self.file_path, username = request.user.username, is_public = self.is_public)
def post(self, request, *args, **kwargs): self.set_context_props(request, **kwargs) ufs = request.FILES mgr = FileManager(self.agave_client) mfs, fs = mgr.upload_files(ufs, system_id = self.filesystem, path = self.file_path) return self.render_to_json_response({'files': [o.to_dict() for o in fs], 'filesMeta': [o.to_dict(get_id = True) for o in mfs]})
def index_permissions(system_id, path, username, bottom_up = False, levels = 0): try: user = get_user_model().objects.get(username = username) ag = user.agave_oauth.client mngr = FileManager(agave_client = ag) mngr.index_permissions(system_id, path, username, bottom_up = bottom_up, levels = levels) except ObjectDoesNotExist: logger.exception('Unable to locate local user=%s' % username)
def share(system_id, path, username, permission, me): logger.debug('Sharing file/folder: {}. To: {}. Owner: {}'.format(path, username, me)) path_shared = path try: user = get_user_model().objects.get(username=me) ag = user.agave_oauth.client mngr = FileManager(agave_client = ag) rep = mngr.share(system_id = system_id, path = path, username = username, permission = permission, me = me) meta_obj, ret = mngr.get(system_id, path, me, False) if meta_obj.format != 'folder': path_shared = meta_obj.path logger.debug('Successfully updated permissions: {}'.format(rep)) DataEvent.send_generic_event({ 'username_from': me, 'username_to': username, 'permission': permission, 'action': 'share_finished', 'path': path, 'action_link': { 'label': 'View Files', 'value': '/data/my/#/Shared with me/' + path_shared}, 'html':[ {'Message': 'Your files have been shared.'}, {'Action': 'Sharing Finished'}, {'Shared with': username}, {'Permissions set': permission}, ] }, [me]) DataEvent.send_generic_event({ 'username_from': me, 'username_to': username, 'permission': permission, 'action': 'share_finished', 'path': path, 'action_link': { 'label': 'View Files', 'value': '/data/my/#/Shared with me/' + path_shared}, 'html':[ {'Message': '{} have shared some files with you.'.format(me)}, {'Action': 'Sharing Finished'}, {'Shared with': username}, {'Permissions set': permission}, ], 'toast': { 'type': 'info', 'msg': me + ' has shared files with you.' } }, [username]) except ObjectDoesNotExist: logger.exception('Unable to locate local user=%s' % username)
def check_or_create_agave_home_dir(username): logger.info("Checking home directory for user=%s on default storage systemId=%s" % ( username, settings.AGAVE_STORAGE_SYSTEM)) try: # TODO should use DS Files API for this ag = Agave(api_server=settings.AGAVE_TENANT_BASEURL, token=settings.AGAVE_SUPER_TOKEN) body = {'action': 'mkdir', 'path': username} ag.files.manage(systemId=settings.AGAVE_STORAGE_SYSTEM, filePath='', body=body) # add dir to index fm = FileManager(agave_client=ag) fm.index(settings.AGAVE_STORAGE_SYSTEM, username, username, levels=1) except (HTTPError, AgaveException): logger.exception('Failed to create home directory.', extra={'user': username, 'systemId': settings.AGAVE_STORAGE_SYSTEM})
def get(self, request, *args, **kwargs): self.set_context_props(request, **kwargs) mgr = FileManager(agave_client = self.agave_client) if self.file_path == request.user.username: mgr.check_shared_folder(system_id = self.filesystem, username = request.user.username) if request.GET.get('refresh', None): mgr.index(self.filesystem, self.file_path, request.user.username, levels = 1) l = mgr.list_path(system_id = self.filesystem, path = self.file_path, username = request.user.username, special_dir = self.special_dir, is_public = self.is_public) if isinstance(l, GeneratorType): response = [o.to_dict(get_id = True) for o in l] else: tasks.index.delay(self.filesystem, self.file_path, request.user.username, pems = True, levels = 1) response = [o.to_dict(pems = False) for o in l] return self.render_to_json_response(response, status = 200)
def delete(self, request, *args, **kwargs): self.set_context_props(request, **kwargs) mf = FileManager(agave_client = self.agave_client) o = mf.delete(self.filesystem, self.file_path, request.user.username) return self.render_to_json_response(o.to_dict(get_id = True))
def copy_box_item(self, username, box_item_type, box_item_id, target_system_id, target_path): user = get_user_model().objects.get(username=username) client = user.box_user_token.client try: op = getattr(client, box_item_type) item = op(box_item_id).get() except AttributeError as e: logger.error('Invalid box_item_type') self.update_state(state='FAILURE', meta={'exc': e}) return agave_client = user.agave_oauth.client fm = FileManager(agave_client=agave_client) if box_item_type == 'file': try: import_url = item.get_shared_link_download_url() safe_name = get_valid_filename(item.name) import_resp = agave_client.files.importData(systemId=target_system_id, filePath=target_path, fileName=safe_name, urlToIngest=import_url) async_resp = AgaveAsyncResponse(agave_client, import_resp) async_status = async_resp.result(600) if async_status == 'FAILED': logger.error('Box File Transfer failed: %s' % target_path) else: file_path = '%s/%s' % (target_path, item.name) logger.info('Indexing Box File Transfer %s' % file_path) fm.index(settings.AGAVE_STORAGE_SYSTEM, file_path, username, levels=1) except BoxAPIException as e: logger.error('Unable to get download link from Box') self.update_state(state='FAILURE', meta={'exc': e}) except HTTPError as e: logger.error('Agave.files.importData raised HTTPError') self.update_state(state='FAILURE', meta={'exc': e}) except (TimeoutError, Error) as e: logger.error('Agave.files.importData failed to complete') self.update_state(state='FAILURE', meta={'exc': e}) else: # box_item_type == 'folder' # create directory for the folder try: safe_name = get_valid_filename(item.name) mf, f = fm.mkdir(target_path, safe_name, target_system_id, username, raise_if_exists=False) logger.info('Created directory "{0}"; scheduling transfer of contents', f.full_path) except HTTPError as e: logger.error('Agave.files.manage(mkdir) failed') self.update_state(state='FAILURE', meta={'exc': e}) # schedule to copy all of it's items limit = 10 offset = 0 while True: item_collection = item.get_items(limit=limit, offset=offset) for it in item_collection: args = (username, it.type, it.object_id, target_system_id, f.full_path) copy_box_item.apply_async(args=args, countdown=offset*2) if len(item_collection) == limit: offset += limit else: break
def call_api(request, service): try: agave = request.user.agave_oauth.client if service == 'apps': app_id = request.GET.get('app_id') if app_id: data = agave.apps.get(appId=app_id) lic_type = _app_license_type(app_id) data['license'] = { 'type': lic_type } if lic_type is not None: lic = request.user.licenses.filter(license_type=lic_type).first() data['license']['enabled'] = lic is not None else: public_only = request.GET.get('publicOnly') if public_only == 'true': data = agave.apps.list(publicOnly='true') else: data = agave.apps.list() elif service == 'monitors': target = request.GET.get('target') ds_admin_client = Agave(api_server=getattr(settings, 'AGAVE_TENANT_BASEURL'), token=getattr(settings, 'AGAVE_SUPER_TOKEN')) data = ds_admin_client.monitors.list(target=target) elif service == 'meta': app_id = request.GET.get('app_id') if request.method == 'GET': if app_id: data = agave.meta.get(appId=app_id) lic_type = _app_license_type(app_id) data['license'] = { 'type': lic_type } if lic_type is not None: lic = request.user.licenses.filter(license_type=lic_type).first() data['license']['enabled'] = lic is not None else: query = request.GET.get('q') data = agave.meta.listMetadata(q=query) elif request.method == 'POST': meta_post = json.loads(request.body) meta_uuid = meta_post.get('uuid') if meta_uuid: del meta_post['uuid'] data = agave.meta.updateMetadata(uuid=meta_uuid, body=meta_post) else: data = agave.meta.addMetadata(body=meta_post) elif request.method == 'DELETE': meta_uuid = request.GET.get('uuid') if meta_uuid: data = agave.meta.deleteMetadata(uuid=meta_uuid) elif service == 'files': system_id = request.GET.get('system_id') file_path = request.GET.get('file_path', '') special_dir = None if system_id == 'designsafe.storage.default': if shared_with_me in file_path: special_dir = shared_with_me file_path = '/'.join(file_path.split('/')[2:]) elif file_path == '': file_path = request.user.username file_path = file_path.strip('/') if file_path == '': file_path = '/' logger.debug('Listing "agave://%s/%s"...' % (system_id, file_path)) # Agave Files call # data = agave.files.list(systemId=system_id, filePath=file_path) # ElasticSearch call try: fm = FileManager(agave) listing = fm.list_path(system_id=system_id, path=file_path, username=request.user.username, special_dir=special_dir, is_public=system_id == 'nees.public' ) data = [f.to_dict() for f in listing] # filter out empty projects if system_id == 'nees.public' and file_path == '/': data = [f for f in data if 'projecTitle' in f and not f['projecTitle'].startswith('EMPTY PROJECT')] # TODO type of "Shared with me" should be "dir" not "folder" for d in data: d.update((k, 'dir') for k, v in six.iteritems(d) if k == 'type' and v == 'folder') if special_dir: data = [{ 'name': '.', 'path': special_dir + file_path.strip('/'), 'systemId': system_id, 'type': 'dir', }] + data elif file_path != request.user.username: data = [{ 'name': '.', 'path': file_path, 'systemId': system_id, 'type': 'dir', }] + data except: data = [] elif service == 'jobs': if request.method == 'DELETE': job_id = request.GET.get('job_id') data = agave.jobs.delete(jobId=job_id) elif request.method == 'POST': job_post = json.loads(request.body) job_id = job_post.get('job_id') # cancel job / stop job if job_id: data = agave.jobs.manage(jobId=job_id, body='{"action":"stop"}') # submit job elif job_post: # cleaning archive path value if 'archivePath' in job_post: parsed = urlparse(job_post['archivePath']) if parsed.path.startswith('/'): # strip leading '/' archive_path = parsed.path[1:] else: archive_path = parsed.path if not archive_path.startswith(request.user.username): archive_path = '{}/{}'.format( request.user.username, archive_path) job_post['archivePath'] = archive_path if parsed.netloc: job_post['archiveSystem'] = parsed.netloc else: job_post['archivePath'] = \ '{}/archive/jobs/{}/${{JOB_NAME}}-${{JOB_ID}}'.format( request.user.username, datetime.now().strftime('%Y-%m-%d')) # check for running licensed apps lic_type = _app_license_type(job_post['appId']) if lic_type is not None: lic = request.user.licenses.filter(license_type=lic_type).first() job_post['parameters']['_license'] = lic.license_as_str() # url encode inputs if job_post['inputs']: for key, value in six.iteritems(job_post['inputs']): parsed = urlparse(value) if parsed.scheme: job_post['inputs'][key] = '{}://{}{}'.format( parsed.scheme, parsed.netloc, urllib.quote(parsed.path)) else: job_post['inputs'][key] = urllib.quote(parsed.path) try: data = submit_job(request, request.user.username, job_post) except JobSubmitError as e: data = e.json() logger.error('Failed to submit job {0}'.format(data)) return HttpResponse(json.dumps(data), content_type='application/json', status=e.status_code) # list jobs (via POST?) else: limit = request.GET.get('limit', 10) offset = request.GET.get('offset', 0) data = agave.jobs.list(limit=limit, offset=offset) elif request.method == 'GET': job_id = request.GET.get('job_id') # get specific job info if job_id: data = agave.jobs.get(jobId=job_id) q = {"associationIds": job_id} job_meta = agave.meta.listMetadata(q=json.dumps(q)) data['_embedded'] = {"metadata": job_meta} archive_system_path = '{}/{}'.format(data['archiveSystem'], data['archivePath']) data['archiveUrl'] = reverse( 'designsafe_data:data_depot') data['archiveUrl'] += 'agave/{}/'.format(archive_system_path) # list jobs else: limit = request.GET.get('limit', 10) offset = request.GET.get('offset', 0) data = agave.jobs.list(limit=limit, offset=offset) else: return HttpResponse('Unexpected service: %s' % service, status=400) else: return HttpResponse('Unexpected service: %s' % service, status=400) except HTTPError as e: logger.error('Failed to execute {0} API call due to HTTPError={1}'.format( service, e.message)) return HttpResponse(json.dumps(e.message), content_type='application/json', status=400) except AgaveException as e: logger.error('Failed to execute {0} API call due to AgaveException={1}'.format( service, e.message)) return HttpResponse(json.dumps(e.message), content_type='application/json', status=400) except Exception as e: logger.error('Failed to execute {0} API call due to Exception={1}'.format( service, e.message)) return HttpResponse( json.dumps({'status': 'error', 'message': '{}'.format(e.message)}), content_type='application/json', status=400) return HttpResponse(json.dumps(data, cls=DjangoJSONEncoder), content_type='application/json')