def fix_tags_no_path(entity): if entity['name'] == 'designsafe.project': proj_other = BaseFileResource.listing(service_account(), system="project-{}".format(entity['uuid']), path="") for child in proj_other.children: try: pub_file = BaseFileResource.listing(service_account(), system="designsafe.storage.published", path="{}{}".format(project_id, child.path)) proj_file = BaseFileResource.listing(service_account(), system="project-{}".format(entity['uuid']), path=child.path) for tag in entity['value']['fileTags']: if tag['fileUuid'] == proj_file.uuid: tag['fileUuid'] = pub_file.uuid except Exception as err: LOG.info('error: {}'.format(err)) continue else: for fobj in entity['fileObjs']: try: pub_file = BaseFileResource.listing(service_account(), system="designsafe.storage.published", path="{}{}".format(project_id, fobj['path'])) proj_file = BaseFileResource.listing(service_account(), system="project-{}".format(pub_dict['project']['uuid']), path=fobj['path']) for tag in entity['value']['fileTags']: if tag['fileUuid'] == proj_file.uuid: tag['fileUuid'] = pub_file.uuid except Exception as err: LOG.info('error: {}'.format(err)) continue
def mkdir(self, system, file_path, dir_name): f = BaseFileResource(self._ag, system, file_path) resp = f.mkdir(dir_name) reindex_agave.apply_async(kwargs={ 'username': '******', 'file_id': '{}/{}'.format(system, file_path) }, queue='indexing') return resp
def upload(self, system, file_path, upload_file): f = BaseFileResource(self._ag, system, file_path) resp = f.upload(upload_file) reindex_agave.apply_async(kwargs={ 'username': '******', 'file_id': '{}/{}'.format(system, file_path), 'levels': 1 }, queue='indexing') return resp
def upload(self, system, file_path, upload_file): f = BaseFileResource(self._ag, system, file_path) resp = f.upload(upload_file) agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': os.path.dirname(resp.path), 'recurse': False }, queue='indexing') return resp
def mkdir(self, system, file_path, dir_name): f = BaseFileResource(self._ag, system, file_path) resp = f.mkdir(dir_name) agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': f.path, 'recurse': False }, queue='indexing') return resp
def delete(self, system, path): f = BaseFileResource(self._ag, system, path) resp = f.delete() parent_path = '/'.join(path.strip('/').split('/')[:-1]) agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': os.path.dirname(f.path), 'recurse': False }, queue='indexing') return resp
def copy(self, system, file_path, dest_path=None, dest_name=None): f = BaseFileResource.listing(self._ag, system, file_path) # default to same path if dest_path is None: dest_path = f.path # default to same name if dest_name is None: dest_name = f.name # if same path and name, add suffix to file name if dest_name == f.name and dest_path == f.path: dest_name = '{0}_copy{1}'.format(*os.path.splitext(dest_name)) copied_file = f.copy(dest_path, dest_name) # schedule celery task to index new copy reindex_agave.apply_async(kwargs={ 'username': '******', 'file_id': '{}/{}/{}'.format(system, dest_path.strip('/'), dest_name) }, queue='indexing') return copied_file
def copy(self, system, file_path, dest_path=None, dest_name=None): f = BaseFileResource.listing(self._ag, system, file_path) # default to same path if dest_path is None: dest_path = f.path # default to same name if dest_name is None: dest_name = f.name # if same path and name, add suffix to file name if dest_name == f.name and dest_path == f.path: dest_name = '{0}_copy{1}'.format(*os.path.splitext(dest_name)) copied_file = f.copy(dest_path, dest_name) agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': os.path.dirname(copied_file.path), 'recurse': False }, queue='indexing') if copied_file.format == 'folder': agave_indexer.apply_async(kwargs={ 'systemId': system, 'filePath': copied_file.path, 'recurse': True }, routing_key='indexing') return copied_file
def import_data(self, system, file_path, from_system, from_file_path): file_path = file_path or '/' if file_path != '/': file_path = file_path.strip('/') from_file_path = from_file_path.strip('/') f = BaseFileResource.listing(self._ag, system, file_path) res = f.import_data(from_system, from_file_path) file_name = from_file_path.split('/')[-1] agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': os.path.dirname(res.path), 'recurse': False }, queue='indexing') if res.format == 'folder': agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': res.path, 'recurse': True }, queue='indexing') return res
def move(self, system, file_path, dest_path, dest_name=None): f = BaseFileResource.listing(self._ag, system, file_path) resp = f.move(dest_path, dest_name) parent_path = '/'.join(file_path.strip('/').split('/')[:-1]) parent_path = parent_path.strip('/') or '/' agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': os.path.dirname(f.path), 'recurse': False }, queue='indexing') agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': os.path.dirname(resp.path), 'recurse': False }, queue='indexing') if resp.format == 'folder': agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': resp.path }, queue='indexing') return resp
def share(self, system, file_path, username, permission): f = BaseFileResource(self._ag, system, file_path) recursive = True pem = BaseFilePermissionResource(self._ag, f, recursive=recursive) pem.username = username pem.permission_bit = permission resp = pem.save() return resp
def trash(self, system, file_path, trash_path): name = os.path.basename(file_path) f = BaseFileResource(self._ag, system, file_path) # first ensure trash_path exists BaseFileResource.ensure_path(self._ag, system, trash_path) # check if file with same name exists in trash; expect 404 try: check = os.path.join(trash_path, name) BaseFileResource.listing(self._ag, system, check) # if we did not 404, then we have a conflict; append date to name name_ext = os.path.splitext(name) timestamp = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') name = '{0} {1}{2}'.format(name_ext[0], timestamp, name_ext[1]) except HTTPError as e: if e.response.status_code != 404: raise resp = f.move(trash_path, name) parent_path = '/'.join(file_path.strip('/').split('/')[:-1]) parent_path = parent_path.strip('/') or '/' agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': os.path.dirname(f.path), 'recurse': False }, queue='indexing') return resp
def project_directory(self): """ Queries for the File object that represents the root of this Project's files. :return: The project's root dir :rtype: :class:`BaseFileResource` """ if self._project_directory is None: self._project_directory = BaseFileResource.listing( system=self.project_system_id, path='/', agave_client=self._agave) return self._project_directory
def fix_tags_path(entity): for tag in entity['value']['fileTags']: try: pub_file = BaseFileResource.listing( service_account(), system="designsafe.storage.published", path="{}{}".format(project_id, tag['path'])) tag['fileUuid'] = pub_file.uuid except Exception as err: logger.info('error: {}'.format(err)) continue
def delete(self, system, path): resp = BaseFileResource(self._ag, system, path).delete() parent_path = '/'.join(path.strip('/').split('/')[:-1]) reindex_agave.apply_async(kwargs={ 'username': '******', 'file_id': '{}/{}'.format(system, parent_path), 'levels': 1 }, queue='indexing') return resp
def rename(self, system, file_path, rename_to): f = BaseFileResource.listing(self._ag, system, file_path) resp = f.rename(rename_to) parent_path = '/'.join(file_path.strip('/').split('/')[:-1]) agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': system, 'filePath': os.path.dirname(resp.path), 'recurse': False }, queue='indexing') return resp
def share(self, system, file_path, username, permission): f = BaseFileResource(self._ag, system, file_path) recursive = True pem = BaseFilePermissionResource(self._ag, f, recursive=recursive) pem.username = username pem.permission_bit = permission resp = pem.save() reindex_agave.apply_async(kwargs={ 'username': '******', 'file_id': '{}/{}'.format(system, file_path) }, queue='indexing') return resp
def post(self, request, project_id, name): """ :param request: :return: """ ag = get_service_account_client() if request.is_ajax(): post_data = json.loads(request.body) else: post_data = request.POST.copy() entity = post_data.get('entity') try: model_cls = self._lookup_model(name) logger.debug('entity: %s', entity) model = model_cls(value=entity) logger.debug('model uuid: %s', model.uuid) file_uuids = [] if 'filePaths' in entity: file_paths = entity.get('filePaths', []) project_system = ''.join(['project-', project_id]) user_ag = request.user.agave_oauth.client for file_path in file_paths: file_obj = BaseFileResource.listing( user_ag, project_system, file_path) file_uuids.append(file_obj.uuid) for file_uuid in file_uuids: model.files.add(file_uuid) model.associate(file_uuids) model.project.add(project_id) model.associate(project_id) saved = model.save(ag) resp = model_cls(**saved) #TODO: This should happen in a celery task and implemented in a manager #Get project's metadata permissions pems = BaseMetadataPermissionResource.list_permissions( project_id, ag) #Loop permissions and set them in whatever metadata object we're saving for pem in pems: _pem = BaseMetadataPermissionResource(resp.uuid, ag) _pem.username = pem.username _pem.read = pem.read _pem.write = pem.write _pem.save() except ValueError: return HttpResponseBadRequest('Entity not valid.') return JsonResponse(resp.to_body_dict(), safe=False)
def rename(self, system, file_path, rename_to): f = BaseFileResource.listing(self._ag, system, file_path) resp = f.rename(rename_to) parent_path = '/'.join(file_path.strip('/').split('/')[:-1]) reindex_agave.apply_async(kwargs={ 'username': '******', 'file_id': '{}/{}'.format(system, parent_path), 'levels': 1 }, queue='indexing') return resp
def get(self, request, file_path='', project_id=None, system_id=None, project_system_id=None, file_mgr_name=None): """ :return: The root directory for the Project's data :rtype: JsonResponse """ ag = request.user.agave_oauth.client if project_system_id is None: p = Project(ag, uuid=project_id) project_system_id = p.project_system_id listing = BaseFileResource.listing(ag, project_system_id, file_path) return JsonResponse(listing, encoder=AgaveJSONEncoder, safe=False)
def post(self, request, project_id, name): """ Create project related metadata object. :param request: :return: """ sa_client = get_service_account_client() if request.is_ajax(): post_data = json.loads(request.body) else: post_data = request.POST.copy() entity = post_data.get('entity') try: model_cls = project_lookup_model(name=name) model = model_cls(value=entity) file_uuids = [] if 'filePaths' in entity: file_paths = entity.get('filePaths', []) project_system = ''.join(['project-', project_id]) client = request.user.agave_oauth.client for file_path in file_paths: file_obj = BaseFileResource.listing( client, project_system, file_path) file_uuids.append(file_obj.uuid) for file_uuid in file_uuids: model.files.add(file_uuid) model.associate(file_uuids) model.project.add(project_id) model.associate(project_id) saved = model.save(sa_client) resp = model_cls(**saved) # TODO: We should stop using these "Resources" and just use agavepy methods. pems = BaseMetadataPermissionResource.list_permissions( project_id, sa_client) # Loop permissions and set them in whatever metadata object we're saving for pem in pems: _pem = BaseMetadataPermissionResource(resp.uuid, sa_client) _pem.username = pem.username _pem.read = pem.read _pem.write = pem.write _pem.save() except ValueError: return HttpResponseBadRequest('Entity not valid.') return JsonResponse(resp.to_body_dict(), safe=False)
def import_data(self, system, file_path, from_system, from_file_path): file_path = file_path or '/' if file_path != '/': file_path = file_path.strip('/') from_file_path = from_file_path.strip('/') f = BaseFileResource.listing(self._ag, system, file_path) res = f.import_data(from_system, from_file_path) file_name = from_file_path.split('/')[-1] reindex_agave.apply_async(kwargs={ 'username': '******', 'file_id': '{}/{}'.format(system, os.path.join(file_path, file_name)) }, queue='indexing') return res
def move(self, system, file_path, dest_path, dest_name=None): f = BaseFileResource.listing(self._ag, system, file_path) resp = f.move(dest_path, dest_name) parent_path = '/'.join(file_path.strip('/').split('/')[:-1]) parent_path = parent_path.strip('/') or '/' reindex_agave.apply_async(kwargs={ 'username': '******', 'file_id': '{}/{}'.format(system, parent_path), 'levels': 1 }, queue='indexing') reindex_agave.apply_async(kwargs={ 'username': '******', 'file_id': '{}/{}'.format(system, os.path.join(dest_path, resp.name)), 'levels': 1 }, queue='indexing') return resp
def list_permissions(self, system, file_path): f = BaseFileResource(self._ag, system, file_path) return BaseFilePermissionResource.list_permissions(self._ag, f)
def post(self, request): """ Create a new Project. Projects and the root File directory for a Project should be owned by the portal, with roles/permissions granted to the creating user. 1. Create the metadata record for the project 2. Create a directory on the projects storage system named after the metadata uuid 3. Associate the metadata uuid and file uuid :param request: :return: The newly created project :rtype: JsonResponse """ # portal service account needs to create the objects on behalf of the user ag = get_service_account_client() if request.is_ajax(): post_data = json.loads(request.body) else: post_data = request.POST.copy() # create Project (metadata) metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'project_create', 'info': { 'postData': post_data } }) prj = BaseProject() prj.manager().set_client(ag) prj.save(ag) project_uuid = prj.uuid prj.title = post_data.get('title') prj.award_number = post_data.get('awardNumber', '') prj.project_type = post_data.get('projectType', 'other') prj.associated_projects = post_data.get('associatedProjects', {}) prj.description = post_data.get('description', '') prj.pi = post_data.get('pi') prj.keywords = post_data.get('keywords', '') prj.project_id = post_data.get('projectId', '') prj.save(ag) # create Project Directory on Managed system metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'base_directory_create', 'info': { 'systemId': Project.STORAGE_SYSTEM_ID, 'uuid': prj.uuid } }) project_storage_root = BaseFileResource(ag, Project.STORAGE_SYSTEM_ID, '/') project_storage_root.mkdir(prj.uuid) # Wrap Project Directory as private system for project project_system_tmpl = template_project_storage_system(prj) project_system_tmpl['storage']['rootDir'] = \ project_system_tmpl['storage']['rootDir'].format(project_uuid) metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'private_system_create', 'info': { 'id': project_system_tmpl.get('id'), 'site': project_system_tmpl.get('site'), 'default': project_system_tmpl.get('default'), 'status': project_system_tmpl.get('status'), 'description': project_system_tmpl.get('description'), 'name': project_system_tmpl.get('name'), 'globalDefault': project_system_tmpl.get('globalDefault'), 'available': project_system_tmpl.get('available'), 'public': project_system_tmpl.get('public'), 'type': project_system_tmpl.get('type'), 'storage': { 'homeDir': project_system_tmpl.get('storage', {}).get('homeDir'), 'rootDir': project_system_tmpl.get('storage', {}).get('rootDir') } } }) ag.systems.add(body=project_system_tmpl) # grant initial permissions for creating user and PI, if exists metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'initial_pems_create', 'info': { 'collab': request.user.username, 'pi': prj.pi } }) prj.add_team_members([request.user.username]) tasks.set_facl_project.apply_async( args=[prj.uuid, [request.user.username]], queue='api') if prj.pi and prj.pi != request.user.username: prj.add_team_members([prj.pi]) tasks.set_facl_project.apply_async(args=[prj.uuid, [prj.pi]], queue='api') collab_users = get_user_model().objects.filter(username=prj.pi) if collab_users: collab_user = collab_users[0] try: collab_user.profile.send_mail( "[Designsafe-CI] You have been added to a project!", "<p>You have been added to the project <em> {title} </em> as PI</p><p>You can visit the project using this url <a href=\"{url}\">{url}</a>" .format(title=prj.title, url=request.build_absolute_uri( reverse('designsafe_data:data_depot') + '/projects/%s/' % (prj.uuid, )))) except DesignSafeProfile.DoesNotExist as err: logger.info("Could not send email to user %s", collab_user) body = "<p>You have been added to the project <em> {title} </em> as PI</p><p>You can visit the project using this url <a href=\"{url}\">{url}</a>".format( title=prj.title, url=request.build_absolute_uri( reverse('designsafe_data:data_depot') + '/projects/%s/' % (prj.uuid, ))) send_mail( "[Designsafe-CI] You have been added to a project!", body, settings.DEFAULT_FROM_EMAIL, [collab_user.email], html_message=body) prj.add_admin('prjadmin') tasks.set_project_id.apply_async(args=[prj.uuid], queue="api") return JsonResponse(prj.to_body_dict(), safe=False)
def project_data_listing(self, path='/'): return BaseFileResource.listing(system=self.project_system_id, path=path, agave_client=self._agave)
def post(self, request, file_mgr_name, system_id, file_path): if file_mgr_name == AgaveFileManager.NAME \ or file_mgr_name == 'public': if not request.user.is_authenticated: return HttpResponseForbidden('Login required') agave_client = request.user.agave_oauth.client fm = AgaveFileManager(agave_client=agave_client) if request.FILES: upload_file = request.FILES['file'] upload_dir = file_path relative_path = request.POST.get('relative_path', None) if relative_path: # user uploaded a folder structure; ensure path exists upload_dir = os.path.join(file_path, os.path.dirname(relative_path)) BaseFileResource.ensure_path(agave_client, system_id, upload_dir) metrics.info('Data Depot', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'data_depot_folder_upload', 'info': { 'filePath': file_path, 'relativePath': os.path.dirname(relative_path), 'systemId': system_id, 'uploadDir': upload_dir } }) try: result = fm.upload(system_id, upload_dir, upload_file) metrics.info('Data Depot', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'data_depot_file_upload', 'info': { 'systemId': system_id, 'uploadDir': upload_dir, 'uploadFile': upload_file } }) result['system'] = result['systemId'] result_file = BaseFileResource(agave_client, **result) event_data = { Notification.EVENT_TYPE: 'data_depot', Notification.OPERATION: 'data_depot_file_upload', Notification.STATUS: Notification.SUCCESS, Notification.USER: request.user.username, Notification.MESSAGE: 'File Upload was successful.', Notification.EXTRA: result_file.to_dict() } Notification.objects.create(**event_data) except HTTPError as e: logger.error(e.response.text) event_data = { Notification.EVENT_TYPE: 'data_depot', Notification.OPERATION: 'data_depot_file_upload', Notification.STATUS: Notification.ERROR, Notification.USER: request.user.username, Notification.MESSAGE: 'There was an error uploading one or more file(s).', Notification.EXTRA: { 'system': system_id, 'file_path': file_path } } Notification.objects.create(**event_data) return HttpResponseBadRequest(e.response.text) return JsonResponse({'status': 'ok'}) return HttpResponseBadRequest("Unsupported operation")
def download(self, system, path): file_obj = BaseFileResource.listing(self._ag, system, path) postit = file_obj.download_postit() return postit
def post(self, request): """ Create a new Project. Projects and the root File directory for a Project should be owned by the portal, with roles/permissions granted to the creating user. 1. Create the metadata record for the project 2. Create a directory on the projects storage system named after the metadata uuid 3. Associate the metadata uuid and file uuid :param request: :return: The newly created project :rtype: JsonResponse """ # portal service account needs to create the objects on behalf of the user ag = get_service_account_client() if request.is_ajax(): post_data = json.loads(request.body) else: post_data = request.POST.copy() # create Project (metadata) metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'project_create', 'info': { 'postData': post_data } }) prj_model = project_lookup_model({ 'name': 'designsafe.project', 'value': post_data }) prj = prj_model(value=post_data) project_uuid = prj.uuid prj.manager().set_client(ag) prj.save(ag) # create Project Directory on Managed system metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'base_directory_create', 'info': { 'systemId': Project.STORAGE_SYSTEM_ID, 'uuid': prj.uuid } }) project_storage_root = BaseFileResource(ag, Project.STORAGE_SYSTEM_ID, '/') project_storage_root.mkdir(prj.uuid) # Wrap Project Directory as private system for project project_system_tmpl = template_project_storage_system(prj) project_system_tmpl['storage']['rootDir'] = \ project_system_tmpl['storage']['rootDir'].format(project_uuid) metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'private_system_create', 'info': { 'id': project_system_tmpl.get('id'), 'site': project_system_tmpl.get('site'), 'default': project_system_tmpl.get('default'), 'status': project_system_tmpl.get('status'), 'description': project_system_tmpl.get('description'), 'name': project_system_tmpl.get('name'), 'globalDefault': project_system_tmpl.get('globalDefault'), 'available': project_system_tmpl.get('available'), 'public': project_system_tmpl.get('public'), 'type': project_system_tmpl.get('type'), 'storage': { 'homeDir': project_system_tmpl.get('storage', {}).get('homeDir'), 'rootDir': project_system_tmpl.get('storage', {}).get('rootDir') } } }) ag.systems.add(body=project_system_tmpl) # grant initial permissions for creating user and PI, if exists metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'initial_pems_create', 'info': { 'collab': request.user.username, 'pi': prj.pi } }) if getattr(prj, 'copi', None): prj.add_co_pis(prj.copi) elif getattr(prj, 'co_pis', None): prj.add_co_pis(prj.co_pis) if getattr(prj, 'team', None): prj.add_team_members(prj.team) elif getattr(prj, 'team_members', None): prj.add_team_members(prj.team_members) prj._add_team_members_pems([prj.pi]) if request.user.username not in list( set(prj.co_pis + prj.team_members + [prj.pi])): # Add creator to project as team member prj.add_team_members([request.user.username]) # Email collaborators chain( tasks.set_project_id.s(prj.uuid).set(queue="api") | tasks.email_collaborator_added_to_project.s( prj.uuid, prj.title, request.build_absolute_uri('{}/projects/{}/'.format( reverse('designsafe_data:data_depot'), prj.uuid)), [ u for u in list(set(prj.co_pis + prj.team_members + [prj.pi])) if u != request.user.username ], [])).apply_async() tasks.set_facl_project.apply_async(args=[ prj.uuid, list(set(prj.co_pis + prj.team_members + [prj.pi])) ], queue='api') prj.add_admin('prjadmin') return JsonResponse(prj.to_body_dict(), safe=False)
def listing(self, system, file_path, offset=0, limit=100, **kwargs): return BaseFileResource.listing(self._ag, system, file_path, offset, limit)