def check_project_files_meta_pems(self, project_uuid): from designsafe.apps.api.agave.models.files import BaseFileMetadata logger.debug('Checking metadata pems linked to a project') service = get_service_account_client() metas = BaseFileMetadata.search(service, {'associationIds': project_uuid, 'name': BaseFileMetadata.NAME}) for meta in metas: meta.match_pems_to_project(project_uuid)
def delete(self, request, project_id): if request.is_ajax(): post_data = json.loads(request.body) else: post_data = request.POST.copy() ag = get_service_account_client() project = Project.from_uuid(agave_client=ag, uuid=project_id) project.remove_collaborator(post_data.get('username')) tasks.check_project_files_meta_pems(project.uuid) return JsonResponse({'status': 'ok'})
def get(self, request, file_mgr_name, system_id, file_path): if file_mgr_name == AgaveFileManager.NAME \ or file_mgr_name == 'public': if not request.user.is_authenticated(): return HttpResponseForbidden('Log in required') # List permissions as the portal user rather than logged in user. # This also works around the issue where pems on private systems are # inconsistent. fm = AgaveFileManager(agave_client=get_service_account_client()) pems = fm.list_permissions(system_id, file_path) return JsonResponse(pems, encoder=AgaveJSONEncoder, safe=False) return HttpResponseBadRequest("Unsupported operation")
def get(self, request, system_id=None): params = request.GET.copy() if request.user.is_authenticated(): ag = request.user.agave_oauth.client if system_id is None: systems = BaseSystemResource.list(ag, **params) return JsonResponse(systems, encoder=AgaveJSONEncoder, safe=False) else: system = BaseSystemResource.from_id(ag, system_id) return JsonResponse(system, encoder=AgaveJSONEncoder, safe=False) else: # Force public=true params.pop('public', None) ag = get_service_account_client() systems = BaseSystemResource.list(ag, public=True, **params) return JsonResponse(systems, encoder=AgaveJSONEncoder, safe=False)
def post(self, request, project_id): """ :param request: :return: """ ag = get_service_account_client() if request.is_ajax(): post_data = json.loads(request.body) else: post_data = request.POST.copy() # save Project (metadata) p = Project.from_uuid(ag, project_id) p.title = post_data.get('title') new_pi = post_data.get('pi', None) if p.pi != new_pi: p.pi = new_pi p.add_collaborator(new_pi) p.save() return JsonResponse(p, encoder=AgaveJSONEncoder, safe=False)
def post(self, request): """ Create a new Project. Projects and the root File directory for a Project should be owned by the portal, with roles/permissions granted to the creating user. 1. Create the metadata record for the project 2. Create a directory on the projects storage system named after the metadata uuid 3. Associate the metadata uuid and file uuid :param request: :return: The newly created project :rtype: JsonResponse """ # portal service account needs to create the objects on behalf of the user ag = get_service_account_client() if request.is_ajax(): post_data = json.loads(request.body) else: post_data = request.POST.copy() # create Project (metadata) metrics.info('projects', extra={'user' : request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'metadata_create', 'info': {'postData': post_data} }) p = Project(ag) p.title = post_data.get('title') p.pi = post_data.get('pi', None) p.save() # create Project Directory on Managed system metrics.info('projects', extra={'user' : request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'base_directory_create', 'info': { 'systemId': Project.STORAGE_SYSTEM_ID, 'uuid': p.uuid }}) project_storage_root = BaseFileResource(ag, Project.STORAGE_SYSTEM_ID, '/') project_storage_root.mkdir(p.uuid) # Wrap Project Directory as private system for project project_system_tmpl = template_project_storage_system(p) metrics.info('projects', extra={'user' : request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'private_system_create', 'info': { 'id': project_system_tmpl.get('id'), 'site': project_system_tmpl.get('site'), 'default': project_system_tmpl.get('default'), 'status': project_system_tmpl.get('status'), 'description': project_system_tmpl.get('description'), 'name': project_system_tmpl.get('name'), 'globalDefault': project_system_tmpl.get('globalDefault'), 'available': project_system_tmpl.get('available'), 'public': project_system_tmpl.get('public'), 'type': project_system_tmpl.get('type'), 'storage': { 'homeDir': project_system_tmpl.get('storage', {}).get('homeDir'), 'rootDir': project_system_tmpl.get('storage', {}).get('rootDir') } }}) ag.systems.add(body=project_system_tmpl) # grant initial permissions for creating user and PI, if exists project_system_tmpl = template_project_storage_system(p) metrics.info('projects', extra={'user' : request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'initial_pems_create', 'info': {'collab': request.user.username, 'pi': p.pi} }) p.add_collaborator(request.user.username) if p.pi and p.pi != request.user.username: p.add_collaborator(p.pi) return JsonResponse(p, encoder=AgaveJSONEncoder, safe=False)
def check_published_files(project_id, revision=None, selected_files=None): #get list of files that should be in the publication es_client = new_es_client() publication = BaseESPublication(project_id=project_id, revision=revision, using=es_client) if selected_files: #it's type other, use this for comparison filepaths = selected_files else: filepaths = publication.related_file_paths() #empty dirs missing_files = [] existing_files = [] empty_folders = [] #strip leading forward slash from file paths updated_filepaths = [ file_path.strip('/') for file_path in filepaths if (file_path != '.Trash') ] pub_directory = '/corral-repl/tacc/NHERI/published/{}'.format(project_id) if revision: pub_directory += 'v{}'.format(revision) #navigate through publication files paths and #compare to the previous list of files for pub_file in updated_filepaths: file_to_check = os.path.join(pub_directory, pub_file) try: if os.path.isfile(file_to_check): existing_files.append(pub_file) elif os.path.isdir(file_to_check): #check directory for items in it dir_list = os.listdir(file_to_check) if dir_list != []: existing_files.append(pub_file) else: empty_folders.append(pub_file) else: missing_files.append(pub_file) except OSError as exc: logger.info(exc) #send email if there are files/folders missing/empty if (missing_files or empty_folders): #log for potential later queries logger.info("check_published_files missing files: " + project_id + " " + str(missing_files)) logger.info("check_published_files empty folders: " + project_id + " " + str(empty_folders)) #send email to dev admins service = get_service_account_client() prj_admins = settings.DEV_PROJECT_ADMINS_EMAIL for admin in prj_admins: email_body = """ <p>Hello,</p> <p> The following project has been published with either missing files/folders or empty folders: <br> <b>{prjID} - revision {revision}</b> <br> Path to publication files: {pubFiles} </p> <p> These are the missing files/folders for this publication: <br> {missingFiles} </p> <p> These are the empty folders for this publication: <br> {emptyFolders} </p> This is a programmatically generated message. Do NOT reply to this message. """.format(pubFiles=pub_directory, prjID=project_id, missingFiles=missing_files, emptyFolders=empty_folders, revision=revision) send_mail( "DesignSafe Alert: Published Project has missing files/folders", email_body, settings.DEFAULT_FROM_EMAIL, [admin], html_message=email_body)
def copy_publication_files_to_corral(self, project_id): from designsafe.apps.api.agave.filemanager.public_search_index import Publication import shutil publication = Publication(project_id=project_id) filepaths = publication.related_file_paths() if not len(filepaths): res = get_service_account_client().files.list( systemId='project-{project_uuid}'.format( project_uuid=publication.project.uuid), filePath='/') filepaths = [ _file.path.strip('/') for _file in res if (_file.name != '.' and _file.name != 'Trash') ] filepaths = list(set(filepaths)) filepaths = sorted(filepaths) base_path = ''.join(['/', publication.projectId]) os.chmod('/corral-repl/tacc/NHERI/published', 0755) prefix_dest = '/corral-repl/tacc/NHERI/published/{}'.format(project_id) if not os.path.isdir(prefix_dest): os.mkdir(prefix_dest) prefix_src = '/corral-repl/tacc/NHERI/projects/{}'.format( publication.project['uuid']) for filepath in filepaths: local_src_path = '{}/{}'.format(prefix_src, filepath) local_dst_path = '{}/{}'.format(prefix_dest, filepath) logger.info('Trying to copy: %s to %s', local_src_path, local_dst_path) if os.path.isdir(local_src_path): try: #os.mkdir(local_dst_path) if not os.path.isdir(os.path.dirname(local_dst_path)): os.makedirs(os.path.dirname(local_dst_path)) shutil.copytree(local_src_path, local_dst_path) for root, dirs, files in os.walk(local_dst_path): for d in dirs: os.chmod(os.path.join(root, d), 0555) for f in files: os.chmod(os.path.join(root, f), 0444) os.chmod(local_dst_path, 0555) except OSError as exc: logger.info(exc) except IOError as exc: logger.info(exc) else: try: if not os.path.isdir(os.path.dirname(local_dst_path)): os.makedirs(os.path.dirname(local_dst_path)) for root, dirs, files in os.walk( os.path.dirname(local_dst_path)): for d in dirs: os.chmod(os.path.join(root, d), 0555) for f in files: os.chmod(os.path.join(root, f), 0444) shutil.copy(local_src_path, local_dst_path) os.chmod(local_dst_path, 0444) except OSError as exc: logger.info(exc) except IOError as exc: logger.info(exc) os.chmod(prefix_dest, 0555) os.chmod('/corral-repl/tacc/NHERI/published', 0555) save_to_fedora.apply_async(args=[project_id]) agave_indexer.apply_async(kwargs={ 'username': '******', 'systemId': 'designsafe.storage.published', 'filePath': '/' + project_id, 'recurse': True }, queue='indexing')
def check_project_meta_pems(self, metadata_uuid): from designsafe.apps.api.agave.models.files import BaseFileMetadata logger.debug('Checking single metadata pems linked to a project %s', metadata_uuid) service = get_service_account_client() bfm = BaseFileMetadata.from_uuid(service, metadata_uuid) bfm.match_pems_to_project()
def post(self, request): """ Create a new Project. Projects and the root File directory for a Project should be owned by the portal, with roles/permissions granted to the creating user. 1. Create the metadata record for the project 2. Create a directory on the projects storage system named after the metadata uuid 3. Associate the metadata uuid and file uuid :param request: :return: The newly created project :rtype: JsonResponse """ # portal service account needs to create the objects on behalf of the user ag = get_service_account_client() if request.is_ajax(): post_data = json.loads(request.body) else: post_data = request.POST.copy() # create Project (metadata) metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'project_create', 'info': { 'postData': post_data } }) prj_model = project_lookup_model({ 'name': 'designsafe.project', 'value': post_data }) prj = prj_model(value=post_data) project_uuid = prj.uuid prj.manager().set_client(ag) prj.save(ag) # create Project Directory on Managed system metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'base_directory_create', 'info': { 'systemId': Project.STORAGE_SYSTEM_ID, 'uuid': prj.uuid } }) project_storage_root = BaseFileResource(ag, Project.STORAGE_SYSTEM_ID, '/') project_storage_root.mkdir(prj.uuid) # Wrap Project Directory as private system for project project_system_tmpl = template_project_storage_system(prj) project_system_tmpl['storage']['rootDir'] = \ project_system_tmpl['storage']['rootDir'].format(project_uuid) metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'private_system_create', 'info': { 'id': project_system_tmpl.get('id'), 'site': project_system_tmpl.get('site'), 'default': project_system_tmpl.get('default'), 'status': project_system_tmpl.get('status'), 'description': project_system_tmpl.get('description'), 'name': project_system_tmpl.get('name'), 'globalDefault': project_system_tmpl.get('globalDefault'), 'available': project_system_tmpl.get('available'), 'public': project_system_tmpl.get('public'), 'type': project_system_tmpl.get('type'), 'storage': { 'homeDir': project_system_tmpl.get('storage', {}).get('homeDir'), 'rootDir': project_system_tmpl.get('storage', {}).get('rootDir') } } }) ag.systems.add(body=project_system_tmpl) # grant initial permissions for creating user and PI, if exists metrics.info('projects', extra={ 'user': request.user.username, 'sessionId': getattr(request.session, 'session_key', ''), 'operation': 'initial_pems_create', 'info': { 'collab': request.user.username, 'pi': prj.pi } }) if getattr(prj, 'copi', None): prj.add_co_pis(prj.copi) elif getattr(prj, 'co_pis', None): prj.add_co_pis(prj.co_pis) if getattr(prj, 'team', None): prj.add_team_members(prj.team) elif getattr(prj, 'team_members', None): prj.add_team_members(prj.team_members) prj._add_team_members_pems([prj.pi]) if request.user.username not in list( set(prj.co_pis + prj.team_members + [prj.pi])): # Add creator to project as team member prj.add_team_members([request.user.username]) # Email collaborators tasks.email_collaborator_added_to_project.apply_async(args=[ prj.title, request.build_absolute_uri('{}/projects/{}/'.format( reverse('designsafe_data:data_depot'), prj.uuid)), [ u for u in list(set(prj.co_pis + prj.team_members + [prj.pi])) if u != request.user.username ], [] ]) tasks.set_facl_project.apply_async(args=[ prj.uuid, list(set(prj.co_pis + prj.team_members + [prj.pi])) ], queue='api') prj.add_admin('prjadmin') tasks.set_project_id.apply_async(args=[prj.uuid], queue="api") return JsonResponse(prj.to_body_dict(), safe=False)
def get_client(self): self.client = get_service_account_client() return self.client
def check_project_meta_pems(self, metadata_uuid): from designsafe.apps.data.models.agave.files import BaseFileMetadata logger.debug('Checking single metadata pems linked to a project %s', metadata_uuid) service = get_service_account_client() bfm = BaseFileMetadata.from_uuid(service, metadata_uuid) bfm.match_pems_to_project()
def post(self, request, project_id): """ Update a Project. Projects and the root File directory for a Project should be owned by the portal, with roles/permissions granted to the creating user. 1. Get the metadata record for the project 2. Get the metadata permissions for the project 3. Update the metadata record with changes from the post data and initilize the appropriate project class. 4. Use the metadata permissions and the metadata record to determine which users to add and/or remove from the project 5. Update file metadata permissions 6. Set ACLs on the project 7. Email users who have been added to the project :param request: :return: """ if request.is_ajax(): post_data = json.loads(request.body) else: post_data = request.POST.copy() client = request.user.agave_oauth.client sa_client = get_service_account_client( ) # service account for updating user permissions meta_obj = client.meta.getMetadata(uuid=project_id) meta_perms = client.meta.listMetadataPermissions(uuid=project_id) # update the meta_obj for key, value in post_data.items(): camel_key = to_camel_case(key) meta_obj.value[camel_key] = value # get users to add/remove admins = ['ds_admin', 'prjadmin'] users_with_access = [x['username'] for x in meta_perms] updated_users = list( set(meta_obj['value']['teamMembers'] + meta_obj['value']['coPis'] + [meta_obj['value']['pi']])) add_perm_usrs = [ u for u in updated_users + admins if u not in users_with_access ] rm_perm_usrs = [ u for u in users_with_access if u not in updated_users + admins ] prj_class = project_lookup_model(meta_obj) project = prj_class(value=meta_obj.value, uuid=project_id) project.manager().set_client(sa_client) try: ds_user = get_user_model().objects.get(username=project.pi) except: return HttpResponseBadRequest('Project update requires a valid PI') # remove permissions for users not on project and add permissions for new members if rm_perm_usrs: project._remove_team_members_pems(rm_perm_usrs) if add_perm_usrs: project._add_team_members_pems(add_perm_usrs) tasks.check_project_files_meta_pems.apply_async(args=[project.uuid], queue='api') tasks.set_facl_project.apply_async(args=[project.uuid, add_perm_usrs], queue='api') tasks.email_collaborator_added_to_project.apply_async(args=[ project.project_id, project.uuid, project.title, request.build_absolute_uri('{}/projects/{}/'.format( reverse('designsafe_data:data_depot'), project.uuid)), add_perm_usrs, [] ]) project.save(client) return JsonResponse(project.to_body_dict())