def check_project_files_meta_pems(self, project_uuid):
from designsafe.apps.api.agave.models.files import BaseFileMetadata
logger.debug('Checking metadata pems linked to a project')
service = get_service_account_client()
metas = BaseFileMetadata.search(service, {'associationIds': project_uuid,
'name': BaseFileMetadata.NAME})
for meta in metas:
meta.match_pems_to_project(project_uuid)
def delete(self, request, project_id):
if request.is_ajax():
post_data = json.loads(request.body)
else:
post_data = request.POST.copy()
ag = get_service_account_client()
project = Project.from_uuid(agave_client=ag, uuid=project_id)
project.remove_collaborator(post_data.get('username'))
tasks.check_project_files_meta_pems(project.uuid)
return JsonResponse({'status': 'ok'})
def get(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public':
if not request.user.is_authenticated():
return HttpResponseForbidden('Log in required')
# List permissions as the portal user rather than logged in user.
# This also works around the issue where pems on private systems are
# inconsistent.
fm = AgaveFileManager(agave_client=get_service_account_client())
pems = fm.list_permissions(system_id, file_path)
return JsonResponse(pems, encoder=AgaveJSONEncoder, safe=False)
return HttpResponseBadRequest("Unsupported operation")
def get(self, request, system_id=None):
params = request.GET.copy()
if request.user.is_authenticated():
ag = request.user.agave_oauth.client
if system_id is None:
systems = BaseSystemResource.list(ag, **params)
return JsonResponse(systems, encoder=AgaveJSONEncoder, safe=False)
else:
system = BaseSystemResource.from_id(ag, system_id)
return JsonResponse(system, encoder=AgaveJSONEncoder, safe=False)
else:
# Force public=true
params.pop('public', None)
ag = get_service_account_client()
systems = BaseSystemResource.list(ag, public=True, **params)
return JsonResponse(systems, encoder=AgaveJSONEncoder, safe=False)
def post(self, request, project_id):
"""
:param request:
:return:
"""
ag = get_service_account_client()
if request.is_ajax():
post_data = json.loads(request.body)
else:
post_data = request.POST.copy()
# save Project (metadata)
p = Project.from_uuid(ag, project_id)
p.title = post_data.get('title')
new_pi = post_data.get('pi', None)
if p.pi != new_pi:
p.pi = new_pi
p.add_collaborator(new_pi)
p.save()
return JsonResponse(p, encoder=AgaveJSONEncoder, safe=False)
def post(self, request):
"""
Create a new Project. Projects and the root File directory for a Project should
be owned by the portal, with roles/permissions granted to the creating user.
1. Create the metadata record for the project
2. Create a directory on the projects storage system named after the metadata uuid
3. Associate the metadata uuid and file uuid
:param request:
:return: The newly created project
:rtype: JsonResponse
"""
# portal service account needs to create the objects on behalf of the user
ag = get_service_account_client()
if request.is_ajax():
post_data = json.loads(request.body)
else:
post_data = request.POST.copy()
# create Project (metadata)
metrics.info('projects',
extra={'user' : request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': 'metadata_create',
'info': {'postData': post_data} })
p = Project(ag)
p.title = post_data.get('title')
p.pi = post_data.get('pi', None)
p.save()
# create Project Directory on Managed system
metrics.info('projects',
extra={'user' : request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': 'base_directory_create',
'info': {
'systemId': Project.STORAGE_SYSTEM_ID,
'uuid': p.uuid
}})
project_storage_root = BaseFileResource(ag, Project.STORAGE_SYSTEM_ID, '/')
project_storage_root.mkdir(p.uuid)
# Wrap Project Directory as private system for project
project_system_tmpl = template_project_storage_system(p)
metrics.info('projects',
extra={'user' : request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': 'private_system_create',
'info': {
'id': project_system_tmpl.get('id'),
'site': project_system_tmpl.get('site'),
'default': project_system_tmpl.get('default'),
'status': project_system_tmpl.get('status'),
'description': project_system_tmpl.get('description'),
'name': project_system_tmpl.get('name'),
'globalDefault': project_system_tmpl.get('globalDefault'),
'available': project_system_tmpl.get('available'),
'public': project_system_tmpl.get('public'),
'type': project_system_tmpl.get('type'),
'storage': {
'homeDir': project_system_tmpl.get('storage', {}).get('homeDir'),
'rootDir': project_system_tmpl.get('storage', {}).get('rootDir')
}
}})
ag.systems.add(body=project_system_tmpl)
# grant initial permissions for creating user and PI, if exists
project_system_tmpl = template_project_storage_system(p)
metrics.info('projects',
extra={'user' : request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': 'initial_pems_create',
'info': {'collab': request.user.username, 'pi': p.pi} })
p.add_collaborator(request.user.username)
if p.pi and p.pi != request.user.username:
p.add_collaborator(p.pi)
return JsonResponse(p, encoder=AgaveJSONEncoder, safe=False)
def check_published_files(project_id, revision=None, selected_files=None):
#get list of files that should be in the publication
es_client = new_es_client()
publication = BaseESPublication(project_id=project_id,
revision=revision,
using=es_client)
if selected_files:
#it's type other, use this for comparison
filepaths = selected_files
else:
filepaths = publication.related_file_paths()
#empty dirs
missing_files = []
existing_files = []
empty_folders = []
#strip leading forward slash from file paths
updated_filepaths = [
file_path.strip('/') for file_path in filepaths
if (file_path != '.Trash')
]
pub_directory = '/corral-repl/tacc/NHERI/published/{}'.format(project_id)
if revision:
pub_directory += 'v{}'.format(revision)
#navigate through publication files paths and
#compare to the previous list of files
for pub_file in updated_filepaths:
file_to_check = os.path.join(pub_directory, pub_file)
try:
if os.path.isfile(file_to_check):
existing_files.append(pub_file)
elif os.path.isdir(file_to_check):
#check directory for items in it
dir_list = os.listdir(file_to_check)
if dir_list != []:
existing_files.append(pub_file)
else:
empty_folders.append(pub_file)
else:
missing_files.append(pub_file)
except OSError as exc:
logger.info(exc)
#send email if there are files/folders missing/empty
if (missing_files or empty_folders):
#log for potential later queries
logger.info("check_published_files missing files: " + project_id +
" " + str(missing_files))
logger.info("check_published_files empty folders: " + project_id +
" " + str(empty_folders))
#send email to dev admins
service = get_service_account_client()
prj_admins = settings.DEV_PROJECT_ADMINS_EMAIL
for admin in prj_admins:
email_body = """
<p>Hello,</p>
<p>
The following project has been published with either missing files/folders or empty folders:
<br>
<b>{prjID} - revision {revision}</b>
<br>
Path to publication files: {pubFiles}
</p>
<p>
These are the missing files/folders for this publication:
<br>
{missingFiles}
</p>
<p>
These are the empty folders for this publication:
<br>
{emptyFolders}
</p>
This is a programmatically generated message. Do NOT reply to this message.
""".format(pubFiles=pub_directory,
prjID=project_id,
missingFiles=missing_files,
emptyFolders=empty_folders,
revision=revision)
send_mail(
"DesignSafe Alert: Published Project has missing files/folders",
email_body,
settings.DEFAULT_FROM_EMAIL, [admin],
html_message=email_body)
def copy_publication_files_to_corral(self, project_id):
from designsafe.apps.api.agave.filemanager.public_search_index import Publication
import shutil
publication = Publication(project_id=project_id)
filepaths = publication.related_file_paths()
if not len(filepaths):
res = get_service_account_client().files.list(
systemId='project-{project_uuid}'.format(
project_uuid=publication.project.uuid),
filePath='/')
filepaths = [
_file.path.strip('/') for _file in res
if (_file.name != '.' and _file.name != 'Trash')
]
filepaths = list(set(filepaths))
filepaths = sorted(filepaths)
base_path = ''.join(['/', publication.projectId])
os.chmod('/corral-repl/tacc/NHERI/published', 0755)
prefix_dest = '/corral-repl/tacc/NHERI/published/{}'.format(project_id)
if not os.path.isdir(prefix_dest):
os.mkdir(prefix_dest)
prefix_src = '/corral-repl/tacc/NHERI/projects/{}'.format(
publication.project['uuid'])
for filepath in filepaths:
local_src_path = '{}/{}'.format(prefix_src, filepath)
local_dst_path = '{}/{}'.format(prefix_dest, filepath)
logger.info('Trying to copy: %s to %s', local_src_path, local_dst_path)
if os.path.isdir(local_src_path):
try:
#os.mkdir(local_dst_path)
if not os.path.isdir(os.path.dirname(local_dst_path)):
os.makedirs(os.path.dirname(local_dst_path))
shutil.copytree(local_src_path, local_dst_path)
for root, dirs, files in os.walk(local_dst_path):
for d in dirs:
os.chmod(os.path.join(root, d), 0555)
for f in files:
os.chmod(os.path.join(root, f), 0444)
os.chmod(local_dst_path, 0555)
except OSError as exc:
logger.info(exc)
except IOError as exc:
logger.info(exc)
else:
try:
if not os.path.isdir(os.path.dirname(local_dst_path)):
os.makedirs(os.path.dirname(local_dst_path))
for root, dirs, files in os.walk(
os.path.dirname(local_dst_path)):
for d in dirs:
os.chmod(os.path.join(root, d), 0555)
for f in files:
os.chmod(os.path.join(root, f), 0444)
shutil.copy(local_src_path, local_dst_path)
os.chmod(local_dst_path, 0444)
except OSError as exc:
logger.info(exc)
except IOError as exc:
logger.info(exc)
os.chmod(prefix_dest, 0555)
os.chmod('/corral-repl/tacc/NHERI/published', 0555)
save_to_fedora.apply_async(args=[project_id])
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': 'designsafe.storage.published',
'filePath': '/' + project_id,
'recurse': True
},
queue='indexing')
def check_project_meta_pems(self, metadata_uuid):
from designsafe.apps.api.agave.models.files import BaseFileMetadata
logger.debug('Checking single metadata pems linked to a project %s', metadata_uuid)
service = get_service_account_client()
bfm = BaseFileMetadata.from_uuid(service, metadata_uuid)
bfm.match_pems_to_project()
def post(self, request):
"""
Create a new Project. Projects and the root File directory for a Project should
be owned by the portal, with roles/permissions granted to the creating user.
1. Create the metadata record for the project
2. Create a directory on the projects storage system named after the metadata uuid
3. Associate the metadata uuid and file uuid
:param request:
:return: The newly created project
:rtype: JsonResponse
"""
# portal service account needs to create the objects on behalf of the user
ag = get_service_account_client()
if request.is_ajax():
post_data = json.loads(request.body)
else:
post_data = request.POST.copy()
# create Project (metadata)
metrics.info('projects',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key',
''),
'operation': 'project_create',
'info': {
'postData': post_data
}
})
prj_model = project_lookup_model({
'name': 'designsafe.project',
'value': post_data
})
prj = prj_model(value=post_data)
project_uuid = prj.uuid
prj.manager().set_client(ag)
prj.save(ag)
# create Project Directory on Managed system
metrics.info('projects',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key',
''),
'operation': 'base_directory_create',
'info': {
'systemId': Project.STORAGE_SYSTEM_ID,
'uuid': prj.uuid
}
})
project_storage_root = BaseFileResource(ag, Project.STORAGE_SYSTEM_ID,
'/')
project_storage_root.mkdir(prj.uuid)
# Wrap Project Directory as private system for project
project_system_tmpl = template_project_storage_system(prj)
project_system_tmpl['storage']['rootDir'] = \
project_system_tmpl['storage']['rootDir'].format(project_uuid)
metrics.info('projects',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key',
''),
'operation': 'private_system_create',
'info': {
'id':
project_system_tmpl.get('id'),
'site':
project_system_tmpl.get('site'),
'default':
project_system_tmpl.get('default'),
'status':
project_system_tmpl.get('status'),
'description':
project_system_tmpl.get('description'),
'name':
project_system_tmpl.get('name'),
'globalDefault':
project_system_tmpl.get('globalDefault'),
'available':
project_system_tmpl.get('available'),
'public':
project_system_tmpl.get('public'),
'type':
project_system_tmpl.get('type'),
'storage': {
'homeDir':
project_system_tmpl.get('storage',
{}).get('homeDir'),
'rootDir':
project_system_tmpl.get('storage',
{}).get('rootDir')
}
}
})
ag.systems.add(body=project_system_tmpl)
# grant initial permissions for creating user and PI, if exists
metrics.info('projects',
extra={
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key',
''),
'operation': 'initial_pems_create',
'info': {
'collab': request.user.username,
'pi': prj.pi
}
})
if getattr(prj, 'copi', None):
prj.add_co_pis(prj.copi)
elif getattr(prj, 'co_pis', None):
prj.add_co_pis(prj.co_pis)
if getattr(prj, 'team', None):
prj.add_team_members(prj.team)
elif getattr(prj, 'team_members', None):
prj.add_team_members(prj.team_members)
prj._add_team_members_pems([prj.pi])
if request.user.username not in list(
set(prj.co_pis + prj.team_members + [prj.pi])):
# Add creator to project as team member
prj.add_team_members([request.user.username])
# Email collaborators
tasks.email_collaborator_added_to_project.apply_async(args=[
prj.title,
request.build_absolute_uri('{}/projects/{}/'.format(
reverse('designsafe_data:data_depot'), prj.uuid)),
[
u for u in list(set(prj.co_pis + prj.team_members + [prj.pi]))
if u != request.user.username
], []
])
tasks.set_facl_project.apply_async(args=[
prj.uuid,
list(set(prj.co_pis + prj.team_members + [prj.pi]))
],
queue='api')
prj.add_admin('prjadmin')
tasks.set_project_id.apply_async(args=[prj.uuid], queue="api")
return JsonResponse(prj.to_body_dict(), safe=False)
def get_client(self):
self.client = get_service_account_client()
return self.client
def check_project_meta_pems(self, metadata_uuid):
from designsafe.apps.data.models.agave.files import BaseFileMetadata
logger.debug('Checking single metadata pems linked to a project %s', metadata_uuid)
service = get_service_account_client()
bfm = BaseFileMetadata.from_uuid(service, metadata_uuid)
bfm.match_pems_to_project()
def post(self, request, project_id):
"""
Update a Project. Projects and the root File directory for a Project should
be owned by the portal, with roles/permissions granted to the creating user.
1. Get the metadata record for the project
2. Get the metadata permissions for the project
3. Update the metadata record with changes from the post data and initilize the
appropriate project class.
4. Use the metadata permissions and the metadata record to determine which users
to add and/or remove from the project
5. Update file metadata permissions
6. Set ACLs on the project
7. Email users who have been added to the project
:param request:
:return:
"""
if request.is_ajax():
post_data = json.loads(request.body)
else:
post_data = request.POST.copy()
client = request.user.agave_oauth.client
sa_client = get_service_account_client(
) # service account for updating user permissions
meta_obj = client.meta.getMetadata(uuid=project_id)
meta_perms = client.meta.listMetadataPermissions(uuid=project_id)
# update the meta_obj
for key, value in post_data.items():
camel_key = to_camel_case(key)
meta_obj.value[camel_key] = value
# get users to add/remove
admins = ['ds_admin', 'prjadmin']
users_with_access = [x['username'] for x in meta_perms]
updated_users = list(
set(meta_obj['value']['teamMembers'] + meta_obj['value']['coPis'] +
[meta_obj['value']['pi']]))
add_perm_usrs = [
u for u in updated_users + admins if u not in users_with_access
]
rm_perm_usrs = [
u for u in users_with_access if u not in updated_users + admins
]
prj_class = project_lookup_model(meta_obj)
project = prj_class(value=meta_obj.value, uuid=project_id)
project.manager().set_client(sa_client)
try:
ds_user = get_user_model().objects.get(username=project.pi)
except:
return HttpResponseBadRequest('Project update requires a valid PI')
# remove permissions for users not on project and add permissions for new members
if rm_perm_usrs:
project._remove_team_members_pems(rm_perm_usrs)
if add_perm_usrs:
project._add_team_members_pems(add_perm_usrs)
tasks.check_project_files_meta_pems.apply_async(args=[project.uuid],
queue='api')
tasks.set_facl_project.apply_async(args=[project.uuid, add_perm_usrs],
queue='api')
tasks.email_collaborator_added_to_project.apply_async(args=[
project.project_id, project.uuid, project.title,
request.build_absolute_uri('{}/projects/{}/'.format(
reverse('designsafe_data:data_depot'), project.uuid)),
add_perm_usrs, []
])
project.save(client)
return JsonResponse(project.to_body_dict())
