def set_context_props(self, request, **kwargs):
super(MetadataMixin, self).set_context_props(request, **kwargs)
mgr = FileManager(self.agave_client)
return mgr.get(system_id = self.filesystem,
path = self.file_path,
username = request.user.username,
is_public = self.is_public)
def post(self, request, *args, **kwargs):
self.set_context_props(request, **kwargs)
ufs = request.FILES
mgr = FileManager(self.agave_client)
mfs, fs = mgr.upload_files(ufs, system_id = self.filesystem, path = self.file_path)
return self.render_to_json_response({'files': [o.to_dict() for o in fs],
'filesMeta': [o.to_dict(get_id = True) for o in mfs]})
def index_permissions(system_id, path, username, bottom_up = False, levels = 0):
try:
user = get_user_model().objects.get(username = username)
ag = user.agave_oauth.client
mngr = FileManager(agave_client = ag)
mngr.index_permissions(system_id, path, username, bottom_up = bottom_up, levels = levels)
except ObjectDoesNotExist:
logger.exception('Unable to locate local user=%s' % username)
def share(system_id, path, username, permission, me):
logger.debug('Sharing file/folder: {}. To: {}. Owner: {}'.format(path, username, me))
path_shared = path
try:
user = get_user_model().objects.get(username=me)
ag = user.agave_oauth.client
mngr = FileManager(agave_client = ag)
rep = mngr.share(system_id = system_id, path = path,
username = username, permission = permission,
me = me)
meta_obj, ret = mngr.get(system_id, path, me, False)
if meta_obj.format != 'folder':
path_shared = meta_obj.path
logger.debug('Successfully updated permissions: {}'.format(rep))
DataEvent.send_generic_event({
'username_from': me,
'username_to': username,
'permission': permission,
'action': 'share_finished',
'path': path,
'action_link': { 'label': 'View Files', 'value': '/data/my/#/Shared with me/' + path_shared},
'html':[
{'Message': 'Your files have been shared.'},
{'Action': 'Sharing Finished'},
{'Shared with': username},
{'Permissions set': permission},
]
},
[me])
DataEvent.send_generic_event({
'username_from': me,
'username_to': username,
'permission': permission,
'action': 'share_finished',
'path': path,
'action_link': { 'label': 'View Files', 'value': '/data/my/#/Shared with me/' + path_shared},
'html':[
{'Message': '{} have shared some files with you.'.format(me)},
{'Action': 'Sharing Finished'},
{'Shared with': username},
{'Permissions set': permission},
],
'toast': {
'type': 'info',
'msg': me + ' has shared files with you.'
}
},
[username])
except ObjectDoesNotExist:
logger.exception('Unable to locate local user=%s' % username)
def check_or_create_agave_home_dir(username):
logger.info("Checking home directory for user=%s on default storage systemId=%s" % (
username, settings.AGAVE_STORAGE_SYSTEM))
try:
# TODO should use DS Files API for this
ag = Agave(api_server=settings.AGAVE_TENANT_BASEURL,
token=settings.AGAVE_SUPER_TOKEN)
body = {'action': 'mkdir', 'path': username}
ag.files.manage(systemId=settings.AGAVE_STORAGE_SYSTEM,
filePath='', body=body)
# add dir to index
fm = FileManager(agave_client=ag)
fm.index(settings.AGAVE_STORAGE_SYSTEM, username, username, levels=1)
except (HTTPError, AgaveException):
logger.exception('Failed to create home directory.',
extra={'user': username,
'systemId': settings.AGAVE_STORAGE_SYSTEM})
def get(self, request, *args, **kwargs):
self.set_context_props(request, **kwargs)
mgr = FileManager(agave_client = self.agave_client)
if self.file_path == request.user.username:
mgr.check_shared_folder(system_id = self.filesystem,
username = request.user.username)
if request.GET.get('refresh', None):
mgr.index(self.filesystem, self.file_path, request.user.username, levels = 1)
l = mgr.list_path(system_id = self.filesystem,
path = self.file_path,
username = request.user.username,
special_dir = self.special_dir,
is_public = self.is_public)
if isinstance(l, GeneratorType):
response = [o.to_dict(get_id = True) for o in l]
else:
tasks.index.delay(self.filesystem, self.file_path, request.user.username, pems = True, levels = 1)
response = [o.to_dict(pems = False) for o in l]
return self.render_to_json_response(response, status = 200)
def delete(self, request, *args, **kwargs):
self.set_context_props(request, **kwargs)
mf = FileManager(agave_client = self.agave_client)
o = mf.delete(self.filesystem, self.file_path, request.user.username)
return self.render_to_json_response(o.to_dict(get_id = True))
def copy_box_item(self, username, box_item_type, box_item_id, target_system_id,
target_path):
user = get_user_model().objects.get(username=username)
client = user.box_user_token.client
try:
op = getattr(client, box_item_type)
item = op(box_item_id).get()
except AttributeError as e:
logger.error('Invalid box_item_type')
self.update_state(state='FAILURE', meta={'exc': e})
return
agave_client = user.agave_oauth.client
fm = FileManager(agave_client=agave_client)
if box_item_type == 'file':
try:
import_url = item.get_shared_link_download_url()
safe_name = get_valid_filename(item.name)
import_resp = agave_client.files.importData(systemId=target_system_id,
filePath=target_path,
fileName=safe_name,
urlToIngest=import_url)
async_resp = AgaveAsyncResponse(agave_client, import_resp)
async_status = async_resp.result(600)
if async_status == 'FAILED':
logger.error('Box File Transfer failed: %s' % target_path)
else:
file_path = '%s/%s' % (target_path, item.name)
logger.info('Indexing Box File Transfer %s' % file_path)
fm.index(settings.AGAVE_STORAGE_SYSTEM, file_path, username, levels=1)
except BoxAPIException as e:
logger.error('Unable to get download link from Box')
self.update_state(state='FAILURE', meta={'exc': e})
except HTTPError as e:
logger.error('Agave.files.importData raised HTTPError')
self.update_state(state='FAILURE', meta={'exc': e})
except (TimeoutError, Error) as e:
logger.error('Agave.files.importData failed to complete')
self.update_state(state='FAILURE', meta={'exc': e})
else: # box_item_type == 'folder'
# create directory for the folder
try:
safe_name = get_valid_filename(item.name)
mf, f = fm.mkdir(target_path, safe_name, target_system_id, username,
raise_if_exists=False)
logger.info('Created directory "{0}"; scheduling transfer of contents',
f.full_path)
except HTTPError as e:
logger.error('Agave.files.manage(mkdir) failed')
self.update_state(state='FAILURE', meta={'exc': e})
# schedule to copy all of it's items
limit = 10
offset = 0
while True:
item_collection = item.get_items(limit=limit, offset=offset)
for it in item_collection:
args = (username, it.type, it.object_id, target_system_id,
f.full_path)
copy_box_item.apply_async(args=args, countdown=offset*2)
if len(item_collection) == limit:
offset += limit
else:
break
def call_api(request, service):
try:
agave = request.user.agave_oauth.client
if service == 'apps':
app_id = request.GET.get('app_id')
if app_id:
data = agave.apps.get(appId=app_id)
lic_type = _app_license_type(app_id)
data['license'] = {
'type': lic_type
}
if lic_type is not None:
lic = request.user.licenses.filter(license_type=lic_type).first()
data['license']['enabled'] = lic is not None
else:
public_only = request.GET.get('publicOnly')
if public_only == 'true':
data = agave.apps.list(publicOnly='true')
else:
data = agave.apps.list()
elif service == 'monitors':
target = request.GET.get('target')
ds_admin_client = Agave(api_server=getattr(settings, 'AGAVE_TENANT_BASEURL'), token=getattr(settings, 'AGAVE_SUPER_TOKEN'))
data = ds_admin_client.monitors.list(target=target)
elif service == 'meta':
app_id = request.GET.get('app_id')
if request.method == 'GET':
if app_id:
data = agave.meta.get(appId=app_id)
lic_type = _app_license_type(app_id)
data['license'] = {
'type': lic_type
}
if lic_type is not None:
lic = request.user.licenses.filter(license_type=lic_type).first()
data['license']['enabled'] = lic is not None
else:
query = request.GET.get('q')
data = agave.meta.listMetadata(q=query)
elif request.method == 'POST':
meta_post = json.loads(request.body)
meta_uuid = meta_post.get('uuid')
if meta_uuid:
del meta_post['uuid']
data = agave.meta.updateMetadata(uuid=meta_uuid, body=meta_post)
else:
data = agave.meta.addMetadata(body=meta_post)
elif request.method == 'DELETE':
meta_uuid = request.GET.get('uuid')
if meta_uuid:
data = agave.meta.deleteMetadata(uuid=meta_uuid)
elif service == 'files':
system_id = request.GET.get('system_id')
file_path = request.GET.get('file_path', '')
special_dir = None
if system_id == 'designsafe.storage.default':
if shared_with_me in file_path:
special_dir = shared_with_me
file_path = '/'.join(file_path.split('/')[2:])
elif file_path == '':
file_path = request.user.username
file_path = file_path.strip('/')
if file_path == '':
file_path = '/'
logger.debug('Listing "agave://%s/%s"...' % (system_id, file_path))
# Agave Files call
# data = agave.files.list(systemId=system_id, filePath=file_path)
# ElasticSearch call
try:
fm = FileManager(agave)
listing = fm.list_path(system_id=system_id,
path=file_path,
username=request.user.username,
special_dir=special_dir,
is_public=system_id == 'nees.public'
)
data = [f.to_dict() for f in listing]
# filter out empty projects
if system_id == 'nees.public' and file_path == '/':
data = [f for f in data if 'projecTitle' in f and
not f['projecTitle'].startswith('EMPTY PROJECT')]
# TODO type of "Shared with me" should be "dir" not "folder"
for d in data:
d.update((k, 'dir') for k, v in six.iteritems(d)
if k == 'type' and v == 'folder')
if special_dir:
data = [{
'name': '.',
'path': special_dir + file_path.strip('/'),
'systemId': system_id,
'type': 'dir',
}] + data
elif file_path != request.user.username:
data = [{
'name': '.',
'path': file_path,
'systemId': system_id,
'type': 'dir',
}] + data
except:
data = []
elif service == 'jobs':
if request.method == 'DELETE':
job_id = request.GET.get('job_id')
data = agave.jobs.delete(jobId=job_id)
elif request.method == 'POST':
job_post = json.loads(request.body)
job_id = job_post.get('job_id')
# cancel job / stop job
if job_id:
data = agave.jobs.manage(jobId=job_id, body='{"action":"stop"}')
# submit job
elif job_post:
# cleaning archive path value
if 'archivePath' in job_post:
parsed = urlparse(job_post['archivePath'])
if parsed.path.startswith('/'):
# strip leading '/'
archive_path = parsed.path[1:]
else:
archive_path = parsed.path
if not archive_path.startswith(request.user.username):
archive_path = '{}/{}'.format(
request.user.username, archive_path)
job_post['archivePath'] = archive_path
if parsed.netloc:
job_post['archiveSystem'] = parsed.netloc
else:
job_post['archivePath'] = \
'{}/archive/jobs/{}/${{JOB_NAME}}-${{JOB_ID}}'.format(
request.user.username,
datetime.now().strftime('%Y-%m-%d'))
# check for running licensed apps
lic_type = _app_license_type(job_post['appId'])
if lic_type is not None:
lic = request.user.licenses.filter(license_type=lic_type).first()
job_post['parameters']['_license'] = lic.license_as_str()
# url encode inputs
if job_post['inputs']:
for key, value in six.iteritems(job_post['inputs']):
parsed = urlparse(value)
if parsed.scheme:
job_post['inputs'][key] = '{}://{}{}'.format(
parsed.scheme, parsed.netloc, urllib.quote(parsed.path))
else:
job_post['inputs'][key] = urllib.quote(parsed.path)
try:
data = submit_job(request, request.user.username, job_post)
except JobSubmitError as e:
data = e.json()
logger.error('Failed to submit job {0}'.format(data))
return HttpResponse(json.dumps(data),
content_type='application/json',
status=e.status_code)
# list jobs (via POST?)
else:
limit = request.GET.get('limit', 10)
offset = request.GET.get('offset', 0)
data = agave.jobs.list(limit=limit, offset=offset)
elif request.method == 'GET':
job_id = request.GET.get('job_id')
# get specific job info
if job_id:
data = agave.jobs.get(jobId=job_id)
q = {"associationIds": job_id}
job_meta = agave.meta.listMetadata(q=json.dumps(q))
data['_embedded'] = {"metadata": job_meta}
archive_system_path = '{}/{}'.format(data['archiveSystem'],
data['archivePath'])
data['archiveUrl'] = reverse(
'designsafe_data:data_depot')
data['archiveUrl'] += 'agave/{}/'.format(archive_system_path)
# list jobs
else:
limit = request.GET.get('limit', 10)
offset = request.GET.get('offset', 0)
data = agave.jobs.list(limit=limit, offset=offset)
else:
return HttpResponse('Unexpected service: %s' % service, status=400)
else:
return HttpResponse('Unexpected service: %s' % service, status=400)
except HTTPError as e:
logger.error('Failed to execute {0} API call due to HTTPError={1}'.format(
service, e.message))
return HttpResponse(json.dumps(e.message),
content_type='application/json',
status=400)
except AgaveException as e:
logger.error('Failed to execute {0} API call due to AgaveException={1}'.format(
service, e.message))
return HttpResponse(json.dumps(e.message), content_type='application/json',
status=400)
except Exception as e:
logger.error('Failed to execute {0} API call due to Exception={1}'.format(
service, e.message))
return HttpResponse(
json.dumps({'status': 'error', 'message': '{}'.format(e.message)}),
content_type='application/json', status=400)
return HttpResponse(json.dumps(data, cls=DjangoJSONEncoder),
content_type='application/json')
