def get(self, request, file_mgr_name, system_id=None, file_path=None):
if file_mgr_name == AgaveFileManager.NAME:
if not request.user.is_authenticated:
return HttpResponseForbidden('Login required')
fm = AgaveFileManager(agave_client=request.user.agave_oauth.client)
if system_id is None:
system_id = AgaveFileManager.DEFAULT_SYSTEM_ID
if file_path is None:
file_path = request.user.username
if system_id == AgaveFileManager.DEFAULT_SYSTEM_ID and \
(file_path.strip('/') == '$SHARE' or
file_path.strip('/').split('/')[0] != request.user.username):
listing = ElasticFileManager.listing(
system=system_id,
file_path=file_path,
user_context=request.user.username)
return JsonResponse(listing)
else:
offset = int(request.GET.get('offset', 0))
limit = int(request.GET.get('limit', 100))
listing = fm.listing(system=system_id,
file_path=file_path,
offset=offset,
limit=limit)
return JsonResponse(listing,
encoder=AgaveJSONEncoder,
safe=False)
return HttpResponseBadRequest()
def get(self, request, file_mgr_name, system_id, file_path):
fm_cls = FileLookupManager(file_mgr_name)
fm = fm_cls(None)
if fm.requires_auth and not request.user.is_authenticated:
return HttpResponseForbidden('Login Required.')
if file_mgr_name == 'agave':
all = request.GET.get("all")
client = request.user.agave_oauth.client
fmgr = AgaveFileManager(agave_client=client)
file_obj = fmgr.listing(system_id, file_path)
if all:
file_uuid = file_obj.uuid
query = {"associationIds": file_uuid}
objs = client.meta.listMetadata(q=json.dumps(query))
return JsonResponse([o.value for o in objs], safe=False)
file_dict = file_obj.to_dict()
file_dict['keywords'] = file_obj.metadata.value['keywords']
return JsonResponse(file_dict)
elif file_mgr_name in ['public', 'community', 'published']:
pems = [{'username': '******',
'permission': {'read': True,
'write': False,
'execute': False}}]
if request.user.is_authenticated:
pems.append({'username': request.user.username,
'permission': {'read': True,
'write': False,
'execute': False}})
return JsonResponse(pems, safe=False)
return HttpResponseBadRequest('Unsupported file manager.')
def listing(self, system, file_path, offset=0, limit=100, status='published'):
file_path = file_path or '/'
logger.debug('file_path: %s', file_path)
if file_path == '/':
# listing = PublicObject.listing(system, file_path,
# offset=offset, limit=limit)
publications = Publication.listing(status)
legacy_publications = LegacyPublication.listing(offset, limit)
# show new publications on top; don't re-display when scrolling down
if offset == 0:
listing_iterator = itertools.chain(publications, legacy_publications)
else:
listing_iterator = legacy_publications
listing = PublicDocumentListing(listing_iterator, system, file_path)
else:
fmgr = AgaveFileManager(self._ag)
listing = fmgr.listing(system, file_path, offset, limit, status=status)
""""
try:
#_list = PublicObject.listing(
# system, file_path, offset=offset, limit=limit
#)
_list = None
#logger.debug(_list._doc.to_dict())
#listing._wrapped['metadata'] = _list.metadata()
#listing.trail = _list.trail()
#logger.debug(listing._wrapped['metadata']['project'])
except TransportError:
pass
"""
return listing
def post(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public':
if not request.user.is_authenticated():
return HttpResponseForbidden('Log in required')
agave_client = request.user.agave_oauth.client
fm = AgaveFileManager(agave_client=agave_client)
if request.FILES:
upload_file = request.FILES['file']
upload_dir = file_path
relative_path = request.POST.get('relative_path', None)
if relative_path:
# user uploaded a folder structure; ensure path exists
upload_dir = os.path.join(file_path, os.path.dirname(relative_path))
BaseFileResource.ensure_path(agave_client, system_id, upload_dir)
try:
result = fm.upload(system_id, upload_dir, upload_file)
except HTTPError as e:
logger.error(e.response.text)
return HttpResponseBadRequest(e.response.text)
return JsonResponse({'status': 'ok'})
return HttpResponseBadRequest("Unsupported operation")
def put(self, request, file_mgr_name, system_id, file_path):
post_body = json.loads(request.body)
metadata = post_body.get('metadata', {})
if file_mgr_name == ElasticFileManager.NAME or not metadata:
if not request.user.is_authenticated:
return HttpResponseForbidden('Login required')
fmgr = AgaveFileManager(
agave_client=request.user.agave_oauth.client)
try:
file_obj = fmgr.listing(system_id, file_path)
file_obj.metadata.update(metadata)
file_dict = file_obj.to_dict()
file_dict['keyword'] = file_obj.metadata.value['keywords']
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session, 'session_key', ''),
'operation':
'data_depot_metadata_update',
'info': {
'systemId': system_id,
'filePath': file_path,
'metadata': metadata
}
})
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.OPERATION: 'data_depot_metadata_update',
Notification.STATUS: Notification.SUCCESS,
Notification.USER: request.user.username,
Notification.MESSAGE: 'Metadata was updated successfully.',
Notification.EXTRA: {
'systemId': system_id,
'filePath': file_path,
'metadata': metadata
}
}
Notification.objects.create(**event_data)
except HTTPError as err:
logger.debug(err.response.text)
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.STATUS: Notification.ERROR,
Notification.OPERATION: 'data_depot_metadata_update',
Notification.USER: request.user.username,
Notification.MESSAGE: 'Metadata was updated successfully.',
Notification.EXTRA: {
'systemId': system_id,
'filePath': file_path,
'metadata': metadata
}
}
Notification.objects.create(**event_data)
return JsonResponse(file_dict)
return HttpResponseBadRequest('Unsupported file manager.')
def get(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == ElasticFileManager.NAME:
if not request.user.is_authenticated():
return HttpResponseForbidden('Log in required')
fmgr = AgaveFileManager(agave_client=request.user.agave_oauth.client)
file_obj = fmgr.listing(system_id, file_path)
file_dict = file_obj.to_dict()
file_dict['keywords'] = file_obj.metadata.value['keywords']
return JsonResponse(file_dict)
return HttpResponseBadRequest('Unsupported file manager.')
def post(self, request, file_mgr_name, system_id, file_path):
if request.is_ajax():
body = json.loads(request.body)
else:
body = request.POST.copy()
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public':
if not request.user.is_authenticated:
return HttpResponseForbidden('Login required')
fm = AgaveFileManager(agave_client=request.user.agave_oauth.client)
username = body.get('username')
permission = body.get('permission')
try:
pem = fm.share(system_id, file_path, username, permission)
metrics.info('Data Depot',
extra = {
'user': request.user.username,
'sessionId': getattr(request.session, 'session_key', ''),
'operation': 'data_depot_share',
'info': {
'systemId': system_id,
'filePath': file_path}
})
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.OPERATION: 'data_depot_share',
Notification.STATUS: Notification.SUCCESS,
Notification.USER: request.user.username,
Notification.MESSAGE: '{} permissions were granted to {}.'.format(permission, username),
Notification.EXTRA: {'system': system_id,
'file_path': file_path,
'username': username,
'permission': permission}
}
Notification.objects.create(**event_data)
except HTTPError as err:
logger.debug(err.response.text)
event_data = {
Notification.EVENT_TYPE: 'data_depot_share',
Notification.STATUS: Notification.ERROR,
Notification.OPERATION: 'data_depot_share',
Notification.USER: request.user.username,
Notification.MESSAGE: 'There was an error updating permissions for a file/folder.',
Notification.EXTRA: {'message': err.response.text}
}
Notification.objects.create(**event_data)
return HttpResponseBadRequest(err.response.text)
return JsonResponse(pem, encoder=AgaveJSONEncoder, safe=False)
return HttpResponseBadRequest("Unsupported operation")
def get(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == ElasticFileManager.NAME:
if not request.user.is_authenticated:
return HttpResponseForbidden('Login required')
fmgr = AgaveFileManager(
agave_client=request.user.agave_oauth.client)
file_obj = fmgr.listing(system_id, file_path)
file_dict = file_obj.to_dict()
file_dict['keywords'] = file_obj.metadata.value['keywords']
return JsonResponse(file_dict)
return HttpResponseBadRequest('Unsupported file manager.')
def delete(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == AgaveFileManager.NAME:
if not request.user.is_authenticated():
return HttpResponseForbidden('Log in required')
fm = AgaveFileManager(agave_client=request.user.agave_oauth.client)
try:
fm.delete(system_id, file_path)
return JsonResponse({'status': 'ok'})
except HTTPError as e:
logger.exception(e.response.text)
return HttpResponseBadRequest(e.response.text)
return HttpResponseBadRequest("Unsupported operation")
def delete(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == AgaveFileManager.NAME:
if not request.user.is_authenticated:
return HttpResponseForbidden('Login required')
fm = AgaveFileManager(agave_client=request.user.agave_oauth.client)
try:
fm.delete(system_id, file_path)
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session, 'session_key', ''),
'operation':
'agave_file_delete',
'info': {
'systemId': system_id,
'filePath': file_path
}
})
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.OPERATION: 'data_depot_delete',
Notification.STATUS: Notification.SUCCESS,
Notification.USER: request.user.username,
Notification.MESSAGE: 'File/folder was deleted.',
Notification.EXTRA: {
'message': 'ok'
}
}
Notification.objects.create(**event_data)
return JsonResponse({'status': 'ok'})
except HTTPError as e:
logger.exception(e.response.text)
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.OPERATION: 'data_depot_delete',
Notification.STATUS: Notification.ERROR,
Notification.USER: request.user.username,
Notification.MESSAGE:
'There was an error deleting a file/folder.',
Notification.EXTRA: {
'message': 'ok'
}
}
Notification.objects.create(**event_data)
return HttpResponseBadRequest(e.response.text)
return HttpResponseBadRequest("Unsupported operation")
def get(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public':
if not request.user.is_authenticated:
return HttpResponseForbidden('Login required')
# List permissions as the portal user rather than logged in user.
# This also works around the issue where pems on private systems are
# inconsistent.
fm = AgaveFileManager(agave_client=get_service_account_client())
pems = fm.list_permissions(system_id, file_path)
return JsonResponse(pems, encoder=AgaveJSONEncoder, safe=False)
return HttpResponseBadRequest("Unsupported operation.")
def get(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public':
if not request.user.is_authenticated():
return HttpResponseForbidden('Log in required')
# List permissions as the portal user rather than logged in user.
# This also works around the issue where pems on private systems are
# inconsistent.
fm = AgaveFileManager(agave_client=get_service_account_client())
pems = fm.list_permissions(system_id, file_path)
return JsonResponse(pems, encoder=AgaveJSONEncoder, safe=False)
return HttpResponseBadRequest("Unsupported operation")
def get(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == ElasticFileManager.NAME:
all = request.GET.get("all")
client = request.user.agave_oauth.client
fmgr = AgaveFileManager(agave_client=client)
file_obj = fmgr.listing(system_id, file_path)
if all:
file_uuid = file_obj.uuid
query = {"associationIds": file_uuid}
objs = client.meta.listMetadata(q=json.dumps(query))
return JsonResponse([o.value for o in objs], safe=False)
file_dict = file_obj.to_dict()
file_dict['keywords'] = file_obj.metadata.value['keywords']
return JsonResponse(file_dict)
return HttpResponseBadRequest('Unsupported file manager.')
def put(self, request, file_mgr_name, system_id, file_path):
post_body = json.loads(request.body)
metadata = post_body.get('metadata', {})
logger.debug('metadata: %s' % (metadata, ))
if file_mgr_name == ElasticFileManager.NAME or not metadata:
if not request.user.is_authenticated():
return HttpResponseForbidden('Log in required')
fmgr = AgaveFileManager(agave_client=request.user.agave_oauth.client)
file_obj = fmgr.listing(system_id, file_path)
file_obj.metadata.update(metadata)
file_dict = file_obj.to_dict()
file_dict['keyword'] = file_obj.metadata.value['keywords']
return JsonResponse(file_dict)
return HttpResponseBadRequest('Unsupported file manager.')
def get(self, request, file_mgr_name, system_id=None, file_path=None):
if file_mgr_name == AgaveFileManager.NAME:
if not request.user.is_authenticated:
return HttpResponseForbidden('Login required')
fm = AgaveFileManager(agave_client=request.user.agave_oauth.client)
if system_id is None:
system_id = AgaveFileManager.DEFAULT_SYSTEM_ID
if file_path is None:
file_path = request.user.username
if system_id == AgaveFileManager.DEFAULT_SYSTEM_ID and \
(file_path.strip('/') == '$SHARE' or
file_path.strip('/').split('/')[0] != request.user.username):
listing = ElasticFileManager.listing(
system=system_id,
file_path=file_path,
user_context=request.user.username)
return JsonResponse(listing)
else:
query_string = request.GET.get('query_string')
offset = int(request.GET.get('offset', 0))
limit = int(request.GET.get('limit', 100))
if (not query_string) or (query_string == ""):
listing = fm.listing(system=system_id,
file_path=file_path,
offset=offset,
limit=limit)
else:
query_string = request.GET.get('query_string')
# Performing an Agave listing here prevents a race condition.
listing = fm.listing(system=system_id,
file_path='/',
offset=offset,
limit=limit)
efmgr = ElasticFileManager()
listing = efmgr.search_in_project(system_id,
query_string,
offset=offset,
limit=limit)
return JsonResponse(listing,
encoder=AgaveJSONEncoder,
safe=False)
return HttpResponseBadRequest()
def post(self, request, file_mgr_name, system_id, file_path):
if request.is_ajax():
body = json.loads(request.body)
else:
body = request.POST.copy()
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public':
if not request.user.is_authenticated():
return HttpResponseForbidden('Log in required')
fm = AgaveFileManager(agave_client=request.user.agave_oauth.client)
username = body.get('username')
permission = body.get('permission')
pem = fm.share(system_id, file_path, username, permission)
return JsonResponse(pem, encoder=AgaveJSONEncoder, safe=False)
return HttpResponseBadRequest("Unsupported operation")
def listing(self, system, file_path, offset=0, limit=100, status='published'):
file_path = file_path or '/'
logger.debug('file_path: %s', file_path)
if file_path == '/':
listing = PublicObject.listing(system, file_path,
offset=offset, limit=limit)
publications = Publication.listing(status)
if file_path == '/':
listing.children = itertools.chain(publications, listing.children)
else:
fmgr = AgaveFileManager(self._ag)
listing = fmgr.listing(system, file_path, offset, limit, status=status)
try:
_list = PublicObject.listing(
system, file_path, offset=offset, limit=limit
)
listing._wrapped['metadata'] = _list.metadata()
listing.trail = _list.trail()
except TransportError:
pass
return listing
def get(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public':
if not request.user.is_authenticated():
return HttpResponseForbidden('Log in required')
fm = AgaveFileManager(agave_client=request.user.agave_oauth.client)
f = fm.listing(system_id, file_path)
if request.GET.get('preview', False):
context = {
'file': f
}
if f.ext in BaseFileResource.SUPPORTED_IMAGE_PREVIEW_EXTS:
context['image_preview'] = f.download_postit(force=False, lifetime=360)
elif f.ext in BaseFileResource.SUPPORTED_TEXT_PREVIEW_EXTS:
content = f.download()
try:
encoded = content.encode('utf-8')
except UnicodeError:
try:
encoding = chardet.detect(content)['encoding']
encoded = content.decode(encoding).encode('utf-8')
except UnicodeError:
logger.exception('Failed to preview file',
extra={'file_mgr_name': file_mgr_name,
'system_id': system_id,
'file_path': file_path})
encoded = u'Sorry! We were unable to preview this file due ' \
u'to a unrecognized content encoding. Please ' \
u'download the file to view its contents.'
context['text_preview'] = encoded
elif f.ext in BaseFileResource.SUPPORTED_OBJECT_PREVIEW_EXTS:
context['object_preview'] = f.download_postit(force=False, lifetime=360)
return render(request, 'designsafe/apps/api/agave/preview.html', context)
else:
return HttpResponseRedirect(fm.download(system_id, file_path))
return HttpResponseBadRequest("Unsupported operation")
def get(self, request, file_mgr_name, system_id=None, file_path=None):
if file_mgr_name == AgaveFileManager.NAME:
if not request.user.is_authenticated():
return HttpResponseForbidden('Log in required')
fm = AgaveFileManager(agave_client=request.user.agave_oauth.client)
if system_id is None:
system_id = AgaveFileManager.DEFAULT_SYSTEM_ID
if file_path is None:
file_path = request.user.username
if system_id == AgaveFileManager.DEFAULT_SYSTEM_ID and \
(file_path.strip('/') == '$SHARE' or
file_path.strip('/').split('/')[0] != request.user.username):
listing = ElasticFileManager.listing(system=system_id,
file_path=file_path,
user_context=request.user.username)
return JsonResponse(listing)
else:
try:
offset = int(request.GET.get('offset', 0))
limit = int(request.GET.get('limit', 100))
listing = fm.listing(system=system_id, file_path=file_path,
offset=offset, limit=limit)
return JsonResponse(listing, encoder=AgaveJSONEncoder, safe=False)
except HTTPError as e:
logger.exception('Unable to list files')
if e.response.status_code == 403:
return HttpResponseForbidden(e.response.text)
elif e.response.status_code >= 500:
return HttpResponseServerError(e.response.text)
elif e.response.status_code >= 400:
return HttpResponseBadRequest(e.response.text)
return HttpResponseBadRequest()
def put(self, request, file_mgr_name, system_id, file_path):
if request.is_ajax():
body = json.loads(request.body)
else:
body = request.POST.copy()
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public' \
or file_mgr_name == 'community' \
or file_mgr_name == 'published':
if not request.user.is_authenticated:
if file_mgr_name in ['public', 'community', 'published']:
ag = get_user_model().objects.get(
username='******').agave_oauth.client
else:
return HttpResponseForbidden('Login required')
else:
ag = request.user.agave_oauth.client
fm = AgaveFileManager(agave_client=ag)
action = body.get('action', '')
if action == 'copy':
try:
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.OPERATION: 'data_depot_copy',
Notification.STATUS: Notification.SUCCESS,
Notification.USER: request.user.username,
Notification.MESSAGE: 'Data was copied.',
}
if body.get('system') is None:
external = body.get('resource')
if external not in ['box', 'dropbox', 'googledrive']:
return HttpResponseBadRequest(
"External resource not available.")
if external == 'googledrive':
dest_file_id = body.get('id')
else:
dest_file_id = body.get('path')
external_resource_upload.apply_async(kwargs={
'username':
request.user.username,
'dest_resource':
external,
'src_file_id':
os.path.join(system_id, file_path.strip('/')),
'dest_file_id':
dest_file_id
},
queue='files')
event_data[
Notification.
MESSAGE] = 'Data copy was scheduled. This may take a few minutes.'
event_data[Notification.EXTRA] = {
'resource':
external,
'dest_file_id':
body.get('path'),
'src_file_id':
os.path.join(system_id, file_path.strip('/'))
}
elif body.get('system') != system_id:
if body.get('ipynb'):
path = '/{}'.format(request.user.username)
else:
path = body.get('path')
copied = fm.import_data(body.get('system'), path,
system_id, file_path)
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session,
'session_key', ''),
'operation':
'data_depot_copy',
'info': {
'destSystemId':
body.get('system'),
'destFilePath': path,
'fromSystemId': system_id,
'fromFilePath': file_path
}
})
event_data[Notification.EXTRA] = copied.to_dict()
else:
copied = fm.copy(system_id, file_path,
body.get('path'), body.get('name'))
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session,
'session_key', ''),
'operation':
'data_depot_copy',
'info': {
'destSystemId': system_id,
'destFilePath': file_path,
'fromSystemId':
body.get('system'),
'fromFilePath': body.get('path')
}
})
event_data[Notification.EXTRA] = copied.to_dict()
notification = Notification.objects.create(**event_data)
notification.save()
return JsonResponse(copied,
encoder=AgaveJSONEncoder,
safe=False)
except HTTPError as e:
logger.exception(e.response.text)
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.OPERATION: 'data_depot_copy',
Notification.STATUS: Notification.ERROR,
Notification.USER: request.user.username,
Notification.MESSAGE:
'There was an error copying data.',
Notification.EXTRA: {
'message': e.response.text
}
}
Notification.objects.create(**event_data)
return HttpResponseBadRequest(e.response.text)
elif action == 'download':
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session, 'session_key', ''),
'operation':
'agave_file_download',
'info': {
'systemId': system_id,
'filePath': file_path
}
})
return JsonResponse(
{'href': fm.download(system_id, file_path)})
elif action == 'mkdir':
try:
dir_name = body.get('name')
new_dir = fm.mkdir(system_id, file_path, dir_name)
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session, 'session_key',
''),
'operation':
'agave_file_mkdir',
'info': {
'systemId': system_id,
'filePath': file_path,
'dirName': dir_name
}
})
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.OPERATION: 'data_depot_mkdir',
Notification.STATUS: Notification.SUCCESS,
Notification.USER: request.user.username,
Notification.MESSAGE: 'Directory created.',
Notification.EXTRA: new_dir.to_dict()
}
Notification.objects.create(**event_data)
return JsonResponse(new_dir,
encoder=AgaveJSONEncoder,
safe=False)
except HTTPError as e:
logger.exception(e.response.text)
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.OPERATION: 'data_depot_mkdir',
Notification.STATUS: Notification.ERROR,
Notification.USER: request.user.username,
Notification.MESSAGE: 'Error creating directory.',
Notification.EXTRA: {
'message': e.response.text
}
}
Notification.objects.create(**event_data)
return HttpResponseBadRequest(e.response.text)
elif action == 'move':
try:
if body.get('system') != system_id:
moved = fm.import_data(body.get('system'),
body.get('path'), system_id,
file_path)
fm.delete(system_id, file_path)
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session,
'session_key', ''),
'operation':
'agave_file_copy_move',
'info': {
'destSystemId':
body.get('system'),
'destFilePath': body.get('path'),
'fromSystemId': system_id,
'fromFilePath': file_path
}
})
else:
moved = fm.move(system_id, file_path, body.get('path'),
body.get('name'))
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session,
'session_key', ''),
'operation':
'agave_file_copy',
'info': {
'destSystemId': system_id,
'destFilePath': file_path,
'fromSystemId':
body.get('system'),
'fromFilePath': body.get('path')
}
})
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.STATUS: Notification.SUCCESS,
Notification.OPERATION: 'data_depot_move',
Notification.USER: request.user.username,
Notification.MESSAGE: 'Data has been moved.',
Notification.EXTRA: moved.to_dict()
}
Notification.objects.create(**event_data)
return JsonResponse(moved,
encoder=AgaveJSONEncoder,
safe=False)
except HTTPError as e:
logger.exception(e.response.text)
event_data = {
Notification.EVENT_TYPE: 'data_depot_move',
Notification.STATUS: Notification.ERROR,
Notification.OPERATION: 'data_depot_move',
Notification.USER: request.user.username,
Notification.MESSAGE:
'There was an error moving your data.',
Notification.EXTRA: {
'message': e.response.text
}
}
Notification.objects.create(**event_data)
return HttpResponseBadRequest(e.response.text)
elif action == 'preview':
try:
file_listing = fm.listing(system_id, file_path)
if file_listing.previewable:
preview_url = reverse(
'designsafe_api:files_media',
args=[file_mgr_name, system_id, file_path])
return JsonResponse({
'href':
'{}?preview=true'.format(preview_url),
'postit':
file_listing.download_postit(force=False,
lifetime=360)
})
else:
return HttpResponseBadRequest(
'Preview not available for this item.')
except HTTPError as e:
logger.exception(
'Unable to preview file: {file_path}'.format(
file_path=file))
return HttpResponseBadRequest(e.response.text)
elif action == 'rename':
try:
renamed = fm.rename(system_id, file_path, body.get('name'))
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session, 'session_key',
''),
'operation':
'agave_file_rename',
'info': {
'systemId': system_id,
'filePath': file_path,
'name': body.get('name')
}
})
event_data = {
Notification.EVENT_TYPE: 'data_depot_rename',
Notification.STATUS: Notification.SUCCESS,
Notification.USER: request.user.username,
Notification.MESSAGE: 'File/folder was renamed.',
Notification.EXTRA: renamed.to_dict()
}
Notification.objects.create(**event_data)
return JsonResponse(renamed,
encoder=AgaveJSONEncoder,
safe=False)
except HTTPError as e:
logger.exception(e.response.text)
event_data = {
Notification.EVENT_TYPE: 'data_depot_rename',
Notification.STATUS: Notification.ERROR,
Notification.USER: request.user.username,
Notification.MESSAGE:
'There was an error renaming a file/folder.',
Notification.EXTRA: {
'message': e.response.text
}
}
Notification.objects.create(**event_data)
return HttpResponseBadRequest(e.response.text)
elif action == 'trash':
trash_path = '/Trash'
if system_id == AgaveFileManager.DEFAULT_SYSTEM_ID:
trash_path = request.user.username + '/.Trash'
try:
trashed = fm.trash(system_id, file_path, trash_path)
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session, 'session_key',
''),
'operation':
'agave_file_trash',
'info': {
'systemId': system_id,
'filePath': file_path,
'trashPath': trash_path
}
})
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.STATUS: Notification.SUCCESS,
Notification.OPERATION: 'data_depot_trash',
Notification.USER: request.user.username,
Notification.MESSAGE:
'File/folder was moved to trash.',
Notification.EXTRA: trashed.to_dict()
}
Notification.objects.create(**event_data)
return JsonResponse(trashed,
encoder=AgaveJSONEncoder,
safe=False)
except HTTPError as e:
logger.error(e.response.text)
event_data = {
Notification.EVENT_TYPE: 'data_depot_trash',
Notification.STATUS: Notification.ERROR,
Notification.OPERATION: 'data_depot_trash',
Notification.USER: request.user.username,
Notification.MESSAGE:
'There was an error moving file/folder to trash.',
Notification.EXTRA: {
'message': e.response.text
}
}
Notification.objects.create(**event_data)
return HttpResponseBadRequest(e.response.text)
return HttpResponseBadRequest("Unsupported operation")
def get(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public' \
or file_mgr_name == 'community' \
or file_mgr_name == 'published':
if not request.user.is_authenticated:
if file_mgr_name in ['public', 'community', 'published']:
ag = get_user_model().objects.get(
username='******').agave_oauth.client
else:
return HttpResponseForbidden('Login required')
else:
ag = request.user.agave_oauth.client
fm = AgaveFileManager(agave_client=ag)
f = fm.listing(system_id, file_path)
if request.GET.get('preview', False):
context = {'file': f}
if f.ext in BaseFileResource.SUPPORTED_IMAGE_PREVIEW_EXTS:
context['image_preview'] = f.download_postit(force=False,
lifetime=360)
elif f.ext in BaseFileResource.SUPPORTED_TEXT_PREVIEW_EXTS:
content = f.download()
try:
encoded = content.encode('utf-8')
except UnicodeError:
try:
encoding = chardet.detect(content)['encoding']
encoded = content.decode(encoding).encode('utf-8')
except UnicodeError:
logger.exception('Failed to preview file',
extra={
'file_mgr_name':
file_mgr_name,
'system_id': system_id,
'file_path': file_path
})
encoded = u'Sorry! We were unable to preview this file due ' \
u'to an unrecognized content encoding. Please ' \
u'download the file to view its contents.'
context['text_preview'] = encoded
elif f.ext in BaseFileResource.SUPPORTED_OBJECT_PREVIEW_EXTS:
context['object_preview'] = f.download_postit(force=False,
lifetime=360)
elif f.ext in BaseFileResource.SUPPORTED_MS_OFFICE:
context['iframe_preview'] = 'https://view.officeapps.live.com/op/view.aspx?src={}'\
.format(f.download_postit(force=False, lifetime=360))
elif f.ext in BaseFileResource.SUPPORTED_VIDEO_EXTS:
context['video_preview'] = f.download_postit(force=False,
lifetime=360)
context[
'mimetype'] = BaseFileResource.SUPPORTED_VIDEO_MIMETYPES[
f.ext]
return render(request,
'designsafe/apps/api/agave/preview.html',
context)
else:
url = 'https://agave.designsafe-ci.org/files/v2/media/{system}/{path}'.format(
system=system_id, path=file_path)
# return HttpResponseRedirect(fm.download(system_id, file_path))
resp = HttpResponseRedirect(url)
resp['X-Authorization'] = 'Bearer {token}'.format(
token=request.user.agave_oauth.access_token)
logger.info(resp)
return resp
return HttpResponseBadRequest("Unsupported operation")
def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download file %s from box.' %
(src_file_id, ),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, src_file_id: {}, dest_file_id: {}'.format(
username, src_file_id, dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
# box_fm = BoxFileManager(user)
box_file_type, box_file_id = self.parse_file_id(
file_id=src_file_id)
levels = 0
downloaded_file_path = None
if box_file_type == 'file':
downloaded_file_path = self.download_file(
box_file_id, dest_real_path)
levels = 1
elif box_file_type == 'folder':
downloaded_file_path = self.download_folder(
box_file_id, dest_real_path)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s has been copied from box successfully!' %
(src_file_id, ),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
if not agave_file_path.startswith('/'):
agave_file_path = '/' + agave_file_path
agave_indexer.apply_async(kwargs={
'username':
user.username,
'systemId':
agave_system_id,
'filePath':
os.path.dirname(agave_file_path),
'recurse':
False
},
queue='indexing')
agave_indexer.apply_async(kwargs={
'systemId': agave_system_id,
'filePath': agave_file_path,
'recurse': True
},
routing_key='indexing')
except:
logger.exception('Unexpected task failure: box_download',
extra={
'username': username,
'box_file_id': src_file_id,
'dest_file_id': dest_file_id
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={})
n.save()
raise
def put(self, request, file_mgr_name, system_id, file_path):
if request.is_ajax():
body = json.loads(request.body)
else:
body = request.POST.copy()
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public':
if not request.user.is_authenticated():
return HttpResponseForbidden('Log in required')
fm = AgaveFileManager(agave_client=request.user.agave_oauth.client)
action = body.get('action', '')
if action == 'copy':
try:
if body.get('system') != system_id:
copied = fm.import_data(body.get('system'), body.get('path'),
system_id, file_path)
else:
copied = fm.copy(system_id, file_path, body.get('path'), body.get('name'))
return JsonResponse(copied, encoder=AgaveJSONEncoder, safe=False)
except HTTPError as e:
logger.exception(e.response.text)
return HttpResponseBadRequest(e.response.text)
elif action == 'download':
return JsonResponse({
'href': fm.download(system_id, file_path)
})
elif action == 'mkdir':
try:
new_dir = fm.mkdir(system_id, file_path, body.get('name'))
return JsonResponse(new_dir, encoder=AgaveJSONEncoder, safe=False)
except HTTPError as e:
logger.exception(e.response.text)
return HttpResponseBadRequest(e.response.text)
elif action == 'move':
try:
if body.get('system') != system_id:
moved = fm.import_data(body.get('system'), body.get('path'), system_id, file_path)
fm.delete(system_id, file_path)
else:
moved = fm.move(system_id, file_path, body.get('path'), body.get('name'))
return JsonResponse(moved, encoder=AgaveJSONEncoder, safe=False)
except HTTPError as e:
logger.exception(e.response.text)
return HttpResponseBadRequest(e.response.text)
elif action == 'preview':
try:
file_listing = fm.listing(system_id, file_path)
if file_listing.previewable:
preview_url = reverse('designsafe_api:files_media',
args=[file_mgr_name, system_id, file_path])
return JsonResponse({'href': '{}?preview=true'.format(preview_url)})
else:
return HttpResponseBadRequest('Preview not available for this item')
except HTTPError as e:
logger.exception('Unable to preview file')
return HttpResponseBadRequest(e.response.text)
elif action == 'rename':
try:
renamed = fm.rename(system_id, file_path, body.get('name'))
return JsonResponse(renamed, encoder=AgaveJSONEncoder, safe=False)
except HTTPError as e:
logger.exception(e.response.text)
return HttpResponseBadRequest(e.response.text)
elif action == 'trash':
trash_path = '/Trash'
if system_id == AgaveFileManager.DEFAULT_SYSTEM_ID:
trash_path = request.user.username + '/.Trash'
try:
trashed = fm.trash(system_id, file_path, trash_path)
return JsonResponse(trashed, encoder=AgaveJSONEncoder, safe=False)
except HTTPError as e:
logger.error(e.response.text)
return HttpResponseBadRequest(e.response.text)
return HttpResponseBadRequest("Unsupported operation")
def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
file_type, file_id = self.parse_file_id(file_id=src_file_id)
googledrive_item = self.googledrive_api.files().get(
fileId=file_id, fields="name").execute()
n = Notification(
event_type='data',
status=Notification.INFO,
operation='googledrive_download_start',
message='Starting copy of {} {} from Google Drive.'.format(
file_type, googledrive_item['name']),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, filename: {}, src_file_id: {}, dest_file_id: {}'
.format(username, googledrive_item['name'], src_file_id,
dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = self.download_file(
file_id, dest_real_path, username)
if downloaded_file_path is None:
return None
elif file_type == 'folder':
downloaded_file_path = self.download_folder(
file_id, dest_real_path, username)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='googledrive_download_end',
message='{} "{}" was copied from Google Drive successfully!'.
format(file_type.capitalize(), googledrive_item['name']),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
reindex_agave.apply_async(kwargs={
'username':
user.username,
'file_id':
'{}/{}'.format(agave_system_id, agave_file_path)
},
queue='indexing')
except Exception as e:
logger.exception('Unexpected task failure: googledrive_copy',
extra={
'username': username,
'file_id': src_file_id,
'dest_file_id': dest_file_id,
'error_type': type(e),
'error': str(e)
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_copy_error',
message='We were unable to copy the file from Google Drive. '
'Please try again...',
user=username,
extra={'path': googledrive_item['name']})
n.save()
raise
def upload(self, username, src_file_id, dest_folder_id):
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='googledrive_upload_start',
message='Uploading file %s to Google Drive.' %
(src_file_id, ),
user=username,
extra={})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split src ination file path
src_file_path_comps = src_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = src_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if src_file_path_comps[1:]:
src_real_path = os.path.join(*src_file_path_comps[1:])
else:
src_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
src_real_path = os.path.join(base_mounted_path, project_dir,
src_real_path.strip('/'))
else:
src_real_path = os.path.join(base_mounted_path,
src_real_path.strip('/'))
logger.debug('src_real_path: {}'.format(src_real_path))
logger.debug('dest_folder_id:{}'.format(dest_folder_id))
if dest_folder_id == '':
dest_folder_id = 'root'
file_type, folder_id = self.parse_file_id(
file_id=dest_folder_id.strip('/'))
if os.path.isfile(src_real_path):
self.upload_file(folder_id, src_real_path)
elif os.path.isdir(src_real_path):
self.upload_directory(folder_id, src_real_path)
else:
logger.error('Unable to upload %s: file does not exist!',
src_real_path)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='googledrive_upload_end',
message='File "%s" was copied to Google Drive successfully!' %
(src_file_id, ),
user=username,
extra={})
n.save()
except Exception as err:
logger.exception('Unexpected task failure: googledrive_upload',
extra={
'username': username,
'src_file_id': src_file_id,
'dst_file_id': dest_folder_id,
'error': err
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_upload_error',
message=
'We were unable to upload the specified file to Google Drive. '
'Please try again...',
user=username,
extra={})
n.save()
raise
def post(self, request, file_mgr_name, system_id, file_path):
if file_mgr_name == AgaveFileManager.NAME \
or file_mgr_name == 'public':
if not request.user.is_authenticated:
return HttpResponseForbidden('Login required')
agave_client = request.user.agave_oauth.client
fm = AgaveFileManager(agave_client=agave_client)
if request.FILES:
upload_file = request.FILES['file']
upload_dir = file_path
relative_path = request.POST.get('relative_path', None)
if relative_path:
# user uploaded a folder structure; ensure path exists
upload_dir = os.path.join(file_path,
os.path.dirname(relative_path))
BaseFileResource.ensure_path(agave_client, system_id,
upload_dir)
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session, 'session_key',
''),
'operation':
'data_depot_folder_upload',
'info': {
'filePath':
file_path,
'relativePath':
os.path.dirname(relative_path),
'systemId':
system_id,
'uploadDir':
upload_dir
}
})
try:
result = fm.upload(system_id, upload_dir, upload_file)
metrics.info('Data Depot',
extra={
'user':
request.user.username,
'sessionId':
getattr(request.session, 'session_key',
''),
'operation':
'data_depot_file_upload',
'info': {
'systemId': system_id,
'uploadDir': upload_dir,
'uploadFile': upload_file
}
})
result['system'] = result['systemId']
result_file = BaseFileResource(agave_client, **result)
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.OPERATION: 'data_depot_file_upload',
Notification.STATUS: Notification.SUCCESS,
Notification.USER: request.user.username,
Notification.MESSAGE: 'File Upload was successful.',
Notification.EXTRA: result_file.to_dict()
}
Notification.objects.create(**event_data)
except HTTPError as e:
logger.error(e.response.text)
event_data = {
Notification.EVENT_TYPE: 'data_depot',
Notification.OPERATION: 'data_depot_file_upload',
Notification.STATUS: Notification.ERROR,
Notification.USER: request.user.username,
Notification.MESSAGE:
'There was an error uploading one or more file(s).',
Notification.EXTRA: {
'system': system_id,
'file_path': file_path
}
}
Notification.objects.create(**event_data)
return HttpResponseBadRequest(e.response.text)
return JsonResponse({'status': 'ok'})
return HttpResponseBadRequest("Unsupported operation")
