def resource_unconnected_handler(api):
if api == 'googledrive':
message = 'Connect your Google Drive account <a href="' + reverse(
'googledrive_integration:index') + '">here</a>'
return ApiException(status=400,
message=message,
extra={
'action_url':
reverse('googledrive_integration:index'),
'action_label':
'Connect Google Drive Account'
})
if api == 'dropbox':
message = 'Connect your Dropbox account <a href="' + reverse(
'dropbox_integration:index') + '">here</a>'
return ApiException(status=400,
message=message,
extra={
'action_url':
reverse('dropbox_integration:index'),
'action_label': 'Connect Dropbox.com Account'
})
if api == 'box':
message = 'Connect your Box account <a href="' + reverse(
'box_integration:index') + '">here</a>'
return ApiException(status=400,
message=message,
extra={
'action_url': reverse('box_integration:index'),
'action_label': 'Connect Box.com Account'
})
else:
message = 'There was an error accessing this storage system.'
return ApiException(status=400, message=message)
def listing(self, file_id=None, **kwargs):
"""
Lists contents of a folder or details of a file.
Args:
file_id: The type/id of the Google Drive Object. This should be formatted {type}/{id}
where {type} is one of ['folder', 'file'] and {id} is the numeric Google Drive ID for
the object.
Returns:
Dictionary with two keys;
- resource: File System resource that we're listing
- files: Array of Api file-like objects
"""
default_pems = 'ALL'
try:
if file_id == '/':
# top level dir
file_id = 'root'
file_type, file_id = self.parse_file_id(file_id)
fields = "mimeType, name, id, modifiedTime, fileExtension, size, parents"
googledrive_item = self.googledrive_api.files().get(fileId=file_id, fields=fields).execute()
child_results = self.googledrive_api.files().list(q="'{}' in parents and trashed=False".format(file_id), fields="files({})".format(fields)).execute()
if file_type == 'folder':
children = [GoogleDriveFile(item, parent=googledrive_item, drive=self.googledrive_api).to_dict(default_pems=default_pems)
for item in child_results['files']]
child_folders = sorted([item for item in children if item['type'] == 'folder'], key=lambda k: os.path.splitext(k['name'])[0])
child_files = sorted([item for item in children if item['type'] == 'file'], key=lambda k: os.path.splitext(k['name'])[0])
children = child_folders + child_files
else:
children = None
list_data = GoogleDriveFile(googledrive_item, drive=self.googledrive_api).to_dict(default_pems=default_pems)
if children:
list_data['children'] = children
return list_data
except AssertionError:
raise ApiException(status=404, message='The file you requested does not exist.')
except Exception as e:
if 'invalid_grant' in str(e):
message = 'While you previously granted this application access to Google Drive, ' \
'that grant appears to be no longer valid. Please ' \
'<a href="{}">disconnect and reconnect your Google Drive account</a> ' \
'to continue using Google Drive data.'.format(reverse('googledrive_integration:index'))
raise ApiException(status=401, message=message)
message = 'Unable to communicate with Google Drive: {}'.format(e)
raise ApiException(status=500, message=message)
def listing(self, file_id=None, **kwargs):
"""
Lists contents of a folder or details of a file.
Args:
file_id: The type/id of the Box Object. This should be formatted {type}/{id}
where {type} is one of ['folder', 'file'] and {id} is the numeric Box ID for
the object.
Returns:
Dictionary with two keys;
- resource: File System resource that we're listing
- files: Array of Api file-like objects
"""
default_pems = 'ALL'
try:
file_type, file_id = self.parse_file_id(file_id)
box_op = getattr(self.box_api, file_type)
box_item = box_op(file_id).get()
if file_type == 'folder':
limit = int(kwargs.pop('limit', 100))
offset = int(kwargs.pop('offset', 0))
limit = offset + limit
children = [
BoxFile(item,
parent=box_item).to_dict(default_pems=default_pems)
for item in box_item.get_items(limit, offset=offset)
]
else:
children = None
list_data = BoxFile(box_item).to_dict(default_pems=default_pems)
if children:
list_data['children'] = children
return list_data
except AssertionError:
raise ApiException(
status=404, message='The file you requested does not exist.')
except BoxOAuthException:
# user needs to reconnect with Box
message = 'While you previously granted this application access to Box, ' \
'that grant appears to be no longer valid. Please ' \
'<a href="%s">disconnect and reconnect your Box.com account</a> ' \
'to continue using Box data.' % reverse('box_integration:index')
raise ApiException(status=403, message=message)
except BoxException as e:
message = 'Unable to communicate with Box: %s' % e.message
raise ApiException(status=500, message=message)
def __init__(self, user_obj, **kwargs):
self._user = user_obj
if user_obj.is_anonymous():
raise ApiException(status=403, message='Log in required to access Google Drive files.')
try:
self.googledrive_api = user_obj.googledrive_user_token.client
except GoogleDriveUserToken.DoesNotExist:
message = 'Connect your Google Drive account <a href="'+ reverse('googledrive_integration:index') + '">here</a>'
raise ApiException(status=400, message=message, extra={
'action_url': reverse('googledrive_integration:index'),
'action_label': 'Connect Google Drive Account'
})
def mkdir(cls,
system,
username,
file_path,
dir_name,
agave_client=None,
**kwargs):
body = '{{"action": "mkdir", "path": "{}"}}'.format(dir_name)
try:
logger.debug('Agave: calling: files.manage, args: {}'.format({
'systemId':
system,
'filePath':
urllib2.quote(file_path),
'body':
body
}))
agave_client.files.manage(systemId=system,
filePath=urllib2.quote(file_path),
body=body)
dir_path = os.path.join(file_path, dir_name)
return cls.from_file_path(system, username, dir_path, agave_client)
except (AgaveException, HTTPError) as e:
logger.error(e, exc_info=True, extra=kwargs)
d = {'operation': 'files.list'}
d.update({'systemId': system, 'filePath': file_path, 'body': body})
raise ApiException(e.message, e.response.status_code, extra=d)
def listing(cls, system, file_path, agave_client, **kwargs):
try:
limit = int(kwargs.pop('limit', 100))
offset = int(kwargs.pop('offset', 0))
limit = offset + limit
logger.debug('Agave: calling: files.list, args: {}'.format({
'systemId':
system,
'filePath':
file_path,
'limit':
limit,
'offset':
offset
}))
listing = agave_client.files.list(
systemId=system,
filePath=urllib2.quote(file_path),
limit=limit,
offset=offset)
except (AgaveException, HTTPError) as e:
logger.error(
e,
exc_info=True,
)
d = {'operation': 'files.list'}
d.update({'systemId': system, 'filePath': file_path})
raise ApiException(e.message, e.response.status_code, extra=d)
return [
cls(wrap=o, agave_client=agave_client, **kwargs) for o in listing
]
def move(self, file_id, **kwargs):
raise ApiException('Moving Google Drive files is not supported.',
status=400,
extra={
'file_id': file_id,
'kwargs': kwargs
})
def box_object_for_file_hierarchy_path(self, file_path):
"""
Resolves a hierarchical path to a box_object. For example, given the file_path
"Documents/Project/Testing" this function will iteratively query Box, starting at
All Files (folder/0), and looking for a child with the name of the next element in
the path. If found, the BoxObject is returned. Otherwise, raises.
Args:
file_path: The hierarchical path to the BoxObject
Returns:
The BoxObject
"""
box_object = self.box_api.folder(u'0')
if file_path is None or file_path == '' or file_path == 'All Files':
return box_object
path_c = file_path.split('/')
for c in path_c:
if box_object._item_type != 'folder':
# we've found a file, but there are still more path components to process
raise ApiException(
'The Box path "{0}" does not exist.'.format(file_path),
status=404)
limit = 100
offset = 0
next_object = None
while next_object is None:
children = self.box_api.folder(box_object.object_id).get_items(
limit=limit, offset=offset)
for child in children:
if child.name == c:
next_object = child
break
if len(children) == limit:
offset += limit
elif next_object is None: # this can happen if path doesn't exist
raise ApiException(
'The Box path "{0}" does not exist.'.format(file_path),
status=404)
box_object = next_object
return box_object
def copy(self, file_id, dest_resource, dest_file_id, **kwargs):
"""Copies a file
Copies a file in both the Agave filesystem and the
Elasticsearch index.
:param str file_id:
:param str dest_resource:
:param str dest_file_id:
:returns: dict representation of the original
:class:`designsafe.apps.api.data.agve.file.AgaveFile` instance
:rtype: dict
Examples:
--------
Copy a file. `fm` is an instance of FileManager
>>> fm.copy(file_id='designsafe.storage.default/username/file.jpg',
>>> dest_resource='agave',
>>> dest_file_id='designsafe.storage.default/username/file_copy.jpg')
"""
if self._user.is_anonymous():
raise ApiException(
message='You must log in to perform this action.', status=403)
# can only transfer out of public
from designsafe.apps.api.data import lookup_transfer_service
service = lookup_transfer_service(self.resource, dest_resource)
if service:
args = (self.username, self.resource, file_id, dest_resource,
dest_file_id)
service.apply_async(args=args, queue='files')
return {'message': 'The requested transfer has been scheduled'}
else:
message = 'The requested transfer from %s to %s ' \
'is not supported' % (self.resource, dest_resource)
extra = {
'file_id': file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id
}
raise ApiException(message, status=400, extra=extra)
def SearchLookupManager(name):
"""Lookup data depot manager class"""
manager_names = list(settings.PORTAL_DATA_DEPOT_SEARCH_MANAGERS)
if name not in manager_names:
raise ApiException("Invalid file manager.")
module_str, class_str = settings.PORTAL_DATA_DEPOT_SEARCH_MANAGERS[name].rsplit('.', 1)
module = import_module(module_str)
cls = getattr(module, class_str)
return cls
def __init__(self, user_obj, **kwargs):
super(FileManager, self).__init__()
self._user = user_obj
if user_obj.is_anonymous():
raise ApiException(status=403,
message='Log in required to access Box files.')
try:
self.box_api = user_obj.box_user_token.client
except BoxUserToken.DoesNotExist:
message = 'You need to connect your Box.com account ' \
'before you can access your Box.com files.'
raise ApiException(status=400,
message=message,
extra={
'action_url':
reverse('box_integration:index'),
'action_label': 'Connect Box.com Account'
})
def listing(self, system, file_path='/', offset=0, limit=100, **kwargs):
file_path = file_path or '/'
path_comps = file_path.strip('/').split('/')
if len(path_comps) < 1:
raise ApiException(messsage='Invalid Action', status=400)
#elif len(path_comps) == 1:
# project_id = path_comps[0]
# publication = Publication(project_id=project_id)
# return publication
else:
return super(PublishedFileManager, self).\
listing(system, file_path, offset, limit)
def resource_expired_handler(api):
if api == 'googledrive':
message = 'While you previously granted this application access to Google Drive, ' \
'that grant appears to be no longer valid. Please ' \
'<a href="%s">disconnect and reconnect your Google Drive account</a> ' \
'to continue using Google Drive data.' % reverse('googledrive_integration:index')
return ApiException(status=403, message=message)
if api == 'dropbox':
message = 'While you previously granted this application access to Dropbox, ' \
'that grant appears to be no longer valid. Please ' \
'<a href="%s">disconnect and reconnect your Dropbox.com account</a> ' \
'to continue using Dropbox data.' % reverse('dropbox_integration:index')
raise ApiException(status=403, message=message)
if api == 'box':
# user needs to reconnect with Box
message = 'While you previously granted this application access to Box, ' \
'that grant appears to be no longer valid. Please ' \
'<a href="%s">disconnect and reconnect your Box.com account</a> ' \
'to continue using Box data.' % reverse('box_integration:index')
raise ApiException(status=403, message=message)
else:
message = 'There was an error accessing this storage system.'
return ApiException(status=400, message=message)
def __init__(self, user_obj, **kwargs):
self._user = user_obj
if user_obj.is_anonymous():
raise ApiException(
status=403, message='Log in required to access Dropbox files.')
try:
# self.dropbox_api = user_obj.dropbox_user_token.client
dropbox_token = DropboxUserToken.objects.get(user=user_obj)
self.dropbox_api = Dropbox(dropbox_token.access_token)
except DropboxUserToken.DoesNotExist:
message = 'Connect your Dropbox account <a href="' + reverse(
'dropbox_integration:index') + '">here</a>'
raise ApiException(status=400,
message=message,
extra={
'action_url':
reverse('dropbox_integration:index'),
'action_label':
'Connect Dropbox.com Account'
})
def post(self, request):
"""Post a feedback response
"""
rt = rtUtil.DjangoRt(queue=settings.DJANGO_RT['RT_FEEDBACK_QUEUE'])
data = json.loads(request.body)
email = request.user.email if request.user.is_authenticated else data[
'email']
name = "{} {}".format(
request.user.first_name, request.user.last_name
) if request.user.is_authenticated else data['name']
subject = data['subject']
body = data['body']
project_id = data['projectId']
project_title = data['title']
if subject is None or email is None or body is None:
return HttpResponseBadRequest()
requestor_meta = '%s <%s>' % (name, email)
meta = (
('Opened by', request.user.username),
('Category', 'Project Feedback'),
('Resource', 'DesignSafe'),
('Project ID', project_id),
('Project Title', project_title),
)
header = '\n'.join('[%s] %s' % m for m in meta)
ticket_body = '%s\n\n%s\n\n---\n%s' % (header, body, requestor_meta)
ticket = rtModels.Ticket(subject=subject,
problem_description="\n ".join(
ticket_body.splitlines()),
requestor=email,
cc='')
logger.debug('Creating ticket for user: %s' % ticket)
ticket_id = rt.createTicket(ticket)
if ticket_id > -1:
return HttpResponse("OK")
else:
return ApiException(
status=400,
message="There was a problem submittin your ticket.")
def call_operation(self, operation, raise_agave=False, **kwargs):
a = self.agave_client
op = self.get_operation(a, operation)
try:
logger.debug('Agave: calling {}, args: {}'.format(operation, kwargs))
response = self.exec_operation(op, **kwargs)
except (AgaveException, HTTPError) as e:
logger.error(e, exc_info = True, extra = kwargs)
if e.response.status_code < 500 or raise_agave:
raise
else:
d = {'operation': op}
d.update(kwargs)
raise ApiException(e.message, e.response.status_code,
response=e.response, extra=d)
return response
def update_pems(self, username_to_update, permission, recursive=True):
"""Update permissions.
Args:
username_to_update (string): Username whose permissions we are
going to update
permission (string): New permission.
[READ | WRITE | EXECUTE | READ_WRITE | READ_EXECUTE | WRITE_EXECUTE | ALL | NONE]
Returns:
Class instasnce for chainability
"""
if permission == 'NONE' and len(
self.parent_path.strip('/').split('/')) > 1:
parent_pems = self.from_file_path(self.system, None,
self.parent_path,
self.agave_client).permissions
parent_user_pems = filter(
lambda x: x['username'] == username_to_update and x[
'recursive'], parent_pems)
if parent_user_pems:
raise ApiException(
'Can not remove permissions when the user has RECURSIVE permissions on parent',
400)
permission_body = json.dumps({
'recursive': recursive,
'permission': permission,
'username': username_to_update
})
try:
self.call_operation('files.updatePermissions',
filePath=urllib2.quote(self.full_path),
systemId=self.system,
body=permission_body,
raise_agave=True)
except AgaveException as e:
logger.error('{}: Could not update permissions {}'.format(
e.message, permission_body))
except HTTPError as e:
if e.response.status_code == 502 or e.response.status_code == 503:
logger.warning('{}: Could not update permissions {}'.format(
e.message, permission_body))
else:
raise
return self
def copy(self, file_id, dest_resource, dest_file_id, **kwargs):
# can only transfer out of box
from designsafe.apps.api.data import lookup_transfer_service
service = lookup_transfer_service(self.resource, dest_resource)
if service:
args = (self._user.username, self.resource, file_id, dest_resource,
dest_file_id)
service.apply_async(args=args, queue='files')
return {'message': 'The requested transfer has been scheduled'}
else:
message = 'The requested transfer from %s to %s ' \
'is not supported' % (self.resource, dest_resource)
extra = {
'file_id': file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id
}
raise ApiException(message, status=400, extra=extra)
def parse_file_id(self, path):
if path == '/' or path == '':
file_type, path = 'folder', ''
else:
try:
file_type, path = DropboxFile.parse_file_id(path)
except AssertionError:
# file path is hierarchical; need to find the DropboxObject here
if path[:1] != '/':
path = '/' + path
try:
dropbox_item = DropboxFile(
self.dropbox_api.files_list_folder(path))
return dropbox_item.type, path
except ApiError as e:
if e.error.get_path().is_not_found():
raise ApiException(
'The Dropbox path "{0}" does not exist.'.format(
path),
status=404)
return file_type, path
def from_file_path(cls,
system,
username=None,
file_path=None,
agave_client=None,
**kwargs):
try:
logger.debug('Agave: calling: files.list, args: {}'.format({
'systemId':
system,
'filePath':
urllib2.quote(file_path)
}))
listing = agave_client.files.list(
systemId=system, filePath=urllib2.quote(file_path))
except (AgaveException, HTTPError) as e:
logger.error(e, exc_info=True, extra=kwargs)
d = {'operation': 'files.list'}
d.update({'systemId': system, 'filePath': file_path})
raise ApiException(e.message, e.response.status_code, extra=d)
return cls(agave_client=agave_client, wrap=listing[0], **kwargs)
def listing(self, file_id='', **kwargs):
"""
Lists contents of a folder or details of a file.
Args:
file_id: The type/path of the Dropbox Object. This should be formatted {type}/{path}
where {type} is one of ['folder', 'file'] and {path} is the numeric Dropbox path for
the object.
Returns:
Dictionary with two keys;
- resource: File System resource that we're listing
- files: Array of Api file-like objects
"""
default_pems = 'ALL'
try:
file_type, path = self.parse_file_id(file_id)
dropbox_item = self.dropbox_api.files_list_folder(path)
children = []
if file_type == 'folder':
has_more = dropbox_item.has_more
cursor = dropbox_item.cursor
entries = dropbox_item.entries
while True:
children.extend([
DropboxFile(item,
item.path_display,
parent=dropbox_item).to_dict(
default_pems=default_pems)
for item in entries
])
if has_more:
folder = self.dropbox_api.files_list_folder_continue(
cursor)
entries = folder.entries
has_more = folder.has_more
cursor = folder.cursor
else:
break
else:
children = None
list_data = DropboxFile(dropbox_item,
path).to_dict(default_pems=default_pems)
if children:
list_data['children'] = children
return list_data
except AssertionError:
raise ApiException(
status=404,
message='The file/folder you requested does not exist.')
except AuthError:
# user needs to reconnect with Dropbox
message = 'While you previously granted this application access to Dropbox, ' \
'that grant appears to be no longer valid. Please ' \
'<a href="%s">disconnect and reconnect your Dropbox.com account</a> ' \
'to continue using Dropbox data.' % reverse('dropbox_integration:index')
raise ApiException(status=403, message=message)
except ApiError as e:
if e.error.get_path().is_not_folder():
dropbox_item = self.dropbox_api.files_get_metadata(path)
list_data = DropboxFile(dropbox_item, path)
return list_data
message = 'Unable to communicate with Dropbox: %s' % e.message
raise ApiException(status=500, message=message)
def generic_webhook_handler(request):
event_type = request.POST.get('event_type', None)
if event_type == 'WEB':
# This is for jobs that just point to a URL that gets created
# like the Potree Viewer Application or DCV-based apps
job_owner = request.POST.get('owner', '')
address = request.POST.get('address', '')
job_uuid = request.POST.get('job_uuid', '')
event_data = {
Notification.EVENT_TYPE: event_type,
Notification.STATUS: Notification.INFO,
Notification.OPERATION: 'web_link',
Notification.USER: job_owner,
Notification.MESSAGE: 'Ready to view.',
Notification.ACTION_LINK: address,
Notification.EXTRA: {
'address': address,
'target_uri': address
}
}
elif event_type == 'VNC':
job_owner = request.POST.get('owner', '')
host = request.POST.get('host', '')
port = request.POST.get('port', '')
password = request.POST.get('password', '')
address = request.POST.get('address', '')
job_uuid = password
if host == 'designsafe-exec-01.tacc.utexas.edu':
target_uri = 'https://' + address + '&autoconnect=true'
else:
target_uri = \
'https://tap.tacc.utexas.edu/noVNC/?'\
'host={host}&port={port}&autoconnect=true&encrypt=true&resize=scale&password={pw}' \
.format(host=host, port=port, pw=password)
event_data = {
Notification.EVENT_TYPE: event_type,
Notification.STATUS: Notification.INFO,
Notification.OPERATION: 'web_link',
Notification.USER: job_owner,
Notification.MESSAGE: 'Your VNC session is ready.',
Notification.ACTION_LINK: target_uri,
Notification.EXTRA: {
'host': host,
'port': port,
'address': address,
'password': password,
'target_uri': target_uri,
'associationIds': job_uuid
}
}
else:
return HttpResponse('Unexpected event_type', status=400)
# confirm that there is a corresponding running agave job before sending notification
try:
user = get_user_model().objects.get(username=job_owner)
agave = user.agave_oauth.client
job_data = agave.jobs.get(jobId=job_uuid)
if job_data['owner'] != job_owner or job_data["status"] in [
'FINISHED', 'FAILED', 'STOPPED'
]:
LOGGER.error(
"Agave job (owner='{}', status='{}) for this event (owner='{}') is not valid"
.format(job_data['owner'], job_data['status'], job_owner))
raise ApiException(
"Unable to find a related valid job for this interactive event."
)
except (HTTPError, AgaveException, ApiException) as e:
LOGGER.exception(
"Could not find valid corresponding Agave job for interactive event"
)
return HttpResponse({'message': e.message},
content_type='application/json',
status=400)
n = Notification.objects.create(**event_data)
n.save()
# create metadata for Interactive connection and save to agave metadata
try:
agave_job_meta = {
'name': 'interactiveJobDetails',
'value': event_data,
'associationIds': [job_uuid],
}
user = get_user_model().objects.get(username=job_owner)
agave = user.agave_oauth.client
agave.meta.addMetadata(body=json.dumps(agave_job_meta))
except (HTTPError, AgaveException) as e:
LOGGER.exception(
'Could not add interactive connection data to metadata')
return HttpResponse({'message': e.message},
content_type='application/json',
status=400)
return HttpResponse('OK')
def upload(self, *args, **kwargs):
return ApiException(messsage='Invalid Action', status=400)