def resource_unconnected_handler(api): if api == 'googledrive': message = 'Connect your Google Drive account <a href="' + reverse( 'googledrive_integration:index') + '">here</a>' return ApiException(status=400, message=message, extra={ 'action_url': reverse('googledrive_integration:index'), 'action_label': 'Connect Google Drive Account' }) if api == 'dropbox': message = 'Connect your Dropbox account <a href="' + reverse( 'dropbox_integration:index') + '">here</a>' return ApiException(status=400, message=message, extra={ 'action_url': reverse('dropbox_integration:index'), 'action_label': 'Connect Dropbox.com Account' }) if api == 'box': message = 'Connect your Box account <a href="' + reverse( 'box_integration:index') + '">here</a>' return ApiException(status=400, message=message, extra={ 'action_url': reverse('box_integration:index'), 'action_label': 'Connect Box.com Account' }) else: message = 'There was an error accessing this storage system.' return ApiException(status=400, message=message)
def listing(self, file_id=None, **kwargs): """ Lists contents of a folder or details of a file. Args: file_id: The type/id of the Google Drive Object. This should be formatted {type}/{id} where {type} is one of ['folder', 'file'] and {id} is the numeric Google Drive ID for the object. Returns: Dictionary with two keys; - resource: File System resource that we're listing - files: Array of Api file-like objects """ default_pems = 'ALL' try: if file_id == '/': # top level dir file_id = 'root' file_type, file_id = self.parse_file_id(file_id) fields = "mimeType, name, id, modifiedTime, fileExtension, size, parents" googledrive_item = self.googledrive_api.files().get(fileId=file_id, fields=fields).execute() child_results = self.googledrive_api.files().list(q="'{}' in parents and trashed=False".format(file_id), fields="files({})".format(fields)).execute() if file_type == 'folder': children = [GoogleDriveFile(item, parent=googledrive_item, drive=self.googledrive_api).to_dict(default_pems=default_pems) for item in child_results['files']] child_folders = sorted([item for item in children if item['type'] == 'folder'], key=lambda k: os.path.splitext(k['name'])[0]) child_files = sorted([item for item in children if item['type'] == 'file'], key=lambda k: os.path.splitext(k['name'])[0]) children = child_folders + child_files else: children = None list_data = GoogleDriveFile(googledrive_item, drive=self.googledrive_api).to_dict(default_pems=default_pems) if children: list_data['children'] = children return list_data except AssertionError: raise ApiException(status=404, message='The file you requested does not exist.') except Exception as e: if 'invalid_grant' in str(e): message = 'While you previously granted this application access to Google Drive, ' \ 'that grant appears to be no longer valid. Please ' \ '<a href="{}">disconnect and reconnect your Google Drive account</a> ' \ 'to continue using Google Drive data.'.format(reverse('googledrive_integration:index')) raise ApiException(status=401, message=message) message = 'Unable to communicate with Google Drive: {}'.format(e) raise ApiException(status=500, message=message)
def listing(self, file_id=None, **kwargs): """ Lists contents of a folder or details of a file. Args: file_id: The type/id of the Box Object. This should be formatted {type}/{id} where {type} is one of ['folder', 'file'] and {id} is the numeric Box ID for the object. Returns: Dictionary with two keys; - resource: File System resource that we're listing - files: Array of Api file-like objects """ default_pems = 'ALL' try: file_type, file_id = self.parse_file_id(file_id) box_op = getattr(self.box_api, file_type) box_item = box_op(file_id).get() if file_type == 'folder': limit = int(kwargs.pop('limit', 100)) offset = int(kwargs.pop('offset', 0)) limit = offset + limit children = [ BoxFile(item, parent=box_item).to_dict(default_pems=default_pems) for item in box_item.get_items(limit, offset=offset) ] else: children = None list_data = BoxFile(box_item).to_dict(default_pems=default_pems) if children: list_data['children'] = children return list_data except AssertionError: raise ApiException( status=404, message='The file you requested does not exist.') except BoxOAuthException: # user needs to reconnect with Box message = 'While you previously granted this application access to Box, ' \ 'that grant appears to be no longer valid. Please ' \ '<a href="%s">disconnect and reconnect your Box.com account</a> ' \ 'to continue using Box data.' % reverse('box_integration:index') raise ApiException(status=403, message=message) except BoxException as e: message = 'Unable to communicate with Box: %s' % e.message raise ApiException(status=500, message=message)
def __init__(self, user_obj, **kwargs): self._user = user_obj if user_obj.is_anonymous(): raise ApiException(status=403, message='Log in required to access Google Drive files.') try: self.googledrive_api = user_obj.googledrive_user_token.client except GoogleDriveUserToken.DoesNotExist: message = 'Connect your Google Drive account <a href="'+ reverse('googledrive_integration:index') + '">here</a>' raise ApiException(status=400, message=message, extra={ 'action_url': reverse('googledrive_integration:index'), 'action_label': 'Connect Google Drive Account' })
def mkdir(cls, system, username, file_path, dir_name, agave_client=None, **kwargs): body = '{{"action": "mkdir", "path": "{}"}}'.format(dir_name) try: logger.debug('Agave: calling: files.manage, args: {}'.format({ 'systemId': system, 'filePath': urllib2.quote(file_path), 'body': body })) agave_client.files.manage(systemId=system, filePath=urllib2.quote(file_path), body=body) dir_path = os.path.join(file_path, dir_name) return cls.from_file_path(system, username, dir_path, agave_client) except (AgaveException, HTTPError) as e: logger.error(e, exc_info=True, extra=kwargs) d = {'operation': 'files.list'} d.update({'systemId': system, 'filePath': file_path, 'body': body}) raise ApiException(e.message, e.response.status_code, extra=d)
def listing(cls, system, file_path, agave_client, **kwargs): try: limit = int(kwargs.pop('limit', 100)) offset = int(kwargs.pop('offset', 0)) limit = offset + limit logger.debug('Agave: calling: files.list, args: {}'.format({ 'systemId': system, 'filePath': file_path, 'limit': limit, 'offset': offset })) listing = agave_client.files.list( systemId=system, filePath=urllib2.quote(file_path), limit=limit, offset=offset) except (AgaveException, HTTPError) as e: logger.error( e, exc_info=True, ) d = {'operation': 'files.list'} d.update({'systemId': system, 'filePath': file_path}) raise ApiException(e.message, e.response.status_code, extra=d) return [ cls(wrap=o, agave_client=agave_client, **kwargs) for o in listing ]
def move(self, file_id, **kwargs): raise ApiException('Moving Google Drive files is not supported.', status=400, extra={ 'file_id': file_id, 'kwargs': kwargs })
def box_object_for_file_hierarchy_path(self, file_path): """ Resolves a hierarchical path to a box_object. For example, given the file_path "Documents/Project/Testing" this function will iteratively query Box, starting at All Files (folder/0), and looking for a child with the name of the next element in the path. If found, the BoxObject is returned. Otherwise, raises. Args: file_path: The hierarchical path to the BoxObject Returns: The BoxObject """ box_object = self.box_api.folder(u'0') if file_path is None or file_path == '' or file_path == 'All Files': return box_object path_c = file_path.split('/') for c in path_c: if box_object._item_type != 'folder': # we've found a file, but there are still more path components to process raise ApiException( 'The Box path "{0}" does not exist.'.format(file_path), status=404) limit = 100 offset = 0 next_object = None while next_object is None: children = self.box_api.folder(box_object.object_id).get_items( limit=limit, offset=offset) for child in children: if child.name == c: next_object = child break if len(children) == limit: offset += limit elif next_object is None: # this can happen if path doesn't exist raise ApiException( 'The Box path "{0}" does not exist.'.format(file_path), status=404) box_object = next_object return box_object
def copy(self, file_id, dest_resource, dest_file_id, **kwargs): """Copies a file Copies a file in both the Agave filesystem and the Elasticsearch index. :param str file_id: :param str dest_resource: :param str dest_file_id: :returns: dict representation of the original :class:`designsafe.apps.api.data.agve.file.AgaveFile` instance :rtype: dict Examples: -------- Copy a file. `fm` is an instance of FileManager >>> fm.copy(file_id='designsafe.storage.default/username/file.jpg', >>> dest_resource='agave', >>> dest_file_id='designsafe.storage.default/username/file_copy.jpg') """ if self._user.is_anonymous(): raise ApiException( message='You must log in to perform this action.', status=403) # can only transfer out of public from designsafe.apps.api.data import lookup_transfer_service service = lookup_transfer_service(self.resource, dest_resource) if service: args = (self.username, self.resource, file_id, dest_resource, dest_file_id) service.apply_async(args=args, queue='files') return {'message': 'The requested transfer has been scheduled'} else: message = 'The requested transfer from %s to %s ' \ 'is not supported' % (self.resource, dest_resource) extra = { 'file_id': file_id, 'dest_resource': dest_resource, 'dest_file_id': dest_file_id } raise ApiException(message, status=400, extra=extra)
def SearchLookupManager(name): """Lookup data depot manager class""" manager_names = list(settings.PORTAL_DATA_DEPOT_SEARCH_MANAGERS) if name not in manager_names: raise ApiException("Invalid file manager.") module_str, class_str = settings.PORTAL_DATA_DEPOT_SEARCH_MANAGERS[name].rsplit('.', 1) module = import_module(module_str) cls = getattr(module, class_str) return cls
def __init__(self, user_obj, **kwargs): super(FileManager, self).__init__() self._user = user_obj if user_obj.is_anonymous(): raise ApiException(status=403, message='Log in required to access Box files.') try: self.box_api = user_obj.box_user_token.client except BoxUserToken.DoesNotExist: message = 'You need to connect your Box.com account ' \ 'before you can access your Box.com files.' raise ApiException(status=400, message=message, extra={ 'action_url': reverse('box_integration:index'), 'action_label': 'Connect Box.com Account' })
def listing(self, system, file_path='/', offset=0, limit=100, **kwargs): file_path = file_path or '/' path_comps = file_path.strip('/').split('/') if len(path_comps) < 1: raise ApiException(messsage='Invalid Action', status=400) #elif len(path_comps) == 1: # project_id = path_comps[0] # publication = Publication(project_id=project_id) # return publication else: return super(PublishedFileManager, self).\ listing(system, file_path, offset, limit)
def resource_expired_handler(api): if api == 'googledrive': message = 'While you previously granted this application access to Google Drive, ' \ 'that grant appears to be no longer valid. Please ' \ '<a href="%s">disconnect and reconnect your Google Drive account</a> ' \ 'to continue using Google Drive data.' % reverse('googledrive_integration:index') return ApiException(status=403, message=message) if api == 'dropbox': message = 'While you previously granted this application access to Dropbox, ' \ 'that grant appears to be no longer valid. Please ' \ '<a href="%s">disconnect and reconnect your Dropbox.com account</a> ' \ 'to continue using Dropbox data.' % reverse('dropbox_integration:index') raise ApiException(status=403, message=message) if api == 'box': # user needs to reconnect with Box message = 'While you previously granted this application access to Box, ' \ 'that grant appears to be no longer valid. Please ' \ '<a href="%s">disconnect and reconnect your Box.com account</a> ' \ 'to continue using Box data.' % reverse('box_integration:index') raise ApiException(status=403, message=message) else: message = 'There was an error accessing this storage system.' return ApiException(status=400, message=message)
def __init__(self, user_obj, **kwargs): self._user = user_obj if user_obj.is_anonymous(): raise ApiException( status=403, message='Log in required to access Dropbox files.') try: # self.dropbox_api = user_obj.dropbox_user_token.client dropbox_token = DropboxUserToken.objects.get(user=user_obj) self.dropbox_api = Dropbox(dropbox_token.access_token) except DropboxUserToken.DoesNotExist: message = 'Connect your Dropbox account <a href="' + reverse( 'dropbox_integration:index') + '">here</a>' raise ApiException(status=400, message=message, extra={ 'action_url': reverse('dropbox_integration:index'), 'action_label': 'Connect Dropbox.com Account' })
def post(self, request): """Post a feedback response """ rt = rtUtil.DjangoRt(queue=settings.DJANGO_RT['RT_FEEDBACK_QUEUE']) data = json.loads(request.body) email = request.user.email if request.user.is_authenticated else data[ 'email'] name = "{} {}".format( request.user.first_name, request.user.last_name ) if request.user.is_authenticated else data['name'] subject = data['subject'] body = data['body'] project_id = data['projectId'] project_title = data['title'] if subject is None or email is None or body is None: return HttpResponseBadRequest() requestor_meta = '%s <%s>' % (name, email) meta = ( ('Opened by', request.user.username), ('Category', 'Project Feedback'), ('Resource', 'DesignSafe'), ('Project ID', project_id), ('Project Title', project_title), ) header = '\n'.join('[%s] %s' % m for m in meta) ticket_body = '%s\n\n%s\n\n---\n%s' % (header, body, requestor_meta) ticket = rtModels.Ticket(subject=subject, problem_description="\n ".join( ticket_body.splitlines()), requestor=email, cc='') logger.debug('Creating ticket for user: %s' % ticket) ticket_id = rt.createTicket(ticket) if ticket_id > -1: return HttpResponse("OK") else: return ApiException( status=400, message="There was a problem submittin your ticket.")
def call_operation(self, operation, raise_agave=False, **kwargs): a = self.agave_client op = self.get_operation(a, operation) try: logger.debug('Agave: calling {}, args: {}'.format(operation, kwargs)) response = self.exec_operation(op, **kwargs) except (AgaveException, HTTPError) as e: logger.error(e, exc_info = True, extra = kwargs) if e.response.status_code < 500 or raise_agave: raise else: d = {'operation': op} d.update(kwargs) raise ApiException(e.message, e.response.status_code, response=e.response, extra=d) return response
def update_pems(self, username_to_update, permission, recursive=True): """Update permissions. Args: username_to_update (string): Username whose permissions we are going to update permission (string): New permission. [READ | WRITE | EXECUTE | READ_WRITE | READ_EXECUTE | WRITE_EXECUTE | ALL | NONE] Returns: Class instasnce for chainability """ if permission == 'NONE' and len( self.parent_path.strip('/').split('/')) > 1: parent_pems = self.from_file_path(self.system, None, self.parent_path, self.agave_client).permissions parent_user_pems = filter( lambda x: x['username'] == username_to_update and x[ 'recursive'], parent_pems) if parent_user_pems: raise ApiException( 'Can not remove permissions when the user has RECURSIVE permissions on parent', 400) permission_body = json.dumps({ 'recursive': recursive, 'permission': permission, 'username': username_to_update }) try: self.call_operation('files.updatePermissions', filePath=urllib2.quote(self.full_path), systemId=self.system, body=permission_body, raise_agave=True) except AgaveException as e: logger.error('{}: Could not update permissions {}'.format( e.message, permission_body)) except HTTPError as e: if e.response.status_code == 502 or e.response.status_code == 503: logger.warning('{}: Could not update permissions {}'.format( e.message, permission_body)) else: raise return self
def copy(self, file_id, dest_resource, dest_file_id, **kwargs): # can only transfer out of box from designsafe.apps.api.data import lookup_transfer_service service = lookup_transfer_service(self.resource, dest_resource) if service: args = (self._user.username, self.resource, file_id, dest_resource, dest_file_id) service.apply_async(args=args, queue='files') return {'message': 'The requested transfer has been scheduled'} else: message = 'The requested transfer from %s to %s ' \ 'is not supported' % (self.resource, dest_resource) extra = { 'file_id': file_id, 'dest_resource': dest_resource, 'dest_file_id': dest_file_id } raise ApiException(message, status=400, extra=extra)
def parse_file_id(self, path): if path == '/' or path == '': file_type, path = 'folder', '' else: try: file_type, path = DropboxFile.parse_file_id(path) except AssertionError: # file path is hierarchical; need to find the DropboxObject here if path[:1] != '/': path = '/' + path try: dropbox_item = DropboxFile( self.dropbox_api.files_list_folder(path)) return dropbox_item.type, path except ApiError as e: if e.error.get_path().is_not_found(): raise ApiException( 'The Dropbox path "{0}" does not exist.'.format( path), status=404) return file_type, path
def from_file_path(cls, system, username=None, file_path=None, agave_client=None, **kwargs): try: logger.debug('Agave: calling: files.list, args: {}'.format({ 'systemId': system, 'filePath': urllib2.quote(file_path) })) listing = agave_client.files.list( systemId=system, filePath=urllib2.quote(file_path)) except (AgaveException, HTTPError) as e: logger.error(e, exc_info=True, extra=kwargs) d = {'operation': 'files.list'} d.update({'systemId': system, 'filePath': file_path}) raise ApiException(e.message, e.response.status_code, extra=d) return cls(agave_client=agave_client, wrap=listing[0], **kwargs)
def listing(self, file_id='', **kwargs): """ Lists contents of a folder or details of a file. Args: file_id: The type/path of the Dropbox Object. This should be formatted {type}/{path} where {type} is one of ['folder', 'file'] and {path} is the numeric Dropbox path for the object. Returns: Dictionary with two keys; - resource: File System resource that we're listing - files: Array of Api file-like objects """ default_pems = 'ALL' try: file_type, path = self.parse_file_id(file_id) dropbox_item = self.dropbox_api.files_list_folder(path) children = [] if file_type == 'folder': has_more = dropbox_item.has_more cursor = dropbox_item.cursor entries = dropbox_item.entries while True: children.extend([ DropboxFile(item, item.path_display, parent=dropbox_item).to_dict( default_pems=default_pems) for item in entries ]) if has_more: folder = self.dropbox_api.files_list_folder_continue( cursor) entries = folder.entries has_more = folder.has_more cursor = folder.cursor else: break else: children = None list_data = DropboxFile(dropbox_item, path).to_dict(default_pems=default_pems) if children: list_data['children'] = children return list_data except AssertionError: raise ApiException( status=404, message='The file/folder you requested does not exist.') except AuthError: # user needs to reconnect with Dropbox message = 'While you previously granted this application access to Dropbox, ' \ 'that grant appears to be no longer valid. Please ' \ '<a href="%s">disconnect and reconnect your Dropbox.com account</a> ' \ 'to continue using Dropbox data.' % reverse('dropbox_integration:index') raise ApiException(status=403, message=message) except ApiError as e: if e.error.get_path().is_not_folder(): dropbox_item = self.dropbox_api.files_get_metadata(path) list_data = DropboxFile(dropbox_item, path) return list_data message = 'Unable to communicate with Dropbox: %s' % e.message raise ApiException(status=500, message=message)
def generic_webhook_handler(request): event_type = request.POST.get('event_type', None) if event_type == 'WEB': # This is for jobs that just point to a URL that gets created # like the Potree Viewer Application or DCV-based apps job_owner = request.POST.get('owner', '') address = request.POST.get('address', '') job_uuid = request.POST.get('job_uuid', '') event_data = { Notification.EVENT_TYPE: event_type, Notification.STATUS: Notification.INFO, Notification.OPERATION: 'web_link', Notification.USER: job_owner, Notification.MESSAGE: 'Ready to view.', Notification.ACTION_LINK: address, Notification.EXTRA: { 'address': address, 'target_uri': address } } elif event_type == 'VNC': job_owner = request.POST.get('owner', '') host = request.POST.get('host', '') port = request.POST.get('port', '') password = request.POST.get('password', '') address = request.POST.get('address', '') job_uuid = password if host == 'designsafe-exec-01.tacc.utexas.edu': target_uri = 'https://' + address + '&autoconnect=true' else: target_uri = \ 'https://tap.tacc.utexas.edu/noVNC/?'\ 'host={host}&port={port}&autoconnect=true&encrypt=true&resize=scale&password={pw}' \ .format(host=host, port=port, pw=password) event_data = { Notification.EVENT_TYPE: event_type, Notification.STATUS: Notification.INFO, Notification.OPERATION: 'web_link', Notification.USER: job_owner, Notification.MESSAGE: 'Your VNC session is ready.', Notification.ACTION_LINK: target_uri, Notification.EXTRA: { 'host': host, 'port': port, 'address': address, 'password': password, 'target_uri': target_uri, 'associationIds': job_uuid } } else: return HttpResponse('Unexpected event_type', status=400) # confirm that there is a corresponding running agave job before sending notification try: user = get_user_model().objects.get(username=job_owner) agave = user.agave_oauth.client job_data = agave.jobs.get(jobId=job_uuid) if job_data['owner'] != job_owner or job_data["status"] in [ 'FINISHED', 'FAILED', 'STOPPED' ]: LOGGER.error( "Agave job (owner='{}', status='{}) for this event (owner='{}') is not valid" .format(job_data['owner'], job_data['status'], job_owner)) raise ApiException( "Unable to find a related valid job for this interactive event." ) except (HTTPError, AgaveException, ApiException) as e: LOGGER.exception( "Could not find valid corresponding Agave job for interactive event" ) return HttpResponse({'message': e.message}, content_type='application/json', status=400) n = Notification.objects.create(**event_data) n.save() # create metadata for Interactive connection and save to agave metadata try: agave_job_meta = { 'name': 'interactiveJobDetails', 'value': event_data, 'associationIds': [job_uuid], } user = get_user_model().objects.get(username=job_owner) agave = user.agave_oauth.client agave.meta.addMetadata(body=json.dumps(agave_job_meta)) except (HTTPError, AgaveException) as e: LOGGER.exception( 'Could not add interactive connection data to metadata') return HttpResponse({'message': e.message}, content_type='application/json', status=400) return HttpResponse('OK')
def upload(self, *args, **kwargs): return ApiException(messsage='Invalid Action', status=400)