def get_download_url(self, file_id, **kwargs): file_type, file_id = self.parse_file_id(file_id) if file_type == 'file': googledrive_file = self.googledrive_api.files().get( fileId=file_id, fields="webContentLink, ownedByMe, mimeType, name").execute() # if user owns file, make file viewable by anyone with link to allow user to download the file if googledrive_file['ownedByMe']: file_pems = self.googledrive_api.permissions().list( fileId=file_id).execute()['permissions'] if 'anyoneWithLink' not in [pem['id'] for pem in file_pems]: body = {'role': 'reader', 'type': 'anyone'} update_pems = self.googledrive_api.permissions().create( fileId=file_id, body=body).execute() if 'webContentLink' in googledrive_file: return {'href': googledrive_file['webContentLink']} else: n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_download_error', message= 'Downloading Google-type files is currently unsupported. Convert the file to' ' a standard format and try again.', user=kwargs['username'], extra={'path': googledrive_file['name'] }) # show file in Notification n.save() return None return None
def post(self, request, *args, **kwargs): JOB_EVENT='job' logger.debug('request body: {}'.format(request.body)) try: notification = json.loads(request.body) job_name = notification['name'] status = notification['status'] event = request.GET.get('event') job_id = request.GET.get('job_id') job_owner = notification['owner'] archive_path = notification['archivePath'] except ValueError as e: #for testing ->used when mocking agave notification job_name = request.POST.get('job_name') status = request.POST.get('status') event = request.POST.get('event') job_id = request.POST.get('job_id') job_owner = request.POST.get('job_owner') archive_path = request.POST.get('archivePath') logger.info('job_name: {}'.format(job_name)) logger.info('event: {}'.format(event)) logger.info('job_id: {}'.format(job_id)) n = Notification(event_type = 'job', status = 'INFO', operation = event, message = 'Job %s changed to %s' % (job_name, status), user = job_owner, extra = {'archive_path': archive_path, 'job_id': job_id}) n.save() return HttpResponse('OK')
def download_folder(self, folder_id, download_path, username): """ Recursively download the folder for folder_id, and all of its contents, to the given download_path. :param folder_id: :param download_path: :return: """ googledrive_folder = self.googledrive_api.files().get(fileId=folder_id, fields="name").execute() # convert utf-8 chars safe_dirname = googledrive_folder['name'].encode(sys.getfilesystemencoding(), 'ignore') directory_path = os.path.join(download_path, safe_dirname) logger.debug('Creating directory %s <= googledrive://folder/%s', directory_path, folder_id) try: os.mkdir(directory_path, 0o0755) except OSError as e: if e.errno == 17: # directory already exists? pass else: logger.exception('Error creating directory: %s', directory_path) raise items = self.googledrive_api.files().list(q="'{}' in parents and trashed=False".format(folder_id)).execute() for item in items['files']: if item['mimeType'] == 'application/vnd.google-apps.folder': self.download_folder(item['id'], directory_path, username) else: try: self.download_file(item['id'], directory_path, username) logger.info('Google File download complete: {}'.format('/'.join([directory_path, item['id']]))) except Exception as e: logger.exception('Unexpected task failure: googledrive_download', extra={ 'username': username, 'file_id': item['id'], 'dest_file_id': '/'.join([directory_path, item['id']]), 'error_type': type(e), 'error': str(e) }) n = Notification(event_type='data', status=Notification.ERROR, operation='googledrive_download_error', message='We were unable to download file from Google Drive. ' 'Please try again...', user=username, extra={'path': item['name']}) n.save() raise return directory_path
def post(self, request, *args, **kwargs): body_json = json.loads(request.body) nid = body_json['id'] read = body_json['read'] n = Notification.get(id = nid) n.read = read n.save()
def delete(self, request, *args, **kwargs): body_json = json.loads(request.body) nid = body_json['id'] deleted = body_json['deleted'] n = Notification.get(id = nid) n.deleted = deleted n.save()
def copy(self, username, src_file_id, dest_file_id, **kwargs): try: n = Notification(event_type='data', status=Notification.INFO, operation='box_download_start', message='Starting download file %s from box.' % (src_file_id, ), user=username, extra={}) n.save() logger.debug( 'username: {}, src_file_id: {}, dest_file_id: {}'.format( username, src_file_id, dest_file_id)) user = get_user_model().objects.get(username=username) from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager # Initialize agave filemanager agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client) # Split destination file path dest_file_path_comps = dest_file_id.strip('/').split('/') # If it is an agave file id then the first component is a system id agave_system_id = dest_file_path_comps[0] # Start construction the actual real path into the NSF mount if dest_file_path_comps[1:]: dest_real_path = os.path.join(*dest_file_path_comps[1:]) else: dest_real_path = '/' # Get what the system id maps to base_mounted_path = agave_fm.base_mounted_path(agave_system_id) # Add actual path if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) dest_real_path = os.path.join(base_mounted_path, project_dir, dest_real_path.strip('/')) else: dest_real_path = os.path.join(base_mounted_path, dest_real_path.strip('/')) logger.debug('dest_real_path: {}'.format(dest_real_path)) # box_fm = BoxFileManager(user) box_file_type, box_file_id = self.parse_file_id( file_id=src_file_id) levels = 0 downloaded_file_path = None if box_file_type == 'file': downloaded_file_path = self.download_file( box_file_id, dest_real_path) levels = 1 elif box_file_type == 'folder': downloaded_file_path = self.download_folder( box_file_id, dest_real_path) n = Notification( event_type='data', status=Notification.SUCCESS, operation='box_download_end', message='File %s has been copied from box successfully!' % (src_file_id, ), user=username, extra={}) n.save() if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) project_dir = os.path.join(base_mounted_path.strip('/'), project_dir) agave_file_path = downloaded_file_path.replace( project_dir, '', 1).strip('/') else: agave_file_path = downloaded_file_path.replace( base_mounted_path, '', 1).strip('/') if not agave_file_path.startswith('/'): agave_file_path = '/' + agave_file_path agave_indexer.apply_async(kwargs={ 'username': user.username, 'systemId': agave_system_id, 'filePath': os.path.dirname(agave_file_path), 'recurse': False }, queue='indexing') agave_indexer.apply_async(kwargs={ 'systemId': agave_system_id, 'filePath': agave_file_path, 'recurse': True }, routing_key='indexing') except: logger.exception('Unexpected task failure: box_download', extra={ 'username': username, 'box_file_id': src_file_id, 'dest_file_id': dest_file_id }) n = Notification( event_type='data', status=Notification.ERROR, operation='box_download_error', message='We were unable to get the specified file from box. ' 'Please try again...', user=username, extra={}) n.save() raise
def upload(self, username, src_file_id, dest_folder_id): try: n = Notification(event_type='data', status=Notification.INFO, operation='googledrive_upload_start', message='Uploading file %s to Google Drive.' % (src_file_id, ), user=username, extra={}) n.save() user = get_user_model().objects.get(username=username) from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager # Initialize agave filemanager agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client) # Split src ination file path src_file_path_comps = src_file_id.strip('/').split('/') # If it is an agave file id then the first component is a system id agave_system_id = src_file_path_comps[0] # Start construction the actual real path into the NSF mount if src_file_path_comps[1:]: src_real_path = os.path.join(*src_file_path_comps[1:]) else: src_real_path = '/' # Get what the system id maps to base_mounted_path = agave_fm.base_mounted_path(agave_system_id) # Add actual path if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) src_real_path = os.path.join(base_mounted_path, project_dir, src_real_path.strip('/')) else: src_real_path = os.path.join(base_mounted_path, src_real_path.strip('/')) logger.debug('src_real_path: {}'.format(src_real_path)) logger.debug('dest_folder_id:{}'.format(dest_folder_id)) if dest_folder_id == '': dest_folder_id = 'root' file_type, folder_id = self.parse_file_id( file_id=dest_folder_id.strip('/')) if os.path.isfile(src_real_path): self.upload_file(folder_id, src_real_path) elif os.path.isdir(src_real_path): self.upload_directory(folder_id, src_real_path) else: logger.error('Unable to upload %s: file does not exist!', src_real_path) n = Notification( event_type='data', status=Notification.SUCCESS, operation='googledrive_upload_end', message='File "%s" was copied to Google Drive successfully!' % (src_file_id, ), user=username, extra={}) n.save() except Exception as err: logger.exception('Unexpected task failure: googledrive_upload', extra={ 'username': username, 'src_file_id': src_file_id, 'dst_file_id': dest_folder_id, 'error': err }) n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_upload_error', message= 'We were unable to upload the specified file to Google Drive. ' 'Please try again...', user=username, extra={}) n.save() raise
def download_file(self, file_id, download_directory_path, username): """ Downloads the file for file_id to the given download_path. :param file_id: :param download_directory_path: :return: the full path to the downloaded file """ googledrive_file = self.googledrive_api.files().get( fileId=file_id, fields="name, mimeType").execute() # convert utf-8 chars safe_filename = googledrive_file['name'].encode( sys.getfilesystemencoding(), 'ignore') file_download_path = os.path.join(download_directory_path, safe_filename) logger.debug('Download file %s <= googledrive://file/%s', file_download_path, file_id) if 'vnd.google-apps' in googledrive_file['mimeType']: # if googledrive_file['mimeType'] == 'application/vnd.google-apps.spreadsheet': # mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet' # elif googledrive_file['mimeType'] == 'application/vnd.google-apps.document': # mimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' # elif googledrive_file['mimeType'] == 'application/vnd.google-apps.presentation': # mimeType = 'application/vnd.openxmlformats-officedocument.presentationml.presentation' # else: n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_download_error', message= 'Copying Google-type files is currently unsupported. Export the file to' ' a standard format and try again.', user=username, extra={ 'path': "'{}' of type {}".format(googledrive_file['name'], googledrive_file['mimeType']) }) n.save() return None request = self.googledrive_api.files().get_media(fileId=file_id) # Incremental Partial Download fh = io.FileIO(file_download_path, 'wb') downloader = MediaIoBaseDownload(fh, request) done = False backoff_attempts = 0 while done is False: try: status, done = downloader.next_chunk() # logger.debug('status: {} percent'.format(status.progress())) except HttpError as e: # Incremental backoff for exceeding google api rate limit if "Rate Limit Exceeded" in str(e): logger.debug('RATE LIMIT EXCEEDED') backoff_attempts += 1 time.sleep(backoff_attempts) if backoff_attempts > 10: n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_download_error', message= 'Rate Limit Exceeded. Try again after a few minutes for this file.', user=username, extra={ 'path': "{}".format(googledrive_file['name']) }) n.save() return None elif "Only files with binary content can be downloaded" in str( e): n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_download_error', message= 'Only files with binary content can be downloaded. Convert the file to' ' a standard format and try again.', user=username, extra={ 'path': "'{}' of type {}".format( googledrive_file['name'], googledrive_file['mimeType']) }) n.save() return None else: raise fh.close() return file_download_path
def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_file_id): """ :param self: :param username: :param src_resource: should be 'agave' :param src_file_id: the agave file id :param dest_resource: should be 'box' :param dest_file_id: the box id of the destination folder :return: """ logger.debug('Importing file %s://%s for user %s to %s://%s' % ( src_resource, src_file_id, username, dest_resource, dest_file_id)) try: n = Notification(event_type = 'data', status = Notification.INFO, operation = 'box_upload_start', message = 'Starting import file %s into box.' % src_file_id, user = username, extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), src_resource, src_file_id)}) n.save() user = get_user_model().objects.get(username=username) from designsafe.apps.api.data import BoxFileManager box_fm = BoxFileManager(user) file_type, file_id = box_fm.parse_file_id(dest_file_id) if file_type != 'folder': logger.warn('Cannot import to a file destination!') raise Exception('You can only import files to a folder!', status=400, extra={ 'src_resource': src_resource, 'src_file_id': src_file_id, 'dest_resource': dest_resource, 'dest_file_id': dest_file_id, }) if src_resource == 'agave' or src_resource == 'public': try: logger.debug('Starting upload to Box...') from designsafe.apps.api.data import lookup_file_manager agave_fm = lookup_file_manager(src_resource)(user) file_real_path = agave_fm.get_file_real_path(src_file_id) if os.path.isfile(file_real_path): box_upload_file(box_fm, file_id, file_real_path) elif os.path.isdir(file_real_path): box_upload_directory(box_fm, file_id, file_real_path) else: logger.error('Unable to upload %s: file does not exist!', file_real_path) except: logger.exception('Upload to Box failed!') logger.debug('Box upload task complete.') n = Notification(event_type = 'data', status = Notification.SUCCESS, operation = 'box_upload_end', message = 'File(s) %s succesfully uploaded into box!' % src_file_id, user = username, extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), dest_resource, dest_file_id)}) n.save() except: logger.exception('Unexpected task failure: box_upload', extra={ 'username': username, 'src_resource': src_resource, 'src_file_id': src_file_id, 'dest_resource': dest_resource, 'dest_file_id': dest_file_id, }) n = Notification(event_type = 'data', status = Notification.ERROR, operation = 'box_download_error', message = 'We were unable to get the specified file from box. Please try again...', user = username, extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), src_resource, src_file_id)}) n.save()
def copy(self, username, src_file_id, dest_file_id, **kwargs): try: file_type, file_id = self.parse_file_id(file_id=src_file_id) googledrive_item = self.googledrive_api.files().get( fileId=file_id, fields="name").execute() n = Notification( event_type='data', status=Notification.INFO, operation='googledrive_download_start', message='Starting copy of {} {} from Google Drive.'.format( file_type, googledrive_item['name']), user=username, extra={}) n.save() logger.debug( 'username: {}, filename: {}, src_file_id: {}, dest_file_id: {}' .format(username, googledrive_item['name'], src_file_id, dest_file_id)) user = get_user_model().objects.get(username=username) from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager # Initialize agave filemanager agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client) # Split destination file path dest_file_path_comps = dest_file_id.strip('/').split('/') # If it is an agave file id then the first component is a system id agave_system_id = dest_file_path_comps[0] # Start construction the actual real path into the NSF mount if dest_file_path_comps[1:]: dest_real_path = os.path.join(*dest_file_path_comps[1:]) else: dest_real_path = '/' # Get what the system id maps to base_mounted_path = agave_fm.base_mounted_path(agave_system_id) # Add actual path if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) dest_real_path = os.path.join(base_mounted_path, project_dir, dest_real_path.strip('/')) else: dest_real_path = os.path.join(base_mounted_path, dest_real_path.strip('/')) logger.debug('dest_real_path: {}'.format(dest_real_path)) downloaded_file_path = None if file_type == 'file': downloaded_file_path = self.download_file( file_id, dest_real_path, username) if downloaded_file_path is None: return None elif file_type == 'folder': downloaded_file_path = self.download_folder( file_id, dest_real_path, username) n = Notification( event_type='data', status=Notification.SUCCESS, operation='googledrive_download_end', message='{} "{}" was copied from Google Drive successfully!'. format(file_type.capitalize(), googledrive_item['name']), user=username, extra={}) n.save() if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) project_dir = os.path.join(base_mounted_path.strip('/'), project_dir) agave_file_path = downloaded_file_path.replace( project_dir, '', 1).strip('/') else: agave_file_path = downloaded_file_path.replace( base_mounted_path, '', 1).strip('/') reindex_agave.apply_async(kwargs={ 'username': user.username, 'file_id': '{}/{}'.format(agave_system_id, agave_file_path) }, queue='indexing') except Exception as e: logger.exception('Unexpected task failure: googledrive_copy', extra={ 'username': username, 'file_id': src_file_id, 'dest_file_id': dest_file_id, 'error_type': type(e), 'error': str(e) }) n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_copy_error', message='We were unable to copy the file from Google Drive. ' 'Please try again...', user=username, extra={'path': googledrive_item['name']}) n.save() raise
def share_agave(self, username, file_id, permissions, recursive): try: # n = Notification(event_type = 'data', # status = 'INFO', # operation = 'share_initializing', # message = 'File sharing is initializing. Please wait...', # user = username, # extra = {'target_path': reverse('designsafe_data:data_depot', # args=['agave', file_id])}) # n.save() user = get_user_model().objects.get(username=username) from designsafe.apps.api.data import AgaveFileManager from designsafe.apps.api.data.agave.file import AgaveFile from designsafe.apps.api.data.agave.elasticsearch.documents import Object agave_fm = AgaveFileManager(user) system_id, file_user, file_path = agave_fm.parse_file_id(file_id) f = AgaveFile.from_file_path(system_id, username, file_path, agave_client=agave_fm.agave_client) f_dict = f.to_dict() n = Notification(event_type = 'data', status = 'INFO', operation = 'share_initializing', message = 'File sharing is initializing. Please wait...', user = username, extra = f_dict) n.save() f.share(permissions, recursive) #reindex_agave.apply_async(args=(self.username, file_id)) # self.indexer.index(system, file_path, file_user, pems_indexing=True) esf = Object.from_file_path(system_id, username, file_path) esf.share(username, permissions, recursive) # Notify owner share completed n = Notification(event_type = 'data', status = 'SUCCESS', operation = 'share_finished', message = 'File permissions were updated successfully.', user = username, extra = f_dict) n.save() # Notify users they have new shared files for pem in permissions: if pem['permission'] != 'NONE': message = '%s shared some files with you.' % user.get_full_name() n = Notification(event_type = 'data', status = 'SUCCESS', operation = 'share_finished', message = message, user = pem['user_to_share'], extra = f_dict) n.save() except: logger.error('Error sharing file/folder', exc_info=True, extra = { 'username': username, 'file_id': file_id, 'permissions': permissions }) n = Notification(event_type='data', status=Notification.ERROR, operation='share_error', message='We were unable to share the specified folder/file(s). ' 'Please try again...', user=username, extra={'system': system_id, 'path': file_path }) n.save()
def copy_public_to_mydata(self, username, src_resource, src_file_id, dest_resource, dest_file_id): logger.debug('Scheduled copy of files from %s://%s to %s://%s', src_resource, src_file_id, dest_resource, dest_file_id) try: n = Notification(event_type = 'data', status = 'INFO', operation = 'copy_public_to_mydata_start', message = 'Copying folder/files %s from public data to your private data. Please wait...' % (src_file_id, ), user = username, extra={ 'system': dest_resource, 'id': dest_file_id, }) # extra = {'target_path': '%s%s' %(reverse('designsafe_data:data_depot'), src_file_id)}) n.save() notify_status = 'SUCCESS' from designsafe.apps.api.data import lookup_file_manager source_fm_cls = lookup_file_manager(src_resource) dest_fm_cls = lookup_file_manager(dest_resource) if source_fm_cls and dest_fm_cls: user = get_user_model().objects.get(username=username) source_fm = source_fm_cls(user) dest_fm = dest_fm_cls(user) source_real_path = source_fm.get_file_real_path(src_file_id) dirname = os.path.basename(source_real_path) dest_real_path = os.path.join(dest_fm.get_file_real_path(dest_file_id), dirname) if os.path.isdir(source_real_path): shutil.copytree(source_real_path, dest_real_path) elif os.path.isfile(source_real_path): shutil.copy(source_real_path, dest_real_path) else: notify_status = 'ERROR' logger.error('The request copy source=%s does not exist!', src_resource) system, username, path = dest_fm.parse_file_id(dest_file_id) dest_fm.indexer.index(system, path, username, levels = 1) n = Notification(event_type = 'data', status = notify_status, operation = 'copy_public_to_mydata_end', message = 'Files were copied to your private data.', user = username, extra={ 'system': dest_resource, 'id': dest_file_id, }) # extra = {'target_path': '%s%s' %(reverse('designsafe_data:data_depot'), dest_file_id)}) n.save() else: logger.error('Unable to load file managers for both source=%s and destination=%s', src_resource, dest_resource) n = Notification(event_type = 'data', status = 'ERROR', operation = 'copy_public_to_mydata_error', message = '''There was an error copying the files to your public data. Plese try again.''', user = username, extra={ 'system': dest_resource, 'id': dest_file_id, }) # extra = {}) n.save() except: logger.exception('Unexpected task failure') n = Notification(event_type = 'data', status = 'ERROR', operation = 'copy_public_to_mydata_error', message = '''There was an error copying the files to your public data. Plese try again.''', user = username, extra={ 'system': dest_resource, 'id': dest_file_id, }) # extra = {}) n.save()
def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_file_id): """ :param self: :param username: :param src_resource: should be 'agave' :param src_file_id: the agave file id :param dest_resource: should be 'box' :param dest_file_id: the box id of the destination folder :return: """ logger.debug('Importing file %s://%s for user %s to %s://%s' % ( src_resource, src_file_id, username, dest_resource, dest_file_id)) try: n = Notification(event_type = 'data', status = Notification.INFO, operation = 'box_upload_start', message = 'Starting import of file %s into box.' % src_file_id, user = username, # extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), src_resource, src_file_id)}) extra={'id': src_file_id}) n.save() user = get_user_model().objects.get(username=username) from designsafe.apps.api.data import BoxFileManager box_fm = BoxFileManager(user) file_type, file_id = box_fm.parse_file_id(dest_file_id) if file_type != 'folder': logger.warn('Cannot import to a file destination!') raise Exception('You can only import files to a folder!', status=400, extra={ 'src_resource': src_resource, 'src_file_id': src_file_id, 'dest_resource': dest_resource, 'dest_file_id': dest_file_id, }) if src_resource == 'agave' or src_resource == 'public': try: logger.debug('Starting upload to Box...') from designsafe.apps.api.data import lookup_file_manager agave_fm = lookup_file_manager(src_resource)(user) file_real_path = agave_fm.get_file_real_path(src_file_id) if os.path.isfile(file_real_path): box_upload_file(box_fm, file_id, file_real_path) elif os.path.isdir(file_real_path): box_upload_directory(box_fm, file_id, file_real_path) else: logger.error('Unable to upload %s: file does not exist!', file_real_path) except: logger.exception('Upload to Box failed!') logger.debug('Box upload task complete.') n = Notification(event_type = 'data', status = Notification.SUCCESS, operation = 'box_upload_end', message = 'File(s) %s succesfully uploaded into box!' % src_file_id, user = username, extra={'id': src_file_id}) # extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), dest_resource, dest_file_id)}) n.save() except: logger.exception('Unexpected task failure: box_upload', extra={ 'username': username, 'src_resource': src_resource, 'src_file_id': src_file_id, 'dest_resource': dest_resource, 'dest_file_id': dest_file_id, }) n = Notification(event_type = 'data', status = Notification.ERROR, operation = 'box_download_error', message = 'We were unable to get the specified file from box. Please try again...', user = username, extra={ 'src_resource': src_resource, 'id': src_file_id, 'dest_resource': dest_resource, 'dest_file_id': dest_file_id, }) # extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), src_resource, src_file_id)}) n.save()
def box_download(self, username, src_resource, src_file_id, dest_resource, dest_file_id): """ :param self: :param username: :param src_resource: :param src_file_id: :param dest_resource: :param dest_file_id: :return: """ logger.debug('Downloading %s://%s for user %s to %s://%s', src_resource, src_file_id, username, dest_resource, dest_file_id) try: target_path = reverse('designsafe_data:data_depot', args=[src_resource, src_file_id]) n = Notification(event_type='data', status=Notification.INFO, operation='box_download_start', message='Starting download of file %s from box.' % (src_file_id,), user=username, # extra={'target_path': target_path}) extra={'id': dest_file_id} ) n.save() user = get_user_model().objects.get(username=username) from designsafe.apps.api.data import BoxFileManager, AgaveFileManager agave_fm = AgaveFileManager(user) dest_real_path = agave_fm.get_file_real_path(dest_file_id) box_fm = BoxFileManager(user) file_type, file_id = box_fm.parse_file_id(src_file_id) levels = 0 downloaded_file_path = None if file_type == 'file': downloaded_file_path = box_download_file(box_fm, file_id, dest_real_path) levels = 1 elif file_type == 'folder': downloaded_file_path = box_download_folder(box_fm, file_id, dest_real_path) if downloaded_file_path is not None: downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path) system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id) agave_fm.indexer.index(system_id, file_path, file_user, full_indexing=True, pems_indexing=True, index_full_path=True, levels=levels) target_path = reverse('designsafe_data:data_depot', args=[dest_resource, dest_file_id]) n = Notification(event_type='data', status=Notification.SUCCESS, operation='box_download_end', message='File %s was copied from box successfully!' % (src_file_id, ), user=username, extra={'id': dest_file_id} ) n.save() except: logger.exception('Unexpected task failure: box_download', extra={ 'username': username, 'box_file_id': src_file_id, 'to_resource': dest_resource, 'dest_file_id': dest_file_id }) target_path = reverse('designsafe_data:data_depot', args=[src_resource, src_file_id]) n = Notification(event_type='data', status=Notification.ERROR, operation='box_download_error', message='We were unable to get the specified file from box. ' 'Please try again...', user=username, extra={'system': dest_resource, 'id': dest_file_id, 'src_file_id': src_file_id, 'src_resource': src_resource }) n.save()
def box_download(self, username, src_resource, src_file_id, dest_resource, dest_file_id): """ :param self: :param username: :param src_resource: :param src_file_id: :param dest_resource: :param dest_file_id: :return: """ logger.debug('Downloading %s://%s for user %s to %s://%s', src_resource, src_file_id, username, dest_resource, dest_file_id) try: target_path = reverse('designsafe_data:data_browser', args=[src_resource, src_file_id]) n = Notification(event_type='data', status=Notification.INFO, operation='box_download_start', message='Starting download file %s from box.' % (src_file_id,), user=username, extra={'target_path': target_path}) n.save() user = get_user_model().objects.get(username=username) from designsafe.apps.api.data import BoxFileManager, AgaveFileManager agave_fm = AgaveFileManager(user) dest_real_path = agave_fm.get_file_real_path(dest_file_id) box_fm = BoxFileManager(user) file_type, file_id = box_fm.parse_file_id(src_file_id) levels = 0 downloaded_file_path = None if file_type == 'file': downloaded_file_path = box_download_file(box_fm, file_id, dest_real_path) levels = 1 elif file_type == 'folder': downloaded_file_path = box_download_folder(box_fm, file_id, dest_real_path) if downloaded_file_path is not None: downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path) system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id) agave_fm.indexer.index(system_id, file_path, file_user, full_indexing=True, pems_indexing=True, index_full_path=True, levels=levels) target_path = reverse('designsafe_data:data_browser', args=[dest_resource, dest_file_id]) n = Notification(event_type='data', status=Notification.SUCCESS, operation='box_download_end', message='File %s has been copied from box successfully!' % (src_file_id, ), user=username, extra={'target_path': target_path}) n.save() except: logger.exception('Unexpected task failure: box_download', extra={ 'username': username, 'box_file_id': src_file_id, 'to_resource': dest_resource, 'dest_file_id': dest_file_id }) target_path = reverse('designsafe_data:data_browser', args=[src_resource, src_file_id]) n = Notification(event_type='data', status=Notification.ERROR, operation='box_download_error', message='We were unable to get the specified file from box. ' 'Please try again...', user=username, extra={'target_path': target_path}) n.save()
def copy_public_to_mydata(self, username, src_resource, src_file_id, dest_resource, dest_file_id): logger.debug('Scheduled copy of files from %s://%s to %s://%s', src_resource, src_file_id, dest_resource, dest_file_id) try: n = Notification(event_type = 'data', status = 'INFO', operation = 'copy_public_to_mydata_start', message = 'Copying folder/files %s from public data to your private data. Please wait...' % (src_file_id, ), user = username, extra = {'target_path': '%s%s' %(reverse('designsafe_data:data_browser'), src_file_id)}) n.save() notify_status = 'SUCCESS' from designsafe.apps.api.data import lookup_file_manager source_fm_cls = lookup_file_manager(src_resource) dest_fm_cls = lookup_file_manager(dest_resource) if source_fm_cls and dest_fm_cls: user = get_user_model().objects.get(username=username) source_fm = source_fm_cls(user) dest_fm = dest_fm_cls(user) source_real_path = source_fm.get_file_real_path(src_file_id) dirname = os.path.basename(source_real_path) dest_real_path = os.path.join(dest_fm.get_file_real_path(dest_file_id), dirname) if os.path.isdir(source_real_path): shutil.copytree(source_real_path, dest_real_path) elif os.path.isfile(source_real_path): shutil.copy(source_real_path, dest_real_path) else: notify_status = 'ERROR' logger.error('The request copy source=%s does not exist!', src_resource) system, username, path = dest_fm.parse_file_id(dest_file_id) dest_fm.indexer.index(system, path, username, levels = 1) n = Notification(event_type = 'data', status = notify_status, operation = 'copy_public_to_mydata_end', message = 'Files have been copied to your private data.', user = username, extra = {'target_path': '%s%s' %(reverse('designsafe_data:data_browser'), dest_file_id)}) n.save() else: logger.error('Unable to load file managers for both source=%s and destination=%s', src_resource, dest_resource) n = Notification(event_type = 'data', status = 'ERROR', operation = 'copy_public_to_mydata_error', message = '''There was an error copying the files to your public data. Plese try again.''', user = username, extra = {}) n.save() except: logger.exception('Unexpected task failure') n = Notification(event_type = 'data', status = 'ERROR', operation = 'copy_public_to_mydata_error', message = '''There was an error copying the files to your public data. Plese try again.''', user = username, extra = {}) n.save()
def box_resource_download(self, username, src_file_id, dest_file_id): """ :param self: :param username: :param src_file_id: :param dest_file_id: :return: """ logger.debug('Downloading box://%s for user %s to %s', src_file_id, username, dest_file_id) try: n = Notification(event_type='data', status=Notification.INFO, operation='box_download_start', message='Starting download file %s from box.' % (src_file_id,), user=username, extra={}) n.save() logger.debug('username: {}, src_file_id: {}, dest_file_id: {}'.format(username, src_file_id, dest_file_id)) user = get_user_model().objects.get(username=username) from designsafe.apps.api.external_resources.box.filemanager.manager import \ FileManager as BoxFileManager from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager # Initialize agave filemanager agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client) # Split destination file path dest_file_path_comps = dest_file_id.strip('/').split('/') # If it is an agave file id then the first component is a system id agave_system_id = dest_file_path_comps[0] # Start construction the actual real path into the NSF mount if dest_file_path_comps[1:]: dest_real_path = os.path.join(*dest_file_path_comps[1:]) else: dest_real_path = '/' # Get what the system id maps to base_mounted_path = agave_fm.base_mounted_path(agave_system_id) # Add actual path if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) dest_real_path = os.path.join(base_mounted_path, project_dir, dest_real_path.strip('/')) else: dest_real_path = os.path.join(base_mounted_path, dest_real_path.strip('/')) logger.debug('dest_real_path: {}'.format(dest_real_path)) box_fm = BoxFileManager(user) box_file_type, box_file_id = box_fm.parse_file_id(file_id=src_file_id) levels = 0 downloaded_file_path = None if box_file_type == 'file': downloaded_file_path = box_fm.download_file(box_file_id, dest_real_path) levels = 1 elif box_file_type == 'folder': downloaded_file_path = box_fm.download_folder(box_file_id, dest_real_path) #if downloaded_file_path is not None: # downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path) # system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id) n = Notification(event_type='data', status=Notification.SUCCESS, operation='box_download_end', message='File %s has been copied from box successfully!' % (src_file_id, ), user=username, extra={}) n.save() if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) project_dir = os.path.join(base_mounted_path.strip('/'), project_dir) agave_file_path = downloaded_file_path.replace(project_dir, '', 1).strip('/') else: agave_file_path = downloaded_file_path.replace(base_mounted_path, '', 1).strip('/') reindex_agave.apply_async(kwargs={ 'username': user.username, 'file_id': '{}/{}'.format(agave_system_id, agave_file_path) }) except: logger.exception('Unexpected task failure: box_download', extra={ 'username': username, 'box_file_id': src_file_id, 'dest_file_id': dest_file_id }) n = Notification(event_type='data', status=Notification.ERROR, operation='box_download_error', message='We were unable to get the specified file from box. ' 'Please try again...', user=username, extra={}) n.save()
def share_agave(self, username, file_id, permissions, recursive): try: n = Notification(event_type = 'data', status = 'INFO', operation = 'share_initializing', message = 'File sharing is initializing. Please wait...', user = username, extra = {'target_path': reverse('designsafe_data:data_browser', args=['agave', file_id])}) n.save() user = get_user_model().objects.get(username=username) from designsafe.apps.api.data import AgaveFileManager from designsafe.apps.api.data.agave.file import AgaveFile from designsafe.apps.api.data.agave.elasticsearch.documents import Object agave_fm = AgaveFileManager(user) system_id, file_user, file_path = agave_fm.parse_file_id(file_id) f = AgaveFile.from_file_path(system_id, username, file_path, agave_client=agave_fm.agave_client) f.share(permissions, recursive) #reindex_agave.apply_async(args=(self.username, file_id)) # self.indexer.index(system, file_path, file_user, pems_indexing=True) esf = Object.from_file_path(system_id, username, file_path) esf.share(username, permissions, recursive) # Notify owner share completed n = Notification(event_type = 'data', status = 'SUCCESS', operation = 'share_finished', message = 'File permissions were updated successfully.', user = username, extra = {'target_path': reverse('designsafe_data:data_browser', args=['agave', file_id])}) n.save() # Notify users they have new shared files for pem in permissions: if pem['permission'] != 'NONE': message = '%s shared some files with you.' % user.get_full_name() n = Notification(event_type = 'data', status = 'SUCCESS', operation = 'share_finished', message = message, user = pem['user_to_share'], extra = {'target_path': reverse('designsafe_data:data_browser', args=['agave', file_id])}) n.save() except: logger.error('Error sharing file/folder', exc_info=True, extra = { 'username': username, 'file_id': file_id, 'permissions': permissions }) n = Notification(event_type='data', status=Notification.ERROR, operation='share_error', message='We were unable to share the specified folder/file(s). ' 'Please try again...', user=username, extra={}) n.save()