def copy(self, username, src_file_id, dest_file_id, **kwargs): try: file_type, file_id = self.parse_file_id(file_id=src_file_id) googledrive_item = self.googledrive_api.files().get( fileId=file_id, fields="name").execute() n = Notification( event_type='data', status=Notification.INFO, operation='googledrive_download_start', message='Starting copy of {} {} from Google Drive.'.format( file_type, googledrive_item['name']), user=username, extra={}) n.save() logger.debug( 'username: {}, filename: {}, src_file_id: {}, dest_file_id: {}' .format(username, googledrive_item['name'], src_file_id, dest_file_id)) user = get_user_model().objects.get(username=username) from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager # Initialize agave filemanager agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client) # Split destination file path dest_file_path_comps = dest_file_id.strip('/').split('/') # If it is an agave file id then the first component is a system id agave_system_id = dest_file_path_comps[0] # Start construction the actual real path into the NSF mount if dest_file_path_comps[1:]: dest_real_path = os.path.join(*dest_file_path_comps[1:]) else: dest_real_path = '/' # Get what the system id maps to base_mounted_path = agave_fm.base_mounted_path(agave_system_id) # Add actual path if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) dest_real_path = os.path.join(base_mounted_path, project_dir, dest_real_path.strip('/')) else: dest_real_path = os.path.join(base_mounted_path, dest_real_path.strip('/')) logger.debug('dest_real_path: {}'.format(dest_real_path)) downloaded_file_path = None if file_type == 'file': downloaded_file_path = self.download_file( file_id, dest_real_path, username) if downloaded_file_path is None: return None elif file_type == 'folder': downloaded_file_path = self.download_folder( file_id, dest_real_path, username) n = Notification( event_type='data', status=Notification.SUCCESS, operation='googledrive_download_end', message='{} "{}" was copied from Google Drive successfully!'. format(file_type.capitalize(), googledrive_item['name']), user=username, extra={}) n.save() if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) project_dir = os.path.join(base_mounted_path.strip('/'), project_dir) agave_file_path = downloaded_file_path.replace( project_dir, '', 1).strip('/') else: agave_file_path = downloaded_file_path.replace( base_mounted_path, '', 1).strip('/') reindex_agave.apply_async(kwargs={ 'username': user.username, 'file_id': '{}/{}'.format(agave_system_id, agave_file_path) }, queue='indexing') except Exception as e: logger.exception('Unexpected task failure: googledrive_copy', extra={ 'username': username, 'file_id': src_file_id, 'dest_file_id': dest_file_id, 'error_type': type(e), 'error': str(e) }) n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_copy_error', message='We were unable to copy the file from Google Drive. ' 'Please try again...', user=username, extra={'path': googledrive_item['name']}) n.save() raise
def upload(self, username, src_file_id, dest_folder_id): try: n = Notification(event_type='data', status=Notification.INFO, operation='googledrive_upload_start', message='Uploading file %s to Google Drive.' % (src_file_id, ), user=username, extra={}) n.save() user = get_user_model().objects.get(username=username) from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager # Initialize agave filemanager agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client) # Split src ination file path src_file_path_comps = src_file_id.strip('/').split('/') # If it is an agave file id then the first component is a system id agave_system_id = src_file_path_comps[0] # Start construction the actual real path into the NSF mount if src_file_path_comps[1:]: src_real_path = os.path.join(*src_file_path_comps[1:]) else: src_real_path = '/' # Get what the system id maps to base_mounted_path = agave_fm.base_mounted_path(agave_system_id) # Add actual path if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) src_real_path = os.path.join(base_mounted_path, project_dir, src_real_path.strip('/')) else: src_real_path = os.path.join(base_mounted_path, src_real_path.strip('/')) logger.debug('src_real_path: {}'.format(src_real_path)) logger.debug('dest_folder_id:{}'.format(dest_folder_id)) if dest_folder_id == '': dest_folder_id = 'root' file_type, folder_id = self.parse_file_id( file_id=dest_folder_id.strip('/')) if os.path.isfile(src_real_path): self.upload_file(folder_id, src_real_path) elif os.path.isdir(src_real_path): self.upload_directory(folder_id, src_real_path) else: logger.error('Unable to upload %s: file does not exist!', src_real_path) n = Notification( event_type='data', status=Notification.SUCCESS, operation='googledrive_upload_end', message='File "%s" was copied to Google Drive successfully!' % (src_file_id, ), user=username, extra={}) n.save() except Exception as err: logger.exception('Unexpected task failure: googledrive_upload', extra={ 'username': username, 'src_file_id': src_file_id, 'dst_file_id': dest_folder_id, 'error': err }) n = Notification( event_type='data', status=Notification.ERROR, operation='googledrive_upload_error', message= 'We were unable to upload the specified file to Google Drive. ' 'Please try again...', user=username, extra={}) n.save() raise
def copy(self, username, src_file_id, dest_file_id, **kwargs): try: n = Notification(event_type='data', status=Notification.INFO, operation='box_download_start', message='Starting download file %s from box.' % (src_file_id, ), user=username, extra={}) n.save() logger.debug( 'username: {}, src_file_id: {}, dest_file_id: {}'.format( username, src_file_id, dest_file_id)) user = get_user_model().objects.get(username=username) from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager # Initialize agave filemanager agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client) # Split destination file path dest_file_path_comps = dest_file_id.strip('/').split('/') # If it is an agave file id then the first component is a system id agave_system_id = dest_file_path_comps[0] # Start construction the actual real path into the NSF mount if dest_file_path_comps[1:]: dest_real_path = os.path.join(*dest_file_path_comps[1:]) else: dest_real_path = '/' # Get what the system id maps to base_mounted_path = agave_fm.base_mounted_path(agave_system_id) # Add actual path if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) dest_real_path = os.path.join(base_mounted_path, project_dir, dest_real_path.strip('/')) else: dest_real_path = os.path.join(base_mounted_path, dest_real_path.strip('/')) logger.debug('dest_real_path: {}'.format(dest_real_path)) # box_fm = BoxFileManager(user) box_file_type, box_file_id = self.parse_file_id( file_id=src_file_id) levels = 0 downloaded_file_path = None if box_file_type == 'file': downloaded_file_path = self.download_file( box_file_id, dest_real_path) levels = 1 elif box_file_type == 'folder': downloaded_file_path = self.download_folder( box_file_id, dest_real_path) n = Notification( event_type='data', status=Notification.SUCCESS, operation='box_download_end', message='File %s has been copied from box successfully!' % (src_file_id, ), user=username, extra={}) n.save() if re.search(r'^project-', agave_system_id): project_dir = agave_system_id.replace('project-', '', 1) project_dir = os.path.join(base_mounted_path.strip('/'), project_dir) agave_file_path = downloaded_file_path.replace( project_dir, '', 1).strip('/') else: agave_file_path = downloaded_file_path.replace( base_mounted_path, '', 1).strip('/') if not agave_file_path.startswith('/'): agave_file_path = '/' + agave_file_path agave_indexer.apply_async(kwargs={ 'username': user.username, 'systemId': agave_system_id, 'filePath': os.path.dirname(agave_file_path), 'recurse': False }, queue='indexing') agave_indexer.apply_async(kwargs={ 'systemId': agave_system_id, 'filePath': agave_file_path, 'recurse': True }, routing_key='indexing') except: logger.exception('Unexpected task failure: box_download', extra={ 'username': username, 'box_file_id': src_file_id, 'dest_file_id': dest_file_id }) n = Notification( event_type='data', status=Notification.ERROR, operation='box_download_error', message='We were unable to get the specified file from box. ' 'Please try again...', user=username, extra={}) n.save() raise