def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
file_type, file_id = self.parse_file_id(file_id=src_file_id)
googledrive_item = self.googledrive_api.files().get(
fileId=file_id, fields="name").execute()
n = Notification(
event_type='data',
status=Notification.INFO,
operation='googledrive_download_start',
message='Starting copy of {} {} from Google Drive.'.format(
file_type, googledrive_item['name']),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, filename: {}, src_file_id: {}, dest_file_id: {}'
.format(username, googledrive_item['name'], src_file_id,
dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = self.download_file(
file_id, dest_real_path, username)
if downloaded_file_path is None:
return None
elif file_type == 'folder':
downloaded_file_path = self.download_folder(
file_id, dest_real_path, username)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='googledrive_download_end',
message='{} "{}" was copied from Google Drive successfully!'.
format(file_type.capitalize(), googledrive_item['name']),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
reindex_agave.apply_async(kwargs={
'username':
user.username,
'file_id':
'{}/{}'.format(agave_system_id, agave_file_path)
},
queue='indexing')
except Exception as e:
logger.exception('Unexpected task failure: googledrive_copy',
extra={
'username': username,
'file_id': src_file_id,
'dest_file_id': dest_file_id,
'error_type': type(e),
'error': str(e)
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_copy_error',
message='We were unable to copy the file from Google Drive. '
'Please try again...',
user=username,
extra={'path': googledrive_item['name']})
n.save()
raise
def upload(self, username, src_file_id, dest_folder_id):
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='googledrive_upload_start',
message='Uploading file %s to Google Drive.' %
(src_file_id, ),
user=username,
extra={})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split src ination file path
src_file_path_comps = src_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = src_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if src_file_path_comps[1:]:
src_real_path = os.path.join(*src_file_path_comps[1:])
else:
src_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
src_real_path = os.path.join(base_mounted_path, project_dir,
src_real_path.strip('/'))
else:
src_real_path = os.path.join(base_mounted_path,
src_real_path.strip('/'))
logger.debug('src_real_path: {}'.format(src_real_path))
logger.debug('dest_folder_id:{}'.format(dest_folder_id))
if dest_folder_id == '':
dest_folder_id = 'root'
file_type, folder_id = self.parse_file_id(
file_id=dest_folder_id.strip('/'))
if os.path.isfile(src_real_path):
self.upload_file(folder_id, src_real_path)
elif os.path.isdir(src_real_path):
self.upload_directory(folder_id, src_real_path)
else:
logger.error('Unable to upload %s: file does not exist!',
src_real_path)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='googledrive_upload_end',
message='File "%s" was copied to Google Drive successfully!' %
(src_file_id, ),
user=username,
extra={})
n.save()
except Exception as err:
logger.exception('Unexpected task failure: googledrive_upload',
extra={
'username': username,
'src_file_id': src_file_id,
'dst_file_id': dest_folder_id,
'error': err
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_upload_error',
message=
'We were unable to upload the specified file to Google Drive. '
'Please try again...',
user=username,
extra={})
n.save()
raise
def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download file %s from box.' %
(src_file_id, ),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, src_file_id: {}, dest_file_id: {}'.format(
username, src_file_id, dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
# box_fm = BoxFileManager(user)
box_file_type, box_file_id = self.parse_file_id(
file_id=src_file_id)
levels = 0
downloaded_file_path = None
if box_file_type == 'file':
downloaded_file_path = self.download_file(
box_file_id, dest_real_path)
levels = 1
elif box_file_type == 'folder':
downloaded_file_path = self.download_folder(
box_file_id, dest_real_path)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s has been copied from box successfully!' %
(src_file_id, ),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
if not agave_file_path.startswith('/'):
agave_file_path = '/' + agave_file_path
agave_indexer.apply_async(kwargs={
'username':
user.username,
'systemId':
agave_system_id,
'filePath':
os.path.dirname(agave_file_path),
'recurse':
False
},
queue='indexing')
agave_indexer.apply_async(kwargs={
'systemId': agave_system_id,
'filePath': agave_file_path,
'recurse': True
},
routing_key='indexing')
except:
logger.exception('Unexpected task failure: box_download',
extra={
'username': username,
'box_file_id': src_file_id,
'dest_file_id': dest_file_id
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={})
n.save()
raise
