def external_resource_upload(self, username, dest_resource, src_file_id, dest_file_id):
"""
:param self:
:param username:
:param dest_resource:
:param src_file_id:
:param dest_file_id:
:return:
"""
logger.debug('Initializing external_resource_upload. username: %s, src_file_id: %s, dest_resource: %s, dest_file_id: %s ', username, src_file_id, dest_resource, dest_file_id)
from designsafe.apps.api.external_resources.box.filemanager.manager \
import FileManager as BoxFileManager
from designsafe.apps.api.external_resources.dropbox.filemanager.manager \
import FileManager as DropboxFileManager
from designsafe.apps.api.external_resources.googledrive.filemanager.manager \
import FileManager as GoogleDriveFileManager
user = get_user_model().objects.get(username=username)
if dest_resource == 'box':
fmgr = BoxFileManager(user)
elif dest_resource == 'dropbox':
fmgr = DropboxFileManager(user)
elif dest_resource == 'googledrive':
fmgr = GoogleDriveFileManager(user)
logger.debug('fmgr.upload( %s, %s, %s)', username, src_file_id, dest_file_id)
fmgr.upload(username, src_file_id, dest_file_id)
def external_resource_download(self, file_mgr_name, username, src_file_id, dest_file_id):
"""
:param self:
:param username:
:param src_file_id:
:param dest_file_id:
:return:
"""
logger.debug('Downloading %s://%s for user %s to %s',
file_mgr_name, src_file_id, username, dest_file_id)
from designsafe.apps.api.external_resources.box.filemanager.manager \
import FileManager as BoxFileManager
from designsafe.apps.api.external_resources.dropbox.filemanager.manager \
import FileManager as DropboxFileManager
from designsafe.apps.api.external_resources.googledrive.filemanager.manager \
import FileManager as GoogleDriveFileManager
user = get_user_model().objects.get(username=username)
if file_mgr_name == 'box':
fmgr = BoxFileManager(user)
elif file_mgr_name == 'dropbox':
fmgr = DropboxFileManager(user)
elif file_mgr_name == 'googledrive':
fmgr = GoogleDriveFileManager(user)
fmgr.copy(username, src_file_id, dest_file_id)
def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
"""
:param self:
:param username:
:param src_resource: should be 'agave'
:param src_file_id: the agave file id
:param dest_resource: should be 'box'
:param dest_file_id: the box id of the destination folder
:return:
"""
logger.debug('Importing file %s://%s for user %s to %s://%s' % (
src_resource, src_file_id, username, dest_resource, dest_file_id))
try:
n = Notification(event_type = 'data',
status = Notification.INFO,
operation = 'box_upload_start',
message = 'Starting import of file %s into box.' % src_file_id,
user = username,
# extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), src_resource, src_file_id)})
extra={'id': src_file_id})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import BoxFileManager
box_fm = BoxFileManager(user)
file_type, file_id = box_fm.parse_file_id(dest_file_id)
if file_type != 'folder':
logger.warn('Cannot import to a file destination!')
raise Exception('You can only import files to a folder!', status=400,
extra={
'src_resource': src_resource,
'src_file_id': src_file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id,
})
if src_resource == 'agave' or src_resource == 'public':
try:
logger.debug('Starting upload to Box...')
from designsafe.apps.api.data import lookup_file_manager
agave_fm = lookup_file_manager(src_resource)(user)
file_real_path = agave_fm.get_file_real_path(src_file_id)
if os.path.isfile(file_real_path):
box_upload_file(box_fm, file_id, file_real_path)
elif os.path.isdir(file_real_path):
box_upload_directory(box_fm, file_id, file_real_path)
else:
logger.error('Unable to upload %s: file does not exist!',
file_real_path)
except:
logger.exception('Upload to Box failed!')
logger.debug('Box upload task complete.')
n = Notification(event_type = 'data',
status = Notification.SUCCESS,
operation = 'box_upload_end',
message = 'File(s) %s succesfully uploaded into box!' % src_file_id,
user = username,
extra={'id': src_file_id})
# extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), dest_resource, dest_file_id)})
n.save()
except:
logger.exception('Unexpected task failure: box_upload', extra={
'username': username,
'src_resource': src_resource,
'src_file_id': src_file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id,
})
n = Notification(event_type = 'data',
status = Notification.ERROR,
operation = 'box_download_error',
message = 'We were unable to get the specified file from box. Please try again...',
user = username,
extra={
'src_resource': src_resource,
'id': src_file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id,
})
# extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), src_resource, src_file_id)})
n.save()
def box_download(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
"""
:param self:
:param username:
:param src_resource:
:param src_file_id:
:param dest_resource:
:param dest_file_id:
:return:
"""
logger.debug('Downloading %s://%s for user %s to %s://%s',
src_resource, src_file_id, username, dest_resource, dest_file_id)
try:
target_path = reverse('designsafe_data:data_depot',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download of file %s from box.' % (src_file_id,),
user=username,
# extra={'target_path': target_path})
extra={'id': dest_file_id}
)
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import BoxFileManager, AgaveFileManager
agave_fm = AgaveFileManager(user)
dest_real_path = agave_fm.get_file_real_path(dest_file_id)
box_fm = BoxFileManager(user)
file_type, file_id = box_fm.parse_file_id(src_file_id)
levels = 0
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = box_download_file(box_fm, file_id, dest_real_path)
levels = 1
elif file_type == 'folder':
downloaded_file_path = box_download_folder(box_fm, file_id, dest_real_path)
if downloaded_file_path is not None:
downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
agave_fm.indexer.index(system_id, file_path, file_user, full_indexing=True,
pems_indexing=True, index_full_path=True,
levels=levels)
target_path = reverse('designsafe_data:data_depot',
args=[dest_resource, dest_file_id])
n = Notification(event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s was copied from box successfully!' % (src_file_id, ),
user=username,
extra={'id': dest_file_id}
)
n.save()
except:
logger.exception('Unexpected task failure: box_download', extra={
'username': username,
'box_file_id': src_file_id,
'to_resource': dest_resource,
'dest_file_id': dest_file_id
})
target_path = reverse('designsafe_data:data_depot',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={'system': dest_resource,
'id': dest_file_id,
'src_file_id': src_file_id,
'src_resource': src_resource
})
n.save()
def box_resource_download(self, username, src_file_id, dest_file_id):
"""
:param self:
:param username:
:param src_file_id:
:param dest_file_id:
:return:
"""
logger.debug('Downloading box://%s for user %s to %s',
src_file_id, username, dest_file_id)
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download file %s from box.' % (src_file_id,),
user=username,
extra={})
n.save()
logger.debug('username: {}, src_file_id: {}, dest_file_id: {}'.format(username, src_file_id, dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.external_resources.box.filemanager.manager import \
FileManager as BoxFileManager
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir, dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path, dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
box_fm = BoxFileManager(user)
box_file_type, box_file_id = box_fm.parse_file_id(file_id=src_file_id)
levels = 0
downloaded_file_path = None
if box_file_type == 'file':
downloaded_file_path = box_fm.download_file(box_file_id, dest_real_path)
levels = 1
elif box_file_type == 'folder':
downloaded_file_path = box_fm.download_folder(box_file_id, dest_real_path)
#if downloaded_file_path is not None:
# downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
# system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
n = Notification(event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s has been copied from box successfully!' % (src_file_id, ),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'), project_dir)
agave_file_path = downloaded_file_path.replace(project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(base_mounted_path, '', 1).strip('/')
reindex_agave.apply_async(kwargs={
'username': user.username,
'file_id': '{}/{}'.format(agave_system_id, agave_file_path)
})
except:
logger.exception('Unexpected task failure: box_download', extra={
'username': username,
'box_file_id': src_file_id,
'dest_file_id': dest_file_id
})
n = Notification(event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={})
n.save()
def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
"""
:param self:
:param username:
:param src_resource: should be 'agave'
:param src_file_id: the agave file id
:param dest_resource: should be 'box'
:param dest_file_id: the box id of the destination folder
:return:
"""
logger.debug('Importing file %s://%s for user %s to %s://%s' % (
src_resource, src_file_id, username, dest_resource, dest_file_id))
try:
n = Notification(event_type = 'data',
status = Notification.INFO,
operation = 'box_upload_start',
message = 'Starting import file %s into box.' % src_file_id,
user = username,
extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), src_resource, src_file_id)})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import BoxFileManager
box_fm = BoxFileManager(user)
file_type, file_id = box_fm.parse_file_id(dest_file_id)
if file_type != 'folder':
logger.warn('Cannot import to a file destination!')
raise Exception('You can only import files to a folder!', status=400,
extra={
'src_resource': src_resource,
'src_file_id': src_file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id,
})
if src_resource == 'agave' or src_resource == 'public':
try:
logger.debug('Starting upload to Box...')
from designsafe.apps.api.data import lookup_file_manager
agave_fm = lookup_file_manager(src_resource)(user)
file_real_path = agave_fm.get_file_real_path(src_file_id)
if os.path.isfile(file_real_path):
box_upload_file(box_fm, file_id, file_real_path)
elif os.path.isdir(file_real_path):
box_upload_directory(box_fm, file_id, file_real_path)
else:
logger.error('Unable to upload %s: file does not exist!',
file_real_path)
except:
logger.exception('Upload to Box failed!')
logger.debug('Box upload task complete.')
n = Notification(event_type = 'data',
status = Notification.SUCCESS,
operation = 'box_upload_end',
message = 'File(s) %s succesfully uploaded into box!' % src_file_id,
user = username,
extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), dest_resource, dest_file_id)})
n.save()
except:
logger.exception('Unexpected task failure: box_upload', extra={
'username': username,
'src_resource': src_resource,
'src_file_id': src_file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id,
})
n = Notification(event_type = 'data',
status = Notification.ERROR,
operation = 'box_download_error',
message = 'We were unable to get the specified file from box. Please try again...',
user = username,
extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), src_resource, src_file_id)})
n.save()
def box_download(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
"""
:param self:
:param username:
:param src_resource:
:param src_file_id:
:param dest_resource:
:param dest_file_id:
:return:
"""
logger.debug('Downloading %s://%s for user %s to %s://%s',
src_resource, src_file_id, username, dest_resource, dest_file_id)
try:
target_path = reverse('designsafe_data:data_browser',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download file %s from box.' % (src_file_id,),
user=username,
extra={'target_path': target_path})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import BoxFileManager, AgaveFileManager
agave_fm = AgaveFileManager(user)
dest_real_path = agave_fm.get_file_real_path(dest_file_id)
box_fm = BoxFileManager(user)
file_type, file_id = box_fm.parse_file_id(src_file_id)
levels = 0
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = box_download_file(box_fm, file_id, dest_real_path)
levels = 1
elif file_type == 'folder':
downloaded_file_path = box_download_folder(box_fm, file_id, dest_real_path)
if downloaded_file_path is not None:
downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
agave_fm.indexer.index(system_id, file_path, file_user, full_indexing=True,
pems_indexing=True, index_full_path=True,
levels=levels)
target_path = reverse('designsafe_data:data_browser',
args=[dest_resource, dest_file_id])
n = Notification(event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s has been copied from box successfully!' % (src_file_id, ),
user=username,
extra={'target_path': target_path})
n.save()
except:
logger.exception('Unexpected task failure: box_download', extra={
'username': username,
'box_file_id': src_file_id,
'to_resource': dest_resource,
'dest_file_id': dest_file_id
})
target_path = reverse('designsafe_data:data_browser',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={'target_path': target_path})
n.save()