def get_download_url(self, file_id, **kwargs):
file_type, file_id = self.parse_file_id(file_id)
if file_type == 'file':
googledrive_file = self.googledrive_api.files().get(
fileId=file_id,
fields="webContentLink, ownedByMe, mimeType, name").execute()
# if user owns file, make file viewable by anyone with link to allow user to download the file
if googledrive_file['ownedByMe']:
file_pems = self.googledrive_api.permissions().list(
fileId=file_id).execute()['permissions']
if 'anyoneWithLink' not in [pem['id'] for pem in file_pems]:
body = {'role': 'reader', 'type': 'anyone'}
update_pems = self.googledrive_api.permissions().create(
fileId=file_id, body=body).execute()
if 'webContentLink' in googledrive_file:
return {'href': googledrive_file['webContentLink']}
else:
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_download_error',
message=
'Downloading Google-type files is currently unsupported. Convert the file to'
' a standard format and try again.',
user=kwargs['username'],
extra={'path': googledrive_file['name']
}) # show file in Notification
n.save()
return None
return None
def post(self, request, *args, **kwargs):
JOB_EVENT='job'
logger.debug('request body: {}'.format(request.body))
try:
notification = json.loads(request.body)
job_name = notification['name']
status = notification['status']
event = request.GET.get('event')
job_id = request.GET.get('job_id')
job_owner = notification['owner']
archive_path = notification['archivePath']
except ValueError as e: #for testing ->used when mocking agave notification
job_name = request.POST.get('job_name')
status = request.POST.get('status')
event = request.POST.get('event')
job_id = request.POST.get('job_id')
job_owner = request.POST.get('job_owner')
archive_path = request.POST.get('archivePath')
logger.info('job_name: {}'.format(job_name))
logger.info('event: {}'.format(event))
logger.info('job_id: {}'.format(job_id))
n = Notification(event_type = 'job',
status = 'INFO',
operation = event,
message = 'Job %s changed to %s' % (job_name, status),
user = job_owner,
extra = {'archive_path': archive_path,
'job_id': job_id})
n.save()
return HttpResponse('OK')
def download_folder(self, folder_id, download_path, username):
"""
Recursively download the folder for folder_id, and all of its contents, to the given
download_path.
:param folder_id:
:param download_path:
:return:
"""
googledrive_folder = self.googledrive_api.files().get(fileId=folder_id, fields="name").execute()
# convert utf-8 chars
safe_dirname = googledrive_folder['name'].encode(sys.getfilesystemencoding(), 'ignore')
directory_path = os.path.join(download_path, safe_dirname)
logger.debug('Creating directory %s <= googledrive://folder/%s', directory_path, folder_id)
try:
os.mkdir(directory_path, 0o0755)
except OSError as e:
if e.errno == 17: # directory already exists?
pass
else:
logger.exception('Error creating directory: %s', directory_path)
raise
items = self.googledrive_api.files().list(q="'{}' in parents and trashed=False".format(folder_id)).execute()
for item in items['files']:
if item['mimeType'] == 'application/vnd.google-apps.folder':
self.download_folder(item['id'], directory_path, username)
else:
try:
self.download_file(item['id'], directory_path, username)
logger.info('Google File download complete: {}'.format('/'.join([directory_path, item['id']])))
except Exception as e:
logger.exception('Unexpected task failure: googledrive_download', extra={
'username': username,
'file_id': item['id'],
'dest_file_id': '/'.join([directory_path, item['id']]),
'error_type': type(e),
'error': str(e)
})
n = Notification(event_type='data',
status=Notification.ERROR,
operation='googledrive_download_error',
message='We were unable to download file from Google Drive. '
'Please try again...',
user=username,
extra={'path': item['name']})
n.save()
raise
return directory_path
def post(self, request, *args, **kwargs):
body_json = json.loads(request.body)
nid = body_json['id']
read = body_json['read']
n = Notification.get(id = nid)
n.read = read
n.save()
def delete(self, request, *args, **kwargs):
body_json = json.loads(request.body)
nid = body_json['id']
deleted = body_json['deleted']
n = Notification.get(id = nid)
n.deleted = deleted
n.save()
def post(self, request, *args, **kwargs):
body_json = json.loads(request.body)
nid = body_json['id']
read = body_json['read']
n = Notification.get(id = nid)
n.read = read
n.save()
def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download file %s from box.' %
(src_file_id, ),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, src_file_id: {}, dest_file_id: {}'.format(
username, src_file_id, dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
# box_fm = BoxFileManager(user)
box_file_type, box_file_id = self.parse_file_id(
file_id=src_file_id)
levels = 0
downloaded_file_path = None
if box_file_type == 'file':
downloaded_file_path = self.download_file(
box_file_id, dest_real_path)
levels = 1
elif box_file_type == 'folder':
downloaded_file_path = self.download_folder(
box_file_id, dest_real_path)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s has been copied from box successfully!' %
(src_file_id, ),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
if not agave_file_path.startswith('/'):
agave_file_path = '/' + agave_file_path
agave_indexer.apply_async(kwargs={
'username':
user.username,
'systemId':
agave_system_id,
'filePath':
os.path.dirname(agave_file_path),
'recurse':
False
},
queue='indexing')
agave_indexer.apply_async(kwargs={
'systemId': agave_system_id,
'filePath': agave_file_path,
'recurse': True
},
routing_key='indexing')
except:
logger.exception('Unexpected task failure: box_download',
extra={
'username': username,
'box_file_id': src_file_id,
'dest_file_id': dest_file_id
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={})
n.save()
raise
def upload(self, username, src_file_id, dest_folder_id):
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='googledrive_upload_start',
message='Uploading file %s to Google Drive.' %
(src_file_id, ),
user=username,
extra={})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split src ination file path
src_file_path_comps = src_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = src_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if src_file_path_comps[1:]:
src_real_path = os.path.join(*src_file_path_comps[1:])
else:
src_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
src_real_path = os.path.join(base_mounted_path, project_dir,
src_real_path.strip('/'))
else:
src_real_path = os.path.join(base_mounted_path,
src_real_path.strip('/'))
logger.debug('src_real_path: {}'.format(src_real_path))
logger.debug('dest_folder_id:{}'.format(dest_folder_id))
if dest_folder_id == '':
dest_folder_id = 'root'
file_type, folder_id = self.parse_file_id(
file_id=dest_folder_id.strip('/'))
if os.path.isfile(src_real_path):
self.upload_file(folder_id, src_real_path)
elif os.path.isdir(src_real_path):
self.upload_directory(folder_id, src_real_path)
else:
logger.error('Unable to upload %s: file does not exist!',
src_real_path)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='googledrive_upload_end',
message='File "%s" was copied to Google Drive successfully!' %
(src_file_id, ),
user=username,
extra={})
n.save()
except Exception as err:
logger.exception('Unexpected task failure: googledrive_upload',
extra={
'username': username,
'src_file_id': src_file_id,
'dst_file_id': dest_folder_id,
'error': err
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_upload_error',
message=
'We were unable to upload the specified file to Google Drive. '
'Please try again...',
user=username,
extra={})
n.save()
raise
def download_file(self, file_id, download_directory_path, username):
"""
Downloads the file for file_id to the given download_path.
:param file_id:
:param download_directory_path:
:return: the full path to the downloaded file
"""
googledrive_file = self.googledrive_api.files().get(
fileId=file_id, fields="name, mimeType").execute()
# convert utf-8 chars
safe_filename = googledrive_file['name'].encode(
sys.getfilesystemencoding(), 'ignore')
file_download_path = os.path.join(download_directory_path,
safe_filename)
logger.debug('Download file %s <= googledrive://file/%s',
file_download_path, file_id)
if 'vnd.google-apps' in googledrive_file['mimeType']:
# if googledrive_file['mimeType'] == 'application/vnd.google-apps.spreadsheet':
# mimeType = 'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet'
# elif googledrive_file['mimeType'] == 'application/vnd.google-apps.document':
# mimeType = 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
# elif googledrive_file['mimeType'] == 'application/vnd.google-apps.presentation':
# mimeType = 'application/vnd.openxmlformats-officedocument.presentationml.presentation'
# else:
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_download_error',
message=
'Copying Google-type files is currently unsupported. Export the file to'
' a standard format and try again.',
user=username,
extra={
'path':
"'{}' of type {}".format(googledrive_file['name'],
googledrive_file['mimeType'])
})
n.save()
return None
request = self.googledrive_api.files().get_media(fileId=file_id)
# Incremental Partial Download
fh = io.FileIO(file_download_path, 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
backoff_attempts = 0
while done is False:
try:
status, done = downloader.next_chunk()
# logger.debug('status: {} percent'.format(status.progress()))
except HttpError as e:
# Incremental backoff for exceeding google api rate limit
if "Rate Limit Exceeded" in str(e):
logger.debug('RATE LIMIT EXCEEDED')
backoff_attempts += 1
time.sleep(backoff_attempts)
if backoff_attempts > 10:
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_download_error',
message=
'Rate Limit Exceeded. Try again after a few minutes for this file.',
user=username,
extra={
'path': "{}".format(googledrive_file['name'])
})
n.save()
return None
elif "Only files with binary content can be downloaded" in str(
e):
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_download_error',
message=
'Only files with binary content can be downloaded. Convert the file to'
' a standard format and try again.',
user=username,
extra={
'path':
"'{}' of type {}".format(
googledrive_file['name'],
googledrive_file['mimeType'])
})
n.save()
return None
else:
raise
fh.close()
return file_download_path
def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
"""
:param self:
:param username:
:param src_resource: should be 'agave'
:param src_file_id: the agave file id
:param dest_resource: should be 'box'
:param dest_file_id: the box id of the destination folder
:return:
"""
logger.debug('Importing file %s://%s for user %s to %s://%s' % (
src_resource, src_file_id, username, dest_resource, dest_file_id))
try:
n = Notification(event_type = 'data',
status = Notification.INFO,
operation = 'box_upload_start',
message = 'Starting import file %s into box.' % src_file_id,
user = username,
extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), src_resource, src_file_id)})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import BoxFileManager
box_fm = BoxFileManager(user)
file_type, file_id = box_fm.parse_file_id(dest_file_id)
if file_type != 'folder':
logger.warn('Cannot import to a file destination!')
raise Exception('You can only import files to a folder!', status=400,
extra={
'src_resource': src_resource,
'src_file_id': src_file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id,
})
if src_resource == 'agave' or src_resource == 'public':
try:
logger.debug('Starting upload to Box...')
from designsafe.apps.api.data import lookup_file_manager
agave_fm = lookup_file_manager(src_resource)(user)
file_real_path = agave_fm.get_file_real_path(src_file_id)
if os.path.isfile(file_real_path):
box_upload_file(box_fm, file_id, file_real_path)
elif os.path.isdir(file_real_path):
box_upload_directory(box_fm, file_id, file_real_path)
else:
logger.error('Unable to upload %s: file does not exist!',
file_real_path)
except:
logger.exception('Upload to Box failed!')
logger.debug('Box upload task complete.')
n = Notification(event_type = 'data',
status = Notification.SUCCESS,
operation = 'box_upload_end',
message = 'File(s) %s succesfully uploaded into box!' % src_file_id,
user = username,
extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), dest_resource, dest_file_id)})
n.save()
except:
logger.exception('Unexpected task failure: box_upload', extra={
'username': username,
'src_resource': src_resource,
'src_file_id': src_file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id,
})
n = Notification(event_type = 'data',
status = Notification.ERROR,
operation = 'box_download_error',
message = 'We were unable to get the specified file from box. Please try again...',
user = username,
extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_browser'), src_resource, src_file_id)})
n.save()
def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
file_type, file_id = self.parse_file_id(file_id=src_file_id)
googledrive_item = self.googledrive_api.files().get(
fileId=file_id, fields="name").execute()
n = Notification(
event_type='data',
status=Notification.INFO,
operation='googledrive_download_start',
message='Starting copy of {} {} from Google Drive.'.format(
file_type, googledrive_item['name']),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, filename: {}, src_file_id: {}, dest_file_id: {}'
.format(username, googledrive_item['name'], src_file_id,
dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = self.download_file(
file_id, dest_real_path, username)
if downloaded_file_path is None:
return None
elif file_type == 'folder':
downloaded_file_path = self.download_folder(
file_id, dest_real_path, username)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='googledrive_download_end',
message='{} "{}" was copied from Google Drive successfully!'.
format(file_type.capitalize(), googledrive_item['name']),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
reindex_agave.apply_async(kwargs={
'username':
user.username,
'file_id':
'{}/{}'.format(agave_system_id, agave_file_path)
},
queue='indexing')
except Exception as e:
logger.exception('Unexpected task failure: googledrive_copy',
extra={
'username': username,
'file_id': src_file_id,
'dest_file_id': dest_file_id,
'error_type': type(e),
'error': str(e)
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_copy_error',
message='We were unable to copy the file from Google Drive. '
'Please try again...',
user=username,
extra={'path': googledrive_item['name']})
n.save()
raise
def share_agave(self, username, file_id, permissions, recursive):
try:
# n = Notification(event_type = 'data',
# status = 'INFO',
# operation = 'share_initializing',
# message = 'File sharing is initializing. Please wait...',
# user = username,
# extra = {'target_path': reverse('designsafe_data:data_depot',
# args=['agave', file_id])})
# n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import AgaveFileManager
from designsafe.apps.api.data.agave.file import AgaveFile
from designsafe.apps.api.data.agave.elasticsearch.documents import Object
agave_fm = AgaveFileManager(user)
system_id, file_user, file_path = agave_fm.parse_file_id(file_id)
f = AgaveFile.from_file_path(system_id, username, file_path,
agave_client=agave_fm.agave_client)
f_dict = f.to_dict()
n = Notification(event_type = 'data',
status = 'INFO',
operation = 'share_initializing',
message = 'File sharing is initializing. Please wait...',
user = username,
extra = f_dict)
n.save()
f.share(permissions, recursive)
#reindex_agave.apply_async(args=(self.username, file_id))
# self.indexer.index(system, file_path, file_user, pems_indexing=True)
esf = Object.from_file_path(system_id, username, file_path)
esf.share(username, permissions, recursive)
# Notify owner share completed
n = Notification(event_type = 'data',
status = 'SUCCESS',
operation = 'share_finished',
message = 'File permissions were updated successfully.',
user = username,
extra = f_dict)
n.save()
# Notify users they have new shared files
for pem in permissions:
if pem['permission'] != 'NONE':
message = '%s shared some files with you.' % user.get_full_name()
n = Notification(event_type = 'data',
status = 'SUCCESS',
operation = 'share_finished',
message = message,
user = pem['user_to_share'],
extra = f_dict)
n.save()
except:
logger.error('Error sharing file/folder', exc_info=True,
extra = {
'username': username,
'file_id': file_id,
'permissions': permissions
})
n = Notification(event_type='data',
status=Notification.ERROR,
operation='share_error',
message='We were unable to share the specified folder/file(s). '
'Please try again...',
user=username,
extra={'system': system_id,
'path': file_path
})
n.save()
def copy_public_to_mydata(self, username, src_resource, src_file_id, dest_resource,
dest_file_id):
logger.debug('Scheduled copy of files from %s://%s to %s://%s',
src_resource, src_file_id, dest_resource, dest_file_id)
try:
n = Notification(event_type = 'data',
status = 'INFO',
operation = 'copy_public_to_mydata_start',
message = 'Copying folder/files %s from public data to your private data. Please wait...' % (src_file_id, ),
user = username,
extra={
'system': dest_resource,
'id': dest_file_id,
})
# extra = {'target_path': '%s%s' %(reverse('designsafe_data:data_depot'), src_file_id)})
n.save()
notify_status = 'SUCCESS'
from designsafe.apps.api.data import lookup_file_manager
source_fm_cls = lookup_file_manager(src_resource)
dest_fm_cls = lookup_file_manager(dest_resource)
if source_fm_cls and dest_fm_cls:
user = get_user_model().objects.get(username=username)
source_fm = source_fm_cls(user)
dest_fm = dest_fm_cls(user)
source_real_path = source_fm.get_file_real_path(src_file_id)
dirname = os.path.basename(source_real_path)
dest_real_path = os.path.join(dest_fm.get_file_real_path(dest_file_id),
dirname)
if os.path.isdir(source_real_path):
shutil.copytree(source_real_path, dest_real_path)
elif os.path.isfile(source_real_path):
shutil.copy(source_real_path, dest_real_path)
else:
notify_status = 'ERROR'
logger.error('The request copy source=%s does not exist!', src_resource)
system, username, path = dest_fm.parse_file_id(dest_file_id)
dest_fm.indexer.index(system, path, username, levels = 1)
n = Notification(event_type = 'data',
status = notify_status,
operation = 'copy_public_to_mydata_end',
message = 'Files were copied to your private data.',
user = username,
extra={
'system': dest_resource,
'id': dest_file_id,
})
# extra = {'target_path': '%s%s' %(reverse('designsafe_data:data_depot'), dest_file_id)})
n.save()
else:
logger.error('Unable to load file managers for both source=%s and destination=%s',
src_resource, dest_resource)
n = Notification(event_type = 'data',
status = 'ERROR',
operation = 'copy_public_to_mydata_error',
message = '''There was an error copying the files to your public data.
Plese try again.''',
user = username,
extra={
'system': dest_resource,
'id': dest_file_id,
})
# extra = {})
n.save()
except:
logger.exception('Unexpected task failure')
n = Notification(event_type = 'data',
status = 'ERROR',
operation = 'copy_public_to_mydata_error',
message = '''There was an error copying the files to your public data.
Plese try again.''',
user = username,
extra={
'system': dest_resource,
'id': dest_file_id,
})
# extra = {})
n.save()
def box_upload(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
"""
:param self:
:param username:
:param src_resource: should be 'agave'
:param src_file_id: the agave file id
:param dest_resource: should be 'box'
:param dest_file_id: the box id of the destination folder
:return:
"""
logger.debug('Importing file %s://%s for user %s to %s://%s' % (
src_resource, src_file_id, username, dest_resource, dest_file_id))
try:
n = Notification(event_type = 'data',
status = Notification.INFO,
operation = 'box_upload_start',
message = 'Starting import of file %s into box.' % src_file_id,
user = username,
# extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), src_resource, src_file_id)})
extra={'id': src_file_id})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import BoxFileManager
box_fm = BoxFileManager(user)
file_type, file_id = box_fm.parse_file_id(dest_file_id)
if file_type != 'folder':
logger.warn('Cannot import to a file destination!')
raise Exception('You can only import files to a folder!', status=400,
extra={
'src_resource': src_resource,
'src_file_id': src_file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id,
})
if src_resource == 'agave' or src_resource == 'public':
try:
logger.debug('Starting upload to Box...')
from designsafe.apps.api.data import lookup_file_manager
agave_fm = lookup_file_manager(src_resource)(user)
file_real_path = agave_fm.get_file_real_path(src_file_id)
if os.path.isfile(file_real_path):
box_upload_file(box_fm, file_id, file_real_path)
elif os.path.isdir(file_real_path):
box_upload_directory(box_fm, file_id, file_real_path)
else:
logger.error('Unable to upload %s: file does not exist!',
file_real_path)
except:
logger.exception('Upload to Box failed!')
logger.debug('Box upload task complete.')
n = Notification(event_type = 'data',
status = Notification.SUCCESS,
operation = 'box_upload_end',
message = 'File(s) %s succesfully uploaded into box!' % src_file_id,
user = username,
extra={'id': src_file_id})
# extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), dest_resource, dest_file_id)})
n.save()
except:
logger.exception('Unexpected task failure: box_upload', extra={
'username': username,
'src_resource': src_resource,
'src_file_id': src_file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id,
})
n = Notification(event_type = 'data',
status = Notification.ERROR,
operation = 'box_download_error',
message = 'We were unable to get the specified file from box. Please try again...',
user = username,
extra={
'src_resource': src_resource,
'id': src_file_id,
'dest_resource': dest_resource,
'dest_file_id': dest_file_id,
})
# extra = {'target_path': '%s%s/%s' %(reverse('designsafe_data:data_depot'), src_resource, src_file_id)})
n.save()
def box_download(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
"""
:param self:
:param username:
:param src_resource:
:param src_file_id:
:param dest_resource:
:param dest_file_id:
:return:
"""
logger.debug('Downloading %s://%s for user %s to %s://%s',
src_resource, src_file_id, username, dest_resource, dest_file_id)
try:
target_path = reverse('designsafe_data:data_depot',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download of file %s from box.' % (src_file_id,),
user=username,
# extra={'target_path': target_path})
extra={'id': dest_file_id}
)
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import BoxFileManager, AgaveFileManager
agave_fm = AgaveFileManager(user)
dest_real_path = agave_fm.get_file_real_path(dest_file_id)
box_fm = BoxFileManager(user)
file_type, file_id = box_fm.parse_file_id(src_file_id)
levels = 0
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = box_download_file(box_fm, file_id, dest_real_path)
levels = 1
elif file_type == 'folder':
downloaded_file_path = box_download_folder(box_fm, file_id, dest_real_path)
if downloaded_file_path is not None:
downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
agave_fm.indexer.index(system_id, file_path, file_user, full_indexing=True,
pems_indexing=True, index_full_path=True,
levels=levels)
target_path = reverse('designsafe_data:data_depot',
args=[dest_resource, dest_file_id])
n = Notification(event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s was copied from box successfully!' % (src_file_id, ),
user=username,
extra={'id': dest_file_id}
)
n.save()
except:
logger.exception('Unexpected task failure: box_download', extra={
'username': username,
'box_file_id': src_file_id,
'to_resource': dest_resource,
'dest_file_id': dest_file_id
})
target_path = reverse('designsafe_data:data_depot',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={'system': dest_resource,
'id': dest_file_id,
'src_file_id': src_file_id,
'src_resource': src_resource
})
n.save()
def box_download(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
"""
:param self:
:param username:
:param src_resource:
:param src_file_id:
:param dest_resource:
:param dest_file_id:
:return:
"""
logger.debug('Downloading %s://%s for user %s to %s://%s',
src_resource, src_file_id, username, dest_resource, dest_file_id)
try:
target_path = reverse('designsafe_data:data_browser',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download file %s from box.' % (src_file_id,),
user=username,
extra={'target_path': target_path})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import BoxFileManager, AgaveFileManager
agave_fm = AgaveFileManager(user)
dest_real_path = agave_fm.get_file_real_path(dest_file_id)
box_fm = BoxFileManager(user)
file_type, file_id = box_fm.parse_file_id(src_file_id)
levels = 0
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = box_download_file(box_fm, file_id, dest_real_path)
levels = 1
elif file_type == 'folder':
downloaded_file_path = box_download_folder(box_fm, file_id, dest_real_path)
if downloaded_file_path is not None:
downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
agave_fm.indexer.index(system_id, file_path, file_user, full_indexing=True,
pems_indexing=True, index_full_path=True,
levels=levels)
target_path = reverse('designsafe_data:data_browser',
args=[dest_resource, dest_file_id])
n = Notification(event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s has been copied from box successfully!' % (src_file_id, ),
user=username,
extra={'target_path': target_path})
n.save()
except:
logger.exception('Unexpected task failure: box_download', extra={
'username': username,
'box_file_id': src_file_id,
'to_resource': dest_resource,
'dest_file_id': dest_file_id
})
target_path = reverse('designsafe_data:data_browser',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={'target_path': target_path})
n.save()
def copy_public_to_mydata(self, username, src_resource, src_file_id, dest_resource,
dest_file_id):
logger.debug('Scheduled copy of files from %s://%s to %s://%s',
src_resource, src_file_id, dest_resource, dest_file_id)
try:
n = Notification(event_type = 'data',
status = 'INFO',
operation = 'copy_public_to_mydata_start',
message = 'Copying folder/files %s from public data to your private data. Please wait...' % (src_file_id, ),
user = username,
extra = {'target_path': '%s%s' %(reverse('designsafe_data:data_browser'), src_file_id)})
n.save()
notify_status = 'SUCCESS'
from designsafe.apps.api.data import lookup_file_manager
source_fm_cls = lookup_file_manager(src_resource)
dest_fm_cls = lookup_file_manager(dest_resource)
if source_fm_cls and dest_fm_cls:
user = get_user_model().objects.get(username=username)
source_fm = source_fm_cls(user)
dest_fm = dest_fm_cls(user)
source_real_path = source_fm.get_file_real_path(src_file_id)
dirname = os.path.basename(source_real_path)
dest_real_path = os.path.join(dest_fm.get_file_real_path(dest_file_id),
dirname)
if os.path.isdir(source_real_path):
shutil.copytree(source_real_path, dest_real_path)
elif os.path.isfile(source_real_path):
shutil.copy(source_real_path, dest_real_path)
else:
notify_status = 'ERROR'
logger.error('The request copy source=%s does not exist!', src_resource)
system, username, path = dest_fm.parse_file_id(dest_file_id)
dest_fm.indexer.index(system, path, username, levels = 1)
n = Notification(event_type = 'data',
status = notify_status,
operation = 'copy_public_to_mydata_end',
message = 'Files have been copied to your private data.',
user = username,
extra = {'target_path': '%s%s' %(reverse('designsafe_data:data_browser'), dest_file_id)})
n.save()
else:
logger.error('Unable to load file managers for both source=%s and destination=%s',
src_resource, dest_resource)
n = Notification(event_type = 'data',
status = 'ERROR',
operation = 'copy_public_to_mydata_error',
message = '''There was an error copying the files to your public data.
Plese try again.''',
user = username,
extra = {})
n.save()
except:
logger.exception('Unexpected task failure')
n = Notification(event_type = 'data',
status = 'ERROR',
operation = 'copy_public_to_mydata_error',
message = '''There was an error copying the files to your public data.
Plese try again.''',
user = username,
extra = {})
n.save()
def box_resource_download(self, username, src_file_id, dest_file_id):
"""
:param self:
:param username:
:param src_file_id:
:param dest_file_id:
:return:
"""
logger.debug('Downloading box://%s for user %s to %s',
src_file_id, username, dest_file_id)
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download file %s from box.' % (src_file_id,),
user=username,
extra={})
n.save()
logger.debug('username: {}, src_file_id: {}, dest_file_id: {}'.format(username, src_file_id, dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.external_resources.box.filemanager.manager import \
FileManager as BoxFileManager
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir, dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path, dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
box_fm = BoxFileManager(user)
box_file_type, box_file_id = box_fm.parse_file_id(file_id=src_file_id)
levels = 0
downloaded_file_path = None
if box_file_type == 'file':
downloaded_file_path = box_fm.download_file(box_file_id, dest_real_path)
levels = 1
elif box_file_type == 'folder':
downloaded_file_path = box_fm.download_folder(box_file_id, dest_real_path)
#if downloaded_file_path is not None:
# downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
# system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
n = Notification(event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s has been copied from box successfully!' % (src_file_id, ),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'), project_dir)
agave_file_path = downloaded_file_path.replace(project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(base_mounted_path, '', 1).strip('/')
reindex_agave.apply_async(kwargs={
'username': user.username,
'file_id': '{}/{}'.format(agave_system_id, agave_file_path)
})
except:
logger.exception('Unexpected task failure: box_download', extra={
'username': username,
'box_file_id': src_file_id,
'dest_file_id': dest_file_id
})
n = Notification(event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={})
n.save()
def share_agave(self, username, file_id, permissions, recursive):
try:
n = Notification(event_type = 'data',
status = 'INFO',
operation = 'share_initializing',
message = 'File sharing is initializing. Please wait...',
user = username,
extra = {'target_path': reverse('designsafe_data:data_browser',
args=['agave', file_id])})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import AgaveFileManager
from designsafe.apps.api.data.agave.file import AgaveFile
from designsafe.apps.api.data.agave.elasticsearch.documents import Object
agave_fm = AgaveFileManager(user)
system_id, file_user, file_path = agave_fm.parse_file_id(file_id)
f = AgaveFile.from_file_path(system_id, username, file_path,
agave_client=agave_fm.agave_client)
f.share(permissions, recursive)
#reindex_agave.apply_async(args=(self.username, file_id))
# self.indexer.index(system, file_path, file_user, pems_indexing=True)
esf = Object.from_file_path(system_id, username, file_path)
esf.share(username, permissions, recursive)
# Notify owner share completed
n = Notification(event_type = 'data',
status = 'SUCCESS',
operation = 'share_finished',
message = 'File permissions were updated successfully.',
user = username,
extra = {'target_path': reverse('designsafe_data:data_browser',
args=['agave', file_id])})
n.save()
# Notify users they have new shared files
for pem in permissions:
if pem['permission'] != 'NONE':
message = '%s shared some files with you.' % user.get_full_name()
n = Notification(event_type = 'data',
status = 'SUCCESS',
operation = 'share_finished',
message = message,
user = pem['user_to_share'],
extra = {'target_path': reverse('designsafe_data:data_browser',
args=['agave', file_id])})
n.save()
except:
logger.error('Error sharing file/folder', exc_info=True,
extra = {
'username': username,
'file_id': file_id,
'permissions': permissions
})
n = Notification(event_type='data',
status=Notification.ERROR,
operation='share_error',
message='We were unable to share the specified folder/file(s). '
'Please try again...',
user=username,
extra={})
n.save()