def delete(self, system, path):
resp = BaseFileResource(self._ag, system, path).delete()
parent_path = '/'.join(path.strip('/').split('/')[:-1])
reindex_agave.apply_async(kwargs = {'username': '******',
'file_id': '{}/{}'.format(system, parent_path),
'levels': 1})
return resp
def copy(self, system, file_path, dest_path=None, dest_name=None):
f = BaseFileResource.listing(self._ag, system, file_path)
# default to same path
if dest_path is None:
dest_path = f.path
# default to same name
if dest_name is None:
dest_name = f.name
# if same path and name, add suffix to file name
if dest_name == f.name and dest_path == f.path:
dest_name = '{0}_copy{1}'.format(*os.path.splitext(dest_name))
copied_file = f.copy(dest_path, dest_name)
# schedule celery task to index new copy
reindex_agave.apply_async(kwargs={
'username':
'******',
'file_id':
'{}/{}/{}'.format(system, dest_path.strip('/'), dest_name)
},
queue='indexing')
return copied_file
def upload(self, system, file_path, upload_file):
f = BaseFileResource(self._ag, system, file_path)
resp = f.upload(upload_file)
reindex_agave.apply_async(kwargs = {'username': '******',
'file_id': '{}/{}'.format(system, file_path),
'levels': 1})
return resp
def trash(self, system, file_path, trash_path):
name = os.path.basename(file_path)
f = BaseFileResource(self._ag, system, file_path)
# first ensure trash_path exists
BaseFileResource.ensure_path(self._ag, system, trash_path)
# check if file with same name exists in trash; expect 404
try:
check = os.path.join(trash_path, name)
BaseFileResource.listing(self._ag, system, check)
# if we did not 404, then we have a conflict; append date to name
name_ext = os.path.splitext(name)
timestamp = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
name = '{0} {1}{2}'.format(name_ext[0], timestamp, name_ext[1])
except HTTPError as e:
if e.response.status_code != 404:
raise
resp = f.move(trash_path, name)
parent_path = '/'.join(file_path.strip('/').split('/')[:-1])
parent_path = parent_path.strip('/') or '/'
reindex_agave.apply_async(kwargs = {'username': '******',
'file_id': '{}/{}'.format(system, trash_path),
'levels': 1})
reindex_agave.apply_async(kwargs = {'username': '******',
'file_id': '{}/{}'.format(system, parent_path),
'levels': 1})
return resp
def rename(self, system, file_path, rename_to):
f = BaseFileResource.listing(self._ag, system, file_path)
resp = f.rename(rename_to)
parent_path = '/'.join(file_path.strip('/').split('/')[:-1])
reindex_agave.apply_async(kwargs = {'username': '******',
'file_id': '{}/{}'.format(system, parent_path),
'levels': 1})
return resp
def mkdir(self, system, file_path, dir_name):
f = BaseFileResource(self._ag, system, file_path)
resp = f.mkdir(dir_name)
reindex_agave.apply_async(kwargs={
'username': '******',
'file_id': '{}/{}'.format(system, file_path)
},
queue='indexing')
return resp
def upload(self, system, file_path, upload_file):
f = BaseFileResource(self._ag, system, file_path)
resp = f.upload(upload_file)
reindex_agave.apply_async(kwargs={
'username': '******',
'file_id': '{}/{}'.format(system, file_path),
'levels': 1
},
queue='indexing')
return resp
def share(self, system, file_path, username, permission):
f = BaseFileResource(self._ag, system, file_path)
recursive = True
pem = BaseFilePermissionResource(self._ag, f, recursive=recursive)
pem.username = username
pem.permission_bit = permission
resp = pem.save()
reindex_agave.apply_async(kwargs = {'username': '******',
'file_id': '{}/{}'.format(system, file_path)})
return resp
def import_data(self, system, file_path, from_system, from_file_path):
file_path = file_path or '/'
if file_path != '/':
file_path = file_path.strip('/')
from_file_path = from_file_path.strip('/')
f = BaseFileResource.listing(self._ag, system, file_path)
res = f.import_data(from_system, from_file_path)
file_name = from_file_path.split('/')[-1]
reindex_agave.apply_async(kwargs={'username': '******',
'file_id': '{}/{}'.format(system, os.path.join(file_path, file_name))})
return res
def move(self, system, file_path, dest_path, dest_name=None):
f = BaseFileResource.listing(self._ag, system, file_path)
resp = f.move(dest_path, dest_name)
parent_path = '/'.join(file_path.strip('/').split('/')[:-1])
parent_path = parent_path.strip('/') or '/'
reindex_agave.apply_async(kwargs = {'username': '******',
'file_id': '{}/{}'.format(system, parent_path),
'levels': 1})
reindex_agave.apply_async(kwargs = {'username': '******',
'file_id': '{}/{}'.format(system, os.path.join(dest_path, resp.name)),
'levels': 1})
return resp
def share(self, system, file_path, username, permission):
f = BaseFileResource(self._ag, system, file_path)
recursive = True
pem = BaseFilePermissionResource(self._ag, f, recursive=recursive)
pem.username = username
pem.permission_bit = permission
resp = pem.save()
reindex_agave.apply_async(kwargs={
'username': '******',
'file_id': '{}/{}'.format(system, file_path)
},
queue='indexing')
return resp
def delete(self, system, path):
resp = BaseFileResource(self._ag, system, path).delete()
parent_path = '/'.join(path.strip('/').split('/')[:-1])
reindex_agave.apply_async(kwargs={
'username':
'******',
'file_id':
'{}/{}'.format(system, parent_path),
'levels':
1
},
queue='indexing')
return resp
def rename(self, system, file_path, rename_to):
f = BaseFileResource.listing(self._ag, system, file_path)
resp = f.rename(rename_to)
parent_path = '/'.join(file_path.strip('/').split('/')[:-1])
reindex_agave.apply_async(kwargs={
'username':
'******',
'file_id':
'{}/{}'.format(system, parent_path),
'levels':
1
},
queue='indexing')
return resp
def import_data(self, system, file_path, from_system, from_file_path):
file_path = file_path or '/'
if file_path != '/':
file_path = file_path.strip('/')
from_file_path = from_file_path.strip('/')
f = BaseFileResource.listing(self._ag, system, file_path)
res = f.import_data(from_system, from_file_path)
file_name = from_file_path.split('/')[-1]
reindex_agave.apply_async(kwargs={
'username':
'******',
'file_id':
'{}/{}'.format(system, os.path.join(file_path, file_name))
},
queue='indexing')
return res
def trash(self, system, file_path, trash_path):
name = os.path.basename(file_path)
f = BaseFileResource(self._ag, system, file_path)
# first ensure trash_path exists
BaseFileResource.ensure_path(self._ag, system, trash_path)
# check if file with same name exists in trash; expect 404
try:
check = os.path.join(trash_path, name)
BaseFileResource.listing(self._ag, system, check)
# if we did not 404, then we have a conflict; append date to name
name_ext = os.path.splitext(name)
timestamp = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
name = '{0} {1}{2}'.format(name_ext[0], timestamp, name_ext[1])
except HTTPError as e:
if e.response.status_code != 404:
raise
resp = f.move(trash_path, name)
parent_path = '/'.join(file_path.strip('/').split('/')[:-1])
parent_path = parent_path.strip('/') or '/'
reindex_agave.apply_async(kwargs={
'username': '******',
'file_id': '{}/{}'.format(system, trash_path),
'levels': 1
},
queue='indexing')
reindex_agave.apply_async(kwargs={
'username':
'******',
'file_id':
'{}/{}'.format(system, parent_path),
'levels':
1
},
queue='indexing')
return resp
def copy(self, system, file_path, dest_path=None, dest_name=None):
f = BaseFileResource.listing(self._ag, system, file_path)
# default to same path
if dest_path is None:
dest_path = f.path
# default to same name
if dest_name is None:
dest_name = f.name
# if same path and name, add suffix to file name
if dest_name == f.name and dest_path == f.path:
dest_name = '{0}_copy{1}'.format(*os.path.splitext(dest_name))
copied_file = f.copy(dest_path, dest_name)
# schedule celery task to index new copy
reindex_agave.apply_async(kwargs = {'username': '******',
'file_id': '{}/{}/{}'.format(system, dest_path.strip('/'), dest_name)})
return copied_file
def move(self, system, file_path, dest_path, dest_name=None):
f = BaseFileResource.listing(self._ag, system, file_path)
resp = f.move(dest_path, dest_name)
parent_path = '/'.join(file_path.strip('/').split('/')[:-1])
parent_path = parent_path.strip('/') or '/'
reindex_agave.apply_async(kwargs={
'username':
'******',
'file_id':
'{}/{}'.format(system, parent_path),
'levels':
1
},
queue='indexing')
reindex_agave.apply_async(kwargs={
'username':
'******',
'file_id':
'{}/{}'.format(system, os.path.join(dest_path, resp.name)),
'levels':
1
},
queue='indexing')
return resp
def mkdir(self, system, file_path, dir_name):
f = BaseFileResource(self._ag, system, file_path)
resp = f.mkdir(dir_name)
reindex_agave.apply_async(kwargs = {'username': '******',
'file_id': '{}/{}'.format(system, file_path)})
return resp
def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
file_type, file_id = self.parse_file_id(file_id=src_file_id)
googledrive_item = self.googledrive_api.files().get(
fileId=file_id, fields="name").execute()
n = Notification(
event_type='data',
status=Notification.INFO,
operation='googledrive_download_start',
message='Starting copy of {} {} from Google Drive.'.format(
file_type, googledrive_item['name']),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, filename: {}, src_file_id: {}, dest_file_id: {}'
.format(username, googledrive_item['name'], src_file_id,
dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = self.download_file(
file_id, dest_real_path, username)
if downloaded_file_path is None:
return None
elif file_type == 'folder':
downloaded_file_path = self.download_folder(
file_id, dest_real_path, username)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='googledrive_download_end',
message='{} "{}" was copied from Google Drive successfully!'.
format(file_type.capitalize(), googledrive_item['name']),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
reindex_agave.apply_async(kwargs={
'username':
user.username,
'file_id':
'{}/{}'.format(agave_system_id, agave_file_path)
},
queue='indexing')
except Exception as e:
logger.exception('Unexpected task failure: googledrive_copy',
extra={
'username': username,
'file_id': src_file_id,
'dest_file_id': dest_file_id,
'error_type': type(e),
'error': str(e)
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_copy_error',
message='We were unable to copy the file from Google Drive. '
'Please try again...',
user=username,
extra={'path': googledrive_item['name']})
n.save()
raise
def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
n = Notification(
event_type='data',
status=Notification.INFO,
operation='dropbox_download_start',
message='Starting download file %s from dropbox.' %
(src_file_id, ),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, src_file_id: {}, dest_file_id: {}'.format(
username, src_file_id, dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
file_type, path = self.parse_file_id(src_file_id)
levels = 0
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = self.download_file(path, dest_real_path)
levels = 1
elif file_type == 'folder':
downloaded_file_path = self.download_folder(
path, dest_real_path)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='dropbox_download_end',
message='File %s has been copied from dropbox successfully!' %
(src_file_id, ),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
reindex_agave.apply_async(kwargs={
'username':
user.username,
'file_id':
'{}/{}'.format(agave_system_id, agave_file_path)
},
queue='indexing')
except:
logger.exception('Unexpected task failure: dropbox_download',
extra={
'username': username,
'box_file_id': src_file_id,
'dest_file_id': dest_file_id
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='dropbox_download_error',
message='We were unable to get the specified file from dropbox. '
'Please try again...',
user=username,
extra={})
n.save()
raise