def reindex_agave(self, username, file_id, full_indexing=True,
levels=1, pems_indexing=True, index_full_path=True):
user = get_user_model().objects.get(username=username)
#levels=1
from designsafe.apps.api.data import AgaveFileManager
from designsafe.apps.data.managers.indexer import AgaveIndexer as AgaveFileIndexer
agave_fm = AgaveFileManager(user)
if settings.DEBUG and username == 'ds_admin':
service_client = get_service_account_client()
agave_fm.agave_client = service_client
agave_fm.indexer = AgaveFileIndexer(agave_client=service_client)
system_id, file_user, file_path = agave_fm.parse_file_id(file_id)
if system_id != settings.AGAVE_STORAGE_SYSTEM:
file_id_comps = file_id.strip('/').split('/')
system_id = file_id_comps[0]
file_user = username
if len(file_id_comps) > 1:
file_path = os.path.join(*file_id_comps[1:])
else:
file_path = '/'
agave_fm.indexer.index(system_id, file_path, file_user,
full_indexing = full_indexing,
pems_indexing = pems_indexing,
index_full_path = index_full_path,
levels = levels)
def reindex_agave(self, username, file_id, full_indexing = True,
levels = 0, pems_indexing = True, index_full_path = True):
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import AgaveFileManager
agave_fm = AgaveFileManager(user)
system_id, file_user, file_path = agave_fm.parse_file_id(file_id)
if system_id != settings.AGAVE_STORAGE_SYSTEM:
file_id_comps = file_id.strip('/').split('/')
system_id = file_id_comps[0]
file_user = username
if len(file_id_comps) > 1:
file_path = os.path.join(*file_id_comps[1:])
else:
file_path = '/'
agave_fm.indexer.index(system_id, file_path, file_user,
full_indexing = full_indexing,
pems_indexing = pems_indexing,
index_full_path = index_full_path,
levels = levels)
def reindex_agave(self, username, file_id, full_indexing=True,
levels=1, pems_indexing=True, index_full_path=True):
user = get_user_model().objects.get(username=username)
#levels=1
from designsafe.apps.api.data import AgaveFileManager
agave_fm = AgaveFileManager(user)
system_id, file_user, file_path = agave_fm.parse_file_id(file_id)
if system_id != settings.AGAVE_STORAGE_SYSTEM:
file_id_comps = file_id.strip('/').split('/')
system_id = file_id_comps[0]
file_user = username
if len(file_id_comps) > 1:
file_path = os.path.join(*file_id_comps[1:])
else:
file_path = '/'
agave_fm.indexer.index(system_id, file_path, file_user,
full_indexing = full_indexing,
pems_indexing = pems_indexing,
index_full_path = index_full_path,
levels = levels)
def share_agave(self, username, file_id, permissions, recursive):
try:
# n = Notification(event_type = 'data',
# status = 'INFO',
# operation = 'share_initializing',
# message = 'File sharing is initializing. Please wait...',
# user = username,
# extra = {'target_path': reverse('designsafe_data:data_depot',
# args=['agave', file_id])})
# n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import AgaveFileManager
from designsafe.apps.api.data.agave.file import AgaveFile
from designsafe.apps.api.data.agave.elasticsearch.documents import Object
agave_fm = AgaveFileManager(user)
system_id, file_user, file_path = agave_fm.parse_file_id(file_id)
f = AgaveFile.from_file_path(system_id, username, file_path,
agave_client=agave_fm.agave_client)
f_dict = f.to_dict()
n = Notification(event_type = 'data',
status = 'INFO',
operation = 'share_initializing',
message = 'File sharing is initializing. Please wait...',
user = username,
extra = f_dict)
n.save()
f.share(permissions, recursive)
#reindex_agave.apply_async(args=(self.username, file_id))
# self.indexer.index(system, file_path, file_user, pems_indexing=True)
esf = Object.from_file_path(system_id, username, file_path)
esf.share(username, permissions, recursive)
# Notify owner share completed
n = Notification(event_type = 'data',
status = 'SUCCESS',
operation = 'share_finished',
message = 'File permissions were updated successfully.',
user = username,
extra = f_dict)
n.save()
# Notify users they have new shared files
for pem in permissions:
if pem['permission'] != 'NONE':
message = '%s shared some files with you.' % user.get_full_name()
n = Notification(event_type = 'data',
status = 'SUCCESS',
operation = 'share_finished',
message = message,
user = pem['user_to_share'],
extra = f_dict)
n.save()
except:
logger.error('Error sharing file/folder', exc_info=True,
extra = {
'username': username,
'file_id': file_id,
'permissions': permissions
})
n = Notification(event_type='data',
status=Notification.ERROR,
operation='share_error',
message='We were unable to share the specified folder/file(s). '
'Please try again...',
user=username,
extra={'system': system_id,
'path': file_path
})
n.save()
def box_download(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
"""
:param self:
:param username:
:param src_resource:
:param src_file_id:
:param dest_resource:
:param dest_file_id:
:return:
"""
logger.debug('Downloading %s://%s for user %s to %s://%s',
src_resource, src_file_id, username, dest_resource, dest_file_id)
try:
target_path = reverse('designsafe_data:data_depot',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download of file %s from box.' % (src_file_id,),
user=username,
# extra={'target_path': target_path})
extra={'id': dest_file_id}
)
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import BoxFileManager, AgaveFileManager
agave_fm = AgaveFileManager(user)
dest_real_path = agave_fm.get_file_real_path(dest_file_id)
box_fm = BoxFileManager(user)
file_type, file_id = box_fm.parse_file_id(src_file_id)
levels = 0
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = box_download_file(box_fm, file_id, dest_real_path)
levels = 1
elif file_type == 'folder':
downloaded_file_path = box_download_folder(box_fm, file_id, dest_real_path)
if downloaded_file_path is not None:
downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
agave_fm.indexer.index(system_id, file_path, file_user, full_indexing=True,
pems_indexing=True, index_full_path=True,
levels=levels)
target_path = reverse('designsafe_data:data_depot',
args=[dest_resource, dest_file_id])
n = Notification(event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s was copied from box successfully!' % (src_file_id, ),
user=username,
extra={'id': dest_file_id}
)
n.save()
except:
logger.exception('Unexpected task failure: box_download', extra={
'username': username,
'box_file_id': src_file_id,
'to_resource': dest_resource,
'dest_file_id': dest_file_id
})
target_path = reverse('designsafe_data:data_depot',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={'system': dest_resource,
'id': dest_file_id,
'src_file_id': src_file_id,
'src_resource': src_resource
})
n.save()
def box_resource_download(self, username, src_file_id, dest_file_id):
"""
:param self:
:param username:
:param src_file_id:
:param dest_file_id:
:return:
"""
logger.debug('Downloading box://%s for user %s to %s',
src_file_id, username, dest_file_id)
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download file %s from box.' % (src_file_id,),
user=username,
extra={})
n.save()
logger.debug('username: {}, src_file_id: {}, dest_file_id: {}'.format(username, src_file_id, dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.external_resources.box.filemanager.manager import \
FileManager as BoxFileManager
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir, dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path, dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
box_fm = BoxFileManager(user)
box_file_type, box_file_id = box_fm.parse_file_id(file_id=src_file_id)
levels = 0
downloaded_file_path = None
if box_file_type == 'file':
downloaded_file_path = box_fm.download_file(box_file_id, dest_real_path)
levels = 1
elif box_file_type == 'folder':
downloaded_file_path = box_fm.download_folder(box_file_id, dest_real_path)
#if downloaded_file_path is not None:
# downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
# system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
n = Notification(event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s has been copied from box successfully!' % (src_file_id, ),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'), project_dir)
agave_file_path = downloaded_file_path.replace(project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(base_mounted_path, '', 1).strip('/')
reindex_agave.apply_async(kwargs={
'username': user.username,
'file_id': '{}/{}'.format(agave_system_id, agave_file_path)
})
except:
logger.exception('Unexpected task failure: box_download', extra={
'username': username,
'box_file_id': src_file_id,
'dest_file_id': dest_file_id
})
n = Notification(event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={})
n.save()
def share_agave(self, username, file_id, permissions, recursive):
try:
n = Notification(event_type = 'data',
status = 'INFO',
operation = 'share_initializing',
message = 'File sharing is initializing. Please wait...',
user = username,
extra = {'target_path': reverse('designsafe_data:data_browser',
args=['agave', file_id])})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import AgaveFileManager
from designsafe.apps.api.data.agave.file import AgaveFile
from designsafe.apps.api.data.agave.elasticsearch.documents import Object
agave_fm = AgaveFileManager(user)
system_id, file_user, file_path = agave_fm.parse_file_id(file_id)
f = AgaveFile.from_file_path(system_id, username, file_path,
agave_client=agave_fm.agave_client)
f.share(permissions, recursive)
#reindex_agave.apply_async(args=(self.username, file_id))
# self.indexer.index(system, file_path, file_user, pems_indexing=True)
esf = Object.from_file_path(system_id, username, file_path)
esf.share(username, permissions, recursive)
# Notify owner share completed
n = Notification(event_type = 'data',
status = 'SUCCESS',
operation = 'share_finished',
message = 'File permissions were updated successfully.',
user = username,
extra = {'target_path': reverse('designsafe_data:data_browser',
args=['agave', file_id])})
n.save()
# Notify users they have new shared files
for pem in permissions:
if pem['permission'] != 'NONE':
message = '%s shared some files with you.' % user.get_full_name()
n = Notification(event_type = 'data',
status = 'SUCCESS',
operation = 'share_finished',
message = message,
user = pem['user_to_share'],
extra = {'target_path': reverse('designsafe_data:data_browser',
args=['agave', file_id])})
n.save()
except:
logger.error('Error sharing file/folder', exc_info=True,
extra = {
'username': username,
'file_id': file_id,
'permissions': permissions
})
n = Notification(event_type='data',
status=Notification.ERROR,
operation='share_error',
message='We were unable to share the specified folder/file(s). '
'Please try again...',
user=username,
extra={})
n.save()
def box_download(self, username, src_resource, src_file_id, dest_resource, dest_file_id):
"""
:param self:
:param username:
:param src_resource:
:param src_file_id:
:param dest_resource:
:param dest_file_id:
:return:
"""
logger.debug('Downloading %s://%s for user %s to %s://%s',
src_resource, src_file_id, username, dest_resource, dest_file_id)
try:
target_path = reverse('designsafe_data:data_browser',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download file %s from box.' % (src_file_id,),
user=username,
extra={'target_path': target_path})
n.save()
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.data import BoxFileManager, AgaveFileManager
agave_fm = AgaveFileManager(user)
dest_real_path = agave_fm.get_file_real_path(dest_file_id)
box_fm = BoxFileManager(user)
file_type, file_id = box_fm.parse_file_id(src_file_id)
levels = 0
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = box_download_file(box_fm, file_id, dest_real_path)
levels = 1
elif file_type == 'folder':
downloaded_file_path = box_download_folder(box_fm, file_id, dest_real_path)
if downloaded_file_path is not None:
downloaded_file_id = agave_fm.from_file_real_path(downloaded_file_path)
system_id, file_user, file_path = agave_fm.parse_file_id(downloaded_file_id)
agave_fm.indexer.index(system_id, file_path, file_user, full_indexing=True,
pems_indexing=True, index_full_path=True,
levels=levels)
target_path = reverse('designsafe_data:data_browser',
args=[dest_resource, dest_file_id])
n = Notification(event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s has been copied from box successfully!' % (src_file_id, ),
user=username,
extra={'target_path': target_path})
n.save()
except:
logger.exception('Unexpected task failure: box_download', extra={
'username': username,
'box_file_id': src_file_id,
'to_resource': dest_resource,
'dest_file_id': dest_file_id
})
target_path = reverse('designsafe_data:data_browser',
args=[src_resource, src_file_id])
n = Notification(event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={'target_path': target_path})
n.save()
