def import_data(self, system, file_path, from_system, from_file_path):
file_path = file_path or '/'
if file_path != '/':
file_path = file_path.strip('/')
from_file_path = from_file_path.strip('/')
f = BaseFileResource.listing(self._ag, system, file_path)
res = f.import_data(from_system, from_file_path)
file_name = from_file_path.split('/')[-1]
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': os.path.dirname(res.path),
'recurse': False
},
queue='indexing')
if res.format == 'folder':
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': res.path,
'recurse': True
},
queue='indexing')
return res
def mkdir(client, system, path, dir_name):
"""Create a new directory.
Params
------
client: agavepy.agave.Agave
Tapis client to use for the listing.
system: str
Tapis system ID.
path: str
Path in which to run mkdir.
dir_name: str
Name of the directory
Returns
-------
dict
"""
body = {'action': 'mkdir', 'path': dir_name}
result = client.files.manage(systemId=system,
filePath=urllib.parse.quote(path),
body=body)
agave_indexer.apply_async(kwargs={
'systemId': system,
'filePath': path,
'recurse': False
},
queue='indexing')
return dict(result)
def trash(self, system, file_path, trash_path):
name = os.path.basename(file_path)
f = BaseFileResource(self._ag, system, file_path)
# first ensure trash_path exists
BaseFileResource.ensure_path(self._ag, system, trash_path)
# check if file with same name exists in trash; expect 404
try:
check = os.path.join(trash_path, name)
BaseFileResource.listing(self._ag, system, check)
# if we did not 404, then we have a conflict; append date to name
name_ext = os.path.splitext(name)
timestamp = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
name = '{0} {1}{2}'.format(name_ext[0], timestamp, name_ext[1])
except HTTPError as e:
if e.response.status_code != 404:
raise
resp = f.move(trash_path, name)
parent_path = '/'.join(file_path.strip('/').split('/')[:-1])
parent_path = parent_path.strip('/') or '/'
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': os.path.dirname(f.path),
'recurse': False
},
queue='indexing')
return resp
def rename(client, system, path, new_name):
"""Renames a file. This is performed under the hood by moving the file to
the same parent folder but with a new name.
Params
------
client: agavepy.agave.Agave
Tapis client to use.
system: str
Tapis system ID for the file.
path: str
Path of the file relative to the storage system root.
new_name: str
New name for the file.
Returns
-------
dict
"""
body = {'action': 'rename', 'path': new_name}
rename_result = client.files.manage(systemId=system, filePath=path, body=body)
agave_indexer.apply_async(kwargs={'systemId': system,
'filePath': os.path.dirname(path),
'recurse': False},
queue='indexing'
)
rename_result['nativeFormat'] == 'dir' and agave_indexer.apply_async(kwargs={'systemId': system,
'filePath': rename_result['path'],
'recurse': True},
queue='indexing'
)
return dict(rename_result)
def copy(self, system, file_path, dest_path=None, dest_name=None):
f = BaseFileResource.listing(self._ag, system, file_path)
# default to same path
if dest_path is None:
dest_path = f.path
# default to same name
if dest_name is None:
dest_name = f.name
# if same path and name, add suffix to file name
if dest_name == f.name and dest_path == f.path:
dest_name = '{0}_copy{1}'.format(*os.path.splitext(dest_name))
copied_file = f.copy(dest_path, dest_name)
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': os.path.dirname(copied_file.path),
'recurse': False
},
queue='indexing')
if copied_file.format == 'folder':
agave_indexer.apply_async(kwargs={
'systemId': system,
'filePath': copied_file.path,
'recurse': True
},
routing_key='indexing')
return copied_file
def upload(client,
system,
path,
uploaded_file,
webkit_relative_path=None,
*args,
**kwargs):
"""Upload a file.
Params
------
client: agavepy.agave.Agave
Tapis client to use.
system: str
Tapis system ID for the file.
path: str
Path to upload the file to.
uploaded_file: file
File object to upload.
webkit_relative_path: string
path for file structure of uploaded directory.
Returns
-------
dict
"""
# NOTE:
# Directory and file uploads are not compared against the upload path for an existing listing.
# This could provide a bad user experience when their files are just uploaded and overwrite
# existing data. We will need to be able to handle directory/folder uploads as a single operation.
if webkit_relative_path:
rel_path = os.path.dirname(webkit_relative_path).strip('/')
mkdir(client, system, path, rel_path)
path = os.path.join(path, rel_path)
upload_name = os.path.basename(uploaded_file.name)
resp = client.files.importData(systemId=system,
filePath=urllib.parse.quote(path),
fileName=str(upload_name),
fileToUpload=uploaded_file)
agave_indexer.apply_async(kwargs={
'systemId': system,
'filePath': path,
'recurse': False
},
queue='indexing')
# attempt to gather standardized metadata from upload
try:
metadata = scrape_metadata_from_file(uploaded_file)
if metadata:
create_meta(path=os.path.join('/', path, upload_name),
system=system,
meta=metadata)
return dict(resp)
except:
return dict(resp)
def upload(self, system, file_path, upload_file):
f = BaseFileResource(self._ag, system, file_path)
resp = f.upload(upload_file)
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': os.path.dirname(resp.path),
'recurse': False
},
queue='indexing')
return resp
def mkdir(self, system, file_path, dir_name):
f = BaseFileResource(self._ag, system, file_path)
resp = f.mkdir(dir_name)
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': f.path,
'recurse': False
},
queue='indexing')
return resp
def delete(self, system, path):
f = BaseFileResource(self._ag, system, path)
resp = f.delete()
parent_path = '/'.join(path.strip('/').split('/')[:-1])
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': os.path.dirname(f.path),
'recurse': False
},
queue='indexing')
return resp
def rename(self, system, file_path, rename_to):
f = BaseFileResource.listing(self._ag, system, file_path)
resp = f.rename(rename_to)
parent_path = '/'.join(file_path.strip('/').split('/')[:-1])
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': os.path.dirname(resp.path),
'recurse': False
},
queue='indexing')
return resp
def upload(client, system, path, uploaded_file, webkit_relative_path=None, *args, **kwargs,):
"""Upload a file.
Params
------
client: agavepy.agave.Agave
Tapis client to use.
system: str
Tapis system ID for the file.
path: str
Path to upload the file to.
uploaded_file: file
File object to upload.
Returns
-------
dict
"""
# try:
# listing(client, system=system, path=path)
# except HTTPError as err:
# if err.response.status_code != 404:
# raise
if webkit_relative_path:
rel_path = os.path.dirname(webkit_relative_path).strip('/')
mkdir(client, system, path, rel_path)
path = os.path.join(path, rel_path)
upload_name = os.path.basename(uploaded_file.name)
resp = client.files.importData(systemId=system,
filePath=urllib.parse.quote(path),
fileName=str(upload_name),
fileToUpload=uploaded_file)
agave_indexer.apply_async(kwargs={'systemId': system,
'filePath': path,
'recurse': False},
queue='indexing'
)
return dict(resp)
def move(self, system, file_path, dest_path, dest_name=None):
f = BaseFileResource.listing(self._ag, system, file_path)
resp = f.move(dest_path, dest_name)
parent_path = '/'.join(file_path.strip('/').split('/')[:-1])
parent_path = parent_path.strip('/') or '/'
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': os.path.dirname(f.path),
'recurse': False
},
queue='indexing')
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': os.path.dirname(resp.path),
'recurse': False
},
queue='indexing')
if resp.format == 'folder':
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': system,
'filePath': resp.path
},
queue='indexing')
return resp
def move(client, src_system, src_path, dest_system, dest_path, file_name=None):
"""Move a current file to the given destination.
Params
------
client: agavepy.agave.Agave
Tapis client to use for the listing.
src_system: str
System ID for the file's source.
src_path: str
Path to the source file.
dest_system: str
System ID for the destination.
dest_path: str
Path under which the file should be moved.
file_name: str
New name for the file if desired.
Returns
-------
dict
"""
if file_name is None:
file_name = src_path.strip('/').split('/')[-1]
dest_path_full = os.path.join(dest_path.strip('/'), file_name)
src_path_full = urllib.parse.quote(src_path)
# Handle attempt to move a file into its current path.
if src_system == dest_system and src_path_full == dest_path_full:
return {'system': src_system, 'path': src_path_full, 'name': file_name}
try:
client.files.list(systemId=dest_system,
filePath="{}/{}".format(dest_path, file_name))
# Destination path exists, must make it unique.
_ext = os.path.splitext(file_name)[1].lower()
_name = os.path.splitext(file_name)[0]
now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H-%M-%S')
file_name = '{}_{}{}'.format(_name, now, _ext)
except HTTPError as err:
if err.response.status_code != 404:
raise
full_dest_path = os.path.join(dest_path.strip('/'), file_name)
if src_system == dest_system:
body = {'action': 'move',
'path': full_dest_path}
move_result = client.files.manage(systemId=src_system,
filePath=urllib.parse.quote(
src_path),
body=body)
else:
src_url = 'agave://{}/{}'.format(
src_system,
urllib.parse.quote(src_path)
)
move_result = client.files.importData(
systemId=dest_system,
filePath=urllib.parse.quote(dest_path),
fileName=str(file_name),
urlToIngest=src_url
)
client.files.delete(systemId=src_system,
filePath=urllib.parse.quote(src_path))
if os.path.dirname(src_path) != dest_path or src_path != dest_path:
agave_indexer.apply_async(kwargs={'systemId': src_system,
'filePath': os.path.dirname(src_path),
'recurse': False},
queue='indexing')
agave_indexer.apply_async(kwargs={'systemId': dest_system,
'filePath': os.path.dirname(full_dest_path),
'recurse': False}, routing_key='indexing')
if move_result['nativeFormat'] == 'dir':
agave_indexer.apply_async(kwargs={'systemId': dest_system,
'filePath': full_dest_path, 'recurse': True},
queue='indexing')
return move_result
def rename(client, system, path, new_name):
"""Renames a file. This is performed under the hood by moving the file to
the same parent folder but with a new name.
Params
------
client: agavepy.agave.Agave
Tapis client to use.
system: str
Tapis system ID for the file.
path: str
Path of the file relative to the storage system root.
new_name: str
New name for the file.
Returns
-------
dict
"""
# NOTE: There seems to be inconsistant values in file operation response
# objects returned by tapis. The "nativeFormat" field has returned
# values representing a file, when the operation was carried out on
# a directory...
# listing[0].type == 'file'
# listing[0].type == 'dir'
listing = client.files.list(systemId=system, filePath=path)
path = path.strip('/')
body = {'action': 'rename', 'path': new_name}
rename_result = client.files.manage(systemId=system,
filePath=urllib.parse.quote(
os.path.join('/', path)),
body=body)
update_meta(src_system=system,
src_path=path,
dest_system=system,
dest_path=os.path.join(os.path.dirname(path), new_name))
# if rename_result['nativeFormat'] == 'dir':
if listing[0].type == 'dir':
agave_indexer.apply_async(kwargs={
'systemId': system,
'filePath': os.path.dirname(path),
'recurse': False
},
queue='indexing')
agave_indexer.apply_async(kwargs={
'systemId': system,
'filePath': rename_result['path'],
'recurse': True
},
queue='indexing')
else:
agave_indexer.apply_async(kwargs={
'systemId': system,
'filePath': os.path.dirname(path),
'recurse': False
},
queue='indexing')
return dict(rename_result)
def copy(client, src_system, src_path, dest_system, dest_path):
"""
Copy files and related file metadata
Params
------
client: agavepy.agave.Agave
Tapis client to use for the listing.
src_system: str
System ID for the file's source.
src_path: str
Path to the source file.
dest_system: str
System ID for the destination.
dest_path: str
Path under which the file should be Copied.
Returns
-------
dict
"""
src_file_name = os.path.basename(src_path)
try:
client.files.list(systemId=dest_system,
filePath=os.path.join(dest_path, src_file_name))
dst_file_name = rename_duplicate_path(src_file_name)
full_dest_path = os.path.join(dest_path.strip('/'), dst_file_name)
except:
dst_file_name = src_file_name
full_dest_path = os.path.join(dest_path.strip('/'), src_file_name)
if src_system == dest_system:
body = {'action': 'copy', 'path': full_dest_path}
copy_result = client.files.manage(
systemId=src_system,
filePath=urllib.parse.quote(src_path.strip(
'/')), # don't think we need to strip '/' here...
body=body)
else:
src_url = 'agave://{}/{}'.format(src_system,
urllib.parse.quote(src_path))
copy_result = client.files.importData(
systemId=dest_system,
filePath=urllib.parse.quote(dest_path),
fileName=dst_file_name,
urlToIngest=src_url)
copy_meta(src_system=src_system,
src_path=src_path,
dest_system=dest_system,
dest_path=full_dest_path)
if copy_result['nativeFormat'] == 'dir':
agave_indexer.apply_async(kwargs={
'systemId': dest_system,
'filePath': full_dest_path,
'recurse': True
},
queue='indexing')
else:
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': dest_system,
'filePath': dest_path,
'recurse': False
},
queue='indexing')
return dict(copy_result)
def move(client, src_system, src_path, dest_system, dest_path):
"""
Move files and related file metadata
Params
------
client: agavepy.agave.Agave
Tapis client to use for the listing.
src_system: str
System ID for the file's source.
src_path: str
Path to the source file.
dest_system: str
System ID for the destination.
dest_path: str
Path under which the file should be moved.
Returns
-------
dict
"""
# do not allow moves to the same location or across systems
if (os.path.dirname(src_path) == dest_path.strip('/')
or src_system != dest_system):
return {
'system': src_system,
'path': urllib.parse.quote(src_path),
'name': os.path.basename(src_path)
}
src_file_name = os.path.basename(src_path)
try:
client.files.list(systemId=dest_system,
filePath=os.path.join(dest_path, src_file_name))
dst_file_name = rename_duplicate_path(src_file_name)
full_dest_path = os.path.join(dest_path.strip('/'), dst_file_name)
except:
dst_file_name = src_file_name
full_dest_path = os.path.join(dest_path.strip('/'), src_file_name)
body = {'action': 'move', 'path': full_dest_path}
move_result = client.files.manage(systemId=src_system,
filePath=urllib.parse.quote(src_path),
body=body)
update_meta(src_system=src_system,
src_path=src_path,
dest_system=dest_system,
dest_path=full_dest_path)
if os.path.dirname(
src_path) != full_dest_path or src_path != full_dest_path:
agave_indexer.apply_async(kwargs={
'systemId': src_system,
'filePath': os.path.dirname(src_path),
'recurse': False
},
queue='indexing')
agave_indexer.apply_async(kwargs={
'systemId': dest_system,
'filePath': dest_path,
'recurse': False
},
queue='indexing')
if move_result['nativeFormat'] == 'dir':
agave_indexer.apply_async(kwargs={
'systemId': dest_system,
'filePath': full_dest_path,
'recurse': True
},
queue='indexing')
return move_result
def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
file_type, file_id = self.parse_file_id(file_id=src_file_id)
googledrive_item = self.googledrive_api.files().get(
fileId=file_id, fields="name").execute()
n = Notification(
event_type='data',
status=Notification.INFO,
operation='googledrive_download_start',
message='Starting copy of {} {} from Google Drive.'.format(
file_type, googledrive_item['name']),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, filename: {}, src_file_id: {}, dest_file_id: {}'
.format(username, googledrive_item['name'], src_file_id,
dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
downloaded_file_path = None
if file_type == 'file':
downloaded_file_path = self.download_file(
file_id, dest_real_path, username)
if downloaded_file_path is None:
return None
elif file_type == 'folder':
downloaded_file_path = self.download_folder(
file_id, dest_real_path, username)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='googledrive_download_end',
message='{} "{}" was copied from Google Drive successfully!'.
format(file_type.capitalize(), googledrive_item['name']),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
if not agave_file_path.startswith('/'):
agave_file_path = '/' + agave_file_path
agave_indexer.apply_async(kwargs={
'username':
user.username,
'systemId':
agave_system_id,
'filePath':
os.path.dirname(agave_file_path),
'recurse':
False
},
queue='indexing')
agave_indexer.apply_async(kwargs={
'systemId': agave_system_id,
'filePath': agave_file_path,
'recurse': True
},
routing_key='indexing')
except Exception as e:
logger.exception('Unexpected task failure: googledrive_copy',
extra={
'username': username,
'file_id': src_file_id,
'dest_file_id': dest_file_id,
'error_type': type(e),
'error': str(e)
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='googledrive_copy_error',
message='We were unable to copy the file from Google Drive. '
'Please try again...',
user=username,
extra={'path': googledrive_item['name']})
n.save()
raise
def copy_publication_files_to_corral(self, project_id):
from designsafe.apps.api.agave.filemanager.public_search_index import Publication
import shutil
publication = Publication(project_id=project_id)
filepaths = publication.related_file_paths()
if not len(filepaths):
res = get_service_account_client().files.list(
systemId='project-{project_uuid}'.format(
project_uuid=publication.project.uuid),
filePath='/')
filepaths = [
_file.path.strip('/') for _file in res
if (_file.name != '.' and _file.name != 'Trash')
]
filepaths = list(set(filepaths))
filepaths = sorted(filepaths)
base_path = ''.join(['/', publication.projectId])
os.chmod('/corral-repl/tacc/NHERI/published', 0755)
prefix_dest = '/corral-repl/tacc/NHERI/published/{}'.format(project_id)
if not os.path.isdir(prefix_dest):
os.mkdir(prefix_dest)
prefix_src = '/corral-repl/tacc/NHERI/projects/{}'.format(
publication.project['uuid'])
for filepath in filepaths:
local_src_path = '{}/{}'.format(prefix_src, filepath)
local_dst_path = '{}/{}'.format(prefix_dest, filepath)
logger.info('Trying to copy: %s to %s', local_src_path, local_dst_path)
if os.path.isdir(local_src_path):
try:
#os.mkdir(local_dst_path)
if not os.path.isdir(os.path.dirname(local_dst_path)):
os.makedirs(os.path.dirname(local_dst_path))
shutil.copytree(local_src_path, local_dst_path)
for root, dirs, files in os.walk(local_dst_path):
for d in dirs:
os.chmod(os.path.join(root, d), 0555)
for f in files:
os.chmod(os.path.join(root, f), 0444)
os.chmod(local_dst_path, 0555)
except OSError as exc:
logger.info(exc)
except IOError as exc:
logger.info(exc)
else:
try:
if not os.path.isdir(os.path.dirname(local_dst_path)):
os.makedirs(os.path.dirname(local_dst_path))
for root, dirs, files in os.walk(
os.path.dirname(local_dst_path)):
for d in dirs:
os.chmod(os.path.join(root, d), 0555)
for f in files:
os.chmod(os.path.join(root, f), 0444)
shutil.copy(local_src_path, local_dst_path)
os.chmod(local_dst_path, 0444)
except OSError as exc:
logger.info(exc)
except IOError as exc:
logger.info(exc)
os.chmod(prefix_dest, 0555)
os.chmod('/corral-repl/tacc/NHERI/published', 0555)
save_to_fedora.apply_async(args=[project_id])
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': 'designsafe.storage.published',
'filePath': '/' + project_id,
'recurse': True
},
queue='indexing')
def handle_webhook_request(job):
"""Notifies the user of the job status by instantiating and saving
a Notification instance.
If the job is finished, we also index the job and alert the user to the
URL of the job's location in the data depot.
Args:
job (dict): Dictionary containing the webhook data.
"""
# logger.debug(job)
try:
username = job['owner']
job_id = job['id']
user = get_user_model().objects.get(username=username)
ag = user.agave_oauth.client
# ag_job = ag.jobs.get(jobId=job_id)
try:
job['submitTime'] = str(job['submitTime'])
job['endTime'] = str(job['endTime'])
except KeyError as e:
pass
job_status = job['status']
job_name = job['name']
# logger.debug(current_status)
logger.debug(job_status)
event_data = {
Notification.EVENT_TYPE: 'job',
Notification.JOB_ID: job_id,
Notification.STATUS: '',
Notification.USER: username,
Notification.MESSAGE: '',
Notification.EXTRA: job
}
archive_id = 'agave/%s/%s' % (job['archiveSystem'],
job['archivePath'].split('/'))
if job_status == 'FAILED':
logger.debug('JOB FAILED: id=%s status=%s' % (job_id, job_status))
event_data[Notification.STATUS] = Notification.ERROR
event_data[
Notification.
MESSAGE] = "Job '%s' Failed. Please try again..." % (job_name)
event_data[Notification.OPERATION] = 'job_failed'
with transaction.atomic():
last_notification = Notification.objects.select_for_update(
).filter(jobId=job_id).last()
should_notify = True
if last_notification:
last_status = last_notification.to_dict(
)['extra']['status']
logger.debug('last status: ' + last_status)
if job_status == last_status:
logger.debug('duplicate notification received.')
should_notify = False
if should_notify:
n = Notification.objects.select_for_update().create(
**event_data)
n.save()
elif job_status == 'FINISHED':
logger.debug('JOB STATUS CHANGE: id=%s status=%s' %
(job_id, job_status))
logger.debug('archivePath: {}'.format(job['archivePath']))
target_path = reverse('designsafe_data:data_depot')
os.path.join(target_path, 'agave', archive_id.strip('/'))
event_data[Notification.STATUS] = Notification.SUCCESS
event_data[Notification.EXTRA]['job_status'] = 'FINISHED'
event_data[Notification.EXTRA]['target_path'] = target_path
event_data[
Notification.MESSAGE] = "Job '%s' finished!" % (job_name)
event_data[Notification.OPERATION] = 'job_finished'
with transaction.atomic():
last_notification = Notification.objects.select_for_update(
).filter(jobId=job_id).last()
should_notify = True
if last_notification:
last_status = last_notification.to_dict(
)['extra']['status']
logger.debug('last status: ' + last_status)
if job_status == last_status:
logger.debug('duplicate notification received.')
should_notify = False
if should_notify:
n = Notification.objects.select_for_update().create(
**event_data)
n.save()
logger.debug('Event data with action link %s' % event_data)
try:
logger.debug('Preparing to Index Job Output job=%s',
job_name)
archivePath = '/'.join(
[job['archiveSystem'], job['archivePath']])
agave_indexer.apply_async(kwargs={
'username':
'******',
'systemId':
job['archiveSystem'],
'filePath':
job['archivePath'],
'recurse':
True
},
queue='indexing')
logger.debug('Finished Indexing Job Output job=%s',
job_name)
except Exception as e:
logger.exception('Error indexing job output')
else:
# notify
logger.debug('JOB STATUS CHANGE: id=%s status=%s' %
(job_id, job_status))
event_data[Notification.STATUS] = Notification.INFO
event_data[Notification.MESSAGE] = "Job '%s' updated to %s." % (
job_name, job_status)
event_data[Notification.OPERATION] = 'job_status_update'
with transaction.atomic():
last_notification = Notification.objects.select_for_update(
).filter(jobId=job_id).last()
should_notify = True
if last_notification:
last_status = last_notification.to_dict(
)['extra']['status']
logger.debug('last status: ' + last_status)
if job_status == last_status:
logger.debug('duplicate notification received.')
should_notify = False
if should_notify:
n = Notification.objects.select_for_update().create(
**event_data)
n.save()
logger.debug(n.pk)
except ObjectDoesNotExist:
logger.exception('Unable to locate local user account: %s' % username)
except HTTPError as e:
if e.response.status_code == 404:
logger.warning('Job not found. Cancelling job watch.',
extra={'job_id': job_id})
else:
logger.warning('Agave API error. Retrying...')
except AgaveException as e:
logger.warning('Agave API error. Retrying...')
def copy(self, username, src_file_id, dest_file_id, **kwargs):
try:
n = Notification(event_type='data',
status=Notification.INFO,
operation='box_download_start',
message='Starting download file %s from box.' %
(src_file_id, ),
user=username,
extra={})
n.save()
logger.debug(
'username: {}, src_file_id: {}, dest_file_id: {}'.format(
username, src_file_id, dest_file_id))
user = get_user_model().objects.get(username=username)
from designsafe.apps.api.agave.filemanager.agave import AgaveFileManager
# Initialize agave filemanager
agave_fm = AgaveFileManager(agave_client=user.agave_oauth.client)
# Split destination file path
dest_file_path_comps = dest_file_id.strip('/').split('/')
# If it is an agave file id then the first component is a system id
agave_system_id = dest_file_path_comps[0]
# Start construction the actual real path into the NSF mount
if dest_file_path_comps[1:]:
dest_real_path = os.path.join(*dest_file_path_comps[1:])
else:
dest_real_path = '/'
# Get what the system id maps to
base_mounted_path = agave_fm.base_mounted_path(agave_system_id)
# Add actual path
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
dest_real_path = os.path.join(base_mounted_path, project_dir,
dest_real_path.strip('/'))
else:
dest_real_path = os.path.join(base_mounted_path,
dest_real_path.strip('/'))
logger.debug('dest_real_path: {}'.format(dest_real_path))
# box_fm = BoxFileManager(user)
box_file_type, box_file_id = self.parse_file_id(
file_id=src_file_id)
levels = 0
downloaded_file_path = None
if box_file_type == 'file':
downloaded_file_path = self.download_file(
box_file_id, dest_real_path)
levels = 1
elif box_file_type == 'folder':
downloaded_file_path = self.download_folder(
box_file_id, dest_real_path)
n = Notification(
event_type='data',
status=Notification.SUCCESS,
operation='box_download_end',
message='File %s has been copied from box successfully!' %
(src_file_id, ),
user=username,
extra={})
n.save()
if re.search(r'^project-', agave_system_id):
project_dir = agave_system_id.replace('project-', '', 1)
project_dir = os.path.join(base_mounted_path.strip('/'),
project_dir)
agave_file_path = downloaded_file_path.replace(
project_dir, '', 1).strip('/')
else:
agave_file_path = downloaded_file_path.replace(
base_mounted_path, '', 1).strip('/')
if not agave_file_path.startswith('/'):
agave_file_path = '/' + agave_file_path
agave_indexer.apply_async(kwargs={
'username':
user.username,
'systemId':
agave_system_id,
'filePath':
os.path.dirname(agave_file_path),
'recurse':
False
},
queue='indexing')
agave_indexer.apply_async(kwargs={
'systemId': agave_system_id,
'filePath': agave_file_path,
'recurse': True
},
routing_key='indexing')
except:
logger.exception('Unexpected task failure: box_download',
extra={
'username': username,
'box_file_id': src_file_id,
'dest_file_id': dest_file_id
})
n = Notification(
event_type='data',
status=Notification.ERROR,
operation='box_download_error',
message='We were unable to get the specified file from box. '
'Please try again...',
user=username,
extra={})
n.save()
raise
def copy_publication_files_to_corral(self,
project_id,
revision=None,
selected_files=None):
"""
Takes a project ID and copies project files to a published directory.
:param str project_id: Project ID
:param int revision: The revision number of the publication
:param list of selected_files strings: Only provided if project type == other.
"""
es_client = new_es_client()
publication = BaseESPublication(project_id=project_id,
revision=revision,
using=es_client)
filepaths = publication.related_file_paths()
if not len(filepaths) and selected_files:
# Project is "Other" so we just copy the selected files
filepaths = [
file_path.strip('/') for file_path in selected_files
if (file_path != '.Trash')
]
filepaths = list(set(filepaths))
filepaths = sorted(filepaths)
base_path = ''.join(['/', publication.projectId])
os.chmod('/corral-repl/tacc/NHERI/published', 0o755)
prefix_dest = '/corral-repl/tacc/NHERI/published/{}'.format(project_id)
if revision:
prefix_dest += 'v{}'.format(revision)
if not os.path.isdir(prefix_dest):
os.mkdir(prefix_dest)
prefix_src = '/corral-repl/tacc/NHERI/projects/{}'.format(
publication.project['uuid'])
for filepath in filepaths:
local_src_path = '{}/{}'.format(prefix_src, filepath)
local_dst_path = '{}/{}'.format(prefix_dest, filepath)
logger.info('Trying to copy: %s to %s', local_src_path, local_dst_path)
if os.path.isdir(local_src_path):
try:
#os.mkdir(local_dst_path)
if not os.path.isdir(os.path.dirname(local_dst_path)):
os.makedirs(os.path.dirname(local_dst_path))
shutil.copytree(local_src_path, local_dst_path)
for root, dirs, files in os.walk(local_dst_path):
for d in dirs:
os.chmod(os.path.join(root, d), 0o555)
for f in files:
os.chmod(os.path.join(root, f), 0o444)
os.chmod(local_dst_path, 0o555)
except OSError as exc:
logger.info(exc)
except IOError as exc:
logger.info(exc)
else:
try:
if not os.path.isdir(os.path.dirname(local_dst_path)):
os.makedirs(os.path.dirname(local_dst_path))
for root, dirs, files in os.walk(
os.path.dirname(local_dst_path)):
for d in dirs:
os.chmod(os.path.join(root, d), 0o555)
for f in files:
os.chmod(os.path.join(root, f), 0o444)
shutil.copy(local_src_path, local_dst_path)
os.chmod(local_dst_path, 0o444)
except OSError as exc:
logger.info(exc)
except IOError as exc:
logger.info(exc)
os.chmod(prefix_dest, 0o555)
os.chmod('/corral-repl/tacc/NHERI/published', 0o555)
save_to_fedora.apply_async(args=[project_id, revision])
index_path = '/' + project_id
if revision:
index_path += 'v{}'.format(revision)
agave_indexer.apply_async(kwargs={
'username': '******',
'systemId': 'designsafe.storage.published',
'filePath': index_path,
'recurse': True
},
queue='indexing')
def copy(client, src_system, src_path, dest_system, dest_path, file_name=None, *args, **kwargs):
"""Copies the current file to the provided destination path.
Params
------
client: agavepy.agave.Agave
Tapis client to use for the listing.
src_system: str
System ID for the file's source.
src_path: str
Path to the source file.
dest_system: str
System ID for the destination.
dest_path: str
Path under which the file should be copied.
file_name: str
New name for the file if desired.
Returns
-------
dict
"""
if file_name is None:
file_name = src_path.strip('/').split('/')[-1]
try:
client.files.list(systemId=dest_system,
filePath="{}/{}".format(dest_path, file_name))
# Destination path exists, must make it unique.
_ext = os.path.splitext(file_name)[1].lower()
_name = os.path.splitext(file_name)[0]
now = datetime.datetime.utcnow().strftime('%Y-%m-%d %H-%M-%S')
file_name = '{}_{}{}'.format(_name, now, _ext)
except HTTPError as err:
if err.response.status_code != 404:
raise
full_dest_path = os.path.join(dest_path.strip('/'), file_name)
if src_system == dest_system:
body = {'action': 'copy',
'path': full_dest_path}
copy_result = client.files.manage(systemId=src_system,
filePath=urllib.parse.quote(
src_path.strip('/')),
body=body)
else:
src_url = 'agave://{}/{}'.format(
src_system,
urllib.parse.quote(src_path)
)
copy_result = client.files.importData(
systemId=dest_system,
filePath=urllib.parse.quote(dest_path),
fileName=str(file_name),
urlToIngest=src_url
)
agave_indexer.apply_async(kwargs={'username': '******', 'systemId': dest_system, 'filePath': os.path.dirname(full_dest_path), 'recurse': False}, queue='indexing')
if copy_result['nativeFormat'] == 'dir':
agave_indexer.apply_async(kwargs={'systemId': dest_system, 'filePath': full_dest_path, 'recurse': True}, queue='indexing')
return dict(copy_result)
