def files_import_from_agave(self, system_id, file_path, file_name,
url_to_ingest):
"""
Wrap AgavePy import data file command.
Args:
self: class instance.
system_id: Identifier for Agave storage system.
file_path: Path where file is to be imported.
file_name: Name of the imported file.
url_to_ingest: Agave URL to be ingested.
Returns:
On success: True with no exceptions.
On failure: Throws exception.
"""
response = self._agave.files.importData(systemId=system_id,
filePath=file_path,
fileName=file_name,
urlToIngest=urllib.parse.quote(
str(url_to_ingest or ''),
safe='/:'))
async_response = AgaveAsyncResponse(self._agave, response)
status = async_response.result()
Log.some().debug('import %s: %s -> agave://%s/%s/%s', str(status),
url_to_ingest, system_id, file_path, file_name)
if str(status) == 'FINISHED':
return True
# not finished, try again
raise Exception('agave import failed')
def test_submit_job(agave, test_job):
job = agave.jobs.submit(body=test_job)
validate_job(job)
# create an async object
arsp = AgaveAsyncResponse(agave, job)
# block until job finishes with a timeout of 3 minutes.
assert arsp.result(180) == 'FINISHED'
def test_upload_large_file(agave, credentials):
rsp = agave.files.importData(systemId=credentials['storage'],
filePath=credentials['storage_user'],
fileToUpload=open(
'test_largefile_upload_python_sdk', 'rb'))
arsp = AgaveAsyncResponse(agave, rsp)
status = arsp.result(timeout=120)
assert status == 'FINISHED'
def test_submit_archive_job(agave, test_job, credentials):
test_job['archive'] = True
test_job['archiveSystem'] = credentials['storage']
job = agave.jobs.submit(body=test_job)
validate_job(job)
# create an async object
arsp = AgaveAsyncResponse(agave, job)
# block until job finishes with a timeout of 3 minutes.
assert arsp.result(180) == 'FINISHED'
def files_import_from_local(self, system_id, file_path, file_name,
file_to_upload):
"""
Wrap AgavePy import data file command.
Args:
self: class instance.
system_id: Identifier for Agave storage system.
file_path: Path where file is to be imported.
file_name: Name of the imported file.
file_to_upload: File or folder path to upload to Agave.
Returns:
On success: True with no exceptions.
On failure: Throws exception.
"""
if os.path.isdir(file_to_upload):
# create target directory, which is "file_name"
if not self.files_mkdir(system_id, file_path, file_name):
Log.an().error('cannot create folder at uri: agave://%s%s/%s',
system_id, file_path, file_name)
return False
# walk through local directory structure
for root, dirs, files in os.walk(file_to_upload, topdown=True):
# translate local path to dest path
dest_file_path = os.path.join(file_path, file_name,
root[len(file_to_upload) + 1:])
# upload each file in this directory level
for name in files:
# read file in binary mode to transfer
response = self._agave.files.importData(
systemId=system_id,
filePath=dest_file_path,
fileName=name,
fileToUpload=open('%s/%s' % (root, name), "rb"))
async_response = AgaveAsyncResponse(self._agave, response)
status = async_response.result()
Log.some().debug('import %s: %s/%s -> agave://%s/%s/%s',
str(status), root, name, system_id,
dest_file_path, name)
if status != 'FINISHED':
return False
# create new directory for each directory in this level
for name in dirs:
# create dest directory
if not self.files_mkdir(system_id, dest_file_path, name):
Log.an().error(
'cannot create folder at uri: agave://%s%s/%s',
system_id, dest_file_path, name)
return False
elif os.path.isfile(file_to_upload):
# import single file
response = self._agave.files.importData(systemId=system_id,
filePath=file_path,
fileName=file_name,
fileToUpload=open(
file_to_upload, 'rb'))
async_response = AgaveAsyncResponse(self._agave, response)
status = async_response.result()
Log.some().debug('import %s: %s -> agave://%s/%s/%s', str(status),
file_to_upload, system_id, file_path, file_name)
if status != 'FINISHED':
return False
return True
def import_data(self, from_system, from_path, retries=5, remote_url=None, external_resource=False):
"""Imports data from an external storage system
:param str from_system: System to import from.
:param str from_path: Path to import from.
:param int retries: Maximum retries if something goes wrong.
:returns: Agave File Resource imported.
:rtype: :class:`BaseFile`
.. note:: This function should be used to move data from one
Agave storage system to another Agave storage system.
.. todo:: We should implement a fallback using another type of
data transfer method if this fails.
"""
if not remote_url:
remote_url = 'agave://{}/{}'.format(
from_system,
urllib.parse.quote(from_path)
)
file_name = os.path.split(from_path)[1]
_retries = retries
while _retries > 0:
try:
result = self._ac.files.importData(
systemId=self.system,
filePath=urllib.parse.quote(self.path),
fileName=str(file_name),
urlToIngest=remote_url
)
async_resp = AgaveAsyncResponse(self._ac, result)
async_status = async_resp.result(600)
_retries = 0
except Error as err:
logger.error(
'There was an error importing data. %s. Retrying...',
err,
exc_info=True
)
_retries -= 1
if str(async_status) == 'FAILED':
logger.error(
'Import Data failed from: systemId=%s, filePath=%s. '
'to: systemId=%s, filePath=%s '
'using URI: %s',
from_system,
from_path,
self.system,
self.path,
remote_url,
exc_info=True
)
# If import is coming from an external resource like google drive,
# don't return a listing for every recursive file upload.
if external_resource:
return BaseFile(system=result['systemId'],
path=result['path'],
client=self._ac)
return BaseFile.listing(self._ac, self.system, result['path'])