def test_submit_job(agave, test_job):
job = agave.jobs.submit(body=test_job)
validate_job(job)
# create an async object
arsp = AgaveAsyncResponse(agave, job)
# block until job finishes with a timeout of 3 minutes.
assert arsp.result(180) == 'FINISHED'
def test_submit_job(agave, test_job):
job = agave.jobs.submit(body=test_job)
validate_job(job)
# create an async object
arsp = AgaveAsyncResponse(agave, job)
# block until job finishes with a timeout of 3 minutes.
assert arsp.result(180) == 'FINISHED'
def test_upload_binary_file(agave, credentials):
rsp = agave.files.importData(systemId=credentials['storage'],
filePath=credentials['storage_user'],
fileToUpload=open('test_upload_python_sdk_g_art.mov', 'rb'))
arsp = AgaveAsyncResponse(agave, rsp)
status = arsp.result(timeout=120)
assert status == 'FINISHED'
def test_upload_binary_file(agave, credentials):
rsp = agave.files.importData(systemId=credentials['storage'],
filePath=credentials['storage_user'],
fileToUpload=open('test_upload_python_sdk_g_art.mov', 'rb'))
arsp = AgaveAsyncResponse(agave, rsp)
status = arsp.result(timeout=120)
assert status == 'FINISHED'
def _call(self, systemId, filePath, fileName, urlToIngest):
"""
Wrap agavePy import data file command.
Args:
self: class instance
systemId: Identifier for Agave storage system
filePath: Path where file is to be imported
fileName: Name of the imported file
urlToIngest: Agave URL to be ingested
Returns:
True
"""
response = self._agave.files.importData(systemId=systemId,
filePath=filePath,
fileName=fileName,
urlToIngest=urlToIngest)
async_response = AgaveAsyncResponse(self._agave, response)
status = async_response.result()
Log.some().info('import %s: %s -> agave://%s/%s/%s', str(status),
urlToIngest, systemId, filePath, fileName)
if str(status) == 'FINISHED':
return True
# not finished, try again
raise Exception('agave import failed')
def test_submit_archive_job(agave, test_job, credentials):
test_job['archive'] = True
test_job['archiveSystem'] = credentials['storage']
job = agave.jobs.submit(body=test_job)
validate_job(job)
# create an async object
arsp = AgaveAsyncResponse(agave, job)
# block until job finishes with a timeout of 3 minutes.
assert arsp.result(180) == 'FINISHED'
def test_submit_archive_job(agave, test_job, credentials):
test_job['archive'] = True
test_job['archiveSystem'] = credentials['storage']
job = agave.jobs.submit(body=test_job)
validate_job(job)
# create an async object
arsp = AgaveAsyncResponse(agave, job)
# block until job finishes with a timeout of 3 minutes.
assert arsp.result(180) == 'FINISHED'
def import_data(self, from_system, from_file_path):
remote_url = 'agave://{}/{}'.format(from_system, urllib.quote(from_file_path))
file_name = os.path.split(from_file_path)[1]
logger.debug('SystemId: %s, filePath: %s, fileName: %s, urlToingest: %s',
self.system, self.path, file_name, remote_url)
result = self._agave.files.importData(systemId=self.system,
filePath=urllib.quote(self.path),
fileName=str(file_name),
urlToIngest=remote_url)
async_resp = AgaveAsyncResponse(self._agave, result)
async_status = async_resp.result(600)
if async_status == 'FAILED':
logger.error('Import Data failed; from: %s/%s' % (from_system, from_file_path))
return BaseFileResource.listing(self._agave, self.system, result['path'])
def import_data(self, from_system, from_file_path):
remote_url = 'agave://{}/{}'.format(from_system, urllib.quote(from_file_path))
file_name = os.path.split(from_file_path)[1]
logger.debug('SystemId: %s, filePath: %s, fileName: %s, urlToingest: %s',
self.system, self.path, file_name, remote_url)
result = self._agave.files.importData(systemId=self.system,
filePath=urllib.quote(self.path),
fileName=file_name,
urlToIngest=remote_url)
async_resp = AgaveAsyncResponse(self._agave, result)
async_status = async_resp.result(600)
if async_status == 'FAILED':
logger.error('Import Data failed; from: %s/%s' % (from_system, from_file_path))
return BaseFileResource.listing(self._agave, self.system, result['path'])
def copy_box_item(self, username, box_item_type, box_item_id, target_system_id,
target_path):
user = get_user_model().objects.get(username=username)
client = user.box_user_token.client
try:
op = getattr(client, box_item_type)
item = op(box_item_id).get()
except AttributeError as e:
logger.error('Invalid box_item_type')
self.update_state(state='FAILURE', meta={'exc': e})
return
agave_client = user.agave_oauth.client
fm = FileManager(agave_client=agave_client)
if box_item_type == 'file':
try:
import_url = item.get_shared_link_download_url()
safe_name = get_valid_filename(item.name)
import_resp = agave_client.files.importData(systemId=target_system_id,
filePath=target_path,
fileName=safe_name,
urlToIngest=import_url)
async_resp = AgaveAsyncResponse(agave_client, import_resp)
async_status = async_resp.result(600)
if async_status == 'FAILED':
logger.error('Box File Transfer failed: %s' % target_path)
else:
file_path = '%s/%s' % (target_path, item.name)
logger.info('Indexing Box File Transfer %s' % file_path)
fm.index(settings.AGAVE_STORAGE_SYSTEM, file_path, username, levels=1)
except BoxAPIException as e:
logger.error('Unable to get download link from Box')
self.update_state(state='FAILURE', meta={'exc': e})
except HTTPError as e:
logger.error('Agave.files.importData raised HTTPError')
self.update_state(state='FAILURE', meta={'exc': e})
except (TimeoutError, Error) as e:
logger.error('Agave.files.importData failed to complete')
self.update_state(state='FAILURE', meta={'exc': e})
else: # box_item_type == 'folder'
# create directory for the folder
try:
safe_name = get_valid_filename(item.name)
mf, f = fm.mkdir(target_path, safe_name, target_system_id, username,
raise_if_exists=False)
logger.info('Created directory "{0}"; scheduling transfer of contents',
f.full_path)
except HTTPError as e:
logger.error('Agave.files.manage(mkdir) failed')
self.update_state(state='FAILURE', meta={'exc': e})
# schedule to copy all of it's items
limit = 10
offset = 0
while True:
item_collection = item.get_items(limit=limit, offset=offset)
for it in item_collection:
args = (username, it.type, it.object_id, target_system_id,
f.full_path)
copy_box_item.apply_async(args=args, countdown=offset*2)
if len(item_collection) == limit:
offset += limit
else:
break
######## Make Agave connection #######################
ag = Agave(api_server=URL, token=TOKEN)
############## Before initiating file transfer ###########################
### Before we initiate file transfer, we make sure that the test file is uploaded to the source system
### We use Agave file upload call to upload the file to the source system
### If the test file is already present this call can be skipped and this block can be commented
#filePath: The path of the file relative to the user’s default storage location. (string)
# More details can be found on AgavePy readthedocs
# https://agavepy.readthedocs.io/en/latest/agavepy.files.html#importdata-import-a-file-via-direct-upload-or-importing-from-a-url-to-the-user-s-default-storage-location
FILE_UPLOAD = ag.files.importData(
systemId=AGAVE_SOURCE_SYSTEM,
filePath='/Path where you want to upload file source system',
fileToUpload=open('AbsolutePath/local/SourceFile.txt', 'rb'))
arsp = AgaveAsyncResponse(ag, FILE_UPLOAD)
status = arsp.result(timeout=120)
print(status)
assert status == 'FINISHED'
####################### Agave file transfer##################################
#Transfer file from AGAVE_SOURCE_SYSTEM to AGAVE_DESTINATION_SYSTEM
FILE_TRANSFER = ag.files.importData(systemId=AGAVE_DESTINATION_SYSTEM,
fileName=FILE_DEST,
filePath='',
urlToIngest=URL_TO_INGEST)
arsp = AgaveAsyncResponse(ag, FILE_TRANSFER)
status = arsp.result(timeout=200)
assert status == 'FINISHED'
print(status)
#print("remote path is " + REMOTE_PATH)
def _call(self, systemId, filePath, fileName, fileToUpload):
"""
Wrap agavePy import data file command.
Args:
self: class instance
systemId: Identifier for Agave storage system
filePath: Path where file is to be imported
fileName: Name of the imported file
fileToUpload: File or folder path to upload to Agave
Returns:
On success: True.
On failure: False.
"""
if os.path.isdir(fileToUpload):
# create target directory, which is "fileName"
agwrap_mkdir = AgaveFilesMkDir(self._agave, self._config)
if not agwrap_mkdir.call(systemId, filePath, fileName):
Log.an().error('cannot create folder at uri: agave://%s%s/%s',
systemId, filePath, fileName)
return False
# walk through local directory structure
for root, dirs, files in os.walk(fileToUpload, topdown=True):
# upload each file in this directory level
for name in files:
# translate local path to dest path
dest_file_path = os.path.join(filePath, fileName,
root[len(fileToUpload) + 1:])
# read file in binary mode to transfer
response = self._agave.files.importData(
systemId=systemId,
filePath=dest_file_path,
fileName=name,
fileToUpload=open('%s/%s' % (root, name), "rb"))
async_response = AgaveAsyncResponse(self._agave, response)
status = async_response.result()
Log.some().info('import %s: %s/%s -> agave://%s/%s/%s',
str(status), root, name, systemId,
dest_file_path, name)
if status != 'FINISHED':
return False
# create new directory for each directory in this level
for name in dirs:
# translate local path to dest path
dest_file_path = os.path.join(filePath, fileName,
root[len(fileToUpload) + 1:])
# create dest directory
if not agwrap_mkdir.call(systemId, dest_file_path, name):
Log.an().error(
'cannot create folder at uri: agave://%s%s/%s',
systemId, dest_file_path, name)
return False
elif os.path.isfile(fileToUpload):
# import single file
response = self._agave.files.importData(systemId=systemId,
filePath=filePath,
fileName=fileName,
fileToUpload=open(
fileToUpload, 'rb'))
async_response = AgaveAsyncResponse(self._agave, response)
status = async_response.result()
Log.some().info('import %s: %s -> agave://%s/%s/%s', str(status),
fileToUpload, systemId, filePath, fileName)
if status != 'FINISHED':
return False
return True