def test_submit_job(agave, test_job): job = agave.jobs.submit(body=test_job) validate_job(job) # create an async object arsp = AgaveAsyncResponse(agave, job) # block until job finishes with a timeout of 3 minutes. assert arsp.result(180) == 'FINISHED'
def test_upload_binary_file(agave, credentials): rsp = agave.files.importData(systemId=credentials['storage'], filePath=credentials['storage_user'], fileToUpload=open('test_upload_python_sdk_g_art.mov', 'rb')) arsp = AgaveAsyncResponse(agave, rsp) status = arsp.result(timeout=120) assert status == 'FINISHED'
def _call(self, systemId, filePath, fileName, urlToIngest): """ Wrap agavePy import data file command. Args: self: class instance systemId: Identifier for Agave storage system filePath: Path where file is to be imported fileName: Name of the imported file urlToIngest: Agave URL to be ingested Returns: True """ response = self._agave.files.importData(systemId=systemId, filePath=filePath, fileName=fileName, urlToIngest=urlToIngest) async_response = AgaveAsyncResponse(self._agave, response) status = async_response.result() Log.some().info('import %s: %s -> agave://%s/%s/%s', str(status), urlToIngest, systemId, filePath, fileName) if str(status) == 'FINISHED': return True # not finished, try again raise Exception('agave import failed')
def test_submit_archive_job(agave, test_job, credentials): test_job['archive'] = True test_job['archiveSystem'] = credentials['storage'] job = agave.jobs.submit(body=test_job) validate_job(job) # create an async object arsp = AgaveAsyncResponse(agave, job) # block until job finishes with a timeout of 3 minutes. assert arsp.result(180) == 'FINISHED'
def import_data(self, from_system, from_file_path): remote_url = 'agave://{}/{}'.format(from_system, urllib.quote(from_file_path)) file_name = os.path.split(from_file_path)[1] logger.debug('SystemId: %s, filePath: %s, fileName: %s, urlToingest: %s', self.system, self.path, file_name, remote_url) result = self._agave.files.importData(systemId=self.system, filePath=urllib.quote(self.path), fileName=str(file_name), urlToIngest=remote_url) async_resp = AgaveAsyncResponse(self._agave, result) async_status = async_resp.result(600) if async_status == 'FAILED': logger.error('Import Data failed; from: %s/%s' % (from_system, from_file_path)) return BaseFileResource.listing(self._agave, self.system, result['path'])
######## Make Agave connection ####################### ag = Agave(api_server=URL, token=TOKEN) ############## Before initiating file transfer ########################### ### Before we initiate file transfer, we make sure that the test file is uploaded to the source system ### We use Agave file upload call to upload the file to the source system ### If the test file is already present this call can be skipped and this block can be commented #filePath: The path of the file relative to the user’s default storage location. (string) # More details can be found on AgavePy readthedocs # https://agavepy.readthedocs.io/en/latest/agavepy.files.html#importdata-import-a-file-via-direct-upload-or-importing-from-a-url-to-the-user-s-default-storage-location FILE_UPLOAD = ag.files.importData( systemId=AGAVE_SOURCE_SYSTEM, filePath='/Path where you want to upload file source system', fileToUpload=open('AbsolutePath/local/SourceFile.txt', 'rb')) arsp = AgaveAsyncResponse(ag, FILE_UPLOAD) status = arsp.result(timeout=120) print(status) assert status == 'FINISHED' ####################### Agave file transfer################################## #Transfer file from AGAVE_SOURCE_SYSTEM to AGAVE_DESTINATION_SYSTEM FILE_TRANSFER = ag.files.importData(systemId=AGAVE_DESTINATION_SYSTEM, fileName=FILE_DEST, filePath='', urlToIngest=URL_TO_INGEST) arsp = AgaveAsyncResponse(ag, FILE_TRANSFER) status = arsp.result(timeout=200) assert status == 'FINISHED' print(status) #print("remote path is " + REMOTE_PATH)
def _call(self, systemId, filePath, fileName, fileToUpload): """ Wrap agavePy import data file command. Args: self: class instance systemId: Identifier for Agave storage system filePath: Path where file is to be imported fileName: Name of the imported file fileToUpload: File or folder path to upload to Agave Returns: On success: True. On failure: False. """ if os.path.isdir(fileToUpload): # create target directory, which is "fileName" agwrap_mkdir = AgaveFilesMkDir(self._agave, self._config) if not agwrap_mkdir.call(systemId, filePath, fileName): Log.an().error('cannot create folder at uri: agave://%s%s/%s', systemId, filePath, fileName) return False # walk through local directory structure for root, dirs, files in os.walk(fileToUpload, topdown=True): # upload each file in this directory level for name in files: # translate local path to dest path dest_file_path = os.path.join(filePath, fileName, root[len(fileToUpload) + 1:]) # read file in binary mode to transfer response = self._agave.files.importData( systemId=systemId, filePath=dest_file_path, fileName=name, fileToUpload=open('%s/%s' % (root, name), "rb")) async_response = AgaveAsyncResponse(self._agave, response) status = async_response.result() Log.some().info('import %s: %s/%s -> agave://%s/%s/%s', str(status), root, name, systemId, dest_file_path, name) if status != 'FINISHED': return False # create new directory for each directory in this level for name in dirs: # translate local path to dest path dest_file_path = os.path.join(filePath, fileName, root[len(fileToUpload) + 1:]) # create dest directory if not agwrap_mkdir.call(systemId, dest_file_path, name): Log.an().error( 'cannot create folder at uri: agave://%s%s/%s', systemId, dest_file_path, name) return False elif os.path.isfile(fileToUpload): # import single file response = self._agave.files.importData(systemId=systemId, filePath=filePath, fileName=fileName, fileToUpload=open( fileToUpload, 'rb')) async_response = AgaveAsyncResponse(self._agave, response) status = async_response.result() Log.some().info('import %s: %s -> agave://%s/%s/%s', str(status), fileToUpload, systemId, filePath, fileName) if status != 'FINISHED': return False return True