Exemple #1
0
def test_submit_job(agave, test_job):
    job = agave.jobs.submit(body=test_job)
    validate_job(job)
    # create an async object
    arsp = AgaveAsyncResponse(agave, job)
    # block until job finishes with a timeout of 3 minutes.
    assert arsp.result(180) == 'FINISHED'
Exemple #2
0
def test_submit_job(agave, test_job):
    job = agave.jobs.submit(body=test_job)
    validate_job(job)
    # create an async object
    arsp = AgaveAsyncResponse(agave, job)
    # block until job finishes with a timeout of 3 minutes.
    assert arsp.result(180) == 'FINISHED'
Exemple #3
0
def test_upload_binary_file(agave, credentials):
    rsp = agave.files.importData(systemId=credentials['storage'],
                                 filePath=credentials['storage_user'],
                                 fileToUpload=open('test_upload_python_sdk_g_art.mov', 'rb'))
    arsp = AgaveAsyncResponse(agave, rsp)
    status = arsp.result(timeout=120)
    assert status == 'FINISHED'
Exemple #4
0
def test_upload_binary_file(agave, credentials):
    rsp = agave.files.importData(systemId=credentials['storage'],
                                 filePath=credentials['storage_user'],
                                 fileToUpload=open('test_upload_python_sdk_g_art.mov', 'rb'))
    arsp = AgaveAsyncResponse(agave, rsp)
    status = arsp.result(timeout=120)
    assert status == 'FINISHED'
Exemple #5
0
    def _call(self, systemId, filePath, fileName, urlToIngest):
        """
        Wrap agavePy import data file command.

        Args:
            self: class instance
            systemId: Identifier for Agave storage system
            filePath: Path where file is to be imported
            fileName: Name of the imported file
            urlToIngest: Agave URL to be ingested

        Returns:
            True

        """
        response = self._agave.files.importData(systemId=systemId,
                                                filePath=filePath,
                                                fileName=fileName,
                                                urlToIngest=urlToIngest)
        async_response = AgaveAsyncResponse(self._agave, response)
        status = async_response.result()
        Log.some().info('import %s: %s -> agave://%s/%s/%s', str(status),
                        urlToIngest, systemId, filePath, fileName)
        if str(status) == 'FINISHED':
            return True

        # not finished, try again
        raise Exception('agave import failed')
Exemple #6
0
def test_submit_archive_job(agave, test_job, credentials):
    test_job['archive'] = True
    test_job['archiveSystem'] = credentials['storage']
    job = agave.jobs.submit(body=test_job)
    validate_job(job)
    # create an async object
    arsp = AgaveAsyncResponse(agave, job)
    # block until job finishes with a timeout of 3 minutes.
    assert arsp.result(180) == 'FINISHED'
Exemple #7
0
def test_submit_archive_job(agave, test_job, credentials):
    test_job['archive'] = True
    test_job['archiveSystem'] = credentials['storage']
    job = agave.jobs.submit(body=test_job)
    validate_job(job)
    # create an async object
    arsp = AgaveAsyncResponse(agave, job)
    # block until job finishes with a timeout of 3 minutes.
    assert arsp.result(180) == 'FINISHED'
Exemple #8
0
    def import_data(self, from_system, from_file_path):
        remote_url = 'agave://{}/{}'.format(from_system, urllib.quote(from_file_path))
        file_name = os.path.split(from_file_path)[1]
        logger.debug('SystemId: %s, filePath: %s, fileName: %s, urlToingest: %s',
                     self.system, self.path, file_name, remote_url)
        result = self._agave.files.importData(systemId=self.system,
                                              filePath=urllib.quote(self.path),
                                              fileName=str(file_name),
                                              urlToIngest=remote_url)
        async_resp = AgaveAsyncResponse(self._agave, result)
        async_status = async_resp.result(600)

        if async_status == 'FAILED':
            logger.error('Import Data failed; from: %s/%s' % (from_system, from_file_path))

        return BaseFileResource.listing(self._agave, self.system, result['path'])
Exemple #9
0
    def import_data(self, from_system, from_file_path):
        remote_url = 'agave://{}/{}'.format(from_system, urllib.quote(from_file_path))
        file_name = os.path.split(from_file_path)[1]
        logger.debug('SystemId: %s, filePath: %s, fileName: %s, urlToingest: %s',
                     self.system, self.path, file_name, remote_url)
        result = self._agave.files.importData(systemId=self.system,
                                              filePath=urllib.quote(self.path),
                                              fileName=file_name,
                                              urlToIngest=remote_url)
        async_resp = AgaveAsyncResponse(self._agave, result)
        async_status = async_resp.result(600)

        if async_status == 'FAILED':
            logger.error('Import Data failed; from: %s/%s' % (from_system, from_file_path))

        return BaseFileResource.listing(self._agave, self.system, result['path'])
Exemple #10
0
def copy_box_item(self, username, box_item_type, box_item_id, target_system_id,
                  target_path):

    user = get_user_model().objects.get(username=username)
    client = user.box_user_token.client

    try:
        op = getattr(client, box_item_type)
        item = op(box_item_id).get()
    except AttributeError as e:
        logger.error('Invalid box_item_type')
        self.update_state(state='FAILURE', meta={'exc': e})
        return

    agave_client = user.agave_oauth.client
    fm = FileManager(agave_client=agave_client)

    if box_item_type == 'file':
        try:
            import_url = item.get_shared_link_download_url()
            safe_name = get_valid_filename(item.name)
            import_resp = agave_client.files.importData(systemId=target_system_id,
                                                        filePath=target_path,
                                                        fileName=safe_name,
                                                        urlToIngest=import_url)
            async_resp = AgaveAsyncResponse(agave_client, import_resp)
            async_status = async_resp.result(600)

            if async_status == 'FAILED':
                logger.error('Box File Transfer failed: %s' % target_path)
            else:
                file_path = '%s/%s' % (target_path, item.name)
                logger.info('Indexing Box File Transfer %s' % file_path)
                fm.index(settings.AGAVE_STORAGE_SYSTEM, file_path, username, levels=1)
        except BoxAPIException as e:
            logger.error('Unable to get download link from Box')
            self.update_state(state='FAILURE', meta={'exc': e})
        except HTTPError as e:
            logger.error('Agave.files.importData raised HTTPError')
            self.update_state(state='FAILURE', meta={'exc': e})
        except (TimeoutError, Error) as e:
            logger.error('Agave.files.importData failed to complete')
            self.update_state(state='FAILURE', meta={'exc': e})

    else:  # box_item_type == 'folder'

        # create directory for the folder
        try:
            safe_name = get_valid_filename(item.name)
            mf, f = fm.mkdir(target_path, safe_name, target_system_id, username,
                             raise_if_exists=False)
            logger.info('Created directory "{0}"; scheduling transfer of contents',
                        f.full_path)
        except HTTPError as e:
            logger.error('Agave.files.manage(mkdir) failed')
            self.update_state(state='FAILURE', meta={'exc': e})

        # schedule to copy all of it's items
        limit = 10
        offset = 0
        while True:
            item_collection = item.get_items(limit=limit, offset=offset)
            for it in item_collection:
                args = (username, it.type, it.object_id, target_system_id,
                        f.full_path)
                copy_box_item.apply_async(args=args, countdown=offset*2)
            if len(item_collection) == limit:
                offset += limit
            else:
                break
######## Make Agave connection #######################

ag = Agave(api_server=URL, token=TOKEN)

############## Before initiating file transfer ###########################
### Before we initiate file transfer, we make sure that the test file is uploaded to the source system
### We use Agave file upload call to upload the file to the source system
### If the test file is already present this call can be skipped and this block can be commented
#filePath: The path of the file relative to the user’s default storage location. (string)
# More details can be found on AgavePy readthedocs
# https://agavepy.readthedocs.io/en/latest/agavepy.files.html#importdata-import-a-file-via-direct-upload-or-importing-from-a-url-to-the-user-s-default-storage-location
FILE_UPLOAD = ag.files.importData(
    systemId=AGAVE_SOURCE_SYSTEM,
    filePath='/Path where you want to upload file source system',
    fileToUpload=open('AbsolutePath/local/SourceFile.txt', 'rb'))
arsp = AgaveAsyncResponse(ag, FILE_UPLOAD)
status = arsp.result(timeout=120)
print(status)
assert status == 'FINISHED'

####################### Agave file transfer##################################
#Transfer file from AGAVE_SOURCE_SYSTEM to AGAVE_DESTINATION_SYSTEM
FILE_TRANSFER = ag.files.importData(systemId=AGAVE_DESTINATION_SYSTEM,
                                    fileName=FILE_DEST,
                                    filePath='',
                                    urlToIngest=URL_TO_INGEST)
arsp = AgaveAsyncResponse(ag, FILE_TRANSFER)
status = arsp.result(timeout=200)
assert status == 'FINISHED'
print(status)
#print("remote path is " + REMOTE_PATH)
Exemple #12
0
    def _call(self, systemId, filePath, fileName, fileToUpload):
        """
        Wrap agavePy import data file command.

        Args:
            self: class instance
            systemId: Identifier for Agave storage system
            filePath: Path where file is to be imported
            fileName: Name of the imported file
            fileToUpload: File or folder path to upload to Agave

        Returns:
            On success: True.
            On failure: False.

        """
        if os.path.isdir(fileToUpload):
            # create target directory, which is "fileName"
            agwrap_mkdir = AgaveFilesMkDir(self._agave, self._config)
            if not agwrap_mkdir.call(systemId, filePath, fileName):
                Log.an().error('cannot create folder at uri: agave://%s%s/%s',
                               systemId, filePath, fileName)
                return False

            # walk through local directory structure
            for root, dirs, files in os.walk(fileToUpload, topdown=True):
                # upload each file in this directory level
                for name in files:
                    # translate local path to dest path
                    dest_file_path = os.path.join(filePath, fileName,
                                                  root[len(fileToUpload) + 1:])
                    # read file in binary mode to transfer
                    response = self._agave.files.importData(
                        systemId=systemId,
                        filePath=dest_file_path,
                        fileName=name,
                        fileToUpload=open('%s/%s' % (root, name), "rb"))
                    async_response = AgaveAsyncResponse(self._agave, response)
                    status = async_response.result()
                    Log.some().info('import %s: %s/%s -> agave://%s/%s/%s',
                                    str(status), root, name, systemId,
                                    dest_file_path, name)
                    if status != 'FINISHED':
                        return False

                # create new directory for each directory in this level
                for name in dirs:
                    # translate local path to dest path
                    dest_file_path = os.path.join(filePath, fileName,
                                                  root[len(fileToUpload) + 1:])
                    # create dest directory
                    if not agwrap_mkdir.call(systemId, dest_file_path, name):
                        Log.an().error(
                            'cannot create folder at uri: agave://%s%s/%s',
                            systemId, dest_file_path, name)
                        return False

        elif os.path.isfile(fileToUpload):
            # import single file
            response = self._agave.files.importData(systemId=systemId,
                                                    filePath=filePath,
                                                    fileName=fileName,
                                                    fileToUpload=open(
                                                        fileToUpload, 'rb'))
            async_response = AgaveAsyncResponse(self._agave, response)
            status = async_response.result()
            Log.some().info('import %s: %s -> agave://%s/%s/%s', str(status),
                            fileToUpload, systemId, filePath, fileName)
            if status != 'FINISHED':
                return False

        return True