コード例 #1
0
    def test_check_file_size_needed(self):
        """test that the file size returned from the archive is parsed right"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):
            response = """{
                            "bytes_per_level": "(24L, 0L, 0L, 0L, 0L)", 
                            "ctime": "1444938166", 
                            "file": "/myemsl-dev/bundle/file.1", 
                            "file_storage_media": "disk", 
                            "filesize": "24", 
                            "message": "File was found", 
                            "mtime": "1444938166"
                            }"""
            test_cart = Cart.create(cart_uid='1', status="staging")
            test_file = File.create(cart=test_cart, file_name="1.txt",
                                    bundle_path="/tmp/1/1.txt")
            cart_utils = Cartutils()
            file_size = cart_utils.check_file_size_needed(response, test_file,
                                                          test_cart)
            self.assertEqual(file_size, 24)
            self.assertNotEqual(test_file.status, "error")

            #now check for an error by sending a bad response
            file_size = cart_utils.check_file_size_needed("", test_file,
                                                          test_cart)
            self.assertEqual(file_size, -1)
            self.assertEqual(test_file.status, "error")
コード例 #2
0
    def test_check_file_size_needed(self):
        """test that the file size returned from the archive is parsed right"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):
            response = """{
                            "bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
                            "ctime": "1444938166",
                            "file": "/myemsl-dev/bundle/file.1",
                            "file_storage_media": "disk",
                            "filesize": "24",
                            "message": "File was found",
                            "mtime": "1444938166"
                            }"""
            test_cart = Cart.create(cart_uid='1', status='staging')
            test_file = File.create(cart=test_cart,
                                    file_name='1.txt',
                                    bundle_path='/tmp/1/1.txt')
            cart_utils = Cartutils()
            file_size = cart_utils.check_file_size_needed(
                response, test_file, test_cart)
            self.assertEqual(file_size, 24)
            self.assertNotEqual(test_file.status, 'error')

            #now check for an error by sending a bad response
            file_size = cart_utils.check_file_size_needed(
                '', test_file, test_cart)
            self.assertEqual(file_size, -1)
            self.assertEqual(test_file.status, 'error')
コード例 #3
0
    def test_get_path_size(self):
        """test to see if the path size of a directory is returned"""

        cart_utils = Cartutils()
        path = os.path.dirname(os.path.realpath(__file__))
        rtn = cart_utils.get_path_size(path + '/../')
        self.assertNotEqual(rtn, 0)
コード例 #4
0
    def test_get_path_size(self):
        """test to see if the path size of a directory is returned"""

        cart_utils = Cartutils()
        path = os.path.dirname(os.path.realpath(__file__))
        rtn = cart_utils.get_path_size(path)
        self.assertNotEqual(rtn, 0)
コード例 #5
0
 def test_create_bundle_directories(self):
     """test the  creation of directories where files will be saved"""
     directory_name = '/tmp/fakedir/'
     cart_utils = Cartutils()
     cart_utils.create_bundle_directories(directory_name)
     self.assertEqual(os.path.isdir(directory_name), True)
     os.rmdir(directory_name)
     self.assertEqual(os.path.isdir(directory_name), False)
コード例 #6
0
 def test_create_bundle_directories(self):
     """test the creation of direcoties were files will be saved"""
     directory_name = "/tmp/fakedir/"
     cart_utils = Cartutils()
     cart_utils.create_bundle_directories(directory_name)
     self.assertEqual(os.path.isdir(directory_name), True)
     os.rmdir(directory_name)
     self.assertEqual(os.path.isdir(directory_name), False)
コード例 #7
0
 def test_fix_absolute_path(self):
     """test the correct creation of paths by removing absolute paths"""
     cart_utils = Cartutils()
     return_one = cart_utils.fix_absolute_path("tmp/foo.text")
     return_two = cart_utils.fix_absolute_path("/tmp/foo.text")
     return_three = cart_utils.fix_absolute_path("/tmp/foo.text")
     self.assertEqual(return_one, "tmp/foo.text")
     self.assertEqual(return_two, "tmp/foo.text")
     self.assertNotEqual(return_three, "/tmp/foo.text")
コード例 #8
0
 def test_fix_absolute_path(self):
     """test the correct creation of paths by removing absolute paths"""
     cart_utils = Cartutils()
     return_one = cart_utils.fix_absolute_path('tmp/foo.text')
     return_two = cart_utils.fix_absolute_path('/tmp/foo.text')
     return_three = cart_utils.fix_absolute_path('/tmp/foo.text')
     self.assertEqual(return_one, 'tmp/foo.text')
     self.assertEqual(return_two, 'tmp/foo.text')
     self.assertNotEqual(return_three, '/tmp/foo.text')
コード例 #9
0
    def test_delete_cart_bundle(self):
        """test that trys to delete a cart bundle"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1', status="staging",
                                    bundle_path="/tmp/1/")
            cart_utils = Cartutils()
            os.makedirs(test_cart.bundle_path, 0777)
            deleted = cart_utils.delete_cart_bundle(test_cart)
            self.assertEqual(test_cart.status, "deleted")
            self.assertEqual(deleted, True)
            self.assertEqual(os.path.isdir(test_cart.bundle_path), False)
コード例 #10
0
 def test_bad_create_download_path(self, mock_create_bundle):
     """test the creation of the download path for a cart file"""
     with test_database(SqliteDatabase(':memory:'), (Cart, File)):
         test_cart = Cart.create(cart_uid='1', status='staging')
         test_file = File.create(cart=test_cart,
                                 file_name='1.txt',
                                 bundle_path='/tmp/1/1.txt')
         cart_utils = Cartutils()
         mock_create_bundle.side_effect = OSError(mock.Mock(), 'Error')
         success = cart_utils.create_download_path(test_file, test_cart,
                                                   test_file.bundle_path)
         self.assertEqual(success, False)
コード例 #11
0
    def test_set_file_status(self):
        """test that trys to set a specific files status"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1', status="staging",
                                    bundle_path="/tmp/1/")
            test_file = File.create(cart=test_cart, file_name="1.txt",
                                    bundle_path="/tmp/1/1.txt")
            cart_utils = Cartutils()
            cart_utils.set_file_status(test_file, test_cart, "error", "fake error")
            self.assertEqual(test_file.status, "error")
            self.assertEqual(test_file.error, "fake error")
コード例 #12
0
ファイル: tasks.py プロジェクト: EMSL-MSC/pacifica-cartd
def stage_files(file_ids, uid):
    """Tell the files to be staged on the backend system """
    Cart.database_connect()
    mycart = Cart(cart_uid=uid, status="staging")
    mycart.save()
    #with update or new, need to add in files
    cart_utils = Cartutils()
    cart_utils.update_cart_files(mycart, file_ids)

    get_files_locally.delay(mycart.id)
    prepare_bundle.delay(mycart.id)
    Cart.database_close()
コード例 #13
0
    def test_delete_cart_bundle(self):
        """test that trys to delete a cart bundle"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1',
                                    status='staging',
                                    bundle_path='/tmp/1/')
            cart_utils = Cartutils()
            os.makedirs(test_cart.bundle_path, 0o777)
            deleted = cart_utils.delete_cart_bundle(test_cart)
            self.assertEqual(test_cart.status, 'deleted')
            self.assertEqual(deleted, True)
            self.assertEqual(os.path.isdir(test_cart.bundle_path), False)
コード例 #14
0
    def test_check_space_bad_path(self, mock_disk_usage):
        """test that the error when a bad path"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1', status='staging')
            test_file = File.create(cart=test_cart,
                                    file_name='1.txt',
                                    bundle_path='/tmp/1/1.txt')
            cart_utils = Cartutils()
            mock_disk_usage.side_effect = psutil.Error(mock.Mock())
            rtn = cart_utils.check_space_requirements(test_file, test_cart, 10,
                                                      False)
            self.assertEqual(rtn, False)
            self.assertEqual(test_file.status, 'error')
コード例 #15
0
 def test_prep_bundle_staging(self):
     """test getting bundle ready with a file in staging state"""
     data = json.loads(cart_json_helper())
     file_ids = data['fileids']
     Cart.database_connect()
     mycart = Cart(cart_uid=343, status='staging')
     mycart.save()
     cart_utils = Cartutils()
     cart_utils.update_cart_files(mycart, file_ids)
     get_files_locally(mycart.id)
     for cart_file in File.select().where(File.cart == mycart.id):
         cart_file.status = 'staging'
         cart_file.save()
     cart_utils.prepare_bundle(
         mycart.id)  #hitting more coverage, set files to staged
     for cart_file in File.select().where(File.cart == mycart.id):
         cart_file.status = 'staged'
         cart_file.save()
     cart_utils.prepare_bundle(mycart.id)  #call again after file update
     status = mycart.status
     cartid = mycart.id
     while status == 'staging':
         mycart = Cart.get(Cart.id == cartid)
         status = mycart.status
     Cart.database_close()
     self.assertEqual(status, 'ready')
コード例 #16
0
    def status(self, env, start_response):
        """Get the status of a carts tar file"""
        resp = cart_interface_responses.Responses()
        uid = fix_cart_uid(env['PATH_INFO'])
        is_valid = is_valid_uid(uid)
        if not is_valid:
            self._response = resp.invalid_uid_error_response(
                start_response, uid)
            return self.return_response()

        cart_utils = Cartutils()
        status = cart_utils.cart_status(uid)
        self._response = resp.cart_status_response(start_response, status)
        return self.return_response()
コード例 #17
0
    def test_delete_cart_bundle_fail(self, mock_rmtree):
        """test that trys to delete a cart bundle but fails"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1',
                                    status='staging',
                                    bundle_path='/tmp/1/')
            cart_utils = Cartutils()
            os.makedirs(test_cart.bundle_path, 0o777)
            mock_rmtree.side_effect = OSError(mock.Mock(), 'Error')
            deleted = cart_utils.delete_cart_bundle(test_cart)
            self.assertNotEqual(test_cart.status, 'deleted')
            self.assertEqual(deleted, False)
            self.assertEqual(os.path.isdir(test_cart.bundle_path), True)
コード例 #18
0
def stage_file_task(file_id):
    """Stage the file from the archive, then call status """
    Cart.database_connect()
    try:
        cart_file = File.get(File.id == file_id)
        mycart = cart_file.cart
        cart_utils = Cartutils()
        cart_utils.set_file_status(cart_file, mycart, 'staging', False)
        #make sure cart wasnt deleted before pulling file
        if mycart.deleted_date:
            Cart.database_close()
            return
    except DoesNotExist:
        Cart.database_close()
        return
    archive_request = ArchiveRequests()
    try:
        archive_request.stage_file(cart_file.file_name)
    except requests.exceptions.RequestException as ex:
        error_msg = 'Failed to stage with error: ' + str(ex)
        cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
        Cart.database_close()
        cart_utils.prepare_bundle(mycart.id)
        return
    #successful stage so move on to status
    Cart.database_close()
    status_file_task.delay(file_id)
コード例 #19
0
    def test_create_download_path(self):
        """test the creation of the download path for a cart file"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):
            test_cart = Cart.create(cart_uid='1', status="staging")
            test_file = File.create(cart=test_cart, file_name="1.txt",
                                    bundle_path="/tmp/1/1.txt")
            cart_utils = Cartutils()
            success = cart_utils.create_download_path(test_file, test_cart,
                                                      test_file.bundle_path)
            directory_name = os.path.dirname(test_file.bundle_path)
            self.assertEqual(success, True)
            self.assertEqual(os.path.isdir(directory_name), True)

            os.rmdir(directory_name)
            self.assertEqual(os.path.isdir(directory_name), False)
コード例 #20
0
    def test_cart_no_hash_passed(self):
        """test error with cart with no hash passed"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1',
                                    status='staging',
                                    bundle_path='/tmp/1/')
            cart_utils = Cartutils()

            data = json.loads(
                '{"fileids": [{"id":"foo.txt", "path":"1/2/3/foo.txt", "hashtype":"md5"}]}'
            )

            file_ids = data['fileids']
            retval = cart_utils.update_cart_files(test_cart, file_ids)
            self.assertNotEqual(retval, None)
コード例 #21
0
def stage_files(file_ids, mycart_id):
    """Tell the files to be staged on the backend system """
    Cart.database_connect()
    #with update or new, need to add in files
    mycart = Cart.get(Cart.id == mycart_id)
    cart_utils = Cartutils()
    file_id_error = cart_utils.update_cart_files(mycart, file_ids)
    if not file_id_error:
        get_files_locally.delay(mycart.id)
    else:
        mycart.status = 'error'
        mycart.error = 'Error parsing file Ids with error: ' + str(
            file_id_error)
        mycart.updated_date = datetime.datetime.now()
        mycart.save()
    Cart.database_close()
コード例 #22
0
    def test_create_download_path(self):
        """test the creation of the download path for a cart file"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):
            test_cart = Cart.create(cart_uid='1', status='staging')
            test_file = File.create(cart=test_cart,
                                    file_name='1.txt',
                                    bundle_path='/tmp/1/1.txt')
            cart_utils = Cartutils()
            success = cart_utils.create_download_path(test_file, test_cart,
                                                      test_file.bundle_path)
            directory_name = os.path.dirname(test_file.bundle_path)
            self.assertEqual(success, True)
            self.assertEqual(os.path.isdir(directory_name), True)

            os.rmdir(directory_name)
            self.assertEqual(os.path.isdir(directory_name), False)
コード例 #23
0
    def test_status_details_fail(self):
        """test status details fail"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1',
                                    status='staging',
                                    bundle_path='/tmp/1/')
            test_file = File.create(cart=test_cart,
                                    file_name='1.txt',
                                    bundle_path='/tmp/1/1.txt')
            cart_utils = Cartutils()

            #say file is way to big
            retval = cart_utils.check_status_details(
                test_cart, test_file, 99999999999999999999999999999, 1)
            self.assertEqual(retval, -1)
コード例 #24
0
    def test_set_file_status(self):
        """test that trys to set a specific files status"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1',
                                    status='staging',
                                    bundle_path='/tmp/1/')
            test_file = File.create(cart=test_cart,
                                    file_name='1.txt',
                                    bundle_path='/tmp/1/1.txt')
            cart_utils = Cartutils()

            cart_utils.set_file_status(test_file, test_cart, 'error',
                                       'fake error')
            self.assertEqual(test_file.status, 'error')
            self.assertEqual(test_file.error, 'fake error')
コード例 #25
0
    def delete_cart(self, env, start_response):
        """Delete a cart that has been created"""
        resp = cart_interface_responses.Responses()
        uid = fix_cart_uid(env['PATH_INFO'])
        is_valid = is_valid_uid(uid)
        if not is_valid:
            self._response = resp.invalid_uid_error_response(
                start_response, uid)
            return self.return_response()

        cart_utils = Cartutils()
        message = cart_utils.remove_cart(uid)
        if message is False:
            self._response = resp.cart_not_found(start_response)
        else:
            self._response = resp.cart_delete_response(start_response, message)
        return self.return_response()
コード例 #26
0
    def test_check_space_requirements(self):
        """test that there is enough space on the volume for the file"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1', status="staging")
            test_file = File.create(cart=test_cart, file_name="1.txt",
                                    bundle_path="/tmp/1/1.txt")
            cart_utils = Cartutils()
            rtn = cart_utils.check_space_requirements(test_file, test_cart,
                                                      10, False)
            self.assertEqual(rtn, True)
            self.assertNotEqual(test_file.status, "error")

            #now check for an error by sending a way to large size needed number
            rtn = cart_utils.check_space_requirements(test_file, test_cart,
                                                      9999999999999999999999, False)
            self.assertEqual(rtn, False)
            self.assertEqual(test_file.status, "error")
コード例 #27
0
 def test_cart_deleted_date(self):
     """test getting bundle ready with a file in staging state"""
     data = json.loads(cart_json_helper())
     file_ids = data['fileids']
     Cart.database_connect()
     mycart = Cart(cart_uid=444, status='staging')
     mycart.save()
     cart_utils = Cartutils()
     cart_utils.update_cart_files(mycart, file_ids)
     get_files_locally(mycart.id)
     mycart.status = 'deleted'
     mycart.deleted_date = datetime.datetime.now()
     mycart.save()
     status = mycart.status
     for cart_file in File.select().where(File.cart == mycart.id):
         pull_file(cart_file.id, '/tmp/some/Path', '1111', False)
     Cart.database_close()
     self.assertEqual(status, 'deleted')
コード例 #28
0
    def test_lru_cart_delete(self):
        """test that trys to delete a cart"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1',
                                    status='staging',
                                    bundle_path='/tmp/1/')
            test_cart2 = Cart.create(cart_uid='2',
                                     status='staging',
                                     bundle_path='/tmp/2/',
                                     updated_date=1)
            cart_utils = Cartutils()
            os.makedirs(test_cart2.bundle_path, 0o777)
            retval = cart_utils.lru_cart_delete(test_cart)
            self.assertEqual(retval, True)
            test_c2 = Cart.get(Cart.id == test_cart2.id)
            self.assertEqual(test_c2.status, 'deleted')
            #also hit error block when nothing to delete
            retval = cart_utils.lru_cart_delete(test_cart)
            self.assertEqual(retval, False)
コード例 #29
0
    def test_check_space_requirements(self):
        """test that there is enough space on the volume for the file"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):

            test_cart = Cart.create(cart_uid='1', status='staging')
            test_file = File.create(cart=test_cart,
                                    file_name='1.txt',
                                    bundle_path='/tmp/1/1.txt')
            cart_utils = Cartutils()
            rtn = cart_utils.check_space_requirements(test_file, test_cart, 10,
                                                      False)
            self.assertEqual(rtn, True)
            self.assertNotEqual(test_file.status, 'error')

            #now check for an error by sending a way to large size needed number
            rtn = cart_utils.check_space_requirements(test_file, test_cart,
                                                      9999999999999999999999,
                                                      True)
            self.assertEqual(rtn, False)
            self.assertEqual(test_file.status, 'error')
コード例 #30
0
    def test_check_file_not_ready_pull(self):
        """test that checks to see if a file is not ready to pull
        by checking the archive response"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):
            response = """{
                            "bytes_per_level": "(0L, 24L, 0L, 0L, 0L)",
                            "ctime": "1444938166",
                            "file": "/myemsl-dev/bundle/file.1",
                            "file_storage_media": "tape",
                            "filesize": "24",
                            "message": "File was found",
                            "mtime": "1444938166"
                            }"""
            resp_bad = """{
                            "bytes_per_level": "(0L, 33L, 33L, 0L, 0L)",
                            "ctime": "1444938177",
                            "file": "/myemsl-dev/bundle/file.2",
                            "filesize": "33",
                            "message": "File was found",
                            "mtime": "1444938133"
                            }"""
            test_cart = Cart.create(cart_uid='1', status='staging')
            test_file = File.create(cart=test_cart,
                                    file_name='1.txt',
                                    bundle_path='/tmp/1/1.txt')
            cart_utils = Cartutils()
            ready = cart_utils.check_file_ready_pull(response, test_file,
                                                     test_cart)
            self.assertEqual(ready, False)

            #now check for an error by sending a bad response
            ready = cart_utils.check_file_ready_pull('', test_file, test_cart)
            self.assertEqual(ready, -1)
            self.assertEqual(test_file.status, 'error')

            #now check for an error with storage media
            ready = cart_utils.check_file_ready_pull(resp_bad, test_file,
                                                     test_cart)
            self.assertEqual(ready, -1)
            self.assertEqual(test_file.status, 'error')
コード例 #31
0
    def test_bad_stage(self, mock_delete_cart):
        """test the bad stage of a archive file"""
        with test_database(SqliteDatabase(':memory:'), (Cart, File)):
            test_cart = Cart.create(cart_uid='1', status='staging')

            def fake_database_connect(cls_name):
                """no error"""
                return cls_name

            def fake_database_close(cls_name):
                """no error"""
                return cls_name

            cart.cart_orm.CartBase.database_connect = MethodType(
                fake_database_connect, cart.cart_orm.CartBase)
            cart.cart_orm.CartBase.database_close = MethodType(
                fake_database_close, cart.cart_orm.CartBase)
            cart.cart_orm.CartBase.throw_error = False
            mock_delete_cart.return_value = False
            cart_util = Cartutils()
            return_val = cart_util.remove_cart(test_cart.id)
            self.assertEqual(return_val, None)
コード例 #32
0
 def test_check_file_ready_pull(self):
     """test that checks to see if a file is ready to pull
     by checking the archive response"""
     with test_database(SqliteDatabase(':memory:'), (Cart, File)):
         response = """{
                         "bytes_per_level": "(24L, 0L, 0L, 0L, 0L)",
                         "ctime": "1444938166",
                         "file": "/myemsl-dev/bundle/file.1",
                         "file_storage_media": "disk",
                         "filesize": "24",
                         "message": "File was found",
                         "mtime": "1444938166"
                         }"""
         test_cart = Cart.create(cart_uid='1', status='staging')
         test_file = File.create(cart=test_cart,
                                 file_name='1.txt',
                                 bundle_path='/tmp/1/1.txt')
         cart_utils = Cartutils()
         ready = cart_utils.check_file_ready_pull(response, test_file,
                                                  test_cart)
         self.assertEqual(ready['enough_space'], True)
         self.assertNotEqual(test_file.status, 'error')
コード例 #33
0
def status_file_task(file_id):
    """Status a file from the archive. If ready then pull the file"""
    Cart.database_connect()
    cart_file = File.get(File.id == file_id)
    mycart = cart_file.cart
    cart_utils = Cartutils()
    #make sure cart wasnt deleted before pulling file
    if mycart.deleted_date:
        Cart.database_close()
        return

    #check to see if file is available to pull from archive interface
    archive_request = ArchiveRequests()
    try:
        response = archive_request.status_file(cart_file.file_name)
    except requests.exceptions.RequestException as ex:
        error_msg = 'Failed to status file with error: ' + str(ex)
        cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
        Cart.database_close()
        cart_utils.prepare_bundle(mycart.id)
        return

    ready = cart_utils.check_file_ready_pull(response, cart_file, mycart)

    #Check to see if ready to pull.  If not recall this to check again
    # error on less then 0. No coverage on recall since it just calls the method again
    if ready < 0:
        Cart.database_close()
        cart_utils.prepare_bundle(mycart.id)
        return
    elif not ready:  # pragma: no cover
        Cart.database_close()
        status_file_task.delay(file_id)
        return
    #ready so try to pull file
    pull_file.delay(file_id, ready['filepath'], ready['modtime'], False)
コード例 #34
0
 def test_stage_files(self):
     """test getting bundle files ready"""
     data = json.loads(cart_json_helper())
     file_ids = data['fileids']
     Cart.database_connect()
     mycart = Cart(cart_uid=747, status='staging')
     mycart.save()
     cart_utils = Cartutils()
     cart_utils.update_cart_files(mycart, file_ids)
     stage_files(file_ids, mycart.id)
     cart_utils.prepare_bundle(mycart.id)
     status = mycart.status
     cartid = mycart.id
     while status == 'staging':
         mycart = Cart.get(Cart.id == cartid)
         status = mycart.status
     Cart.database_close()
     self.assertEqual(status, 'ready')
コード例 #35
0
 def test_prep_bundle_error(self):
     """test getting bundle ready with a file in error state"""
     data = json.loads(cart_json_helper())
     file_ids = data['fileids']
     Cart.database_connect()
     mycart = Cart(cart_uid=343, status='staging')
     mycart.save()
     cart_utils = Cartutils()
     cart_utils.update_cart_files(mycart, file_ids)
     get_files_locally(mycart.id)
     for cart_file in File.select().where(File.cart == mycart.id):
         cart_file.status = 'error'
         cart_file.save()
     cart_utils.prepare_bundle(mycart.id)
     status = mycart.status
     cartid = mycart.id
     while status == 'staging':
         mycart = Cart.get(Cart.id == cartid)
         status = mycart.status
     Cart.database_close()
     self.assertEqual(status, 'error')
コード例 #36
0
def pull_file(file_id, filepath, modtime, record_error):
    """Pull a file from the archive  """
    Cart.database_connect()
    try:
        cart_file = File.get(File.id == file_id)
        mycart = cart_file.cart
        cart_utils = Cartutils()
        #make sure cart wasnt deleted before pulling file
        if mycart.deleted_date:
            Cart.database_close()
            return
    except DoesNotExist:
        Cart.database_close()
        return

    archive_request = ArchiveRequests()
    try:
        archive_request.pull_file(cart_file.file_name, filepath,
                                  cart_file.hash_value, cart_file.hash_type)
        cart_utils.set_file_status(cart_file, mycart, 'staged', False)
        os.utime(filepath, (int(float(modtime)), int(float(modtime))))
        Cart.database_close()
    except requests.exceptions.RequestException as ex:
        #if request fails...try a second time, if that fails write error
        if record_error:
            error_msg = 'Failed to pull with error: ' + str(ex)
            cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
            Cart.database_close()
            cart_utils.prepare_bundle(mycart.id)

        else:
            pull_file.delay(file_id, filepath, modtime, True)
            Cart.database_close()

    except ValueError as ex:
        error_msg = 'Failed to pull with error: ' + str(ex)
        cart_utils.set_file_status(cart_file, mycart, 'error', error_msg)
        Cart.database_close()
        cart_utils.prepare_bundle(mycart.id)

    cart_utils.prepare_bundle(mycart.id)
コード例 #37
0
 def test_tar_invalid_cart(self):
     """test pulling a file id that doesnt exist"""
     cart_utils = Cartutils()
     cart_utils.tar_files('8765', True)
     #no action happens on invalid cart to tar, so no assertion to check
     self.assertEqual(True, True)
コード例 #38
0
 def test_bad_makedirs(self, mock_makedirs):
     """test a error return from a file not ready to pull"""
     mock_makedirs.side_effect = OSError(mock.Mock(), 'Error')
     c_util = Cartutils()
     self.assertRaises(OSError, c_util.create_bundle_directories,
                       "fakepath")
コード例 #39
0
 def get(self, env, start_response):
     """Download the tar file created by the cart"""
     resp = cart_interface_responses.Responses()
     rtn_name = None
     if 'filename' in parse_qs(env['QUERY_STRING']):
         rtn_name = os.path.basename(
             parse_qs(env['QUERY_STRING'])['filename'][0])
     else:
         rtn_name = "data_" + datetime.now().strftime(
             '%Y_%m_%d_%H_%M_%S') + ".tar"
     uid = fix_cart_uid(env['PATH_INFO'])
     is_valid = is_valid_uid(uid)
     if not is_valid:
         self._response = resp.invalid_uid_error_response(
             start_response, uid)
         return self.return_response()
     #get the bundle path if available
     cart_utils = Cartutils()
     cart_path = cart_utils.available_cart(uid)
     if cart_path is False:
         #cart not ready
         self._response = resp.unready_cart(start_response)
     elif cart_path is None:
         #cart not found
         self._response = resp.cart_not_found(start_response)
     else:
         if os.path.isdir(cart_path):
             #give back bundle here
             stderr.flush()
             try:
                 #want to stream the tar file out
                 (rpipe, wpipe) = os.pipe()
                 cpid = os.fork()
                 # the fork screws up coverage testing... :(
                 if cpid == 0:  # pragma: no cover
                     # we are the child process
                     #write the data to the pipe
                     os.close(rpipe)
                     wfd = os.fdopen(wpipe, 'wb')
                     mytar = TarFile.open(fileobj=wfd, mode='w|')
                     mytar.add(cart_path,
                               arcname=rtn_name.replace('.tar', ''))
                     mytar.close()
                     #to exit from the fork child without killing the parent
                     #we need to use_exit.  Disabling the pylint for this
                     #so that it doesnt throw an error
                     # pylint: disable=protected-access
                     os._exit(0)
                     # pylint: enable=protected-access
                 # we are the parent
                 os.close(wpipe)
                 #open the pipe as a file
                 rfd = os.fdopen(rpipe, 'rb')
                 start_response(
                     '200 OK',
                     [('Content-Type', 'application/octet-stream'),
                      ('Content-Disposition',
                       'attachment; filename=' + str(rtn_name))])
                 if 'wsgi.file_wrapper' in env:
                     return env['wsgi.file_wrapper'](rfd, BLOCK_SIZE)
                 return iter(lambda: rfd.read(BLOCK_SIZE), '')
             except IOError:
                 self._response = resp.bundle_doesnt_exist(start_response)
         else:
             self._response = resp.bundle_doesnt_exist(start_response)
             return self.return_response()
     return self.return_response()
コード例 #40
0
ファイル: tasks.py プロジェクト: EMSL-MSC/pacifica-cartd
def pull_file(file_id, record_error):
    """Pull a file from the archive  """
    Cart.database_connect()
    try:
        cart_file = File.get(File.id == file_id)
        mycart = cart_file.cart
        cart_utils = Cartutils()
        cart_utils.set_file_status(cart_file, mycart, "staging", False)
        #make sure cart wasnt deleted before pulling file
        if mycart.deleted_date:
            return
    except DoesNotExist:
        Cart.database_close()
        return

    archive_request = ArchiveRequests()
    #stage the file on the archive.  True on success, False on fail
    try:
        archive_request.stage_file(cart_file.file_name)
    except requests.exceptions.RequestException as ex:
        error_msg = "Failed to stage with error: " + str(ex)
        cart_utils.set_file_status(cart_file, mycart, "error", error_msg)
        Cart.database_close()
        return

    #check to see if file is available to pull from archive interface
    try:
        response = archive_request.status_file(cart_file.file_name)
    except requests.exceptions.RequestException as ex:
        error_msg = "Failed to status file with error: " + str(ex)
        cart_utils.set_file_status(cart_file, mycart, "error", error_msg)
        response = 'False'

    size_needed = cart_utils.check_file_size_needed(response, cart_file, mycart)
    mod_time = cart_utils.check_file_modified_time(response, cart_file, mycart)
    #Return from function if the values couldnt be parsed (-1 return)
    if size_needed < 0 or mod_time < 0:
        Cart.database_close()
        return

    ready_to_pull = cart_utils.check_file_ready_pull(
        response, cart_file, mycart)

    #Check to see if ready to pull.  If not recall this to check again
    # error on less then 0
    if ready_to_pull < 0:
        Cart.database_close()
        return
    elif not ready_to_pull:
        pull_file.delay(file_id, False)
        Cart.database_close()
        return

    #create the path the file will be downloaded to
    abs_cart_file_path = os.path.join(
        VOLUME_PATH, str(mycart.id), mycart.cart_uid, cart_file.bundle_path)
    path_created = cart_utils.create_download_path(
        cart_file, mycart, abs_cart_file_path)
    #Check size here and make sure enough space is available.
    enough_space = cart_utils.check_space_requirements(
        cart_file, mycart, size_needed, True)

    if path_created and enough_space:
        try:
            archive_request.pull_file(cart_file.file_name, abs_cart_file_path)
            cart_utils.set_file_status(cart_file, mycart, "staged", False)
            Cart.database_close()
        except requests.exceptions.RequestException as ex:
            #if request fails...try a second time, if that fails write error
            if record_error:
                error_msg = "Failed to pull with error: " + str(ex)
                cart_utils.set_file_status(
                    cart_file, mycart, "error", error_msg)
                Cart.database_close()
            else:
                pull_file.delay(file_id, True)
                Cart.database_close()

        os.utime(abs_cart_file_path, (int(float(mod_time)), int(float(mod_time))))