def test_filemgr_list_files(self, mock_file, mock_api, mock_creds, mock_cfg): """Test list_files""" mgr = FileManager(mock_creds, cfg=mock_cfg) resp = mock.create_autospec(Response) resp.success = False resp.result = RestCallException(None, "test", None) mgr._client.list_files.return_value = resp with self.assertRaises(RestCallException): test = mgr.list_files() self.assertTrue(mgr._client.list_files.called) self.assertFalse(mock_file.called) resp.success = True resp.result = ["test", True, 42, None] test = mgr.list_files() self.assertIsInstance(test, list) mock_file.assert_any_call(mgr._client, "test") mock_file.assert_any_call(mgr._client, True) mock_file.assert_any_call(mgr._client, 42) mock_file.assert_any_call(mgr._client, None) self.assertEqual(mock_file.call_count, 4)
def upload_assets(auth, config): """ Uploads a specified collection of assets either from a set created from scratch, or from the contents of existing directory. :Args: - auth (:class:`.Credentials`): instance of the Credentials class as returned by authentication() - config (:class:`.Configuration`): instance of the Configuration class as returned by logging_mode() """ asset_mgr = FileManager(auth, cfg=config) asset_to_add = asset_mgr.file_from_path(ASSET) # Creates FileCollection object asset_set = asset_mgr.create_file_set(asset_to_add) # Extends a FileCollection object with another FileCollection asset_set.extend(asset_mgr.files_from_dir(ASSET_DIR)) try: # force=true uploads the assets in the asset collection regardless of # whether they have been uploaded before. asset_set.upload(force=True) print("Assets uploaded successfully.") except RestCallException as e: print("failed: {0}".format(e))
def upload_asset(auth, config): """ Checks if a local asset has been uploaded before and if not, performs an upload operation on the single asset. :Args: - auth (:class:`.Credentials`): instance of the Credentials class as returned by authentication() - config (:class:`.Configuration`): instance of the Configuration class as returned by logging_mode() """ asset_mgr = FileManager(auth, cfg=config) asset_to_upload = asset_mgr.file_from_path(ASSET) if asset_to_upload.is_uploaded(): print("{0} already uploaded.".format(asset_to_upload)) return else: conf = "{0} MBs to be uploaded. Continue? (yes/no) ".format( len(asset_to_upload)/1024/1024) if input(conf)[0].lower() == 'y': try: asset_to_upload.upload() print("{0} uploaded successfully.".format(asset_to_upload)) except Exception as e: print("Upload failed: {0}".format(e)) else: print("Upload aborted.") return
def test_filemgr_find_file(self, mock_file, mock_api, mock_creds, mock_cfg): """Test find_file""" mgr = FileManager(mock_creds, cfg=mock_cfg) resp = mock.create_autospec(Response) resp.success = False resp.result = RestCallException(None, "test", None) mgr._client.query_files.return_value = resp with self.assertRaises(RestCallException): res = mgr.find_file("test", "date") mgr._client.query_files.assert_called_with({'FileName':'test', 'Timestamp':'date'}) with self.assertRaises(RestCallException): res = mgr.find_file("test", "date", full_path="path") mgr._client.query_files.assert_called_with({'FileName':'test', 'Timestamp':'date', 'OriginalPath':'path'}) resp.success = True resp.result = [] res = mgr.find_file("test", "date") self.assertEqual(res, []) self.assertFalse(mock_file.called) resp.result = ["testFile", None] res = mgr.find_file("test", "date") self.assertEqual(len(res), 2) mock_file.assert_any_call(mgr._client, "testFile") mock_file.assert_any_call(mgr._client, None)
def test_filemgr_find_files(self, mock_file, mock_api, mock_creds, mock_cfg): """Test find_files""" mgr = FileManager(mock_creds, cfg=mock_cfg) resp = mock.create_autospec(Response) resp.success = False resp.result = RestCallException(None, "test", None) mgr._client.query_files.return_value = resp with self.assertRaises(RestCallException): res = mgr.find_files("test") mgr._client.query_files.assert_called_with("test") with self.assertRaises(RestCallException): res = mgr.find_files([None]) mgr._client.query_files.assert_called_with([None]) resp.success = True resp.result = [] res = mgr.find_files("test") self.assertEqual(res, []) self.assertFalse(mock_file.called) resp.result = ["testFile", None] res = mgr.find_files("test") self.assertEqual(len(res), 2) mock_file.assert_any_call(mgr._client, "testFile") mock_file.assert_any_call(mgr._client, None)
def submit_job(configuration, creds, job_manager): """ Create a new job submission and submit it to the cloud. :Args: - configuration (:class:`batchapps.Configuration`): The generated ImageMagick config to apply to the session. - creds (:class:`batchapps.Credentials`): The appropriate credentials to access the session. :Returns: - A submission response. """ try: asset_mgr = FileManager(creds, cfg=configuration) files = asset_mgr.files_from_dir(_check_valid_dir(ASSET_DIR)) new_job = job_manager.create_job("Image Magic Test", files=files) # Setting various job parameters. new_job.instances = len(files) # 1 vm per file for optimal performance new_job.set_job_file(files[0]) # This sets the file that will be run to start the job. new_job.required_files.upload(threads=4) # Upload all files needed for the job. job_submission = new_job.submit() print("New job submitted with ID: {0}".format(job_submission["jobId"])) return job_submission except RestCallException as exp: raise RuntimeError("Submission failed: {0}".format(exp))
def submit_job(configuration, creds, job_manager): """ Create a new job submission and submit it to the cloud. :Args: - configuration (:class:`batchapps.Configuration`): The generated ImageMagick config to apply to the session. - creds (:class:`batchapps.Credentials`): The appropriate credentials to access the session. :Returns: - A submission response. """ try: asset_mgr = FileManager(creds, cfg=configuration) files = asset_mgr.files_from_dir(_check_valid_dir(ASSET_DIR)) new_job = job_manager.create_job("Image Magic Test", files=files) # Setting various job parameters. new_job.instances = len(files) # 1 vm per file for optimal performance new_job.set_job_file( files[0]) # This sets the file that will be run to start the job. new_job.required_files.upload( threads=4) # Upload all files needed for the job. job_submission = new_job.submit() print("New job submitted with ID: {0}".format(job_submission['jobId'])) return job_submission except RestCallException as exp: raise RuntimeError("Submission failed: {0}".format(exp))
def test_filemgr_file_from_path(self, mock_file, mock_api, mock_creds, mock_cfg): """Test file_from_path""" mgr = FileManager(mock_creds, cfg=mock_cfg) ufile = mgr.file_from_path("c:\\test.txt") mock_file.assert_called_with(mock.ANY, "c:\\test.txt") self.assertIsNotNone(ufile) ufile = mgr.file_from_path(None) mock_file.assert_called_with(mock.ANY, 'None') self.assertIsNotNone(ufile) ufile = mgr.file_from_path(42) mock_file.assert_called_with(mock.ANY, "42") self.assertIsNotNone(ufile)
def start(self, creds): """ Initialize all the addon subpages after authentication is complete. Sets page to HOME. :Args: - creds (:class:`batchapps.Credentials`): Authorised credentials with which API calls will be made. """ job_mgr = JobManager(creds, cfg=self.cfg) asset_mgr = FileManager(creds, cfg=self.cfg) pool_mgr = PoolManager(creds, cfg=self.cfg) self.submission = BatchAppsSubmission(job_mgr, asset_mgr, pool_mgr) self.log.debug("Initialised submission module") self.assets = BatchAppsAssets(asset_mgr) self.log.debug("Initialised assets module") self.history = BatchAppsHistory(job_mgr) self.log.debug("Initialised history module") self.pools = BatchAppsPools(pool_mgr) self.log.debug("Initialised pool module") self.page = "HOME"
def test_filemgr_create_file(self, mock_file, mock_api, mock_creds, mock_cfg): """Test deprecated method create_file""" mgr = FileManager(mock_creds, cfg=mock_cfg) ufile = mgr.create_file("c:\\test.txt") mock_file.assert_called_with(mock.ANY, "c:\\test.txt") self.assertIsNotNone(ufile) ufile = mgr.create_file(None) mock_file.assert_called_with(mock.ANY, 'None') self.assertIsNotNone(ufile) ufile = mgr.create_file(42) mock_file.assert_called_with(mock.ANY, "42") self.assertIsNotNone(ufile)
def test_filemgr_files_from_dir_b(self, mock_api, mock_creds, mock_cfg): """Test files_from_dir""" if not self.use_test_files: self.skipTest("No test files present") mgr = FileManager(mock_creds, cfg=mock_cfg) collection = mgr.files_from_dir(self.test_dir) collection._collection.sort() self.assertEqual(str(collection), "['same.png', 'speech_bubble.png', 'star.png']") collection = mgr.files_from_dir(self.cwd, pattern="*.png") self.assertEqual(str(collection), "[]") collection = mgr.files_from_dir(self.cwd, recursive=True, pattern="*.png") collection._collection.sort() self.assertEqual(str(collection), "['same.png', 'speech_bubble.png', 'star.png']") with self.assertRaises(OSError): mgr.files_from_dir(os.path.join(self.test_dir, "not a dir"))
def test_filemgr_create_file_set(self, mock_file, mock_api, mock_creds, mock_cfg): """Test create_file_set""" mgr = FileManager(mock_creds, cfg=mock_cfg) coll = mgr.create_file_set() self.assertIsNotNone(coll) mock_file.assert_called_with(mock.ANY, *[]) coll = mgr.create_file_set(None) mock_file.assert_called_with(mock.ANY, *[None]) coll = mgr.create_file_set(1, 2, 3) mock_file.assert_called_with(mock.ANY, *[1, 2, 3]) coll = mgr.create_file_set("a", "a", "a") mock_file.assert_called_with(mock.ANY, *['a'])
def submit_job(auth, config): """ Create a new job submission and send it to the cloud. :Args: - auth :class:`.Credentials`: instance of the Credentials class as returned by authentication() - config :class:`.Configuration`: instance of the Configuration class as returned by create_config() """ asset_mgr = FileManager(auth, cfg=config) job_mgr = JobManager(auth, cfg=config) # Converts directory contents to a FileCollection file_collection = asset_mgr.files_from_dir(ASSET_DIR) new_job = job_mgr.create_job("Test Job", files=file_collection) # Set various job parameters. The pre-configured parameters for the # job type can be found using new_job.get_default_params(). new_job.instances = 5 # Number of machines to work on the job. new_job.start = 1 new_job.end = 10 new_job.numFrames = 10 # This sets the file that will be run to start the job. # In this case the first file in the FileCollection. new_job.set_job_file(file_collection[0]) # Upload all files needed for the job. new_job.required_files.upload(threads=4) try: submission = new_job.submit() print("New job submitted with ID: {0}".format(submission['jobId'])) except RestCallException as e: print("Job failed: {0}".format(e))
def test_filemgr_find_file(self, mock_file, mock_api, mock_creds, mock_cfg): """Test find_file""" mgr = FileManager(mock_creds, cfg=mock_cfg) resp = mock.create_autospec(Response) resp.success = False resp.result = RestCallException(None, "test", None) mgr._client.query_files.return_value = resp with self.assertRaises(RestCallException): res = mgr.find_file("test", "date") mgr._client.query_files.assert_called_with({ 'FileName': 'test', 'Timestamp': 'date' }) with self.assertRaises(RestCallException): res = mgr.find_file("test", "date", full_path="path") mgr._client.query_files.assert_called_with({ 'FileName': 'test', 'Timestamp': 'date', 'OriginalPath': 'path' }) resp.success = True resp.result = [] res = mgr.find_file("test", "date") self.assertEqual(res, []) self.assertFalse(mock_file.called) resp.result = ["testFile", None] res = mgr.find_file("test", "date") self.assertEqual(len(res), 2) mock_file.assert_any_call(mgr._client, "testFile") mock_file.assert_any_call(mgr._client, None)
def test_filemgr_files_from_dir_a(self, mock_file, mock_api, mock_creds, mock_cfg, mock_glob, mock_isdir, mock_isfile): """Test files_from_dir""" mgr = FileManager(mock_creds, cfg=mock_cfg) mock_isdir.return_value = False mock_isfile.return_value = True with self.assertRaises(OSError): mgr.files_from_dir(None) with self.assertRaises(OSError): mgr.files_from_dir("") with self.assertRaises(OSError): mgr.files_from_dir(42) if not self.use_test_files: self.skipTest("No test files present") mock_isdir.return_value = True mgr.files_from_dir(os.path.join(self.test_dir, "test_config")) mock_glob.glob.assert_called_with(os.path.join(self.test_dir, "test_config", '*')) mgr.files_from_dir(os.path.join(self.test_dir, "test_config"), recursive=True) mock_glob.glob.assert_any_call(os.path.join(self.test_dir, "test_config", '*')) mock_glob.glob.assert_any_call(os.path.join(self.test_dir, "test_config", "batch_apps.ini", '*')) mock_glob.reset() mock_glob.glob.call_count = 0 mgr.files_from_dir(self.test_dir, recursive=False) mock_glob.glob.assert_any_call(self.test_dir + "\\*") self.assertEqual(mock_glob.glob.call_count, 1) mock_glob.reset() mock_glob.glob.call_count = 0 mgr.files_from_dir(self.test_dir, recursive=True, pattern="*.png") self.assertEqual(mock_glob.glob.call_count, 6) mock_glob.glob.assert_any_call(self.test_dir + "\\*.png") mock_glob.glob.assert_any_call(self.test_dir + "\\test_config\\*.png")
def test_filemgr_files_from_dir_a(self, mock_file, mock_api, mock_creds, mock_cfg, mock_glob, mock_isdir, mock_isfile): """Test files_from_dir""" mgr = FileManager(mock_creds, cfg=mock_cfg) mock_isdir.return_value = False mock_isfile.return_value = True with self.assertRaises(OSError): mgr.files_from_dir(None) with self.assertRaises(OSError): mgr.files_from_dir("") with self.assertRaises(OSError): mgr.files_from_dir(42) if not self.use_test_files: self.skipTest("No test files present") mock_isdir.return_value = True mgr.files_from_dir(os.path.join(self.test_dir, "test_config")) mock_glob.glob.assert_called_with( os.path.join(self.test_dir, "test_config", '*')) mgr.files_from_dir(os.path.join(self.test_dir, "test_config"), recursive=True) mock_glob.glob.assert_any_call( os.path.join(self.test_dir, "test_config", '*')) mock_glob.glob.assert_any_call( os.path.join(self.test_dir, "test_config", "batch_apps.ini", '*')) mock_glob.reset() mock_glob.glob.call_count = 0 mgr.files_from_dir(self.test_dir, recursive=False) mock_glob.glob.assert_any_call(self.test_dir + "\\*") self.assertEqual(mock_glob.glob.call_count, 1) mock_glob.reset() mock_glob.glob.call_count = 0 mgr.files_from_dir(self.test_dir, recursive=True, pattern="*.png") self.assertEqual(mock_glob.glob.call_count, 6) mock_glob.glob.assert_any_call(self.test_dir + "\\*.png") mock_glob.glob.assert_any_call(self.test_dir + "\\test_config\\*.png")