def test_filemgr_list_files(self, mock_file, mock_api, mock_creds,
mock_cfg):
"""Test list_files"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "test", None)
mgr._client.list_files.return_value = resp
with self.assertRaises(RestCallException):
test = mgr.list_files()
self.assertTrue(mgr._client.list_files.called)
self.assertFalse(mock_file.called)
resp.success = True
resp.result = ["test", True, 42, None]
test = mgr.list_files()
self.assertIsInstance(test, list)
mock_file.assert_any_call(mgr._client, "test")
mock_file.assert_any_call(mgr._client, True)
mock_file.assert_any_call(mgr._client, 42)
mock_file.assert_any_call(mgr._client, None)
self.assertEqual(mock_file.call_count, 4)
def upload_assets(auth, config):
"""
Uploads a specified collection of assets either from a set created
from scratch, or from the contents of existing directory.
:Args:
- auth (:class:`.Credentials`): instance of the Credentials class
as returned by authentication()
- config (:class:`.Configuration`): instance of the Configuration
class as returned by logging_mode()
"""
asset_mgr = FileManager(auth, cfg=config)
asset_to_add = asset_mgr.file_from_path(ASSET)
# Creates FileCollection object
asset_set = asset_mgr.create_file_set(asset_to_add)
# Extends a FileCollection object with another FileCollection
asset_set.extend(asset_mgr.files_from_dir(ASSET_DIR))
try:
# force=true uploads the assets in the asset collection regardless of
# whether they have been uploaded before.
asset_set.upload(force=True)
print("Assets uploaded successfully.")
except RestCallException as e:
print("failed: {0}".format(e))
def upload_asset(auth, config):
"""
Checks if a local asset has been uploaded before and if not,
performs an upload operation on the single asset.
:Args:
- auth (:class:`.Credentials`): instance of the Credentials class
as returned by authentication()
- config (:class:`.Configuration`): instance of the Configuration
class as returned by logging_mode()
"""
asset_mgr = FileManager(auth, cfg=config)
asset_to_upload = asset_mgr.file_from_path(ASSET)
if asset_to_upload.is_uploaded():
print("{0} already uploaded.".format(asset_to_upload))
return
else:
conf = "{0} MBs to be uploaded. Continue? (yes/no) ".format(
len(asset_to_upload)/1024/1024)
if input(conf)[0].lower() == 'y':
try:
asset_to_upload.upload()
print("{0} uploaded successfully.".format(asset_to_upload))
except Exception as e:
print("Upload failed: {0}".format(e))
else:
print("Upload aborted.")
return
def test_filemgr_list_files(self,
mock_file,
mock_api,
mock_creds,
mock_cfg):
"""Test list_files"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "test", None)
mgr._client.list_files.return_value = resp
with self.assertRaises(RestCallException):
test = mgr.list_files()
self.assertTrue(mgr._client.list_files.called)
self.assertFalse(mock_file.called)
resp.success = True
resp.result = ["test", True, 42, None]
test = mgr.list_files()
self.assertIsInstance(test, list)
mock_file.assert_any_call(mgr._client, "test")
mock_file.assert_any_call(mgr._client, True)
mock_file.assert_any_call(mgr._client, 42)
mock_file.assert_any_call(mgr._client, None)
self.assertEqual(mock_file.call_count, 4)
def test_filemgr_find_file(self,
mock_file,
mock_api,
mock_creds,
mock_cfg):
"""Test find_file"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "test", None)
mgr._client.query_files.return_value = resp
with self.assertRaises(RestCallException):
res = mgr.find_file("test", "date")
mgr._client.query_files.assert_called_with({'FileName':'test',
'Timestamp':'date'})
with self.assertRaises(RestCallException):
res = mgr.find_file("test", "date", full_path="path")
mgr._client.query_files.assert_called_with({'FileName':'test',
'Timestamp':'date',
'OriginalPath':'path'})
resp.success = True
resp.result = []
res = mgr.find_file("test", "date")
self.assertEqual(res, [])
self.assertFalse(mock_file.called)
resp.result = ["testFile", None]
res = mgr.find_file("test", "date")
self.assertEqual(len(res), 2)
mock_file.assert_any_call(mgr._client, "testFile")
mock_file.assert_any_call(mgr._client, None)
def test_filemgr_find_files(self,
mock_file,
mock_api,
mock_creds,
mock_cfg):
"""Test find_files"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "test", None)
mgr._client.query_files.return_value = resp
with self.assertRaises(RestCallException):
res = mgr.find_files("test")
mgr._client.query_files.assert_called_with("test")
with self.assertRaises(RestCallException):
res = mgr.find_files([None])
mgr._client.query_files.assert_called_with([None])
resp.success = True
resp.result = []
res = mgr.find_files("test")
self.assertEqual(res, [])
self.assertFalse(mock_file.called)
resp.result = ["testFile", None]
res = mgr.find_files("test")
self.assertEqual(len(res), 2)
mock_file.assert_any_call(mgr._client, "testFile")
mock_file.assert_any_call(mgr._client, None)
def submit_job(configuration, creds, job_manager):
"""
Create a new job submission and submit it to the cloud.
:Args:
- configuration (:class:`batchapps.Configuration`): The generated
ImageMagick config to apply to the session.
- creds (:class:`batchapps.Credentials`): The appropriate credentials
to access the session.
:Returns:
- A submission response.
"""
try:
asset_mgr = FileManager(creds, cfg=configuration)
files = asset_mgr.files_from_dir(_check_valid_dir(ASSET_DIR))
new_job = job_manager.create_job("Image Magic Test", files=files)
# Setting various job parameters.
new_job.instances = len(files) # 1 vm per file for optimal performance
new_job.set_job_file(files[0]) # This sets the file that will be run to start the job.
new_job.required_files.upload(threads=4) # Upload all files needed for the job.
job_submission = new_job.submit()
print("New job submitted with ID: {0}".format(job_submission["jobId"]))
return job_submission
except RestCallException as exp:
raise RuntimeError("Submission failed: {0}".format(exp))
def submit_job(configuration, creds, job_manager):
"""
Create a new job submission and submit it to the cloud.
:Args:
- configuration (:class:`batchapps.Configuration`): The generated
ImageMagick config to apply to the session.
- creds (:class:`batchapps.Credentials`): The appropriate credentials
to access the session.
:Returns:
- A submission response.
"""
try:
asset_mgr = FileManager(creds, cfg=configuration)
files = asset_mgr.files_from_dir(_check_valid_dir(ASSET_DIR))
new_job = job_manager.create_job("Image Magic Test", files=files)
# Setting various job parameters.
new_job.instances = len(files) # 1 vm per file for optimal performance
new_job.set_job_file(
files[0]) # This sets the file that will be run to start the job.
new_job.required_files.upload(
threads=4) # Upload all files needed for the job.
job_submission = new_job.submit()
print("New job submitted with ID: {0}".format(job_submission['jobId']))
return job_submission
except RestCallException as exp:
raise RuntimeError("Submission failed: {0}".format(exp))
def test_filemgr_file_from_path(self, mock_file, mock_api, mock_creds,
mock_cfg):
"""Test file_from_path"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
ufile = mgr.file_from_path("c:\\test.txt")
mock_file.assert_called_with(mock.ANY, "c:\\test.txt")
self.assertIsNotNone(ufile)
ufile = mgr.file_from_path(None)
mock_file.assert_called_with(mock.ANY, 'None')
self.assertIsNotNone(ufile)
ufile = mgr.file_from_path(42)
mock_file.assert_called_with(mock.ANY, "42")
self.assertIsNotNone(ufile)
def start(self, creds):
"""
Initialize all the addon subpages after authentication is
complete.
Sets page to HOME.
:Args:
- creds (:class:`batchapps.Credentials`): Authorised credentials
with which API calls will be made.
"""
job_mgr = JobManager(creds, cfg=self.cfg)
asset_mgr = FileManager(creds, cfg=self.cfg)
pool_mgr = PoolManager(creds, cfg=self.cfg)
self.submission = BatchAppsSubmission(job_mgr, asset_mgr, pool_mgr)
self.log.debug("Initialised submission module")
self.assets = BatchAppsAssets(asset_mgr)
self.log.debug("Initialised assets module")
self.history = BatchAppsHistory(job_mgr)
self.log.debug("Initialised history module")
self.pools = BatchAppsPools(pool_mgr)
self.log.debug("Initialised pool module")
self.page = "HOME"
def test_filemgr_create_file(self, mock_file, mock_api, mock_creds,
mock_cfg):
"""Test deprecated method create_file"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
ufile = mgr.create_file("c:\\test.txt")
mock_file.assert_called_with(mock.ANY, "c:\\test.txt")
self.assertIsNotNone(ufile)
ufile = mgr.create_file(None)
mock_file.assert_called_with(mock.ANY, 'None')
self.assertIsNotNone(ufile)
ufile = mgr.create_file(42)
mock_file.assert_called_with(mock.ANY, "42")
self.assertIsNotNone(ufile)
def test_filemgr_files_from_dir_b(self, mock_api, mock_creds, mock_cfg):
"""Test files_from_dir"""
if not self.use_test_files:
self.skipTest("No test files present")
mgr = FileManager(mock_creds, cfg=mock_cfg)
collection = mgr.files_from_dir(self.test_dir)
collection._collection.sort()
self.assertEqual(str(collection),
"['same.png', 'speech_bubble.png', 'star.png']")
collection = mgr.files_from_dir(self.cwd, pattern="*.png")
self.assertEqual(str(collection), "[]")
collection = mgr.files_from_dir(self.cwd,
recursive=True,
pattern="*.png")
collection._collection.sort()
self.assertEqual(str(collection),
"['same.png', 'speech_bubble.png', 'star.png']")
with self.assertRaises(OSError):
mgr.files_from_dir(os.path.join(self.test_dir, "not a dir"))
def test_filemgr_file_from_path(self,
mock_file,
mock_api,
mock_creds,
mock_cfg):
"""Test file_from_path"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
ufile = mgr.file_from_path("c:\\test.txt")
mock_file.assert_called_with(mock.ANY, "c:\\test.txt")
self.assertIsNotNone(ufile)
ufile = mgr.file_from_path(None)
mock_file.assert_called_with(mock.ANY, 'None')
self.assertIsNotNone(ufile)
ufile = mgr.file_from_path(42)
mock_file.assert_called_with(mock.ANY, "42")
self.assertIsNotNone(ufile)
def test_filemgr_create_file(self,
mock_file,
mock_api,
mock_creds,
mock_cfg):
"""Test deprecated method create_file"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
ufile = mgr.create_file("c:\\test.txt")
mock_file.assert_called_with(mock.ANY, "c:\\test.txt")
self.assertIsNotNone(ufile)
ufile = mgr.create_file(None)
mock_file.assert_called_with(mock.ANY, 'None')
self.assertIsNotNone(ufile)
ufile = mgr.create_file(42)
mock_file.assert_called_with(mock.ANY, "42")
self.assertIsNotNone(ufile)
def test_filemgr_create_file_set(self,
mock_file,
mock_api,
mock_creds,
mock_cfg):
"""Test create_file_set"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
coll = mgr.create_file_set()
self.assertIsNotNone(coll)
mock_file.assert_called_with(mock.ANY, *[])
coll = mgr.create_file_set(None)
mock_file.assert_called_with(mock.ANY, *[None])
coll = mgr.create_file_set(1, 2, 3)
mock_file.assert_called_with(mock.ANY, *[1, 2, 3])
coll = mgr.create_file_set("a", "a", "a")
mock_file.assert_called_with(mock.ANY, *['a'])
def submit_job(auth, config):
"""
Create a new job submission and send it to the cloud.
:Args:
- auth :class:`.Credentials`: instance of the Credentials
class as returned by authentication()
- config :class:`.Configuration`: instance of the Configuration
class as returned by create_config()
"""
asset_mgr = FileManager(auth, cfg=config)
job_mgr = JobManager(auth, cfg=config)
# Converts directory contents to a FileCollection
file_collection = asset_mgr.files_from_dir(ASSET_DIR)
new_job = job_mgr.create_job("Test Job", files=file_collection)
# Set various job parameters. The pre-configured parameters for the
# job type can be found using new_job.get_default_params().
new_job.instances = 5 # Number of machines to work on the job.
new_job.start = 1
new_job.end = 10
new_job.numFrames = 10
# This sets the file that will be run to start the job.
# In this case the first file in the FileCollection.
new_job.set_job_file(file_collection[0])
# Upload all files needed for the job.
new_job.required_files.upload(threads=4)
try:
submission = new_job.submit()
print("New job submitted with ID: {0}".format(submission['jobId']))
except RestCallException as e:
print("Job failed: {0}".format(e))
def submit_job(auth, config):
"""
Create a new job submission and send it to the cloud.
:Args:
- auth :class:`.Credentials`: instance of the Credentials
class as returned by authentication()
- config :class:`.Configuration`: instance of the Configuration
class as returned by create_config()
"""
asset_mgr = FileManager(auth, cfg=config)
job_mgr = JobManager(auth, cfg=config)
# Converts directory contents to a FileCollection
file_collection = asset_mgr.files_from_dir(ASSET_DIR)
new_job = job_mgr.create_job("Test Job", files=file_collection)
# Set various job parameters. The pre-configured parameters for the
# job type can be found using new_job.get_default_params().
new_job.instances = 5 # Number of machines to work on the job.
new_job.start = 1
new_job.end = 10
new_job.numFrames = 10
# This sets the file that will be run to start the job.
# In this case the first file in the FileCollection.
new_job.set_job_file(file_collection[0])
# Upload all files needed for the job.
new_job.required_files.upload(threads=4)
try:
submission = new_job.submit()
print("New job submitted with ID: {0}".format(submission['jobId']))
except RestCallException as e:
print("Job failed: {0}".format(e))
def test_filemgr_files_from_dir_b(self, mock_api, mock_creds, mock_cfg):
"""Test files_from_dir"""
if not self.use_test_files:
self.skipTest("No test files present")
mgr = FileManager(mock_creds, cfg=mock_cfg)
collection = mgr.files_from_dir(self.test_dir)
collection._collection.sort()
self.assertEqual(str(collection),
"['same.png', 'speech_bubble.png', 'star.png']")
collection = mgr.files_from_dir(self.cwd, pattern="*.png")
self.assertEqual(str(collection), "[]")
collection = mgr.files_from_dir(self.cwd,
recursive=True,
pattern="*.png")
collection._collection.sort()
self.assertEqual(str(collection),
"['same.png', 'speech_bubble.png', 'star.png']")
with self.assertRaises(OSError):
mgr.files_from_dir(os.path.join(self.test_dir, "not a dir"))
def test_filemgr_find_files(self, mock_file, mock_api, mock_creds,
mock_cfg):
"""Test find_files"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "test", None)
mgr._client.query_files.return_value = resp
with self.assertRaises(RestCallException):
res = mgr.find_files("test")
mgr._client.query_files.assert_called_with("test")
with self.assertRaises(RestCallException):
res = mgr.find_files([None])
mgr._client.query_files.assert_called_with([None])
resp.success = True
resp.result = []
res = mgr.find_files("test")
self.assertEqual(res, [])
self.assertFalse(mock_file.called)
resp.result = ["testFile", None]
res = mgr.find_files("test")
self.assertEqual(len(res), 2)
mock_file.assert_any_call(mgr._client, "testFile")
mock_file.assert_any_call(mgr._client, None)
def test_filemgr_create_file_set(self, mock_file, mock_api, mock_creds,
mock_cfg):
"""Test create_file_set"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
coll = mgr.create_file_set()
self.assertIsNotNone(coll)
mock_file.assert_called_with(mock.ANY, *[])
coll = mgr.create_file_set(None)
mock_file.assert_called_with(mock.ANY, *[None])
coll = mgr.create_file_set(1, 2, 3)
mock_file.assert_called_with(mock.ANY, *[1, 2, 3])
coll = mgr.create_file_set("a", "a", "a")
mock_file.assert_called_with(mock.ANY, *['a'])
def test_filemgr_find_file(self, mock_file, mock_api, mock_creds,
mock_cfg):
"""Test find_file"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
resp = mock.create_autospec(Response)
resp.success = False
resp.result = RestCallException(None, "test", None)
mgr._client.query_files.return_value = resp
with self.assertRaises(RestCallException):
res = mgr.find_file("test", "date")
mgr._client.query_files.assert_called_with({
'FileName': 'test',
'Timestamp': 'date'
})
with self.assertRaises(RestCallException):
res = mgr.find_file("test", "date", full_path="path")
mgr._client.query_files.assert_called_with({
'FileName': 'test',
'Timestamp': 'date',
'OriginalPath': 'path'
})
resp.success = True
resp.result = []
res = mgr.find_file("test", "date")
self.assertEqual(res, [])
self.assertFalse(mock_file.called)
resp.result = ["testFile", None]
res = mgr.find_file("test", "date")
self.assertEqual(len(res), 2)
mock_file.assert_any_call(mgr._client, "testFile")
mock_file.assert_any_call(mgr._client, None)
def test_filemgr_files_from_dir_a(self,
mock_file,
mock_api,
mock_creds,
mock_cfg,
mock_glob,
mock_isdir,
mock_isfile):
"""Test files_from_dir"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
mock_isdir.return_value = False
mock_isfile.return_value = True
with self.assertRaises(OSError):
mgr.files_from_dir(None)
with self.assertRaises(OSError):
mgr.files_from_dir("")
with self.assertRaises(OSError):
mgr.files_from_dir(42)
if not self.use_test_files:
self.skipTest("No test files present")
mock_isdir.return_value = True
mgr.files_from_dir(os.path.join(self.test_dir, "test_config"))
mock_glob.glob.assert_called_with(os.path.join(self.test_dir,
"test_config",
'*'))
mgr.files_from_dir(os.path.join(self.test_dir, "test_config"),
recursive=True)
mock_glob.glob.assert_any_call(os.path.join(self.test_dir,
"test_config",
'*'))
mock_glob.glob.assert_any_call(os.path.join(self.test_dir,
"test_config",
"batch_apps.ini",
'*'))
mock_glob.reset()
mock_glob.glob.call_count = 0
mgr.files_from_dir(self.test_dir, recursive=False)
mock_glob.glob.assert_any_call(self.test_dir + "\\*")
self.assertEqual(mock_glob.glob.call_count, 1)
mock_glob.reset()
mock_glob.glob.call_count = 0
mgr.files_from_dir(self.test_dir,
recursive=True,
pattern="*.png")
self.assertEqual(mock_glob.glob.call_count, 6)
mock_glob.glob.assert_any_call(self.test_dir + "\\*.png")
mock_glob.glob.assert_any_call(self.test_dir + "\\test_config\\*.png")
def test_filemgr_files_from_dir_a(self, mock_file, mock_api, mock_creds,
mock_cfg, mock_glob, mock_isdir,
mock_isfile):
"""Test files_from_dir"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
mock_isdir.return_value = False
mock_isfile.return_value = True
with self.assertRaises(OSError):
mgr.files_from_dir(None)
with self.assertRaises(OSError):
mgr.files_from_dir("")
with self.assertRaises(OSError):
mgr.files_from_dir(42)
if not self.use_test_files:
self.skipTest("No test files present")
mock_isdir.return_value = True
mgr.files_from_dir(os.path.join(self.test_dir, "test_config"))
mock_glob.glob.assert_called_with(
os.path.join(self.test_dir, "test_config", '*'))
mgr.files_from_dir(os.path.join(self.test_dir, "test_config"),
recursive=True)
mock_glob.glob.assert_any_call(
os.path.join(self.test_dir, "test_config", '*'))
mock_glob.glob.assert_any_call(
os.path.join(self.test_dir, "test_config", "batch_apps.ini", '*'))
mock_glob.reset()
mock_glob.glob.call_count = 0
mgr.files_from_dir(self.test_dir, recursive=False)
mock_glob.glob.assert_any_call(self.test_dir + "\\*")
self.assertEqual(mock_glob.glob.call_count, 1)
mock_glob.reset()
mock_glob.glob.call_count = 0
mgr.files_from_dir(self.test_dir, recursive=True, pattern="*.png")
self.assertEqual(mock_glob.glob.call_count, 6)
mock_glob.glob.assert_any_call(self.test_dir + "\\*.png")
mock_glob.glob.assert_any_call(self.test_dir + "\\test_config\\*.png")
