def test_filemgr_files_from_dir_b(self, mock_api, mock_creds, mock_cfg):
"""Test files_from_dir"""
if not self.use_test_files:
self.skipTest("No test files present")
mgr = FileManager(mock_creds, cfg=mock_cfg)
collection = mgr.files_from_dir(self.test_dir)
collection._collection.sort()
self.assertEqual(str(collection),
"['same.png', 'speech_bubble.png', 'star.png']")
collection = mgr.files_from_dir(self.cwd, pattern="*.png")
self.assertEqual(str(collection), "[]")
collection = mgr.files_from_dir(self.cwd,
recursive=True,
pattern="*.png")
collection._collection.sort()
self.assertEqual(str(collection),
"['same.png', 'speech_bubble.png', 'star.png']")
with self.assertRaises(OSError):
mgr.files_from_dir(os.path.join(self.test_dir, "not a dir"))
def test_filemgr_files_from_dir_b(self, mock_api, mock_creds, mock_cfg):
"""Test files_from_dir"""
if not self.use_test_files:
self.skipTest("No test files present")
mgr = FileManager(mock_creds, cfg=mock_cfg)
collection = mgr.files_from_dir(self.test_dir)
collection._collection.sort()
self.assertEqual(str(collection),
"['same.png', 'speech_bubble.png', 'star.png']")
collection = mgr.files_from_dir(self.cwd, pattern="*.png")
self.assertEqual(str(collection), "[]")
collection = mgr.files_from_dir(self.cwd,
recursive=True,
pattern="*.png")
collection._collection.sort()
self.assertEqual(str(collection),
"['same.png', 'speech_bubble.png', 'star.png']")
with self.assertRaises(OSError):
mgr.files_from_dir(os.path.join(self.test_dir, "not a dir"))
def upload_assets(auth, config):
"""
Uploads a specified collection of assets either from a set created
from scratch, or from the contents of existing directory.
:Args:
- auth (:class:`.Credentials`): instance of the Credentials class
as returned by authentication()
- config (:class:`.Configuration`): instance of the Configuration
class as returned by logging_mode()
"""
asset_mgr = FileManager(auth, cfg=config)
asset_to_add = asset_mgr.file_from_path(ASSET)
# Creates FileCollection object
asset_set = asset_mgr.create_file_set(asset_to_add)
# Extends a FileCollection object with another FileCollection
asset_set.extend(asset_mgr.files_from_dir(ASSET_DIR))
try:
# force=true uploads the assets in the asset collection regardless of
# whether they have been uploaded before.
asset_set.upload(force=True)
print("Assets uploaded successfully.")
except RestCallException as e:
print("failed: {0}".format(e))
def submit_job(configuration, creds, job_manager):
"""
Create a new job submission and submit it to the cloud.
:Args:
- configuration (:class:`batchapps.Configuration`): The generated
ImageMagick config to apply to the session.
- creds (:class:`batchapps.Credentials`): The appropriate credentials
to access the session.
:Returns:
- A submission response.
"""
try:
asset_mgr = FileManager(creds, cfg=configuration)
files = asset_mgr.files_from_dir(_check_valid_dir(ASSET_DIR))
new_job = job_manager.create_job("Image Magic Test", files=files)
# Setting various job parameters.
new_job.instances = len(files) # 1 vm per file for optimal performance
new_job.set_job_file(files[0]) # This sets the file that will be run to start the job.
new_job.required_files.upload(threads=4) # Upload all files needed for the job.
job_submission = new_job.submit()
print("New job submitted with ID: {0}".format(job_submission["jobId"]))
return job_submission
except RestCallException as exp:
raise RuntimeError("Submission failed: {0}".format(exp))
def submit_job(configuration, creds, job_manager):
"""
Create a new job submission and submit it to the cloud.
:Args:
- configuration (:class:`batchapps.Configuration`): The generated
ImageMagick config to apply to the session.
- creds (:class:`batchapps.Credentials`): The appropriate credentials
to access the session.
:Returns:
- A submission response.
"""
try:
asset_mgr = FileManager(creds, cfg=configuration)
files = asset_mgr.files_from_dir(_check_valid_dir(ASSET_DIR))
new_job = job_manager.create_job("Image Magic Test", files=files)
# Setting various job parameters.
new_job.instances = len(files) # 1 vm per file for optimal performance
new_job.set_job_file(
files[0]) # This sets the file that will be run to start the job.
new_job.required_files.upload(
threads=4) # Upload all files needed for the job.
job_submission = new_job.submit()
print("New job submitted with ID: {0}".format(job_submission['jobId']))
return job_submission
except RestCallException as exp:
raise RuntimeError("Submission failed: {0}".format(exp))
def submit_job(auth, config):
"""
Create a new job submission and send it to the cloud.
:Args:
- auth :class:`.Credentials`: instance of the Credentials
class as returned by authentication()
- config :class:`.Configuration`: instance of the Configuration
class as returned by create_config()
"""
asset_mgr = FileManager(auth, cfg=config)
job_mgr = JobManager(auth, cfg=config)
# Converts directory contents to a FileCollection
file_collection = asset_mgr.files_from_dir(ASSET_DIR)
new_job = job_mgr.create_job("Test Job", files=file_collection)
# Set various job parameters. The pre-configured parameters for the
# job type can be found using new_job.get_default_params().
new_job.instances = 5 # Number of machines to work on the job.
new_job.start = 1
new_job.end = 10
new_job.numFrames = 10
# This sets the file that will be run to start the job.
# In this case the first file in the FileCollection.
new_job.set_job_file(file_collection[0])
# Upload all files needed for the job.
new_job.required_files.upload(threads=4)
try:
submission = new_job.submit()
print("New job submitted with ID: {0}".format(submission['jobId']))
except RestCallException as e:
print("Job failed: {0}".format(e))
def submit_job(auth, config):
"""
Create a new job submission and send it to the cloud.
:Args:
- auth :class:`.Credentials`: instance of the Credentials
class as returned by authentication()
- config :class:`.Configuration`: instance of the Configuration
class as returned by create_config()
"""
asset_mgr = FileManager(auth, cfg=config)
job_mgr = JobManager(auth, cfg=config)
# Converts directory contents to a FileCollection
file_collection = asset_mgr.files_from_dir(ASSET_DIR)
new_job = job_mgr.create_job("Test Job", files=file_collection)
# Set various job parameters. The pre-configured parameters for the
# job type can be found using new_job.get_default_params().
new_job.instances = 5 # Number of machines to work on the job.
new_job.start = 1
new_job.end = 10
new_job.numFrames = 10
# This sets the file that will be run to start the job.
# In this case the first file in the FileCollection.
new_job.set_job_file(file_collection[0])
# Upload all files needed for the job.
new_job.required_files.upload(threads=4)
try:
submission = new_job.submit()
print("New job submitted with ID: {0}".format(submission['jobId']))
except RestCallException as e:
print("Job failed: {0}".format(e))
def test_filemgr_files_from_dir_a(self,
mock_file,
mock_api,
mock_creds,
mock_cfg,
mock_glob,
mock_isdir,
mock_isfile):
"""Test files_from_dir"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
mock_isdir.return_value = False
mock_isfile.return_value = True
with self.assertRaises(OSError):
mgr.files_from_dir(None)
with self.assertRaises(OSError):
mgr.files_from_dir("")
with self.assertRaises(OSError):
mgr.files_from_dir(42)
if not self.use_test_files:
self.skipTest("No test files present")
mock_isdir.return_value = True
mgr.files_from_dir(os.path.join(self.test_dir, "test_config"))
mock_glob.glob.assert_called_with(os.path.join(self.test_dir,
"test_config",
'*'))
mgr.files_from_dir(os.path.join(self.test_dir, "test_config"),
recursive=True)
mock_glob.glob.assert_any_call(os.path.join(self.test_dir,
"test_config",
'*'))
mock_glob.glob.assert_any_call(os.path.join(self.test_dir,
"test_config",
"batch_apps.ini",
'*'))
mock_glob.reset()
mock_glob.glob.call_count = 0
mgr.files_from_dir(self.test_dir, recursive=False)
mock_glob.glob.assert_any_call(self.test_dir + "\\*")
self.assertEqual(mock_glob.glob.call_count, 1)
mock_glob.reset()
mock_glob.glob.call_count = 0
mgr.files_from_dir(self.test_dir,
recursive=True,
pattern="*.png")
self.assertEqual(mock_glob.glob.call_count, 6)
mock_glob.glob.assert_any_call(self.test_dir + "\\*.png")
mock_glob.glob.assert_any_call(self.test_dir + "\\test_config\\*.png")
def test_filemgr_files_from_dir_a(self, mock_file, mock_api, mock_creds,
mock_cfg, mock_glob, mock_isdir,
mock_isfile):
"""Test files_from_dir"""
mgr = FileManager(mock_creds, cfg=mock_cfg)
mock_isdir.return_value = False
mock_isfile.return_value = True
with self.assertRaises(OSError):
mgr.files_from_dir(None)
with self.assertRaises(OSError):
mgr.files_from_dir("")
with self.assertRaises(OSError):
mgr.files_from_dir(42)
if not self.use_test_files:
self.skipTest("No test files present")
mock_isdir.return_value = True
mgr.files_from_dir(os.path.join(self.test_dir, "test_config"))
mock_glob.glob.assert_called_with(
os.path.join(self.test_dir, "test_config", '*'))
mgr.files_from_dir(os.path.join(self.test_dir, "test_config"),
recursive=True)
mock_glob.glob.assert_any_call(
os.path.join(self.test_dir, "test_config", '*'))
mock_glob.glob.assert_any_call(
os.path.join(self.test_dir, "test_config", "batch_apps.ini", '*'))
mock_glob.reset()
mock_glob.glob.call_count = 0
mgr.files_from_dir(self.test_dir, recursive=False)
mock_glob.glob.assert_any_call(self.test_dir + "\\*")
self.assertEqual(mock_glob.glob.call_count, 1)
mock_glob.reset()
mock_glob.glob.call_count = 0
mgr.files_from_dir(self.test_dir, recursive=True, pattern="*.png")
self.assertEqual(mock_glob.glob.call_count, 6)
mock_glob.glob.assert_any_call(self.test_dir + "\\*.png")
mock_glob.glob.assert_any_call(self.test_dir + "\\test_config\\*.png")
