def test_flavor_selection_application_specific_unavailable(self):
"""
Test that application-specific "instance type" is ignored if unavailable.
"""
# NOTE: `EXTRA_CONF` has to start with a blank line and with the first
# non-whitespace character in column 0, otherwise `ConfigParser` will
# consider it a continuation of the last key/value assignment in
# `RESOURCE_STANZA` ...
EXTRA_CONF = """
large_instance_type = 5cpu-11ram-server
"""
cloud, flavors = self._setup_flavor_selection_tests(EXTRA_CONF)
# no instance type specified, should get smallest one
app1 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
)
flv = cloud.get_instance_type_for_job(app1)
assert flv == flavors[0]
# application-specific instance type specified but unavailable:
# should yield same result as before
app2 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
)
app2.application_name = 'large'
flv = cloud.get_instance_type_for_job(app2)
assert flv == flavors[0]
# application-specific instance type specified,
# but requirements exclude it
app3 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
requested_cores=6,
)
app3.application_name = 'large'
flv = cloud.get_instance_type_for_job(app3)
assert flv == flavors[2]
def test_flavor_selection_application_specific_unavailable():
"""
Test that application-specific "instance type" is ignored if unavailable.
"""
# NOTE: `EXTRA_CONF` has to start with a blank line and with the first
# non-whitespace character in column 0, otherwise `ConfigParser` will
# consider it a continuation of the last key/value assignment in
# `RESOURCE_STANZA` ...
EXTRA_CONF = """
large_instance_type = 5cpu-11ram-server
"""
cloud, flavors = _setup_flavor_selection_tests(EXTRA_CONF)
# no instance type specified, should get smallest one
app1 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
)
flv = cloud.get_instance_type_for_job(app1)
assert flv == flavors[0]
# application-specific instance type specified but unavailable:
# should yield same result as before
app2 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
)
app2.application_name = 'large'
flv = cloud.get_instance_type_for_job(app2)
assert flv == flavors[0]
# application-specific instance type specified,
# but requirements exclude it
app3 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
requested_cores=6,
)
app3.application_name = 'large'
flv = cloud.get_instance_type_for_job(app3)
assert flv == flavors[2]
def test_flavor_selection_generic_instance_type(self):
"""
Test flavor selection when a default flavor is prescribed by config.
"""
# NOTE: `EXTRA_CONF` has to start with a blank line and with the first
# non-whitespace character in column 0, otherwise `ConfigParser` will
# consider it a continuation of the last key/value assignment in
# `RESOURCE_STANZA` ...
EXTRA_CONF = """
instance_type = 4cpu-16ram-hpc
small_instance_type = 1cpu-4ram-hpc
"""
cloud, flavors = self._setup_flavor_selection_tests(EXTRA_CONF)
# no instance type specified, should get configured one
app1 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
)
flv = cloud.get_instance_type_for_job(app1)
assert flv == flavors[1]
# application-specific instance type specified, should override default
app2 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
)
app2.application_name = 'small'
flv = cloud.get_instance_type_for_job(app2)
assert flv == flavors[0]
# requirements exclude default instance type
app3 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
requested_cores=6,
)
app3.application_name = 'large'
flv = cloud.get_instance_type_for_job(app3)
assert flv == flavors[2]
def test_flavor_selection_application_specific():
"""
Test flavor selection when an application-specific flavor is prescribed by config.
"""
# NOTE: `EXTRA_CONF` has to start with a blank line and with the first
# non-whitespace character in column 0, otherwise `ConfigParser` will
# consider it a continuation of the last key/value assignment in
# `RESOURCE_STANZA` ...
EXTRA_CONF = """
large_instance_type = 4cpu-16ram-hpc
"""
cloud, flavors = _setup_flavor_selection_tests(EXTRA_CONF)
# no instance type specified, should get smallest one
app1 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
)
flv = cloud.get_instance_type_for_job(app1)
assert flv == flavors[0]
# application-specific instance type specified
app2 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
)
app2.application_name = 'large'
flv = cloud.get_instance_type_for_job(app2)
assert flv == flavors[1]
# application-specific instance type specified,
# but requirements exclude it
app3 = Application(
['/bin/true'],
inputs=[],
outputs=[],
output_dir='/tmp',
requested_cores=6,
)
app3.application_name = 'large'
flv = cloud.get_instance_type_for_job(app3)
assert flv == flavors[2]
def test_pbs_prologue_and_epilogue_contents_when_not_files(self):
"""Prologue and epilogue scripts are inserted in the submission script
"""
# Ugly hack. We have to list the job dirs to check which one
# is the new one.
jobdir = os.path.expanduser('~/.gc3pie_jobs')
jobs = []
if os.path.isdir(jobdir):
jobs = os.listdir(jobdir)
app = Application(['/bin/true'], [], [], '')
app.application_name = 'app2'
self.core = gc3libs.core.Core(self.cfg)
self.core.select_resource('testpbs')
try:
self.core.submit(app)
except Exception:
# it is normal to have an error since we don't probably
# run a pbs server in this machine.
pass
newjobs = [d for d in os.listdir(jobdir) if d not in jobs]
# There must be only one more job...
assert_equal(len(newjobs), 1)
newjobdir = os.path.join(jobdir, newjobs[0])
self.files_to_remove.append(newjobdir)
# and only one file in it
assert_equal(len(os.listdir(newjobdir)), 1)
# Check the content of the script file
scriptfname = os.path.join(newjobdir, (os.listdir(newjobdir)[0]))
scriptfile = open(scriptfname)
assert re.match(
"#!/bin/sh.*"
"# prologue file `.*` BEGIN.*"
"echo prologue.*"
"# prologue file END.*"
"# inline script BEGIN.*"
"echo prologue app2.*"
"# inline script END.*"
"/bin/true.*"
"# epilogue file `.*` BEGIN.*"
"echo epilogue.*"
"# epilogue file END.*"
"# inline script BEGIN.*"
"echo epilogue app2.*"
"# inline script END",
scriptfile.read(),
re.DOTALL | re.M)
scriptfile.close()
# kill the job
if app.execution.state != Run.State.NEW:
self.core.kill(app)
def test_pbs_prologue_and_epilogue_contents_when_not_files(self):
"""Prologue and epilogue scripts are inserted in the submission script
"""
# Ugly hack. We have to list the job dirs to check which one
# is the new one.
jobdir = os.path.expanduser('~/.gc3pie_jobs')
jobs = []
if os.path.isdir(jobdir):
jobs = os.listdir(jobdir)
app = Application(['/bin/true'], [], [], '')
app.application_name = 'app2'
self.core = gc3libs.core.Core(self.cfg)
self.core.select_resource('testpbs')
try:
self.core.submit(app)
except Exception:
# it is normal to have an error since we don't probably
# run a pbs server in this machine.
pass
newjobs = [d for d in os.listdir(jobdir) if d not in jobs]
# There must be only one more job...
assert len(newjobs) == 1
newjobdir = os.path.join(jobdir, newjobs[0])
self.files_to_remove.append(newjobdir)
# and only one file in it
assert len(os.listdir(newjobdir)) == 1
# Check the content of the script file
scriptfname = os.path.join(newjobdir, (os.listdir(newjobdir)[0]))
scriptfile = open(scriptfname)
assert re.match(
"#!/bin/sh.*"
"# prologue file `.*` BEGIN.*"
"echo prologue.*"
"# prologue file END.*"
"# inline script BEGIN.*"
"echo prologue app2.*"
"# inline script END.*"
"/bin/true.*"
"# epilogue file `.*` BEGIN.*"
"echo epilogue.*"
"# epilogue file END.*"
"# inline script BEGIN.*"
"echo epilogue app2.*"
"# inline script END",
scriptfile.read(),
re.DOTALL | re.M)
scriptfile.close()
# kill the job
if app.execution.state != Run.State.NEW:
self.core.kill(app)
