def check_file_accesspoller(self):
poller = plr.FilePoller(self.tmpdir)
assert self.tmpdir in poller._watched
# test create file
fpath = os.path.join(self.tmpdir, 'foo')
write_contents(fpath, 'test')
_check_events(poller, fpath, ['created'])
assert fpath in poller._watched
# since the `mtime` field only has 1-second resolution, we
# need to pause a bit to ensure the following
# `.get_new_events()` are able to see any modification
sleep(2)
# no new events here
_check_events(poller, fpath, [])
# test modify file
write_contents(fpath, 'test2')
_check_events(poller, fpath, ['modified'])
# test remove file
os.remove(fpath)
_check_events(poller, fpath, ['deleted'])
assert fpath not in poller._watched
def __init__(self,
executable,
cell_diffusion,
public_good_durability,
public_good_diffusion,
death_rate,
make_movie,
num_cells=None,
num_trials=None,
**extra_args):
extra_args.setdefault('requested_cores', 1)
self.executable = executable
self.cell_diffusion = cell_diffusion
self.public_good_diffusion = public_good_diffusion
self.public_good_durability = public_good_durability
self.death_rate = death_rate
arguments = ['./'+os.path.basename(executable),
'-c', str(float(cell_diffusion)),
'-p', str(float(public_good_diffusion)),
'-d', str(int(public_good_durability)),
'-x', str(float(death_rate)),
'-D', 'data']
if make_movie:
arguments.append('-m')
if num_cells:
arguments.extend(['--num-cells', num_cells])
if num_trials:
arguments.extend(['--num-trials', num_trials])
extra_args['jobname'] = "GBiointeract_cdiff:" + \
"%f_pgdiff:%f_dur:%f_deathrate:%f" % (cell_diffusion,
public_good_diffusion,
public_good_durability, death_rate)
if 'output_dir' in extra_args:
extra_args['output_dir'] = os.path.join(
os.path.dirname(extra_args['output_dir']),
extra_args['jobname'])
executable_script = """#!/bin/sh
# Create data directory
mkdir data
# Run the script
%s
# Compress all output files
gzip data/*.txt
""" % str.join(' ', arguments)
try:
(fd, self.tmp_filename) = tempfile.mkstemp(prefix='gc3pie-gbiointeract_')
write_contents(self.tmp_filename, executable_script)
os.chmod(self.tmp_filename, 0755)
except Exception, ex:
gc3libs.log.debug("Error creating execution script."
"Error type: %s. Message: %s" % (type(ex), ex.message))
raise
def setUp(self):
self.scriptdir = join(dirname(abspath(__file__)), 'scripts')
self.client_py = join(self.scriptdir, 'simpleclient.py')
self.daemon_py = join(self.scriptdir, 'simpledaemon.py')
self.script_py = join(self.scriptdir, 'simplescript.py')
self.basedir = tempfile.mkdtemp(
prefix=(__name__ + '.'),
suffix='.tmp.d')
orig_wd = os.getcwd()
os.chdir(self.basedir)
self.cfgfile = join(self.basedir, 'config.ini')
write_contents(self.cfgfile,
"""
[resource/localhost]
enabled = true
auth = none
type = shellcmd
frontend = localhost
transport = local
max_cores_per_job = 2
max_memory_per_core = 8GB
max_walltime = 8h
max_cores = 2
architecture = x86_64
override = False
resourcedir = {basedir}/resource.d
""".format(basedir=self.basedir))
try:
yield
finally:
os.chdir(orig_wd)
shutil.rmtree(self.basedir, ignore_errors=True)
def test_filepoller(self):
poller = plr.FilePoller(self.tmpdir)
assert self.tmpdir in poller._watched
# test create file
fpath = os.path.join(self.tmpdir, 'foo')
write_contents(fpath, 'test')
_check_events(poller, fpath, [['IN_CLOSE_WRITE', 'IN_CREATE']])
assert fpath in poller._watched
# test remove file
os.remove(fpath)
_check_events(poller, fpath, [['IN_DELETE']])
assert fpath not in poller._watched
def test_inotifypoller(self):
poller = plr.INotifyPoller(self.tmpdir)
# test create file
fpath = os.path.join(self.tmpdir, 'foo')
write_contents(fpath, 'test')
_check_events(poller, fpath, ['created'])
# no new events here
_check_events(poller, fpath, [])
# test modify file
write_contents(fpath, 'test2')
_check_events(poller, fpath, ['modified'])
# test remove file
os.remove(fpath)
_check_events(poller, fpath, ['deleted'])
def test_inotifypoller(self):
poller = plr.INotifyPoller(self.tmpdir)
# test create file
fpath = os.path.join(self.tmpdir, 'foo')
write_contents(fpath, 'test')
_check_events(
poller,
fpath,
[
# inotify sends 4 distinct events
['IN_CREATE'],
['IN_OPEN'],
['IN_MODIFY'],
['IN_CLOSE_WRITE']
])
# test remove file
os.remove(fpath)
_check_events(poller, fpath, [['IN_DELETE']])
def _make_temp_run_docker_file(location, input_csv, seed):
"""
Create execution script to control docker execution and post-process
"""
try:
(fd, tmp_filename) = tempfile.mkstemp(prefix="sbj{0}-".format(subject),
dir=location)
write_contents(
tmp_filename,
RUN_SCRIPT.format(data=data,
output=output,
container=docker,
analysis=analysis,
subject=subject))
os.chmod(tmp_filename, 0755)
return tmp_filename
except Exception, ex:
gc3libs.log.debug("Error creating execution script."
"Error type: %s. Message: %s" %
(type(ex), ex.message))
raise
def test_session_based_daemon_inbox(self, daemon, max_wait=10):
_, _, inboxdir = daemon
# Wait until the daemon is up and running.
# It will create the directory, so let's wait until then.
self.check_with_timeout(
condition=lambda: isdir(inboxdir),
timeout=max_wait,
errmsg="Daemon didn't create inbox dir"
)
# create marker file
marker_file = join(inboxdir, 'foo')
write_contents(marker_file, 'whatever')
# wait up to max_wait seconds for task to complete
output_dir = join(self.basedir, 'LSApp.foo')
self.check_with_timeout(
condition=lambda: isdir(output_dir),
timeout=max_wait,
errmsg="Daemon didn't process incoming file"
)
def __init__(self,
executable,
cell_diffusion,
public_good_durability,
public_good_diffusion,
death_rate,
make_movie,
num_cells=None,
num_trials=None,
**extra_args):
extra_args.setdefault('requested_cores', 1)
self.executable = executable
self.cell_diffusion = cell_diffusion
self.public_good_diffusion = public_good_diffusion
self.public_good_durability = public_good_durability
self.death_rate = death_rate
arguments = [
'./' + os.path.basename(executable), '-c',
str(float(cell_diffusion)), '-p',
str(float(public_good_diffusion)), '-d',
str(int(public_good_durability)), '-x',
str(float(death_rate)), '-D', 'data'
]
if make_movie:
arguments.append('-m')
if num_cells:
arguments.extend(['--num-cells', num_cells])
if num_trials:
arguments.extend(['--num-trials', num_trials])
extra_args['jobname'] = "GBiointeract_cdiff:" + \
"%f_pgdiff:%f_dur:%f_deathrate:%f" % (cell_diffusion,
public_good_diffusion,
public_good_durability, death_rate)
if 'output_dir' in extra_args:
extra_args['output_dir'] = os.path.join(
os.path.dirname(extra_args['output_dir']),
extra_args['jobname'])
executable_script = """#!/bin/sh
# Create data directory
mkdir data
# Run the script
%s
# Compress all output files
gzip data/*.txt
""" % str.join(' ', arguments)
try:
(fd, self.tmp_filename) = tempfile.mkstemp(
prefix='gc3pie-gbiointeract_')
write_contents(self.tmp_filename, executable_script)
os.chmod(self.tmp_filename, 0o755)
except Exception, ex:
gc3libs.log.debug("Error creating execution script."
"Error type: %s. Message: %s" %
(type(ex), ex.message))
raise