def test_run(self):
t = load_object_from_file(os.path.join(module_dir, "write.yaml"))
t.run_task({})
for i in range(2):
self.assertTrue(os.path.exists("myfile{}".format(i + 1)))
#Use delete task to remove the files created.
t = load_object_from_file(os.path.join(module_dir, "delete.yaml"))
t.run_task({})
for i in range(2):
self.assertFalse(os.path.exists("myfile{}".format(i + 1)))
def test_run(self):
t = load_object_from_file(os.path.join(module_dir, "write.yaml"))
t.run_task({})
for i in range(2):
self.assertTrue(os.path.exists(f"myfile{i + 1}"))
# Use delete task to remove the files created.
t = load_object_from_file(os.path.join(module_dir, "delete.yaml"))
t.run_task({})
for i in range(2):
self.assertFalse(os.path.exists(f"myfile{i + 1}"))
def test_validate_load_object_from_file(self):
""" validate Firetask, CommonAdapter schemas via load_object_from_file() """
schemas = ['Firetask', 'CommonAdapter']
for schema in schemas:
path = os.path.join(SAMPLES_DIR, schema.lower())
for sfile in os.listdir(path):
if sfile in self.skip_list:
continue
try:
load_object_from_file(os.path.join(path, sfile))
except ValidationError as err:
self.fail('Validation error: '+err.message)
def do_launch(args):
if not args.launchpad_file and os.path.exists(
os.path.join(args.config_dir, 'my_launchpad.yaml')):
args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml')
elif not args.launchpad_file:
args.launchpad_file = LAUNCHPAD_LOC
if not args.fworker_file and os.path.exists(
os.path.join(args.config_dir, 'my_fworker.yaml')):
args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml')
elif not args.fworker_file:
args.fworker_file = FWORKER_LOC
if not args.queueadapter_file and os.path.exists(
os.path.join(args.config_dir, 'my_qadapter.yaml')):
args.queueadapter_file = os.path.join(args.config_dir, 'my_qadapter.yaml')
elif not args.queueadapter_file:
args.queueadapter_file = QUEUEADAPTER_LOC
launchpad = LaunchPad.from_file(
args.launchpad_file) if args.launchpad_file else LaunchPad(
strm_lvl=args.loglvl)
fworker = FWorker.from_file(
args.fworker_file) if args.fworker_file else FWorker()
queueadapter = load_object_from_file(args.queueadapter_file)
args.loglvl = 'CRITICAL' if args.silencer else args.loglvl
if args.command == 'rapidfire':
rapidfire(launchpad, fworker=fworker, qadapter=queueadapter, launch_dir=args.launch_dir,
nlaunches=args.nlaunches, njobs_queue=args.maxjobs_queue,
njobs_block=args.maxjobs_block, sleep_time=args.sleep,
reserve=args.reserve, strm_lvl=args.loglvl, timeout=args.timeout, fill_mode=args.fill_mode)
else:
launch_rocket_to_queue(launchpad, fworker, queueadapter,
args.launch_dir, args.reserve, args.loglvl, False, args.fill_mode, args.fw_id)
def do_launch(args):
if not args.launchpad_file and os.path.exists(
os.path.join(args.config_dir, 'my_launchpad.yaml')):
args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml')
if not args.fworker_file and os.path.exists(
os.path.join(args.config_dir, 'my_fworker.yaml')):
args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml')
if not args.queueadapter_file and os.path.exists(
os.path.join(args.config_dir, 'my_qadapter.yaml')):
args.queueadapter_file = os.path.join(args.config_dir,
'my_qadapter.yaml')
launchpad = LaunchPad.from_file(
args.launchpad_file) if args.launchpad_file else LaunchPad(
strm_lvl=args.loglvl)
fworker = FWorker.from_file(
args.fworker_file) if args.fworker_file else FWorker()
queueadapter = load_object_from_file(args.queueadapter_file)
args.loglvl = 'CRITICAL' if args.silencer else args.loglvl
if args.command == 'rapidfire':
rapidfire(launchpad, fworker, queueadapter, args.launch_dir,
args.nlaunches, args.maxjobs_queue,
args.maxjobs_block, args.sleep, args.reserve, args.loglvl)
else:
launch_rocket_to_queue(launchpad, fworker, queueadapter,
args.launch_dir, args.reserve, args.loglvl, False)
def test_override(self):
p = load_object_from_file(
os.path.join(os.path.dirname(__file__), "pbs_override.yaml"))
self.assertEqual(p._get_status_cmd("my_name"),
['my_qstatus', '-u', 'my_name'])
self.assertEqual(p.q_commands["PBS"]["submit_cmd"], "my_qsubmit")
def do_launch(args):
if not args.launchpad_file and os.path.exists(
os.path.join(args.config_dir, 'my_launchpad.yaml')):
args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml')
elif not args.launchpad_file:
args.launchpad_file = LAUNCHPAD_LOC
if not args.fworker_file and os.path.exists(
os.path.join(args.config_dir, 'my_fworker.yaml')):
args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml')
elif not args.fworker_file:
args.fworker_file = FWORKER_LOC
if not args.queueadapter_file and os.path.exists(
os.path.join(args.config_dir, 'my_qadapter.yaml')):
args.queueadapter_file = os.path.join(args.config_dir, 'my_qadapter.yaml')
elif not args.queueadapter_file:
args.queueadapter_file = QUEUEADAPTER_LOC
launchpad = LaunchPad.from_file(
args.launchpad_file) if args.launchpad_file else LaunchPad(
strm_lvl=args.loglvl)
fworker = FWorker.from_file(
args.fworker_file) if args.fworker_file else FWorker()
queueadapter = load_object_from_file(args.queueadapter_file)
args.loglvl = 'CRITICAL' if args.silencer else args.loglvl
if args.command == 'rapidfire':
rapidfire(launchpad, fworker=fworker, qadapter=queueadapter, launch_dir=args.launch_dir,
nlaunches=args.nlaunches, njobs_queue=args.maxjobs_queue,
njobs_block=args.maxjobs_block, sleep_time=args.sleep,
reserve=args.reserve, strm_lvl=args.loglvl, timeout=args.timeout, fill_mode=args.fill_mode)
else:
launch_rocket_to_queue(launchpad, fworker, queueadapter,
args.launch_dir, args.reserve, args.loglvl, False, args.fill_mode)
def test_status_cmd_pbs(self):
p = load_object_from_file(
os.path.join(os.path.dirname(__file__), "pbs_override.yaml"
) # intentional red herring to test deepcopy
)
p = CommonAdapter(q_type="PBS")
self.assertEqual(p._get_status_cmd("my_name"),
["qstat", "-u", "my_name"])
def test_yaml_load(self):
#Test yaml loading.
p = load_object_from_file(os.path.join(os.path.dirname(__file__),
"pbs.yaml"))
p = CommonAdapter(
q_type="PBS",
q_name="hello",
ppnode="8:ib", nnodes=1,
hello="world", queue="random")
print(p.get_script_str("."))
import yaml
print(yaml.dump(p.to_dict(), default_flow_style=False))
def test_yaml_load(self):
#Test yaml loading.
p = load_object_from_file(
os.path.join(os.path.dirname(__file__), "pbs.yaml"))
p = CommonAdapter(q_type="PBS",
q_name="hello",
ppnode="8:ib",
nnodes=1,
hello="world",
queue="random")
print(p.get_script_str("."))
import yaml
print(yaml.dump(p.to_dict(), default_flow_style=False))
def test_override(self):
p = load_object_from_file(os.path.join(os.path.dirname(__file__),
"pbs_override.yaml"))
self.assertEqual(p._get_status_cmd("my_name"), ['my_qstatus', '-u', 'my_name'])
self.assertEqual(p.q_commands["PBS"]["submit_cmd"], "my_qsubmit")
def test_status_cmd_pbs(self):
p = load_object_from_file(os.path.join(os.path.dirname(__file__), # intentional red herring to test deepcopy
"pbs_override.yaml"))
p = CommonAdapter(q_type="PBS")
self.assertEqual(p._get_status_cmd("my_name"), ['qstat', '-u', 'my_name'])
def watcher_daemon(self, log_file):
log = None
if (log_file):
try:
log = open(log_file, "w")
except:
log = log_file #hope its a filehandle instead!
print >> sys.stderr, "Log file opened!"
#about to fork a process, throw away all handlers.
#fireworks will create a new queue log handler to write to test with
#lil hacky but who cares right now
logging.handlers = []
#FIXME this seems not to have fixed it all the time?
old_sys_stdout = sys.stdout
self.db.client.close()
self.launchpad.connection.close()
print >> sys.stderr, "Connections closed, preparing to fork..."
with daemon.DaemonContext(stdout=log, stderr=log):
logging.handlers = []
dbm = DatabaseManager()
self.db = dbm
#reconnect to mongo after fork
print dbm.find_lpad_config()
self.launchpad = fireworks.LaunchPad.from_file(
dbm.find_lpad_config())
self.qadapter = dbm.find_qadapter()
#add our pid as a running process so new daemons don't get started
dbm.client.admin.authenticate("fireworks", "speakfriendandenter")
db = dbm.client.daemons
#FIXME POSSIBLE CRITICAL RAISE FOR EXTREMELY RAPID WORKFLOW STARTS
#ADD MUTEX?
running_daemons = db.daemons.find({
"user": getpass.getuser()
}).count()
if running_daemons > 0:
#todo, check pid is alive
print >> old_sys_stdout, "Not Forking Daemon- daemon process found"
#don't start daemon
sys.exit(0)
atexit.register(self.cleanup_daemon)
db.daemons.insert_one({
"user": getpass.getuser(),
"pid": os.getpid()
})
while (True):
common_adapter = load_object_from_file(self.qadapter)
launcher_log_dir = os.path.join(FW_WFLOW_LAUNCH_LOC,
getpass.getuser(), "")
queue_launcher.rapidfire(self.launchpad,
fireworks.FWorker(name="LSF"),
common_adapter,
reserve=True,
nlaunches=0,
launch_dir=launcher_log_dir,
sleep_time=10,
njobs_queue=500)
failed_fws = []
time.sleep(50)
# offline_runs = self.launchpad.offline_runs.find({"completed": False, "deprecated": False}, {"launch_id": 1}).count()
# self.launchpad.m_logger.info("%s offline runs found" % offline_runs)
ready_lsf_jobs = self.launchpad.fireworks.find({
"state":
"READY",
"spec._fworker":
"LSF"
}).count()
reserved_lsf_jobs = self.launchpad.fireworks.find({
"state":
"RESERVED",
"spec._fworker":
"LSF"
}).count()
running_lsf_jobs = self.launchpad.fireworks.find({
"state":
"RUNNING",
"spec._fworker":
"LSF"
}).count()
self.launchpad.m_logger.info(
"%s ready, %s running, %s reserved lsf jobs found" %
(ready_lsf_jobs, running_lsf_jobs, reserved_lsf_jobs))
if (ready_lsf_jobs == 0 and reserved_lsf_jobs == 0
and running_lsf_jobs == 0):
break
for l in self.launchpad.offline_runs.find(
{
"completed": False,
"deprecated": False
}, {"launch_id": 1}):
fw = self.launchpad.recover_offline(l['launch_id'], True)
if fw:
failed_fws.append(fw)
self.launchpad.m_logger.info(
"FINISHED recovering offline runs.")
if failed_fws:
self.launchpad.m_logger.info(
"FAILED to recover offline fw_ids: {}".format(
failed_fws))
db.daemons.remove({"user": getpass.getuser()})
