def do_launch(args): if not args.launchpad_file and os.path.exists( os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') if not args.fworker_file and os.path.exists( os.path.join(args.config_dir, 'my_fworker.yaml')): args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml') if not args.queueadapter_file and os.path.exists( os.path.join(args.config_dir, 'my_qadapter.yaml')): args.queueadapter_file = os.path.join(args.config_dir, 'my_qadapter.yaml') launchpad = LaunchPad.from_file( args.launchpad_file) if args.launchpad_file else LaunchPad( strm_lvl=args.loglvl) fworker = FWorker.from_file( args.fworker_file) if args.fworker_file else FWorker() queueadapter = load_object_from_file(args.queueadapter_file) args.loglvl = 'CRITICAL' if args.silencer else args.loglvl if args.command == 'rapidfire': rapidfire(launchpad, fworker, queueadapter, args.launch_dir, args.nlaunches, args.maxjobs_queue, args.maxjobs_block, args.sleep, args.reserve, args.loglvl) else: launch_rocket_to_queue(launchpad, fworker, queueadapter, args.launch_dir, args.reserve, args.loglvl, False)
def do_launch(args): if not args.launchpad_file and os.path.exists( os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') elif not args.launchpad_file: args.launchpad_file = LAUNCHPAD_LOC if not args.fworker_file and os.path.exists( os.path.join(args.config_dir, 'my_fworker.yaml')): args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml') elif not args.fworker_file: args.fworker_file = FWORKER_LOC if not args.queueadapter_file and os.path.exists( os.path.join(args.config_dir, 'my_qadapter.yaml')): args.queueadapter_file = os.path.join(args.config_dir, 'my_qadapter.yaml') elif not args.queueadapter_file: args.queueadapter_file = QUEUEADAPTER_LOC launchpad = LaunchPad.from_file( args.launchpad_file) if args.launchpad_file else LaunchPad( strm_lvl=args.loglvl) fworker = FWorker.from_file( args.fworker_file) if args.fworker_file else FWorker() queueadapter = load_object_from_file(args.queueadapter_file) args.loglvl = 'CRITICAL' if args.silencer else args.loglvl if args.command == 'rapidfire': rapidfire(launchpad, fworker=fworker, qadapter=queueadapter, launch_dir=args.launch_dir, nlaunches=args.nlaunches, njobs_queue=args.maxjobs_queue, njobs_block=args.maxjobs_block, sleep_time=args.sleep, reserve=args.reserve, strm_lvl=args.loglvl, timeout=args.timeout, fill_mode=args.fill_mode) else: launch_rocket_to_queue(launchpad, fworker, queueadapter, args.launch_dir, args.reserve, args.loglvl, False, args.fill_mode)
def do_launch(args): if not args.launchpad_file and os.path.exists( os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') elif not args.launchpad_file: args.launchpad_file = LAUNCHPAD_LOC if not args.fworker_file and os.path.exists( os.path.join(args.config_dir, 'my_fworker.yaml')): args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml') elif not args.fworker_file: args.fworker_file = FWORKER_LOC if not args.queueadapter_file and os.path.exists( os.path.join(args.config_dir, 'my_qadapter.yaml')): args.queueadapter_file = os.path.join(args.config_dir, 'my_qadapter.yaml') elif not args.queueadapter_file: args.queueadapter_file = QUEUEADAPTER_LOC launchpad = LaunchPad.from_file( args.launchpad_file) if args.launchpad_file else LaunchPad( strm_lvl=args.loglvl) fworker = FWorker.from_file( args.fworker_file) if args.fworker_file else FWorker() queueadapter = load_object_from_file(args.queueadapter_file) args.loglvl = 'CRITICAL' if args.silencer else args.loglvl if args.command == 'rapidfire': rapidfire(launchpad, fworker=fworker, qadapter=queueadapter, launch_dir=args.launch_dir, nlaunches=args.nlaunches, njobs_queue=args.maxjobs_queue, njobs_block=args.maxjobs_block, sleep_time=args.sleep, reserve=args.reserve, strm_lvl=args.loglvl, timeout=args.timeout, fill_mode=args.fill_mode) else: launch_rocket_to_queue(launchpad, fworker, queueadapter, args.launch_dir, args.reserve, args.loglvl, False, args.fill_mode, args.fw_id)
def rlaunch(): m_description = 'This program launches one or more Rockets. A Rocket grabs a job from the central database and ' \ 'runs it. The "single-shot" option launches a single Rocket, ' \ 'whereas the "rapidfire" option loops until all FireWorks are completed.' parser = ArgumentParser(description=m_description) subparsers = parser.add_subparsers(help='command', dest='command') single_parser = subparsers.add_parser('singleshot', help='launch a single Rocket') rapid_parser = subparsers.add_parser('rapidfire', help='launch multiple Rockets (loop until all FireWorks complete)') single_parser.add_argument('-f', '--fw_id', help='specific fw_id to run', default=None, type=int) single_parser.add_argument('--offline', help='run in offline mode (FW.json required)', action='store_true') rapid_parser.add_argument('--nlaunches', help='num_launches (int or "infinite"; default 0 is all jobs in DB)', default=0) rapid_parser.add_argument('--sleep', help='sleep time between loops (secs)', default=None, type=int) parser.add_argument('-l', '--launchpad_file', help='path to launchpad file', default=LAUNCHPAD_LOC) parser.add_argument('-w', '--fworker_file', help='path to fworker file', default=FWORKER_LOC) parser.add_argument('-c', '--config_dir', help='path to a directory containing the config file (used if -l, -w unspecified)', default=CONFIG_FILE_DIR) parser.add_argument('--loglvl', help='level to print log messages', default='INFO') parser.add_argument('-s', '--silencer', help='shortcut to mute log messages', action='store_true') args = parser.parse_args() signal.signal(signal.SIGINT, handle_interrupt) # graceful exist on ^C if not args.launchpad_file and os.path.exists(os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') if not args.fworker_file and os.path.exists(os.path.join(args.config_dir, 'my_fworker.yaml')): args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml') args.loglvl = 'CRITICAL' if args.silencer else args.loglvl if args.command == 'singleshot' and args.offline: launchpad = None else: launchpad = LaunchPad.from_file(args.launchpad_file) if args.launchpad_file else LaunchPad(strm_lvl=args.loglvl) if args.fworker_file: fworker = FWorker.from_file(args.fworker_file) else: fworker = FWorker() # prime addr lookups _log = get_fw_logger("rlaunch", stream_level="INFO") _log.info("Hostname/IP lookup (this will take a few seconds)") get_my_host() get_my_ip() if args.command == 'rapidfire': rapidfire(launchpad, fworker, None, args.nlaunches, -1, args.sleep, args.loglvl) else: launch_rocket(launchpad, fworker, args.fw_id, args.loglvl)
def rlaunch(): m_description = 'This program launches one or more Rockets. A Rocket grabs a job from the central database and ' \ 'runs it. The "single-shot" option launches a single Rocket, ' \ 'whereas the "rapidfire" option loops until all FireWorks are completed.' parser = ArgumentParser(description=m_description) subparsers = parser.add_subparsers(help='command', dest='command') single_parser = subparsers.add_parser('singleshot', help='launch a single Rocket') rapid_parser = subparsers.add_parser('rapidfire', help='launch multiple Rockets (loop until all FireWorks complete)') single_parser.add_argument('-f', '--fw_id', help='specific fw_id to run', default=None, type=int) single_parser.add_argument('--offline', help='run in offline mode (FW.json required)', action='store_true') rapid_parser.add_argument('--nlaunches', help='num_launches (int or "infinite"; default 0 is all jobs in DB)', default=0) rapid_parser.add_argument('--sleep', help='sleep time between loops (secs)', default=None, type=int) parser.add_argument('-l', '--launchpad_file', help='path to launchpad file', default=LAUNCHPAD_LOC) parser.add_argument('-w', '--fworker_file', help='path to fworker file', default=FWORKER_LOC) parser.add_argument('-c', '--config_dir', help='path to a directory containing the config file (used if -l, -w unspecified)', default=CONFIG_FILE_DIR) parser.add_argument('--loglvl', help='level to print log messages', default='INFO') parser.add_argument('-s', '--silencer', help='shortcut to mute log messages', action='store_true') args = parser.parse_args() signal.signal(signal.SIGINT, handle_interrupt) # graceful exist on ^C if not args.launchpad_file and os.path.exists(os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') if not args.fworker_file and os.path.exists(os.path.join(args.config_dir, 'my_fworker.yaml')): args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml') args.loglvl = 'CRITICAL' if args.silencer else args.loglvl if args.command == 'singleshot' and args.offline: launchpad = None else: launchpad = LaunchPad.from_file(args.launchpad_file) if args.launchpad_file else LaunchPad(strm_lvl=args.loglvl) if args.fworker_file: fworker = FWorker.from_file(args.fworker_file) else: fworker = FWorker() # prime addr lookups _log = get_fw_logger("rlaunch", stream_level="INFO") _log.info("Hostname/IP lookup (this will take a few seconds)") get_my_host() get_my_ip() if args.command == 'rapidfire': rapidfire(launchpad, fworker, None, args.nlaunches, -1, args.sleep, args.loglvl) else: launch_rocket(launchpad, fworker, args.fw_id, args.loglvl)
def auto_load(cls): sma = SubmissionMongoAdapter.auto_load() l_dir = FWConfig().CONFIG_FILE_DIR l_file = os.path.join(l_dir, 'my_launchpad.yaml') lp = LaunchPad.from_file(l_file) return SubmissionProcessor(sma, lp)
def setUp(self): if os.path.exists(self.scratch_dir): shutil.rmtree(self.scratch_dir) os.makedirs(self.scratch_dir) os.chdir(self.scratch_dir) try: self.lp = LaunchPad.from_file(os.path.join(db_dir, "my_launchpad.yaml")) self.lp.reset("", require_password=False) except: raise unittest.SkipTest( 'Cannot connect to MongoDB! Is the database server running? ' 'Are the credentials correct?')
def __init__(self, launchpad=None): super().__init__() # Here store the launchpad in the class attribute so it can be reused.... if launchpad is not None: self.lpad = launchpad # Keep the launchpad FwScheduler._lpad = launchpad else: # Create and save the launchpad if FwScheduler._lpad is None: FwScheduler._lpad = LaunchPad.from_file(LAUNCHPAD_LOC) self.lpad = FwScheduler._lpad
def mlaunch(): m_description = 'This program launches multiple Rockets simultaneously' parser = ArgumentParser(description=m_description) parser.add_argument('num_jobs', help='the number of jobs to run in parallel', type=int) parser.add_argument('--nlaunches', help='number of FireWorks to run in series per parallel job (int or "infinite"; default 0 is all jobs in DB)', default=0) parser.add_argument('--sleep', help='sleep time between loops in infinite launch mode (secs)', default=None, type=int) parser.add_argument('--timeout', help='timeout (secs) after which to quit (default None)', default=None, type=int) parser.add_argument('-l', '--launchpad_file', help='path to launchpad file', default=LAUNCHPAD_LOC) parser.add_argument('-w', '--fworker_file', help='path to fworker file', default=FWORKER_LOC) parser.add_argument('-c', '--config_dir', help='path to a directory containing the config file (used if -l, -w unspecified)', default=CONFIG_FILE_DIR) parser.add_argument('--loglvl', help='level to print log messages', default='INFO') parser.add_argument('-s', '--silencer', help='shortcut to mute log messages', action='store_true') parser.add_argument('--nodefile', help='nodefile name or environment variable name containing the node file name (for populating FWData only)', default=None, type=str) parser.add_argument('--ppn', help='processors per node (for populating FWData only)', default=1, type=int) args = parser.parse_args() if not args.launchpad_file and args.config_dir and os.path.exists(os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') if not args.fworker_file and args.config_dir and os.path.exists(os.path.join(args.config_dir, 'my_fworker.yaml')): args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml') args.loglvl = 'CRITICAL' if args.silencer else args.loglvl launchpad = LaunchPad.from_file(args.launchpad_file) if args.launchpad_file else LaunchPad(strm_lvl=args.loglvl) if args.fworker_file: fworker = FWorker.from_file(args.fworker_file) else: fworker = FWorker() total_node_list = None if args.nodefile: if args.nodefile in os.environ: args.nodefile = os.environ[args.nodefile] with open(args.nodefile, 'r') as f: total_node_list = [line.strip() for line in f.readlines()] launch_multiprocess(launchpad, fworker, args.loglvl, args.nlaunches, args.num_jobs, args.sleep, total_node_list, args.ppn, timeout=args.timeout)
def mlaunch(): m_description = 'This program launches multiple Rockets simultaneously' parser = ArgumentParser(description=m_description) parser.add_argument('num_jobs', help='the number of jobs to run in parallel', type=int) parser.add_argument('--nlaunches', help='number of FireWorks to run in series per parallel job (int or "infinite"; default 0 is all jobs in DB)', default=0) parser.add_argument('--sleep', help='sleep time between loops in infinite launch mode (secs)', default=None, type=int) parser.add_argument('-l', '--launchpad_file', help='path to launchpad file', default=LAUNCHPAD_LOC) parser.add_argument('-w', '--fworker_file', help='path to fworker file', default=FWORKER_LOC) parser.add_argument('-c', '--config_dir', help='path to a directory containing the config file (used if -l, -w unspecified)', default=CONFIG_FILE_DIR) parser.add_argument('--loglvl', help='level to print log messages', default='INFO') parser.add_argument('-s', '--silencer', help='shortcut to mute log messages', action='store_true') parser.add_argument('--nodefile', help='nodefile name or environment variable name containing the node file name (for populating FWData only)', default=None, type=str) parser.add_argument('--ppn', help='processors per node (for populating FWData only)', default=1, type=int) args = parser.parse_args() if not args.launchpad_file and args.config_dir and os.path.exists(os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') if not args.fworker_file and args.config_dir and os.path.exists(os.path.join(args.config_dir, 'my_fworker.yaml')): args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml') args.loglvl = 'CRITICAL' if args.silencer else args.loglvl launchpad = LaunchPad.from_file(args.launchpad_file) if args.launchpad_file else LaunchPad(strm_lvl=args.loglvl) if args.fworker_file: fworker = FWorker.from_file(args.fworker_file) else: fworker = FWorker() total_node_list = None if args.nodefile: if args.nodefile in os.environ: args.nodefile = os.environ[args.nodefile] with open(args.nodefile, 'r') as f: total_node_list = [line.strip() for line in f.readlines()] launch_multiprocess(launchpad, fworker, args.loglvl, args.nlaunches, args.num_jobs, args.sleep, total_node_list, args.ppn)
def restart_fizzled(): module_dir = os.path.dirname(os.path.abspath(__file__)) lp_f = os.path.join(module_dir, 'my_launchpad.yaml') lpdb = LaunchPad.from_file(lp_f) for fw in lpdb.fireworks.find({"state": "FIZZLED"}, {"fw_id": 1, "spec.task_type": 1}): fw_id = fw['fw_id'] task_type = fw['spec']['task_type'] restart_id = fw_id if 'VASP db insertion' in task_type: restart_id = fw_id - 1 elif 'Controller' in task_type: restart_id = fw_id - 2 lpdb.rerun_fw(restart_id)
def get_lp(args): try: if not args.launchpad_file and os.path.exists(os.path.join(args.config_dir, DEFAULT_LPAD_YAML)): args.launchpad_file = os.path.join(args.config_dir, DEFAULT_LPAD_YAML) if args.launchpad_file: return LaunchPad.from_file(args.launchpad_file) else: args.loglvl = 'CRITICAL' if args.silencer else args.loglvl return LaunchPad(logdir=args.logdir, strm_lvl=args.loglvl) except: traceback.print_exc() err_message = 'FireWorks was not able to connect to MongoDB. Is the server running? The database file specified was {}.'.format(args.launchpad_file) if not args.launchpad_file: err_message += ' Type "lpad init" if you would like to set up a file that specifies location and credentials of your Mongo database (otherwise use default localhost configuration).' raise ValueError(err_message)
def get_lp(args): try: if not args.launchpad_file and os.path.exists(os.path.join(args.config_dir, DEFAULT_LPAD_YAML)): args.launchpad_file = os.path.join(args.config_dir, DEFAULT_LPAD_YAML) if args.launchpad_file: return LaunchPad.from_file(args.launchpad_file) else: args.loglvl = 'CRITICAL' if args.silencer else args.loglvl return LaunchPad(logdir=args.logdir, strm_lvl=args.loglvl) except: traceback.print_exc() err_message = 'FireWorks was not able to connect to MongoDB. Is the server running? The database file specified was {}.'.format(args.launchpad_file) if not args.launchpad_file: err_message += ' Type "lpad init" if you would like to set up a file that specifies location and credentials of your Mongo database (otherwise use default localhost configuration).' raise ValueError(err_message)
def archive_deprecated_fws(): # find all snlgroups that are deprecated, and archive all WFs that have deprecated fw_ids so we don't run them module_dir = os.path.dirname(os.path.abspath(__file__)) snl_f = os.path.join(module_dir, 'snl.yaml') snldb = SNLMongoAdapter.from_file(snl_f) snlgroups = snldb.snlgroups lp_f = os.path.join(module_dir, 'my_launchpad.yaml') lpdb = LaunchPad.from_file(lp_f) for g in snlgroups.find({'canonical_snl.about.remarks':'DEPRECATED'}, {'snlgroup_id': 1}): while lpdb.fireworks.find_one({'spec.snlgroup_id': g['snlgroup_id'], 'state': {'$ne': 'ARCHIVED'}}, {'fw_id': 1}): fw = lpdb.fireworks.find_one({'spec.snlgroup_id': g['snlgroup_id'], 'state': {'$ne': 'ARCHIVED'}}, {'fw_id': 1}) print fw['fw_id'] lpdb.archive_wf(fw['fw_id']) print 'DONE'
def get_colls(): colls = namedtuple('Collections', ['snl', 'snlgroups']) sma = SNLMongoAdapter.from_file(snl_f) lp = LaunchPad.from_file(fw_f) colls.snl = sma.snl colls.snlgroups = sma.snlgroups colls.fireworks = lp.fireworks colls.launches = lp.launches with open(tasks_f) as f2: task_creds = yaml.load(f2) mc = MongoClient(task_creds['host'], task_creds['port']) db = mc[task_creds['database']] db.authenticate(task_creds['admin_user'], task_creds['admin_password']) colls.tasks = db['tasks'] return colls
x = tdb.tasks.find_one({"fw_id": fw_id}, {"analysis": 1}) warnings = x['analysis'].get('warnings', []) warnings.extend(x['analysis']['errors_MP']['signals']) errors = x['analysis'].get('errors', []) errors.extend(x['analysis']['errors_MP']['critical_signals']) warnings = set(warnings) errors = set(errors) warnings = warnings.difference(errors) return set(warnings), set(errors) if __name__ == '__main__': module_dir = os.path.dirname(os.path.abspath(__file__)) lp_f = os.path.join(module_dir, 'my_launchpad.yaml') lpdb = LaunchPad.from_file(lp_f) tasks_f = os.path.join(module_dir, 'tasks_read.yaml') creds = {} with open(tasks_f) as f: creds = yaml.load(f) connection = MongoClient(creds['host'], creds['port']) tdb = connection[creds['db']] tdb.authenticate(creds['username'], creds['password']) except_dict = defaultdict(int) fizzled_fws = []
parser.add_argument('-w', '--fworker_file', help='path to fworker file', default=None) parser.add_argument('--logdir', help='path to a directory for logging', default=None) parser.add_argument('--loglvl', help='level to print log messages', default='INFO') parser.add_argument('--silencer', help='shortcut to mute log messages', action='store_true') args = parser.parse_args() if not args.launchpad_file and os.path.exists('my_launchpad.yaml'): args.launchpad_file = 'my_launchpad.yaml' if not args.fworker_file and os.path.exists('my_fworker.yaml'): args.fworker_file = 'my_fworker.yaml' args.loglvl = 'CRITICAL' if args.silencer else args.loglvl if args.launchpad_file: launchpad = LaunchPad.from_file(args.launchpad_file) else: launchpad = LaunchPad(logdir=args.logdir, strm_lvl=args.loglvl) if args.fworker_file: fworker = FWorker.from_file(args.fworker_file) else: fworker = FWorker() if args.command == 'rapidfire': rapidfire(launchpad, fworker, None, args.logdir, args.loglvl, args.nlaunches, args.sleep) else: launch_rocket(launchpad, fworker, args.logdir, args.loglvl, args.fw_id)
default='INFO') parser.add_argument('--silencer', help='shortcut to mute log messages', action='store_true') args = parser.parse_args() if args.command == 'version': print FW_VERSION else: if not args.launchpad_file and os.path.exists('my_launchpad.yaml'): args.launchpad_file = 'my_launchpad.yaml' if args.launchpad_file: lp = LaunchPad.from_file(args.launchpad_file) else: args.loglvl = 'CRITICAL' if args.silencer else args.loglvl lp = LaunchPad(logdir=args.logdir, strm_lvl=args.loglvl) if args.command == 'reset': lp.reset(args.password) elif args.command == 'detect_fizzled': # TODO: report when fixed print lp.detect_fizzled(args.time, args.fix) elif args.command == 'detect_unreserved': # TODO: report when fixed print lp.detect_unreserved(args.time, args.fix)
def rlaunch(): m_description = 'This program launches one or more Rockets. A Rocket grabs a job from the ' \ 'central database and runs it. The "single-shot" option launches a single Rocket, ' \ 'whereas the "rapidfire" option loops until all FireWorks are completed.' parser = ArgumentParser(description=m_description) subparsers = parser.add_subparsers(help='command', dest='command') single_parser = subparsers.add_parser('singleshot', help='launch a single Rocket') rapid_parser = subparsers.add_parser('rapidfire', help='launch multiple Rockets (loop until all FireWorks complete)') multi_parser = subparsers.add_parser('multi', help='launches multiple Rockets simultaneously') single_parser.add_argument('-f', '--fw_id', help='specific fw_id to run', default=None, type=int) single_parser.add_argument('--offline', help='run in offline mode (FW.json required)', action='store_true') rapid_parser.add_argument('--nlaunches', help='num_launches (int or "infinite"; ' 'default 0 is all jobs in DB)', default=0) rapid_parser.add_argument('--timeout', help='timeout (secs) after which to quit (default None)', default=None, type=int) rapid_parser.add_argument('--max_loops', help='after this many sleep loops, quit even in ' 'infinite nlaunches mode (default -1 is infinite loops)', default=-1, type=int) rapid_parser.add_argument('--sleep', help='sleep time between loops (secs)', default=None, type=int) multi_parser.add_argument('num_jobs', help='the number of jobs to run in parallel', type=int) multi_parser.add_argument('--nlaunches', help='number of FireWorks to run in series per ' 'parallel job (int or "infinite"; default 0 is ' 'all jobs in DB)', default=0) multi_parser.add_argument('--sleep', help='sleep time between loops in infinite launch mode' '(secs)', default=None, type=int) multi_parser.add_argument('--timeout', help='timeout (secs) after which to quit (default None)', default=None, type=int) multi_parser.add_argument('--nodefile', help='nodefile name or environment variable name ' 'containing the node file name (for populating' ' FWData only)', default=None, type=str) multi_parser.add_argument('--ppn', help='processors per node (for populating FWData only)', default=1, type=int) multi_parser.add_argument('--exclude_current_node', help="Don't use the script launching node" "as compute node", action="store_true") parser.add_argument('-l', '--launchpad_file', help='path to launchpad file', default=LAUNCHPAD_LOC) parser.add_argument('-w', '--fworker_file', help='path to fworker file', default=FWORKER_LOC) parser.add_argument('-c', '--config_dir', help='path to a directory containing the config file ' '(used if -l, -w unspecified)', default=CONFIG_FILE_DIR) parser.add_argument('--loglvl', help='level to print log messages', default='INFO') parser.add_argument('-s', '--silencer', help='shortcut to mute log messages', action='store_true') args = parser.parse_args() signal.signal(signal.SIGINT, handle_interrupt) # graceful exit on ^C if not args.launchpad_file and os.path.exists(os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') if not args.fworker_file and os.path.exists(os.path.join(args.config_dir, 'my_fworker.yaml')): args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml') args.loglvl = 'CRITICAL' if args.silencer else args.loglvl if args.command == 'singleshot' and args.offline: launchpad = None else: launchpad = LaunchPad.from_file(args.launchpad_file) if args.launchpad_file else LaunchPad( strm_lvl=args.loglvl) if args.fworker_file: fworker = FWorker.from_file(args.fworker_file) else: fworker = FWorker() # prime addr lookups _log = get_fw_logger("rlaunch", stream_level="INFO") _log.info("Hostname/IP lookup (this will take a few seconds)") get_my_host() get_my_ip() if args.command == 'rapidfire': rapidfire(launchpad, fworker=fworker, m_dir=None, nlaunches=args.nlaunches, max_loops=args.max_loops, sleep_time=args.sleep, strm_lvl=args.loglvl, timeout=args.timeout) elif args.command == 'multi': total_node_list = None if args.nodefile: if args.nodefile in os.environ: args.nodefile = os.environ[args.nodefile] with open(args.nodefile, 'r') as f: total_node_list = [line.strip() for line in f.readlines()] launch_multiprocess(launchpad, fworker, args.loglvl, args.nlaunches, args.num_jobs, args.sleep, total_node_list, args.ppn, timeout=args.timeout, exclude_current_node=args.exclude_current_node) else: launch_rocket(launchpad, fworker, args.fw_id, args.loglvl)
def launch_workflow( self, launchpad_dir="", k_product=50, job=None, user_incar_settings=None, potcar_functional="PBE", additional_handlers=[], ): """ Creates a list of Fireworks. Each Firework represents calculations that will be done on a slab system of a compound in a specific orientation. Each Firework contains a oriented unit cell relaxation job and a WriteSlabVaspInputs which creates os. Firework(s) depending on whether or not Termination=True. Vasp outputs from all slab and oriented unit cell calculations will then be inserted into a database. Args: launchpad_dir (str path): The path to my_launchpad.yaml. Defaults to the current working directory containing your runs k_product: kpts[0][0]*a. Decide k density without kpoint0, default to 50 cwd: (str path): The curent working directory. Location of where you want your vasp outputs to be. job (VaspJob): The command (cmd) entered into VaspJob object. Default is specifically set for running vasp jobs on Carver at NERSC (use aprun for Hopper or Edison). user_incar_settings(dict): A dict specifying additional incar settings, default to None (ediff_per_atom=False) potcar_functional (str): default to PBE """ launchpad = LaunchPad.from_file(os.path.join(os.environ["HOME"], launchpad_dir, "my_launchpad.yaml")) if self.reset: launchpad.reset("", require_password=False) # Scratch directory reffered to by custodian. # May be different on non-Nersc systems. if not job: job = VaspJob(["mpirun", "-n", "64", "vasp"], auto_npar=False, copy_magmom=True) handlers = [ VaspErrorHandler(), NonConvergingErrorHandler(), UnconvergedErrorHandler(), PotimErrorHandler(), PositiveEnergyErrorHandler(), FrozenJobErrorHandler(timeout=3600), ] if additional_handlers: handlers.extend(additional_handlers) cust_params = { "custodian_params": {"scratch_dir": os.path.join("/global/scratch2/sd/", os.environ["USER"])}, "jobs": job.double_relaxation_run(job.vasp_cmd, auto_npar=False), "handlers": handlers, "max_errors": 100, } # will return a list of jobs # instead of just being one job fws = [] for key in self.miller_dict.keys(): # Enumerate through all compounds in the dictionary, # the key is the compositional formula of the compound print key for miller_index in self.miller_dict[key]: # Enumerates through all miller indices we # want to create slabs of that compound from print str(miller_index) max_norm = max(miller_index) if self.max_normal_search else None # Whether or not we want to use the # max_normal_search algorithm from surface.py print "true or false max norm is ", max_norm, self.max_normal_search slab = SlabGenerator( self.unit_cells_dict[key][0], miller_index, self.ssize, self.vsize, max_normal_search=max_norm ) oriented_uc = slab.oriented_unit_cell if self.fail_safe and len(oriented_uc) > 199: break # This method only creates the oriented unit cell, the # slabs are created in the WriteSlabVaspInputs task. # WriteSlabVaspInputs will create the slabs from # the contcar of the oriented unit cell calculation handler = [] tasks = [] folderbulk = "/%s_%s_k%s_s%sv%s_%s%s%s" % ( oriented_uc.composition.reduced_formula, "bulk", k_product, self.ssize, self.vsize, str(miller_index[0]), str(miller_index[1]), str(miller_index[2]), ) cwd = os.getcwd() if self.get_bulk_e: tasks.extend( [ WriteUCVaspInputs( oriented_ucell=oriented_uc, folder=folderbulk, cwd=cwd, user_incar_settings=user_incar_settings, potcar_functional=potcar_functional, k_product=k_product, ), RunCustodianTask(dir=folderbulk, cwd=cwd, **cust_params), VaspSlabDBInsertTask( struct_type="oriented_unit_cell", loc=folderbulk, cwd=cwd, miller_index=miller_index, **self.vaspdbinsert_params ), ] ) # Slab will inherit average final magnetic moment # of the bulk from outcar, will have to generalize # this for systems with different elements later # element = oriented_uc.species[0] # out = Outcar(cwd+folderbulk) # out_mag = out.magnetization # tot_mag = [mag['tot'] for mag in out_mag] # magmom = np.mean(tot_mag) # user_incar_settings['MAGMOM'] = {element: magmom} tasks.append( WriteSlabVaspInputs( folder=folderbulk, cwd=cwd, user_incar_settings=user_incar_settings, terminations=self.terminations, custodian_params=cust_params, vaspdbinsert_parameters=self.vaspdbinsert_params, potcar_functional=potcar_functional, k_product=k_product, miller_index=miller_index, min_slab_size=self.ssize, min_vacuum_size=self.vsize, ucell=self.unit_cells_dict[key][0], ) ) fw = Firework(tasks, name=folderbulk) fws.append(fw) wf = Workflow(fws, name="Surface Calculations") launchpad.add_wf(wf)
def add_to_db(self): from fireworks.core.launchpad import LaunchPad launchpad_file = os.path.join(os.environ['FW_CONFIG_DIR'], 'my_launchpad.yaml') lp = LaunchPad.from_file(launchpad_file) lp.add_wf(self.wf)
import os from pymatgen.core import Structure from custodian.vasp.jobs import VaspJob from fireworks.core.firework import Firework, Workflow from fireworks.core.launchpad import LaunchPad from bs_wflows.bs_task import MPRelaxationVASPInputTask, MPStaticVASPInputTask, MPNonSCFVASPInputTask, \ RunCustodianTask, TransferResultsTask __author__ = 'Zhenbin Wang' __date__ = "3/20/17" __email__ = "z9wang at ucsd.edu" MODULE_DIR = os.path.dirname(os.path.abspath(__file__)) launchpad = LaunchPad.from_file( os.path.join(os.environ["HOME"], ".fireworks", "my_launchpad.yaml")) struct_file = os.path.join(MODULE_DIR, "test_files/ICSD_182730_Si.cif") material_id = "material_id" s = Structure.from_file(struct_file) vasp_jobs = VaspJob( ["srun", "-n", "32", "-c", "16", "--cpu_bind=cores", "vasp_std"], auto_npar=False) double_relaxations = VaspJob.double_relaxation_run(vasp_cmd=vasp_jobs, auto_npar=False) scratch_dir = "/global/cscratch1/sd/{}/temp_project".format(os.environ["USER"]) fw1 = Firework([ MPRelaxationVASPInputTask(structure=s.as_dict()),
def setup(cls): module_dir = os.path.dirname(__file__) cls.lp = LaunchPad.from_file(os.path.join(module_dir, 'my_launchpad.yaml'))
rapid_parser.add_argument('--nlaunches', help='num_launches (int or "infinite")') rapid_parser.add_argument('--sleep', help='sleep time between loops', default=60, type=int) args = parser.parse_args() if not args.launchpad_file and os.path.exists('my_launchpad.yaml'): args.launchpad_file = 'my_launchpad.yaml' if not args.fworker_file and os.path.exists('my_fworker.yaml'): args.fworker_file = 'my_fworker.yaml' launchpad = LaunchPad.from_file( args.launchpad_file) if args.launchpad_file else None fworker = FWorker.from_file( args.fworker_file) if args.fworker_file else FWorker() rocket_params = QueueParams.from_file(args.queue_params_file) args.loglvl = 'CRITICAL' if args.silencer else args.loglvl # TODO: the number of arguments here is crazy! if args.command == 'rapidfire': rapidfire(rocket_params, args.launch_dir, args.njobs_queue, args.njobs_block, args.loglvl, args.nlaunches, args.sleep, launchpad, fworker, args.reserve) else: launch_rocket_to_queue(rocket_params, args.launch_dir, args.loglvl, launchpad, fworker, args.reserve)
from pymacy.fireworks.md_workflows import MDWorkflowManager from pymatgen.transformations.standard_transformations import SupercellTransformation from pymatgen.transformations.site_transformations import RemoveSitesTransformation from fireworks.core.launchpad import LaunchPad import os wfm = MDWorkflowManager(49, category="tscc-condo") wf = wfm.add_temperature([1], [900]) lpad = LaunchPad.from_file(os.path.join(os.environ["HOME"], ".fireworks", "my_launchpad.yaml")) lpad.add_wf(wf)
import os, json from pymongo import DESCENDING, ASCENDING from fireworks.fw_config import CONFIG_FILE_DIR, SORT_FWS from fireworks.core.fworker import FWorker from fireworks.core.launchpad import LaunchPad from pymongo import ReturnDocument launchpad = LaunchPad.from_file( os.path.join(CONFIG_FILE_DIR, 'my_launchpad.yaml')) fworker = FWorker.from_file(os.path.join(CONFIG_FILE_DIR, 'my_fworker.yaml')) #print launchpad._get_a_fw_to_run(query=fworker.query, checkout=False) m_query = dict(fworker.query) m_query['state'] = 'READY' sortby = [("spec._priority", DESCENDING)] if SORT_FWS.upper() == "FIFO": sortby.append(("created_on", ASCENDING)) elif SORT_FWS.upper() == "FILO": sortby.append(("created_on", DESCENDING)) #print json.dumps(m_query, indent=4) projection = { '_id': 0, 'fw_id': 1, 'spec._fworker': 1, 'spec.task_type': 1, 'spec._queueadapter': 1, 'spec.mpsnl.about.remarks': 1, 'spec.snl.about.remarks': 1, 'spec.prev_vasp_dir': 1, 'updated_on': 1, 'state': 1 }
import os, json from pymongo import DESCENDING, ASCENDING from fireworks.fw_config import CONFIG_FILE_DIR, SORT_FWS from fireworks.core.fworker import FWorker from fireworks.core.launchpad import LaunchPad from pymongo import ReturnDocument launchpad = LaunchPad.from_file(os.path.join(CONFIG_FILE_DIR, 'my_launchpad.yaml')) fworker = FWorker.from_file(os.path.join(CONFIG_FILE_DIR, 'my_fworker.yaml')) #print launchpad._get_a_fw_to_run(query=fworker.query, checkout=False) m_query = dict(fworker.query) m_query['state'] = 'READY' sortby = [("spec._priority", DESCENDING)] if SORT_FWS.upper() == "FIFO": sortby.append(("created_on", ASCENDING)) elif SORT_FWS.upper() == "FILO": sortby.append(("created_on", DESCENDING)) #print json.dumps(m_query, indent=4) projection = { '_id': 0, 'fw_id': 1, 'spec._fworker': 1, 'spec.task_type': 1, 'spec._queueadapter': 1, 'spec.mpsnl.about.remarks': 1, 'spec.snl.about.remarks': 1, 'spec.prev_vasp_dir': 1, 'updated_on': 1, 'state': 1 } fw_ids = [] for idoc, doc in enumerate(launchpad.fireworks.find(m_query, projection=projection, sort=sortby).limit(100)): #print doc if 'walltime' in doc['spec']['_queueadapter']: walltime = doc['spec']['_queueadapter']['walltime'] if int(walltime.split(':')[0]) > 48: launchpad.fireworks.find_one_and_update(
def add_to_db(self): from fireworks.core.launchpad import LaunchPad launchpad_file = os.path.join(os.environ['FW_CONFIG_DIR'], 'my_launchpad.yaml') lp = LaunchPad.from_file(launchpad_file) lp.add_wf(self.wf)
def mlaunch(): m_description = 'This program launches multiple Rockets simultaneously' parser = ArgumentParser(description=m_description) parser.add_argument('num_jobs', help='the number of jobs to run in parallel', type=int) parser.add_argument('--nlaunches', help='number of FireWorks to run in series per parallel job ' '(int or "infinite"; default 0 is all jobs in DB)', default=0) parser.add_argument('--sleep', help='sleep time between loops in infinite launch mode (secs)', default=None, type=int) parser.add_argument('--timeout', help='timeout (secs) after which to quit (default None)', default=None, type=int) parser.add_argument('-l', '--launchpad_file', help='path to launchpad file', default=LAUNCHPAD_LOC) parser.add_argument('-w', '--fworker_file', help='path to fworker file', default=FWORKER_LOC) parser.add_argument('-c', '--config_dir', help='path to a directory containing the config file ' '(used if -l, -w unspecified)', default=CONFIG_FILE_DIR) parser.add_argument('--loglvl', help='level to print log messages', default='INFO') parser.add_argument('-s', '--silencer', help='shortcut to mute log messages', action='store_true') parser.add_argument('--nodefile', help='nodefile name or environment variable name containing ' 'the node file name (for populating FWData only)', default=None, type=str) parser.add_argument('--ppn', help='processors per node (for populating FWData only)', default=1, type=int) parser.add_argument('--exclude_current_node', help="Don't use the script launching node as compute node", action="store_true") try: import argcomplete argcomplete.autocomplete(parser) # This supports bash autocompletion. To enable this, pip install # argcomplete, activate global completion, or add # eval "$(register-python-argcomplete mlaunch)" # into your .bash_profile or .bashrc except ImportError: pass args = parser.parse_args() if not args.launchpad_file and args.config_dir and os.path.exists(os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') if not args.fworker_file and args.config_dir and os.path.exists(os.path.join(args.config_dir, 'my_fworker.yaml')): args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml') args.loglvl = 'CRITICAL' if args.silencer else args.loglvl launchpad = LaunchPad.from_file(args.launchpad_file) if args.launchpad_file else LaunchPad(strm_lvl=args.loglvl) if args.fworker_file: fworker = FWorker.from_file(args.fworker_file) else: fworker = FWorker() total_node_list = None if args.nodefile: if args.nodefile in os.environ: args.nodefile = os.environ[args.nodefile] with open(args.nodefile, 'r') as f: total_node_list = [line.strip() for line in f.readlines()] launch_multiprocess(launchpad, fworker, args.loglvl, args.nlaunches, args.num_jobs, args.sleep, total_node_list, args.ppn, timeout=args.timeout, exclude_current_node=args.exclude_current_node)
x = tdb.tasks.find_one({"fw_id": fw_id}, {"analysis": 1}) warnings = x['analysis'].get('warnings', []) warnings.extend(x['analysis']['errors_MP']['signals']) errors = x['analysis'].get('errors', []) errors.extend(x['analysis']['errors_MP']['critical_signals']) warnings = set(warnings) errors = set(errors) warnings = warnings.difference(errors) return set(warnings), set(errors) if __name__ == '__main__': module_dir = os.path.dirname(os.path.abspath(__file__)) lp_f = os.path.join(module_dir, 'my_launchpad.yaml') lpdb = LaunchPad.from_file(lp_f) tasks_f = os.path.join(module_dir, 'tasks_read.yaml') creds = {} with open(tasks_f) as f: creds = yaml.load(f) connection = MongoClient(creds['host'], creds['port']) tdb = connection[creds['db']] tdb.authenticate(creds['username'], creds['password']) except_dict = defaultdict(int) fizzled_fws = [] for f in lpdb.fireworks.find({"state": "FIZZLED"}, {"fw_id": 1}): fizzled_fws.append(f['fw_id'])
def mlaunch(): m_description = 'This program launches multiple Rockets simultaneously' parser = ArgumentParser(description=m_description) parser.add_argument('num_jobs', help='the number of jobs to run in parallel', type=int) parser.add_argument( '--nlaunches', help='number of FireWorks to run in series per parallel job ' '(int or "infinite"; default 0 is all jobs in DB)', default=0) parser.add_argument( '--sleep', help='sleep time between loops in infinite launch mode (secs)', default=None, type=int) parser.add_argument( '--timeout', help='timeout (secs) after which to quit (default None)', default=None, type=int) parser.add_argument('-l', '--launchpad_file', help='path to launchpad file', default=LAUNCHPAD_LOC) parser.add_argument('-w', '--fworker_file', help='path to fworker file', default=FWORKER_LOC) parser.add_argument('-c', '--config_dir', help='path to a directory containing the config file ' '(used if -l, -w unspecified)', default=CONFIG_FILE_DIR) parser.add_argument('--loglvl', help='level to print log messages', default='INFO') parser.add_argument('-s', '--silencer', help='shortcut to mute log messages', action='store_true') parser.add_argument( '--nodefile', help='nodefile name or environment variable name containing ' 'the node file name (for populating FWData only)', default=None, type=str) parser.add_argument( '--ppn', help='processors per node (for populating FWData only)', default=1, type=int) parser.add_argument( '--exclude_current_node', help="Don't use the script launching node as compute node", action="store_true") try: import argcomplete argcomplete.autocomplete(parser) # This supports bash autocompletion. To enable this, pip install # argcomplete, activate global completion, or add # eval "$(register-python-argcomplete mlaunch)" # into your .bash_profile or .bashrc except ImportError: pass args = parser.parse_args() if not args.launchpad_file and args.config_dir and os.path.exists( os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') if not args.fworker_file and args.config_dir and os.path.exists( os.path.join(args.config_dir, 'my_fworker.yaml')): args.fworker_file = os.path.join(args.config_dir, 'my_fworker.yaml') args.loglvl = 'CRITICAL' if args.silencer else args.loglvl launchpad = LaunchPad.from_file( args.launchpad_file) if args.launchpad_file else LaunchPad( strm_lvl=args.loglvl) if args.fworker_file: fworker = FWorker.from_file(args.fworker_file) else: fworker = FWorker() total_node_list = None if args.nodefile: if args.nodefile in os.environ: args.nodefile = os.environ[args.nodefile] with open(args.nodefile, 'r') as f: total_node_list = [line.strip() for line in f.readlines()] launch_multiprocess(launchpad, fworker, args.loglvl, args.nlaunches, args.num_jobs, args.sleep, total_node_list, args.ppn, timeout=args.timeout, exclude_current_node=args.exclude_current_node)
def rlaunch(): m_description = ( "This program launches one or more Rockets. A Rocket retrieves a job from the " 'central database and runs it. The "single-shot" option launches a single Rocket, ' 'whereas the "rapidfire" option loops until all FireWorks are completed.' ) parser = ArgumentParser(description=m_description) subparsers = parser.add_subparsers(help="command", dest="command") single_parser = subparsers.add_parser("singleshot", help="launch a single Rocket") rapid_parser = subparsers.add_parser( "rapidfire", help="launch multiple Rockets (loop until all FireWorks complete)") multi_parser = subparsers.add_parser( "multi", help="launches multiple Rockets simultaneously") single_parser.add_argument("-f", "--fw_id", help="specific fw_id to run", default=None, type=int) single_parser.add_argument("--offline", help="run in offline mode (FW.json required)", action="store_true") single_parser.add_argument("--pdb", help="shortcut to invoke debugger on error", action="store_true") rapid_parser.add_argument("--nlaunches", help='num_launches (int or "infinite"; ' "default 0 is all jobs in DB)", default=0) rapid_parser.add_argument( "--timeout", help="timeout (secs) after which to quit (default None)", default=None, type=int) rapid_parser.add_argument( "--max_loops", help= "after this many sleep loops, quit even in infinite nlaunches mode (default -1 is infinite loops)", default=-1, type=int, ) rapid_parser.add_argument("--sleep", help="sleep time between loops (secs)", default=None, type=int) rapid_parser.add_argument( "--local_redirect", help="Redirect stdout and stderr to the launch directory", action="store_true") multi_parser.add_argument("num_jobs", help="the number of jobs to run in parallel", type=int) multi_parser.add_argument( "--nlaunches", help="number of FireWorks to run in series per " 'parallel job (int or "infinite"; default 0 is ' "all jobs in DB)", default=0, ) multi_parser.add_argument( "--sleep", help="sleep time between loops in infinite launch mode (secs)", default=None, type=int) multi_parser.add_argument( "--timeout", help="timeout (secs) after which to quit (default None)", default=None, type=int) multi_parser.add_argument( "--nodefile", help="nodefile name or environment variable name " "containing the node file name (for populating" " FWData only)", default=None, type=str, ) multi_parser.add_argument( "--ppn", help="processors per node (for populating FWData only)", default=1, type=int) multi_parser.add_argument( "--exclude_current_node", help="Don't use the script launching node as compute node", action="store_true") multi_parser.add_argument( "--local_redirect", help="Redirect stdout and stderr to the launch directory", action="store_true") parser.add_argument("-l", "--launchpad_file", help="path to launchpad file") parser.add_argument("-w", "--fworker_file", help="path to fworker file") parser.add_argument( "-c", "--config_dir", help= "path to a directory containing the config file (used if -l, -w unspecified)", default=CONFIG_FILE_DIR, ) parser.add_argument("--loglvl", help="level to print log messages", default="INFO") parser.add_argument("-s", "--silencer", help="shortcut to mute log messages", action="store_true") try: import argcomplete argcomplete.autocomplete(parser) # This supports bash autocompletion. To enable this, pip install # argcomplete, activate global completion, or add # eval "$(register-python-argcomplete rlaunch)" # into your .bash_profile or .bashrc except ImportError: pass args = parser.parse_args() signal.signal(signal.SIGINT, handle_interrupt) # graceful exit on ^C if not args.launchpad_file and os.path.exists( os.path.join(args.config_dir, "my_launchpad.yaml")): args.launchpad_file = os.path.join(args.config_dir, "my_launchpad.yaml") elif not args.launchpad_file: args.launchpad_file = LAUNCHPAD_LOC if not args.fworker_file and os.path.exists( os.path.join(args.config_dir, "my_fworker.yaml")): args.fworker_file = os.path.join(args.config_dir, "my_fworker.yaml") elif not args.fworker_file: args.fworker_file = FWORKER_LOC args.loglvl = "CRITICAL" if args.silencer else args.loglvl if args.command == "singleshot" and args.offline: launchpad = None else: launchpad = LaunchPad.from_file( args.launchpad_file) if args.launchpad_file else LaunchPad( strm_lvl=args.loglvl) if args.fworker_file: fworker = FWorker.from_file(args.fworker_file) else: fworker = FWorker() # prime addr lookups _log = get_fw_logger("rlaunch", stream_level="INFO") _log.info("Hostname/IP lookup (this will take a few seconds)") get_my_host() get_my_ip() if args.command == "rapidfire": rapidfire( launchpad, fworker=fworker, m_dir=None, nlaunches=args.nlaunches, max_loops=args.max_loops, sleep_time=args.sleep, strm_lvl=args.loglvl, timeout=args.timeout, local_redirect=args.local_redirect, ) elif args.command == "multi": total_node_list = None if args.nodefile: if args.nodefile in os.environ: args.nodefile = os.environ[args.nodefile] with open(args.nodefile) as f: total_node_list = [line.strip() for line in f.readlines()] launch_multiprocess( launchpad, fworker, args.loglvl, args.nlaunches, args.num_jobs, args.sleep, total_node_list, args.ppn, timeout=args.timeout, exclude_current_node=args.exclude_current_node, local_redirect=args.local_redirect, ) else: launch_rocket(launchpad, fworker, args.fw_id, args.loglvl, pdb_on_exception=args.pdb)
import time, yaml, sys from fireworks.core.launchpad import LaunchPad from fireworks.core.firework import Firework, Workflow from mpworks.firetasks.controller_tasks import AddEStructureTask from fireworks.utilities.fw_utilities import get_slug from mpworks.snl_utils.snl_mongo import SNLMongoAdapter from pymongo import MongoClient from collections import Counter from datetime import datetime from fnmatch import fnmatch # DONE manually: "mp-987" -> fw_id: 119629 lpdb = LaunchPad.from_file('/global/homes/m/matcomp/mp_prod/config/config_Mendel/my_launchpad.yaml') spec = {'task_type': 'Controller: add Electronic Structure v2', '_priority': 100000} sma = SNLMongoAdapter.from_file('/global/homes/m/matcomp/mp_prod/config/dbs/snl_db.yaml') with open('/global/homes/m/matcomp/mp_prod/materials_db_prod.yaml') as f: creds = yaml.load(f) client = MongoClient(creds['host'], creds['port']) db = client[creds['db']] db.authenticate(creds['username'], creds['password']) materials = db['materials'] tasks = db['tasks'] def append_wf(fw_id, parent_fw_id=None): wf = lpdb.workflows.find_one({'nodes':fw_id}, {'parent_links':1,'links':1,'name':1}) try: if parent_fw_id is None: parent_fw_id = wf['parent_links'][str(fw_id)][-1] # non-defused AddEStructureTask v2 already in children? for child_fw_id in wf['links'][str(parent_fw_id)]:
from fireworks.core.launchpad import LaunchPad from fireworks.core.firework import Firework, Workflow from mpworks.firetasks.controller_tasks import AddEStructureTask from fireworks.utilities.fw_utilities import get_slug from mpworks.snl_utils.snl_mongo import SNLMongoAdapter from pymongo import MongoClient from collections import Counter from datetime import datetime from fnmatch import fnmatch from custodian.vasp.handlers import VaspErrorHandler cwd = os.getcwd() # DONE manually: "mp-987" -> fw_id: 119629 lpdb = LaunchPad.from_file( '/global/homes/m/matcomp/mp_prod/config/config_Mendel/my_launchpad.yaml') spec = { 'task_type': 'Controller: add Electronic Structure v2', '_priority': 100000 } sma = SNLMongoAdapter.from_file( '/global/homes/m/matcomp/mp_prod/config/dbs/snl_db.yaml') with open('/global/homes/m/matcomp/mp_prod/materials_db_prod.yaml') as f: creds = yaml.load(f) client = MongoClient(creds['host'], creds['port']) db = client[creds['db']] db.authenticate(creds['username'], creds['password']) materials = db['materials'] tasks = db['tasks'] print materials.count()
def setup(cls): module_dir = os.path.dirname(__file__) cls.lp = LaunchPad.from_file( os.path.join(module_dir, 'my_launchpad.yaml'))
def arlaunch(): """ Function rapid-fire job launching """ m_description = 'This program launches one or more Rockets. A Rocket retrieves a job from the ' \ 'central database and runs it. The "single-shot" option launches a single Rocket, ' \ 'whereas the "rapidfire" option loops until all FireWorks are completed.' parser = ArgumentParser(description=m_description) subparsers = parser.add_subparsers(help='command', dest='command') single_parser = subparsers.add_parser('singleshot', help='launch a single Rocket') rapid_parser = subparsers.add_parser( 'rapidfire', help='launch multiple Rockets (loop until all FireWorks complete)') multi_parser = subparsers.add_parser( 'multi', help='launches multiple Rockets simultaneously') single_parser.add_argument('-f', '--fw_id', help='specific fw_id to run', default=None, type=int) single_parser.add_argument('--offline', help='run in offline mode (FW.json required)', action='store_true') single_parser.add_argument('--pdb', help='shortcut to invoke debugger on error', action='store_true') rapid_parser.add_argument('--nlaunches', help='num_launches (int or "infinite"; ' 'default 0 is all jobs in DB)', default=0) rapid_parser.add_argument( '--timeout', help='timeout (secs) after which to quit (default None)', default=None, type=int) rapid_parser.add_argument( '--max_loops', help='after this many sleep loops, quit even in ' 'infinite nlaunches mode (default -1 is infinite loops)', default=-1, type=int) rapid_parser.add_argument('--sleep', help='sleep time between loops (secs)', default=None, type=int) rapid_parser.add_argument( '--local_redirect', help="Redirect stdout and stderr to the launch directory", action="store_true") multi_parser.add_argument('num_jobs', help='the number of jobs to run in parallel', type=int) multi_parser.add_argument('--nlaunches', help='number of FireWorks to run in series per ' 'parallel job (int or "infinite"; default 0 is ' 'all jobs in DB)', default=0) multi_parser.add_argument( '--sleep', help='sleep time between loops in infinite launch mode' '(secs)', default=None, type=int) multi_parser.add_argument( '--timeout', help='timeout (secs) after which to quit (default None)', default=None, type=int) multi_parser.add_argument( '--nodefile', help='nodefile name or environment variable name ' 'containing the node file name (for populating' ' FWData only)', default=None, type=str) multi_parser.add_argument( '--ppn', help='processors per node (for populating FWData only)', default=1, type=int) multi_parser.add_argument('--exclude_current_node', help="Don't use the script launching node" "as compute node", action="store_true") multi_parser.add_argument( '--local_redirect', help="Redirect stdout and stderr to the launch directory", action="store_true") parser.add_argument('-l', '--launchpad_file', help='path to launchpad file') parser.add_argument('-w', '--fworker_file', required=True, help='path to fworker file') parser.add_argument('-c', '--config_dir', help='path to a directory containing the config file ' '(used if -l, -w unspecified)', default=CONFIG_FILE_DIR) parser.add_argument('--loglvl', help='level to print log messages', default='INFO') parser.add_argument('-s', '--silencer', help='shortcut to mute log messages', action='store_true') try: import argcomplete argcomplete.autocomplete(parser) # This supports bash autocompletion. To enable this, pip install # argcomplete, activate global completion, or add # eval "$(register-python-argcomplete rlaunch)" # into your .bash_profile or .bashrc except ImportError: pass args = parser.parse_args() signal.signal(signal.SIGINT, handle_interrupt) # graceful exit on ^C if not args.launchpad_file and os.path.exists( os.path.join(args.config_dir, 'my_launchpad.yaml')): args.launchpad_file = os.path.join(args.config_dir, 'my_launchpad.yaml') elif not args.launchpad_file: args.launchpad_file = LAUNCHPAD_LOC args.loglvl = 'CRITICAL' if args.silencer else args.loglvl if args.command == 'singleshot' and args.offline: launchpad = None else: launchpad = LaunchPad.from_file( args.launchpad_file) if args.launchpad_file else LaunchPad( strm_lvl=args.loglvl) fworker = AiiDAFWorker.from_file(args.fworker_file) # prime addr lookups _log = get_fw_logger("rlaunch", stream_level="INFO") _log.info("Hostname/IP lookup (this will take a few seconds)") get_my_host() get_my_ip() if args.command == 'rapidfire': rapidfire(launchpad, fworker=fworker, m_dir=None, nlaunches=args.nlaunches, max_loops=args.max_loops, sleep_time=args.sleep, strm_lvl=args.loglvl, timeout=args.timeout, local_redirect=args.local_redirect) elif args.command == 'multi': total_node_list = None if args.nodefile: if args.nodefile in os.environ: args.nodefile = os.environ[args.nodefile] with open(args.nodefile, 'r') as fhandle: total_node_list = [ line.strip() for line in fhandle.readlines() ] launch_multiprocess(launchpad, fworker, args.loglvl, args.nlaunches, args.num_jobs, args.sleep, total_node_list, args.ppn, timeout=args.timeout, exclude_current_node=args.exclude_current_node, local_redirect=args.local_redirect) else: launch_rocket(launchpad, fworker, args.fw_id, args.loglvl, pdb_on_exception=args.pdb)
def launch_workflow(self, launchpad_dir="", k_product=50, job=None, user_incar_settings=None, potcar_functional='PBE', additional_handlers=[]): """ Creates a list of Fireworks. Each Firework represents calculations that will be done on a slab system of a compound in a specific orientation. Each Firework contains a oriented unit cell relaxation job and a WriteSlabVaspInputs which creates os. Firework(s) depending on whether or not Termination=True. Vasp outputs from all slab and oriented unit cell calculations will then be inserted into a database. Args: launchpad_dir (str path): The path to my_launchpad.yaml. Defaults to the current working directory containing your runs k_product: kpts[0][0]*a. Decide k density without kpoint0, default to 50 cwd: (str path): The curent working directory. Location of where you want your vasp outputs to be. job (VaspJob): The command (cmd) entered into VaspJob object. Default is specifically set for running vasp jobs on Carver at NERSC (use aprun for Hopper or Edison). user_incar_settings(dict): A dict specifying additional incar settings, default to None (ediff_per_atom=False) potcar_functional (str): default to PBE """ launchpad = LaunchPad.from_file( os.path.join(os.environ["HOME"], launchpad_dir, "my_launchpad.yaml")) if self.reset: launchpad.reset('', require_password=False) # Scratch directory reffered to by custodian. # May be different on non-Nersc systems. if not job: job = VaspJob(["mpirun", "-n", "64", "vasp"], auto_npar=False, copy_magmom=True) handlers = [ VaspErrorHandler(), NonConvergingErrorHandler(), UnconvergedErrorHandler(), PotimErrorHandler(), PositiveEnergyErrorHandler(), FrozenJobErrorHandler(timeout=3600) ] if additional_handlers: handlers.extend(additional_handlers) cust_params = { "custodian_params": { "scratch_dir": os.path.join("/global/scratch2/sd/", os.environ["USER"]) }, "jobs": job.double_relaxation_run(job.vasp_cmd, auto_npar=False), "handlers": handlers, "max_errors": 100 } # will return a list of jobs # instead of just being one job fws = [] for key in self.miller_dict.keys(): # Enumerate through all compounds in the dictionary, # the key is the compositional formula of the compound print key for miller_index in self.miller_dict[key]: # Enumerates through all miller indices we # want to create slabs of that compound from print str(miller_index) max_norm = max( miller_index) if self.max_normal_search else None # Whether or not we want to use the # max_normal_search algorithm from surface.py print 'true or false max norm is ', max_norm, self.max_normal_search slab = SlabGenerator(self.unit_cells_dict[key][0], miller_index, self.ssize, self.vsize, max_normal_search=max_norm) oriented_uc = slab.oriented_unit_cell if self.fail_safe and len(oriented_uc) > 199: break # This method only creates the oriented unit cell, the # slabs are created in the WriteSlabVaspInputs task. # WriteSlabVaspInputs will create the slabs from # the contcar of the oriented unit cell calculation handler = [] tasks = [] folderbulk = '/%s_%s_k%s_s%sv%s_%s%s%s' % ( oriented_uc.composition.reduced_formula, 'bulk', k_product, self.ssize, self.vsize, str(miller_index[0]), str(miller_index[1]), str(miller_index[2])) cwd = os.getcwd() if self.get_bulk_e: tasks.extend([ WriteUCVaspInputs( oriented_ucell=oriented_uc, folder=folderbulk, cwd=cwd, user_incar_settings=user_incar_settings, potcar_functional=potcar_functional, k_product=k_product), RunCustodianTask(dir=folderbulk, cwd=cwd, **cust_params), VaspSlabDBInsertTask(struct_type="oriented_unit_cell", loc=folderbulk, cwd=cwd, miller_index=miller_index, **self.vaspdbinsert_params) ]) # Slab will inherit average final magnetic moment # of the bulk from outcar, will have to generalize # this for systems with different elements later # element = oriented_uc.species[0] # out = Outcar(cwd+folderbulk) # out_mag = out.magnetization # tot_mag = [mag['tot'] for mag in out_mag] # magmom = np.mean(tot_mag) # user_incar_settings['MAGMOM'] = {element: magmom} tasks.append( WriteSlabVaspInputs( folder=folderbulk, cwd=cwd, user_incar_settings=user_incar_settings, terminations=self.terminations, custodian_params=cust_params, vaspdbinsert_parameters=self.vaspdbinsert_params, potcar_functional=potcar_functional, k_product=k_product, miller_index=miller_index, min_slab_size=self.ssize, min_vacuum_size=self.vsize, ucell=self.unit_cells_dict[key][0])) fw = Firework(tasks, name=folderbulk) fws.append(fw) wf = Workflow(fws, name='Surface Calculations') launchpad.add_wf(wf)
def fix(): # initialize databases module_dir = os.path.dirname(os.path.abspath(__file__)) snl_f = os.path.join(module_dir, 'snl.yaml') snldb = SNLMongoAdapter.from_file(snl_f) snl = snldb.snl snlgroups = snldb.snlgroups tasks_f = os.path.join(module_dir, 'tasks.yaml') with open(tasks_f) as f2: task_creds = yaml.load(f2) mc = MongoClient(task_creds['host'], task_creds['port']) db = mc[task_creds['database']] db.authenticate(task_creds['admin_user'], task_creds['admin_password']) tasks = db['tasks'] tasks_f = os.path.join(module_dir, 'tasks.yaml') with open(tasks_f) as f2: task_creds = yaml.load(f2) mc = MongoClient(task_creds['host'], task_creds['port']) db = mc[task_creds['database']] db.authenticate(task_creds['admin_user'], task_creds['admin_password']) tasks = db['tasks'] lp_f = os.path.join(module_dir, 'my_launchpad.yaml') lpdb = LaunchPad.from_file(lp_f) fws = lpdb.fireworks launches = lpdb.launches sb_f = os.path.join(module_dir, 'submission.yaml') sbdb = SubmissionMongoAdapter.from_file(sb_f) submissions = sbdb.jobs bad_crystal_ids = [] crystals_file = os.path.join(module_dir, 'bad_crystals.txt') with open(crystals_file) as f: for line in f: bad_crystal_ids.append(int(line.strip())) for c_id in bad_crystal_ids: if c_id == 100892 or c_id == 100202: print 'SKIP' else: # FIX SNL for s in snl.find({'about._materialsproject.deprecated.crystal_id_deprecated': c_id}, {'snl_id': 1}): snl.update({'snl_id': s['snl_id']}, {'$pushAll': {"about.remarks": ['DEPRECATED', 'SEVERE BUG IN ICSD CONVERSION']}}) # FIX SNLGROUPS for s in snlgroups.find({'canonical_snl.about._materialsproject.deprecated.crystal_id_deprecated': c_id}, {'snlgroup_id': 1}): snlgroups.update({'snlgroup_id': s['snlgroup_id']}, {'$pushAll': {"canonical_snl.about.remarks": ['DEPRECATED', 'SEVERE BUG IN ICSD CONVERSION']}}) # FIX FWs pt 1 for s in fws.find({'spec.mpsnl.about._materialsproject.deprecated.crystal_id_deprecated': c_id}, {'fw_id': 1}): fws.update({'fw_id': s['fw_id']}, {'$pushAll': {"spec.mpsnl.about.remarks": ['DEPRECATED', 'SEVERE BUG IN ICSD CONVERSION']}}) # FIX FWs pt 2 for s in fws.find({'spec.force_mpsnl.about._materialsproject.deprecated.crystal_id_deprecated': c_id}, {'fw_id': 1}): fws.update({'fw_id': s['fw_id']}, {'$pushAll': {"spec.force_mpsnl.about.remarks": ['DEPRECATED', 'SEVERE BUG IN ICSD CONVERSION']}}) # FIX Launches for s in launches.find({'action.update_spec.mpsnl.about._materialsproject.deprecated.crystal_id_deprecated': c_id}, {'launch_id': 1}): launches.update({'launch_id': s['launch_id']}, {'$pushAll': {"action.update_spec.mpsnl.about.remarks": ['DEPRECATED', 'SEVERE BUG IN ICSD CONVERSION']}}) # FIX TASKS for s in tasks.find({'snl.about._materialsproject.deprecated.crystal_id_deprecated': c_id}, {'task_id': 1}): tasks.update({'task_id': s['task_id']}, {'$pushAll': {"snl.about.remarks": ['DEPRECATED', 'SEVERE BUG IN ICSD CONVERSION']}}) tasks.update({'task_id': s['task_id']}, {'$pushAll': {"snl_final.about.remarks": ['DEPRECATED', 'SEVERE BUG IN ICSD CONVERSION']}}) # FIX SUBMISSIONS for s in submissions.find({'about._materialsproject.deprecated.crystal_id_deprecated': c_id}, {'submission_id': 1}): submissions.update({'submission_id': s['submission_id']}, {'$pushAll': {"about.remarks": ['DEPRECATED', 'SEVERE BUG IN ICSD CONVERSION']}}) print 'FIXED', c_id
parser.add_argument('--logdir', help='path to a directory for logging', default=None) parser.add_argument('--loglvl', help='level to print log messages', default='INFO') parser.add_argument('--silencer', help='shortcut to mute log messages', action='store_true') parser.add_argument('-r', '--reserve', help='reserve a fw', action='store_true') parser.add_argument('-l', '--launchpad_file', help='path to launchpad file', default=None) parser.add_argument('-w', '--fworker_file', help='path to fworker file', default=None) rapid_parser.add_argument('-q', '--njobs_queue', help='maximum jobs to keep in queue for this user', default=10, type=int) rapid_parser.add_argument('-b', '--njobs_block', help='maximum jobs to put in a block', default=500, type=int) rapid_parser.add_argument('--nlaunches', help='num_launches (int or "infinite")') rapid_parser.add_argument('--sleep', help='sleep time between loops', default=60, type=int) args = parser.parse_args() if not args.launchpad_file and os.path.exists('my_launchpad.yaml'): args.launchpad_file = 'my_launchpad.yaml' if not args.fworker_file and os.path.exists('my_fworker.yaml'): args.fworker_file = 'my_fworker.yaml' launchpad = LaunchPad.from_file(args.launchpad_file) if args.launchpad_file else None fworker = FWorker.from_file(args.fworker_file) if args.fworker_file else FWorker() rocket_params = QueueParams.from_file(args.queue_params_file) args.loglvl = 'CRITICAL' if args.silencer else args.loglvl # TODO: the number of arguments here is crazy! if args.command == 'rapidfire': rapidfire(rocket_params, args.launch_dir, args.njobs_queue, args.njobs_block, args.loglvl, args.nlaunches, args.sleep, launchpad, fworker, args.reserve) else: launch_rocket_to_queue(rocket_params, args.launch_dir, args.loglvl, launchpad, fworker, args.reserve)