def start_gmw(self, conn): self.param_dict['gmw'] = os.path.join(os.getenv('HOME'), self.param_dict['user'], self.param_dict['mwd'], self.param_dict['gmw']) print os.environ sys.stdout.flush() args = """--worker_name {worker_name} """+\ """--worker_id {worker_id} """+\ """--dbpass {dbpass} """+\ """--dbname {dbname} --dbhost {dbhost} --dbuser {dbuser} """+\ """--gen_host {gen_host} """ +\ """--prefix {prefix}/{dbname} """+\ """--mwd {mwd} """ +\ """--user {user} """ +\ """--avg_seq_jobs {avg_seq_jobs} """ +\ """--res_config_name {res_config_name} """ # start a screen here start_local_screen = """screen -S {worker_name} -d -m """+\ """env PYTHONPATH={pythonpaths} GMX_MAXBACKUP=-1 GPHOME=/opt/greenplum-db/. """ +\ """PATH={PATH} LD_LIBRARY_PATH={LD_LIBRARY_PATH} SSH_AUTH_SOCK={SSH_AUTH_SOCK} """.format(**os.environ) +\ """/opt/greenplum-db/ext/python/bin/python {gmw} """ + args # ssh into the worker host and start a screen start_remote_screen = """screen -S {worker_name} -d -m ssh {worker_host} """+\ """env PYTHONPATH={pythonpaths} GMX_MAXBACKUP=-1 """ +\ """"/usr/bin/python {gmw} """ + args + """" """ template = start_local_screen if self.param_dict['worker_host'] == 'localhost' else start_remote_screen mddb_utils.run_cmd(template.format(**self.param_dict))
def quit_existing_workers(self): conn = mddb_utils.get_dbconn(self.param_dict['dbname'], self.param_dict['dbuser'], self.param_dict['dbhost'], self.param_dict['dbpass']) cur = conn.cursor() cur.execute("select worker_name from Workers") for rec in cur: gw_name = rec[0] mddb_utils.run_cmd("screen -X -S {0} quit".format(gw_name)) cur.execute("truncate Workers cascade") cur.execute("select setval((select pg_get_serial_sequence('workers', 'worker_id')), 1, false)") conn.commit() cur.close() conn.close()
def run(self, input_params): print input_params in_fn = input_params['simulation_in_fn'] pdb2gmx = input_params['gromacs_pdb2gmx'] grompp = input_params['gromacs_grompp'] mdrun = input_params['gromacs_mdrun'] gro_fn = input_params['gro_file_name'] tpr_fn = input_params['tpr_file_name'] xtc_fn = input_params['xtc_file_name'] trr_fn = input_params['trr_file_name'] pdb_fn = input_params['pdb_file_name'] mdp_fn = input_params['simulation_in_fn'] top_fn = input_params['top1_file_name'] self.reformat_pdb1(pdb_fn) mddb_utils.run_cmd(pdb2gmx + " -f " + pdb_fn + " -o " + gro_fn +\ " -p " + top_fn + "-ignh -ff amber99sb -water none") if not os.path.isfile(top_fn): top_fn = input_params['top2_file_name'] mddb_utils.run_cmd(grompp + " -v -f " + mdp_fn + " -c " + gro_fn +\ " -p " + top_fn + " -o " + tpr_fn) tmp_xtc_fn = "/tmp" + xtc_fn; tmp_trr_fn = "/tmp" + trr_fn; try: os.makedirs(os.path.dirname(tmp_xtc_fn)) except: pass log_fn = input_params['simulation_in_fn'] + '.log' sim_cmd = mdrun + " -s " + tpr_fn + " -x " + tmp_xtc_fn + " -o " + tmp_trr_fn + " > " + log_fn mddb_utils.run_cmd(sim_cmd) mddb_utils.run_cmd("cp {0} {1}".format(tmp_xtc_fn, xtc_fn)) mddb_utils.run_cmd("cp {0} {1}".format(tmp_trr_fn, trr_fn)) ret_dict = \ { 'trj_fn': trr_fn ,'sim_fn': gro_fn ,'log_fn': log_fn } return ret_dict
def init_database(self, init_files): dbname = self.param_dict['dbname'] dbdir = os.path.join(os.getenv('HOME'), self.param_dict['user'], self.param_dict['mwd'], self.param_dict['dbdir']) p = mddb_utils.run_cmd("dropdb {0}".format(dbname)) p = mddb_utils.run_cmd("createdb {0}".format(dbname)) madpack = '/usr/local/madlib/bin/madpack' #mddb_utils.run_sql("alter role {0} with password '12345'".format(self.param_dict['dbuser']), dbname) #mddb_utils.run_cmd("{0} -p greenplum -c {1}/12345@{2}/{3} install".format( # madpack, self.param_dict['dbuser'], 'localhost4', dbname)) #mddb_utils.run_sql("alter role {0} with password null".format(self.param_dict['dbuser']), dbname) for f in init_files: mddb_utils.load_sql("{0}/{1}".format(dbdir,f), dbname)
def run_docking_combo_remote(job_dict): fn_dict = prepare_filenames(job_dict) job_json_fn = fn_dict['job_json'] remote_dir = mddb_utils.run_cmd("ssh hhpc 'echo -n $INDIR'") print job_dict qsub.create_lddb_job_json(job_json_fn, job_dict) job_json_fn_remote = remote_dir + "/" + os.path.basename(job_json_fn) qsub_fn = fn_dict['qsub'] qsub.create_lddb_qsub_script(qsub_fn, job_json_fn_remote) qsub_fn_remote = remote_dir + "/" + os.path.basename(qsub_fn) mol_fn = job_dict['mol_fn'] sub_dir = str(job_dict['c_id']) fl = get_amber_file_names(in_dir + "/" + sub_dir, receptor_id, mol_fn) fl = fl + [job_json_fn, qsub_fn] map(lambda x: mddb_utils.run_cmd("scp {0} hhpc:{1}".format(x, remote_dir)), fl) mddb_utils.run_cmd("ssh hhpc qsub {0}".format(qsub_fn_remote)) time.sleep(50) return ''
def start_sampleloop(self): conn = mddb_utils.get_dbconn(self.param_dict['dbname'], self.param_dict['dbuser'], self.param_dict['dbhost'], self.param_dict['dbpass']) prefix = os.path.join(os.getenv('HOME'), self.param_dict['user'], controller.Controller.mwd) pp = controller.Controller.centos_pythonpaths.format(prefix) env = "PYTHONPATH={0} GPHOME=/opt/greenplum-db/. PGHOST=localhost4 PGOPTIONS=--client-min-messages=error ".format(pp) env = env + "PATH={PATH} LD_LIBRARY_PATH={LD_LIBRARY_PATH} ".format(**os.environ) self.param_dict['env'] = env sample_loop_cmd = ("""screen -S {dbname}_{user}_sample_loop -d -m """+\ """env {env}""" +\ """/opt/greenplum-db/ext/python/bin/python """+ os.path.abspath( __file__ ) +\ """ --mode loop """+\ """ --dbname {dbname} """+\ """ --dbpass {dbpass} """+\ """ 2>&1 | tee /tmp/{dbname}_{user}_sampleloop.log >&1 """).format(**self.param_dict); mddb_utils.run_cmd(sample_loop_cmd) mddb_utils.execute_query(conn, "select controlparams_upsert('sampleloop', 'running')") conn.commit() conn.close()
def fetch_results(): st = mddb_utils.run_cmd("ssh hhpc python2.6 nutanong/mddb/scheduler/lddb_sqlite_utils.py --mode getresults") sql_stms = st.split("\n") conn = mddb_utils.get_dbconn(dbname, dbuser, dbhost, dbpass) cur = conn.cursor() for stm in sql_stms: print stm try: cur.execute(stm) except: pass conn.commit() cur.close() conn.close()
def num_finished_remote_jobs(): n = mddb_utils.run_cmd("""ssh hhpc 'sqlite3 /home/yahmad/lddb_jobs/db/docking.db "select count(*) from FinishedJobsView"' """) return int(n)
def quit_everything(self): mddb_utils.run_cmd('screen -S {0}_{1}_sample_loop -X quit'.format(self.param_dict['dbname'], self.param_dict['user'])) super(MDDBController, self).quit_everything()
def reformat_pdb1(self, pdb_fn): mddb_utils.run_cmd("cp {0} {0}.bak".format(pdb_fn)) script_fn = pdb_fn + '.R' self.gen_conversion_script(pdb_fn, pdb_fn + '2', script_fn) mddb_utils.run_cmd("Rscript " + script_fn) mddb_utils.run_cmd("mv {0}2 {0}".format(pdb_fn))