def launch_workers(self): target_jobs, = self.master_db.query_master_fetchone('SELECT max_jobs FROM queues WHERE name = "default"') self.master_db.update_list_ensemble_dbs() if len(self.master_db.active_dbs) == 0: utils.inline_msg("MSG", "No active dbs... sleeping ") return current_count = self.active_threads() to_launch = target_jobs - current_count if to_launch >= 0: utils.newline_msg("STATUS", "[n_jobs=%d] run=%d ::: new=%d" % (target_jobs,current_count,to_launch ) ) else: utils.newline_msg("STATUS", "[n_jobs=%d] run=%d :!: exceed" % (target_jobs,current_count)) for i_t in range(to_launch): self.lock.acquire() pick = self.master_db.pick_ensemble() status = pick.get_updated_status() if status['process_not_run'] == 0: print "+D+ ----- %s " % (pick.full_name) self.master_db.query_master_db('UPDATE dbs SET status= ? WHERE full_name = ?', "D", pick.full_name) return self.lock.release() nt = SPGRunningAtom(pick, self.lock) # nt = SPGRunningAtom(pick, lock=self.get_lock( pick ) ) nt.start()
def get_flags_and_db(self, c, init_db=True): flags, args = self.parse_command_line(c) # if len(args)== 1: # utils.newline_msg("ERR", "a single file was expected or could not parse flags") # return flags, args, None db_name = args[-1] args = args[:-1] if db_name.isdigit(): id = int(db_name) rdb = self.master_db.result_dbs filtered = [x for x in rdb if rdb[x]['id'] == id] if filtered: db_name = filtered[0] else: utils.newline_msg( "ERR", "database with id '%s' doesn't exist." % db_name) return flags, args.append(db_name), None else: full_name, path, base_name, extension = utils.translate_name( db_name) # print "do_init::: ",self.translate_name(i_arg) db_name = "%s/%s.spgql" % (path, base_name) # sim_name = "%s/%s.spg" % (path, base_name) if not os.path.exists(db_name): utils.newline_msg( "ERR", "database with name '%s' doesn't exist." % utils.shorten_name(db_name)) return flags, args.append(db_name), None return flags, args, self.EnsembleConstructor(db_name, init_db)
def dump_result(self): """ loads the next parameter atom from a parameter ensemble""" #:::~ status can be either #:::~ 'N': not run #:::~ 'R': running #:::~ 'D': successfully run (done) #:::~ 'E': run but with non-zero error code if self.return_code != 0: self.query_set_run_status("E") return for line in self.output: table_name, output_columns = self.parse_output_line(line) output_columns = [self.current_spg_uid, self.current_spg_vsid, self.current_spg_rep] + output_columns cc = 'INSERT INTO %s (%s) VALUES (%s) ' % (table_name, ", ".join(self.table_columns[table_name]), ", ".join(["'%s'" % str(i) for i in output_columns])) # print cc # try: self.execute_query(cc) self.query_set_run_status("D", self.current_spg_uid, self.run_time) except sql.OperationalError as e: v = str(e).split() nv, nc = int(v[0]), int(v[3]) utils.newline_msg("DB", "Fatal, '%d' values for the '%d' output columns %s expected " % (nv-3,nc-3, self.table_columns[table_name][3:])) sys.exit(1)
def plot_all_join_outputs(self, Plotter): newline_msg("INF", "%s - %s - %s" % (self.separated_vars, self.coalesced_vars, self.x_axis)) table_name = self.base_name + "_results.csv" ctp = Plotter(table_name) ctp.x_axis = self.x_axis ctp.y_axis = self.output_columns ctp.separated_vars = self.separated_vars ctp.coalesced_vars = self.coalesced_vars ctp.settings = self.settings plot_fname = self.base_name + "_plot" if len(ctp.separated_vars) > 1: plot_fname += "_" + "_".join(ctp.separated_vars) if len(ctp.separated_vars) == 1: plot_fname += "_" + ctp.separated_vars[0] plot_fname += ".pdf" newline_msg("OUT", plot_fname) ctp.plot_all_join_outputs(output_name=plot_fname)
def do_init(self, c): """init [-flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]] Generates a new database out of a parameters.dat FLAGS::: -skip-master: the new results.sqlite is created, but gets not registered in master -purge: deletes the results.sqlite db, if it already existed """ flags,c = self.parse_command_line(c) i_arg = c[0] try: i_arg, db_name = self.translate_name(i_arg) except: utils.newline_msg("ERR", "results db '%s' doesn't exist. Can not init it" ) return if "purge" in flags: try: os.remove(db_name) except: utils.newline_msg("WRN", "db '%s' could not be removed... skipping"%db_name) self.do_deregister(i_arg) if len(c) >1: self.do_set( ":".join( c[1:] ) ) parser = EnsembleCSVBuilder(stream = open(i_arg), db_name=db_name) parser.init_db( )
def __init__(self, arg, command=None): full_name, self.path, self.base_name, extension = utils.translate_name( arg) MultIteratorParser.__init__(self, open(f"{self.base_name}.spg")) if command is not None: self.command = command self.command, ext = os.path.splitext(self.command) if not utils.check_params.consistency(self.command, self): utils.newline_msg("ERR", "simulation configuration is not consistent.") sys.exit(1) self.input_configuration = utils.read_input_configuration( f"{self.command}.input") self.default_parameters = utils.SPGSettings() for k, v in self.input_configuration.items(): if v.var_type != "str": self.default_parameters[k] = eval(f"{v.var_type}({v.default})") else: self.default_parameters[k] = v.default self.stdout_configuration = utils.read_output_configuration( self.command)
def get_flags_and_db(self, c,init_db = True): flags, args = self.parse_command_line(c) # if len(args)== 1: # utils.newline_msg("ERR", "a single file was expected or could not parse flags") # return flags, args, None db_name = args[-1] args = args[:-1] if db_name.isdigit(): id = int(db_name) rdb = self.master_db.result_dbs filtered = [x for x in rdb if rdb[x]['id'] == id] if filtered: db_name = filtered[0] else: utils.newline_msg("ERR", "database with id '%s' doesn't exist." % db_name) return flags, args.append(db_name), None else: full_name, path, base_name, extension = utils.translate_name(db_name) # print "do_init::: ",self.translate_name(i_arg) db_name = "%s/%s.spgql" % (path, base_name) # sim_name = "%s/%s.spg" % (path, base_name) if not os.path.exists(db_name): utils.newline_msg("ERR", "database with name '%s' doesn't exist." % utils.shorten_name(db_name)) return flags, args.append(db_name), None return flags, args, self.EnsembleConstructor(db_name, init_db)
def dump_result(self): """ loads the next parameter atom from a parameter ensemble""" #:::~ status can be either #:::~ 'N': not run #:::~ 'R': running #:::~ 'D': successfully run (done) #:::~ 'E': run but with non-zero error code if self.return_code != 0: self.query_set_run_status("E") return for line in self.output: table_name, output_columns = self.parse_output_line(line) # print( "DUMP", line , table_name, output_columns ) output_columns = [self.current_spg_uid, self.current_spg_vsid, self.current_spg_rep] + output_columns # print(self.table_columns[table_name],["'%s'" % str(i) for i in output_columns]) cc = 'INSERT INTO %s (%s) VALUES (%s) ' % (table_name, ", ".join(self.table_columns[table_name]), ", ".join(["'%s'" % str(i) for i in output_columns])) # print(cc) # # self.execute_query(cc) try: self.execute_query(cc) self.query_set_run_status("D", self.current_spg_uid, self.run_time) except sql.OperationalError as e: v = str(e).split() nv, nc = int(v[0]), int(v[3]) utils.newline_msg("DB", "Fatal, '%d' values for the '%d' output columns %s expected " % (nv-3,nc-3, self.table_columns[table_name][3:])) sys.exit(1)
def plot_all_join_outputs(self, Plotter, mean): newline_msg( "INF", "%s - %s - %s" % (self.separated_vars, self.coalesced_vars, self.x_axis)) table_name = self.base_name + "_results.csv" ctp = Plotter(table_name) ctp.x_axis = self.x_axis ctp.y_axis = self.output_columns ctp.separated_vars = self.separated_vars ctp.coalesced_vars = self.coalesced_vars ctp.settings = self.settings plot_fname = self.base_name + "_plot" if len(ctp.separated_vars) > 1: plot_fname += "_" + "_".join(ctp.separated_vars) if len(ctp.separated_vars) == 1: plot_fname += "_" + ctp.separated_vars[0] plot_fname += ".pdf" newline_msg("OUT", plot_fname) if mean: ctp.mean() ctp.plot_all_join_outputs(output_name=plot_fname)
def __init__(self, db_name, timeout=5): full_name, self.path, self.base_name, extension = utils.translate_name( db_name) self.db_name = "%s/%s.spgql" % (self.path, self.base_name) sim_name = "%s/%s.spg" % (self.path, self.base_name) MultIteratorParser.__init__(self, open(sim_name)) # print self.data # self.path, foo = os.path.split( os.path.abspath(db_name) ) # self.base_name, foo = os.path.splitext( foo ) # print check_params.consistency(self.command, self) if not check_params.consistency(self.command, self): utils.newline_msg("ERR", "simulation configuration is not consistent.") sys.exit(1) all_outputs = check_params.read_output_configuration(self.command) self.stdout_contents = dict() for k in all_outputs: table = k.output_table if table not in self.stdout_contents.leys(): self.stdout_contents[table] = k self.connection = sql.connect(self.db_name, timeout=timeout, check_same_thread=False) self.cursor = self.connection.cursor()
def plot_all(self, Plotter): newline_msg("INF", "%s - %s - %s" % (self.separated_vars, self.coalesced_vars, self.x_axis)) table_name = "%s_%s.csv"%(self.base_name, self.table_name ) ctp = Plotter(table_name) ctp.x_axis = self.x_axis ctp.y_axis = self.output_columns ctp.separated_vars = self.separated_vars ctp.coalesced_vars = self.coalesced_vars ctp.settings = self.settings plot_fname = "%s_%s_plot"%(self.base_name, self.table_name ) if len(ctp.separated_vars) > 1: plot_fname += "_" + "_".join(ctp.separated_vars) if len(ctp.separated_vars) == 1: plot_fname += "_" + ctp.separated_vars[0] plot_fname += ".pdf" newline_msg("OUT", plot_fname) ctp.plot_all(output_name=plot_fname)
def __init__(self, parameter_file): self.parameter_file = parameter_file self.mit = spgb.MultIteratorParser( open(parameter_file) ) self.variables = self.mit.varying_items() self.settings, foo = self.get_settings(self.mit.command, "inputs") settings_output, self.output_columns = self.get_settings(self.mit.command, "stdout") newline_msg( "INF", "output columns: %s"% enumerate(self.output_columns) ) self.settings.update(settings_output) self.x_axis = self.variables[-1] x_axis_iter = self.mit.data[ self.mit.position_of( self.x_axis ) ] if x_axis_iter.type == "*": self.add_setting( self.x_axis, "scale = log") self.add_setting( self.x_axis, "lim = (%s,%s)"%(x_axis_iter.xmin,x_axis_iter.xmax)) try: self.coalesced_vars = [ self.variables[-2] ] except: self.coalesced_vars = [] try: self.separated_vars = self.variables[:-2] except: self.separated_vars = []
def do_load(self,c): """loads a queue as active one""" self.__update_queue_list() if len( filter(lambda x: x[1] == c, self.queues) ): self.current_queue = c else: newline_msg("ERR", "queue '%s' not found"%c) print "loaded '%s'"%self.current_queue
def __init__(self, key, previous, value): self.key = key self.previous = previous self.value = value utils.newline_msg( "ERR", "conflict in %s name (in db '%s', in param '%s')" % (key, previous, value))
def do_setup_output_column(self,c): """sets which columns to generate output from""" if not self.current_param_db: utils.newline_msg("WRN", "current db not set... skipping") return c = c.split(",") if not set(c).issubset( self.current_param_db.output_column ): utils.newline_msg("ERR", "the column(s) is (are) not in the output: %s"%( set(c) - set( self.current_param_db.output_column )) ) self.output_column = c
def do_add(self, c): """adds queue queue""" self.__update_queue_list() if len( filter(lambda x: x[1] == c, self.queues) ): newline_msg("ERR", "queue '%s' already exists"%queue, 2) return self.master_db.execute_query( "INSERT INTO queues (name, max_jobs, status) VALUES (?,?,?)",c, 1, 'S') os.makedirs("%s/queue/%s"%(VAR_PATH,c)) self.current_queue = c
def do_setup_output_column(self,c): """sets which columns to generate output from""" if not self.current_param_db: utils.newline_msg("WRN", "current db not set... it must be loaded first",2) return flags, args = self.parse_command_line(c) cols = args[0].split(",") if not set(cols).issubset( self.current_param_db.table_columns[self.table] ): utils.newline_msg("ERR", "the column(s) is (are) not in the output: %s"%( set(cols) - set( self.current_param_db.table_columns[self.table] )) ) self.table_columns = cols
def __init__(self, stream=None, db_name = "results.sqlite", timeout = 5): MultIteratorParser.__init__(self, stream) # print check_params.consistency(self.command, self) if not check_params.consistency(self.command, self): utils.newline_msg("ERR","parameters.dat file is not consistent.") sys.exit(1) self.stdout_contents = check_params.contents_in_output(self.command) self.connection = sql.connect(db_name, timeout = timeout) self.cursor = self.connection.cursor()
def launch_workers(self): target_jobs, = self.master_db.query_master_fetchone('SELECT max_jobs FROM queues WHERE name = "default"') self.master_db.update_list_ensemble_dbs() if len(self.master_db.active_dbs) == 0: utils.inline_msg("MSG", "No active dbs... sleeping ") return current_count = self.active_threads() # print "+++++++++++", to_launch vec_to_launch = [] launch = defaultdict(lambda: 0) running = {} for ae in self.master_db.active_dbs: ens = self.master_db.result_dbs[ae ] running[ ens['id'] ] = self.active_processes[ ae ] qty_to_launch = int( m.floor(0.5 + target_jobs*ens['weight']/self.master_db.normalising) - self.active_processes[ ae ] ) vec_to_launch += qty_to_launch * [ae] launch[ ens['id'] ] += qty_to_launch # for id in launch: # print "+++ (%d) %d + %d = //%d//, "%( id, launch[id], running[id],launch[id]+running[id] ) # print to_launch = len(vec_to_launch) if to_launch >= 0: utils.newline_msg("STATUS", utils.str_color( "@green[n_jobs=%d] run=%d %s ::: new=%d" % (target_jobs,current_count, dict(running),to_launch) ) ) else: utils.newline_msg("STATUS", utils.str_color( "@yellow[n_jobs=%d] run=%d :!: exceeded number" % (target_jobs,current_count)) ) # print to_launch, len( vec_to_launch ), launch # for i_t in range(to_launch): for ae in vec_to_launch: pick = self.master_db.EnsembleConstructor(ae, init_db=True) self.lock.acquire() # pick = self.master_db.pick_ensemble() pick.test_run = self.test_run status = pick.get_updated_status() if status['process_not_run'] == 0: print "+D+ ----- %s " % (pick.full_name) self.master_db.query_master_db('UPDATE dbs SET status= ? WHERE full_name = ?', "D", pick.full_name) return self.lock.release() nt = SPGRunningAtom(pick, self.lock, self.active_processes) # nt.test_run = self.test_run # nt = SPGRunningAtom(pick, lock=self.get_lock( pick ) ) nt.start()
def setup_separated_output(self, conf): try: i = int(conf) separated = self.variables[:i+1] except: separated = conf.split(",") if set(separated).issubset( set(self.variables) ): self.separated_vars = separated self.coalesced_vars = [ i for i in self.variables if ( i not in self.separated_vars ) and ( i not in self.in_table_vars ) ] else: utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(separated)-set(self.variables) )
def do_run_script(self,c): """executes a script file with commands accepted in this cmdline parser""" if not os.path.exists(c): utils.newline_msg("FIL", "file doesn't exist") return for l in open(c): l = l.strip() if not l: continue if l[0] == "#": continue self.onecmd(l.strip())
def update_ensemble_list(self): self.normalising = 0. self.active_dbs = [] for i in self.result_dbs.keys(): if self.result_dbs[i] == None: del self.result_dbs[i] utils.newline_msg("MSG", "removing db '%s' from the running list"%i) continue if self.result_dbs[i].status == 'R': self.normalising += self.result_dbs[ i ].weight self.active_dbs.append( self.result_dbs[ i ] )
def do_conf(self,c): """prints the current configuration""" if not self.current_param_db: utils.newline_msg("WRN", "current db not set... skipping") return print " -- db: %s"%( self.shorten_name( self.current_param_db.full_name ) ) print " + variables = %s "%( ", ".join(self.current_param_db.variables ) ) print " + entities = %s "%( ", ".join(self.current_param_db.entities ) ) print " + columns = %s "%( ", ".join(self.current_param_db.output_column ) ) print " + split_columns = %s / raw_data = %s / restrict_by_val = %s"%(self.split_columns, self.raw_data, self.restrict_by_val) print " + vars (separated-coalesced-in_table) = %s - %s - %s "%(self.current_param_db.separated_vars, self.current_param_db.coalesced_vars, self.current_param_db.in_table_vars)
def do_run_script(self, c): """executes a script file with commands accepted in this cmdline parser""" if not os.path.exists(c): utils.newline_msg("FIL", "file doesn't exist") return for l in open(c): l = l.strip() if not l: continue if l[0] == "#": continue self.onecmd(l.strip())
def get_db(self, c): """it returns the db name (or None) of a database identified either from its id or """ try: full_name, path, base_name, extension = utils.translate_name(c.strip()) db_name = "%s/%s.spgql" % (path, base_name) sim_name = "%s/%s.spg" % (path, base_name) except: utils.newline_msg("ERR", "results db '%s' doesn't exist." % c) return None return self.EnsembleConstructor(db_name, init_db=True)
def __init__(self, full_name = "", id=-1, weight=1., queue = '*', status = 'R', repeat = 1, init_db = True): ParameterEnsemble.__init__(self, full_name , id, weight, queue , status , repeat , init_db ) # self.init_db() os.chdir(self.path) if os.path.exists("./%s" % self.command): self.bin_dir = "." elif os.path.exists("%s%/bin/%s" % (BINARY_PATH, self.command)): self.bin_dir = "%s%/bin" % (BINARY_PATH) else: utils.newline_msg("ERR", "Fatal, binary '%s' not found" % self.command) sys.exit(1)
def __init__(self, full_name = "", id=-1, weight=1., queue = '*', status = 'R', repeat = 1, init_db = True): ParameterEnsemble.__init__(self, full_name , id, weight, queue , status , repeat , init_db ) # self.init_db() os.chdir(self.path) if os.path.exists("./%s" % self.command): self.bin_dir = "." # elif os.path.exists("%s%/bin/%s" % (BINARY_PATH, self.command)): # self.bin_dir = "%s%/bin" % (BINARY_PATH) else: utils.newline_msg("ERR", "Fatal, binary '%s' not found" % self.command) sys.exit(1)
def do_set_as_var(self,c): """ Sets a (set of) non-variables as variable """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... skipping") return ls_vars = c.split(",") if not set(ls_vars).issubset( set(self.current_param_db.entities) ): utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(ls_vars)-set(self.current_param_db.entities) ) return for v in ls_vars: self.current_param_db.execute_query( 'UPDATE entities SET varies=1 WHERE name = ?', v) self.current_param_db.init_db()
def do_load(self,c): """loads one of the registered databases from the master""" c = c.split() if len(c) >1: utils.newline_msg("ERR", "only one db can be loaded at a time", 2) return ret = self.get_db_from_cmdline(c[0]) if ret: self.current_param_db = ret print " --- loaded db '%s'"%self.shorten_name( self.current_param_db.full_name ) else: utils.newline_msg("ERR", "db does not exist", 2)
def harvest_atoms(self): ls_atoms = os.listdir("%s/run"%(VAR_PATH) ) self.harvested_atoms = len(ls_atoms) for i_d in ls_atoms: pd = ParameterAtom(i_d) try: pd.load(src = 'run') except: utils.newline_msg("WRN", "could not pickle '%s'...skipping"%i_d, 2) continue a_db =self.result_dbs[pd.full_db_name] pd.dump_result_in_ensemble( a_db )
def get_db(self, c): """it returns the db name (or None) of a database identified either from its id or """ try: full_name, path, base_name, extension = utils.translate_name( c.strip()) db_name = "%s/%s.spgql" % (path, base_name) sim_name = "%s/%s.spg" % (path, base_name) except: utils.newline_msg("ERR", "results db '%s' doesn't exist." % c) return None return self.EnsembleConstructor(db_name, init_db=True)
def execute_query(self, query, *args): self.__connect_db() # print(query) # print(*args) try: ret = [i for i in self.cursor.execute(query, args)] except: utils.newline_msg("ERR QUERY", "%s - %s"%(query, args)) flog = open("query_error.log", "wa" ) utils.newline_msg("ERR QUERY", "%s - %s"%(query, args), stream=flog) flog.close() self.__close_db() return ret
def do_set(self, c): """sets a VAR1=VALUE1[:VAR2=VALUE2] sets a value in the currently loaded database """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... skipping") return ret = utils.parse_to_dict(c, allowed_keys=self.possible_keys) if not ret: return for k in ret.keys(): self.__dict__[k] = ret[k]
def do_set_as_var(self,c): """ Sets a (set of) non-variables as variable """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... it must be loaded first",2) return flags, args = self.parse_command_line(c) ls_vars = args[0].split(",") if not set(ls_vars).issubset( set(self.current_param_db.entities) ): utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(ls_vars)-set(self.current_param_db.entities) ) return for v in ls_vars: self.current_param_db.query_master_db('UPDATE entities SET varies=1 WHERE name = ?', v) self.current_param_db.init_db()
def launch_process(self): pwd = os.path.abspath(".") if self.create_trees(): dir = utils.generate_string(self.values, self.directory_vars, joining_string="/") if not os.path.exists(dir): os.makedirs(dir) os.chdir(dir) configuration_filename = "input_%.8d.dat" % (self.current_run_id) output_filename = "output_%.8d.dat" % (self.current_run_id) # print configuration_filename fconf = open(configuration_filename, "w") for k in self.values.keys(): print >> fconf, k, utils.replace_values(self.values[k], self.values) fconf.close() # except: # utils.newline_msg("WRN", "could not load '%s'"%configuration_filename) # return cmd = "%s/%s -i %s > %s" % (BINARY_PATH, self.command, configuration_filename, output_filename) os.system(cmd) ret_code = 0 # proc = Popen(cmd, shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE ) # proc.wait() # ret_code = proc.returncode try: output = [i.strip() for i in open(output_filename).readline().split()] os.remove(configuration_filename) os.remove(output_filename) except: utils.newline_msg("WRN", "could not read '%s' - may it be other process accessed it" % output_filename) return if self.directory_vars: os.chdir(pwd) # if ret_code == 0: all_d = [self.current_valuesset_id] all_d.extend(output) # print self.output_column, all_d cc = "INSERT INTO results (values_set_id, %s) VALUES (%s) " % ( ", ".join(self.output_column), ", ".join([str(i) for i in all_d]), ) flog = open(self.full_name.replace("sqlite", "log"), "aw") print >> flog, "%.8d: %s --> %s" % (self.current_valuesset_id, self.values, output) try: self.execute_query(cc) self.execute_query('UPDATE run_status SET status ="D" WHERE id = %d' % self.current_run_id) except: self.execute_query('UPDATE run_status SET status ="E" WHERE id = %d' % self.current_run_id)
def get_db_from_cmdline(self, c): """it returns the db name (or None) of a database identified either from its id or """ try: id = int(c) filtered = filter(lambda x: x.id == id, self.master_db.result_dbs.values() ) if filtered: return filtered[0] except: foo, db_name = self.translate_name(c) if self.master_db.result_dbs.has_key(db_name): return self.master_db.result_dbs[db_name] else: return self.EnsembleConstructor(db_name, init_db = True) utils.newline_msg("WRN", "database '%s' is not registered, loading it anyhow"%self.shorten_name(db_name)) return None
def do_setup_vars_separated(self,c): """sets up which variables are going to have a separated directory save_table [-single_flag] FLAGS::: -restore: puts only the last variable in the output_table -empty: sets nothing as separated variables """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... it must be loaded first",2) return flags, args = self.parse_command_line(c) if "restore" in flags: self.current_param_db.setup_vars_separated(",".join(self.current_param_db.variables[:-1])) elif "empty" in flags: self.current_param_db.setup_vars_separated("") else: self.current_param_db.setup_vars_separated(args[0])
def do_set(self, c): """sets a VAR1=VALUE1[:VAR2=VALUE2] sets a value in the currently loaded database """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... skipping") return ret = utils.parse_to_dict(c, allowed_keys=self.possible_keys) if not ret: return for k in ret.keys(): self.current_param_db.__dict__[k] = ret[k] if k == "repeat": continue # repeat is not in the master db (should it be added) self.master_db.execute_query( 'UPDATE dbs SET %s= ? WHERE id = ?'%k, ret[k], self.current_param_db.id )
def do_init(self, c): """init [--flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]] Generates a new database out of a simulation.dat FLAGS::: --purge: deletes the spgql database, if it already exists --repeat=REPEAT repeats the parameter generation REPEAT times """ flags, db_arg = self.parse_command_line(c) if len(db_arg) != 1: utils.newline_msg("WRN", "init must be called with a database", 2) return db_arg = db_arg[0] # i_arg = c[0] full_name, path, base_name, extension = utils.translate_name(db_arg) # print "do_init::: ",self.translate_name(i_arg) full_db_name = "%s/%s.spgql" % (path, base_name) sim_name = "%s/%s.spg" % (path, base_name) if os.path.exists(full_db_name) and "purge" not in flags: utils.newline_msg( "ERR", "database '%s' already exists. Cannot init it twice" % utils.shorten_name(full_db_name), 2) return if not os.path.exists(sim_name): utils.newline_msg( "ERR", "configuration '%s' doesn't exist. Cannot init it" % utils.shorten_name(sim_name), 2) return if "purge" in flags: try: self.do_deregister(db_arg) os.remove(full_db_name) except: utils.newline_msg( "WRN", "database '%s' could not be removed... skipping" % full_db_name) if 'repeat' in flags: repeat = int(flags['repeat']) else: repeat = 1 parser = MultIteratorDBBuilder(db_name=full_db_name) parser.init_db() parser.fill_status(repeat=repeat) current_param_db = ParameterEnsemble(full_db_name, init_db=True) current_param_db.repeat = repeat # if len(c) > 1: self.do_set(":".join(c[1:])) self.master_db.write_ensemble_to_master(current_param_db) self.master_db.update_list_ensemble_dbs() print(" **-- init - %d: '%s' " % (current_param_db.id, utils.shorten_name(current_param_db.full_name)))
def get_settings(self, exec_file, part="stdout"): """ keysColumns = ["type","label","help","scale","repeat", "lim"] the structure of the columns in the files are as follows: name of the variable, and a colon separated list of -optional- options type: of the plot if xy, one column is used, xydy two columns are used label: to be used in the plotting script scale: comma separated list of minimum and maximum values repeat: how many columns are to be taken by the parser help: a string containing an explanation of the variable""" possible_keys = set( ["type", "label", "help", "scale", "repeat", "datatype", "lim"]) ret = {} exec_file, ext = os.path.splitext(exec_file) try: cfgFile = "%s.%s" % (exec_file, part) except: cfgFile = "%s/%s.%s" % (CONFIG_DIR, exec_file, part) sorted_cols = [] for line in open(cfgFile): if len(line.strip()) == 0: continue l = [i.strip() for i in line.split(":")] name = l.pop(0) sorted_cols.append(name) values = {} # {"type":"xy","datatype":"float"} for o in l: # print o, l k, v = o.split("=") k = k.strip() v = v.strip() if k not in possible_keys: newline_msg( "SYN", "in column '%s', unrecognised key '%s'" % (name, k)) sys.exit(1) if k == "lim": values[k] = eval(v) else: values[k] = v ret[name] = values return ret, sorted_cols
def do_setup_vars_in_table(self,c): """sets up the variables that output into the table as independent columns save_table [-single_flag] FLAGS::: --all: puts all variables in the output_table --restore: puts only the last variable in the output_table """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... it must be loaded first",2) return flags, args = self.parse_command_line(c) if "all" in flags: self.current_param_db.setup_vars_in_table(",".join(self.current_param_db.variables) ) elif "restore" in flags: self.current_param_db.setup_vars_in_table(self.current_param_db.variables[-1]) else: self.current_param_db.setup_vars_in_table( args[0] )
def do_setup_output_column(self, c): """sets which columns to generate output from""" if not self.current_param_db: utils.newline_msg("WRN", "current db not set... it must be loaded first", 2) return flags, args = self.parse_command_line(c) cols = args[0].split(",") if not set(cols).issubset( self.current_param_db.table_columns[self.table]): utils.newline_msg( "ERR", "the column(s) is (are) not in the output: %s" % (set(cols) - set(self.current_param_db.table_columns[self.table]))) self.table_columns = cols
def do_setup_vars_coalesced(self,c): """sets up which variables are coalesced into the same file save_table [-single_flag] FLAGS::: -restore: puts only the last variable in the output_table -empty: sets nothing as separated variables """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... it must be loaded first",2) return flags, args = self.parse_command_line(c) if ("restore" in flags) or ("empty" in flags): self.current_param_db.setup_vars_coalesced("") # elif "empty" in flags: # self.current_param_db.setup_vars_coalesced("") else: self.current_param_db.setup_vars_coalesced(args[0])
def do_load(self,c): """load DB_NAME|DB_ID loads one of the registered databases from the master""" flags, args = self.parse_command_line(c) if len(args) >1: utils.newline_msg("ERR", "only one db can be loaded at a time", 2) return ret = self.get_db(args[0]) if ret: self.current_param_db = ret print " --- loaded: '%s'"% utils.shorten_name(ret.full_name) os.chdir(self.current_param_db.path) else: utils.newline_msg("ERR", "'%s' does not exist"%ret, 2)
def do_load(self, c): """load DB_NAME|DB_ID loads one of the registered databases from the master""" flags, args = self.parse_command_line(c) if len(args) > 1: utils.newline_msg("ERR", "only one db can be loaded at a time", 2) return ret = self.get_db(args[0]) if ret: self.current_param_db = ret print(" --- loaded: '%s'" % utils.shorten_name(ret.full_name)) os.chdir(self.current_param_db.path) else: utils.newline_msg("ERR", "'%s' does not exist" % ret, 2)
def setup_vars_coalesced(self, conf): """Which variables are coalesced into the same files, orphaned variables are sent into the separated ones""" if conf.strip() != "" : coalesced = conf.split(",") else: coalesced = [] if set(coalesced).issubset( set(self.variables) ): self.coalesced_vars = coalesced self.separated_vars = [ i for i in self.separated_vars if ( i not in self.coalesced_vars ) ] self.vars_in_table = [i for i in self.vars_in_table if (i not in self.coalesced_vars)] orphaned = set(self.variables) - set(self.separated_vars) - set(self.vars_in_table) - set(self.coalesced_vars) if len(orphaned) > 0: utils.newline_msg("VAR", "orphaned variables '%s' added to separated variables"%orphaned, indent=4) for i in orphaned: self.separated_vars.append(i) print(" +- structure = %s - %s - %s "%(self.separated_vars, self.coalesced_vars, self.vars_in_table)) else: utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(coalesced)-set(self.variables) )
def do_setup_vars_coalesced(self, c): """sets up which variables are coalesced into the same file save_table [-single_flag] FLAGS::: -restore: puts only the last variable in the output_table -empty: sets nothing as separated variables """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... it must be loaded first", 2) return flags, args = self.parse_command_line(c) if ("restore" in flags) or ("empty" in flags): self.current_param_db.setup_vars_coalesced("") # elif "empty" in flags: # self.current_param_db.setup_vars_coalesced("") else: self.current_param_db.setup_vars_coalesced(args[0])
def do_setup_vars_separated(self, c): """sets up which variables are going to have a separated directory save_table [-single_flag] FLAGS::: -restore: puts only the last variable in the output_table -empty: sets nothing as separated variables """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... it must be loaded first", 2) return flags, args = self.parse_command_line(c) if "restore" in flags: self.current_param_db.setup_vars_separated(",".join( self.current_param_db.variables[:-1])) elif "empty" in flags: self.current_param_db.setup_vars_separated("") else: self.current_param_db.setup_vars_separated(args[0])
def do_set_as_var(self, c): """ Sets a (set of) non-variables as variable """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... it must be loaded first", 2) return flags, args = self.parse_command_line(c) ls_vars = args[0].split(",") if not set(ls_vars).issubset(set(self.current_param_db.entities)): utils.newline_msg( "VAR", "the variables '%s' are not recognised" % set(ls_vars) - set(self.current_param_db.entities)) return for v in ls_vars: self.current_param_db.query_master_db( 'UPDATE entities SET varies=1 WHERE name = ?', v) self.current_param_db.init_db()
def parse_output_line(self, output_line): """ parses a line from output. Returns a tuple containing: table of output, column names of output, output values to be inserted in table""" output_columns = output_line.strip().split() table_name = "results" if output_columns[0][0] == "@": table_name = output_columns[0][1:] output_columns.pop(0) if table_name not in self.table_columns: utils.newline_msg("ERR", "DB does not contain table named '%s'"%table_name) sys.exit(1) # try: # output_column_names = [ i[0] for i in self.execute_query("SELECT column FROM output_tables WHERE name = '%s'"%(table_name)) ] # except: # utils.newline_msg("ERR", "DB does not contain table named '%s'"%table_name) # sys.exit(1) return table_name, output_columns
def setup_vars_in_table(self, conf): """which are the variables that are inside of the output file, orphaned variables are sent into the coalesced ones""" if conf.strip() != "" : in_table_vars = conf.split(",") else: in_table_vars = [] if set(in_table_vars).issubset( set(self.variables) ): self.vars_in_table = in_table_vars self.coalesced_vars = [i for i in self.coalesced_vars if (i not in self.vars_in_table)] self.separated_vars = [i for i in self.separated_vars if (i not in self.vars_in_table)] orphaned = set(self.variables) - set(self.separated_vars) - set(self.vars_in_table) - set(self.coalesced_vars) if len(orphaned) > 0: utils.newline_msg("VAR", "orphaned variables '%s' added to separated variables"%orphaned, indent=4) for i in orphaned: self.coalesced_vars.append(i) print(" +- structure = %s - %s - %s "%(self.separated_vars, self.coalesced_vars, self.vars_in_table)) else: # print in_table_vars, conf utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(in_table_vars)-set(self.variables) )
def do_register(self, c): """registers a given results database into the master database""" flags, cmds, ensemble = self.get_flags_and_db(c) if ensemble is None: # utils.newline_msg("ERR", "no database supplied ... skipping") return if ensemble.full_name in self.master_db.result_dbs: utils.newline_msg( "WRN", "skipping... database '%s' is already registered" % utils.shorten_name(ensemble.full_name), 2) return # current_param_db = ParameterEnsemble(db_name, init_db=True) self.master_db.write_ensemble_to_master(ensemble) self.master_db.update_list_ensemble_dbs() print(" *--- registered - %d: '%s' " % (ensemble.id, utils.shorten_name(ensemble.full_name)))
def do_setup_vars_in_table(self, c): """sets up the variables that output into the table as independent columns save_table [-single_flag] FLAGS::: --all: puts all variables in the output_table --restore: puts only the last variable in the output_table """ if not self.current_param_db: utils.newline_msg("WRN", "current db not set... it must be loaded first", 2) return flags, args = self.parse_command_line(c) if "all" in flags: self.current_param_db.setup_vars_in_table(",".join( self.current_param_db.variables)) elif "restore" in flags: self.current_param_db.setup_vars_in_table( self.current_param_db.variables[-1]) else: self.current_param_db.setup_vars_in_table(args[0])
def do_set_weight(self, c): flags, args, ensemble = self.get_flags_and_db(c) # print flags, args, ensemble if ensemble == None: return try: new_weight = float(args[0]) except: utils.newline_msg("ERR", "cannot parse weight") return # print "UPDATE dbs SET weight=%f WHERE full_name = '%s' " % ( new_weight, ensemble.full_name ) try: self.master_db.query_master_db( "UPDATE dbs SET weight=%f WHERE full_name = '%s' " % (new_weight, ensemble.full_name)) except: utils.newline_msg("ERR", "cannot parse command") return self.master_db.update_list_ensemble_dbs()
def do_conf(self, c): """prints the current configuration""" flags, args = self.parse_command_line(c) if not self.current_param_db: if len(args) == 0: utils.newline_msg( "WRN", "database not loaded nor provided. skipping", 2) return elif len(args) > 1: utils.newline_msg("ERR", "only one db can be loaded at a time", 2) return else: self.do_load(args[0]) else: if len(args) > 0: utils.newline_msg("ERR", "only one db can be loaded at a time", 2) return print(" -- db: %s" % (os.path.relpath(self.current_param_db.full_name, "."))) print(" + variables = %s " % (", ".join(self.current_param_db.variables))) print(" + entities = %s " % (", ".join(self.current_param_db.entities))) print(" + tables found: %s " % (", ".join(list(self.current_param_db.table_columns.keys())))) print(" + vars (separated-coalesced-in_table) = %s - %s - %s " % (self.current_param_db.separated_vars, self.current_param_db.coalesced_vars, self.current_param_db.vars_in_table))
def init_db(self): try: (self.command, ) = self.execute_query_fetchone( "SELECT value FROM information WHERE key = 'command'" ) except: utils.newline_msg("FATAL", "Cannot retrieve information from database '%s'..."%utils.shorten_name(self.db_name)) sys.exit(2) #:::~ get the names of the columns sel = self.execute_query("SELECT name FROM entities ORDER BY id") self.entities = [ i[0] for i in sel ] #:::~ get the names of the columns sel = self.execute_query("SELECT name FROM entities WHERE varies = 1 ORDER BY id") self.variables = [ i[0] for i in sel ] #:::~ get the names of the outputs self.table_columns = {} table_names = [i[0] for i in self.execute_query("SELECT DISTINCT name from output_tables")] ### print table_names for table in table_names: fa = self.execute_query("SELECT column FROM output_tables WHERE name = '%s';"%table) self.table_columns[table] = ["spg_uid", "spg_vsid", "spg_rep"] + [i[0] for i in fa]
def do_import_output_table(self, c): """ Usage: import_output_table table_name.csv database.spqql Loads the results output in table_name.csv into database.spgql """ flags, args = self.parse_command_line(c) if not self.current_param_db: if len(args) != 2: utils.newline_msg( "WRN", "database not loaded nor provided. skipping", 2) return self.current_param_db = self.get_db(args[1]) elif len(args) > 1: utils.newline_msg("ERR", "only one db can be loaded at a time", 2) return self.current_param_db.update_results_from_data(args[0], sep=self.sep)
def add_setting(self, var, line): possible_keys = set(["type","label","help","scale","repeat","datatype", "lim"]) if var not in self.settings: self.settings[ var ] = {} ret_name = self.settings[ var ] l = [ i.strip() for i in line.split(":")] for o in l: k,v = o.split("=") k=k.strip() v=v.strip() if k not in possible_keys: spgu.newline_msg("SYN","in column '%s', unrecognised key '%s'"%(var,k)) sys.exit(1) if k == "lim": ret_name[k] = eval(v) else: ret_name[k]=v
def __init__(self, parameter_file): self.parameter_file = parameter_file self.base_name, foo = os.path.splitext(parameter_file) self.mit = spgb.MultIteratorParser(open(parameter_file)) self.variables = self.mit.get_variables() try: self.settings, foo = self.get_settings(self.mit.command, "input") except: self.settings = {} newline_msg("INF", "no 'input' file found: %s") settings_output, self.output_columns = self.get_settings( self.mit.command, "stdout") newline_msg("INF", "output columns: %s" % self.output_columns) self.settings.update(settings_output) self.x_axis = self.variables[-1] x_axis_iter = self.mit.data[self.mit.position_of(self.x_axis)] if x_axis_iter.type == "*": self.add_setting(self.x_axis, "scale = log") self.add_setting( self.x_axis, "lim = (%s,%s)" % (x_axis_iter.xmin, x_axis_iter.xmax)) try: self.coalesced_vars = [self.variables[-2]] except: self.coalesced_vars = [] try: self.separated_vars = self.variables[:-2] except: self.separated_vars = []
def do_set_table(self, c): """sets the output table sets a value in the currently loaded database If key = sep (separator for csv table) "blank" means a single space""" c = c.strip() if c == "help": utils.newline_msg("HELP", " possible_keys = %s" % self.possible_keys) return if not self.current_param_db: utils.newline_msg("WRN", "current db not set... skipping", 2) return if c not in list(self.current_param_db.table_columns.keys()): utils.newline_msg( "ERR", "table '%s' not among the ones found in the DB: (%s)" % (c, ", ".join(list( self.current_param_db.table_columns.keys())))) return self.table_columns = self.current_param_db.table_columns[c][1:]