def launch_workers(self):
target_jobs, = self.master_db.query_master_fetchone('SELECT max_jobs FROM queues WHERE name = "default"')
self.master_db.update_list_ensemble_dbs()
if len(self.master_db.active_dbs) == 0:
utils.inline_msg("MSG", "No active dbs... sleeping ")
return
current_count = self.active_threads()
to_launch = target_jobs - current_count
if to_launch >= 0:
utils.newline_msg("STATUS", "[n_jobs=%d] run=%d ::: new=%d" % (target_jobs,current_count,to_launch ) )
else:
utils.newline_msg("STATUS", "[n_jobs=%d] run=%d :!: exceed" % (target_jobs,current_count))
for i_t in range(to_launch):
self.lock.acquire()
pick = self.master_db.pick_ensemble()
status = pick.get_updated_status()
if status['process_not_run'] == 0:
print "+D+ ----- %s " % (pick.full_name)
self.master_db.query_master_db('UPDATE dbs SET status= ? WHERE full_name = ?', "D", pick.full_name)
return
self.lock.release()
nt = SPGRunningAtom(pick, self.lock)
# nt = SPGRunningAtom(pick, lock=self.get_lock( pick ) )
nt.start()
def get_flags_and_db(self, c, init_db=True):
flags, args = self.parse_command_line(c)
# if len(args)== 1:
# utils.newline_msg("ERR", "a single file was expected or could not parse flags")
# return flags, args, None
db_name = args[-1]
args = args[:-1]
if db_name.isdigit():
id = int(db_name)
rdb = self.master_db.result_dbs
filtered = [x for x in rdb if rdb[x]['id'] == id]
if filtered:
db_name = filtered[0]
else:
utils.newline_msg(
"ERR", "database with id '%s' doesn't exist." % db_name)
return flags, args.append(db_name), None
else:
full_name, path, base_name, extension = utils.translate_name(
db_name)
# print "do_init::: ",self.translate_name(i_arg)
db_name = "%s/%s.spgql" % (path, base_name)
# sim_name = "%s/%s.spg" % (path, base_name)
if not os.path.exists(db_name):
utils.newline_msg(
"ERR", "database with name '%s' doesn't exist." %
utils.shorten_name(db_name))
return flags, args.append(db_name), None
return flags, args, self.EnsembleConstructor(db_name, init_db)
def dump_result(self):
""" loads the next parameter atom from a parameter ensemble"""
#:::~ status can be either
#:::~ 'N': not run
#:::~ 'R': running
#:::~ 'D': successfully run (done)
#:::~ 'E': run but with non-zero error code
if self.return_code != 0:
self.query_set_run_status("E")
return
for line in self.output:
table_name, output_columns = self.parse_output_line(line)
output_columns = [self.current_spg_uid, self.current_spg_vsid, self.current_spg_rep] + output_columns
cc = 'INSERT INTO %s (%s) VALUES (%s) ' % (table_name, ", ".join(self.table_columns[table_name]),
", ".join(["'%s'" % str(i) for i in output_columns]))
# print cc
#
try:
self.execute_query(cc)
self.query_set_run_status("D", self.current_spg_uid, self.run_time)
except sql.OperationalError as e:
v = str(e).split()
nv, nc = int(v[0]), int(v[3])
utils.newline_msg("DB", "Fatal, '%d' values for the '%d' output columns %s expected " % (nv-3,nc-3, self.table_columns[table_name][3:]))
sys.exit(1)
def plot_all_join_outputs(self, Plotter):
newline_msg("INF", "%s - %s - %s" % (self.separated_vars, self.coalesced_vars, self.x_axis))
table_name = self.base_name + "_results.csv"
ctp = Plotter(table_name)
ctp.x_axis = self.x_axis
ctp.y_axis = self.output_columns
ctp.separated_vars = self.separated_vars
ctp.coalesced_vars = self.coalesced_vars
ctp.settings = self.settings
plot_fname = self.base_name + "_plot"
if len(ctp.separated_vars) > 1:
plot_fname += "_" + "_".join(ctp.separated_vars)
if len(ctp.separated_vars) == 1:
plot_fname += "_" + ctp.separated_vars[0]
plot_fname += ".pdf"
newline_msg("OUT", plot_fname)
ctp.plot_all_join_outputs(output_name=plot_fname)
def do_init(self, c):
"""init [-flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]]
Generates a new database out of a parameters.dat
FLAGS::: -skip-master: the new results.sqlite is created, but gets not registered in master
-purge: deletes the results.sqlite db, if it already existed
"""
flags,c = self.parse_command_line(c)
i_arg = c[0]
try:
i_arg, db_name = self.translate_name(i_arg)
except:
utils.newline_msg("ERR", "results db '%s' doesn't exist. Can not init it" )
return
if "purge" in flags:
try:
os.remove(db_name)
except:
utils.newline_msg("WRN", "db '%s' could not be removed... skipping"%db_name)
self.do_deregister(i_arg)
if len(c) >1: self.do_set( ":".join( c[1:] ) )
parser = EnsembleCSVBuilder(stream = open(i_arg), db_name=db_name)
parser.init_db( )
def __init__(self, arg, command=None):
full_name, self.path, self.base_name, extension = utils.translate_name(
arg)
MultIteratorParser.__init__(self, open(f"{self.base_name}.spg"))
if command is not None:
self.command = command
self.command, ext = os.path.splitext(self.command)
if not utils.check_params.consistency(self.command, self):
utils.newline_msg("ERR",
"simulation configuration is not consistent.")
sys.exit(1)
self.input_configuration = utils.read_input_configuration(
f"{self.command}.input")
self.default_parameters = utils.SPGSettings()
for k, v in self.input_configuration.items():
if v.var_type != "str":
self.default_parameters[k] = eval(f"{v.var_type}({v.default})")
else:
self.default_parameters[k] = v.default
self.stdout_configuration = utils.read_output_configuration(
self.command)
def get_flags_and_db(self, c,init_db = True):
flags, args = self.parse_command_line(c)
# if len(args)== 1:
# utils.newline_msg("ERR", "a single file was expected or could not parse flags")
# return flags, args, None
db_name = args[-1]
args = args[:-1]
if db_name.isdigit():
id = int(db_name)
rdb = self.master_db.result_dbs
filtered = [x for x in rdb if rdb[x]['id'] == id]
if filtered:
db_name = filtered[0]
else:
utils.newline_msg("ERR", "database with id '%s' doesn't exist." % db_name)
return flags, args.append(db_name), None
else:
full_name, path, base_name, extension = utils.translate_name(db_name)
# print "do_init::: ",self.translate_name(i_arg)
db_name = "%s/%s.spgql" % (path, base_name)
# sim_name = "%s/%s.spg" % (path, base_name)
if not os.path.exists(db_name):
utils.newline_msg("ERR", "database with name '%s' doesn't exist." % utils.shorten_name(db_name))
return flags, args.append(db_name), None
return flags, args, self.EnsembleConstructor(db_name, init_db)
def dump_result(self):
""" loads the next parameter atom from a parameter ensemble"""
#:::~ status can be either
#:::~ 'N': not run
#:::~ 'R': running
#:::~ 'D': successfully run (done)
#:::~ 'E': run but with non-zero error code
if self.return_code != 0:
self.query_set_run_status("E")
return
for line in self.output:
table_name, output_columns = self.parse_output_line(line)
# print( "DUMP", line , table_name, output_columns )
output_columns = [self.current_spg_uid, self.current_spg_vsid, self.current_spg_rep] + output_columns
# print(self.table_columns[table_name],["'%s'" % str(i) for i in output_columns])
cc = 'INSERT INTO %s (%s) VALUES (%s) ' % (table_name, ", ".join(self.table_columns[table_name]),
", ".join(["'%s'" % str(i) for i in output_columns]))
# print(cc)
#
# self.execute_query(cc)
try:
self.execute_query(cc)
self.query_set_run_status("D", self.current_spg_uid, self.run_time)
except sql.OperationalError as e:
v = str(e).split()
nv, nc = int(v[0]), int(v[3])
utils.newline_msg("DB", "Fatal, '%d' values for the '%d' output columns %s expected " % (nv-3,nc-3, self.table_columns[table_name][3:]))
sys.exit(1)
def plot_all_join_outputs(self, Plotter, mean):
newline_msg(
"INF", "%s - %s - %s" %
(self.separated_vars, self.coalesced_vars, self.x_axis))
table_name = self.base_name + "_results.csv"
ctp = Plotter(table_name)
ctp.x_axis = self.x_axis
ctp.y_axis = self.output_columns
ctp.separated_vars = self.separated_vars
ctp.coalesced_vars = self.coalesced_vars
ctp.settings = self.settings
plot_fname = self.base_name + "_plot"
if len(ctp.separated_vars) > 1:
plot_fname += "_" + "_".join(ctp.separated_vars)
if len(ctp.separated_vars) == 1:
plot_fname += "_" + ctp.separated_vars[0]
plot_fname += ".pdf"
newline_msg("OUT", plot_fname)
if mean:
ctp.mean()
ctp.plot_all_join_outputs(output_name=plot_fname)
def __init__(self, db_name, timeout=5):
full_name, self.path, self.base_name, extension = utils.translate_name(
db_name)
self.db_name = "%s/%s.spgql" % (self.path, self.base_name)
sim_name = "%s/%s.spg" % (self.path, self.base_name)
MultIteratorParser.__init__(self, open(sim_name))
# print self.data
# self.path, foo = os.path.split( os.path.abspath(db_name) )
# self.base_name, foo = os.path.splitext( foo )
# print check_params.consistency(self.command, self)
if not check_params.consistency(self.command, self):
utils.newline_msg("ERR",
"simulation configuration is not consistent.")
sys.exit(1)
all_outputs = check_params.read_output_configuration(self.command)
self.stdout_contents = dict()
for k in all_outputs:
table = k.output_table
if table not in self.stdout_contents.leys():
self.stdout_contents[table] = k
self.connection = sql.connect(self.db_name,
timeout=timeout,
check_same_thread=False)
self.cursor = self.connection.cursor()
def plot_all(self, Plotter):
newline_msg("INF", "%s - %s - %s" % (self.separated_vars, self.coalesced_vars, self.x_axis))
table_name = "%s_%s.csv"%(self.base_name, self.table_name )
ctp = Plotter(table_name)
ctp.x_axis = self.x_axis
ctp.y_axis = self.output_columns
ctp.separated_vars = self.separated_vars
ctp.coalesced_vars = self.coalesced_vars
ctp.settings = self.settings
plot_fname = "%s_%s_plot"%(self.base_name, self.table_name )
if len(ctp.separated_vars) > 1:
plot_fname += "_" + "_".join(ctp.separated_vars)
if len(ctp.separated_vars) == 1:
plot_fname += "_" + ctp.separated_vars[0]
plot_fname += ".pdf"
newline_msg("OUT", plot_fname)
ctp.plot_all(output_name=plot_fname)
def __init__(self, parameter_file):
self.parameter_file = parameter_file
self.mit = spgb.MultIteratorParser( open(parameter_file) )
self.variables = self.mit.varying_items()
self.settings, foo = self.get_settings(self.mit.command, "inputs")
settings_output, self.output_columns = self.get_settings(self.mit.command, "stdout")
newline_msg( "INF", "output columns: %s"% enumerate(self.output_columns) )
self.settings.update(settings_output)
self.x_axis = self.variables[-1]
x_axis_iter = self.mit.data[ self.mit.position_of( self.x_axis ) ]
if x_axis_iter.type == "*":
self.add_setting( self.x_axis, "scale = log")
self.add_setting( self.x_axis, "lim = (%s,%s)"%(x_axis_iter.xmin,x_axis_iter.xmax))
try:
self.coalesced_vars = [ self.variables[-2] ]
except:
self.coalesced_vars = []
try:
self.separated_vars = self.variables[:-2]
except:
self.separated_vars = []
def do_load(self,c):
"""loads a queue as active one"""
self.__update_queue_list()
if len( filter(lambda x: x[1] == c, self.queues) ):
self.current_queue = c
else:
newline_msg("ERR", "queue '%s' not found"%c)
print "loaded '%s'"%self.current_queue
def __init__(self, key, previous, value):
self.key = key
self.previous = previous
self.value = value
utils.newline_msg(
"ERR", "conflict in %s name (in db '%s', in param '%s')" %
(key, previous, value))
def do_setup_output_column(self,c):
"""sets which columns to generate output from"""
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... skipping")
return
c = c.split(",")
if not set(c).issubset( self.current_param_db.output_column ):
utils.newline_msg("ERR", "the column(s) is (are) not in the output: %s"%( set(c) - set( self.current_param_db.output_column )) )
self.output_column = c
def do_add(self, c):
"""adds queue queue"""
self.__update_queue_list()
if len( filter(lambda x: x[1] == c, self.queues) ):
newline_msg("ERR", "queue '%s' already exists"%queue, 2)
return
self.master_db.execute_query( "INSERT INTO queues (name, max_jobs, status) VALUES (?,?,?)",c, 1, 'S')
os.makedirs("%s/queue/%s"%(VAR_PATH,c))
self.current_queue = c
def do_setup_output_column(self,c):
"""sets which columns to generate output from"""
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... it must be loaded first",2)
return
flags, args = self.parse_command_line(c)
cols = args[0].split(",")
if not set(cols).issubset( self.current_param_db.table_columns[self.table] ):
utils.newline_msg("ERR", "the column(s) is (are) not in the output: %s"%( set(cols) - set( self.current_param_db.table_columns[self.table] )) )
self.table_columns = cols
def __init__(self, stream=None, db_name = "results.sqlite", timeout = 5):
MultIteratorParser.__init__(self, stream)
# print check_params.consistency(self.command, self)
if not check_params.consistency(self.command, self):
utils.newline_msg("ERR","parameters.dat file is not consistent.")
sys.exit(1)
self.stdout_contents = check_params.contents_in_output(self.command)
self.connection = sql.connect(db_name, timeout = timeout)
self.cursor = self.connection.cursor()
def launch_workers(self):
target_jobs, = self.master_db.query_master_fetchone('SELECT max_jobs FROM queues WHERE name = "default"')
self.master_db.update_list_ensemble_dbs()
if len(self.master_db.active_dbs) == 0:
utils.inline_msg("MSG", "No active dbs... sleeping ")
return
current_count = self.active_threads()
# print "+++++++++++", to_launch
vec_to_launch = []
launch = defaultdict(lambda: 0)
running = {}
for ae in self.master_db.active_dbs:
ens = self.master_db.result_dbs[ae ]
running[ ens['id'] ] = self.active_processes[ ae ]
qty_to_launch = int( m.floor(0.5 + target_jobs*ens['weight']/self.master_db.normalising) - self.active_processes[ ae ] )
vec_to_launch += qty_to_launch * [ae]
launch[ ens['id'] ] += qty_to_launch
# for id in launch:
# print "+++ (%d) %d + %d = //%d//, "%( id, launch[id], running[id],launch[id]+running[id] )
# print
to_launch = len(vec_to_launch)
if to_launch >= 0:
utils.newline_msg("STATUS", utils.str_color( "@green[n_jobs=%d] run=%d %s ::: new=%d" % (target_jobs,current_count, dict(running),to_launch) ) )
else:
utils.newline_msg("STATUS", utils.str_color( "@yellow[n_jobs=%d] run=%d :!: exceeded number" % (target_jobs,current_count)) )
# print to_launch, len( vec_to_launch ), launch
# for i_t in range(to_launch):
for ae in vec_to_launch:
pick = self.master_db.EnsembleConstructor(ae, init_db=True)
self.lock.acquire()
# pick = self.master_db.pick_ensemble()
pick.test_run = self.test_run
status = pick.get_updated_status()
if status['process_not_run'] == 0:
print "+D+ ----- %s " % (pick.full_name)
self.master_db.query_master_db('UPDATE dbs SET status= ? WHERE full_name = ?', "D", pick.full_name)
return
self.lock.release()
nt = SPGRunningAtom(pick, self.lock, self.active_processes)
# nt.test_run = self.test_run
# nt = SPGRunningAtom(pick, lock=self.get_lock( pick ) )
nt.start()
def setup_separated_output(self, conf):
try:
i = int(conf)
separated = self.variables[:i+1]
except:
separated = conf.split(",")
if set(separated).issubset( set(self.variables) ):
self.separated_vars = separated
self.coalesced_vars = [ i for i in self.variables if ( i not in self.separated_vars ) and ( i not in self.in_table_vars ) ]
else:
utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(separated)-set(self.variables) )
def do_run_script(self,c):
"""executes a script file with commands accepted in this cmdline parser"""
if not os.path.exists(c):
utils.newline_msg("FIL", "file doesn't exist")
return
for l in open(c):
l = l.strip()
if not l: continue
if l[0] == "#": continue
self.onecmd(l.strip())
def update_ensemble_list(self):
self.normalising = 0.
self.active_dbs = []
for i in self.result_dbs.keys():
if self.result_dbs[i] == None:
del self.result_dbs[i]
utils.newline_msg("MSG", "removing db '%s' from the running list"%i)
continue
if self.result_dbs[i].status == 'R':
self.normalising += self.result_dbs[ i ].weight
self.active_dbs.append( self.result_dbs[ i ] )
def do_conf(self,c):
"""prints the current configuration"""
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... skipping")
return
print " -- db: %s"%( self.shorten_name( self.current_param_db.full_name ) )
print " + variables = %s "%( ", ".join(self.current_param_db.variables ) )
print " + entities = %s "%( ", ".join(self.current_param_db.entities ) )
print " + columns = %s "%( ", ".join(self.current_param_db.output_column ) )
print " + split_columns = %s / raw_data = %s / restrict_by_val = %s"%(self.split_columns, self.raw_data, self.restrict_by_val)
print " + vars (separated-coalesced-in_table) = %s - %s - %s "%(self.current_param_db.separated_vars, self.current_param_db.coalesced_vars, self.current_param_db.in_table_vars)
def do_run_script(self, c):
"""executes a script file with commands accepted in this cmdline parser"""
if not os.path.exists(c):
utils.newline_msg("FIL", "file doesn't exist")
return
for l in open(c):
l = l.strip()
if not l: continue
if l[0] == "#": continue
self.onecmd(l.strip())
def get_db(self, c):
"""it returns the db name (or None) of a database identified either from its id or """
try:
full_name, path, base_name, extension = utils.translate_name(c.strip())
db_name = "%s/%s.spgql" % (path, base_name)
sim_name = "%s/%s.spg" % (path, base_name)
except:
utils.newline_msg("ERR", "results db '%s' doesn't exist." % c)
return None
return self.EnsembleConstructor(db_name, init_db=True)
def __init__(self, full_name = "", id=-1, weight=1., queue = '*', status = 'R', repeat = 1, init_db = True):
ParameterEnsemble.__init__(self, full_name , id, weight, queue , status , repeat , init_db )
# self.init_db()
os.chdir(self.path)
if os.path.exists("./%s" % self.command):
self.bin_dir = "."
elif os.path.exists("%s%/bin/%s" % (BINARY_PATH, self.command)):
self.bin_dir = "%s%/bin" % (BINARY_PATH)
else:
utils.newline_msg("ERR", "Fatal, binary '%s' not found" % self.command)
sys.exit(1)
def __init__(self, full_name = "", id=-1, weight=1., queue = '*', status = 'R', repeat = 1, init_db = True):
ParameterEnsemble.__init__(self, full_name , id, weight, queue , status , repeat , init_db )
# self.init_db()
os.chdir(self.path)
if os.path.exists("./%s" % self.command):
self.bin_dir = "."
# elif os.path.exists("%s%/bin/%s" % (BINARY_PATH, self.command)):
# self.bin_dir = "%s%/bin" % (BINARY_PATH)
else:
utils.newline_msg("ERR", "Fatal, binary '%s' not found" % self.command)
sys.exit(1)
def do_set_as_var(self,c):
""" Sets a (set of) non-variables as variable """
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... skipping")
return
ls_vars = c.split(",")
if not set(ls_vars).issubset( set(self.current_param_db.entities) ):
utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(ls_vars)-set(self.current_param_db.entities) )
return
for v in ls_vars:
self.current_param_db.execute_query( 'UPDATE entities SET varies=1 WHERE name = ?', v)
self.current_param_db.init_db()
def do_load(self,c):
"""loads one of the registered databases from the master"""
c = c.split()
if len(c) >1:
utils.newline_msg("ERR", "only one db can be loaded at a time", 2)
return
ret = self.get_db_from_cmdline(c[0])
if ret:
self.current_param_db = ret
print " --- loaded db '%s'"%self.shorten_name( self.current_param_db.full_name )
else:
utils.newline_msg("ERR", "db does not exist", 2)
def harvest_atoms(self):
ls_atoms = os.listdir("%s/run"%(VAR_PATH) )
self.harvested_atoms = len(ls_atoms)
for i_d in ls_atoms:
pd = ParameterAtom(i_d)
try:
pd.load(src = 'run')
except:
utils.newline_msg("WRN", "could not pickle '%s'...skipping"%i_d, 2)
continue
a_db =self.result_dbs[pd.full_db_name]
pd.dump_result_in_ensemble( a_db )
def get_db(self, c):
"""it returns the db name (or None) of a database identified either from its id or """
try:
full_name, path, base_name, extension = utils.translate_name(
c.strip())
db_name = "%s/%s.spgql" % (path, base_name)
sim_name = "%s/%s.spg" % (path, base_name)
except:
utils.newline_msg("ERR", "results db '%s' doesn't exist." % c)
return None
return self.EnsembleConstructor(db_name, init_db=True)
def execute_query(self, query, *args):
self.__connect_db()
# print(query)
# print(*args)
try:
ret = [i for i in self.cursor.execute(query, args)]
except:
utils.newline_msg("ERR QUERY", "%s - %s"%(query, args))
flog = open("query_error.log", "wa" )
utils.newline_msg("ERR QUERY", "%s - %s"%(query, args), stream=flog)
flog.close()
self.__close_db()
return ret
def do_set(self, c):
"""sets a VAR1=VALUE1[:VAR2=VALUE2]
sets a value in the currently loaded database """
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... skipping")
return
ret = utils.parse_to_dict(c, allowed_keys=self.possible_keys)
if not ret:
return
for k in ret.keys():
self.__dict__[k] = ret[k]
def do_set_as_var(self,c):
""" Sets a (set of) non-variables as variable """
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... it must be loaded first",2)
return
flags, args = self.parse_command_line(c)
ls_vars = args[0].split(",")
if not set(ls_vars).issubset( set(self.current_param_db.entities) ):
utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(ls_vars)-set(self.current_param_db.entities) )
return
for v in ls_vars:
self.current_param_db.query_master_db('UPDATE entities SET varies=1 WHERE name = ?', v)
self.current_param_db.init_db()
def launch_process(self):
pwd = os.path.abspath(".")
if self.create_trees():
dir = utils.generate_string(self.values, self.directory_vars, joining_string="/")
if not os.path.exists(dir):
os.makedirs(dir)
os.chdir(dir)
configuration_filename = "input_%.8d.dat" % (self.current_run_id)
output_filename = "output_%.8d.dat" % (self.current_run_id)
# print configuration_filename
fconf = open(configuration_filename, "w")
for k in self.values.keys():
print >> fconf, k, utils.replace_values(self.values[k], self.values)
fconf.close()
# except:
# utils.newline_msg("WRN", "could not load '%s'"%configuration_filename)
# return
cmd = "%s/%s -i %s > %s" % (BINARY_PATH, self.command, configuration_filename, output_filename)
os.system(cmd)
ret_code = 0
# proc = Popen(cmd, shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE )
# proc.wait()
# ret_code = proc.returncode
try:
output = [i.strip() for i in open(output_filename).readline().split()]
os.remove(configuration_filename)
os.remove(output_filename)
except:
utils.newline_msg("WRN", "could not read '%s' - may it be other process accessed it" % output_filename)
return
if self.directory_vars:
os.chdir(pwd)
# if ret_code == 0:
all_d = [self.current_valuesset_id]
all_d.extend(output)
# print self.output_column, all_d
cc = "INSERT INTO results (values_set_id, %s) VALUES (%s) " % (
", ".join(self.output_column),
", ".join([str(i) for i in all_d]),
)
flog = open(self.full_name.replace("sqlite", "log"), "aw")
print >> flog, "%.8d: %s --> %s" % (self.current_valuesset_id, self.values, output)
try:
self.execute_query(cc)
self.execute_query('UPDATE run_status SET status ="D" WHERE id = %d' % self.current_run_id)
except:
self.execute_query('UPDATE run_status SET status ="E" WHERE id = %d' % self.current_run_id)
def get_db_from_cmdline(self, c):
"""it returns the db name (or None) of a database identified either from its id or """
try:
id = int(c)
filtered = filter(lambda x: x.id == id, self.master_db.result_dbs.values() )
if filtered:
return filtered[0]
except:
foo, db_name = self.translate_name(c)
if self.master_db.result_dbs.has_key(db_name):
return self.master_db.result_dbs[db_name]
else:
return self.EnsembleConstructor(db_name, init_db = True)
utils.newline_msg("WRN", "database '%s' is not registered, loading it anyhow"%self.shorten_name(db_name))
return None
def do_setup_vars_separated(self,c):
"""sets up which variables are going to have a separated directory
save_table [-single_flag]
FLAGS::: -restore: puts only the last variable in the output_table
-empty: sets nothing as separated variables """
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... it must be loaded first",2)
return
flags, args = self.parse_command_line(c)
if "restore" in flags:
self.current_param_db.setup_vars_separated(",".join(self.current_param_db.variables[:-1]))
elif "empty" in flags:
self.current_param_db.setup_vars_separated("")
else:
self.current_param_db.setup_vars_separated(args[0])
def do_set(self, c):
"""sets a VAR1=VALUE1[:VAR2=VALUE2]
sets a value in the currently loaded database """
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... skipping")
return
ret = utils.parse_to_dict(c, allowed_keys=self.possible_keys)
if not ret:
return
for k in ret.keys():
self.current_param_db.__dict__[k] = ret[k]
if k == "repeat": continue # repeat is not in the master db (should it be added)
self.master_db.execute_query( 'UPDATE dbs SET %s= ? WHERE id = ?'%k, ret[k], self.current_param_db.id )
def do_init(self, c):
"""init [--flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]]
Generates a new database out of a simulation.dat
FLAGS::: --purge: deletes the spgql database, if it already exists
--repeat=REPEAT repeats the parameter generation REPEAT times
"""
flags, db_arg = self.parse_command_line(c)
if len(db_arg) != 1:
utils.newline_msg("WRN", "init must be called with a database", 2)
return
db_arg = db_arg[0]
# i_arg = c[0]
full_name, path, base_name, extension = utils.translate_name(db_arg)
# print "do_init::: ",self.translate_name(i_arg)
full_db_name = "%s/%s.spgql" % (path, base_name)
sim_name = "%s/%s.spg" % (path, base_name)
if os.path.exists(full_db_name) and "purge" not in flags:
utils.newline_msg(
"ERR", "database '%s' already exists. Cannot init it twice" %
utils.shorten_name(full_db_name), 2)
return
if not os.path.exists(sim_name):
utils.newline_msg(
"ERR", "configuration '%s' doesn't exist. Cannot init it" %
utils.shorten_name(sim_name), 2)
return
if "purge" in flags:
try:
self.do_deregister(db_arg)
os.remove(full_db_name)
except:
utils.newline_msg(
"WRN", "database '%s' could not be removed... skipping" %
full_db_name)
if 'repeat' in flags:
repeat = int(flags['repeat'])
else:
repeat = 1
parser = MultIteratorDBBuilder(db_name=full_db_name)
parser.init_db()
parser.fill_status(repeat=repeat)
current_param_db = ParameterEnsemble(full_db_name, init_db=True)
current_param_db.repeat = repeat
# if len(c) > 1: self.do_set(":".join(c[1:]))
self.master_db.write_ensemble_to_master(current_param_db)
self.master_db.update_list_ensemble_dbs()
print(" **-- init - %d: '%s' " %
(current_param_db.id,
utils.shorten_name(current_param_db.full_name)))
def get_settings(self, exec_file, part="stdout"):
"""
keysColumns = ["type","label","help","scale","repeat", "lim"]
the structure of the columns in the files are as follows:
name of the variable, and a colon separated list of -optional- options
type: of the plot if xy, one column is used, xydy two columns are used
label: to be used in the plotting script
scale: comma separated list of minimum and maximum values
repeat: how many columns are to be taken by the parser
help: a string containing an explanation of the variable"""
possible_keys = set(
["type", "label", "help", "scale", "repeat", "datatype", "lim"])
ret = {}
exec_file, ext = os.path.splitext(exec_file)
try:
cfgFile = "%s.%s" % (exec_file, part)
except:
cfgFile = "%s/%s.%s" % (CONFIG_DIR, exec_file, part)
sorted_cols = []
for line in open(cfgFile):
if len(line.strip()) == 0: continue
l = [i.strip() for i in line.split(":")]
name = l.pop(0)
sorted_cols.append(name)
values = {} # {"type":"xy","datatype":"float"}
for o in l:
# print o, l
k, v = o.split("=")
k = k.strip()
v = v.strip()
if k not in possible_keys:
newline_msg(
"SYN",
"in column '%s', unrecognised key '%s'" % (name, k))
sys.exit(1)
if k == "lim":
values[k] = eval(v)
else:
values[k] = v
ret[name] = values
return ret, sorted_cols
def do_setup_vars_in_table(self,c):
"""sets up the variables that output into the table as independent columns
save_table [-single_flag]
FLAGS::: --all: puts all variables in the output_table
--restore: puts only the last variable in the output_table
"""
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... it must be loaded first",2)
return
flags, args = self.parse_command_line(c)
if "all" in flags:
self.current_param_db.setup_vars_in_table(",".join(self.current_param_db.variables) )
elif "restore" in flags:
self.current_param_db.setup_vars_in_table(self.current_param_db.variables[-1])
else:
self.current_param_db.setup_vars_in_table( args[0] )
def do_setup_output_column(self, c):
"""sets which columns to generate output from"""
if not self.current_param_db:
utils.newline_msg("WRN",
"current db not set... it must be loaded first",
2)
return
flags, args = self.parse_command_line(c)
cols = args[0].split(",")
if not set(cols).issubset(
self.current_param_db.table_columns[self.table]):
utils.newline_msg(
"ERR", "the column(s) is (are) not in the output: %s" %
(set(cols) -
set(self.current_param_db.table_columns[self.table])))
self.table_columns = cols
def do_setup_vars_coalesced(self,c):
"""sets up which variables are coalesced into the same file
save_table [-single_flag]
FLAGS::: -restore: puts only the last variable in the output_table
-empty: sets nothing as separated variables """
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... it must be loaded first",2)
return
flags, args = self.parse_command_line(c)
if ("restore" in flags) or ("empty" in flags):
self.current_param_db.setup_vars_coalesced("")
# elif "empty" in flags:
# self.current_param_db.setup_vars_coalesced("")
else:
self.current_param_db.setup_vars_coalesced(args[0])
def do_load(self,c):
"""load DB_NAME|DB_ID
loads one of the registered databases from the master"""
flags, args = self.parse_command_line(c)
if len(args) >1:
utils.newline_msg("ERR", "only one db can be loaded at a time", 2)
return
ret = self.get_db(args[0])
if ret:
self.current_param_db = ret
print " --- loaded: '%s'"% utils.shorten_name(ret.full_name)
os.chdir(self.current_param_db.path)
else:
utils.newline_msg("ERR", "'%s' does not exist"%ret, 2)
def do_load(self, c):
"""load DB_NAME|DB_ID
loads one of the registered databases from the master"""
flags, args = self.parse_command_line(c)
if len(args) > 1:
utils.newline_msg("ERR", "only one db can be loaded at a time", 2)
return
ret = self.get_db(args[0])
if ret:
self.current_param_db = ret
print(" --- loaded: '%s'" % utils.shorten_name(ret.full_name))
os.chdir(self.current_param_db.path)
else:
utils.newline_msg("ERR", "'%s' does not exist" % ret, 2)
def setup_vars_coalesced(self, conf):
"""Which variables are coalesced into the same files, orphaned variables are sent into the separated ones"""
if conf.strip() != "" :
coalesced = conf.split(",")
else:
coalesced = []
if set(coalesced).issubset( set(self.variables) ):
self.coalesced_vars = coalesced
self.separated_vars = [ i for i in self.separated_vars if ( i not in self.coalesced_vars ) ]
self.vars_in_table = [i for i in self.vars_in_table if (i not in self.coalesced_vars)]
orphaned = set(self.variables) - set(self.separated_vars) - set(self.vars_in_table) - set(self.coalesced_vars)
if len(orphaned) > 0:
utils.newline_msg("VAR", "orphaned variables '%s' added to separated variables"%orphaned, indent=4)
for i in orphaned: self.separated_vars.append(i)
print(" +- structure = %s - %s - %s "%(self.separated_vars, self.coalesced_vars, self.vars_in_table))
else:
utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(coalesced)-set(self.variables) )
def do_setup_vars_coalesced(self, c):
"""sets up which variables are coalesced into the same file
save_table [-single_flag]
FLAGS::: -restore: puts only the last variable in the output_table
-empty: sets nothing as separated variables """
if not self.current_param_db:
utils.newline_msg("WRN",
"current db not set... it must be loaded first",
2)
return
flags, args = self.parse_command_line(c)
if ("restore" in flags) or ("empty" in flags):
self.current_param_db.setup_vars_coalesced("")
# elif "empty" in flags:
# self.current_param_db.setup_vars_coalesced("")
else:
self.current_param_db.setup_vars_coalesced(args[0])
def do_setup_vars_separated(self, c):
"""sets up which variables are going to have a separated directory
save_table [-single_flag]
FLAGS::: -restore: puts only the last variable in the output_table
-empty: sets nothing as separated variables """
if not self.current_param_db:
utils.newline_msg("WRN",
"current db not set... it must be loaded first",
2)
return
flags, args = self.parse_command_line(c)
if "restore" in flags:
self.current_param_db.setup_vars_separated(",".join(
self.current_param_db.variables[:-1]))
elif "empty" in flags:
self.current_param_db.setup_vars_separated("")
else:
self.current_param_db.setup_vars_separated(args[0])
def do_set_as_var(self, c):
""" Sets a (set of) non-variables as variable """
if not self.current_param_db:
utils.newline_msg("WRN",
"current db not set... it must be loaded first",
2)
return
flags, args = self.parse_command_line(c)
ls_vars = args[0].split(",")
if not set(ls_vars).issubset(set(self.current_param_db.entities)):
utils.newline_msg(
"VAR", "the variables '%s' are not recognised" % set(ls_vars) -
set(self.current_param_db.entities))
return
for v in ls_vars:
self.current_param_db.query_master_db(
'UPDATE entities SET varies=1 WHERE name = ?', v)
self.current_param_db.init_db()
def parse_output_line(self, output_line):
""" parses a line from output. Returns a tuple containing: table of output, column names of output, output values to be inserted in table"""
output_columns = output_line.strip().split()
table_name = "results"
if output_columns[0][0] == "@":
table_name = output_columns[0][1:]
output_columns.pop(0)
if table_name not in self.table_columns:
utils.newline_msg("ERR", "DB does not contain table named '%s'"%table_name)
sys.exit(1)
# try:
# output_column_names = [ i[0] for i in self.execute_query("SELECT column FROM output_tables WHERE name = '%s'"%(table_name)) ]
# except:
# utils.newline_msg("ERR", "DB does not contain table named '%s'"%table_name)
# sys.exit(1)
return table_name, output_columns
def setup_vars_in_table(self, conf):
"""which are the variables that are inside of the output file, orphaned variables are sent into the coalesced ones"""
if conf.strip() != "" :
in_table_vars = conf.split(",")
else:
in_table_vars = []
if set(in_table_vars).issubset( set(self.variables) ):
self.vars_in_table = in_table_vars
self.coalesced_vars = [i for i in self.coalesced_vars if (i not in self.vars_in_table)]
self.separated_vars = [i for i in self.separated_vars if (i not in self.vars_in_table)]
orphaned = set(self.variables) - set(self.separated_vars) - set(self.vars_in_table) - set(self.coalesced_vars)
if len(orphaned) > 0:
utils.newline_msg("VAR", "orphaned variables '%s' added to separated variables"%orphaned, indent=4)
for i in orphaned: self.coalesced_vars.append(i)
print(" +- structure = %s - %s - %s "%(self.separated_vars, self.coalesced_vars, self.vars_in_table))
else:
# print in_table_vars, conf
utils.newline_msg("VAR", "the variables '%s' are not recognised"%set(in_table_vars)-set(self.variables) )
def do_register(self, c):
"""registers a given results database into the master database"""
flags, cmds, ensemble = self.get_flags_and_db(c)
if ensemble is None:
# utils.newline_msg("ERR", "no database supplied ... skipping")
return
if ensemble.full_name in self.master_db.result_dbs:
utils.newline_msg(
"WRN", "skipping... database '%s' is already registered" %
utils.shorten_name(ensemble.full_name), 2)
return
# current_param_db = ParameterEnsemble(db_name, init_db=True)
self.master_db.write_ensemble_to_master(ensemble)
self.master_db.update_list_ensemble_dbs()
print(" *--- registered - %d: '%s' " %
(ensemble.id, utils.shorten_name(ensemble.full_name)))
def do_setup_vars_in_table(self, c):
"""sets up the variables that output into the table as independent columns
save_table [-single_flag]
FLAGS::: --all: puts all variables in the output_table
--restore: puts only the last variable in the output_table
"""
if not self.current_param_db:
utils.newline_msg("WRN",
"current db not set... it must be loaded first",
2)
return
flags, args = self.parse_command_line(c)
if "all" in flags:
self.current_param_db.setup_vars_in_table(",".join(
self.current_param_db.variables))
elif "restore" in flags:
self.current_param_db.setup_vars_in_table(
self.current_param_db.variables[-1])
else:
self.current_param_db.setup_vars_in_table(args[0])
def do_set_weight(self, c):
flags, args, ensemble = self.get_flags_and_db(c)
# print flags, args, ensemble
if ensemble == None:
return
try:
new_weight = float(args[0])
except:
utils.newline_msg("ERR", "cannot parse weight")
return
# print "UPDATE dbs SET weight=%f WHERE full_name = '%s' " % ( new_weight, ensemble.full_name )
try:
self.master_db.query_master_db(
"UPDATE dbs SET weight=%f WHERE full_name = '%s' " %
(new_weight, ensemble.full_name))
except:
utils.newline_msg("ERR", "cannot parse command")
return
self.master_db.update_list_ensemble_dbs()
def do_conf(self, c):
"""prints the current configuration"""
flags, args = self.parse_command_line(c)
if not self.current_param_db:
if len(args) == 0:
utils.newline_msg(
"WRN", "database not loaded nor provided. skipping", 2)
return
elif len(args) > 1:
utils.newline_msg("ERR", "only one db can be loaded at a time",
2)
return
else:
self.do_load(args[0])
else:
if len(args) > 0:
utils.newline_msg("ERR", "only one db can be loaded at a time",
2)
return
print(" -- db: %s" %
(os.path.relpath(self.current_param_db.full_name, ".")))
print(" + variables = %s " %
(", ".join(self.current_param_db.variables)))
print(" + entities = %s " %
(", ".join(self.current_param_db.entities)))
print(" + tables found: %s " %
(", ".join(list(self.current_param_db.table_columns.keys()))))
print(" + vars (separated-coalesced-in_table) = %s - %s - %s " %
(self.current_param_db.separated_vars,
self.current_param_db.coalesced_vars,
self.current_param_db.vars_in_table))
def init_db(self):
try:
(self.command, ) = self.execute_query_fetchone( "SELECT value FROM information WHERE key = 'command'" )
except:
utils.newline_msg("FATAL", "Cannot retrieve information from database '%s'..."%utils.shorten_name(self.db_name))
sys.exit(2)
#:::~ get the names of the columns
sel = self.execute_query("SELECT name FROM entities ORDER BY id")
self.entities = [ i[0] for i in sel ]
#:::~ get the names of the columns
sel = self.execute_query("SELECT name FROM entities WHERE varies = 1 ORDER BY id")
self.variables = [ i[0] for i in sel ]
#:::~ get the names of the outputs
self.table_columns = {}
table_names = [i[0] for i in self.execute_query("SELECT DISTINCT name from output_tables")]
### print table_names
for table in table_names:
fa = self.execute_query("SELECT column FROM output_tables WHERE name = '%s';"%table)
self.table_columns[table] = ["spg_uid", "spg_vsid", "spg_rep"] + [i[0] for i in fa]
def do_import_output_table(self, c):
"""
Usage:
import_output_table table_name.csv database.spqql
Loads the results output in table_name.csv into database.spgql
"""
flags, args = self.parse_command_line(c)
if not self.current_param_db:
if len(args) != 2:
utils.newline_msg(
"WRN", "database not loaded nor provided. skipping", 2)
return
self.current_param_db = self.get_db(args[1])
elif len(args) > 1:
utils.newline_msg("ERR", "only one db can be loaded at a time", 2)
return
self.current_param_db.update_results_from_data(args[0], sep=self.sep)
def add_setting(self, var, line):
possible_keys = set(["type","label","help","scale","repeat","datatype", "lim"])
if var not in self.settings:
self.settings[ var ] = {}
ret_name = self.settings[ var ]
l = [ i.strip() for i in line.split(":")]
for o in l:
k,v = o.split("=")
k=k.strip()
v=v.strip()
if k not in possible_keys:
spgu.newline_msg("SYN","in column '%s', unrecognised key '%s'"%(var,k))
sys.exit(1)
if k == "lim":
ret_name[k] = eval(v)
else:
ret_name[k]=v
def __init__(self, parameter_file):
self.parameter_file = parameter_file
self.base_name, foo = os.path.splitext(parameter_file)
self.mit = spgb.MultIteratorParser(open(parameter_file))
self.variables = self.mit.get_variables()
try:
self.settings, foo = self.get_settings(self.mit.command, "input")
except:
self.settings = {}
newline_msg("INF", "no 'input' file found: %s")
settings_output, self.output_columns = self.get_settings(
self.mit.command, "stdout")
newline_msg("INF", "output columns: %s" % self.output_columns)
self.settings.update(settings_output)
self.x_axis = self.variables[-1]
x_axis_iter = self.mit.data[self.mit.position_of(self.x_axis)]
if x_axis_iter.type == "*":
self.add_setting(self.x_axis, "scale = log")
self.add_setting(
self.x_axis,
"lim = (%s,%s)" % (x_axis_iter.xmin, x_axis_iter.xmax))
try:
self.coalesced_vars = [self.variables[-2]]
except:
self.coalesced_vars = []
try:
self.separated_vars = self.variables[:-2]
except:
self.separated_vars = []
def do_set_table(self, c):
"""sets the output table
sets a value in the currently loaded database
If key = sep (separator for csv table) "blank" means a single space"""
c = c.strip()
if c == "help":
utils.newline_msg("HELP",
" possible_keys = %s" % self.possible_keys)
return
if not self.current_param_db:
utils.newline_msg("WRN", "current db not set... skipping", 2)
return
if c not in list(self.current_param_db.table_columns.keys()):
utils.newline_msg(
"ERR", "table '%s' not among the ones found in the DB: (%s)" %
(c, ", ".join(list(
self.current_param_db.table_columns.keys()))))
return
self.table_columns = self.current_param_db.table_columns[c][1:]
