def do_save_table(self,c):
"""saves the table of values"""
for i in self.current_param_db:
if self.split_columns:
for column in self.output_column:
gen_d = utils.generate_string(i, self.current_param_db.separated_vars, joining_string = "/" )
if gen_d : gen_d+= "/"
gen_s = utils.generate_string(i, self.current_param_db.coalesced_vars, joining_string = "_" )
output_fname = "%s%s-%s-%s.dat"%(gen_d, self.prefix, column, gen_s)
output_fname = output_fname.replace("_-","_")
output_fname = output_fname.replace("-.",".")
d,f = os.path.split(output_fname)
if d != "" and not os.path.exists(d): os.makedirs(d)
data = self.current_param_db.result_table(restrict_to_values = i, raw_data = self.raw_data, restrict_by_val = self.restrict_by_val, output_column = [column] )
np.savetxt( output_fname, data)
else:
data = self.current_param_db.result_table(restrict_to_values = i, raw_data = self.raw_data, restrict_by_val = self.restrict_by_val, output_column = self.output_column )
gen_d = utils.generate_string(i, self.current_param_db.separated_vars, joining_string = "/" )
if gen_d : gen_d+= "/"
gen_s = utils.generate_string(i, self.current_param_db.coalesced_vars, joining_string = "_" )
output_fname = "%s%s-%s.dat"%(gen_d, self.prefix, gen_s)
output_fname = output_fname.replace("-.",".")
output_fname = output_fname.replace("_-","_")
d,f = os.path.split(output_fname)
if d != "" and not os.path.exists(d): os.makedirs(d)
np.savetxt( output_fname, data)
def launch_process(self, configuration_filename):
os.chdir(self.path)
if self.create_tree():
dir_n = utils.generate_string(self.values, self.variables, joining_string = "/")
if not os.path.exists(dir_n):
os.makedirs(dir_n)
os.chdir(dir_n)
# configuration_filename = "input_%s_%d.dat"%(self.db_name, self.current_run_id)
fconf = open(configuration_filename,"w")
for k in self.values.keys():
print >> fconf, k, utils.replace_in_string(self.values[k], self.values)
fconf.close()
cmd = "%s/%s -i %s"%(BINARY_PATH, self.command, configuration_filename )
proc = Popen(cmd, shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE )
# poll = proc.poll()
# while poll is None:
# time.sleep(1)
# utils.newline_msg( "SLP", "%s -- %s -- %s"%(cmd,self.path,poll))
# poll = proc.poll()
# print self.command, self.path, self.db_name, configuration_filename , self.values, <$$$$$$$
# print self.current_run_id, self.current_variables_id, self.entities, configuration_filename
self.return_code = proc.wait()
# print self.return_code
self.output = [i.strip() for i in proc.stdout.readline().split()]
self.stderr = "\n".join([i.strip() for i in proc.stderr.readline().split()])
# self.return_code = 0
# self.output = ""
os.remove(configuration_filename)
def __plot(self):
try:
self.dict_of_params = utils.load_config( "%s/spg-conf/%s.params"%(CONFIG_DIR, self.current_param_db.command[4:-3] ), "texlabel" )
except: self.dict_of_params = {}
try:
self.dict_of_stdout = utils.load_config( "%s/spg-conf/%s.stdout"%(CONFIG_DIR, self.current_param_db.command[4:-3] ), "texlabel" )
except: self.dict_of_stdout = {}
# for oc in self.figures.keys():
# plt.close( self.figures[oc] )
plt.clf()
self.figures = {}
for i_restrict in self.current_param_db:
fig_label = utils.generate_string(i_restrict, self.current_param_db.separated_vars, separator = "=", joining_string = " " )
if not self.figures.has_key(fig_label):
self.figures[ fig_label ] = PyplotGraphicsUnit(fig_label, self.n_rows, len(self.output_column) )
for column in self.output_column:
print column, self.n_rows, len(self.output_column)
self.figures[ fig_label ].add_subplot(column)
self.figures[ fig_label ].subplots[column].x_label = self.current_param_db.variables[-1]
self.figures[ fig_label ].subplots[column].y_label = column
self.figures[ fig_label ].subplots[column].x_scale = self.plot_x_scale
self.figures[ fig_label ].subplots[column].y_scale = self.plot_y_scale
self.figures[ fig_label ].subplots[column].x_range = (self.plot_x_min, self.plot_x_max)
self.figures[ fig_label ].subplots[column].y_range = (self.plot_y_min, self.plot_y_max)
self.figures[ fig_label ].subplots[column].refresh_style()
if self.dict_of_stdout.has_key(column):
self.figures[ fig_label ].subplots[column].plot_object.set_ylabel( "$%s$"%self.dict_of_stdout[column] )
else:
self.figures[ fig_label ].subplots[column].plot_object.set_ylabel( "$%s$"%column )
if self.dict_of_params.has_key(self.current_param_db.in_table_vars[0] ):
self.figures[ fig_label ].subplots[column].plot_object.set_xlabel( "$%s$"%self.dict_of_params[ self.current_param_db.in_table_vars[0] ] )
else:
self.figures[ fig_label ].subplots[column].plot_object.set_xlabel( "$%s$"%self.current_param_db.in_table_vars[0] )
for column in self.output_column:# self.figures[column] = plt.figure()
curve_label = utils.generate_string(i_restrict, self.current_param_db.coalesced_vars, separator = "=", joining_string = " " )
data = self.current_param_db.result_table(restrict_to_values = i_restrict, raw_data = self.raw_data, restrict_by_val = self.restrict_by_val, output_column = [column] )
# print data
self.figures[fig_label].subplots[column].add_curve( curve_label, data[:,0], data[:,1] )
def launch_process(self):
pwd = os.path.abspath(".")
if self.create_trees():
dir = utils.generate_string(self.values, self.directory_vars, joining_string="/")
if not os.path.exists(dir):
os.makedirs(dir)
os.chdir(dir)
configuration_filename = "input_%.8d.dat" % (self.current_run_id)
output_filename = "output_%.8d.dat" % (self.current_run_id)
# print configuration_filename
fconf = open(configuration_filename, "w")
for k in self.values.keys():
print >> fconf, k, utils.replace_values(self.values[k], self.values)
fconf.close()
# except:
# utils.newline_msg("WRN", "could not load '%s'"%configuration_filename)
# return
cmd = "%s/%s -i %s > %s" % (BINARY_PATH, self.command, configuration_filename, output_filename)
os.system(cmd)
ret_code = 0
# proc = Popen(cmd, shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE )
# proc.wait()
# ret_code = proc.returncode
try:
output = [i.strip() for i in open(output_filename).readline().split()]
os.remove(configuration_filename)
os.remove(output_filename)
except:
utils.newline_msg("WRN", "could not read '%s' - may it be other process accessed it" % output_filename)
return
if self.directory_vars:
os.chdir(pwd)
# if ret_code == 0:
all_d = [self.current_valuesset_id]
all_d.extend(output)
# print self.output_column, all_d
cc = "INSERT INTO results (values_set_id, %s) VALUES (%s) " % (
", ".join(self.output_column),
", ".join([str(i) for i in all_d]),
)
flog = open(self.full_name.replace("sqlite", "log"), "aw")
print >> flog, "%.8d: %s --> %s" % (self.current_valuesset_id, self.values, output)
try:
self.execute_query(cc)
self.execute_query('UPDATE run_status SET status ="D" WHERE id = %d' % self.current_run_id)
except:
self.execute_query('UPDATE run_status SET status ="E" WHERE id = %d' % self.current_run_id)
def do_save_table(self,c):
"""save_table [-flag1 -flag2]
saves the table values in ascii format. default is averaging column values by variable values
FLAGS::: --noheader: does not output column names in the output file
--raw: raw values without average
--full: all simulation ids and variables are output in table (implies raw)
--id: only the id is output
--sep: column separator ('blank' for space)
"""
flags, args = self.parse_command_line(c)
if not self.current_param_db:
if len(args) == 0:
utils.newline_msg("WRN", "database not loaded nor provided. skipping", 2)
return
elif len(args) > 1:
utils.newline_msg("ERR", "only one db can be loaded at a time", 2)
return
else:
self.do_load(args[0])
else:
if len(args) > 0:
utils.newline_msg("ERR", "only one db can be loaded at a time", 2)
return
if len( set([ 'raw', "full", "id" ]).intersection( flags ) ) > 1:
utils.newline_msg("ERR", "only one flag [raw, full, id] can be active at a time" )
return
table_selector = "grouped_vars"
if "raw" in flags:
table_selector = "raw_vars"
elif "id" in flags:
table_selector = "only_uid"
elif "full" in flags:
table_selector = "full"
if "sep" in flags:
self.do_set_separator(flags["sep"])
for outer_params in self.current_param_db:
for table in self.current_param_db.table_columns.keys():
# print i
gen_d = utils.generate_string(outer_params, self.current_param_db.separated_vars, joining_string = "/" )
if gen_d:
gen_d+= "/"
gen_s = utils.generate_string(outer_params, self.current_param_db.coalesced_vars, joining_string="_")
output_fname = utils.fix_filename("%s%s_%s-%s.csv" % (gen_d, self.current_param_db.base_name, table, gen_s))
d,f = os.path.split(output_fname)
if d != "" and not os.path.exists(d): os.makedirs(d)
header, data= self.current_param_db.result_table(table = table , table_selector = table_selector)
print " +- table: '%s'" % (output_fname)
writer = csv.writer(open(output_fname, "w"), delimiter=self.sep, lineterminator="\n")
if not( "noheader" in flags or "append" in flags):
writer.writerow( header )
writer.writerows( data )
def do_save_table(self, c):
"""save_table [-flag1 -flag2]
saves the table values in ascii format. default is averaging column values by variable values
FLAGS::: --noheader: does not output column names in the output file
--raw: raw values without average
--full: all simulation ids and variables are output in table (implies raw)
--id: only the id is output
--sep: column separator ('blank' for space)
"""
flags, args = self.parse_command_line(c)
if not self.current_param_db:
if len(args) == 0:
utils.newline_msg(
"WRN", "database not loaded nor provided. skipping", 2)
return
elif len(args) > 1:
utils.newline_msg("ERR", "only one db can be loaded at a time",
2)
return
else:
self.do_load(args[0])
else:
if len(args) > 0:
utils.newline_msg("ERR", "only one db can be loaded at a time",
2)
return
if len(set(['raw', "full", "id"]).intersection(flags)) > 1:
utils.newline_msg(
"ERR", "only one flag [raw, full, id] can be active at a time")
return
table_selector = "grouped_vars"
if "raw" in flags:
table_selector = "raw_vars"
elif "id" in flags:
table_selector = "only_uid"
elif "full" in flags:
table_selector = "full"
if "sep" in flags:
self.do_set_separator(flags["sep"])
for outer_params in self.current_param_db:
for table in list(self.current_param_db.table_columns.keys()):
# print i
gen_d = utils.generate_string(
outer_params,
self.current_param_db.separated_vars,
joining_string="/")
if gen_d:
gen_d += "/"
gen_s = utils.generate_string(
outer_params,
self.current_param_db.coalesced_vars,
joining_string="_")
output_fname = utils.fix_filename(
"%s%s_%s-%s.csv" %
(gen_d, self.current_param_db.base_name, table, gen_s))
d, f = os.path.split(output_fname)
if d != "" and not os.path.exists(d): os.makedirs(d)
header, data = self.current_param_db.result_table(
table=table, table_selector=table_selector)
print(" +- table: '%s'" % (output_fname))
writer = csv.writer(open(output_fname, "w"),
delimiter=self.sep,
lineterminator="\n")
if not ("noheader" in flags or "append" in flags):
writer.writerow(header)
writer.writerows(data)