def values_set_table(self):
id_cols = ["spg_uid"]
local_vars_in_table = self.entities
local_var_query = ""
if len(local_vars_in_table) > 0:
local_var_query = "%s " % ",".join(["v.%s" % v for v in local_vars_in_table])
query = "SELECT rs.spg_vsid, rs.spg_rep, rs.id, %s FROM run_status AS rs, values_set AS v WHERE rs.spg_vsid = v.id " % (local_var_query)
query = query.replace("''", "'").replace("'\"", "'")
# print query
base_table = self.execute_query(query)
query_cols = ["spg_vsid", "spg_rep", "spg_uid"] + local_vars_in_table
ret_table = [
[ utils.replace_values(el, dict( list(zip( query_cols, row )) ) ) for el in row[2:] ]
for row in base_table ]
header = id_cols + local_vars_in_table
return header, ret_table
def launch_process(self, configuration_filename):
os.chdir(self.path)
if self.create_tree():
dir_n = utils.generate_string(self.values, self.variables, joining_string = "/")
if not os.path.exists(dir_n):
os.makedirs(dir_n)
os.chdir(dir_n)
# configuration_filename = "input_%s_%d.dat"%(self.db_name, self.current_run_id)
fconf = open(configuration_filename,"w")
for k in self.values.keys():
print >> fconf, k, utils.replace_values(self.values[k], self.values)
fconf.close()
cmd = "%s/%s -i %s"%(BINARY_PATH, self.command, configuration_filename )
proc = Popen(cmd, shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE )
# poll = proc.poll()
# while poll is None:
# time.sleep(1)
# utils.newline_msg( "SLP", "%s -- %s -- %s"%(cmd,self.path,poll))
# poll = proc.poll()
# print self.command, self.path, self.db_name, configuration_filename , self.values, <$$$$$$$
# print self.current_run_id, self.current_variables_id, self.entities, configuration_filename
self.return_code = proc.wait()
# print self.return_code
self.output = [i.strip() for i in proc.stdout.readline().split()]
self.stderr = "\n".join([i.strip() for i in proc.stderr.readline().split()])
# self.return_code = 0
# self.output = ""
os.remove(configuration_filename)
def values_set_table(self):
id_cols = ["spg_uid"]
local_vars_in_table = self.entities
local_var_query = ""
if len(local_vars_in_table) > 0:
local_var_query = "%s " % ",".join(["v.%s" % v for v in local_vars_in_table])
query = "SELECT rs.spg_vsid, rs.spg_rep, rs.id, %s FROM run_status AS rs, values_set AS v WHERE rs.spg_vsid = v.id " % (local_var_query)
query = query.replace("''", "'").replace("'\"", "'")
# print query
base_table = self.execute_query(query)
query_cols = ["spg_vsid", "spg_rep", "spg_uid"] + local_vars_in_table
ret_table = [
[ utils.replace_values(el, dict( zip( query_cols, row ) ) ) for el in row[2:] ]
for row in base_table ]
header = id_cols + local_vars_in_table
return header, ret_table
def launch_process(self, remove_files = True):
os.chdir(self.path)
configuration_filename = "%s_%d.tmp_input" % (self.base_name, self.current_spg_uid)
fconf = open(configuration_filename, "w")
for k in self.values.keys():
print >> fconf, k, utils.replace_values(self.values[k], self)
fconf.close()
fname_stdout = "%s_%s.tmp_stdout"% (self.base_name, self.current_spg_uid)
fname_stderr = "%s_%s.tmp_stderr"% (self.base_name, self.current_spg_uid)
file_stdout = open(fname_stdout, "w")
file_stderr = open(fname_stderr, "w")
cmd = "%s/%s %s" % (self.bin_dir, self.command, configuration_filename)
started_time = time.time()
proc = Popen(cmd, shell=True, stdin=PIPE, stdout=file_stdout, stderr=file_stderr, cwd=self.path, preexec_fn = preexec_function) #, env = {'PYTHONPATH':"${PYTHONPATH}:%s"%ROOT_DIR})
self.return_code = proc.wait()
finish_time = time.time()
file_stdout.close()
file_stderr.close()
self.output = [i.strip() for i in open(fname_stdout, "r")]
self.stderr = [i.strip() for i in open(fname_stderr, "r")]
if remove_files:
os.remove(configuration_filename)
os.remove( fname_stdout )
os.remove( fname_stderr )
self.run_time = finish_time - started_time
if self.run_time < 0: self.run_time = None
def launch_process(self):
# pwd = os.path.abspath(".")
# if self.directory_vars or self.create_trees():
# dir = utils.generate_string(self.values,self.directory_vars, joining_string = "/")
# if not os.path.exists(dir): os.makedirs(dir)
# os.chdir(dir)
configuration_filename = "input_%.8d.dat" % (self.current_valuesset_id)
fconf = open(configuration_filename, "w")
for k in self.values.keys():
print >> fconf, k, utils.replace_values(self.values[k], self.values)
fconf.close()
def launch_process(self):
pwd = os.path.abspath(".")
if self.create_trees():
dir = utils.generate_string(self.values, self.directory_vars, joining_string="/")
if not os.path.exists(dir):
os.makedirs(dir)
os.chdir(dir)
configuration_filename = "input_%.8d.dat" % (self.current_run_id)
output_filename = "output_%.8d.dat" % (self.current_run_id)
# print configuration_filename
fconf = open(configuration_filename, "w")
for k in self.values.keys():
print >> fconf, k, utils.replace_values(self.values[k], self.values)
fconf.close()
# except:
# utils.newline_msg("WRN", "could not load '%s'"%configuration_filename)
# return
cmd = "%s/%s -i %s > %s" % (BINARY_PATH, self.command, configuration_filename, output_filename)
os.system(cmd)
ret_code = 0
# proc = Popen(cmd, shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE )
# proc.wait()
# ret_code = proc.returncode
try:
output = [i.strip() for i in open(output_filename).readline().split()]
os.remove(configuration_filename)
os.remove(output_filename)
except:
utils.newline_msg("WRN", "could not read '%s' - may it be other process accessed it" % output_filename)
return
if self.directory_vars:
os.chdir(pwd)
# if ret_code == 0:
all_d = [self.current_valuesset_id]
all_d.extend(output)
# print self.output_column, all_d
cc = "INSERT INTO results (values_set_id, %s) VALUES (%s) " % (
", ".join(self.output_column),
", ".join([str(i) for i in all_d]),
)
flog = open(self.full_name.replace("sqlite", "log"), "aw")
print >> flog, "%.8d: %s --> %s" % (self.current_valuesset_id, self.values, output)
try:
self.execute_query(cc)
self.execute_query('UPDATE run_status SET status ="D" WHERE id = %d' % self.current_run_id)
except:
self.execute_query('UPDATE run_status SET status ="E" WHERE id = %d' % self.current_run_id)
def launch_process(self, current_run_id, current_vsid, current_rep, values):
os.chdir(self.path)
configuration_filename = "%s_%d.tmp_input" % (self.base_name,current_run_id)
fconf = open(configuration_filename, "w")
for k in list(self.values.keys()):
print(k, utils.replace_values(values[k], self), file=fconf)
fconf.close()
fname_stdout = "%s_%s.tmp_stdout" % (self.base_name, current_run_id)
fname_stderr = "%s_%s.tmp_stderr" % (self.base_name, current_run_id)
file_stdout = open(fname_stdout, "w")
file_stderr = open(fname_stderr, "w")
started_time = time.time()
cmd = "%s/%s %s" % (self.bin_dir, self.command, configuration_filename)
proc = Popen(cmd, shell=True, stdin=PIPE, stdout=file_stdout, stderr=file_stderr, cwd=self.path, preexec_fn = preexec_function)
return_code = proc.wait()
finish_time = time.time()
file_stdout.close()
file_stderr.close()
output = [i.strip() for i in open(fname_stdout, "r")]
stderr = [i.strip() for i in open(fname_stderr, "r")]
if self.test_run:
if not os.path.exists("configuration-%s"%self.base_name):
os.mkdir("configuration-%s"%self.base_name)
os.rename(configuration_filename, "configuration-%s/%s"%(self.base_name,configuration_filename))
os.rename(fname_stdout, "configuration-%s/%s" % (self.base_name, fname_stdout))
os.rename(fname_stderr, "configuration-%s/%s" % (self.base_name, fname_stderr))
else:
os.remove(configuration_filename)
os.remove(fname_stdout)
os.remove(fname_stderr)
run_time = finish_time - started_time
if run_time < 0: run_time = None
return current_run_id, current_vsid, current_rep, output, stderr, run_time, return_code
def launch_process(self, current_run_id, current_vsid, current_rep, values):
os.chdir(self.path)
configuration_filename = "%s_%d.tmp_input" % (self.base_name,current_run_id)
fconf = open(configuration_filename, "w")
for k in self.values.keys():
print >> fconf, k, utils.replace_values(values[k], self)
fconf.close()
fname_stdout = "%s_%s.tmp_stdout" % (self.base_name, current_run_id)
fname_stderr = "%s_%s.tmp_stderr" % (self.base_name, current_run_id)
file_stdout = open(fname_stdout, "w")
file_stderr = open(fname_stderr, "w")
started_time = time.time()
cmd = "%s/%s %s" % (self.bin_dir, self.command, configuration_filename)
proc = Popen(cmd, shell=True, stdin=PIPE, stdout=file_stdout, stderr=file_stderr, cwd=self.path, preexec_fn = preexec_function)
return_code = proc.wait()
finish_time = time.time()
file_stdout.close()
file_stderr.close()
output = [i.strip() for i in open(fname_stdout, "r")]
stderr = [i.strip() for i in open(fname_stderr, "r")]
if self.test_run:
if not os.path.exists("configuration-%s"%self.base_name):
os.mkdir("configuration-%s"%self.base_name)
os.rename(configuration_filename, "configuration-%s/%s"%(self.base_name,configuration_filename))
os.rename(fname_stdout, "configuration-%s/%s" % (self.base_name, fname_stdout))
os.rename(fname_stderr, "configuration-%s/%s" % (self.base_name, fname_stderr))
else:
os.remove(configuration_filename)
os.remove(fname_stdout)
os.remove(fname_stderr)
run_time = finish_time - started_time
if run_time < 0: run_time = None
return current_run_id, current_vsid, current_rep, output, stderr, run_time, return_code
def launch_process(self, remove_files = True):
os.chdir(self.path)
configuration_filename = "%s_%d.tmp_input" % (self.base_name, self.current_spg_uid)
fconf = open(configuration_filename, "w")
for k in list(self.values.keys()):
print(k, utils.replace_values(self.values[k], self), file=fconf)
fconf.close()
fname_stdout = "%s_%s.tmp_stdout"% (self.base_name, self.current_spg_uid)
fname_stderr = "%s_%s.tmp_stderr"% (self.base_name, self.current_spg_uid)
file_stdout = open(fname_stdout, "w")
file_stderr = open(fname_stderr, "w")
cmd = "%s/%s %s" % (self.bin_dir, self.command, configuration_filename)
started_time = time.time()
proc = Popen(cmd, shell=True, stdin=PIPE, stdout=file_stdout, stderr=file_stderr, cwd=self.path, preexec_fn = preexec_function) #, env = {'PYTHONPATH':"${PYTHONPATH}:%s"%ROOT_DIR})
self.return_code = proc.wait()
finish_time = time.time()
file_stdout.close()
file_stderr.close()
self.output = [i.strip() for i in open(fname_stdout, "r")]
# print("OUT <<< ", self.output)
self.stderr = [i.strip() for i in open(fname_stderr, "r")]
if remove_files:
os.remove(configuration_filename)
os.remove( fname_stdout )
os.remove( fname_stderr )
self.run_time = finish_time - started_time
if self.run_time < 0: self.run_time = None
def init_db(self):
#:::~ Table with the information related to the database
self.cursor.execute(
"CREATE TABLE IF NOT EXISTS information "
"(id INTEGER PRIMARY KEY, key CHAR(64), value CHAR(128))")
self.check_and_insert_information('version', database_version)
self.check_and_insert_information('command', self.command)
self.check_and_insert_information('path', self.path)
self.check_and_insert_information('base_name', self.base_name)
#:::~ Table with the defined entities
self.cursor.execute(
"CREATE TABLE IF NOT EXISTS entities "
"(id INTEGER PRIMARY KEY, name CHAR(64), varies INTEGER)")
self.cursor.execute("SELECT COUNT(*) FROM entities ")
n_items = self.cursor.fetchone()
if n_items[0] == 0: # table has not been filled
for i in self.data:
varies = 1 if (i.__class__ != IterConstant) else 0
self.cursor.execute(
"INSERT INTO entities (name, varies) VALUES (?,?)",
(i.name, varies))
else:
self.cursor.execute("SELECT name FROM entities ")
entities = set([i[0] for i in self.cursor])
s_names = set(self.names)
if entities != set(self.names):
utils.newline_msg(
"ERR", "parameter (was %s, is %s)" % (entities, s_names))
sys.exit(1)
self.connection.commit()
elements = "CREATE TABLE IF NOT EXISTS values_set (id INTEGER PRIMARY KEY, %s )" % (
", ".join(["%s CHAR(64) " % i for i in self.names]))
# print elements
self.cursor.execute(elements)
elements = "INSERT INTO values_set ( %s ) VALUES (%s)" % (", ".join(
["%s " % i for i in self.names]), ", ".join("?"
for i in self.names))
# :::~ (CT) Index creation code
for i in self.data:
if i.__class__ == IterConstant: continue
self.cursor.execute(
"CREATE INDEX IF NOT EXISTS idxvs_%s_id ON values_set (%s) " %
(i.name, i.name))
self.possible_varying_ids = []
for i in self:
self.cursor.execute(elements, [
utils.replace_values(self[j],
self,
vars_to_skip=set(
['spg_uid', 'spg_vsid', 'spg_rep']))
for j in self.names
])
self.possible_varying_ids.append(self.cursor.lastrowid)
self.connection.commit()
self.cursor.execute(
"CREATE TABLE IF NOT EXISTS output_tables (id INTEGER PRIMARY KEY, name CHAR(128) , column CHAR(128) )"
)
for results_table in list(self.stdout_contents.keys()):
table_contents = self.stdout_contents[results_table]
# self.number_of_columns = 0
# for ic, iv in table_contents:
# if iv["type"] == "xy":
# self.number_of_columns += 1
# if iv["type"] == "xydy":
# self.number_of_columns += 2
results = "CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY, spg_uid INTEGER, spg_vsid INTEGER, spg_rep INTEGER, %s, FOREIGN KEY(spg_uid) REFERENCES run_status(id), FOREIGN KEY(spg_vsid) REFERENCES values_set(id))" % (
results_table, ", ".join([
"%s %s" % (ic, iv["datatype"]) for ic, iv in table_contents
]))
self.cursor.execute(results)
self.connection.commit()
for results_table in list(self.stdout_contents.keys()):
# self.cursor.execute("INSERT INTO output_tables ( name, column ) VALUES (?, ?)",( results_table, 'spg_runid' ) )
table_contents = self.stdout_contents[results_table]
for ic, iv in table_contents:
self.cursor.execute(
"INSERT INTO output_tables ( name, column ) VALUES (?, ?)",
(results_table, ic))
self.connection.commit()
self.cursor.execute(
"CREATE TABLE IF NOT EXISTS run_status (id INTEGER PRIMARY KEY, spg_vsid INTEGER, spg_rep INTEGER, status CHAR(1), run_time FLOAT, "
"FOREIGN KEY (spg_vsid) REFERENCES values_set(id) )")
self.connection.commit()
def init_db(self):
#:::~ Table with the information related to the database
self.cursor.execute("CREATE TABLE IF NOT EXISTS information "
"(id INTEGER PRIMARY KEY, key CHAR(64), value CHAR(128))"
)
self.check_and_insert_information('version', database_version)
self.check_and_insert_information('command', self.command )
self.check_and_insert_information('path', self.path)
self.check_and_insert_information('base_name', self.base_name)
#:::~ Table with the defined entities
self.cursor.execute("CREATE TABLE IF NOT EXISTS entities "
"(id INTEGER PRIMARY KEY, name CHAR(64), varies INTEGER)"
)
self.cursor.execute( "SELECT COUNT(*) FROM entities ")
n_items = self.cursor.fetchone()
if n_items[0] == 0: # table has not been filled
for i in self.data:
varies = 1 if (i.__class__ != IterConstant) else 0
self.cursor.execute( "INSERT INTO entities (name, varies) VALUES (?,?)",(i.name, varies) )
else:
self.cursor.execute( "SELECT name FROM entities ")
entities = [i[0] for i in self.cursor]
s_names = set(self.names)
if entities != set(self.names):
utils.newline_msg("ERR", "parameter (was %s, is %s)"%(entities, s_names))
sys.exit(1)
self.connection.commit()
elements = "CREATE TABLE IF NOT EXISTS values_set (id INTEGER PRIMARY KEY, %s )"%( ", ".join([ "%s CHAR(64) "%i for i in self.names ] ) )
# print elements
self.cursor.execute(elements)
elements = "INSERT INTO values_set ( %s ) VALUES (%s)"%( ", ".join([ "%s "%i for i in self.names ] ), ", ".join( "?" for i in self.names ) )
# :::~ (CT) Index creation code
for i in self.data:
if i.__class__ == IterConstant: continue
self.cursor.execute( "CREATE INDEX IF NOT EXISTS idxvs_%s_id ON values_set (%s) "%(i.name,i.name) )
self.possible_varying_ids = []
for i in self:
self.cursor.execute( elements, [ utils.replace_values(self[j], self, vars_to_skip = set(['spg_uid','spg_vsid','spg_rep'] ) ) for j in self.names] )
self.possible_varying_ids.append(self.cursor.lastrowid)
self.connection.commit()
self.cursor.execute("CREATE TABLE IF NOT EXISTS output_tables (id INTEGER PRIMARY KEY, name CHAR(128) , column CHAR(128) )")
for results_table in self.stdout_contents.keys():
table_contents = self.stdout_contents[ results_table ]
# self.number_of_columns = 0
# for ic, iv in table_contents:
# if iv["type"] == "xy":
# self.number_of_columns += 1
# if iv["type"] == "xydy":
# self.number_of_columns += 2
results = "CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY, spg_uid INTEGER, spg_vsid INTEGER, spg_rep INTEGER, %s, FOREIGN KEY(spg_uid) REFERENCES run_status(id), FOREIGN KEY(spg_vsid) REFERENCES values_set(id))"%(results_table, ", ".join([ "%s %s"%(ic,iv["datatype"]) for ic, iv in table_contents ]) )
self.cursor.execute(results)
self.connection.commit()
for results_table in self.stdout_contents.keys():
# self.cursor.execute("INSERT INTO output_tables ( name, column ) VALUES (?, ?)",( results_table, 'spg_runid' ) )
table_contents = self.stdout_contents[ results_table ]
for ic, iv in table_contents:
self.cursor.execute("INSERT INTO output_tables ( name, column ) VALUES (?, ?)",( results_table,ic ) )
self.connection.commit()
self.cursor.execute("CREATE TABLE IF NOT EXISTS run_status (id INTEGER PRIMARY KEY, spg_vsid INTEGER, spg_rep INTEGER, status CHAR(1), run_time FLOAT, "
"FOREIGN KEY (spg_vsid) REFERENCES values_set(id) )")
self.connection.commit()
def init_db(self):
#:::~ Table with the name of the executable
self.cursor.execute("CREATE TABLE IF NOT EXISTS executable "
"(id INTEGER PRIMARY KEY, name CHAR(64))"
)
self.cursor.execute( "SELECT name FROM executable " )
prev_val = self.cursor.fetchone()
if prev_val :
if prev_val[0] != self.command:
utils.newline_msg("ERR","conflict in executable name (in db '%s', in param '%s')"%(prev_val, self.command))
else:
self.cursor.execute("INSERT INTO executable (name) VALUES (?)",(self.command,))
self.connection.commit()
#:::~ Table with the defined entities
self.cursor.execute("CREATE TABLE IF NOT EXISTS entities "
"(id INTEGER PRIMARY KEY, name CHAR(64), varies INTEGER)"
)
self.cursor.execute( "SELECT COUNT(*) FROM entities ")
n_items = self.cursor.fetchone()
if n_items[0] == 0: # table has not been filled
for i in self.data:
varies = 1 if (i.__class__ != IterConstant) else 0
self.cursor.execute( "INSERT INTO entities (name, varies) VALUES (?,?)",(i.name, varies) )
else:
self.cursor.execute( "SELECT name FROM entities ")
entities = set([i[0] for i in self.cursor])
s_names = set(self.names)
if entities != set(self.names):
utils.newline_msg("ERR", "parameter (was %s, is %s)"%(entities, s_names))
sys.exit(1)
self.connection.commit()
elements = "CREATE TABLE IF NOT EXISTS values_set (id INTEGER PRIMARY KEY, %s )"%( ", ".join([ "%s CHAR(64)"%i for i in self.names ] ) )
self.cursor.execute(elements)
elements = "INSERT INTO values_set ( %s ) VALUES (%s)"%( ", ".join([ "%s "%i for i in self.names ] ), ", ".join( "?" for i in self.names ) )
self.possible_varying_ids = []
# i_try = 0
for i in self:
self.cursor.execute( elements, [ utils.replace_values(self[i], self) for i in self.names] )
self.possible_varying_ids.append(self.cursor.lastrowid)
self.connection.commit()
#if not commited:
# utils.newline_msg("ERR", "database didn't unlock, exiting")
self.number_of_columns = 0
for ic, iv in self.stdout_contents:
if iv["type"] == "xy":
self.number_of_columns += 1
if iv["type"] == "xydy":
self.number_of_columns += 2
results = "CREATE TABLE IF NOT EXISTS results (id INTEGER PRIMARY KEY, values_set_id INTEGER, %s , FOREIGN KEY(values_set_id) REFERENCES values_set(id))"%( ", ".join([ "%s CHAR(64)"%ic for ic, iv in self.stdout_contents ] ) )
self.cursor.execute(results)
self.connection.commit()
self.cursor.execute("CREATE TABLE IF NOT EXISTS run_status (id INTEGER PRIMARY KEY, values_set_id INTEGER, status CHAR(1), "
"FOREIGN KEY (values_set_id ) REFERENCES values_set(id) )")
self.connection.commit()
