def values_set_table(self): id_cols = ["spg_uid"] local_vars_in_table = self.entities local_var_query = "" if len(local_vars_in_table) > 0: local_var_query = "%s " % ",".join(["v.%s" % v for v in local_vars_in_table]) query = "SELECT rs.spg_vsid, rs.spg_rep, rs.id, %s FROM run_status AS rs, values_set AS v WHERE rs.spg_vsid = v.id " % (local_var_query) query = query.replace("''", "'").replace("'\"", "'") # print query base_table = self.execute_query(query) query_cols = ["spg_vsid", "spg_rep", "spg_uid"] + local_vars_in_table ret_table = [ [ utils.replace_values(el, dict( list(zip( query_cols, row )) ) ) for el in row[2:] ] for row in base_table ] header = id_cols + local_vars_in_table return header, ret_table
def launch_process(self, configuration_filename): os.chdir(self.path) if self.create_tree(): dir_n = utils.generate_string(self.values, self.variables, joining_string = "/") if not os.path.exists(dir_n): os.makedirs(dir_n) os.chdir(dir_n) # configuration_filename = "input_%s_%d.dat"%(self.db_name, self.current_run_id) fconf = open(configuration_filename,"w") for k in self.values.keys(): print >> fconf, k, utils.replace_values(self.values[k], self.values) fconf.close() cmd = "%s/%s -i %s"%(BINARY_PATH, self.command, configuration_filename ) proc = Popen(cmd, shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE ) # poll = proc.poll() # while poll is None: # time.sleep(1) # utils.newline_msg( "SLP", "%s -- %s -- %s"%(cmd,self.path,poll)) # poll = proc.poll() # print self.command, self.path, self.db_name, configuration_filename , self.values, <$$$$$$$ # print self.current_run_id, self.current_variables_id, self.entities, configuration_filename self.return_code = proc.wait() # print self.return_code self.output = [i.strip() for i in proc.stdout.readline().split()] self.stderr = "\n".join([i.strip() for i in proc.stderr.readline().split()]) # self.return_code = 0 # self.output = "" os.remove(configuration_filename)
def values_set_table(self): id_cols = ["spg_uid"] local_vars_in_table = self.entities local_var_query = "" if len(local_vars_in_table) > 0: local_var_query = "%s " % ",".join(["v.%s" % v for v in local_vars_in_table]) query = "SELECT rs.spg_vsid, rs.spg_rep, rs.id, %s FROM run_status AS rs, values_set AS v WHERE rs.spg_vsid = v.id " % (local_var_query) query = query.replace("''", "'").replace("'\"", "'") # print query base_table = self.execute_query(query) query_cols = ["spg_vsid", "spg_rep", "spg_uid"] + local_vars_in_table ret_table = [ [ utils.replace_values(el, dict( zip( query_cols, row ) ) ) for el in row[2:] ] for row in base_table ] header = id_cols + local_vars_in_table return header, ret_table
def launch_process(self, remove_files = True): os.chdir(self.path) configuration_filename = "%s_%d.tmp_input" % (self.base_name, self.current_spg_uid) fconf = open(configuration_filename, "w") for k in self.values.keys(): print >> fconf, k, utils.replace_values(self.values[k], self) fconf.close() fname_stdout = "%s_%s.tmp_stdout"% (self.base_name, self.current_spg_uid) fname_stderr = "%s_%s.tmp_stderr"% (self.base_name, self.current_spg_uid) file_stdout = open(fname_stdout, "w") file_stderr = open(fname_stderr, "w") cmd = "%s/%s %s" % (self.bin_dir, self.command, configuration_filename) started_time = time.time() proc = Popen(cmd, shell=True, stdin=PIPE, stdout=file_stdout, stderr=file_stderr, cwd=self.path, preexec_fn = preexec_function) #, env = {'PYTHONPATH':"${PYTHONPATH}:%s"%ROOT_DIR}) self.return_code = proc.wait() finish_time = time.time() file_stdout.close() file_stderr.close() self.output = [i.strip() for i in open(fname_stdout, "r")] self.stderr = [i.strip() for i in open(fname_stderr, "r")] if remove_files: os.remove(configuration_filename) os.remove( fname_stdout ) os.remove( fname_stderr ) self.run_time = finish_time - started_time if self.run_time < 0: self.run_time = None
def launch_process(self): # pwd = os.path.abspath(".") # if self.directory_vars or self.create_trees(): # dir = utils.generate_string(self.values,self.directory_vars, joining_string = "/") # if not os.path.exists(dir): os.makedirs(dir) # os.chdir(dir) configuration_filename = "input_%.8d.dat" % (self.current_valuesset_id) fconf = open(configuration_filename, "w") for k in self.values.keys(): print >> fconf, k, utils.replace_values(self.values[k], self.values) fconf.close()
def launch_process(self): pwd = os.path.abspath(".") if self.create_trees(): dir = utils.generate_string(self.values, self.directory_vars, joining_string="/") if not os.path.exists(dir): os.makedirs(dir) os.chdir(dir) configuration_filename = "input_%.8d.dat" % (self.current_run_id) output_filename = "output_%.8d.dat" % (self.current_run_id) # print configuration_filename fconf = open(configuration_filename, "w") for k in self.values.keys(): print >> fconf, k, utils.replace_values(self.values[k], self.values) fconf.close() # except: # utils.newline_msg("WRN", "could not load '%s'"%configuration_filename) # return cmd = "%s/%s -i %s > %s" % (BINARY_PATH, self.command, configuration_filename, output_filename) os.system(cmd) ret_code = 0 # proc = Popen(cmd, shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE ) # proc.wait() # ret_code = proc.returncode try: output = [i.strip() for i in open(output_filename).readline().split()] os.remove(configuration_filename) os.remove(output_filename) except: utils.newline_msg("WRN", "could not read '%s' - may it be other process accessed it" % output_filename) return if self.directory_vars: os.chdir(pwd) # if ret_code == 0: all_d = [self.current_valuesset_id] all_d.extend(output) # print self.output_column, all_d cc = "INSERT INTO results (values_set_id, %s) VALUES (%s) " % ( ", ".join(self.output_column), ", ".join([str(i) for i in all_d]), ) flog = open(self.full_name.replace("sqlite", "log"), "aw") print >> flog, "%.8d: %s --> %s" % (self.current_valuesset_id, self.values, output) try: self.execute_query(cc) self.execute_query('UPDATE run_status SET status ="D" WHERE id = %d' % self.current_run_id) except: self.execute_query('UPDATE run_status SET status ="E" WHERE id = %d' % self.current_run_id)
def launch_process(self, current_run_id, current_vsid, current_rep, values): os.chdir(self.path) configuration_filename = "%s_%d.tmp_input" % (self.base_name,current_run_id) fconf = open(configuration_filename, "w") for k in list(self.values.keys()): print(k, utils.replace_values(values[k], self), file=fconf) fconf.close() fname_stdout = "%s_%s.tmp_stdout" % (self.base_name, current_run_id) fname_stderr = "%s_%s.tmp_stderr" % (self.base_name, current_run_id) file_stdout = open(fname_stdout, "w") file_stderr = open(fname_stderr, "w") started_time = time.time() cmd = "%s/%s %s" % (self.bin_dir, self.command, configuration_filename) proc = Popen(cmd, shell=True, stdin=PIPE, stdout=file_stdout, stderr=file_stderr, cwd=self.path, preexec_fn = preexec_function) return_code = proc.wait() finish_time = time.time() file_stdout.close() file_stderr.close() output = [i.strip() for i in open(fname_stdout, "r")] stderr = [i.strip() for i in open(fname_stderr, "r")] if self.test_run: if not os.path.exists("configuration-%s"%self.base_name): os.mkdir("configuration-%s"%self.base_name) os.rename(configuration_filename, "configuration-%s/%s"%(self.base_name,configuration_filename)) os.rename(fname_stdout, "configuration-%s/%s" % (self.base_name, fname_stdout)) os.rename(fname_stderr, "configuration-%s/%s" % (self.base_name, fname_stderr)) else: os.remove(configuration_filename) os.remove(fname_stdout) os.remove(fname_stderr) run_time = finish_time - started_time if run_time < 0: run_time = None return current_run_id, current_vsid, current_rep, output, stderr, run_time, return_code
def launch_process(self, current_run_id, current_vsid, current_rep, values): os.chdir(self.path) configuration_filename = "%s_%d.tmp_input" % (self.base_name,current_run_id) fconf = open(configuration_filename, "w") for k in self.values.keys(): print >> fconf, k, utils.replace_values(values[k], self) fconf.close() fname_stdout = "%s_%s.tmp_stdout" % (self.base_name, current_run_id) fname_stderr = "%s_%s.tmp_stderr" % (self.base_name, current_run_id) file_stdout = open(fname_stdout, "w") file_stderr = open(fname_stderr, "w") started_time = time.time() cmd = "%s/%s %s" % (self.bin_dir, self.command, configuration_filename) proc = Popen(cmd, shell=True, stdin=PIPE, stdout=file_stdout, stderr=file_stderr, cwd=self.path, preexec_fn = preexec_function) return_code = proc.wait() finish_time = time.time() file_stdout.close() file_stderr.close() output = [i.strip() for i in open(fname_stdout, "r")] stderr = [i.strip() for i in open(fname_stderr, "r")] if self.test_run: if not os.path.exists("configuration-%s"%self.base_name): os.mkdir("configuration-%s"%self.base_name) os.rename(configuration_filename, "configuration-%s/%s"%(self.base_name,configuration_filename)) os.rename(fname_stdout, "configuration-%s/%s" % (self.base_name, fname_stdout)) os.rename(fname_stderr, "configuration-%s/%s" % (self.base_name, fname_stderr)) else: os.remove(configuration_filename) os.remove(fname_stdout) os.remove(fname_stderr) run_time = finish_time - started_time if run_time < 0: run_time = None return current_run_id, current_vsid, current_rep, output, stderr, run_time, return_code
def launch_process(self, remove_files = True): os.chdir(self.path) configuration_filename = "%s_%d.tmp_input" % (self.base_name, self.current_spg_uid) fconf = open(configuration_filename, "w") for k in list(self.values.keys()): print(k, utils.replace_values(self.values[k], self), file=fconf) fconf.close() fname_stdout = "%s_%s.tmp_stdout"% (self.base_name, self.current_spg_uid) fname_stderr = "%s_%s.tmp_stderr"% (self.base_name, self.current_spg_uid) file_stdout = open(fname_stdout, "w") file_stderr = open(fname_stderr, "w") cmd = "%s/%s %s" % (self.bin_dir, self.command, configuration_filename) started_time = time.time() proc = Popen(cmd, shell=True, stdin=PIPE, stdout=file_stdout, stderr=file_stderr, cwd=self.path, preexec_fn = preexec_function) #, env = {'PYTHONPATH':"${PYTHONPATH}:%s"%ROOT_DIR}) self.return_code = proc.wait() finish_time = time.time() file_stdout.close() file_stderr.close() self.output = [i.strip() for i in open(fname_stdout, "r")] # print("OUT <<< ", self.output) self.stderr = [i.strip() for i in open(fname_stderr, "r")] if remove_files: os.remove(configuration_filename) os.remove( fname_stdout ) os.remove( fname_stderr ) self.run_time = finish_time - started_time if self.run_time < 0: self.run_time = None
def init_db(self): #:::~ Table with the information related to the database self.cursor.execute( "CREATE TABLE IF NOT EXISTS information " "(id INTEGER PRIMARY KEY, key CHAR(64), value CHAR(128))") self.check_and_insert_information('version', database_version) self.check_and_insert_information('command', self.command) self.check_and_insert_information('path', self.path) self.check_and_insert_information('base_name', self.base_name) #:::~ Table with the defined entities self.cursor.execute( "CREATE TABLE IF NOT EXISTS entities " "(id INTEGER PRIMARY KEY, name CHAR(64), varies INTEGER)") self.cursor.execute("SELECT COUNT(*) FROM entities ") n_items = self.cursor.fetchone() if n_items[0] == 0: # table has not been filled for i in self.data: varies = 1 if (i.__class__ != IterConstant) else 0 self.cursor.execute( "INSERT INTO entities (name, varies) VALUES (?,?)", (i.name, varies)) else: self.cursor.execute("SELECT name FROM entities ") entities = set([i[0] for i in self.cursor]) s_names = set(self.names) if entities != set(self.names): utils.newline_msg( "ERR", "parameter (was %s, is %s)" % (entities, s_names)) sys.exit(1) self.connection.commit() elements = "CREATE TABLE IF NOT EXISTS values_set (id INTEGER PRIMARY KEY, %s )" % ( ", ".join(["%s CHAR(64) " % i for i in self.names])) # print elements self.cursor.execute(elements) elements = "INSERT INTO values_set ( %s ) VALUES (%s)" % (", ".join( ["%s " % i for i in self.names]), ", ".join("?" for i in self.names)) # :::~ (CT) Index creation code for i in self.data: if i.__class__ == IterConstant: continue self.cursor.execute( "CREATE INDEX IF NOT EXISTS idxvs_%s_id ON values_set (%s) " % (i.name, i.name)) self.possible_varying_ids = [] for i in self: self.cursor.execute(elements, [ utils.replace_values(self[j], self, vars_to_skip=set( ['spg_uid', 'spg_vsid', 'spg_rep'])) for j in self.names ]) self.possible_varying_ids.append(self.cursor.lastrowid) self.connection.commit() self.cursor.execute( "CREATE TABLE IF NOT EXISTS output_tables (id INTEGER PRIMARY KEY, name CHAR(128) , column CHAR(128) )" ) for results_table in list(self.stdout_contents.keys()): table_contents = self.stdout_contents[results_table] # self.number_of_columns = 0 # for ic, iv in table_contents: # if iv["type"] == "xy": # self.number_of_columns += 1 # if iv["type"] == "xydy": # self.number_of_columns += 2 results = "CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY, spg_uid INTEGER, spg_vsid INTEGER, spg_rep INTEGER, %s, FOREIGN KEY(spg_uid) REFERENCES run_status(id), FOREIGN KEY(spg_vsid) REFERENCES values_set(id))" % ( results_table, ", ".join([ "%s %s" % (ic, iv["datatype"]) for ic, iv in table_contents ])) self.cursor.execute(results) self.connection.commit() for results_table in list(self.stdout_contents.keys()): # self.cursor.execute("INSERT INTO output_tables ( name, column ) VALUES (?, ?)",( results_table, 'spg_runid' ) ) table_contents = self.stdout_contents[results_table] for ic, iv in table_contents: self.cursor.execute( "INSERT INTO output_tables ( name, column ) VALUES (?, ?)", (results_table, ic)) self.connection.commit() self.cursor.execute( "CREATE TABLE IF NOT EXISTS run_status (id INTEGER PRIMARY KEY, spg_vsid INTEGER, spg_rep INTEGER, status CHAR(1), run_time FLOAT, " "FOREIGN KEY (spg_vsid) REFERENCES values_set(id) )") self.connection.commit()
def init_db(self): #:::~ Table with the information related to the database self.cursor.execute("CREATE TABLE IF NOT EXISTS information " "(id INTEGER PRIMARY KEY, key CHAR(64), value CHAR(128))" ) self.check_and_insert_information('version', database_version) self.check_and_insert_information('command', self.command ) self.check_and_insert_information('path', self.path) self.check_and_insert_information('base_name', self.base_name) #:::~ Table with the defined entities self.cursor.execute("CREATE TABLE IF NOT EXISTS entities " "(id INTEGER PRIMARY KEY, name CHAR(64), varies INTEGER)" ) self.cursor.execute( "SELECT COUNT(*) FROM entities ") n_items = self.cursor.fetchone() if n_items[0] == 0: # table has not been filled for i in self.data: varies = 1 if (i.__class__ != IterConstant) else 0 self.cursor.execute( "INSERT INTO entities (name, varies) VALUES (?,?)",(i.name, varies) ) else: self.cursor.execute( "SELECT name FROM entities ") entities = [i[0] for i in self.cursor] s_names = set(self.names) if entities != set(self.names): utils.newline_msg("ERR", "parameter (was %s, is %s)"%(entities, s_names)) sys.exit(1) self.connection.commit() elements = "CREATE TABLE IF NOT EXISTS values_set (id INTEGER PRIMARY KEY, %s )"%( ", ".join([ "%s CHAR(64) "%i for i in self.names ] ) ) # print elements self.cursor.execute(elements) elements = "INSERT INTO values_set ( %s ) VALUES (%s)"%( ", ".join([ "%s "%i for i in self.names ] ), ", ".join( "?" for i in self.names ) ) # :::~ (CT) Index creation code for i in self.data: if i.__class__ == IterConstant: continue self.cursor.execute( "CREATE INDEX IF NOT EXISTS idxvs_%s_id ON values_set (%s) "%(i.name,i.name) ) self.possible_varying_ids = [] for i in self: self.cursor.execute( elements, [ utils.replace_values(self[j], self, vars_to_skip = set(['spg_uid','spg_vsid','spg_rep'] ) ) for j in self.names] ) self.possible_varying_ids.append(self.cursor.lastrowid) self.connection.commit() self.cursor.execute("CREATE TABLE IF NOT EXISTS output_tables (id INTEGER PRIMARY KEY, name CHAR(128) , column CHAR(128) )") for results_table in self.stdout_contents.keys(): table_contents = self.stdout_contents[ results_table ] # self.number_of_columns = 0 # for ic, iv in table_contents: # if iv["type"] == "xy": # self.number_of_columns += 1 # if iv["type"] == "xydy": # self.number_of_columns += 2 results = "CREATE TABLE IF NOT EXISTS %s (id INTEGER PRIMARY KEY, spg_uid INTEGER, spg_vsid INTEGER, spg_rep INTEGER, %s, FOREIGN KEY(spg_uid) REFERENCES run_status(id), FOREIGN KEY(spg_vsid) REFERENCES values_set(id))"%(results_table, ", ".join([ "%s %s"%(ic,iv["datatype"]) for ic, iv in table_contents ]) ) self.cursor.execute(results) self.connection.commit() for results_table in self.stdout_contents.keys(): # self.cursor.execute("INSERT INTO output_tables ( name, column ) VALUES (?, ?)",( results_table, 'spg_runid' ) ) table_contents = self.stdout_contents[ results_table ] for ic, iv in table_contents: self.cursor.execute("INSERT INTO output_tables ( name, column ) VALUES (?, ?)",( results_table,ic ) ) self.connection.commit() self.cursor.execute("CREATE TABLE IF NOT EXISTS run_status (id INTEGER PRIMARY KEY, spg_vsid INTEGER, spg_rep INTEGER, status CHAR(1), run_time FLOAT, " "FOREIGN KEY (spg_vsid) REFERENCES values_set(id) )") self.connection.commit()
def init_db(self): #:::~ Table with the name of the executable self.cursor.execute("CREATE TABLE IF NOT EXISTS executable " "(id INTEGER PRIMARY KEY, name CHAR(64))" ) self.cursor.execute( "SELECT name FROM executable " ) prev_val = self.cursor.fetchone() if prev_val : if prev_val[0] != self.command: utils.newline_msg("ERR","conflict in executable name (in db '%s', in param '%s')"%(prev_val, self.command)) else: self.cursor.execute("INSERT INTO executable (name) VALUES (?)",(self.command,)) self.connection.commit() #:::~ Table with the defined entities self.cursor.execute("CREATE TABLE IF NOT EXISTS entities " "(id INTEGER PRIMARY KEY, name CHAR(64), varies INTEGER)" ) self.cursor.execute( "SELECT COUNT(*) FROM entities ") n_items = self.cursor.fetchone() if n_items[0] == 0: # table has not been filled for i in self.data: varies = 1 if (i.__class__ != IterConstant) else 0 self.cursor.execute( "INSERT INTO entities (name, varies) VALUES (?,?)",(i.name, varies) ) else: self.cursor.execute( "SELECT name FROM entities ") entities = set([i[0] for i in self.cursor]) s_names = set(self.names) if entities != set(self.names): utils.newline_msg("ERR", "parameter (was %s, is %s)"%(entities, s_names)) sys.exit(1) self.connection.commit() elements = "CREATE TABLE IF NOT EXISTS values_set (id INTEGER PRIMARY KEY, %s )"%( ", ".join([ "%s CHAR(64)"%i for i in self.names ] ) ) self.cursor.execute(elements) elements = "INSERT INTO values_set ( %s ) VALUES (%s)"%( ", ".join([ "%s "%i for i in self.names ] ), ", ".join( "?" for i in self.names ) ) self.possible_varying_ids = [] # i_try = 0 for i in self: self.cursor.execute( elements, [ utils.replace_values(self[i], self) for i in self.names] ) self.possible_varying_ids.append(self.cursor.lastrowid) self.connection.commit() #if not commited: # utils.newline_msg("ERR", "database didn't unlock, exiting") self.number_of_columns = 0 for ic, iv in self.stdout_contents: if iv["type"] == "xy": self.number_of_columns += 1 if iv["type"] == "xydy": self.number_of_columns += 2 results = "CREATE TABLE IF NOT EXISTS results (id INTEGER PRIMARY KEY, values_set_id INTEGER, %s , FOREIGN KEY(values_set_id) REFERENCES values_set(id))"%( ", ".join([ "%s CHAR(64)"%ic for ic, iv in self.stdout_contents ] ) ) self.cursor.execute(results) self.connection.commit() self.cursor.execute("CREATE TABLE IF NOT EXISTS run_status (id INTEGER PRIMARY KEY, values_set_id INTEGER, status CHAR(1), " "FOREIGN KEY (values_set_id ) REFERENCES values_set(id) )") self.connection.commit()