def do_init(self, c): """init [--flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]] Generates a new database out of a simulation.dat FLAGS::: --purge: deletes the spgql database, if it already exists --repeat=REPEAT repeats the parameter generation REPEAT times """ flags, db_arg = self.parse_command_line(c) if len(db_arg) != 1: utils.newline_msg("WRN", "init must be called with a database", 2) return db_arg = db_arg[0] # i_arg = c[0] full_name, path, base_name, extension = utils.translate_name(db_arg) # print "do_init::: ",self.translate_name(i_arg) full_db_name = "%s/%s.spgql" % (path, base_name) sim_name = "%s/%s.spg" % (path, base_name) if os.path.exists(full_db_name) and "purge" not in flags: utils.newline_msg( "ERR", "database '%s' already exists. Cannot init it twice" % utils.shorten_name(full_db_name), 2) return if not os.path.exists(sim_name): utils.newline_msg( "ERR", "configuration '%s' doesn't exist. Cannot init it" % utils.shorten_name(sim_name), 2) return if "purge" in flags: try: self.do_deregister(db_arg) os.remove(full_db_name) except: utils.newline_msg( "WRN", "database '%s' could not be removed... skipping" % full_db_name) if 'repeat' in flags: repeat = int(flags['repeat']) else: repeat = 1 parser = MultIteratorDBBuilder(db_name=full_db_name) parser.init_db() parser.fill_status(repeat=repeat) current_param_db = ParameterEnsemble(full_db_name, init_db=True) current_param_db.repeat = repeat # if len(c) > 1: self.do_set(":".join(c[1:])) self.master_db.write_ensemble_to_master(current_param_db) self.master_db.update_list_ensemble_dbs() print(" **-- init - %d: '%s' " % (current_param_db.id, utils.shorten_name(current_param_db.full_name)))
def do_init(self, c): """init [--flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]] Generates a new database out of a simulation.dat FLAGS::: --purge: deletes the spgql database, if it already exists --repeat=REPEAT repeats the parameter generation REPEAT times """ flags, db_arg = self.parse_command_line(c) if len(db_arg) != 1: utils.newline_msg("WRN", "init must be called with a database", 2) return db_arg = db_arg[0] # i_arg = c[0] full_name, path, base_name, extension = utils.translate_name(db_arg) # print "do_init::: ",self.translate_name(i_arg) full_db_name = "%s/%s.spgql" % (path, base_name) sim_name = "%s/%s.spg" % (path, base_name) if os.path.exists(full_db_name) and "purge" not in flags: utils.newline_msg( "ERR", "database '%s' already exists. Cannot init it twice" % utils.shorten_name(full_db_name), 2 ) return if not os.path.exists(sim_name): utils.newline_msg( "ERR", "configuration '%s' doesn't exist. Cannot init it" % utils.shorten_name(sim_name), 2 ) return if "purge" in flags: self.do_deregister(db_arg) try: os.remove(full_db_name) except: utils.newline_msg("WRN", "database '%s' could not be removed... skipping" % full_db_name) if "repeat" in flags: repeat = int(flags["repeat"]) else: repeat = 1 parser = MultIteratorDBBuilder(db_name=full_db_name) parser.init_db() parser.fill_status(repeat=repeat) current_param_db = ParameterEnsemble(full_db_name, init_db=True) current_param_db.repeat = repeat # if len(c) > 1: self.do_set(":".join(c[1:])) self.master_db.write_ensemble_to_master(current_param_db) self.master_db.update_list_ensemble_dbs() print " **-- init - %d: '%s' " % (current_param_db.id, utils.shorten_name(current_param_db.full_name))
def get_flags_and_db(self, c, init_db=True): flags, args = self.parse_command_line(c) # if len(args)== 1: # utils.newline_msg("ERR", "a single file was expected or could not parse flags") # return flags, args, None db_name = args[-1] args = args[:-1] if db_name.isdigit(): id = int(db_name) rdb = self.master_db.result_dbs filtered = [x for x in rdb if rdb[x]['id'] == id] if filtered: db_name = filtered[0] else: utils.newline_msg( "ERR", "database with id '%s' doesn't exist." % db_name) return flags, args.append(db_name), None else: full_name, path, base_name, extension = utils.translate_name( db_name) # print "do_init::: ",self.translate_name(i_arg) db_name = "%s/%s.spgql" % (path, base_name) # sim_name = "%s/%s.spg" % (path, base_name) if not os.path.exists(db_name): utils.newline_msg( "ERR", "database with name '%s' doesn't exist." % utils.shorten_name(db_name)) return flags, args.append(db_name), None return flags, args, self.EnsembleConstructor(db_name, init_db)
def do_info(self, c): """info REGEXP prints the information of the results databases, filtered by a regular expression, or its id """ flags, cmds, ensemble = self.get_flags_and_db(c) # ensemble = self.get_db(db_name) if ensemble is None: # utils.newline_msg("ERR", "database not found... aborting") return db_status = ensemble.get_updated_status() param_db_id = self.master_db.query_master_fetchone("SELECT id, status, weight FROM dbs WHERE full_name = ?", ensemble.full_name) if param_db_id is None: param_db_id = "X" else: [param_db_id, status, weight] = param_db_id print " ---%5s: %s" % (param_db_id, utils.shorten_name(ensemble.full_name)) frac_done = float(db_status['process_done']) / float(db_status['value_set']) n_repet = db_status['value_set_with_rep'] / db_status['value_set'] print " -+ status = %s / weight: %5.5f " % (status, weight) print " -+ total = %d*%d / done: %d (%.5f) - running: %d - error: %d " % ( db_status['value_set'], n_repet, db_status['process_done'], frac_done, db_status['process_running'], db_status['process_error']) try: print " -+ time = %f / mean: %f - min: %f - max: %f" % ( db_status['total_run_time'], db_status['avg_run_time'], db_status['min_run_time'], db_status['max_run_time']) except: pass
def get_flags_and_db(self, c): flags, args = self.parse_command_line(c) # if len(args)== 1: # utils.newline_msg("ERR", "a single file was expected or could not parse flags") # return flags, args, None db_name = args[-1] args = args[:-1] if db_name.isdigit(): id = int(db_name) rdb = self.master_db.result_dbs filtered = [x for x in rdb if rdb[x]["id"] == id] if filtered: db_name = filtered[0] else: utils.newline_msg("ERR", "database with id '%s' doesn't exist." % db_name) return flags, args.append(db_name), None else: full_name, path, base_name, extension = utils.translate_name(db_name) # print "do_init::: ",self.translate_name(i_arg) db_name = "%s/%s.spgql" % (path, base_name) # sim_name = "%s/%s.spg" % (path, base_name) if not os.path.exists(db_name): utils.newline_msg("ERR", "database with name '%s' doesn't exist." % utils.shorten_name(db_name)) return flags, args.append(db_name), None return flags, args, self.EnsembleConstructor(db_name, init_db=True)
def do_register(self, c): """registers a given results database into the master database""" flags, cmds, ensemble = self.get_flags_and_db(c) if ensemble is None: # utils.newline_msg("ERR", "no database supplied ... skipping") return if self.master_db.result_dbs.has_key(ensemble.full_name): utils.newline_msg( "WRN", "skipping... database '%s' is already registered" % utils.shorten_name(ensemble.full_name), 2 ) return # current_param_db = ParameterEnsemble(db_name, init_db=True) self.master_db.write_ensemble_to_master(ensemble) self.master_db.update_list_ensemble_dbs() print " *--- registered - %d: '%s' " % (ensemble.id, utils.shorten_name(ensemble.full_name))
def do_register(self, c): """registers a given results database into the master database""" flags, cmds, ensemble = self.get_flags_and_db(c) if ensemble is None: # utils.newline_msg("ERR", "no database supplied ... skipping") return if ensemble.full_name in self.master_db.result_dbs: utils.newline_msg( "WRN", "skipping... database '%s' is already registered" % utils.shorten_name(ensemble.full_name), 2) return # current_param_db = ParameterEnsemble(db_name, init_db=True) self.master_db.write_ensemble_to_master(ensemble) self.master_db.update_list_ensemble_dbs() print(" *--- registered - %d: '%s' " % (ensemble.id, utils.shorten_name(ensemble.full_name)))
def do_load(self, c): """load DB_NAME|DB_ID loads one of the registered databases from the master""" flags, args = self.parse_command_line(c) if len(args) > 1: utils.newline_msg("ERR", "only one db can be loaded at a time", 2) return ret = self.get_db(args[0]) if ret: self.current_param_db = ret print(" --- loaded: '%s'" % utils.shorten_name(ret.full_name)) os.chdir(self.current_param_db.path) else: utils.newline_msg("ERR", "'%s' does not exist" % ret, 2)
def do_load(self,c): """load DB_NAME|DB_ID loads one of the registered databases from the master""" flags, args = self.parse_command_line(c) if len(args) >1: utils.newline_msg("ERR", "only one db can be loaded at a time", 2) return ret = self.get_db(args[0]) if ret: self.current_param_db = ret print " --- loaded: '%s'"% utils.shorten_name(ret.full_name) os.chdir(self.current_param_db.path) else: utils.newline_msg("ERR", "'%s' does not exist"%ret, 2)
def do_ls(self, c): """ls REGEXP|DB_ID lists the databases already registered in the master database and the possible ones found in the current directory""" ls_res_db = self.filter_db_list( filter = c ) if ls_res_db: print " --- registered dbs" for i in sorted( ls_res_db ): # :::~FIXME workaround for non-existing dbs curr_db = self.master_db.result_dbs[i] short_name = utils.shorten_name(i) try: print "%5d: %s (%5.5f)"%(curr_db['id'], short_name , curr_db['weight'] ) except: print "%5d: %s "%(curr_db['id'], short_name ) BaseSPGCommandLine.do_ls(self, c )
def do_ls(self, c): full_ls = fnmatch.filter(os.listdir("."), "*.spgql") full_ls.extend( fnmatch.filter( os.listdir("."), "*.spg" ) ) ls_res_db = set() ls_keys = set() for el in full_ls: full_name, path, base_name, extension = utils.translate_name(el) full_base = "%s/%s"%(path, base_name) if full_base not in ls_keys: ls_res_db.add(utils.shorten_name(full_name)) ls_keys.add( full_base ) if c: ls_res_db = fnmatch.filter(ls_res_db, c ) if ls_res_db: print " --- cwd matches found" for i in sorted( ls_res_db ): print " : %s "% i
def do_ls(self, c): """ls REGEXP|DB_ID lists the databases already registered in the master database and the possible ones found in the current directory""" ls_res_db = self.filter_db_list(filter=c) if ls_res_db: print(" --- registered dbs") for i in sorted(ls_res_db): # :::~FIXME workaround for non-existing dbs curr_db = self.master_db.result_dbs[i] short_name = utils.shorten_name(i) try: print("%5d: %s (%5.5f)" % (curr_db['id'], short_name, curr_db['weight'])) except: print("%5d: %s " % (curr_db['id'], short_name)) BaseSPGCommandLine.do_ls(self, c)
def __set_status(self, c, st): # if not c: # ls_res_db = [ self.current_param_db.full_name ] # else: # ls_res_db = self.filter_db_list( filter = c ) # if not ls_res_db: return # # for i in ls_res_db: flags, cmds, ensemble = self.get_flags_and_db(c) # ensemble = self.get_db(db_name) if ensemble is None: # utils.newline_msg("ERR", "database not found... aborting") return ensemble.status = st print " +--- '%s' [status : %s ]" % (utils.shorten_name(ensemble.full_name), st) self.master_db.query_master_db('UPDATE dbs SET status= ? WHERE full_name = ?', st, ensemble.full_name)
def do_ls(self, c): full_ls = fnmatch.filter(os.listdir("."), "*.spgql") full_ls.extend(fnmatch.filter(os.listdir("."), "*.spg")) ls_res_db = set() ls_keys = set() for el in full_ls: full_name, path, base_name, extension = utils.translate_name(el) full_base = "%s/%s" % (path, base_name) if full_base not in ls_keys: ls_res_db.add(utils.shorten_name(full_name)) ls_keys.add(full_base) if c: ls_res_db = fnmatch.filter(ls_res_db, c) if ls_res_db: print(" --- cwd matches found") for i in sorted(ls_res_db): print(" : %s " % i)
def do_info(self, c): """info REGEXP prints the information of the results databases, filtered by a regular expression, or its id """ flags, cmds, ensemble = self.get_flags_and_db(c) # ensemble = self.get_db(db_name) if ensemble is None: # utils.newline_msg("ERR", "database not found... aborting") return db_status = ensemble.get_updated_status() param_db_id = self.master_db.query_master_fetchone( "SELECT id, status, weight FROM dbs WHERE full_name = ?", ensemble.full_name) if param_db_id is None: param_db_id = "X" else: [param_db_id, status, weight] = param_db_id print(" ---%5s: %s" % (param_db_id, utils.shorten_name(ensemble.full_name))) frac_done = float(db_status['process_done']) / float( db_status['value_set']) n_repet = db_status['value_set_with_rep'] / db_status['value_set'] print(" -+ status = %s / weight: %5.5f " % (status, weight)) print( " -+ total = %d*%d / done: %d (%.5f) - running: %d - error: %d " % (db_status['value_set'], n_repet, db_status['process_done'], frac_done, db_status['process_running'], db_status['process_error'])) try: print(" -+ time = %f / mean: %f - min: %f - max: %f" % (db_status['total_run_time'], db_status['avg_run_time'], db_status['min_run_time'], db_status['max_run_time'])) except: pass
def __set_status(self, c, st): # if not c: # ls_res_db = [ self.current_param_db.full_name ] # else: # ls_res_db = self.filter_db_list( filter = c ) # if not ls_res_db: return # # for i in ls_res_db: flags, cmds, ensemble = self.get_flags_and_db(c) # ensemble = self.get_db(db_name) if ensemble is None: # utils.newline_msg("ERR", "database not found... aborting") return ensemble.status = st print(" +--- '%s' [status : %s ]" % (utils.shorten_name(ensemble.full_name), st)) self.master_db.query_master_db( 'UPDATE dbs SET status= ? WHERE full_name = ?', st, ensemble.full_name)
def init_db(self): try: (self.command, ) = self.execute_query_fetchone( "SELECT value FROM information WHERE key = 'command'" ) except: utils.newline_msg("FATAL", "Cannot retrieve information from database '%s'..."%utils.shorten_name(self.db_name)) sys.exit(2) #:::~ get the names of the columns sel = self.execute_query("SELECT name FROM entities ORDER BY id") self.entities = [ i[0] for i in sel ] #:::~ get the names of the columns sel = self.execute_query("SELECT name FROM entities WHERE varies = 1 ORDER BY id") self.variables = [ i[0] for i in sel ] #:::~ get the names of the outputs self.table_columns = {} table_names = [i[0] for i in self.execute_query("SELECT DISTINCT name from output_tables")] ### print table_names for table in table_names: fa = self.execute_query("SELECT column FROM output_tables WHERE name = '%s';"%table) self.table_columns[table] = ["spg_uid", "spg_vsid", "spg_rep"] + [i[0] for i in fa]
def init_db(self): try: (self.command, ) = self.execute_query_fetchone( "SELECT value FROM information WHERE key = 'command'" ) except: utils.newline_msg("FATAL", "Cannot retreive information from database '%s'..."%utils.shorten_name(self.db_name)) sys.exit(2) #:::~ get the names of the columns sel = self.execute_query("SELECT name FROM entities ORDER BY id") self.entities = [ i[0] for i in sel ] #:::~ get the names of the columns sel = self.execute_query("SELECT name FROM entities WHERE varies = 1 ORDER BY id") self.variables = [ i[0] for i in sel ] #:::~ get the names of the outputs self.table_columns = {} table_names = [i[0] for i in self.execute_query("SELECT DISTINCT name from output_tables")] ### print table_names for table in table_names: fa = self.execute_query("SELECT column FROM output_tables WHERE name = '%s';"%table) self.table_columns[table] = ["spg_uid", "spg_vsid", "spg_rep"] + [i[0] for i in fa]