def do_init(self, c):
"""init [--flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]]
Generates a new database out of a simulation.dat
FLAGS::: --purge: deletes the spgql database, if it already exists
--repeat=REPEAT repeats the parameter generation REPEAT times
"""
flags, db_arg = self.parse_command_line(c)
if len(db_arg) != 1:
utils.newline_msg("WRN", "init must be called with a database", 2)
return
db_arg = db_arg[0]
# i_arg = c[0]
full_name, path, base_name, extension = utils.translate_name(db_arg)
# print "do_init::: ",self.translate_name(i_arg)
full_db_name = "%s/%s.spgql" % (path, base_name)
sim_name = "%s/%s.spg" % (path, base_name)
if os.path.exists(full_db_name) and "purge" not in flags:
utils.newline_msg(
"ERR", "database '%s' already exists. Cannot init it twice" %
utils.shorten_name(full_db_name), 2)
return
if not os.path.exists(sim_name):
utils.newline_msg(
"ERR", "configuration '%s' doesn't exist. Cannot init it" %
utils.shorten_name(sim_name), 2)
return
if "purge" in flags:
try:
self.do_deregister(db_arg)
os.remove(full_db_name)
except:
utils.newline_msg(
"WRN", "database '%s' could not be removed... skipping" %
full_db_name)
if 'repeat' in flags:
repeat = int(flags['repeat'])
else:
repeat = 1
parser = MultIteratorDBBuilder(db_name=full_db_name)
parser.init_db()
parser.fill_status(repeat=repeat)
current_param_db = ParameterEnsemble(full_db_name, init_db=True)
current_param_db.repeat = repeat
# if len(c) > 1: self.do_set(":".join(c[1:]))
self.master_db.write_ensemble_to_master(current_param_db)
self.master_db.update_list_ensemble_dbs()
print(" **-- init - %d: '%s' " %
(current_param_db.id,
utils.shorten_name(current_param_db.full_name)))
def do_init(self, c):
"""init [--flag ...] PARAMETERS_NAME|DB_NAME [VAR1=VALUE1[:VAR2=VALUE2]]
Generates a new database out of a simulation.dat
FLAGS::: --purge: deletes the spgql database, if it already exists
--repeat=REPEAT repeats the parameter generation REPEAT times
"""
flags, db_arg = self.parse_command_line(c)
if len(db_arg) != 1:
utils.newline_msg("WRN", "init must be called with a database", 2)
return
db_arg = db_arg[0]
# i_arg = c[0]
full_name, path, base_name, extension = utils.translate_name(db_arg)
# print "do_init::: ",self.translate_name(i_arg)
full_db_name = "%s/%s.spgql" % (path, base_name)
sim_name = "%s/%s.spg" % (path, base_name)
if os.path.exists(full_db_name) and "purge" not in flags:
utils.newline_msg(
"ERR", "database '%s' already exists. Cannot init it twice" % utils.shorten_name(full_db_name), 2
)
return
if not os.path.exists(sim_name):
utils.newline_msg(
"ERR", "configuration '%s' doesn't exist. Cannot init it" % utils.shorten_name(sim_name), 2
)
return
if "purge" in flags:
self.do_deregister(db_arg)
try:
os.remove(full_db_name)
except:
utils.newline_msg("WRN", "database '%s' could not be removed... skipping" % full_db_name)
if "repeat" in flags:
repeat = int(flags["repeat"])
else:
repeat = 1
parser = MultIteratorDBBuilder(db_name=full_db_name)
parser.init_db()
parser.fill_status(repeat=repeat)
current_param_db = ParameterEnsemble(full_db_name, init_db=True)
current_param_db.repeat = repeat
# if len(c) > 1: self.do_set(":".join(c[1:]))
self.master_db.write_ensemble_to_master(current_param_db)
self.master_db.update_list_ensemble_dbs()
print " **-- init - %d: '%s' " % (current_param_db.id, utils.shorten_name(current_param_db.full_name))
def get_flags_and_db(self, c, init_db=True):
flags, args = self.parse_command_line(c)
# if len(args)== 1:
# utils.newline_msg("ERR", "a single file was expected or could not parse flags")
# return flags, args, None
db_name = args[-1]
args = args[:-1]
if db_name.isdigit():
id = int(db_name)
rdb = self.master_db.result_dbs
filtered = [x for x in rdb if rdb[x]['id'] == id]
if filtered:
db_name = filtered[0]
else:
utils.newline_msg(
"ERR", "database with id '%s' doesn't exist." % db_name)
return flags, args.append(db_name), None
else:
full_name, path, base_name, extension = utils.translate_name(
db_name)
# print "do_init::: ",self.translate_name(i_arg)
db_name = "%s/%s.spgql" % (path, base_name)
# sim_name = "%s/%s.spg" % (path, base_name)
if not os.path.exists(db_name):
utils.newline_msg(
"ERR", "database with name '%s' doesn't exist." %
utils.shorten_name(db_name))
return flags, args.append(db_name), None
return flags, args, self.EnsembleConstructor(db_name, init_db)
def do_info(self, c):
"""info REGEXP
prints the information of the results databases, filtered by a regular expression, or its id """
flags, cmds, ensemble = self.get_flags_and_db(c)
# ensemble = self.get_db(db_name)
if ensemble is None:
# utils.newline_msg("ERR", "database not found... aborting")
return
db_status = ensemble.get_updated_status()
param_db_id = self.master_db.query_master_fetchone("SELECT id, status, weight FROM dbs WHERE full_name = ?", ensemble.full_name)
if param_db_id is None:
param_db_id = "X"
else:
[param_db_id, status, weight] = param_db_id
print " ---%5s: %s" % (param_db_id, utils.shorten_name(ensemble.full_name))
frac_done = float(db_status['process_done']) / float(db_status['value_set'])
n_repet = db_status['value_set_with_rep'] / db_status['value_set']
print " -+ status = %s / weight: %5.5f " % (status, weight)
print " -+ total = %d*%d / done: %d (%.5f) - running: %d - error: %d " % (
db_status['value_set'], n_repet, db_status['process_done'], frac_done,
db_status['process_running'], db_status['process_error'])
try:
print " -+ time = %f / mean: %f - min: %f - max: %f" % (
db_status['total_run_time'], db_status['avg_run_time'], db_status['min_run_time'],
db_status['max_run_time'])
except:
pass
def get_flags_and_db(self, c):
flags, args = self.parse_command_line(c)
# if len(args)== 1:
# utils.newline_msg("ERR", "a single file was expected or could not parse flags")
# return flags, args, None
db_name = args[-1]
args = args[:-1]
if db_name.isdigit():
id = int(db_name)
rdb = self.master_db.result_dbs
filtered = [x for x in rdb if rdb[x]["id"] == id]
if filtered:
db_name = filtered[0]
else:
utils.newline_msg("ERR", "database with id '%s' doesn't exist." % db_name)
return flags, args.append(db_name), None
else:
full_name, path, base_name, extension = utils.translate_name(db_name)
# print "do_init::: ",self.translate_name(i_arg)
db_name = "%s/%s.spgql" % (path, base_name)
# sim_name = "%s/%s.spg" % (path, base_name)
if not os.path.exists(db_name):
utils.newline_msg("ERR", "database with name '%s' doesn't exist." % utils.shorten_name(db_name))
return flags, args.append(db_name), None
return flags, args, self.EnsembleConstructor(db_name, init_db=True)
def do_register(self, c):
"""registers a given results database into the master database"""
flags, cmds, ensemble = self.get_flags_and_db(c)
if ensemble is None:
# utils.newline_msg("ERR", "no database supplied ... skipping")
return
if self.master_db.result_dbs.has_key(ensemble.full_name):
utils.newline_msg(
"WRN", "skipping... database '%s' is already registered" % utils.shorten_name(ensemble.full_name), 2
)
return
# current_param_db = ParameterEnsemble(db_name, init_db=True)
self.master_db.write_ensemble_to_master(ensemble)
self.master_db.update_list_ensemble_dbs()
print " *--- registered - %d: '%s' " % (ensemble.id, utils.shorten_name(ensemble.full_name))
def do_register(self, c):
"""registers a given results database into the master database"""
flags, cmds, ensemble = self.get_flags_and_db(c)
if ensemble is None:
# utils.newline_msg("ERR", "no database supplied ... skipping")
return
if ensemble.full_name in self.master_db.result_dbs:
utils.newline_msg(
"WRN", "skipping... database '%s' is already registered" %
utils.shorten_name(ensemble.full_name), 2)
return
# current_param_db = ParameterEnsemble(db_name, init_db=True)
self.master_db.write_ensemble_to_master(ensemble)
self.master_db.update_list_ensemble_dbs()
print(" *--- registered - %d: '%s' " %
(ensemble.id, utils.shorten_name(ensemble.full_name)))
def do_load(self, c):
"""load DB_NAME|DB_ID
loads one of the registered databases from the master"""
flags, args = self.parse_command_line(c)
if len(args) > 1:
utils.newline_msg("ERR", "only one db can be loaded at a time", 2)
return
ret = self.get_db(args[0])
if ret:
self.current_param_db = ret
print(" --- loaded: '%s'" % utils.shorten_name(ret.full_name))
os.chdir(self.current_param_db.path)
else:
utils.newline_msg("ERR", "'%s' does not exist" % ret, 2)
def do_load(self,c):
"""load DB_NAME|DB_ID
loads one of the registered databases from the master"""
flags, args = self.parse_command_line(c)
if len(args) >1:
utils.newline_msg("ERR", "only one db can be loaded at a time", 2)
return
ret = self.get_db(args[0])
if ret:
self.current_param_db = ret
print " --- loaded: '%s'"% utils.shorten_name(ret.full_name)
os.chdir(self.current_param_db.path)
else:
utils.newline_msg("ERR", "'%s' does not exist"%ret, 2)
def do_ls(self, c):
"""ls REGEXP|DB_ID
lists the databases already registered in the master database and the possible ones found in the current directory"""
ls_res_db = self.filter_db_list( filter = c )
if ls_res_db:
print " --- registered dbs"
for i in sorted( ls_res_db ):
# :::~FIXME workaround for non-existing dbs
curr_db = self.master_db.result_dbs[i]
short_name = utils.shorten_name(i)
try:
print "%5d: %s (%5.5f)"%(curr_db['id'], short_name , curr_db['weight'] )
except:
print "%5d: %s "%(curr_db['id'], short_name )
BaseSPGCommandLine.do_ls(self, c )
def do_ls(self, c):
full_ls = fnmatch.filter(os.listdir("."), "*.spgql")
full_ls.extend( fnmatch.filter( os.listdir("."), "*.spg" ) )
ls_res_db = set()
ls_keys = set()
for el in full_ls:
full_name, path, base_name, extension = utils.translate_name(el)
full_base = "%s/%s"%(path, base_name)
if full_base not in ls_keys:
ls_res_db.add(utils.shorten_name(full_name))
ls_keys.add( full_base )
if c:
ls_res_db = fnmatch.filter(ls_res_db, c )
if ls_res_db:
print " --- cwd matches found"
for i in sorted( ls_res_db ):
print " : %s "% i
def do_ls(self, c):
"""ls REGEXP|DB_ID
lists the databases already registered in the master database and the possible ones found in the current directory"""
ls_res_db = self.filter_db_list(filter=c)
if ls_res_db:
print(" --- registered dbs")
for i in sorted(ls_res_db):
# :::~FIXME workaround for non-existing dbs
curr_db = self.master_db.result_dbs[i]
short_name = utils.shorten_name(i)
try:
print("%5d: %s (%5.5f)" %
(curr_db['id'], short_name, curr_db['weight']))
except:
print("%5d: %s " % (curr_db['id'], short_name))
BaseSPGCommandLine.do_ls(self, c)
def __set_status(self, c, st):
# if not c:
# ls_res_db = [ self.current_param_db.full_name ]
# else:
# ls_res_db = self.filter_db_list( filter = c )
# if not ls_res_db: return
#
# for i in ls_res_db:
flags, cmds, ensemble = self.get_flags_and_db(c)
# ensemble = self.get_db(db_name)
if ensemble is None:
# utils.newline_msg("ERR", "database not found... aborting")
return
ensemble.status = st
print " +--- '%s' [status : %s ]" % (utils.shorten_name(ensemble.full_name), st)
self.master_db.query_master_db('UPDATE dbs SET status= ? WHERE full_name = ?', st, ensemble.full_name)
def do_ls(self, c):
full_ls = fnmatch.filter(os.listdir("."), "*.spgql")
full_ls.extend(fnmatch.filter(os.listdir("."), "*.spg"))
ls_res_db = set()
ls_keys = set()
for el in full_ls:
full_name, path, base_name, extension = utils.translate_name(el)
full_base = "%s/%s" % (path, base_name)
if full_base not in ls_keys:
ls_res_db.add(utils.shorten_name(full_name))
ls_keys.add(full_base)
if c:
ls_res_db = fnmatch.filter(ls_res_db, c)
if ls_res_db:
print(" --- cwd matches found")
for i in sorted(ls_res_db):
print(" : %s " % i)
def do_info(self, c):
"""info REGEXP
prints the information of the results databases, filtered by a regular expression, or its id """
flags, cmds, ensemble = self.get_flags_and_db(c)
# ensemble = self.get_db(db_name)
if ensemble is None:
# utils.newline_msg("ERR", "database not found... aborting")
return
db_status = ensemble.get_updated_status()
param_db_id = self.master_db.query_master_fetchone(
"SELECT id, status, weight FROM dbs WHERE full_name = ?",
ensemble.full_name)
if param_db_id is None:
param_db_id = "X"
else:
[param_db_id, status, weight] = param_db_id
print(" ---%5s: %s" %
(param_db_id, utils.shorten_name(ensemble.full_name)))
frac_done = float(db_status['process_done']) / float(
db_status['value_set'])
n_repet = db_status['value_set_with_rep'] / db_status['value_set']
print(" -+ status = %s / weight: %5.5f " % (status, weight))
print(
" -+ total = %d*%d / done: %d (%.5f) - running: %d - error: %d "
% (db_status['value_set'], n_repet, db_status['process_done'],
frac_done, db_status['process_running'],
db_status['process_error']))
try:
print(" -+ time = %f / mean: %f - min: %f - max: %f" %
(db_status['total_run_time'], db_status['avg_run_time'],
db_status['min_run_time'], db_status['max_run_time']))
except:
pass
def __set_status(self, c, st):
# if not c:
# ls_res_db = [ self.current_param_db.full_name ]
# else:
# ls_res_db = self.filter_db_list( filter = c )
# if not ls_res_db: return
#
# for i in ls_res_db:
flags, cmds, ensemble = self.get_flags_and_db(c)
# ensemble = self.get_db(db_name)
if ensemble is None:
# utils.newline_msg("ERR", "database not found... aborting")
return
ensemble.status = st
print(" +--- '%s' [status : %s ]" %
(utils.shorten_name(ensemble.full_name), st))
self.master_db.query_master_db(
'UPDATE dbs SET status= ? WHERE full_name = ?', st,
ensemble.full_name)
def init_db(self):
try:
(self.command, ) = self.execute_query_fetchone( "SELECT value FROM information WHERE key = 'command'" )
except:
utils.newline_msg("FATAL", "Cannot retrieve information from database '%s'..."%utils.shorten_name(self.db_name))
sys.exit(2)
#:::~ get the names of the columns
sel = self.execute_query("SELECT name FROM entities ORDER BY id")
self.entities = [ i[0] for i in sel ]
#:::~ get the names of the columns
sel = self.execute_query("SELECT name FROM entities WHERE varies = 1 ORDER BY id")
self.variables = [ i[0] for i in sel ]
#:::~ get the names of the outputs
self.table_columns = {}
table_names = [i[0] for i in self.execute_query("SELECT DISTINCT name from output_tables")]
### print table_names
for table in table_names:
fa = self.execute_query("SELECT column FROM output_tables WHERE name = '%s';"%table)
self.table_columns[table] = ["spg_uid", "spg_vsid", "spg_rep"] + [i[0] for i in fa]
def init_db(self):
try:
(self.command, ) = self.execute_query_fetchone( "SELECT value FROM information WHERE key = 'command'" )
except:
utils.newline_msg("FATAL", "Cannot retreive information from database '%s'..."%utils.shorten_name(self.db_name))
sys.exit(2)
#:::~ get the names of the columns
sel = self.execute_query("SELECT name FROM entities ORDER BY id")
self.entities = [ i[0] for i in sel ]
#:::~ get the names of the columns
sel = self.execute_query("SELECT name FROM entities WHERE varies = 1 ORDER BY id")
self.variables = [ i[0] for i in sel ]
#:::~ get the names of the outputs
self.table_columns = {}
table_names = [i[0] for i in self.execute_query("SELECT DISTINCT name from output_tables")]
### print table_names
for table in table_names:
fa = self.execute_query("SELECT column FROM output_tables WHERE name = '%s';"%table)
self.table_columns[table] = ["spg_uid", "spg_vsid", "spg_rep"] + [i[0] for i in fa]