def __init__(self, connection = None):
MasterDB.__init__(self, connection)
self.current_counter = 0
res = self.cursor.execute("SELECT last FROM infiles WHERE id = 1").fetchone()
if res == None:
self.cursor.execute("INSERT INTO infiles (last) VALUES (0)")
self.connection.commit()
def __init__(self):
cmd.Cmd.__init__(self)
self.prompt = "| spg-queue :::~ "
self.current_queue = None
self.master_db = MasterDB()
#self.possible_keys = ['max_jobs', 'status']
self.__update_queue_list()
def __init__(self, EnsembleConstructor = ParameterEnsemble):
cmd.Cmd.__init__(self)
self.EnsembleConstructor = EnsembleConstructor
self.current_param_db = None
self.master_db = MasterDB(EnsembleConstructor = EnsembleConstructor)
class BaseDBCommandParser(cmd.Cmd):
"""DB command handler"""
def __init__(self, EnsembleConstructor = ParameterEnsemble):
cmd.Cmd.__init__(self)
self.EnsembleConstructor = EnsembleConstructor
self.current_param_db = None
self.master_db = MasterDB(EnsembleConstructor = EnsembleConstructor)
def parse_command_line(self, st):
"""returns command the flags set under a command and the arguments"""
cmd = []
flags = set()
for ic in st.strip().split():
if ic[0] == "-":
flags.add( ic.strip("-") )
else:
cmd.append( ic )
return flags, cmd
def shorten_name(self, st):
return os.path.relpath(st,RUN_DIR)
def lengthen_name(self, st):
return "%s/%s"%(RUN_DIR, st)
def translate_name( self,st):
"""translates the parameters filename and the database name
into the other and viceversa (returns a duple: param_name, db_name)"""
full_name = os.path.realpath( st )
# if not os.path.exists(full_name):
# full_name = self.lengthen_name(full_name)
if not os.path.exists(full_name):
utils.newline_msg("ERR","database '%s' does not exist"%st)
return None
path, st = os.path.split(full_name)
if ".sqlite" in st:
par_name = st.replace("results","").replace(".sqlite","")
par_name = "parameters%s.dat"%par_name
return "%s/%s"%(path,par_name) , "%s/%s"%(path,st)
else:
db_name = st.replace("parameters","").replace(".dat","")
db_name = "results%s.sqlite"%db_name
return "%s/%s"%(path,st) , "%s/%s"%(path,db_name)
def update_active_result_db(self, c):
c = c.strip()
if not c: return
try:
param_name, db_name = self.translate_name(c)
except:
# utils.newline_msg("ERR", "results db '%s' doesn't exist. Can not load it" )
return
if os.path.exists( db_name ):
self.current_param_db = ParameterEnsemble( db_name )
elif os.path.exists( param_name ) and not os.path.exists( db_name ):
self.current_param_db = ParameterEnsemble( db_name , db_init = False)
return
def filter_db_list(self, ls = None, filter = None):
if ls == None:
ls = self.master_db.result_dbs.keys()
try:
id = int(filter)
rdb = self.master_db.result_dbs
filtered = [ x for x in ls if rdb.has_key(x) and rdb[x] is not None and rdb[x].id == id ]
return filtered
except:
ret = [ self.shorten_name(i) for i in ls ]
if filter:
ret = fnmatch.filter(ret, filter)
return sorted( [ self.lengthen_name( i ) for i in ret ] )
def get_db_from_cmdline(self, c):
"""it returns the db name (or None) of a database identified either from its id or """
try:
id = int(c)
rdb = self.master_db.result_dbs
filtered = [ rdb[x] for x in rdb.keys() if rdb[x] is not None and rdb[x].id == id ]
if filtered: return filtered[0]
else: return None
except:
try:
foo, db_name = self.translate_name(c)
except:
utils.newline_msg("ERR", "results db '%s' doesn't exist."%c )
return
if self.master_db.result_dbs.has_key(db_name):
return self.master_db.result_dbs[db_name]
else:
return self.EnsembleConstructor(db_name, init_db = True)
utils.newline_msg("WRN", "database '%s' is not registered, loading it anyhow"%self.shorten_name(db_name))
return None
# :::~ FIXME!: Hast to be implemented
# def complete(self, text):
# completions = self.master_db.result_dbs.keys()
# if text:
# completions = [ f
# for f in completions
# if f.startswith(text)
# ]
# return completions
def do_ls(self, c):
"""ls REGEXP|DB_ID
lists the databases already registered in the master database and the possible ones found in the current directory"""
ls_res_db = self.filter_db_list( filter = c )
if ls_res_db:
print " --- registered dbs"
for i in sorted( ls_res_db ):
#print "%5d: %s"%(i_id, i_name)
# :::~FIXME workaround for non-existing dbs
try:
curr_db = self.master_db.result_dbs[i]
except:
continue
if not curr_db: continue
try:
print "%5d: %s (%5.5f)"%(curr_db.id, self.shorten_name( curr_db.full_name ), curr_db.weight )
except:
print "%5d: %s (%5s)"%(curr_db.id, self.shorten_name( curr_db.full_name ), curr_db.weight )
ls_res_db = fnmatch.filter( os.listdir("."), "results*.sqlite" )
ls_res_db.extend( fnmatch.filter( os.listdir("."), "parameter*.dat" ) )
ls_res_db = self.filter_db_list( ls_res_db, filter = c )
if ls_res_db:
print " --- cwd dbs"
for i in sorted( ls_res_db ):
print " : %s "%self.shorten_name(i)
def do_load(self,c):
"""load DB_NAME|DB_ID
loads one of the registered databases from the master"""
c = c.split()
if len(c) >1:
utils.newline_msg("ERR", "only one db can be loaded at a time", 2)
return
ret = self.get_db_from_cmdline(c[0])
if ret:
self.current_param_db = ret
print " --- loaded db '%s'"%self.shorten_name( self.current_param_db.full_name )
else:
utils.newline_msg("ERR", "db does not exist", 2)
# :::~ FIXME!: Hast to be implemented
# def complete_load(self, text, line, begidx, endidx):
# return self.complete(text)
def do_info(self, c):
"""info REGEXP
prints the information of the results databases, filtered by a regular expression, or its id """
if not c:
if not self.current_param_db:
return
else:
ls_res_db = [ self.current_param_db.full_name ]
else:
ls_res_db = self.filter_db_list( filter = c )
if not ls_res_db: return
for i in ls_res_db:
curr_db = self.master_db.result_dbs[i]
self.master_db.update_result_db( curr_db )
print " ---%5d: %s"%( curr_db.id, os.path.relpath(curr_db.full_name,RUN_DIR) )
frac_done = float(curr_db.stat_processes_done) / float(curr_db.stat_values_set)
n_repet = curr_db.stat_values_set_with_rep/ curr_db.stat_values_set
print " -+ status: %s / total = %d*%d / D: %d (%.5f) - R: %d - E: %d "%(curr_db.status, curr_db.stat_values_set,n_repet , curr_db.stat_processes_done, frac_done, curr_db.stat_processes_running, curr_db.stat_processes_error )
print " -+ queue = %s / w=%5.5f "%(curr_db.queue, curr_db.weight )
def do_EOF(self, line):
return True
def do_shell(self, line):
"""Runs a shell command"""
output = os.popen(line).read()
print output
def do_cd(self,line):
""" Changes into a given directory """
try:
os.chdir(line)
except:
utils.newline_msg("DIR", "no directory named '%s'"%line, 2)
def complete_cd(self, text, line, begidx, endidx):
completions = filter(lambda x: os.path.isdir(x) , os.listdir(".") )
if text:
completions = [ f
for f in completions
if f.startswith(text)
]
return completions
def do_run_script(self,c):
"""executes a script file with commands accepted in this cmdline parser"""
if not os.path.exists(c):
utils.newline_msg("FIL", "file doesn't exist")
return
for l in open(c):
l = l.strip()
if not l: continue
if l[0] == "#": continue
self.onecmd(l.strip())
class QueueCommandParser(cmd.Cmd):
"""command processor for the queues."""
def __init__(self):
cmd.Cmd.__init__(self)
self.prompt = "| spg-queue :::~ "
self.current_queue = None
self.master_db = MasterDB()
#self.possible_keys = ['max_jobs', 'status']
self.__update_queue_list()
def __update_queue_list(self):
self.queues = self.master_db.execute_query("SELECT * FROM queues ORDER BY id")
def do_load(self,c):
"""loads a queue as active one"""
self.__update_queue_list()
if len( filter(lambda x: x[1] == c, self.queues) ):
self.current_queue = c
else:
newline_msg("ERR", "queue '%s' not found"%c)
print "loaded '%s'"%self.current_queue
def completedefault(self, text, line, begidx, endidx):
self.__update_queue_list()
completions = [ name for (id, name, max_jobs, status) in self.queues]
if text:
completions = [ f
for f in completions
if f.startswith(text)
]
return completions
def do_ls(self, c):
"""lists the queues in the master database """
self.__update_queue_list()
for (id, name, max_jobs, status) in self.queues:
print " %16s - J: %2d - S: '%s'"%( name, max_jobs, status)
### CREATE TABLE IF NOT EXISTS queues
### ( id INTEGER PRIMARY KEY, name CHAR(64), max_jobs INTEGER,
### status CHAR(1) )
def do_add(self, c):
"""adds queue queue"""
self.__update_queue_list()
if len( filter(lambda x: x[1] == c, self.queues) ):
newline_msg("ERR", "queue '%s' already exists"%queue, 2)
return
self.master_db.execute_query( "INSERT INTO queues (name, max_jobs, status) VALUES (?,?,?)",c, 1, 'S')
os.makedirs("%s/queue/%s"%(VAR_PATH,c))
self.current_queue = c
def do_set_max_jobs(self, c):
"""sets the maximum number of jobs in the given queue
usage: [regexp] N_JOBS"""
c = c.split()
if len(c) == 1 and self.current_queue:
max_jobs = int(c[0])
self.master_db.execute_query( 'UPDATE queues SET max_jobs= ? WHERE name = ?', max_jobs, self.current_queue )
elif len(c) == 2:
re = c[0]
max_jobs = int(c[1])
# print re, max_jobs
lsq = [ n for (id, n, mj, s) in self.queues ]
lsq = fnmatch.filter(lsq, re)
# print lsq
for q in lsq:
#print status, q
self.master_db.execute_query( 'UPDATE queues SET max_jobs= ? WHERE name = ?', max_jobs, q )
def do_clean_pool(cmd, name, params):
proc = Popen("rm -f %s/run/*"%VAR_PATH, shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE )
proc.wait()
self.__update_queue_list()
for (id, name, max_jobs, status) in self.queues:
proc = Popen("rm -f %s/queue/%s/*"%(VAR_PATH,name), shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE )
proc.wait()
def do_remove(self, c):
"""sets the maximum number of jobs in the given queue
usage: [regexp] N_JOBS"""
c = c.split()
if len(c) == 0 and self.current_queue:
self.master_db.execute_query( 'DELETE FROM queues WHERE name = ?', self.current_queue )
elif len(c) == 1:
re = c[0]
# print re, max_jobs
lsq = [ n for (id, n, mj, s) in self.queues ]
lsq = fnmatch.filter(lsq, re)
# print lsq
for q in lsq:
#print status, q
self.master_db.execute_query( 'DELETE FROM queues WHERE name = ?', q )
# ret = utils.parse_to_dict(c, allowed_keys=self.possible_keys)
# for k in ret:
# self.values[k] = ret[k]
# print "%s = %s" %(k,ret[k])
# if self.current_queue:
# if ret.has_key('max_jobs'):
# if ret.has_key('status'):
# self.master_db.execute_query( 'UPDATE queues SET status= ? WHERE name = ?', self.current_queue )
# self.master_db.execute( 'UPDATE dbs SET status="%s" WHERE id = %s'% ( self.current_param_db.status, self.current_param_db.id ) )
# self.doc_header = "default values: %s"%(self.values )
def __set_status(self, status, name = None):
#print name, status
if not name and self.current_queue:
# print 'UPDATE queues SET status= ? WHERE name = ?'
self.master_db.execute_query( 'UPDATE queues SET status= ? WHERE name = ?', status, self.current_queue )
return
lsq = [ n for (id, n, max_jobs, s) in self.queues ]
if name:
lsq = fnmatch.filter(lsq, name)
# print lsq
for q in lsq:
# print status, q
self.master_db.execute_query( 'UPDATE queues SET status= ? WHERE name = ?', status, q )
def do_stop(self, c):
"""stops the currently loaded registered database"""
self.__set_status('S', c)
def do_start(self, c):
"""starts the currently loaded registered database"""
self.__set_status('R', c)
def do_pause(self, c):
"""pauses the currently loaded registered database"""
self.__set_status('P', c)
def do_EOF(self, line):
return True
