def __init__(self, connection = None): MasterDB.__init__(self, connection) self.current_counter = 0 res = self.cursor.execute("SELECT last FROM infiles WHERE id = 1").fetchone() if res == None: self.cursor.execute("INSERT INTO infiles (last) VALUES (0)") self.connection.commit()
def __init__(self): cmd.Cmd.__init__(self) self.prompt = "| spg-queue :::~ " self.current_queue = None self.master_db = MasterDB() #self.possible_keys = ['max_jobs', 'status'] self.__update_queue_list()
def __init__(self, EnsembleConstructor = ParameterEnsemble): cmd.Cmd.__init__(self) self.EnsembleConstructor = EnsembleConstructor self.current_param_db = None self.master_db = MasterDB(EnsembleConstructor = EnsembleConstructor)
class BaseDBCommandParser(cmd.Cmd): """DB command handler""" def __init__(self, EnsembleConstructor = ParameterEnsemble): cmd.Cmd.__init__(self) self.EnsembleConstructor = EnsembleConstructor self.current_param_db = None self.master_db = MasterDB(EnsembleConstructor = EnsembleConstructor) def parse_command_line(self, st): """returns command the flags set under a command and the arguments""" cmd = [] flags = set() for ic in st.strip().split(): if ic[0] == "-": flags.add( ic.strip("-") ) else: cmd.append( ic ) return flags, cmd def shorten_name(self, st): return os.path.relpath(st,RUN_DIR) def lengthen_name(self, st): return "%s/%s"%(RUN_DIR, st) def translate_name( self,st): """translates the parameters filename and the database name into the other and viceversa (returns a duple: param_name, db_name)""" full_name = os.path.realpath( st ) # if not os.path.exists(full_name): # full_name = self.lengthen_name(full_name) if not os.path.exists(full_name): utils.newline_msg("ERR","database '%s' does not exist"%st) return None path, st = os.path.split(full_name) if ".sqlite" in st: par_name = st.replace("results","").replace(".sqlite","") par_name = "parameters%s.dat"%par_name return "%s/%s"%(path,par_name) , "%s/%s"%(path,st) else: db_name = st.replace("parameters","").replace(".dat","") db_name = "results%s.sqlite"%db_name return "%s/%s"%(path,st) , "%s/%s"%(path,db_name) def update_active_result_db(self, c): c = c.strip() if not c: return try: param_name, db_name = self.translate_name(c) except: # utils.newline_msg("ERR", "results db '%s' doesn't exist. Can not load it" ) return if os.path.exists( db_name ): self.current_param_db = ParameterEnsemble( db_name ) elif os.path.exists( param_name ) and not os.path.exists( db_name ): self.current_param_db = ParameterEnsemble( db_name , db_init = False) return def filter_db_list(self, ls = None, filter = None): if ls == None: ls = self.master_db.result_dbs.keys() try: id = int(filter) rdb = self.master_db.result_dbs filtered = [ x for x in ls if rdb.has_key(x) and rdb[x] is not None and rdb[x].id == id ] return filtered except: ret = [ self.shorten_name(i) for i in ls ] if filter: ret = fnmatch.filter(ret, filter) return sorted( [ self.lengthen_name( i ) for i in ret ] ) def get_db_from_cmdline(self, c): """it returns the db name (or None) of a database identified either from its id or """ try: id = int(c) rdb = self.master_db.result_dbs filtered = [ rdb[x] for x in rdb.keys() if rdb[x] is not None and rdb[x].id == id ] if filtered: return filtered[0] else: return None except: try: foo, db_name = self.translate_name(c) except: utils.newline_msg("ERR", "results db '%s' doesn't exist."%c ) return if self.master_db.result_dbs.has_key(db_name): return self.master_db.result_dbs[db_name] else: return self.EnsembleConstructor(db_name, init_db = True) utils.newline_msg("WRN", "database '%s' is not registered, loading it anyhow"%self.shorten_name(db_name)) return None # :::~ FIXME!: Hast to be implemented # def complete(self, text): # completions = self.master_db.result_dbs.keys() # if text: # completions = [ f # for f in completions # if f.startswith(text) # ] # return completions def do_ls(self, c): """ls REGEXP|DB_ID lists the databases already registered in the master database and the possible ones found in the current directory""" ls_res_db = self.filter_db_list( filter = c ) if ls_res_db: print " --- registered dbs" for i in sorted( ls_res_db ): #print "%5d: %s"%(i_id, i_name) # :::~FIXME workaround for non-existing dbs try: curr_db = self.master_db.result_dbs[i] except: continue if not curr_db: continue try: print "%5d: %s (%5.5f)"%(curr_db.id, self.shorten_name( curr_db.full_name ), curr_db.weight ) except: print "%5d: %s (%5s)"%(curr_db.id, self.shorten_name( curr_db.full_name ), curr_db.weight ) ls_res_db = fnmatch.filter( os.listdir("."), "results*.sqlite" ) ls_res_db.extend( fnmatch.filter( os.listdir("."), "parameter*.dat" ) ) ls_res_db = self.filter_db_list( ls_res_db, filter = c ) if ls_res_db: print " --- cwd dbs" for i in sorted( ls_res_db ): print " : %s "%self.shorten_name(i) def do_load(self,c): """load DB_NAME|DB_ID loads one of the registered databases from the master""" c = c.split() if len(c) >1: utils.newline_msg("ERR", "only one db can be loaded at a time", 2) return ret = self.get_db_from_cmdline(c[0]) if ret: self.current_param_db = ret print " --- loaded db '%s'"%self.shorten_name( self.current_param_db.full_name ) else: utils.newline_msg("ERR", "db does not exist", 2) # :::~ FIXME!: Hast to be implemented # def complete_load(self, text, line, begidx, endidx): # return self.complete(text) def do_info(self, c): """info REGEXP prints the information of the results databases, filtered by a regular expression, or its id """ if not c: if not self.current_param_db: return else: ls_res_db = [ self.current_param_db.full_name ] else: ls_res_db = self.filter_db_list( filter = c ) if not ls_res_db: return for i in ls_res_db: curr_db = self.master_db.result_dbs[i] self.master_db.update_result_db( curr_db ) print " ---%5d: %s"%( curr_db.id, os.path.relpath(curr_db.full_name,RUN_DIR) ) frac_done = float(curr_db.stat_processes_done) / float(curr_db.stat_values_set) n_repet = curr_db.stat_values_set_with_rep/ curr_db.stat_values_set print " -+ status: %s / total = %d*%d / D: %d (%.5f) - R: %d - E: %d "%(curr_db.status, curr_db.stat_values_set,n_repet , curr_db.stat_processes_done, frac_done, curr_db.stat_processes_running, curr_db.stat_processes_error ) print " -+ queue = %s / w=%5.5f "%(curr_db.queue, curr_db.weight ) def do_EOF(self, line): return True def do_shell(self, line): """Runs a shell command""" output = os.popen(line).read() print output def do_cd(self,line): """ Changes into a given directory """ try: os.chdir(line) except: utils.newline_msg("DIR", "no directory named '%s'"%line, 2) def complete_cd(self, text, line, begidx, endidx): completions = filter(lambda x: os.path.isdir(x) , os.listdir(".") ) if text: completions = [ f for f in completions if f.startswith(text) ] return completions def do_run_script(self,c): """executes a script file with commands accepted in this cmdline parser""" if not os.path.exists(c): utils.newline_msg("FIL", "file doesn't exist") return for l in open(c): l = l.strip() if not l: continue if l[0] == "#": continue self.onecmd(l.strip())
class QueueCommandParser(cmd.Cmd): """command processor for the queues.""" def __init__(self): cmd.Cmd.__init__(self) self.prompt = "| spg-queue :::~ " self.current_queue = None self.master_db = MasterDB() #self.possible_keys = ['max_jobs', 'status'] self.__update_queue_list() def __update_queue_list(self): self.queues = self.master_db.execute_query("SELECT * FROM queues ORDER BY id") def do_load(self,c): """loads a queue as active one""" self.__update_queue_list() if len( filter(lambda x: x[1] == c, self.queues) ): self.current_queue = c else: newline_msg("ERR", "queue '%s' not found"%c) print "loaded '%s'"%self.current_queue def completedefault(self, text, line, begidx, endidx): self.__update_queue_list() completions = [ name for (id, name, max_jobs, status) in self.queues] if text: completions = [ f for f in completions if f.startswith(text) ] return completions def do_ls(self, c): """lists the queues in the master database """ self.__update_queue_list() for (id, name, max_jobs, status) in self.queues: print " %16s - J: %2d - S: '%s'"%( name, max_jobs, status) ### CREATE TABLE IF NOT EXISTS queues ### ( id INTEGER PRIMARY KEY, name CHAR(64), max_jobs INTEGER, ### status CHAR(1) ) def do_add(self, c): """adds queue queue""" self.__update_queue_list() if len( filter(lambda x: x[1] == c, self.queues) ): newline_msg("ERR", "queue '%s' already exists"%queue, 2) return self.master_db.execute_query( "INSERT INTO queues (name, max_jobs, status) VALUES (?,?,?)",c, 1, 'S') os.makedirs("%s/queue/%s"%(VAR_PATH,c)) self.current_queue = c def do_set_max_jobs(self, c): """sets the maximum number of jobs in the given queue usage: [regexp] N_JOBS""" c = c.split() if len(c) == 1 and self.current_queue: max_jobs = int(c[0]) self.master_db.execute_query( 'UPDATE queues SET max_jobs= ? WHERE name = ?', max_jobs, self.current_queue ) elif len(c) == 2: re = c[0] max_jobs = int(c[1]) # print re, max_jobs lsq = [ n for (id, n, mj, s) in self.queues ] lsq = fnmatch.filter(lsq, re) # print lsq for q in lsq: #print status, q self.master_db.execute_query( 'UPDATE queues SET max_jobs= ? WHERE name = ?', max_jobs, q ) def do_clean_pool(cmd, name, params): proc = Popen("rm -f %s/run/*"%VAR_PATH, shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE ) proc.wait() self.__update_queue_list() for (id, name, max_jobs, status) in self.queues: proc = Popen("rm -f %s/queue/%s/*"%(VAR_PATH,name), shell = True, stdin = PIPE, stdout = PIPE, stderr = PIPE ) proc.wait() def do_remove(self, c): """sets the maximum number of jobs in the given queue usage: [regexp] N_JOBS""" c = c.split() if len(c) == 0 and self.current_queue: self.master_db.execute_query( 'DELETE FROM queues WHERE name = ?', self.current_queue ) elif len(c) == 1: re = c[0] # print re, max_jobs lsq = [ n for (id, n, mj, s) in self.queues ] lsq = fnmatch.filter(lsq, re) # print lsq for q in lsq: #print status, q self.master_db.execute_query( 'DELETE FROM queues WHERE name = ?', q ) # ret = utils.parse_to_dict(c, allowed_keys=self.possible_keys) # for k in ret: # self.values[k] = ret[k] # print "%s = %s" %(k,ret[k]) # if self.current_queue: # if ret.has_key('max_jobs'): # if ret.has_key('status'): # self.master_db.execute_query( 'UPDATE queues SET status= ? WHERE name = ?', self.current_queue ) # self.master_db.execute( 'UPDATE dbs SET status="%s" WHERE id = %s'% ( self.current_param_db.status, self.current_param_db.id ) ) # self.doc_header = "default values: %s"%(self.values ) def __set_status(self, status, name = None): #print name, status if not name and self.current_queue: # print 'UPDATE queues SET status= ? WHERE name = ?' self.master_db.execute_query( 'UPDATE queues SET status= ? WHERE name = ?', status, self.current_queue ) return lsq = [ n for (id, n, max_jobs, s) in self.queues ] if name: lsq = fnmatch.filter(lsq, name) # print lsq for q in lsq: # print status, q self.master_db.execute_query( 'UPDATE queues SET status= ? WHERE name = ?', status, q ) def do_stop(self, c): """stops the currently loaded registered database""" self.__set_status('S', c) def do_start(self, c): """starts the currently loaded registered database""" self.__set_status('R', c) def do_pause(self, c): """pauses the currently loaded registered database""" self.__set_status('P', c) def do_EOF(self, line): return True