class RunController(threading.Thread):
def __init__(self, log_level = logging.INFO, log_name = "MetLab"):
super(RunController, self).__init__()
self.log_name = log_name
self.log = logging.getLogger( log_name )
self.log.setLevel( log_level )
self._stop = threading.Event()
self.current = None
self.state = 'idle'
self.run_queue = {}
self.retval = {}
self.process_counter = 0
self.wd = None
self._init_db()
def _init_db(self):
for table in DATABASE_SCHEMA:
self._query( table )
def _query(self, query, *args):
con = None
data = None
try:
con = sqlite3.connect( DATABASE )
cur = con.cursor()
cur.execute(query, tuple(args))
data = cur.fetchall()
if not data:
con.commit()
except sqlite3.Error as e:
self.log.error("Database error: %s" % e)
except Exception as e:
self.log.error("Exception in _query: %s" % e)
finally:
if con:
con.close()
return data
def get_paths(self):
return self._query("SELECT `name`,`path` FROM paths")
def get_retval(self, pid):
if int(pid) in self.retval:
return self.retval[int(pid)]
return None
def queue(self, cmd):
pid = self.process_counter
if cmd[0][0] == '.':
cmd[0] = os.path.abspath(cmd[0])
self.run_queue[pid] = cmd
self.log.info("adding step: %s" % cmd)
if hasattr(self, "project_id"):
self._query("INSERT INTO steps(project_id, command) VALUES (?, ?)", str(self.project_id), " ".join(cmd))
self.process_counter += 1
return pid
def clear(self):
self.run_queue = {}
def run(self, wd = None):
self.log.info("Starting RunController")
try:
while True:
time.sleep(0.2)
if self._stop.isSet():
break
if self.current and self.current.status == 'completed':
if self.current.retval:
self.retval[self.current.pid] = self.current.retval
self.log.info("Retval: %s" % self.current.retval)
self.current_process = None
self.current.join()
self.current = None
if self.current and self.current.status in ['failed', 'aborted']:
self.log.error("%s %s" % (self.current.name, self.current.status))
self.current_process = None
self.current.join()
self.current = None
self.run_queue.clear()
if self.run_queue and not self.current:
pid = sorted(self.run_queue.keys())[0]
cmd = self.run_queue[pid]
if cmd[0] == 'mkdir':
try:
os.stat(cmd[1])
except:
os.makedirs(cmd[1])
elif cmd[0] == 'cd':
os.chdir(cmd[1])
else:
cwd = os.getcwd()
os.chdir(self.wd)
self.current = External(cmd[0], cmd[1:], pid = pid, log_name = self.log_name, wd=self.wd)
self.current.start()
while not self.current.started:
time.sleep(0.1)
os.chdir(cwd)
del self.run_queue[pid]
self.state = 'running'
if not self.run_queue and not self.current and self.state == 'running':
self.state = 'finished'
if self.current:
self.current.stop()
self.current.join()
except Exception as e:
print "RunController: %s" % e
self.log.info("RunController finishing")
def set_wd(self, dir_name):
self.log.info("Setting new WD to %s" % dir_name)
if dir_name[0] == ".":
self.wd = "%s%s" % (os.getcwd(), dir_name[1:])
elif dir_name[0] != "/":
self.wd = "%s/%s" % (os.getcwd(), dir_name)
else:
self.wd = dir_name
def stop(self):
self._stop.set()
def stop_current(self):
if self.current:
self.current.stop()
def start_project(self, name):
self.queue(["mkdir", name])
self._query("INSERT INTO projects (name, directory, started) VALUES ('%s', '%s', %i)" % (name, self.wd, time.time()))
self.project_id = self._query("SELECT id FROM projects ORDER BY id DESC LIMIT 1")[0][0]
self.log.info("Settings project_id = %i" % self.project_id)
class RunController(threading.Thread):
def __init__(self, log_level=logging.INFO, log_name="MetLab"):
super(RunController, self).__init__()
self.log_name = log_name
self.log = logging.getLogger(log_name)
self.log.setLevel(log_level)
self._stop = threading.Event()
self.current = None
self.state = 'idle'
self.run_queue = {}
self.retval = {}
self.process_counter = 0
self.wd = None
self._init_db()
def _init_db(self):
for table in DATABASE_SCHEMA:
self._query(table)
def _query(self, query, *args):
con = None
data = None
try:
con = sqlite3.connect(DATABASE)
cur = con.cursor()
cur.execute(query, tuple(args))
data = cur.fetchall()
if not data:
con.commit()
except sqlite3.Error as e:
self.log.error("Database error: %s" % e)
except Exception as e:
self.log.error("Exception in _query: %s" % e)
finally:
if con:
con.close()
return data
def get_paths(self):
return self._query("SELECT `name`,`path` FROM paths")
def get_retval(self, pid):
if int(pid) in self.retval:
return self.retval[int(pid)]
return None
def queue(self, cmd):
pid = self.process_counter
if cmd[0][0] == '.':
cmd[0] = os.path.abspath(cmd[0])
self.run_queue[pid] = cmd
self.log.info("adding step: %s" % cmd)
if hasattr(self, "project_id"):
self._query("INSERT INTO steps(project_id, command) VALUES (?, ?)",
str(self.project_id), " ".join(cmd))
self.process_counter += 1
return pid
def clear(self):
self.run_queue = {}
def run(self, wd=None):
self.log.info("Starting RunController")
try:
while True:
time.sleep(0.2)
if self._stop.isSet():
break
if self.current and self.current.status == 'completed':
if self.current.retval:
self.retval[self.current.pid] = self.current.retval
self.log.info("Retval: %s" % self.current.retval)
self.current_process = None
self.current.join()
self.current = None
if self.current and self.current.status == 'aborted':
self.log.warning("%s aborted without finishing" %
self.current.name)
if self.run_queue and not self.current:
pid = sorted(self.run_queue.keys())[0]
cmd = self.run_queue[pid]
if cmd[0] == 'mkdir':
try:
os.stat(cmd[1])
except:
os.makedirs(cmd[1])
elif cmd[0] == 'cd':
os.chdir(cmd[1])
else:
self.current = External(cmd[0],
cmd[1:],
pid=pid,
log_name=self.log_name)
cwd = os.getcwd()
if self.wd:
os.chdir(self.wd)
self.current.start()
while not self.current.started:
time.sleep(0.1)
os.chdir(cwd)
del self.run_queue[pid]
self.state = 'running'
if not self.run_queue and not self.current and self.state == 'running':
self.state = 'finished'
if self.current:
self.current.stop()
self.current.join()
except Exception as e:
print "RunController: %s" % e
self.log.info("RunController finishing")
def set_wd(self, dir_name):
if dir_name[0] == ".":
self.wd = "%s%s" % (os.getcwd(), dir_name[1:])
elif dir_name[0] != "/":
self.wd = "%s/%s" % (os.getcwd(), dir_name)
else:
self.wd = dir_name
def stop(self):
self._stop.set()
def stop_current(self):
if self.current:
self.current.stop()
def start_project(self, name):
self.queue(["mkdir", name])
self._query(
"INSERT INTO projects (name, directory, started) VALUES ('%s', '%s', %i)"
% (name, self.wd, time.time()))
self.project_id = self._query(
"SELECT id FROM projects ORDER BY id DESC LIMIT 1")[0][0]
self.log.info("Settings project_id = %i" % self.project_id)
