def timer(timestampqueue: mp.Queue, writeconnectioninfo):
lsttime = None
granularity = 60
thispid = os.getpid()
pidregister(thispid, writeconnectioninfo)
# ###
# debugtimenow = dt.datetime \
# .now(pytz.timezone('America/Chicago')) \
# .replace(tzinfo=None, second=0, microsecond=0)
# timestampqueue.put(debugtimenow)
# ###
try:
while True:
timenow = dt.datetime \
.now(pytz.timezone('America/Chicago')) \
.replace(tzinfo=None, microsecond=0)
if lsttime is None:
lsttime = timenow.replace(second=0)
if (timenow - lsttime).seconds / granularity >= 1:
lsttime = timenow.replace(second=0)
timestampqueue.put(timenow)
time.sleep(0.1)
finally:
query = "UPDATE schedulerDB.realtime_status SET pid_timer={};".format(
0)
mycon.db_write(writeconnectioninfo, query)
def register(EXTERNAL_IP):
dbconnectioninfo = [DATABASES['scheduler_control']['HOST'],
int(DATABASES['scheduler_control']['PORT']),
DATABASES['scheduler_control']['USER'],
DATABASES['scheduler_control']['PASSWORD'],
DATABASES['scheduler_control']['NAME']]
#EXTERNAL_IP = ipgetter.myip()
querystr='''select count(*) from {}.registered_scheduler
where machine_ip = \'{}\''''.format(dbconnectioninfo[4],EXTERNAL_IP)
res = mycon.db_read(dbconnectioninfo, querystr)[0][0]
if res == 1: ## Exists
querystr='''UPDATE {}.registered_scheduler SET
machine_port={}, machine_role='{}'
where machine_ip = '{}' '''\
.format(dbconnectioninfo[4],SOCKET_SERVER_PORT, NODE_TYPE, EXTERNAL_IP)
mycon.db_write(dbconnectioninfo, querystr)
else: ## Not Exists
querystr='''INSERT INTO {}.registered_scheduler
(machine_ip, machine_port, machine_role) VALUES ('{}',{},'{}')'''\
.format(dbconnectioninfo[4],EXTERNAL_IP, SOCKET_SERVER_PORT, NODE_TYPE)
mycon.db_write(dbconnectioninfo, querystr)
def send_dep_pass(self, conninfo):
assert self.showid is not None, 'showid is None'
timenowstr = dt.datetime.now(
pytz.timezone('America/Chicago')).strftime("%Y-%m-%d %H:%M:%S")
querystr = "UPDATE schedulerDB.jobchain_lst_run SET dep_pass_ts=\'{}\' WHERE id={};".format(
timenowstr, self.showid)
mycon.db_write(conninfo, querystr)
def scheduler_status_init(self, thispid):
bkpflag = self.envinfo[0]
typestr = 'main'
if bkpflag == 1:
typestr = 'bkp'
query = "INSERT INTO schedulerDB.realtime_status (name,pid_index,debugmode) " \
"VALUES (\'{}\',{},\'{}\');".format(typestr, thispid, 'debug')
myconn.db_write(self.writeconnectioninfo, query)
def sendlog(writeconninfo, msg, showid=None):
msgstr = str(msg)[-4000:]
msgstr.strip()
msgstr = msgstr.replace('\'', '\"')
msgstr = MySQLdb.escape_string(msgstr)
querystr = "INSERT INTO schedulerDB.schedulerMessage (msg) VALUES (\'{}\')".format(
msgstr.decode('ascii', 'ignore'))
if showid != None:
querystr = "INSERT INTO schedulerDB.schedulerMessage " \
"(run_id, msg) VALUES ({},\'{}\')".format(showid, msgstr.decode('ascii', 'ignore'))
mycon.db_write(writeconninfo, querystr)
def logsubmit(writeconninfo, jc_id, cond, showid):
if cond == 0:
cond = 0
log = 'Success'
else:
cond = 1
log = 'Failed'
timenowstr = dt.datetime.now(pytz.timezone('America/Chicago')).strftime("%Y-%m-%d %H:%M:%S")
# querystr = "INSERT INTO schedulerDB.logtable (log,jobchain_id,run_result,entry_time) \
# VALUES (\"{}\",{},{},\'{}\');".format(log, jc_id, cond, timenowstr)
# cur.execute(querystr)
querystr = "UPDATE schedulerDB.jobchain_lst_run SET last_run_end=\'{}\', status=\'{}\' \
WHERE id={};".format(timenowstr, log, showid)
mycon.db_write(writeconninfo, querystr)
def sendlstrun(self, conninfo, timestr, pid): # dbwrite
querystr = "INSERT INTO schedulerDB.jobchain_lst_run (jobchain_id, last_run_start, pid) " \
"VALUES ({},\'{}\', {});".format(
self.id,
timestr,
pid
)
self.showid = mycon.db_write(conninfo, querystr)
def idle(self):
bkpflag = self.envinfo[0]
typestr = 'main'
if bkpflag == 1:
typestr = 'bkp'
try:
while True:
query = "UPDATE schedulerDB.realtime_status SET timequeuesize={}, replyqueuesize={}, " \
"index_last_ping_time=NOW() " \
"WHERE name = '{}';".format(self.timestampqueue.qsize(),
self.replyqueue.qsize(),
typestr)
myconn.db_write(self.writeconnectioninfo, query)
time.sleep(20)
finally:
query = "UPDATE schedulerDB.realtime_status SET pid_index={} WHERE name = '{}';".format(
0, typestr)
myconn.db_write(self.writeconnectioninfo, query)
def receiver(replyqueue, writeconnectioninfo, tracklck,
runningjobdict):
thispid = os.getpid()
pidregister(thispid,writeconnectioninfo)
try:
while True:
recv = replyqueue.get()
tracklck.acquire()
try:
runningjobdict[str(recv[0])] = 0
finally:
tracklck.release()
if recv[1] != 0:
util.sendlog(writeconnectioninfo, recv[2], recv[3])
util.logsubmit(writeconnectioninfo, recv[0], recv[1], recv[3])
finally:
query = "UPDATE schedulerDB.realtime_status SET pid_receiver={};".format(0)
mycon.db_write(writeconnectioninfo, query)
def scheduler_status_check(self):
bkpflag = self.envinfo[0]
thispid = os.getpid()
retflag = False
typestr = 'main'
if bkpflag == 1:
typestr = 'bkp'
query = "SELECT pid_index FROM schedulerDB.realtime_status " \
"WHERE name = '{}';".format(typestr)
retans = myconn.db_read(self.writeconnectioninfo, query)
if len(retans) == 0:
self.scheduler_status_init(thispid)
retflag = True
else:
dbpid = retans[0][0]
if dbpid == 0:
query = "UPDATE schedulerDB.realtime_status SET pid_index={} WHERE name = '{}';".format(
thispid, typestr)
myconn.db_write(self.writeconnectioninfo, query)
retflag = True
return retflag
def sqlfetcher(timestampqueue: mp.Queue, replyqueue, readconnectioninfo, writeconnectioninfo, envinfo,
tracklck, runningjobdict
):
thispid = os.getpid()
pidregister(thispid, writeconnectioninfo)
bkpflag = envinfo[0]
runningprocessdict = {}
typestr = 'main'
flagstr = 'main_flag'
if bkpflag == 1:
typestr = 'bkp'
flagstr = 'BKP_flag'
try:
while True:
exetime = timestampqueue.get() # get timestamp
query = \
"SELECT id, timestr, dep_ot ,job_desc " \
"FROM schedulerDB.jobchain_table " \
"WHERE switch='ON' and {}=1; ".format(flagstr)
read_cur = mycon.db_read(readconnectioninfo, query)
query = \
"SELECT jobchain_id, MAX(last_run_start) FROM schedulerDB.jobchain_lst_run " \
"GROUP BY jobchain_id ORDER BY jobchain_id;"
write_cur = mycon.db_read(writeconnectioninfo, query)
if len(write_cur) == 0:
tempnp = np.array([[-1, -1]])
else:
tempnp = np.array(write_cur)
pdframe = pd.DataFrame(tempnp, index=tempnp[:, 0])
jobChainlst = []
for row in read_cur:
lstrun = None
if row[0] in pdframe.index:
lstrun = pdframe.loc[[row[0]]].values[0, 1]
jobChainlst.append(mj.Myjobchain(row[0], row[1], lstrun, row[2], row[3]))
errorlst = []
for eachjc in jobChainlst:
if jobchain_timestrcheck(eachjc) is False:
errorlst.append((1, eachjc))
## Turn off that jobchain
querystr = '''update jobchain_table set switch = 'OFF' where id = {};'''.format(eachjc.id)
mycon.db_write(writeconnectioninfo, querystr)
continue
if jobcheck(eachjc, exetime) is False:
continue
tracklck.acquire()
try:
if str(eachjc.id) in runningjobdict and runningjobdict[str(eachjc.id)] == 1:
errorlst.append((0, eachjc))
else:
runningjobdict[str(eachjc.id)] = 1
# worker.worker(eachjc, replyqueue,
# readconnectioninfo,
# writeconnectioninfo,
# envinfo)
p = mp.Process(target=worker.worker,
args=(eachjc, replyqueue,
readconnectioninfo,
writeconnectioninfo,
envinfo))
runningprocessdict[str(eachjc.id)] = p
runningprocessdict[str(eachjc.id)].start()
finally:
tracklck.release()
for eacherrjc in errorlst:
if eacherrjc[0] == 0:
util.sendlog(writeconnectioninfo, 'Job Id: {} not finished'.format(eacherrjc[1].id))
elif eacherrjc[0] == 1:
util.sendlog(writeconnectioninfo, 'Job Id: {} invalid timestr'.format(eacherrjc[1].id))
finally:
query = "UPDATE schedulerDB.realtime_status SET pid_sqlfetcher={};".format(0)
mycon.db_write(writeconnectioninfo, query)
def pidregister(thispid, conninfo):
query = "UPDATE schedulerDB.realtime_status SET pid_sqlfetcher={};".format(thispid)
mycon.db_write(conninfo, query)