def main(g_params):#{{{
submitjoblogfile = "%s/submitted_seq.log"%(path_log)
runjoblogfile = "%s/runjob_log.log"%(path_log)
finishedjoblogfile = "%s/finished_job.log"%(path_log)
if not os.path.exists(path_cache):
os.mkdir(path_cache)
loop = 0
while 1:
if os.path.exists("%s/CACHE_CLEANING_IN_PROGRESS"%(path_result)):#pause when cache cleaning is in progress
continue
# load the config file if exists
configfile = "%s/config/config.json"%(basedir)
config = {}
if os.path.exists(configfile):
text = myfunc.ReadFile(configfile)
config = json.loads(text)
if rootname_progname in config:
g_params.update(config[rootname_progname])
if os.path.exists(black_iplist_file):
g_params['blackiplist'] = myfunc.ReadIDList(black_iplist_file)
os.environ['TZ'] = g_params['TZ']
time.tzset()
avail_computenode = webcom.ReadComputeNode(computenodefile) # return value is a dict
g_params['vip_user_list'] = myfunc.ReadIDList2(vip_email_file, col=0)
num_avail_node = len(avail_computenode)
webcom.loginfo("loop %d"%(loop), gen_logfile)
isOldRstdirDeleted = False
if loop % g_params['STATUS_UPDATE_FREQUENCY'][0] == g_params['STATUS_UPDATE_FREQUENCY'][1]:
qdcom.RunStatistics_basic(webserver_root, gen_logfile, gen_errfile)
isOldRstdirDeleted = webcom.DeleteOldResult(path_result, path_log,
gen_logfile, MAX_KEEP_DAYS=g_params['MAX_KEEP_DAYS'])
webcom.CleanServerFile(path_static, gen_logfile, gen_errfile)
if 'DEBUG_ARCHIVE' in g_params and g_params['DEBUG_ARCHIVE']:
webcom.loginfo("Run ArchiveLogFile, path_log=%s, threshold_logfilesize=%d"%(path_log, threshold_logfilesize), gen_logfile)
webcom.ArchiveLogFile(path_log, threshold_logfilesize=threshold_logfilesize)
qdcom.CreateRunJoblog(loop, isOldRstdirDeleted, g_params)
# Get number of jobs submitted to the remote server based on the
# runjoblogfile
runjobidlist = myfunc.ReadIDList2(runjoblogfile,0)
remotequeueDict = {}
for node in avail_computenode:
remotequeueDict[node] = []
for jobid in runjobidlist:
rstdir = "%s/%s"%(path_result, jobid)
remotequeue_idx_file = "%s/remotequeue_seqindex.txt"%(rstdir)
if os.path.exists(remotequeue_idx_file):
content = myfunc.ReadFile(remotequeue_idx_file)
lines = content.split('\n')
for line in lines:
strs = line.split('\t')
if len(strs)>=5:
node = strs[1]
remotejobid = strs[2]
if node in remotequeueDict:
remotequeueDict[node].append(remotejobid)
cntSubmitJobDict = {} # format of cntSubmitJobDict {'node_ip': [INT, INT, STR]}
for node in avail_computenode:
queue_method = avail_computenode[node]['queue_method']
num_queue_job = len(remotequeueDict[node])
if num_queue_job >= 0:
cntSubmitJobDict[node] = [num_queue_job,
g_params['MAX_SUBMIT_JOB_PER_NODE'], queue_method]
else:
cntSubmitJobDict[node] = [g_params['MAX_SUBMIT_JOB_PER_NODE'],
g_params['MAX_SUBMIT_JOB_PER_NODE'], queue_method]
# entries in runjoblogfile includes jobs in queue or running
hdl = myfunc.ReadLineByBlock(runjoblogfile)
if not hdl.failure:
lines = hdl.readlines()
while lines != None:
for line in lines:
strs = line.split("\t")
if len(strs) >= 11:
jobid = strs[0]
email = strs[4]
try:
numseq = int(strs[5])
except:
numseq = 1
try:
numseq_this_user = int(strs[10])
except:
numseq_this_user = 1
rstdir = "%s/%s"%(path_result, jobid)
finishtagfile = "%s/%s"%(rstdir, "runjob.finish")
status = strs[1]
webcom.loginfo("CompNodeStatus: %s"%(str(cntSubmitJobDict)), gen_logfile)
runjob_lockfile = "%s/%s/%s.lock"%(path_result, jobid, "runjob.lock")
if os.path.exists(runjob_lockfile):
msg = "runjob_lockfile %s exists, ignore the job %s" %(runjob_lockfile, jobid)
webcom.loginfo(msg, gen_logfile)
continue
#if IsHaveAvailNode(cntSubmitJobDict):
if not g_params['DEBUG_NO_SUBMIT']:
qdcom.SubmitJob(jobid, cntSubmitJobDict, numseq_this_user, g_params)
qdcom.GetResult(jobid, g_params) # the start tagfile is written when got the first result
qdcom.CheckIfJobFinished(jobid, numseq, email, g_params)
lines = hdl.readlines()
hdl.close()
myfunc.WriteFile("sleep for %d seconds\n"%(g_params['SLEEP_INTERVAL']), gen_logfile, "a", True)
time.sleep(g_params['SLEEP_INTERVAL'])
loop += 1
return 0
from libpredweb import myfunc
if __name__ == '__main__':
progname=os.path.basename(sys.argv[0])
general_usage = """
usage: %s TESTMODE options
"""%(sys.argv[0])
numArgv = len(sys.argv)
if numArgv <= 1:
print(general_usage)
sys.exit(1)
TESTMODE=sys.argv[1]
if TESTMODE == "readcomputenode":
infile = sys.argv[2]
dt = webcom.ReadComputeNode(infile)
print(dt)
if TESTMODE == "pdb2seq":
pdbfile = sys.argv[2]
seq = myfunc.PDB2Seq(pdbfile)
print(seq)
if TESTMODE == "urlretrieve":
url = sys.argv[2]
outfile = sys.argv[3]
timeout = int(sys.argv[4])
try:
myfunc.urlretrieve(url, outfile, timeout)
except Exception as e:
print("retrieve %s failed with errmsg=%s"%(url, str(e)) )