def __init__(self, rootdir, name, config_path=None):
assert (os.environ.has_key('JUNOTOP'))
assert (os.environ.has_key('DistJETPATH'))
self.id = None
self.flag = None
self.cfg = None
self.name = name
self.app_boot = []
self.res_dir = "" # the directory of result
self.args = {
} # key:value = key:index of command, value= list of args[]
self.data = {
} # key:value = key:index of command, value= list of data[]
self.input_path = None
self._task_id = 0
self.scheduler = None
self.specifiedWorker = None
self.log = logger.getLogger(self.name, applog=True)
if (config_path and os.path.exists(config_path)):
self.app_config = Config.AppConf(config_path)
self.log.info('[App] Load App configure file %s' % config_path)
else:
self.app_config = None
self.log.info('[App] No configure file need to be loaded...')
self.status = {'scheduler': None, 'boot': None, 'resdir': None}
#'data': None}
if os.path.exists(os.path.abspath(rootdir)):
self.rootdir = os.path.abspath(rootdir)
self.status['resdir'] = True
else:
self.log.error('Can not find root dir=%s' % rootdir)
def __init__(self, rootdir, name, config_path=None):
if not config_path or not os.path.exists(os.path.abspath(config_path)):
#self.log.warning('[ProdApp] Cannot find config file = %s, use default'%config_path)
config_path = os.environ[
'DistJETPATH'] + '/Application/ProdApp/config.ini'
super(ProdApp, self).__init__(rootdir, name, config_path)
self.task_reslist = {}
self.workflow = []
self.driver_dir = []
self.driver = {} # driver_name: scripts_list
self.sample_list = []
#self.JUNOTOP=os.environ['JUNOTOP']
self.setStatus('boot')
self.setStatus('data')
self.cfg = Config.AppConf(config_path, 'ProdApp')
self.njob = self.cfg.get('njobs')
if self.cfg.get('sample_list') is not None:
self.sample_list.extend(
self.cfg.get('sample_list').strip().split(' '))
else:
self.sample_list.extend(self.cfg.getSections())
self.tags = {}
for sample in self.sample_list:
self.tags[sample] = []
self.tags[sample].extend(
self.cfg.get('tags', sample).strip().split(' '))
def __init__(self, rootdir, name, config_path=None):
if not config_path or not os.path.exists(os.path.abspath(config_path)):
config_path = '/root/mpi/server/Application/AnalysisApp/config.ini'
super(AnaApp, self).__init__(rootdir, name, config_path=config_path)
self.step_script = script_dict
self.app_config = Config.AppConf(config_path)
self.anascp = {}
self.data_path = self.app_config.get('input_dir')
self.workflow = self.app_config.get('workflow').split()
self.__load = True
for step in self.workflow:
if self.app_config.get(step) is not None:
self.step_script[step] = self.app_config.get(step)
if not os.path.exists(self.data_path):
self.log.error("[Analysis App] Data path not found, failed")
self.__load = False
else:
self.setStatus('boot')
self.setStatus('data')
from python.JobMaster import JobMaster
applications = []
if module.__dict__.has_key('run') and callable(module.__dict__['run']):
app = module.run(app_config_path)
print app
applications.extend(app)
else:
print('@master: No callable function "run" in app module, exit')
exit()
if config_path == 'null' or not os.path.exists(os.path.abspath(config_path)):
print('@master: Cannot find configuration file [%s]' %
os.path.abspath(config_path))
config_path = os.getenv('DistJETPATH') + '/config/config.ini'
Conf.set_inipath(config_path)
cfg = Conf.Config()
master = JobMaster(applications=applications)
if master.getRunFlag():
#with open("running.log","w+") as rf:
# rf.write('True')
# rf.flush()
print('@master: start running')
master.startProcessing()
else:
print('@master: Load Application error, exit')
if os.path.exists(os.environ['DistJETPATH'] + '/config.ini'):
os.remove(os.environ['DistJETPATH'] + '/config.ini')
import python.Util.logger as logger
if len(sys.argv) >= 4 and sys.argv[3] in ['info','debug']:
logger.setlevel(sys.argv[3])
else:
logger.setlevel('info')
if len(sys.argv)>=5 and sys.argv[4] in [True,False]:
logger.setConsole(sys.argv[4])
else:
logger.setConsole(False)
import python.Util.Config as CONF
#CONF.Config.setCfg('Rundir',os.getcwd())
cfg_path = sys.argv[2]
if sys.argv[2] != 'null' and not os.path.exists(sys.argv[2]):
CONF.set_inipath(os.path.abspath(sys.argv[2]))
CONF.set_inipath(cfg_path)
cfg = CONF.Config()
'''
import time
times=0
while 'running.log' not in os.listdir('.'):
time.sleep(1)
times+=1
if times > 10:
print('@agent: Agent cannot find master,exit')
exit()
continue
'''
import python.WorkerAgent as WA