def __init__(self, cfg_params, jobs): self.cfg_params = cfg_params self.copy_data = int(cfg_params.get("USER.copy_data", 0)) self.check_RemoteDir = int(cfg_params.get("USER.check_user_remote_dir", 0)) nj_list = [] # get updated status from server try: from StatusServer import StatusServer stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass if jobs == "bad": nj_list = self.checkBadJob(nj_list) else: nj_list = self.checkAllowedJob(jobs, nj_list) SubmitterServer.__init__(self, cfg_params, nj_list, "range") return
def run(self): # get updated status from server try: stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass # check whether the action is allowable self.check() # notify the server to clean the task csCommunicator = ServerCommunicator(self.server_name, self.server_port, self.cfg_params) taskuuid = str(common._db.queryTask('name')) try: csCommunicator.cleanTask(taskuuid) except Exception, e: msg = "Client Server comunication failed about cleanJobs: task \n" + taskuuid msg += "Only local working directory will be removed." common.logger.debug(msg) pass
def __init__(self, cfg_params, jobs): self.cfg_params = cfg_params self.copy_data = int(cfg_params.get('USER.copy_data', 0)) self.check_RemoteDir = int( cfg_params.get('USER.check_user_remote_dir', 0)) nj_list = [] # get updated status from server try: from StatusServer import StatusServer stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass if (jobs == 'bad'): nj_list = self.checkBadJob(nj_list) else: nj_list = self.checkAllowedJob(jobs, nj_list) SubmitterServer.__init__(self, cfg_params, nj_list, 'range') return
def run(self): """ The main method of the class: kill a complete task """ common.logger.debug("Killer::run() called") # get updated status from server #inherited from StatusServer try: from StatusServer import StatusServer stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass task = common._db.getTask(self.range) toBeKilled = [] for job in task.jobs: #if job.runningJob['status'] not in ['C','E','KK','K','SU','SA','NS']: # commented for fast-kill at registration ,'SSE']: if job.runningJob['state'] in ['SubSuccess', 'SubRequested']: toBeKilled.append(job['jobId']) else: common.logger.info("Not possible to kill Job #"+str(job['jobId'])+\ " : Last action was: "+str(job.runningJob['state'])+\ " Status is "+str(job.runningJob['statusScheduler'])) pass if len(toBeKilled) > 0: ## register proxy ## csCommunicator = ServerCommunicator(self.server_name, self.server_port, self.cfg_params) taskuuid = str(common._db.queryTask('name')) ret = csCommunicator.killJobs(taskuuid, toBeKilled) del csCommunicator if ret != 0: msg = "ClientServer ERROR: %d raised during the communication.\n" % ret raise CrabException(msg) # printout the command result common._db.updateRunJob_(toBeKilled, [{ 'state': 'KillRequested' }] * len(toBeKilled)) common.logger.info( "Kill request for %d jobs succesfully sent to the server\n" % len(toBeKilled)) return
def run(self): """ The main method of the class: kill a complete task """ common.logger.debug( "Killer::run() called") # get updated status from server #inherited from StatusServer try: from StatusServer import StatusServer stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass task = common._db.getTask(self.range) toBeKilled = [] for job in task.jobs: #if job.runningJob['status'] not in ['C','E','KK','K','SU','SA','NS']: # commented for fast-kill at registration ,'SSE']: if job.runningJob['state'] in ['SubSuccess','SubRequested']: toBeKilled.append(job['jobId']) else: common.logger.info("Not possible to kill Job #"+str(job['jobId'])+\ " : Last action was: "+str(job.runningJob['state'])+\ " Status is "+str(job.runningJob['statusScheduler'])) pass if len(toBeKilled)>0: ## register proxy ## csCommunicator = ServerCommunicator(self.server_name, self.server_port, self.cfg_params) taskuuid = str(common._db.queryTask('name')) ret = csCommunicator.killJobs( taskuuid, toBeKilled) del csCommunicator if ret != 0: msg = "ClientServer ERROR: %d raised during the communication.\n"%ret raise CrabException(msg) # printout the command result common._db.updateRunJob_(toBeKilled, [{'state':'KillRequested'}]*len(toBeKilled)) common.logger.info("Kill request for %d jobs succesfully sent to the server\n"%len(toBeKilled) ) return
def __init__(self, cfg_params, jobs): self.cfg_params = cfg_params nj_list = [] # get updated status from server try: from StatusServer import StatusServer stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass nj_list = self.checkAllowedJob(jobs, nj_list) SubmitterServer.__init__(self, cfg_params, nj_list, 'range') return
def collectLogging(self): # get updated status from server try: from StatusServer import StatusServer stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass #create once storage interaction object seEl = None loc = None try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug( str(ex)) msg = "ERROR: Unable to create SE source interface \n" raise CrabException(msg)
def __init__(self, cfg_params, jobs): self.cfg_params = cfg_params nj_list = [] # get updated status from server try: from StatusServer import StatusServer stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass nj_list = self.checkAllowedJob(jobs,nj_list) SubmitterServer.__init__(self, cfg_params, nj_list, 'range') return
def collectLogging(self): # get updated status from server try: from StatusServer import StatusServer stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass #create once storage interaction object seEl = None loc = None try: seEl = SElement(self.storage_name, self.storage_proto, self.storage_port) except Exception, ex: common.logger.debug(str(ex)) msg = "ERROR: Unable to create SE source interface \n" raise CrabException(msg)
def run(self): # get updated status from server try: stat = StatusServer(self.cfg_params) warning_msg = stat.resynchClientSide() if warning_msg is not None: common.logger.info(warning_msg) except: pass # check whether the action is allowable self.check() # notify the server to clean the task csCommunicator = ServerCommunicator(self.server_name, self.server_port, self.cfg_params) taskuuid = str(common._db.queryTask('name')) try: csCommunicator.cleanTask(taskuuid) except Exception, e: msg = "Client Server comunication failed about cleanJobs: task \n" + taskuuid msg += "Only local working directory will be removed." common.logger.debug( msg) pass
def main(stdscr): parser = argparse.ArgumentParser() parser.add_argument("-p", "--preload", help="preload with a saved file") parser.add_argument("-b", "--boot", help="Skip the boot screen", action="store_true") group = parser.add_mutually_exclusive_group() group.add_argument("-s", "--server", help="run this instance as a status server") group.add_argument("-a", "--status_address", help="address of the status server") parser.add_argument("-n", "--name", help="give this team a name", default="Test") args = parser.parse_args() curses.start_color() curses.init_pair(1, curses.COLOR_BLACK, curses.COLOR_CYAN) curses.init_pair(2, curses.COLOR_BLACK, curses.COLOR_RED) curses.init_pair(3, curses.COLOR_BLACK, curses.COLOR_GREEN) curses.init_pair(4, curses.COLOR_BLACK, curses.COLOR_BLUE) curses.init_pair(5, curses.COLOR_BLACK, curses.COLOR_YELLOW) curses.init_pair(6, curses.COLOR_WHITE, curses.COLOR_BLACK) curses.init_pair(7, curses.COLOR_BLACK, curses.COLOR_MAGENTA) curses.init_pair(8, curses.COLOR_BLACK, curses.COLOR_WHITE) BLACK_ON_CYAN = curses.color_pair(1) BLACK_ON_RED = curses.color_pair(2) BLACK_ON_GREEN = curses.color_pair(3) BLACK_ON_BLUE = curses.color_pair(4) BLACK_ON_YELLOW = curses.color_pair(5) WHITE_ON_BLACK = curses.color_pair(6) BLACK_ON_MAGENTA = curses.color_pair(7) BLACK_ON_WHITE = curses.color_pair(8) class WinDim: def __init__(self, y0, x0, y1, x1): self.x0 = x0 self.y0 = y0 self.x1 = x1 self.y1 = y1 self.cols = x1-x0 self.lines = y1-y0 BOAT_WIN = WinDim(2, 1, int(curses.LINES - 5 ), int(curses.COLS * 0.8)) MENU_WIN = WinDim(BOAT_WIN.y1+1, 1, curses.LINES - 1, curses.COLS - 1) MORSE_WIN = WinDim(2, BOAT_WIN.x1+1, BOAT_WIN.y1, curses.COLS - 1) if not args.boot: stdscr.addstr(int(curses.LINES /2 ), int(curses.COLS / 2)-11, "IN MEMORY OF DAN BLACK") stdscr.refresh() sleep(3) stdscr.erase() if args.name: stdscr.addstr(0, 0, "Enigma: Save our Convoy | Team: " + args.name) else: stdscr.addstr(0, 0, "Enigma: Save our Convoy") boatwin = curses.newwin(BOAT_WIN.lines, BOAT_WIN.cols, BOAT_WIN.y0, BOAT_WIN.x0) rectangle(stdscr, BOAT_WIN.y0-1, BOAT_WIN.x0-1, BOAT_WIN.y1, BOAT_WIN.x1) morsewin = curses.newwin(MORSE_WIN.lines, MORSE_WIN.cols, MORSE_WIN.y0, MORSE_WIN.x0) rectangle(stdscr, MORSE_WIN.y0-1, MORSE_WIN.x0-1, MORSE_WIN.y1, MORSE_WIN.x1) menuwin = curses.newwin(MENU_WIN.lines, MENU_WIN.cols, MENU_WIN.y0, MENU_WIN.x0) stdscr.refresh() stdscr.nodelay(True) curses.cbreak() boatwin.refresh() if args.preload and args.server is None: grid = Grid(boatwin, convoi_color = BLACK_ON_GREEN, mine_color = BLACK_ON_YELLOW, uboat_color = BLACK_ON_RED, uboat_danger_color=BLACK_ON_MAGENTA, path_color = BLACK_ON_CYAN, default_color = WHITE_ON_BLACK, preload_file=args.preload, status_addr=args.status_address, name=args.name ) else: grid = Grid(boatwin, convoi_color=BLACK_ON_GREEN, mine_color=BLACK_ON_YELLOW, uboat_color=BLACK_ON_RED, uboat_danger_color=BLACK_ON_MAGENTA, path_color=BLACK_ON_CYAN, default_color=WHITE_ON_BLACK, status_addr=args.status_address, name=args.name ) if not args.server: morse = MorseController(morsewin, receive_array_color=[BLACK_ON_GREEN, BLACK_ON_RED], receive_array_strings=["RECEIVE", "SEND"], status_color=BLACK_ON_YELLOW, speed_color=BLACK_ON_BLUE, volume_color=BLACK_ON_BLUE, backround=WHITE_ON_BLACK) status_server = None else: morse = None status_server = StatusServer(args.server, grid, morsewin, solution=args.preload, highlght_color=BLACK_ON_GREEN, default_color=WHITE_ON_BLACK) menu = Menu(menuwin, morse=morse, grid=grid, status_server=status_server, stdscr=stdscr, name=args.name) menu.run()
def initializeActions_(self, opts): """ For each user action instantiate a corresponding object and put it in the action dictionary. """ for opt in opts.keys(): val = opts[opt] if ( opt == '-create' ): if self.flag_continue: msg = 'Cannot create an existing project. \n' raise CrabException(msg) if val and val != 'all': msg = 'Per default, CRAB will create all jobs as specified in the crab.cfg file, not the command line!' common.logger.info(msg) msg = 'Submission will still take into account the number of jobs specified on the command line!\n' common.logger.info(msg) ncjobs = 'all' #wmbs if int(self.automation) == 1: msg = 'Your jobs will be created on the server. Jobs informations will be known later.' common.logger.info(msg) # wmbs from Requestor import Requestor # Instantiate Creator object self.creator = Requestor(self.job_type_name, self.cfg_params, ncjobs) self.actions[opt] = self.creator # Create and initialize JobList common.job_list = JobList(1,self.creator.jobType()) else: from Creator import Creator # Instantiate Creator object self.creator = Creator(self.job_type_name, self.cfg_params, ncjobs) self.actions[opt] = self.creator # create jobs in the DB common._db.createJobs_(self.creator.nJobsL()) # Create and initialize JobList common.job_list = JobList(common._db.nJobs(), self.creator.jobType()) ## jobs specs not needed if using WMBS self.creator.writeJobsSpecsToDB() taskinfo={} taskinfo['cfgName'] = common.work_space.jobDir()+"/"+self.creator.jobType().configFilename() taskinfo['dataset'] = self.cfg_params['CMSSW.datasetpath'] common.job_list.setScriptNames(self.job_type_name+'.sh') common.job_list.setCfgNames(self.creator.jobType().configFilename()) common._db.updateTask_(taskinfo) pass elif ( opt == '-submit' ): ## Dealt with val == int so that -submit N means submit N jobs and not job # N if (self.UseServer== 1): from SubmitterServer import SubmitterServer self.actions[opt] = SubmitterServer(self.cfg_params, self.parseRange_(val), val) else: from Submitter import Submitter # Instantiate Submitter object self.actions[opt] = Submitter(self.cfg_params, self.parseRange_(val), val) # Create and initialize JobList if len(common.job_list) == 0 : common.job_list = JobList(common._db.nJobs(), None) pass pass elif ( opt == '-list' ): ''' Print the relevant infos of a range-all jobs/task ''' jobs = self.parseRange_(val) common._db.dump(jobs) pass elif ( opt == '-printId' ): ''' Print the unique name of the task if crab is used as client Print the SID list of all the jobs ''' #jid=False #if val == 'full': jid=True jid=True common._db.queryID(self.UseServer,jid) elif ( opt == '-status' ): if (self.UseServer== 1): from StatusServer import StatusServer self.actions[opt] = StatusServer(self.cfg_params, val) else: from Status import Status self.actions[opt] = Status(self.cfg_params, val) elif ( opt == '-kill' ): if val: if val =='all': jobs = common._db.nJobs("list") else: jobs = self.parseRange_(val) pass else: raise CrabException("Warning: with '-kill' you _MUST_ specify a job range or 'all'") pass if (self.UseServer== 1): from KillerServer import KillerServer self.actions[opt] = KillerServer(self.cfg_params,jobs) else: from Killer import Killer self.actions[opt] = Killer(self.cfg_params,jobs) elif ( opt == '-stopWorkflow' ): if self.UseServer==1 and int(self.automation)==1: from WorkflowHandler import WorkflowHandler self.actions[opt] = WorkflowHandler(self.cfg_params) else: raise CrabException("ERROR: you can use this command only if you are running an automated workflow using CRABSERVER") pass elif ( opt == '-getoutput' or opt == '-get'): if val=='all' or val==None or val=='': jobs = 'all' else: jobs = self.parseRange_(val) if (self.UseServer== 1): from GetOutputServer import GetOutputServer self.actions[opt] = GetOutputServer(self.cfg_params,jobs) else: from GetOutput import GetOutput self.actions[opt] = GetOutput(self.cfg_params,jobs) elif ( opt == '-resubmit' ): if val: if val=='all': jobs = common._db.nJobs('list') if val=='bad': jobs = 'bad' else: jobs = self.parseRange_(val) if (self.UseServer== 1): from ResubmitterServer import ResubmitterServer self.actions[opt] = ResubmitterServer(self.cfg_params, jobs) else: from Resubmitter import Resubmitter self.actions[opt] = Resubmitter(self.cfg_params, jobs) else: common.logger.info("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'") common.logger.info("WARNING: _all_ job specified in the range will be resubmitted!!!") pass pass elif ( opt == '-forceResubmit' ): if val: if val=='all': jobs = common._db.nJobs('list') else: jobs = self.parseRange_(val) if (self.UseServer== 1): from ResubmitterServerForced import ResubmitterServerForced self.actions[opt] = ResubmitterServerForced(self.cfg_params, jobs) else: from ResubmitterForced import ResubmitterForced self.actions[opt] = ResubmitterForced(self.cfg_params, jobs) else: common.logger.info("Warning: with '-resubmit' you _MUST_ specify a job range or 'all'") common.logger.info("WARNING: _all_ job specified in the range will be resubmitted!!!") pass pass elif ( opt in ['-testJdl','-listMatch', '-match']): jobs = self.parseRange_(val) if len(jobs) != 0: # Instantiate Checker object from Checker import Checker self.actions[opt] = Checker(self.cfg_params, jobs) elif ( opt == '-postMortem' ): if val: jobs = self.parseRange_(val) if len( jobs ) > 1: raise CrabException("Only single job id allowed for %s command!" % opt ) else: raise CrabException("Warning: please specify a job id") pass if (self.UseServer== 1): from PostMortemServer import PostMortemServer self.actions[opt] = PostMortemServer(self.cfg_params,jobs) else: from PostMortem import PostMortem self.actions[opt] = PostMortem(self.cfg_params, jobs) elif ( opt == '-clean' ): if val != None: raise CrabException("No range allowed for '-clean'") if (self.UseServer== 1): from CleanerServer import CleanerServer self.actions[opt] = CleanerServer(self.cfg_params) else: from Cleaner import Cleaner self.actions[opt] = Cleaner(self.cfg_params) elif ( opt in ['-printJdl','-createJdl']): """ Materialize JDL """ ## Temporary: if opt == '-printJdl': common.logger.info("WARNING: -printJdl option is deprecated : please use -createJdl \n") if val =='all' or val == None or val == '': jobs = common._db.nJobs("list") else: jobs = self.parseRange_(val) pass from JdlWriter import JdlWriter self.actions[opt] = JdlWriter(self.cfg_params, jobs) elif ( opt == '-publish'): from Publisher import Publisher self.actions[opt] = Publisher(self.cfg_params) ### FEDE FOR PUBLICATION WITH NO INPUT FILES### elif ( opt == '-publishNoInp'): from Publisher import Publisher self.cfg_params['USER.no_inp'] = 1 self.actions[opt] = Publisher(self.cfg_params) elif ( opt == '-checkPublication' ): from InspectDBS import InspectDBS self.actions[opt] = InspectDBS(self.cfg_params) elif ( opt == '-copyData' ): if val =='all' or val == None or val == '': jobs = common._db.nJobs("list") else: jobs = self.parseRange_(val) if (self.UseServer== 1): from StatusServer import StatusServer status = StatusServer(self.cfg_params) else: # from Status import Status # status = Status(self.cfg_params) status=None from CopyData import CopyData self.actions[opt] = CopyData(self.cfg_params, jobs,status) elif ( opt == '-validateCfg' ): from ValidateCfg import ValidateCfg config= {'pset' : self.cfg_params.get('CMSSW.pset','None')} if val : config['pset']=val self.actions[opt] = ValidateCfg(config) elif ( opt == '-renewCredential' ): if (self.UseServer== 1): from CredentialRenew import CredentialRenew self.actions[opt] = CredentialRenew(self.cfg_params) else: msg = "The option [-renewProxy] can be used only with the server modality!" raise CrabException(msg) elif ( opt == '-report' ): if (self.UseServer== 1): from StatusServer import StatusServer StatusServer(self.cfg_params).query(display=False) else: # cause a core dump.... #from Status import Status #Status(self.cfg_params).query(display=False) pass from Reporter import Reporter self.actions[opt] = Reporter(self.cfg_params) elif ( opt == '-cleanCache' ): from CacheCleaner import CacheCleaner self.actions[opt] = CacheCleaner() elif ( opt == '-uploadLog' ): jobid = -1 jobs = self.parseRange_(val) if len( jobs ) > 1: common.logger.info("Only single job id allowed for %s command!" % opt ) elif len (jobs) == 1: jobid = jobs[0] from ReportUploader import ReportUploader self.actions[opt] = ReportUploader( self.cfg_params, jobid ) pass return