def stopWorkflow(self, urgency): log.debug("VanillaCondorWorkflowMonitor:stopWorkflow") transmit = events.EventTransmitter(self._eventBrokerHost, self._shutdownTopic) root = PropertySet() root.setInt("level",urgency) root.setString("STATUS","shut things down") event = events.CommandEvent(self.runid, self.originatorId, 0, root) transmit.publishEvent(event)
def testOneVisit(self): # Create a list of clipboards, stage lists, and queue lists for each slice clipboards = [Clipboard() for i in xrange(self.universeSize)] stageLists = [[] for i in xrange(self.universeSize)] queueLists = [] for i in xrange(self.universeSize): queueList = [Queue() for j in xrange(len(self.stages) + 1)] queueList[0].addDataset(clipboards[i]) queueLists.append(queueList) # Create and initialize stages for each slice for stageClass, policy, i in izip(self.stages, self.policies, xrange(len(self.stages))): for stageList, queueList, rank in izip(stageLists, queueLists, xrange(self.universeSize)): stage = stageClass(i, policy) stage.setRun(self.runId) stage.setUniverseSize(self.universeSize) stage.setRank(rank - 1) stage.initialize(queueList[i + 1], queueList[i]) stageList.append(stage) # Create the association pipeline trigger event dateObs = DateTime.now().mjd(DateTime.TAI) triggerAssociationEvent = PropertySet() triggerAssociationEvent.setInt('visitId', self.visitId) triggerAssociationEvent.setDouble('dateObs', dateObs) triggerAssociationEvent.setString('filter', self.filter) triggerAssociationEvent.setDouble('ra', self.ra) triggerAssociationEvent.setDouble('decl', self.dec) # Create the event triggering the match against moving object predictions triggerMatchMopsPredsEvent = PropertySet() triggerMatchMopsPredsEvent.setInt('visitId', self.visitId) # Add the events to clipboard of each stage for clip in clipboards: clip.put('triggerAssociationEvent', triggerAssociationEvent) clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent) assert self.universeSize > 1 masterStageList = stageLists.pop(0) # Run the pipeline (worker slices are run one after the other) for masterStage, workerStages in izip(masterStageList, izip(*stageLists)): masterStage.preprocess() map(lambda x: x.process(), workerStages) masterStage.postprocess() # Close log to avoid bogus memory-leak reports log.Log.closeDefaultLog()
def testOneVisit(self): # Create a list of clipboards, stage lists, and queue lists for each slice clipboards = [Clipboard() for i in xrange(self.universeSize)] stageLists = [[] for i in xrange(self.universeSize)] queueLists = [] for i in xrange(self.universeSize): queueList = [Queue() for j in xrange(len(self.stages) + 1)] queueList[0].addDataset(clipboards[i]) queueLists.append(queueList) # Create and initialize stages for each slice for stageClass, policy, i in izip(self.stages, self.policies, xrange(len(self.stages))): for stageList, queueList, rank in izip(stageLists, queueLists, xrange(self.universeSize)): stage = stageClass(i, policy) stage.setRun(self.runId) stage.setUniverseSize(self.universeSize) stage.setRank(rank - 1) stage.initialize(queueList[i+1], queueList[i]) stageList.append(stage) # Create the association pipeline trigger event dateObs = DateTime.now().mjd(DateTime.TAI) triggerAssociationEvent = PropertySet() triggerAssociationEvent.setInt('visitId', self.visitId) triggerAssociationEvent.setDouble('dateObs', dateObs) triggerAssociationEvent.setString('filter', self.filter) triggerAssociationEvent.setDouble('ra', self.ra) triggerAssociationEvent.setDouble('decl', self.dec) # Create the event triggering the match against moving object predictions triggerMatchMopsPredsEvent = PropertySet() triggerMatchMopsPredsEvent.setInt('visitId', self.visitId) # Add the events to clipboard of each stage for clip in clipboards: clip.put('triggerAssociationEvent', triggerAssociationEvent) clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent) assert self.universeSize > 1 masterStageList = stageLists.pop(0) # Run the pipeline (worker slices are run one after the other) for masterStage, workerStages in izip(masterStageList, izip(*stageLists)): masterStage.preprocess() map(lambda x: x.process(), workerStages) masterStage.postprocess() # Close log to avoid bogus memory-leak reports log.Log.closeDefaultLog()
def finalMessageReceived(self, propSet): log = propSet.get("LOGGER") if log != "orca.control": return False status = propSet.get("STATUS") if status != "eol": return False eventSystem = events.EventSystem.getDefaultEventSystem() id = eventSystem.createOriginatorId() root = PropertySet() root.setString("logger.status", "done") root.setInt("logger.pid", os.getpid()) event = events.StatusEvent(self.runid, id, root) self.transmitter.publishEvent(event) return True
def setUp(self): # Turn on tracing log.Trace.setVerbosity('', 10) log.ScreenLog.createDefaultLog(True, log.Log.INFO) # Eventually, these should be read from a policy somewhere self.dbServer = 'lsst10.ncsa.uiuc.edu' self.dbPort = '3306' self.dbType = 'mysql' if not DbAuth.available(self.dbServer, self.dbPort): self.fail("Cannot access database server %s:%s" % (self.dbServer, self.dbPort)) # Construct test run database name self.runId = DbAuth.username(self.dbServer, self.dbPort) +\ time.strftime("_test_ap_%y%m%d_%H%M%S", time.gmtime()) # Tweak these to run on different input data, or with a different number of slices self.universeSize = 2 self.visitId = 708125 self.filter = 'u' self.ra = 333.880166667 self.dec = -17.7374166667 self.dbUrlPrefix = ''.join( [self.dbType, '://', self.dbServer, ':', self.dbPort, '/']) self.dbUrl = self.dbUrlPrefix + self.runId self.substitutions = { 'visitId': self.visitId, 'filter': self.filter, 'runId': self.runId } # Create a database specifically for the test (copy relevant # tables from the test_ap database) mysqlStatements = [ """CREATE DATABASE %(runId)s""", """USE %(runId)s""", """CREATE TABLE VarObject LIKE test_ap.Object""", """CREATE TABLE NonVarObject LIKE test_ap.Object""", """CREATE TABLE DIASource LIKE test_ap.DIASource""", """CREATE TABLE prv_Filter LIKE test_ap.prv_Filter""", """INSERT INTO prv_Filter SELECT * FROM test_ap.prv_Filter""", """CREATE TABLE _tmp_v%(visitId)d_DIASource LIKE test_ap._tmp_v%(visitId)d_DIASource""", """INSERT INTO _tmp_v%(visitId)d_DIASource SELECT * FROM test_ap._tmp_v%(visitId)d_DIASource""", """CREATE TABLE _tmp_v%(visitId)d_Preds LIKE test_ap._tmp_v%(visitId)d_Preds""", """INSERT INTO _tmp_v%(visitId)d_Preds SELECT * FROM test_ap._tmp_v%(visitId)d_Preds""", """CREATE TABLE _tmpl_MatchPair LIKE test_ap._tmpl_MatchPair""", """CREATE TABLE _tmpl_IdPair LIKE test_ap._tmpl_IdPair""", """CREATE TABLE _tmpl_InMemoryObject LIKE test_ap._tmpl_InMemoryObject""", """CREATE TABLE _tmpl_InMemoryMatchPair LIKE test_ap._tmpl_InMemoryMatchPair""", """CREATE TABLE _tmpl_InMemoryId LIKE test_ap._tmpl_InMemoryId""", """CREATE TABLE _ap_DIASourceToObjectMatches LIKE test_ap._ap_DIASourceToObjectMatches""", """CREATE TABLE _ap_PredToDIASourceMatches LIKE test_ap._ap_PredToDIASourceMatches""", """CREATE TABLE _ap_DIASourceToNewObject LIKE test_ap._ap_DIASourceToNewObject""", """CREATE TABLE _mops_Prediction LIKE test_ap._mops_Prediction""" ] db = DbStorage() db.setPersistLocation(LogicalLocation(self.dbUrlPrefix + 'test_ap')) try: for stmt in mysqlStatements: db.executeSql(stmt % self.substitutions) # Specify list of stages ... self.stages = [ ap.LoadStage, InputStage, ap.MatchDiaSourcesStage, OutputStage, InputStage, ap.MatchMopsPredsStage, OutputStage, ap.StoreStage ] # and read in stage policy for each stage policyDir = os.path.join(os.environ['AP_DIR'], 'pipeline', 'examples', 'policy') self.policies = [ Policy(os.path.join(policyDir, 'LoadStage.paf')), Policy(os.path.join(policyDir, 'MatchDiaSourcesStageInput.paf')), None, Policy( os.path.join(policyDir, 'MatchDiaSourcesStageOutput.paf')), Policy(os.path.join(policyDir, 'MatchMopsPredsStageInput.paf')), None, Policy(os.path.join(policyDir, 'MatchMopsPredsStageOutput.paf')), Policy(os.path.join(policyDir, 'StoreStage.paf')) ] # construct PropertySet for string interpolation psSubs = PropertySet() psSubs.setInt('visitId', self.visitId) psSubs.setString('runId', self.runId) psSubs.setString('filter', self.filter) psSubs.setString('work', '.') psSubs.setString('input', '/tmp') psSubs.setString('output', '/tmp') psSubs.setString('update', '/tmp') psSubs.setString('dbUrl', self.dbUrl) LogicalLocation.setLocationMap(psSubs) except: # cleanup database in case of error db.executeSql("DROP DATABASE %(runId)s" % self.substitutions) raise
# but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the LSST License Statement and # the GNU General Public License along with this program. If not, # see <http://www.lsstcorp.org/LegalNotices/>. # import lsst.ctrl.events as events from lsst.daf.base import PropertySet import sys # usage: python bin/workerdone.py brokerHost runid pipelineName if __name__ == "__main__": topic = "orca.monitor" host = sys.argv[1] runid = sys.argv[2] pipelineName = sys.argv[3] trans = events.EventTransmitter(host, topic) eventSystem = events.EventSystem.getDefaultEventSystem() root = PropertySet() root.setString("pipeline", pipelineName) id = eventSystem.createOriginatorId() event = events.StatusEvent(runid, id, root) trans.publishEvent(event)
def setUp(self): # Turn on tracing log.Trace.setVerbosity('', 10) log.ScreenLog.createDefaultLog(True, log.Log.INFO) # Eventually, these should be read from a policy somewhere self.dbServer = 'lsst10.ncsa.uiuc.edu' self.dbPort = '3306' self.dbType = 'mysql' if not DbAuth.available(self.dbServer, self.dbPort): self.fail("Cannot access database server %s:%s" % (self.dbServer, self.dbPort)) # Construct test run database name self.runId = DbAuth.username(self.dbServer, self.dbPort) +\ time.strftime("_test_ap_%y%m%d_%H%M%S", time.gmtime()) # Tweak these to run on different input data, or with a different number of slices self.universeSize = 2 self.visitId = 708125 self.filter = 'u' self.ra = 333.880166667 self.dec = -17.7374166667 self.dbUrlPrefix = ''.join([self.dbType, '://', self.dbServer, ':', self.dbPort, '/']) self.dbUrl = self.dbUrlPrefix + self.runId self.substitutions = { 'visitId': self.visitId, 'filter': self.filter, 'runId': self.runId } # Create a database specifically for the test (copy relevant # tables from the test_ap database) mysqlStatements = [ """CREATE DATABASE %(runId)s""", """USE %(runId)s""", """CREATE TABLE VarObject LIKE test_ap.Object""", """CREATE TABLE NonVarObject LIKE test_ap.Object""", """CREATE TABLE DIASource LIKE test_ap.DIASource""", """CREATE TABLE prv_Filter LIKE test_ap.prv_Filter""", """INSERT INTO prv_Filter SELECT * FROM test_ap.prv_Filter""", """CREATE TABLE _tmp_v%(visitId)d_DIASource LIKE test_ap._tmp_v%(visitId)d_DIASource""", """INSERT INTO _tmp_v%(visitId)d_DIASource SELECT * FROM test_ap._tmp_v%(visitId)d_DIASource""", """CREATE TABLE _tmp_v%(visitId)d_Preds LIKE test_ap._tmp_v%(visitId)d_Preds""", """INSERT INTO _tmp_v%(visitId)d_Preds SELECT * FROM test_ap._tmp_v%(visitId)d_Preds""", """CREATE TABLE _tmpl_MatchPair LIKE test_ap._tmpl_MatchPair""", """CREATE TABLE _tmpl_IdPair LIKE test_ap._tmpl_IdPair""", """CREATE TABLE _tmpl_InMemoryObject LIKE test_ap._tmpl_InMemoryObject""", """CREATE TABLE _tmpl_InMemoryMatchPair LIKE test_ap._tmpl_InMemoryMatchPair""", """CREATE TABLE _tmpl_InMemoryId LIKE test_ap._tmpl_InMemoryId""", """CREATE TABLE _ap_DIASourceToObjectMatches LIKE test_ap._ap_DIASourceToObjectMatches""", """CREATE TABLE _ap_PredToDIASourceMatches LIKE test_ap._ap_PredToDIASourceMatches""", """CREATE TABLE _ap_DIASourceToNewObject LIKE test_ap._ap_DIASourceToNewObject""", """CREATE TABLE _mops_Prediction LIKE test_ap._mops_Prediction""" ] db = DbStorage() db.setPersistLocation(LogicalLocation(self.dbUrlPrefix + 'test_ap')) try: for stmt in mysqlStatements: db.executeSql(stmt % self.substitutions) # Specify list of stages ... self.stages = [ ap.LoadStage, InputStage, ap.MatchDiaSourcesStage, OutputStage, InputStage, ap.MatchMopsPredsStage, OutputStage, ap.StoreStage ] # and read in stage policy for each stage policyDir = os.path.join(os.environ['AP_DIR'], 'pipeline', 'examples', 'policy') self.policies = [ Policy(os.path.join(policyDir,'LoadStage.paf')), Policy(os.path.join(policyDir,'MatchDiaSourcesStageInput.paf')), None, Policy(os.path.join(policyDir,'MatchDiaSourcesStageOutput.paf')), Policy(os.path.join(policyDir,'MatchMopsPredsStageInput.paf')), None, Policy(os.path.join(policyDir,'MatchMopsPredsStageOutput.paf')), Policy(os.path.join(policyDir,'StoreStage.paf')) ] # construct PropertySet for string interpolation psSubs = PropertySet() psSubs.setInt('visitId', self.visitId) psSubs.setString('runId', self.runId) psSubs.setString('filter', self.filter) psSubs.setString('work', '.') psSubs.setString('input', '/tmp') psSubs.setString('output', '/tmp') psSubs.setString('update', '/tmp') psSubs.setString('dbUrl', self.dbUrl) LogicalLocation.setLocationMap(psSubs) except: # cleanup database in case of error db.executeSql("DROP DATABASE %(runId)s" % self.substitutions) raise