def stopWorkflow(self, urgency):
log.debug("VanillaCondorWorkflowMonitor:stopWorkflow")
transmit = events.EventTransmitter(self._eventBrokerHost, self._shutdownTopic)
root = PropertySet()
root.setInt("level",urgency)
root.setString("STATUS","shut things down")
event = events.CommandEvent(self.runid, self.originatorId, 0, root)
transmit.publishEvent(event)
def testOneVisit(self):
# Create a list of clipboards, stage lists, and queue lists for each slice
clipboards = [Clipboard() for i in xrange(self.universeSize)]
stageLists = [[] for i in xrange(self.universeSize)]
queueLists = []
for i in xrange(self.universeSize):
queueList = [Queue() for j in xrange(len(self.stages) + 1)]
queueList[0].addDataset(clipboards[i])
queueLists.append(queueList)
# Create and initialize stages for each slice
for stageClass, policy, i in izip(self.stages, self.policies,
xrange(len(self.stages))):
for stageList, queueList, rank in izip(stageLists, queueLists,
xrange(self.universeSize)):
stage = stageClass(i, policy)
stage.setRun(self.runId)
stage.setUniverseSize(self.universeSize)
stage.setRank(rank - 1)
stage.initialize(queueList[i + 1], queueList[i])
stageList.append(stage)
# Create the association pipeline trigger event
dateObs = DateTime.now().mjd(DateTime.TAI)
triggerAssociationEvent = PropertySet()
triggerAssociationEvent.setInt('visitId', self.visitId)
triggerAssociationEvent.setDouble('dateObs', dateObs)
triggerAssociationEvent.setString('filter', self.filter)
triggerAssociationEvent.setDouble('ra', self.ra)
triggerAssociationEvent.setDouble('decl', self.dec)
# Create the event triggering the match against moving object predictions
triggerMatchMopsPredsEvent = PropertySet()
triggerMatchMopsPredsEvent.setInt('visitId', self.visitId)
# Add the events to clipboard of each stage
for clip in clipboards:
clip.put('triggerAssociationEvent', triggerAssociationEvent)
clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent)
assert self.universeSize > 1
masterStageList = stageLists.pop(0)
# Run the pipeline (worker slices are run one after the other)
for masterStage, workerStages in izip(masterStageList,
izip(*stageLists)):
masterStage.preprocess()
map(lambda x: x.process(), workerStages)
masterStage.postprocess()
# Close log to avoid bogus memory-leak reports
log.Log.closeDefaultLog()
def testOneVisit(self):
# Create a list of clipboards, stage lists, and queue lists for each slice
clipboards = [Clipboard() for i in xrange(self.universeSize)]
stageLists = [[] for i in xrange(self.universeSize)]
queueLists = []
for i in xrange(self.universeSize):
queueList = [Queue() for j in xrange(len(self.stages) + 1)]
queueList[0].addDataset(clipboards[i])
queueLists.append(queueList)
# Create and initialize stages for each slice
for stageClass, policy, i in izip(self.stages, self.policies, xrange(len(self.stages))):
for stageList, queueList, rank in izip(stageLists, queueLists, xrange(self.universeSize)):
stage = stageClass(i, policy)
stage.setRun(self.runId)
stage.setUniverseSize(self.universeSize)
stage.setRank(rank - 1)
stage.initialize(queueList[i+1], queueList[i])
stageList.append(stage)
# Create the association pipeline trigger event
dateObs = DateTime.now().mjd(DateTime.TAI)
triggerAssociationEvent = PropertySet()
triggerAssociationEvent.setInt('visitId', self.visitId)
triggerAssociationEvent.setDouble('dateObs', dateObs)
triggerAssociationEvent.setString('filter', self.filter)
triggerAssociationEvent.setDouble('ra', self.ra)
triggerAssociationEvent.setDouble('decl', self.dec)
# Create the event triggering the match against moving object predictions
triggerMatchMopsPredsEvent = PropertySet()
triggerMatchMopsPredsEvent.setInt('visitId', self.visitId)
# Add the events to clipboard of each stage
for clip in clipboards:
clip.put('triggerAssociationEvent', triggerAssociationEvent)
clip.put('triggerMatchMopsPredsEvent', triggerMatchMopsPredsEvent)
assert self.universeSize > 1
masterStageList = stageLists.pop(0)
# Run the pipeline (worker slices are run one after the other)
for masterStage, workerStages in izip(masterStageList, izip(*stageLists)):
masterStage.preprocess()
map(lambda x: x.process(), workerStages)
masterStage.postprocess()
# Close log to avoid bogus memory-leak reports
log.Log.closeDefaultLog()
def finalMessageReceived(self, propSet):
log = propSet.get("LOGGER")
if log != "orca.control":
return False
status = propSet.get("STATUS")
if status != "eol":
return False
eventSystem = events.EventSystem.getDefaultEventSystem()
id = eventSystem.createOriginatorId()
root = PropertySet()
root.setString("logger.status", "done")
root.setInt("logger.pid", os.getpid())
event = events.StatusEvent(self.runid, id, root)
self.transmitter.publishEvent(event)
return True
def setUp(self):
# Turn on tracing
log.Trace.setVerbosity('', 10)
log.ScreenLog.createDefaultLog(True, log.Log.INFO)
# Eventually, these should be read from a policy somewhere
self.dbServer = 'lsst10.ncsa.uiuc.edu'
self.dbPort = '3306'
self.dbType = 'mysql'
if not DbAuth.available(self.dbServer, self.dbPort):
self.fail("Cannot access database server %s:%s" %
(self.dbServer, self.dbPort))
# Construct test run database name
self.runId = DbAuth.username(self.dbServer, self.dbPort) +\
time.strftime("_test_ap_%y%m%d_%H%M%S", time.gmtime())
# Tweak these to run on different input data, or with a different number of slices
self.universeSize = 2
self.visitId = 708125
self.filter = 'u'
self.ra = 333.880166667
self.dec = -17.7374166667
self.dbUrlPrefix = ''.join(
[self.dbType, '://', self.dbServer, ':', self.dbPort, '/'])
self.dbUrl = self.dbUrlPrefix + self.runId
self.substitutions = {
'visitId': self.visitId,
'filter': self.filter,
'runId': self.runId
}
# Create a database specifically for the test (copy relevant
# tables from the test_ap database)
mysqlStatements = [
"""CREATE DATABASE %(runId)s""", """USE %(runId)s""",
"""CREATE TABLE VarObject LIKE test_ap.Object""",
"""CREATE TABLE NonVarObject LIKE test_ap.Object""",
"""CREATE TABLE DIASource LIKE test_ap.DIASource""",
"""CREATE TABLE prv_Filter LIKE test_ap.prv_Filter""",
"""INSERT INTO prv_Filter SELECT * FROM test_ap.prv_Filter""",
"""CREATE TABLE _tmp_v%(visitId)d_DIASource
LIKE test_ap._tmp_v%(visitId)d_DIASource""",
"""INSERT INTO _tmp_v%(visitId)d_DIASource
SELECT * FROM test_ap._tmp_v%(visitId)d_DIASource""",
"""CREATE TABLE _tmp_v%(visitId)d_Preds
LIKE test_ap._tmp_v%(visitId)d_Preds""",
"""INSERT INTO _tmp_v%(visitId)d_Preds
SELECT * FROM test_ap._tmp_v%(visitId)d_Preds""",
"""CREATE TABLE _tmpl_MatchPair LIKE test_ap._tmpl_MatchPair""",
"""CREATE TABLE _tmpl_IdPair LIKE test_ap._tmpl_IdPair""",
"""CREATE TABLE _tmpl_InMemoryObject LIKE test_ap._tmpl_InMemoryObject""",
"""CREATE TABLE _tmpl_InMemoryMatchPair LIKE test_ap._tmpl_InMemoryMatchPair""",
"""CREATE TABLE _tmpl_InMemoryId LIKE test_ap._tmpl_InMemoryId""",
"""CREATE TABLE _ap_DIASourceToObjectMatches LIKE test_ap._ap_DIASourceToObjectMatches""",
"""CREATE TABLE _ap_PredToDIASourceMatches LIKE test_ap._ap_PredToDIASourceMatches""",
"""CREATE TABLE _ap_DIASourceToNewObject LIKE test_ap._ap_DIASourceToNewObject""",
"""CREATE TABLE _mops_Prediction LIKE test_ap._mops_Prediction"""
]
db = DbStorage()
db.setPersistLocation(LogicalLocation(self.dbUrlPrefix + 'test_ap'))
try:
for stmt in mysqlStatements:
db.executeSql(stmt % self.substitutions)
# Specify list of stages ...
self.stages = [
ap.LoadStage, InputStage, ap.MatchDiaSourcesStage, OutputStage,
InputStage, ap.MatchMopsPredsStage, OutputStage, ap.StoreStage
]
# and read in stage policy for each stage
policyDir = os.path.join(os.environ['AP_DIR'], 'pipeline',
'examples', 'policy')
self.policies = [
Policy(os.path.join(policyDir, 'LoadStage.paf')),
Policy(os.path.join(policyDir,
'MatchDiaSourcesStageInput.paf')), None,
Policy(
os.path.join(policyDir, 'MatchDiaSourcesStageOutput.paf')),
Policy(os.path.join(policyDir,
'MatchMopsPredsStageInput.paf')), None,
Policy(os.path.join(policyDir,
'MatchMopsPredsStageOutput.paf')),
Policy(os.path.join(policyDir, 'StoreStage.paf'))
]
# construct PropertySet for string interpolation
psSubs = PropertySet()
psSubs.setInt('visitId', self.visitId)
psSubs.setString('runId', self.runId)
psSubs.setString('filter', self.filter)
psSubs.setString('work', '.')
psSubs.setString('input', '/tmp')
psSubs.setString('output', '/tmp')
psSubs.setString('update', '/tmp')
psSubs.setString('dbUrl', self.dbUrl)
LogicalLocation.setLocationMap(psSubs)
except:
# cleanup database in case of error
db.executeSql("DROP DATABASE %(runId)s" % self.substitutions)
raise
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the LSST License Statement and
# the GNU General Public License along with this program. If not,
# see <http://www.lsstcorp.org/LegalNotices/>.
#
import lsst.ctrl.events as events
from lsst.daf.base import PropertySet
import sys
# usage: python bin/workerdone.py brokerHost runid pipelineName
if __name__ == "__main__":
topic = "orca.monitor"
host = sys.argv[1]
runid = sys.argv[2]
pipelineName = sys.argv[3]
trans = events.EventTransmitter(host, topic)
eventSystem = events.EventSystem.getDefaultEventSystem()
root = PropertySet()
root.setString("pipeline", pipelineName)
id = eventSystem.createOriginatorId()
event = events.StatusEvent(runid, id, root)
trans.publishEvent(event)
def setUp(self):
# Turn on tracing
log.Trace.setVerbosity('', 10)
log.ScreenLog.createDefaultLog(True, log.Log.INFO)
# Eventually, these should be read from a policy somewhere
self.dbServer = 'lsst10.ncsa.uiuc.edu'
self.dbPort = '3306'
self.dbType = 'mysql'
if not DbAuth.available(self.dbServer, self.dbPort):
self.fail("Cannot access database server %s:%s" % (self.dbServer, self.dbPort))
# Construct test run database name
self.runId = DbAuth.username(self.dbServer, self.dbPort) +\
time.strftime("_test_ap_%y%m%d_%H%M%S", time.gmtime())
# Tweak these to run on different input data, or with a different number of slices
self.universeSize = 2
self.visitId = 708125
self.filter = 'u'
self.ra = 333.880166667
self.dec = -17.7374166667
self.dbUrlPrefix = ''.join([self.dbType, '://', self.dbServer, ':', self.dbPort, '/'])
self.dbUrl = self.dbUrlPrefix + self.runId
self.substitutions = { 'visitId': self.visitId,
'filter': self.filter,
'runId': self.runId }
# Create a database specifically for the test (copy relevant
# tables from the test_ap database)
mysqlStatements = [
"""CREATE DATABASE %(runId)s""",
"""USE %(runId)s""",
"""CREATE TABLE VarObject LIKE test_ap.Object""",
"""CREATE TABLE NonVarObject LIKE test_ap.Object""",
"""CREATE TABLE DIASource LIKE test_ap.DIASource""",
"""CREATE TABLE prv_Filter LIKE test_ap.prv_Filter""",
"""INSERT INTO prv_Filter SELECT * FROM test_ap.prv_Filter""",
"""CREATE TABLE _tmp_v%(visitId)d_DIASource
LIKE test_ap._tmp_v%(visitId)d_DIASource""",
"""INSERT INTO _tmp_v%(visitId)d_DIASource
SELECT * FROM test_ap._tmp_v%(visitId)d_DIASource""",
"""CREATE TABLE _tmp_v%(visitId)d_Preds
LIKE test_ap._tmp_v%(visitId)d_Preds""",
"""INSERT INTO _tmp_v%(visitId)d_Preds
SELECT * FROM test_ap._tmp_v%(visitId)d_Preds""",
"""CREATE TABLE _tmpl_MatchPair LIKE test_ap._tmpl_MatchPair""",
"""CREATE TABLE _tmpl_IdPair LIKE test_ap._tmpl_IdPair""",
"""CREATE TABLE _tmpl_InMemoryObject LIKE test_ap._tmpl_InMemoryObject""",
"""CREATE TABLE _tmpl_InMemoryMatchPair LIKE test_ap._tmpl_InMemoryMatchPair""",
"""CREATE TABLE _tmpl_InMemoryId LIKE test_ap._tmpl_InMemoryId""",
"""CREATE TABLE _ap_DIASourceToObjectMatches LIKE test_ap._ap_DIASourceToObjectMatches""",
"""CREATE TABLE _ap_PredToDIASourceMatches LIKE test_ap._ap_PredToDIASourceMatches""",
"""CREATE TABLE _ap_DIASourceToNewObject LIKE test_ap._ap_DIASourceToNewObject""",
"""CREATE TABLE _mops_Prediction LIKE test_ap._mops_Prediction"""
]
db = DbStorage()
db.setPersistLocation(LogicalLocation(self.dbUrlPrefix + 'test_ap'))
try:
for stmt in mysqlStatements:
db.executeSql(stmt % self.substitutions)
# Specify list of stages ...
self.stages = [ ap.LoadStage,
InputStage,
ap.MatchDiaSourcesStage,
OutputStage,
InputStage,
ap.MatchMopsPredsStage,
OutputStage,
ap.StoreStage ]
# and read in stage policy for each stage
policyDir = os.path.join(os.environ['AP_DIR'], 'pipeline', 'examples', 'policy')
self.policies = [ Policy(os.path.join(policyDir,'LoadStage.paf')),
Policy(os.path.join(policyDir,'MatchDiaSourcesStageInput.paf')),
None,
Policy(os.path.join(policyDir,'MatchDiaSourcesStageOutput.paf')),
Policy(os.path.join(policyDir,'MatchMopsPredsStageInput.paf')),
None,
Policy(os.path.join(policyDir,'MatchMopsPredsStageOutput.paf')),
Policy(os.path.join(policyDir,'StoreStage.paf')) ]
# construct PropertySet for string interpolation
psSubs = PropertySet()
psSubs.setInt('visitId', self.visitId)
psSubs.setString('runId', self.runId)
psSubs.setString('filter', self.filter)
psSubs.setString('work', '.')
psSubs.setString('input', '/tmp')
psSubs.setString('output', '/tmp')
psSubs.setString('update', '/tmp')
psSubs.setString('dbUrl', self.dbUrl)
LogicalLocation.setLocationMap(psSubs)
except:
# cleanup database in case of error
db.executeSql("DROP DATABASE %(runId)s" % self.substitutions)
raise
