def setUp(self):
# Create a random bus
self.busname = "%s-%s" % (sys.argv[0], str(uuid.uuid4())[:8])
self.bus = ToBus(self.busname, {
"create": "always",
"delete": "always",
"node": {
"type": "topic"
}
})
self.bus.open()
self.addCleanup(self.bus.close)
# Define the services we use
self.status_service = "%s/%s.TaskGetSpecification" % (
self.busname, DEFAULT_OTDB_SERVICENAME)
# ================================
# Setup mock parset service
# ================================
# Nr of parsets requested, to detect multiple requests for the same parset, or of superfluous parsets
self.requested_parsets = 0
def TaskSpecificationService(OtdbID):
if OtdbID == 1:
predecessors = "[2,3]"
elif OtdbID == 2:
predecessors = "[3]"
elif OtdbID == 3:
predecessors = "[]"
else:
raise Exception("Invalid OtdbID: %s" % OtdbID)
self.requested_parsets += 1
return {
"Version.number": "1",
PARSET_PREFIX + "Observation.ObsID": str(OtdbID),
PARSET_PREFIX + "Observation.Scheduler.predecessors":
predecessors,
}
parset_service = Service(DEFAULT_OTDB_SERVICENAME +
".TaskSpecification",
TaskSpecificationService,
busname=self.busname)
parset_service.start_listening()
self.addCleanup(parset_service.stop_listening)
# ================================
# Setup listener to catch result
# of our service
# ================================
listener = RATaskSpecifiedBusListener(busname=self.busname)
listener.start_listening()
self.addCleanup(listener.stop_listening)
self.trigger = MethodTrigger(listener, "onTaskSpecified")
def setUp(self):
# Create a random bus
self.busname = "%s-%s" % (sys.argv[0], str(uuid.uuid4())[:8])
self.bus = ToBus(self.busname, { "create": "always", "delete": "always", "node": { "type": "topic" } })
self.bus.open()
self.addCleanup(self.bus.close)
# Define the services we use
self.status_service = "%s/%s.TaskGetSpecification" % (self.busname,DEFAULT_OTDB_SERVICENAME)
# ================================
# Setup mock parset service
# ================================
# Nr of parsets requested, to detect multiple requests for the same parset, or of superfluous parsets
self.requested_parsets = 0
def TaskSpecificationService( OtdbID ):
if OtdbID == 1:
predecessors = "[2,3]"
elif OtdbID == 2:
predecessors = "[3]"
elif OtdbID == 3:
predecessors = "[]"
else:
raise Exception("Invalid OtdbID: %s" % OtdbID)
self.requested_parsets += 1
return {
"Version.number": "1",
PARSET_PREFIX + "Observation.ObsID": str(OtdbID),
PARSET_PREFIX + "Observation.Scheduler.predecessors": predecessors,
}
parset_service = Service(DEFAULT_OTDB_SERVICENAME+".TaskSpecification", TaskSpecificationService, busname=self.busname)
parset_service.start_listening()
self.addCleanup(parset_service.stop_listening)
# ================================
# Setup listener to catch result
# of our service
# ================================
listener = RATaskSpecifiedBusListener(busname=self.busname)
listener.start_listening()
self.addCleanup(listener.stop_listening)
self.trigger = MethodTrigger(listener, "onTaskSpecified")
def createService(busname=DEFAULT_SSDB_BUSNAME,servicename=DEFAULT_SSDB_SERVICENAME, dbcreds=None, broker=None):
return Service(servicename,
DataMonitorQueryService,
busname=busname,
numthreads=1,
broker=broker,
handler_args={'dbcreds': dbcreds},
use_service_methods=True)
def createService(busname=DEFAULT_BUSNAME,
servicename=DEFAULT_SERVICENAME,
broker=None):
return Service(servicename=servicename,
servicehandler=ResourceEstimatorHandler,
busname=busname,
broker=broker,
numthreads=1,
verbose=True)
def createService(busname=DEFAULT_BUSNAME,
servicename=DEFAULT_SERVICENAME,
broker=None,
dbcreds=None,
log_queries=False,
verbose=False):
return Service(servicename,
RADBHandler,
busname=busname,
broker=broker,
use_service_methods=True,
numthreads=4,
handler_args={
'dbcreds': dbcreds,
'log_queries': log_queries
},
verbose=verbose)
def createService(busname=DEFAULT_MOMQUERY_BUSNAME,
servicename=DEFAULT_MOMQUERY_SERVICENAME,
dbcreds=None,
handler=None,
broker=None):
'''create the GetProjectDetails on given busname
:param string busname: name of the bus on which this service listens
:param string servicename: name of the service
:param Credentials dbcreds: Credentials for the MoM database.
:param ProjectDetailsQueryHandler handler: ProjectDetailsQueryHandler class Type, or mock like type
:rtype: lofar.messaging.Service'''
if not handler:
handler = ProjectDetailsQueryHandler
return Service(servicename,
handler,
busname=busname,
numthreads=2,
use_service_methods=True,
verbose=False,
broker=broker,
handler_args={'dbcreds': dbcreds})
# DictFunc
try:
result = DictFunc(25)
except InvalidArgType as e:
pass
result = DictFunc({'mies': "meisje", "aap": 125, "noot": [2, 3]})
if result != {'mies': "MEISJE", "aap": 125, "noot": [2, 3]}:
raise Exception("Dict function failed:{}".format(result))
print "Functions tested outside RPC: All OK"
# Used settings
busname = sys.argv[1] if len(sys.argv) > 1 else "simpletest"
# Register functs as a service handler listening at busname and ServiceName
serv1 = Service("ErrorService", ErrorFunc, busname=busname, numthreads=1)
serv2 = Service("ExceptionService",
ExceptionFunc,
busname=busname,
numthreads=1)
serv3 = Service("StringService", StringFunc, busname=busname, numthreads=1)
serv4 = Service("ListService", ListFunc, busname=busname, numthreads=1)
serv5 = Service("DictService", DictFunc, busname=busname, numthreads=1)
# 'with' sets up the connection context and defines the scope of the service.
with nested(serv1, serv2, serv3, serv4, serv5):
# Start listening in the background. This will start as many threads as defined by the instance
serv1.start_listening()
serv2.start_listening()
serv3.start_listening()
serv4.start_listening()
def setUp(self):
# Create a random bus
self.busname = "%s-%s" % (sys.argv[0], str(uuid.uuid4())[:8])
self.bus = ToBus(self.busname, { "create": "always", "delete": "always", "node": { "type": "topic" } })
self.bus.open()
self.addCleanup(self.bus.close)
# Patch SLURM
class MockSlurm(object):
def __init__(self, *args, **kwargs):
self.scheduled_jobs = {}
def submit(self, jobName, *args, **kwargs):
print "Schedule SLURM job '%s': %s, %s" % (jobName, args, kwargs)
self.scheduled_jobs[jobName] = (args, kwargs)
# Return job ID
return "42"
def jobid(self, jobname):
if jobname in ["1", "2", "3"]:
return jobname
# "4" is an observation, so no SLURM job
return None
patcher = patch('lofar.mac.PipelineControl.Slurm')
patcher.start().side_effect = MockSlurm
self.addCleanup(patcher.stop)
# Catch functions to prevent running executables
patcher = patch('lofar.mac.PipelineControl.Parset.dockerImage')
patcher.start().return_value = "lofar-pipeline:trunk"
self.addCleanup(patcher.stop)
# ================================
# Setup mock otdb service
# ================================
class MockOTDBService(MessageHandlerInterface):
def __init__(self, notification_bus):
"""
notification_bus: bus to send state changes to
"""
super(MockOTDBService, self).__init__()
self.service2MethodMap = {
"TaskGetSpecification": self.TaskGetSpecification,
"TaskSetStatus": self.TaskSetStatus,
}
self.notification_bus = notification_bus
def TaskGetSpecification(self, OtdbID):
print "***** TaskGetSpecification(%s) *****" % (OtdbID,)
if OtdbID == 1:
predecessors = "[2,3,4]"
elif OtdbID == 2:
predecessors = "[3]"
elif OtdbID == 3:
predecessors = "[]"
elif OtdbID == 4:
return { "TaskSpecification": {
"Version.number": "1",
PARSET_PREFIX + "Observation.otdbID": str(OtdbID),
PARSET_PREFIX + "Observation.Scheduler.predecessors": "[]",
PARSET_PREFIX + "Observation.processType": "Observation",
PARSET_PREFIX + "Observation.Cluster.ProcessingCluster.clusterName": "CEP4",
PARSET_PREFIX + "Observation.stopTime": "2016-01-01 01:00:00",
} }
else:
raise Exception("Invalid OtdbID: %s" % OtdbID)
return { "TaskSpecification": {
"Version.number": "1",
PARSET_PREFIX + "Observation.otdbID": str(OtdbID),
PARSET_PREFIX + "Observation.Scheduler.predecessors": predecessors,
PARSET_PREFIX + "Observation.processType": "Pipeline",
PARSET_PREFIX + "Observation.Cluster.ProcessingCluster.clusterName": "CEP4",
} }
def TaskSetStatus(self, OtdbID, NewStatus, UpdateTimestamps):
print "***** TaskSetStatus(%s,%s) *****" % (OtdbID, NewStatus)
# Broadcast the state change
content = { "treeID" : OtdbID, "state" : NewStatus, "time_of_change" : datetime.datetime.utcnow() }
msg = EventMessage(context=DEFAULT_OTDB_NOTIFICATION_SUBJECT, content=content)
self.notification_bus.send(msg)
return {'OtdbID':OtdbID, 'MomID':None, 'Success':True}
service = Service(DEFAULT_OTDB_SERVICENAME,
MockOTDBService,
busname=self.busname,
use_service_methods=True,
handler_args={ "notification_bus": self.bus })
service.start_listening()
self.addCleanup(service.stop_listening)
# ================================
# Setup listener to catch result
# of our service
# ================================
listener = OTDBBusListener(busname=self.busname)
listener.start_listening()
self.addCleanup(listener.stop_listening)
self.trigger = MethodTrigger(listener, "onObservationQueued")
)
parser.add_option("-S",
"--servicename",
dest="servicename",
type="string",
default=DEFAULT_OTDB_SERVICENAME,
help="Name for this service. [default: %default")
parser.add_option('-V',
'--verbose',
dest='verbose',
action='store_true',
help='verbose logging')
# Add options of dbcredentials: --database, --host, ...
parser.add_option_group(dbcredentials.options_group(parser))
(options, args) = parser.parse_args()
setQpidLogLevel(logging.INFO)
dbcreds = dbcredentials.parse_options(options)
print "###dbcreds:", dbcreds
with Service(options.servicename,
PostgressMessageHandler,
busname=options.busname,
use_service_methods=True,
numthreads=1,
handler_args={"dbcreds": dbcreds},
verbose=True):
waitForInterrupt()
logger.info("Stopped the OTDB services")
def setUp(self):
# Create a random bus
self.busname = "%s-%s" % (sys.argv[0], str(uuid.uuid4())[:8])
self.bus = ToBus(self.busname, {
"create": "always",
"delete": "always",
"node": {
"type": "topic"
}
})
self.bus.open()
self.addCleanup(self.bus.close)
# Patch SLURM
class MockSlurm(object):
def __init__(self, *args, **kwargs):
self.scheduled_jobs = {}
def submit(self, jobName, *args, **kwargs):
print "Schedule SLURM job '%s': %s, %s" % (jobName, args,
kwargs)
self.scheduled_jobs[jobName] = (args, kwargs)
# Return job ID
return "42"
def jobid(self, jobname):
if jobname in ["1", "2", "3"]:
return jobname
# "4" is an observation, so no SLURM job
return None
patcher = patch('lofar.mac.PipelineControl.Slurm')
patcher.start().side_effect = MockSlurm
self.addCleanup(patcher.stop)
# Catch functions to prevent running executables
patcher = patch('lofar.mac.PipelineControl.Parset.dockerImage')
patcher.start().return_value = "lofar-pipeline:trunk"
self.addCleanup(patcher.stop)
# ================================
# Setup mock otdb service
# ================================
class MockOTDBService(MessageHandlerInterface):
def __init__(self, notification_bus):
"""
notification_bus: bus to send state changes to
"""
super(MockOTDBService, self).__init__()
self.service2MethodMap = {
"TaskGetSpecification": self.TaskGetSpecification,
"TaskSetStatus": self.TaskSetStatus,
}
self.notification_bus = notification_bus
def TaskGetSpecification(self, OtdbID):
print "***** TaskGetSpecification(%s) *****" % (OtdbID, )
if OtdbID == 1:
predecessors = "[2,3,4]"
elif OtdbID == 2:
predecessors = "[3]"
elif OtdbID == 3:
predecessors = "[]"
elif OtdbID == 4:
return {
"TaskSpecification": {
"Version.number":
"1",
PARSET_PREFIX + "Observation.otdbID":
str(OtdbID),
PARSET_PREFIX + "Observation.Scheduler.predecessors":
"[]",
PARSET_PREFIX + "Observation.processType":
"Observation",
PARSET_PREFIX + "Observation.Cluster.ProcessingCluster.clusterName":
"CEP4",
PARSET_PREFIX + "Observation.stopTime":
"2016-01-01 01:00:00",
}
}
else:
raise Exception("Invalid OtdbID: %s" % OtdbID)
return {
"TaskSpecification": {
"Version.number":
"1",
PARSET_PREFIX + "Observation.otdbID":
str(OtdbID),
PARSET_PREFIX + "Observation.Scheduler.predecessors":
predecessors,
PARSET_PREFIX + "Observation.processType":
"Pipeline",
PARSET_PREFIX + "Observation.Cluster.ProcessingCluster.clusterName":
"CEP4",
}
}
def TaskSetStatus(self, OtdbID, NewStatus, UpdateTimestamps):
print "***** TaskSetStatus(%s,%s) *****" % (OtdbID, NewStatus)
# Broadcast the state change
content = {
"treeID": OtdbID,
"state": NewStatus,
"time_of_change": datetime.datetime.utcnow()
}
msg = EventMessage(context=DEFAULT_OTDB_NOTIFICATION_SUBJECT,
content=content)
self.notification_bus.send(msg)
return {'OtdbID': OtdbID, 'MomID': None, 'Success': True}
service = Service(DEFAULT_OTDB_SERVICENAME,
MockOTDBService,
busname=self.busname,
use_service_methods=True,
handler_args={"notification_bus": self.bus})
service.start_listening()
self.addCleanup(service.stop_listening)
# ================================
# Setup listener to catch result
# of our service
# ================================
listener = OTDBBusListener(busname=self.busname)
listener.start_listening()
self.addCleanup(listener.stop_listening)
self.trigger = MethodTrigger(listener, "onObservationQueued")
# DictFunc
try:
result = DictFunc(25)
except InvalidArgType as e:
pass
result = DictFunc({'mies' : "meisje", "aap" : 125, "noot" : [2, 3]})
if result != {'mies' : "MEISJE", "aap" : 125, "noot" : [2, 3]}:
raise Exception("Dict function failed:{}".format(result))
print "Functions tested outside RPC: All OK"
# Used settings
busname = sys.argv[1] if len(sys.argv) > 1 else "simpletest"
# Register functs as a service handler listening at busname and ServiceName
serv1 = Service("ErrorService", ErrorFunc, busname=busname, numthreads=1)
serv2 = Service("ExceptionService", ExceptionFunc, busname=busname, numthreads=1)
serv3 = Service("StringService", StringFunc, busname=busname, numthreads=1)
serv4 = Service("ListService", ListFunc, busname=busname, numthreads=1)
serv5 = Service("DictService", DictFunc, busname=busname, numthreads=1)
# 'with' sets up the connection context and defines the scope of the service.
with nested(serv1, serv2, serv3, serv4, serv5):
# Start listening in the background. This will start as many threads as defined by the instance
serv1.start_listening()
serv2.start_listening()
serv3.start_listening()
serv4.start_listening()
serv5.start_listening()
# Redo all tests but via through RPC