def __init__(self,
busname=DEFAULT_NOTIFICATION_BUSNAME,
notification_prefix=DEFAULT_NOTIFICATION_PREFIX,
dbcreds=None,
broker=None):
super(RADBPGListener, self).__init__(dbcreds.host, dbcreds.database,
dbcreds.user, dbcreds.password)
self.notification_prefix = notification_prefix
self.event_bus = ToBus(busname, broker=broker)
self.subscribe('task_update_with_task_view', self.onTaskUpdated)
self.subscribe('task_insert_with_task_view', self.onTaskInserted)
self.subscribe('task_delete', self.onTaskDeleted)
# when the specification starttime and endtime are updated, then that effects the task as well
# so subscribe to specification_update, and use task_view as view_for_row
self.subscribe('specification_update_with_task_view',
self.onSpecificationUpdated)
self.subscribe('resource_claim_update_with_resource_claim_view',
self.onResourceClaimUpdated)
self.subscribe('resource_claim_insert_with_resource_claim_view',
self.onResourceClaimInserted)
self.subscribe('resource_claim_delete', self.onResourceClaimDeleted)
self.subscribe('resource_availability_update',
self.onResourceAvailabilityUpdated)
self.subscribe('resource_capacity_update',
self.onResourceCapacityUpdated)
def __init__(self, exchange=DEFAULT_BUSNAME, broker=DEFAULT_BROKER):
"""
:param exchange: name of the exchange to listen on.
:param broker: name of the broker to connect to.
"""
super().__init__()
self.otdbrpc = OTDBRPC.create(exchange=exchange, broker=broker)
self.radbrpc = RADBRPC.create(exchange=exchange, broker=broker)
self.momrpc = MoMQueryRPC.create(exchange=exchange, broker=broker)
self.send_bus = ToBus(exchange=exchange, broker=broker)
def __init__(self,
servicename,
otdb_busname=None,
my_busname=None,
**kwargs):
super(RATaskSpecified, self).__init__(busname=otdb_busname,
subject="TaskStatus",
**kwargs)
self.parset_rpc = RPC(service="TaskSpecification",
busname=otdb_busname)
self.send_bus = ToBus("%s/%s" % (my_busname, servicename))
def _send_notification(user, host, project, trigger_id, metadata):
try:
content = {
"user": user,
"host": host,
"project": project,
"trigger_id": trigger_id,
"metadata": metadata
}
msg = EventMessage(subject=DEFAULT_TRIGGER_NOTIFICATION_SUBJECT,
content=content)
with ToBus(exchange=DEFAULT_BUSNAME,
broker=DEFAULT_BROKER) as notification_bus:
notification_bus.send(msg)
except Exception as err:
logger.error("Could not send notification ->" + str(err))
def __init__(self, job, momClient, ltaClient,
exchange=DEFAULT_BUSNAME,
broker=DEFAULT_BROKER,
user=getpass.getuser(),
globus_timeout=GLOBUS_TIMEOUT,
minimal_SIP=False):
self.status = IngestPipeline.STATUS_INITIALIZING
self.hostname = socket.gethostname()
self.job = job
self.momClient = momClient
self.ltaClient = ltaClient
self.user = user
if not self.user:
self.user=getpass.getuser()
self.globus_timeout = globus_timeout
self.minimal_SIP = minimal_SIP
self.event_bus = ToBus(exchange, broker=broker, connection_log_level=logging.DEBUG)
self.Project = job['Project']
self.DataProduct = job['DataProduct']
self.FileName = job['FileName']
self.JobId = job['JobId']
self.ArchiveId = int(job['ArchiveId'])
self.ObsId = int(job['ObservationId'])
self.ExportID = job['ExportID']
self.Type = job["Type"]
self.HostLocation = job['Location'].partition(':')[0]
self.Location = job['Location'].partition(':')[2]
self.ticket = ''
self.FileSize = '0'
self.MD5Checksum = ''
self.Adler32Checksum = ''
self.ChecksumResult = False
self.SIP = ''
self.PrimaryUri = ''
self.SecondaryUri = ''
self.lta_site = ''
def __init__(
self,
otdb_notification_busname=DEFAULT_OTDB_NOTIFICATION_BUSNAME,
otdb_notification_subject=DEFAULT_OTDB_NOTIFICATION_SUBJECT,
otdb_service_busname=DEFAULT_OTDB_SERVICE_BUSNAME,
otdb_service_subject=DEFAULT_OTDB_SERVICENAME,
notification_busname=DEFAULT_RA_TASK_SPECIFIED_NOTIFICATION_BUSNAME,
notification_subject=DEFAULT_RA_TASK_SPECIFIED_NOTIFICATION_SUBJECT,
broker=None,
**kwargs):
super(RATaskSpecified,
self).__init__(busname=otdb_notification_busname,
subject=otdb_notification_subject,
**kwargs)
self.otdbrpc = OTDBRPC(
busname=otdb_service_busname,
servicename=otdb_service_subject,
broker=broker
) ## , ForwardExceptions=True hardcoded in RPCWrapper right now
self.send_bus = ToBus("%s/%s" %
(notification_busname, notification_subject))
def __init__(self,
cache_path='.du_cache.py',
mountpoint=CEP4_DATA_MOUNTPOINT,
exchange=DEFAULT_BUSNAME,
broker=DEFAULT_BROKER):
self._cache_path = cache_path
self.otdb_listener = OTDBBusListener(
_CacheManagerOTDBEventMessageHandler,
handler_kwargs={'cache_manager': self},
exchange=exchange,
broker=broker,
num_threads=1)
self.dm_listener = DataManagementBusListener(
_CacheManagerDataManagementEventMessageHandler,
handler_kwargs={'cache_manager': self},
exchange=exchange,
broker=broker,
num_threads=1)
self.event_bus = ToBus(exchange=exchange, broker=broker)
self._updateCacheThread = None
self._running = False
self._cacheLock = RLock()
self._cache = {'path_du_results': {}, 'otdb_id2path': {}}
self._last_cache_write_timestamp = datetime.datetime(1970, 1, 1)
self._readCacheFromDisk()
# dict to hold threading Events per path to prevent expensive multiple du calls for each path
self._du_threading_events = {}
self._du_threading_events_lock = RLock()
self.disk_usage = DiskUsage(mountpoint=mountpoint,
exchange=exchange,
broker=broker)
def __init__(self,
exchange=DEFAULT_BUSNAME,
mom_credentials=None,
lta_credentials=None,
user=None,
broker=None,
max_nr_of_parallel_jobs=MAX_NR_OF_JOBS):
self.user = user
if not self.user:
self.user = getpass.getuser()
self.mom_credentials = mom_credentials
self.lta_credentials = lta_credentials
self.event_bus = ToBus(exchange=exchange, broker=broker)
self.max_nr_of_parallel_jobs = max_nr_of_parallel_jobs
self.__running_jobs = {}
self.__lock = Lock()
self.__prev_bytes_sent = _getBytesSent()
self.__prev_bytes_sent_timestamp = datetime.utcnow()
self.__prev_used_bandwidth = 0.0
self.__running_jobs_log_timestamp = datetime.utcnow()
def __init__(self,
exchange=DEFAULT_BUSNAME,
dbcreds=None,
broker=DEFAULT_BROKER):
super(RADBPGListener, self).__init__(dbcreds=dbcreds)
self.event_bus = ToBus(exchange=exchange, broker=broker)
self.radb = RADatabase(dbcreds=dbcreds)
self.subscribe('task_update', self.onTaskUpdated)
self.subscribe('task_insert', self.onTaskInserted)
self.subscribe('task_delete', self.onTaskDeleted)
self.subscribe('task_predecessor_insert_column_task_id',
self.onTaskPredecessorChanged)
self.subscribe('task_predecessor_update_column_task_id',
self.onTaskPredecessorChanged)
self.subscribe('task_predecessor_delete_column_task_id',
self.onTaskPredecessorChanged)
self.subscribe('task_predecessor_insert_column_predecessor_id',
self.onTaskSuccessorChanged)
self.subscribe('task_predecessor_update_column_predecessor_id',
self.onTaskSuccessorChanged)
self.subscribe('task_predecessor_delete_column_predecessor_id',
self.onTaskSuccessorChanged)
# when the specification starttime and endtime are updated, then that effects the task as well
self.subscribe('specification_update', self.onSpecificationUpdated)
self.subscribe('resource_claim_update', self.onResourceClaimUpdated)
self.subscribe('resource_claim_insert', self.onResourceClaimInserted)
self.subscribe('resource_claim_delete', self.onResourceClaimDeleted)
self.subscribe('resource_availability_update',
self.onResourceAvailabilityUpdated)
self.subscribe('resource_capacity_update',
self.onResourceCapacityUpdated)
from lofar.triggerservices.trigger_service_rpc import TriggerRPC
from lofar.specificationservices.specification_service_rpc import SpecificationRPC
from lofar.mom.momqueryservice.momqueryrpc import MoMQueryRPC
from lofar.messaging import ToBus, EventMessage, RPCException, DEFAULT_BROKER, DEFAULT_BUSNAME
import logging
import traceback
logging.basicConfig(level=logging.DEBUG)
logger = logging.getLogger(__name__)
triggerrpc = TriggerRPC()
specrpc = SpecificationRPC()
momrpc = MoMQueryRPC()
from .config import TRIGGER_SUBMISSION_NOTIFICATION_SUBJECT
notification_bus = ToBus()
# The base URL for triggers specifications
base_url = 'https://lofar.astron.nl/mom3/user/main/explorer/setUpExplorer.do?action=open&object=FP1_CF'
if "LOFARENV" in os.environ:
lofar_environment = os.environ['LOFARENV']
if lofar_environment != "PRODUCTION":
base_url = 'http://lofartest.control.lofar:8080/mom3/user/main/explorer/setUpExplorer.do?action=open&object=FP1_CF'
class TriggerListView(views.APIView):
def __init__(self, **kwargs):
super(TriggerListView, self).__init__(**kwargs)
notification_bus.open()
# Check the invocation arguments
parser = OptionParser("%prog [options]")
parser.add_option("-B", "--busname", dest="busname", type="string", default=DEFAULT_OTDB_NOTIFICATION_BUSNAME,
help="Busname or queue-name the status changes are published on. [default: %default]")
parser.add_option_group(dbcredentials.options_group(parser))
(options, args) = parser.parse_args()
dbcreds = dbcredentials.parse_options(options)
# Set signalhandler to stop the program in a neat way.
signal.signal(signal.SIGINT, signal_handler)
alive = True
connected = False
otdb_connection = None
with ToBus(options.busname) as send_bus:
while alive:
while alive and not connected:
# Connect to the database
try:
otdb_connection = pg.connect(**dbcreds.pg_connect_options())
connected = True
logger.info("Connected to database %s" % (dbcreds,))
# Get list of allowed tree states
allowed_states = {}
for (state_nr, name) in otdb_connection.query("select id,name from treestate").getresult():
allowed_states[state_nr] = name
except (TypeError, SyntaxError, pg.InternalError), e:
connected = False
logger.error("Not connected to database %s, retry in 5 seconds: %s" % (dbcreds, e))
def setUp(self):
# Create a random bus
self.busname = "%s-%s" % (sys.argv[0], str(uuid.uuid4())[:8])
self.bus = ToBus(self.busname, {
"create": "always",
"delete": "always",
"node": {
"type": "topic"
}
})
self.bus.open()
self.addCleanup(self.bus.close)
# Patch SLURM
class MockSlurm(object):
def __init__(self, *args, **kwargs):
self.scheduled_jobs = {}
def submit(self, jobName, *args, **kwargs):
print "Schedule SLURM job '%s': %s, %s" % (jobName, args,
kwargs)
self.scheduled_jobs[jobName] = (args, kwargs)
# Return job ID
return "42"
def jobid(self, jobname):
if jobname in ["1", "2", "3"]:
return jobname
# "4" is an observation, so no SLURM job
return None
patcher = patch('lofar.mac.PipelineControl.Slurm')
patcher.start().side_effect = MockSlurm
self.addCleanup(patcher.stop)
# Catch functions to prevent running executables
patcher = patch('lofar.mac.PipelineControl.Parset.dockerImage')
patcher.start().return_value = "lofar-pipeline:trunk"
self.addCleanup(patcher.stop)
# ================================
# Setup mock otdb service
# ================================
class MockOTDBService(MessageHandlerInterface):
def __init__(self, notification_bus):
"""
notification_bus: bus to send state changes to
"""
super(MockOTDBService, self).__init__()
self.service2MethodMap = {
"TaskGetSpecification": self.TaskGetSpecification,
"TaskSetStatus": self.TaskSetStatus,
}
self.notification_bus = notification_bus
def TaskGetSpecification(self, OtdbID):
print "***** TaskGetSpecification(%s) *****" % (OtdbID, )
if OtdbID == 1:
predecessors = "[2,3,4]"
elif OtdbID == 2:
predecessors = "[3]"
elif OtdbID == 3:
predecessors = "[]"
elif OtdbID == 4:
return {
"TaskSpecification": {
"Version.number":
"1",
PARSET_PREFIX + "Observation.otdbID":
str(OtdbID),
PARSET_PREFIX + "Observation.Scheduler.predecessors":
"[]",
PARSET_PREFIX + "Observation.processType":
"Observation",
PARSET_PREFIX + "Observation.Cluster.ProcessingCluster.clusterName":
"CEP4",
PARSET_PREFIX + "Observation.stopTime":
"2016-01-01 01:00:00",
}
}
else:
raise Exception("Invalid OtdbID: %s" % OtdbID)
return {
"TaskSpecification": {
"Version.number":
"1",
PARSET_PREFIX + "Observation.otdbID":
str(OtdbID),
PARSET_PREFIX + "Observation.Scheduler.predecessors":
predecessors,
PARSET_PREFIX + "Observation.processType":
"Pipeline",
PARSET_PREFIX + "Observation.Cluster.ProcessingCluster.clusterName":
"CEP4",
}
}
def TaskSetStatus(self, OtdbID, NewStatus, UpdateTimestamps):
print "***** TaskSetStatus(%s,%s) *****" % (OtdbID, NewStatus)
# Broadcast the state change
content = {
"treeID": OtdbID,
"state": NewStatus,
"time_of_change": datetime.datetime.utcnow()
}
msg = EventMessage(context=DEFAULT_OTDB_NOTIFICATION_SUBJECT,
content=content)
self.notification_bus.send(msg)
return {'OtdbID': OtdbID, 'MomID': None, 'Success': True}
service = Service(DEFAULT_OTDB_SERVICENAME,
MockOTDBService,
busname=self.busname,
use_service_methods=True,
handler_args={"notification_bus": self.bus})
service.start_listening()
self.addCleanup(service.stop_listening)
# ================================
# Setup listener to catch result
# of our service
# ================================
listener = OTDBBusListener(busname=self.busname)
listener.start_listening()
self.addCleanup(listener.stop_listening)
self.trigger = MethodTrigger(listener, "onObservationQueued")
def create_service(busname, dbcreds, state_file_path='~/.lofar/otdb_treestatusevent_state'):
alive = True
connected = False
otdb_connection = None
with ToBus(busname) as send_bus:
while alive:
while alive and not connected:
# Connect to the database
try:
otdb_connection = pg.connect(**dbcreds.pg_connect_options())
connected = True
logger.info("Connected to database %s" % (dbcreds,))
# Get list of allowed tree states
allowed_states = {}
for (state_nr, name) in otdb_connection.query("select id,name from treestate").getresult():
allowed_states[state_nr] = name
except (TypeError, SyntaxError, pg.InternalError) as e:
connected = False
logger.error("Not connected to database %s, retry in 5 seconds: %s" % (dbcreds, e))
time.sleep(5)
# When we are connected we can poll the database
if connected:
# Get start_time (= creation time of last retrieved record if any)
try:
treestatuseventfilename = os.path.expanduser(state_file_path)
with open(treestatuseventfilename, 'r') as f:
line = f.readline()
if line.rfind('.') > 0:
start_time = datetime.datetime.strptime(line, "%Y-%m-%d %H:%M:%S.%f")
else:
start_time = datetime.datetime.strptime(line, "%Y-%m-%d %H:%M:%S")
except Exception as e:
logger.warning(e)
# start scanning from events since 'now'
# this timestamp will be stored in the treestatuseventfilename file
start_time = datetime.datetime.utcnow()
try:
logger.info("creating %s" % (treestatuseventfilename,))
if not os.path.exists(os.path.dirname(treestatuseventfilename)):
os.makedirs(os.path.dirname(treestatuseventfilename))
with open(treestatuseventfilename, 'w') as f:
f.write(start_time.strftime("%Y-%m-%d %H:%M:%S"))
except Exception as e:
logger.error(e)
try:
logger.debug("start_time=%s, polling database" % (start_time,))
record_list = PollForStatusChanges(start_time, otdb_connection)
for (treeid, state, modtime, creation) in record_list:
content = { "treeID" : treeid, "state" : allowed_states.get(state, "unknown_state"),
"time_of_change" : modtime }
msg = EventMessage(subject=DEFAULT_OTDB_NOTIFICATION_SUBJECT, content=content)
logger.info("sending message treeid %s state %s modtime %s" % (treeid, allowed_states.get(state, "unknown_state"), modtime))
send_bus.send(msg)
logger.debug("new start_time:=%s" % (creation,))
try:
with open(treestatuseventfilename, 'w') as f:
f.write(creation)
except Exception as e:
logger.error(e)
except FunctionError as exc_info:
logger.error(exc_info)
except Exception as e:
logger.error(e)
# Redetermine the database status.
connected = (otdb_connection and otdb_connection.status == 1)
time.sleep(2)