startTime = time.time()
# see if we find any OUSs with state=DeliveryInProgress, substate=ProductsIngested
dbcon = DbConnection(baseUrl)
dbName = 'status-entities'
selector = {
"selector": {
"$and": [{
"state": "DeliveryInProgress"
}, {
"substate": "ProductsIngested"
}]
},
"fields": ["entityId", "timestamp"]
}
retcode, ousStatuses = dbcon.find(dbName, selector)
if retcode != 200:
raise RuntimeError("find: %s: error %d: %s" % (dbName, retcode, OUSs))
# For each OUS status entity we found, see if all data was actually replicated here
if len(ousStatuses) > 0:
startTime = time.time() # Reset startTime for incremental waiting
ousStatuses = sorted(ousStatuses, key=compareByTimestamp)
for ous in ousStatuses:
ousUID = ous['entityId']
# ts = ous['timestamp']
# print( ">>> found", ousUID, ts )
# Retrieve the list of products from the delivery status
encodedUID = dbdrwutils.encode(ousUID)
dbName = 'delivery-status'
class MqConnection():
"""
Implements a RabbitMQ-like message queue.
Implementation is based on CouchDB
Constructor args:
host: where the queue server is running
queueName: name of queue to communicate on
"""
try:
# This may or may not work -- it's some third party service I found somwewhere
# It will fail if queried too often, like more than once per second
myIP = urllib.request.urlopen('http://api.infoip.io/ip').read().decode(
'utf8')
except Exception:
myIP = "0.0.0.0"
def __init__(self, host, queueName, listenTo=None, sendTo=None):
self.host = host
self.queueName = queueName
self.listenTo = None
self.sendTo = None
if listenTo != None:
self.listenTo = listenTo
if sendTo != None:
self.sendTo = sendTo
self.dbcon = DbConnection(baseUrl)
# print( " [x] Created queue %s on %s" % ( self.queueName, self.host ))
def send(self, messageBody, selector=None, addMsgbackID=False):
'''
Send a message to some other filter listening on the queue for the selector
'''
if selector == None:
selector = self.sendTo
if selector == None:
raise RuntimeError("No selectors to send to")
now = nowISO()
# Are we breadcasting to a group?
if not selector.endswith('.*'):
# NO, just to a single receiver
return self._send(now, selector, addMsgbackID, messageBody)
else:
# YES, let's retrieve the group and send to all its participants
# TODO: cache group definition somewhere instead of querying
# the database every time
retcode, group = self.dbcon.findOne(self.queueName, selector)
if retcode == 404:
raise RuntimeError("Group not found: %" % selector)
messages = []
for member in group['members']:
m = self._send(now, member, addMsgbackID, messageBody)
messages.append(m)
return messages
def _send(self, now, selector, addMsgbackID, messageBody):
message = {}
msgbackID = str(uuid.uuid4()).replace("-", "")
message['creationTimestamp'] = now
message['originIP'] = MqConnection.myIP
message['selector'] = selector
message['consumed'] = False
if addMsgbackID:
message['msgbackID'] = msgbackID
message['body'] = messageBody
messageID = now + "-" + msgbackID
retcode, msg = self.dbcon.save(self.queueName, messageID, message)
if retcode != 201:
raise RuntimeError("Msg send failed: DB error: %s: %s" %
(retcode, msg))
return message
def getNext(self,
selector,
consume=True,
fullMessage=False,
condition=None):
"""
Listen on the queue for for new messages, return the oldest we find.
Args:
selector: defines what messages to listen to
consume: if True, the message will be marked as consumed and no other
listener will receive (default is True)
fullMessage: if True, the message's metadata will be passed in as well and
the actual message will be in the 'body' field (default False)
condition: boolean function to be invoked before starting to listen, will cause the
thread to sleep if the condition is false
"""
messages = []
callTime = time.time()
# print( ">>> callTime: " + str(callTime) )
selector = {
"selector": {
"$and": [{
"selector": {
"$regex": selector
}
}, {
"consumed": False
}]
}
#,
# "sort": [{"creationTimestamp":"desc"}]
#
# We should let the server sort the results but that
# requires an index to be created and I don't care about
# that right now -- amchavan, 13-Jul-2018
#
# TODO: revisit this if needed
}
# See if we can even start listening: if we have a conditional expression
# and it evaluates to False we need to wait a bit
while (condition and (condition() == False)):
time.sleep(dbdrwutils.incrementalSleep(callTime))
while True:
retcode, messages = self.dbcon.find(self.queueName, selector)
# print( ">>> selector:", selector, "found:", len(messages))
if retcode == 200:
if len(messages) != 0:
break
else:
time.sleep(dbdrwutils.incrementalSleep(callTime))
else:
raise RuntimeError("Msg read failed: DB error: %s: %s" %
(retcode, messages))
# print( ">>> found: ", messages )
# print( ">>> found: ", len( messages ))
messages.sort(key=lambda x: x['creationTimestamp']) # Oldest first
ret = messages[0]
if consume:
ret['consumed'] = True
self.dbcon.save(self.queueName, ret["_id"], ret)
# print( ">>> found: ", ret )
if fullMessage:
return ret
return ret['body']
def listen(self,
callback,
selector=None,
consume=True,
fullMessage=False,
condition=None):
"""
Listen on the queueName for messages matching the selector and process them.
Args:
callback: function to process the message with
selector: defines what messages to listen to
consume: UNUSED: if True, the message will be marked as consumed and no other
listener will receive (default is True)
fullMessage: UNUSED: if True, the message's metadata will be passed in as well and
the actual message will be in the 'body' field (default False)
condition: boolean function to be invoked before starting to listen, will cause the
thread to sleep if the condition is false
"""
if selector == None:
selector = self.listenTo
if selector == None:
raise RuntimeError("No selectors to listen to")
while True:
# print( ">>> waiting for message on queue '%s' matching selector '%s' ..." % (self.queueName, selector))
message = self.getNext(selector,
consume,
fullMessage=fullMessage,
condition=condition)
# print( ">>> got", message )
callback(message)
def joinGroup(self, groupName, listener=None):
"""
Join a group as a listener. Messages sent to the group will be
passed on to this instance as well.
Group names must end with '.*'
Arg listener defaults to the value of the listenTo
constructor arg.
"""
if groupName == None:
raise RuntimeError("No group name to join")
if not groupName.endswith(".*"):
raise RuntimeError("Group names must end with .*")
if listener == None:
listener = self.listenTo
if listener == None:
raise RuntimeError("No listener to join as")
# We want to join the group groupName as listener
# First let's see if that group exists, otherwise we'll create it
retcode, group = self.dbcon.findOne(self.queueName, groupName)
if retcode == 404:
print(">>> not found: group=%s" % groupName)
group = {}
# group['groupName'] = groupName
group['members'] = [listener]
self.dbcon.save(self.queueName, groupName, group)
print(">>> created: group=%s" % groupName)
return
# Found a group with that name: if needed, add ourselves to it
print(">>> found: group=%s" % group)
members = group['members']
if not listener in members:
group['members'].append(listener)
self.dbcon.save(self.queueName, groupName, group)
print(">>> added ourselves as members: group=%s" % group)
else:
print(">>> already in members list: listener=%s" % listener)
class QA2():
def __init__(self):
self.weblogsBaseUrl = "http://localhost:8000"
self.baseUrl = "http://localhost:5984" # CouchDB
self.dbconn = DbConnection(self.baseUrl)
self.dbName = "pipeline-reports"
self.xtss = ExecutorClient('localhost', 'msgq', 'xtss')
self.select = "pipeline.report.JAO"
self.mq = MqConnection('localhost', 'msgq', select)
def start(self):
# Launch the listener in the background
print(' [*] Waiting for messages matching %s' % (self.select))
dbdrwutils.bgRun(self.mq.listen, (self.callback,))
# This is the program's text-based UI
# Loop forever:
# Show Pipeline runs awaiting review
# Ask for an OUS UID
# Lookup the most recent PL execution for that
# Print it out
# Ask for Fail, Pass, or SemiPass
# Set the OUS state accordingly
while True:
print()
print()
print('------------------------------------------')
print()
print("OUSs ready to be reviewed")
ouss = self.findReadyForReview()
if (ouss == None or len(ouss) == 0):
print("(none)")
else:
for ous in ouss:
print(ous['entityId'])
print()
ousUID = input('Please enter an OUS UID: ')
plReport = self.findMostRecentPlReport(ousUID)
if plReport == None:
print("No Pipeline executions for OUS", ousUID)
continue
# We are reviewing this OUS, set its state accordingly
dbdrwutils.setState(self.xtss, ousUID, "Reviewing")
timestamp = plReport['timestamp']
report = dbdrwutils.b64decode(plReport['encodedReport'])
productsDir = plReport['productsDir']
source = plReport['source']
print("Pipeline report for UID %s, processed %s" % (ousUID,timestamp))
print(report)
print()
print("Weblog available at: %s/weblogs/%s" % (self.weblogsBaseUrl, dbdrwutils.makeWeblogName(ousUID, timestamp)))
print()
while True:
reply = input("Enter [F]ail, [P]ass, [S]emipass, [C]ancel: ")
reply = reply[0:1].upper()
if ((reply=='F') or (reply=='P') or (reply=='S') or (reply=='C')):
break
if reply == 'C':
continue
# Set the OUS state according to the QA2 flag
self.processQA2flag(ousUID, reply)
if reply == 'F':
continue
# Tell the Product Ingestor that it should ingest those Pipeline products
selector = "ingest.JAO"
message = '{"ousUID" : "%s", "timestamp" : "%s", "productsDir" : "%s"}' % \
(ousUID, timestamp, productsDir)
message = {}
message["ousUID"] = ousUID
message["timestamp"] = timestamp
message["productsDir"] = productsDir
self.mq.send(message, selector)
# Wait some, mainly for effect
waitTime = random.randint(3,8)
time.sleep(waitTime)
# Now we can set the state of the OUS to DeliveryInProgress
dbdrwutils.setState(self.xtss, ousUID, "DeliveryInProgress")
def savePlReport(self, ousUID, timestamp, encodedReport, productsDir, source):
'''
Saves a pipeline run report to 'Oracle'
'''
plReport = {}
plReport['ousUID'] = ousUID
plReport['timestamp'] = timestamp
plReport['encodedReport'] = encodedReport
plReport['productsDir'] = productsDir
plReport['source'] = source
plReportID = timestamp + "." + ousUID
retcode,msg = self.dbconn.save(self.dbName, plReportID, plReport)
if retcode != 201:
raise RuntimeError("Error saving Pipeline report: %d, %s" % (retcode,msg))
def findMostRecentPlReport(self, ousUID):
selector = { "selector": { "ousUID": ousUID }}
retcode,reports = self.dbconn.find(self.dbName, selector)
if len(reports) == 0:
return None
if retcode != 200:
print(reports)
return None
# Find the most recent report and return it
reports.sort(key=lambda x: x['timestamp'], reverse=True)
return reports[0]
def findReadyForReview(self):
selector = {
"selector": {
"state": "ReadyForReview"
}
}
retcode,ouss = self.dbconn.find("status-entities", selector)
if len(ouss) == 0:
return None
if retcode != 200:
print(ouss)
return None
ouss.sort(key=lambda x: x['entityId'])
return ouss
def processQA2flag(self, ousUID, flag):
"Flag should be one of 'F' (fail), 'P' (pass) or 'S' (semi-pass)"
newState = "ReadyForProcessing" if (flag == "F") else "Verified"
print(">>> Setting the state of", ousUID, "to", newState)
# Set the OUS state according to the input flag
dbdrwutils.setState(self.xtss, ousUID, newState)
if flag == "F":
dbdrwutils.setSubstate(self.xtss, ousUID, "") # Clear Pipeline recipe
def callback(self, message):
"""
Message is a JSON object:
ousUID is the UID of the OUS
source is the executive where the Pipeline was running
report is the report's XML text, BASE64-encoded
timestamp is the Pipeline run's timestamp
productsDir is the name of the products directory for that Pipeline run
For instance
{
"ousUID" : "uid://X1/X1/Xaf",
"source" : "EU",
"report" : "Cjw/eG1sIHZlcnNpb2..."
"timestamp" : "2018-07-19T08:50:10.228",
"productsDir": "2015.1.00657.S_2018_07_19T08_50_10.228"
}
"""
# print(">>> message:", message)
ousUID = message["ousUID"]
source = message["source"]
encodedReport = message["report"]
timestamp = message["timestamp"]
productsDir = message["productsDir"]
# report = dbdrwutils.b64decode(encodedReport)
# print(">>> report:", report)
# Save the report to Oracle
self.savePlReport(ousUID, timestamp, encodedReport, productsDir, source)
print(">>> AQUA/QA2: saved PL report: ousUID=%s, timestamp=%s" % (ousUID,timestamp))
class DRA():
def __init__(self, location):
self._baseUrl = "http://localhost:5984" # CouchDB
self._dbconn = DbConnection(baseUrl)
self._xtss = ExecutorClient('localhost', 'msgq', 'xtss')
self._mq = MqConnection('localhost', 'msgq')
self._broker = RabbitMqMessageBroker()
self.location = location
def findReadyForPipelineProcessing(self):
"""
Returns all ReadyForProcessing OUSs with a Pipeline Recipe, if
any are found; None otherwise
"""
selector = {
"selector": {
"state": "ReadyForProcessing",
"substate": {
"$regex": "^Pipeline"
}
}
}
retcode, ouss = self._dbconn.find("status-entities", selector)
if len(ouss) == 0:
return None
if retcode != 200:
print(ouss)
return None
ouss.sort(key=lambda x: x['entityId'])
return ouss
def start(self):
# This is the program's text-based UI
# Loop forever:
# Show Pipeline runs ready for processing
# Ask for an OUS UID
# Set:
# state=Processing
# PL_PROCESSING_EXECUTIVE=$DRAWS_LOCATION
while True:
print()
print()
print('------------------------------------------')
print()
print("ReadyForProcessing OUSs")
ouss = self.findReadyForPipelineProcessing()
if (ouss == None or len(ouss) == 0):
print("(none)")
sys.exit()
else:
ousMap = {}
for ous in ouss:
entityId = ous['entityId']
print(entityId)
ousMap[entityId] = ous
print()
ousUID = input(
'Please enter an OUS UID, will be processed at %s: ' %
self.location)
if not (ousUID in ousMap):
print("No OUS with UID='%s'" % (ousUID))
continue
ous = ousMap[ousUID]
# We are going to process this OUS, set its state and processing executive accordingly
dbdrwutils.setState(self._xtss, ousUID, "Processing")
dbdrwutils.setExecutive(self._xtss, ousUID, self.location)
# Launch the Pipeline Driver on Torque/Maui (pretending it listens to messages)
message = {}
message['ousUID'] = ousUID
message['recipe'] = ous["substate"]
message['progID'] = ous["progID"]
# msgTemplate = '{ "ousUID" : "%s", "recipe" : "%s", "progID" : "%s" }'
# message = msgTemplate % (ousUID, ous["substate"], ous["progID"])
torque = "pipeline.process." + self.location
dbdrwutils.sendMsgToSelector(message, torque, self._mq)
# Wait some, mainly for effect
waitTime = random.randint(3, 8)
time.sleep(waitTime)
class BatchHelper():
recipes = [
"ManualCalibration",
# Most manual recipes commented out because we don't want
# too many of them in this mockup
#"ManualImaging",
#"ManualSingleDish",
#"ManualCombination",
"PipelineCalibration",
"PipelineImaging",
"PipelineSingleDish",
"PipelineCombination",
"PipelineCalAndImg"
]
def __init__(self):
self.baseUrl = "http://localhost:5984" # CouchDB
self.dbconn = DbConnection(self.baseUrl)
def start(self):
self.setRecipes()
def findReadyForProcessingNoSubstate(self):
"""
Returns a ReadyForProcessing OUSs with no substate, if any are
found; None otherwise
"""
selector = {
"selector": {
"state": "ReadyForProcessing",
"substate": {"$or": [{ "$eq": None }, { "$eq": "" }]}
}
}
retcode,ouss = self.dbconn.find("status-entities", selector)
if len(ouss) == 0:
return None
if retcode != 200:
print(ouss)
return None
ouss.sort(key=lambda x: x['entityId'])
return ouss[0]
def computeRecipe(self, ous):
"Just pick a recipe at random"
return random.choice(BatchHelper.recipes)
def setRecipes(self):
"""
Runs on a background thread.
Loop forever:
Look for ReadyForProcessing OUSs with no Pipeline Recipe
If you find one:
Compute the Pipeline recipe for that OUS
Set it
Sleep some time
"""
while True:
ous = self.findReadyForProcessingNoSubstate()
if ous != None:
ous["substate"] = self.computeRecipe(ous)
self.dbconn.save("status-entities", ous["_id"], ous)
print(">>> OUS:", ous["_id"], "recipe:", ous["substate"])
time.sleep(5)