def writeMetadata(progID, ousUID, timestamp, dataProduct):
# No error checking!
dbcon = DbConnection(baseUrl)
dbName = "products-metadata"
metadata = {}
metadata['progID'] = progID
metadata['ousUID'] = ousUID
metadata['timestamp'] = timestamp
retcode, retmsg = dbcon.save(dbName, dataProduct, metadata)
if retcode != 201:
raise RuntimeError("setSubstate: error %d, %s" % (retcode, retmsg))
def writeDeliveryStatus(progID,
ousUID,
timestamp,
dataProducts,
complete=False):
# No error checking!
dbcon = DbConnection(baseUrl)
dbName = "delivery-status"
delStatus = {}
delStatus['progID'] = progID
delStatus['timestamp'] = timestamp
delStatus['dataProducts'] = sorted(dataProducts)
delStatus['complete'] = complete
delStatus['ousUID'] = ousUID
retcode, retmsg = dbcon.save(dbName, ousUID, delStatus)
if retcode != 201:
raise RuntimeError("setSubstate: error %d, %s" % (retcode, retmsg))
class NgasConnection():
'''
A simple API for NGAS
'''
def __init__( self ):
self.dbName = "ngas"
self.dbcon = DbConnection( baseUrl )
def __readBinaryFileIntoString( self, filename ):
with open(filename, mode='rb') as file: # b is important -> binary
fileContent = file.read()
s = base64.b64encode(fileContent).decode()
return s
def __writeStringAsBinaryFile( self, s, filename ):
b = base64.b64decode( s )
with open( filename, mode='wb' ) as file:
file.write( b )
def put( self, pathname ):
s = self.__readBinaryFileIntoString( pathname )
basename = os.path.basename( pathname )
file = {}
# file['filename'] = basename
file['encodedContents'] = s
file['writeTimestamp'] = dbdrwutils.nowISO()
# print( ">>> attempting save(): dbName: %s, basename: %s, file: %s" % (self.dbName, basename, file) )
retcode,msg = self.dbcon.save( self.dbName, basename, file )
# print( ">>> ngas retcode:", retcode, "msg:", msg )
return 0 if (retcode==201) else retcode
def check( self, id ):
"Return true if we have a file with the given ID, false otherwise"
retcode,files = self.dbcon.findOne( self.dbName, id )
if retcode != 200:
raise RuntimeError( "NGAS: error %d: %s" % ( retcode,files ))
return True if ( len(files) > 0 ) else False
class XTSS():
def __init__(self):
self._baseUrl = "http://localhost:5984" # CouchDB
self._dbcon = DbConnection(self._baseUrl)
self._dbName = "status-entities"
self._broker = RabbitMqMessageBroker()
self._subscriber = Subscriber(self._broker, 'xtss.transitions', 'xtss')
def start(self):
executor = Executor('localhost', 'msgq', 'xtss', self.xtss)
print(" [x] Awaiting RPC requests to 'xtss'")
executor.run()
def __nowISO(self):
return datetime.datetime.utcnow().isoformat()[:-3]
def findOUSStatus(self, ousUID):
"Find an OUSStatus with the given ID, raise an error if none are found"
retcode,ousStatus = self._dbcon.findOne(self._dbName, ousUID)
if retcode == 404:
raise RuntimeError("OUS not found: %s" % ousUID)
return ousStatus
def setField(self, ousUID, fieldName, fieldValue):
"Set the value of a field of an OUSStatus, update its timestamp"
ousStatus = self.findOUSStatus(ousUID)
ousStatus[fieldName] = fieldValue
ousStatus['timestamp'] = self.__nowISO()
retcode,msg = self._dbcon.save(self._dbName, ousUID, ousStatus)
return retcode
def setState(self, ousUID, state):
"Set the state of an OUSStatus"
return self.setField ousUID, 'state', state)
def setSubstate(self, ousUID, substate):
"Set the substate of an OUSStatus"
return self.setField(ousUID, 'substate', substate)
def setExecutive(self, ousUID, executive):
"Set the Executive of an OUSStatus"
return self.setFlag(self, ousUID, 'PL_PROCESSING_EXECUTIVE', executive)
def clearExecutive(self, ousUID):
"Clear the Executive of an OUSStatus"
return self.clearFlag ousUID, 'PL_PROCESSING_EXECUTIVE')
def setFlag(self, ousUID, name, value):
"Set an OUSStatus flag"
ousStatus = self.findOUSStatus ousUID)
if 'flags' in ousStatus:
flags = ousStatus['flags']
else:
flags = {}
flags[name] = value
return self.setField(ousUID, 'flags', flags)
def clearFlag(self, ousUID, name):
"Clear an OUSStatus flag"
ousStatus = self.findOUSStatus(ousUID)
if 'flags' in ousStatus:
flags = ousStatus['flags']
else:
flags = {}
if name in flags:
del flags[name]
return self.setField(ousUID, 'flags', flags)
def findByStateSubstate(self, state, substate):
"""
Returns a return code and, if all was well and the code is 200,
all OUSs with the given state and substate; note substate is
interpreted as a regexp
"""
selector = {
"selector": {
"state": state,
"substate": { "$regex": substate }
class MqConnection():
"""
Implements a RabbitMQ-like message queue.
Implementation is based on CouchDB
Constructor args:
host: where the queue server is running
queueName: name of queue to communicate on
"""
try:
# This may or may not work -- it's some third party service I found somwewhere
# It will fail if queried too often, like more than once per second
myIP = urllib.request.urlopen('http://api.infoip.io/ip').read().decode(
'utf8')
except Exception:
myIP = "0.0.0.0"
def __init__(self, host, queueName, listenTo=None, sendTo=None):
self.host = host
self.queueName = queueName
self.listenTo = None
self.sendTo = None
if listenTo != None:
self.listenTo = listenTo
if sendTo != None:
self.sendTo = sendTo
self.dbcon = DbConnection(baseUrl)
# print( " [x] Created queue %s on %s" % ( self.queueName, self.host ))
def send(self, messageBody, selector=None, addMsgbackID=False):
'''
Send a message to some other filter listening on the queue for the selector
'''
if selector == None:
selector = self.sendTo
if selector == None:
raise RuntimeError("No selectors to send to")
now = nowISO()
# Are we breadcasting to a group?
if not selector.endswith('.*'):
# NO, just to a single receiver
return self._send(now, selector, addMsgbackID, messageBody)
else:
# YES, let's retrieve the group and send to all its participants
# TODO: cache group definition somewhere instead of querying
# the database every time
retcode, group = self.dbcon.findOne(self.queueName, selector)
if retcode == 404:
raise RuntimeError("Group not found: %" % selector)
messages = []
for member in group['members']:
m = self._send(now, member, addMsgbackID, messageBody)
messages.append(m)
return messages
def _send(self, now, selector, addMsgbackID, messageBody):
message = {}
msgbackID = str(uuid.uuid4()).replace("-", "")
message['creationTimestamp'] = now
message['originIP'] = MqConnection.myIP
message['selector'] = selector
message['consumed'] = False
if addMsgbackID:
message['msgbackID'] = msgbackID
message['body'] = messageBody
messageID = now + "-" + msgbackID
retcode, msg = self.dbcon.save(self.queueName, messageID, message)
if retcode != 201:
raise RuntimeError("Msg send failed: DB error: %s: %s" %
(retcode, msg))
return message
def getNext(self,
selector,
consume=True,
fullMessage=False,
condition=None):
"""
Listen on the queue for for new messages, return the oldest we find.
Args:
selector: defines what messages to listen to
consume: if True, the message will be marked as consumed and no other
listener will receive (default is True)
fullMessage: if True, the message's metadata will be passed in as well and
the actual message will be in the 'body' field (default False)
condition: boolean function to be invoked before starting to listen, will cause the
thread to sleep if the condition is false
"""
messages = []
callTime = time.time()
# print( ">>> callTime: " + str(callTime) )
selector = {
"selector": {
"$and": [{
"selector": {
"$regex": selector
}
}, {
"consumed": False
}]
}
#,
# "sort": [{"creationTimestamp":"desc"}]
#
# We should let the server sort the results but that
# requires an index to be created and I don't care about
# that right now -- amchavan, 13-Jul-2018
#
# TODO: revisit this if needed
}
# See if we can even start listening: if we have a conditional expression
# and it evaluates to False we need to wait a bit
while (condition and (condition() == False)):
time.sleep(dbdrwutils.incrementalSleep(callTime))
while True:
retcode, messages = self.dbcon.find(self.queueName, selector)
# print( ">>> selector:", selector, "found:", len(messages))
if retcode == 200:
if len(messages) != 0:
break
else:
time.sleep(dbdrwutils.incrementalSleep(callTime))
else:
raise RuntimeError("Msg read failed: DB error: %s: %s" %
(retcode, messages))
# print( ">>> found: ", messages )
# print( ">>> found: ", len( messages ))
messages.sort(key=lambda x: x['creationTimestamp']) # Oldest first
ret = messages[0]
if consume:
ret['consumed'] = True
self.dbcon.save(self.queueName, ret["_id"], ret)
# print( ">>> found: ", ret )
if fullMessage:
return ret
return ret['body']
def listen(self,
callback,
selector=None,
consume=True,
fullMessage=False,
condition=None):
"""
Listen on the queueName for messages matching the selector and process them.
Args:
callback: function to process the message with
selector: defines what messages to listen to
consume: UNUSED: if True, the message will be marked as consumed and no other
listener will receive (default is True)
fullMessage: UNUSED: if True, the message's metadata will be passed in as well and
the actual message will be in the 'body' field (default False)
condition: boolean function to be invoked before starting to listen, will cause the
thread to sleep if the condition is false
"""
if selector == None:
selector = self.listenTo
if selector == None:
raise RuntimeError("No selectors to listen to")
while True:
# print( ">>> waiting for message on queue '%s' matching selector '%s' ..." % (self.queueName, selector))
message = self.getNext(selector,
consume,
fullMessage=fullMessage,
condition=condition)
# print( ">>> got", message )
callback(message)
def joinGroup(self, groupName, listener=None):
"""
Join a group as a listener. Messages sent to the group will be
passed on to this instance as well.
Group names must end with '.*'
Arg listener defaults to the value of the listenTo
constructor arg.
"""
if groupName == None:
raise RuntimeError("No group name to join")
if not groupName.endswith(".*"):
raise RuntimeError("Group names must end with .*")
if listener == None:
listener = self.listenTo
if listener == None:
raise RuntimeError("No listener to join as")
# We want to join the group groupName as listener
# First let's see if that group exists, otherwise we'll create it
retcode, group = self.dbcon.findOne(self.queueName, groupName)
if retcode == 404:
print(">>> not found: group=%s" % groupName)
group = {}
# group['groupName'] = groupName
group['members'] = [listener]
self.dbcon.save(self.queueName, groupName, group)
print(">>> created: group=%s" % groupName)
return
# Found a group with that name: if needed, add ourselves to it
print(">>> found: group=%s" % group)
members = group['members']
if not listener in members:
group['members'].append(listener)
self.dbcon.save(self.queueName, groupName, group)
print(">>> added ourselves as members: group=%s" % group)
else:
print(">>> already in members list: listener=%s" % listener)
class QA2():
def __init__(self):
self.weblogsBaseUrl = "http://localhost:8000"
self.baseUrl = "http://localhost:5984" # CouchDB
self.dbconn = DbConnection(self.baseUrl)
self.dbName = "pipeline-reports"
self.xtss = ExecutorClient('localhost', 'msgq', 'xtss')
self.select = "pipeline.report.JAO"
self.mq = MqConnection('localhost', 'msgq', select)
def start(self):
# Launch the listener in the background
print(' [*] Waiting for messages matching %s' % (self.select))
dbdrwutils.bgRun(self.mq.listen, (self.callback,))
# This is the program's text-based UI
# Loop forever:
# Show Pipeline runs awaiting review
# Ask for an OUS UID
# Lookup the most recent PL execution for that
# Print it out
# Ask for Fail, Pass, or SemiPass
# Set the OUS state accordingly
while True:
print()
print()
print('------------------------------------------')
print()
print("OUSs ready to be reviewed")
ouss = self.findReadyForReview()
if (ouss == None or len(ouss) == 0):
print("(none)")
else:
for ous in ouss:
print(ous['entityId'])
print()
ousUID = input('Please enter an OUS UID: ')
plReport = self.findMostRecentPlReport(ousUID)
if plReport == None:
print("No Pipeline executions for OUS", ousUID)
continue
# We are reviewing this OUS, set its state accordingly
dbdrwutils.setState(self.xtss, ousUID, "Reviewing")
timestamp = plReport['timestamp']
report = dbdrwutils.b64decode(plReport['encodedReport'])
productsDir = plReport['productsDir']
source = plReport['source']
print("Pipeline report for UID %s, processed %s" % (ousUID,timestamp))
print(report)
print()
print("Weblog available at: %s/weblogs/%s" % (self.weblogsBaseUrl, dbdrwutils.makeWeblogName(ousUID, timestamp)))
print()
while True:
reply = input("Enter [F]ail, [P]ass, [S]emipass, [C]ancel: ")
reply = reply[0:1].upper()
if ((reply=='F') or (reply=='P') or (reply=='S') or (reply=='C')):
break
if reply == 'C':
continue
# Set the OUS state according to the QA2 flag
self.processQA2flag(ousUID, reply)
if reply == 'F':
continue
# Tell the Product Ingestor that it should ingest those Pipeline products
selector = "ingest.JAO"
message = '{"ousUID" : "%s", "timestamp" : "%s", "productsDir" : "%s"}' % \
(ousUID, timestamp, productsDir)
message = {}
message["ousUID"] = ousUID
message["timestamp"] = timestamp
message["productsDir"] = productsDir
self.mq.send(message, selector)
# Wait some, mainly for effect
waitTime = random.randint(3,8)
time.sleep(waitTime)
# Now we can set the state of the OUS to DeliveryInProgress
dbdrwutils.setState(self.xtss, ousUID, "DeliveryInProgress")
def savePlReport(self, ousUID, timestamp, encodedReport, productsDir, source):
'''
Saves a pipeline run report to 'Oracle'
'''
plReport = {}
plReport['ousUID'] = ousUID
plReport['timestamp'] = timestamp
plReport['encodedReport'] = encodedReport
plReport['productsDir'] = productsDir
plReport['source'] = source
plReportID = timestamp + "." + ousUID
retcode,msg = self.dbconn.save(self.dbName, plReportID, plReport)
if retcode != 201:
raise RuntimeError("Error saving Pipeline report: %d, %s" % (retcode,msg))
def findMostRecentPlReport(self, ousUID):
selector = { "selector": { "ousUID": ousUID }}
retcode,reports = self.dbconn.find(self.dbName, selector)
if len(reports) == 0:
return None
if retcode != 200:
print(reports)
return None
# Find the most recent report and return it
reports.sort(key=lambda x: x['timestamp'], reverse=True)
return reports[0]
def findReadyForReview(self):
selector = {
"selector": {
"state": "ReadyForReview"
}
}
retcode,ouss = self.dbconn.find("status-entities", selector)
if len(ouss) == 0:
return None
if retcode != 200:
print(ouss)
return None
ouss.sort(key=lambda x: x['entityId'])
return ouss
def processQA2flag(self, ousUID, flag):
"Flag should be one of 'F' (fail), 'P' (pass) or 'S' (semi-pass)"
newState = "ReadyForProcessing" if (flag == "F") else "Verified"
print(">>> Setting the state of", ousUID, "to", newState)
# Set the OUS state according to the input flag
dbdrwutils.setState(self.xtss, ousUID, newState)
if flag == "F":
dbdrwutils.setSubstate(self.xtss, ousUID, "") # Clear Pipeline recipe
def callback(self, message):
"""
Message is a JSON object:
ousUID is the UID of the OUS
source is the executive where the Pipeline was running
report is the report's XML text, BASE64-encoded
timestamp is the Pipeline run's timestamp
productsDir is the name of the products directory for that Pipeline run
For instance
{
"ousUID" : "uid://X1/X1/Xaf",
"source" : "EU",
"report" : "Cjw/eG1sIHZlcnNpb2..."
"timestamp" : "2018-07-19T08:50:10.228",
"productsDir": "2015.1.00657.S_2018_07_19T08_50_10.228"
}
"""
# print(">>> message:", message)
ousUID = message["ousUID"]
source = message["source"]
encodedReport = message["report"]
timestamp = message["timestamp"]
productsDir = message["productsDir"]
# report = dbdrwutils.b64decode(encodedReport)
# print(">>> report:", report)
# Save the report to Oracle
self.savePlReport(ousUID, timestamp, encodedReport, productsDir, source)
print(">>> AQUA/QA2: saved PL report: ousUID=%s, timestamp=%s" % (ousUID,timestamp))
## ./launcher.py 2015.1.00657.S uid://X1/X1/Xb2
parser = argparse.ArgumentParser(
description='Starter component, creates status entities')
parser.add_argument(dest="progID", help="ID of the project containing the OUS")
parser.add_argument(dest="ousUID",
help="ID of the OUS that should be processed")
args = parser.parse_args()
ousUID = args.ousUID
progID = args.progID
dbName = "status-entities"
baseUrl = "http://localhost:5984" # CouchDB
dbcon = DbConnection(baseUrl)
# If we have one already, delete it
retcode, ous = dbcon.findOne(dbName, ousUID)
if retcode == 200:
dbcon.delete(dbName, ousUID, ous["_rev"])
# Prepare a new record and write it
ous = {}
ous['entityId'] = ousUID
ous['progID'] = progID
ous['state'] = "ReadyForProcessing"
ous['substate'] = None
ous['flags'] = {}
ous['timestamp'] = dbdrwutils.nowISO()
r, t = dbcon.save(dbName, ousUID, ous)
print(">>> retcode:", r)
class BatchHelper():
recipes = [
"ManualCalibration",
# Most manual recipes commented out because we don't want
# too many of them in this mockup
#"ManualImaging",
#"ManualSingleDish",
#"ManualCombination",
"PipelineCalibration",
"PipelineImaging",
"PipelineSingleDish",
"PipelineCombination",
"PipelineCalAndImg"
]
def __init__(self):
self.baseUrl = "http://localhost:5984" # CouchDB
self.dbconn = DbConnection(self.baseUrl)
def start(self):
self.setRecipes()
def findReadyForProcessingNoSubstate(self):
"""
Returns a ReadyForProcessing OUSs with no substate, if any are
found; None otherwise
"""
selector = {
"selector": {
"state": "ReadyForProcessing",
"substate": {"$or": [{ "$eq": None }, { "$eq": "" }]}
}
}
retcode,ouss = self.dbconn.find("status-entities", selector)
if len(ouss) == 0:
return None
if retcode != 200:
print(ouss)
return None
ouss.sort(key=lambda x: x['entityId'])
return ouss[0]
def computeRecipe(self, ous):
"Just pick a recipe at random"
return random.choice(BatchHelper.recipes)
def setRecipes(self):
"""
Runs on a background thread.
Loop forever:
Look for ReadyForProcessing OUSs with no Pipeline Recipe
If you find one:
Compute the Pipeline recipe for that OUS
Set it
Sleep some time
"""
while True:
ous = self.findReadyForProcessingNoSubstate()
if ous != None:
ous["substate"] = self.computeRecipe(ous)
self.dbconn.save("status-entities", ous["_id"], ous)
print(">>> OUS:", ous["_id"], "recipe:", ous["substate"])
time.sleep(5)