class APAFunc(Task):
def __init__(self):
self._mq = MqConnection('localhost', 'msgq')
self._xtss = ExecutorClient('localhost', 'msgq', 'xtss')
def execute(self, params):
self._params = params
#Check parameters!
self._check(params, [
'progID', 'ousUID', 'location', 'pipelineRunDir', 'replicatedCache'
])
self._progID = params['progID']
self._ousUID = params['ousUID']
self._pipelineRunDirectory = params['pipelineRunDir']
self._replicatedCache = params['replicatedCache']
self._location = params['location']
# Copy the products directory to the replicating cache directory
# and signal that to the JAO cache
productsDir = self._findProductsDir(self._progID)
productsBasedir = os.path.basename(productsDir)
repCacheDir = os.path.join(self._replicatedCache, productsBasedir)
print(">>> PipelineDriver: Products dir name:", productsDir)
print(">>> PipelineDriver: Replicating dir name:", repCacheDir)
self._copyAndReplaceDir(productsDir, repCacheDir)
# message = '{"fileType":"productsdir", "cachedAt":"%s", "name": "%s"}' % (self._location,productsBasedir)
message = {}
message["fileType"] = "productsdir"
message["cachedAt"] = self._location
message["name"] = productsBasedir
selector = "cached.JAO"
self._mq.send(message, selector)
# Copy the weblog to the replicating cache directory
# and signal that to the JAO *and* the local cache (if
# they are not one and the same)
weblog = self._findWeblog(productsDir, self._ousUID)
print(">>> PipelineDriver: weblog: copying", weblog, "to",
self._replicatedCache)
shutil.copy(weblog, self._replicatedCache)
# message = '{"fileType":"weblog", "cachedAt":"%s", "name": "%s"}' % (self._location, os.path.basename(weblog))
message = {}
message["fileType"] = "weblog"
message["cachedAt"] = self._location
message["name"] = os.path.basename(weblog)
selector = "cached.JAO"
self._mq.send(message, selector)
if self._replicatedCache != "JAO":
selector = "cached.%s" % self._location
self._mq.send(message, selector)
# Send the XML text of the pipeline report to AQUA at JAO
# We need to BASE64-encode it because it will be wrapped in a JSON field
timestamp = self._getTimestamp(productsBasedir)
plReportFile = self._findPipelineReport(productsDir, self._ousUID)
plReport = dbdrwutils.readTextFileIntoString(plReportFile)
plReport = dbdrwutils.b64encode(plReport)
message = '''
{
"ousUID" : "%s",
"timestamp" : "%s",
"source" : "%s",
"report" : "%s",
"productsDir": "%s"
}
''' % (self._ousUID, timestamp, self._replicatedCache, plReport,
productsBasedir)
message = json.loads(message) # convert to a Python dict
selector = "pipeline.report.JAO"
self._mq.send(message, selector)
# We are done, set the OUS state to ReadyForReview
dbdrwutils.setState(self._xtss, self._ousUID, "ReadyForReview")
def _isProductsDirectory(self, f, progID):
return (not os.path.isfile(os.path.join(self._pipelineRunDirectory, f))
and f.startswith(progID))
def _findProductsDir(self, progID):
"Get the most recent product directory"
allFiles = os.listdir(self._pipelineRunDirectory)
prodDirs = [
f for f in allFiles if self._isProductsDirectory(f, progID)
]
prodDir = sorted(prodDirs)[-1:]
prodDir = prodDir[0]
print(">>> PipelineDriver: prodDir:", prodDir)
return os.path.join(self._pipelineRunDirectory, prodDir)
def _findWeblog(self, productsDir, ousUID):
# DEMO ONLY: the "products" subdirectory should be looked for
# here we just take the hardcoded path
ousUID = dbdrwutils.encode(ousUID)
productsDir = os.path.join(productsDir, "SOUS", "GOUS", ousUID,
"products")
for file in os.listdir(productsDir):
print(">>> PipelineDriver: file:", file)
if (file.startswith("weblog-") and file.endswith(".zip")):
return (os.path.join(productsDir, file))
raise RuntimeError("No weblog found in %s" % productsDir)
def _findPipelineReport(self, productsDir, ousUID):
# DEMO ONLY: the "products" subdirectory should be looked for
# here we just take the hardcoded path
ousUID = dbdrwutils.encode(ousUID)
productsDir = os.path.join(productsDir, "SOUS", "GOUS", ousUID,
"products")
for file in os.listdir(productsDir):
print(">>> PipelineDriver: file:", file)
if (file.startswith("pl-report-") and file.endswith(".xml")):
return (os.path.join(productsDir, file))
raise RuntimeError("No pipeline report found in %s" % productsDir)
def _copyAndReplaceDir(self, from_path, to_path):
if os.path.exists(to_path):
shutil.rmtree(to_path)
shutil.copytree(from_path, to_path)
def _getTimestamp(self, productsDirName):
'''
If productsDirName is something like '2015.1.00657.S_2018_07_19T08_50_10.228'
will return 2018-07-19T08:50:10.228
'''
n = productsDirName.index('_')
timestamp = productsDirName[n + 1:]
timestamp = timestamp.replace('_', '-', 2)
timestamp = timestamp.replace('_', ':')
return timestamp
reply = input("Enter [F]ail, [P]ass, [S]emipass, [C]ancel: ")
reply = reply[0:1].upper()
if ((reply == 'F') or (reply == 'P') or (reply == 'S')
or (reply == 'C')):
break
if reply == 'C':
continue
# Set the OUS state according to the QA2 flag
processQA2flag(ousUID, reply)
if reply == 'F':
continue
# Tell the Product Ingestor that it should ingest those Pipeline products
selector = "ingest.JAO"
message = '{"ousUID" : "%s", "timestamp" : "%s", "productsDir" : "%s"}' % \
(ousUID, timestamp, productsDir)
message = {}
message["ousUID"] = ousUID
message["timestamp"] = timestamp
message["productsDir"] = productsDir
mq.send(message, selector)
# Wait some, mainly for effect
waitTime = random.randint(3, 8)
time.sleep(waitTime)
# Now we can set the state of the OUS to DeliveryInProgress
dbdrwutils.setState(xtss, ousUID, "DeliveryInProgress")
class QA2():
def __init__(self):
self.weblogsBaseUrl = "http://localhost:8000"
self.baseUrl = "http://localhost:5984" # CouchDB
self.dbconn = DbConnection(self.baseUrl)
self.dbName = "pipeline-reports"
self.xtss = ExecutorClient('localhost', 'msgq', 'xtss')
self.select = "pipeline.report.JAO"
self.mq = MqConnection('localhost', 'msgq', select)
def start(self):
# Launch the listener in the background
print(' [*] Waiting for messages matching %s' % (self.select))
dbdrwutils.bgRun(self.mq.listen, (self.callback,))
# This is the program's text-based UI
# Loop forever:
# Show Pipeline runs awaiting review
# Ask for an OUS UID
# Lookup the most recent PL execution for that
# Print it out
# Ask for Fail, Pass, or SemiPass
# Set the OUS state accordingly
while True:
print()
print()
print('------------------------------------------')
print()
print("OUSs ready to be reviewed")
ouss = self.findReadyForReview()
if (ouss == None or len(ouss) == 0):
print("(none)")
else:
for ous in ouss:
print(ous['entityId'])
print()
ousUID = input('Please enter an OUS UID: ')
plReport = self.findMostRecentPlReport(ousUID)
if plReport == None:
print("No Pipeline executions for OUS", ousUID)
continue
# We are reviewing this OUS, set its state accordingly
dbdrwutils.setState(self.xtss, ousUID, "Reviewing")
timestamp = plReport['timestamp']
report = dbdrwutils.b64decode(plReport['encodedReport'])
productsDir = plReport['productsDir']
source = plReport['source']
print("Pipeline report for UID %s, processed %s" % (ousUID,timestamp))
print(report)
print()
print("Weblog available at: %s/weblogs/%s" % (self.weblogsBaseUrl, dbdrwutils.makeWeblogName(ousUID, timestamp)))
print()
while True:
reply = input("Enter [F]ail, [P]ass, [S]emipass, [C]ancel: ")
reply = reply[0:1].upper()
if ((reply=='F') or (reply=='P') or (reply=='S') or (reply=='C')):
break
if reply == 'C':
continue
# Set the OUS state according to the QA2 flag
self.processQA2flag(ousUID, reply)
if reply == 'F':
continue
# Tell the Product Ingestor that it should ingest those Pipeline products
selector = "ingest.JAO"
message = '{"ousUID" : "%s", "timestamp" : "%s", "productsDir" : "%s"}' % \
(ousUID, timestamp, productsDir)
message = {}
message["ousUID"] = ousUID
message["timestamp"] = timestamp
message["productsDir"] = productsDir
self.mq.send(message, selector)
# Wait some, mainly for effect
waitTime = random.randint(3,8)
time.sleep(waitTime)
# Now we can set the state of the OUS to DeliveryInProgress
dbdrwutils.setState(self.xtss, ousUID, "DeliveryInProgress")
def savePlReport(self, ousUID, timestamp, encodedReport, productsDir, source):
'''
Saves a pipeline run report to 'Oracle'
'''
plReport = {}
plReport['ousUID'] = ousUID
plReport['timestamp'] = timestamp
plReport['encodedReport'] = encodedReport
plReport['productsDir'] = productsDir
plReport['source'] = source
plReportID = timestamp + "." + ousUID
retcode,msg = self.dbconn.save(self.dbName, plReportID, plReport)
if retcode != 201:
raise RuntimeError("Error saving Pipeline report: %d, %s" % (retcode,msg))
def findMostRecentPlReport(self, ousUID):
selector = { "selector": { "ousUID": ousUID }}
retcode,reports = self.dbconn.find(self.dbName, selector)
if len(reports) == 0:
return None
if retcode != 200:
print(reports)
return None
# Find the most recent report and return it
reports.sort(key=lambda x: x['timestamp'], reverse=True)
return reports[0]
def findReadyForReview(self):
selector = {
"selector": {
"state": "ReadyForReview"
}
}
retcode,ouss = self.dbconn.find("status-entities", selector)
if len(ouss) == 0:
return None
if retcode != 200:
print(ouss)
return None
ouss.sort(key=lambda x: x['entityId'])
return ouss
def processQA2flag(self, ousUID, flag):
"Flag should be one of 'F' (fail), 'P' (pass) or 'S' (semi-pass)"
newState = "ReadyForProcessing" if (flag == "F") else "Verified"
print(">>> Setting the state of", ousUID, "to", newState)
# Set the OUS state according to the input flag
dbdrwutils.setState(self.xtss, ousUID, newState)
if flag == "F":
dbdrwutils.setSubstate(self.xtss, ousUID, "") # Clear Pipeline recipe
def callback(self, message):
"""
Message is a JSON object:
ousUID is the UID of the OUS
source is the executive where the Pipeline was running
report is the report's XML text, BASE64-encoded
timestamp is the Pipeline run's timestamp
productsDir is the name of the products directory for that Pipeline run
For instance
{
"ousUID" : "uid://X1/X1/Xaf",
"source" : "EU",
"report" : "Cjw/eG1sIHZlcnNpb2..."
"timestamp" : "2018-07-19T08:50:10.228",
"productsDir": "2015.1.00657.S_2018_07_19T08_50_10.228"
}
"""
# print(">>> message:", message)
ousUID = message["ousUID"]
source = message["source"]
encodedReport = message["report"]
timestamp = message["timestamp"]
productsDir = message["productsDir"]
# report = dbdrwutils.b64decode(encodedReport)
# print(">>> report:", report)
# Save the report to Oracle
self.savePlReport(ousUID, timestamp, encodedReport, productsDir, source)
print(">>> AQUA/QA2: saved PL report: ousUID=%s, timestamp=%s" % (ousUID,timestamp))
#!/usr/bin/env python3
import sys
import argparse
sys.path.insert(0, "../shared")
from dbmsgq import MqConnection
parser = argparse.ArgumentParser(description='')
parser.add_argument(dest="groupName")
args = parser.parse_args()
queue = MqConnection("localhost", "msgq")
queue.joinGroup(args.groupName, listener="listener.NA")
queue.joinGroup(args.groupName, listener="listener.EA")
queue.joinGroup(args.groupName, listener="listener.EU")
msg = {}
msg['text'] = "all is fine"
queue.send(msg, "selector.*")
queue.send(msg, "selector.NA")