def processQA2flag(self, ousUID, flag):
"Flag should be one of 'F' (fail), 'P' (pass) or 'S' (semi-pass)"
newState = "ReadyForProcessing" if (flag == "F") else "Verified"
print(">>> Setting the state of", ousUID, "to", newState)
# Set the OUS state according to the input flag
dbdrwutils.setState(self.xtss, ousUID, newState)
if flag == "F":
dbdrwutils.setSubstate(self.xtss, ousUID, "") # Clear Pipeline recipe
def start(self):
# This is the program's text-based UI
# Loop forever:
# Show Pipeline runs ready for processing
# Ask for an OUS UID
# Set:
# state=Processing
# PL_PROCESSING_EXECUTIVE=$DRAWS_LOCATION
while True:
print()
print()
print('------------------------------------------')
print()
print("ReadyForProcessing OUSs")
ouss = self.findReadyForPipelineProcessing()
if (ouss == None or len(ouss) == 0):
print("(none)")
sys.exit()
else:
ousMap = {}
for ous in ouss:
entityId = ous['entityId']
print(entityId)
ousMap[entityId] = ous
print()
ousUID = input(
'Please enter an OUS UID, will be processed at %s: ' %
self.location)
if not (ousUID in ousMap):
print("No OUS with UID='%s'" % (ousUID))
continue
ous = ousMap[ousUID]
# We are going to process this OUS, set its state and processing executive accordingly
dbdrwutils.setState(self._xtss, ousUID, "Processing")
dbdrwutils.setExecutive(self._xtss, ousUID, self.location)
# Launch the Pipeline Driver on Torque/Maui (pretending it listens to messages)
message = {}
message['ousUID'] = ousUID
message['recipe'] = ous["substate"]
message['progID'] = ous["progID"]
# msgTemplate = '{ "ousUID" : "%s", "recipe" : "%s", "progID" : "%s" }'
# message = msgTemplate % (ousUID, ous["substate"], ous["progID"])
torque = "pipeline.process." + self.location
dbdrwutils.sendMsgToSelector(message, torque, self._mq)
# Wait some, mainly for effect
waitTime = random.randint(3, 8)
time.sleep(waitTime)
def _pipeline(self, ts):
res = True
try:
ts.start(['comp','exec'])
completed = subprocess.run( [self.pipelineExecutable, self._progID, self._ousUID, self._recipe], cwd=self.pipelineRunDirectory)
ret = completed.returncode
if ret != 0:
ts.fail(['comp','exec'])
res = False
print( ">>> PipelineDriver: Pipeline returned:", ret )
if ret != 2:
raise RuntimeError( ">>> PipelineDriver: Pipeline returned: %d" % ret )
# Pipeline returned 2 -- processing failed
# Push the OUS to ProcessingProblem, and we're done
dbdrwutils.setState( self.xtss, self._ousUID, "ProcessingProblem" )
except Exception as e:
res = False
ts.fail(['comp','exec'])
ex = TaskException.handle(e)
ts.setEx(ex)
ts.stop(['comp','exec'])
return res
def wrapPipeline(progID, ousUID, recipe):
# Run the pipeline on our OUS
ret = runPipeline(progID, ousUID, recipe, pipelineRunDirectory)
if ret != 0:
print(">>> PipelineDriver: Pipeline returned:", ret)
if ret != 2:
raise RuntimeError(">>> PipelineDriver: Pipeline returned: %d" %
ret)
# Pipeline returned 2 -- processing failed
# Push the OUS to ProcessingProblem, and we're done
dbdrwutils.setState(xtss, ousUID, "ProcessingProblem")
return
### Processing was successful!
# Copy the products directory to the replicating cache directory
# and signal that to the JAO cache
productsDir = findProductsDir(progID)
productsBasedir = os.path.basename(productsDir)
repCacheDir = os.path.join(replicatedCache, productsBasedir)
print(">>> PipelineDriver: Products dir name:", productsDir)
print(">>> PipelineDriver: Replicating dir name:", repCacheDir)
copyAndReplaceDir(productsDir, repCacheDir)
# message = '{"fileType":"productsdir", "cachedAt":"%s", "name": "%s"}' % (location,productsBasedir)
message = {}
message["fileType"] = "productsdir"
message["cachedAt"] = location
message["name"] = productsBasedir
selector = "cached.JAO"
mq.send(message, selector)
# Copy the weblog to the replicating cache directory
# and signal that to the JAO *and* the local cache (if
# they are not one and the same)
weblog = findWeblog(productsDir, ousUID)
print(">>> PipelineDriver: weblog: copying", weblog, "to", replicatedCache)
shutil.copy(weblog, replicatedCache)
# message = '{"fileType":"weblog", "cachedAt":"%s", "name": "%s"}' % (location, os.path.basename( weblog ))
message = {}
message["fileType"] = "weblog"
message["cachedAt"] = location
message["name"] = os.path.basename(weblog)
selector = "cached.JAO"
mq.send(message, selector)
if replicatedCache != "JAO":
selector = "cached.%s" % location
mq.send(message, selector)
# Send the XML text of the pipeline report to AQUA at JAO
# We need to BASE64-encode it because it will be wrapped in a JSON field
timestamp = getTimestamp(productsBasedir)
plReportFile = findPipelineReport(productsDir, ousUID)
plReport = dbdrwutils.readTextFileIntoString(plReportFile)
plReport = dbdrwutils.b64encode(plReport)
message = '''
{
"ousUID" : "%s",
"timestamp" : "%s",
"source" : "%s",
"report" : "%s",
"productsDir": "%s"
}
''' % (ousUID, timestamp, replicatedCache, plReport, productsBasedir)
message = json.loads(message) # convert to a Python dict
selector = "pipeline.report.JAO"
mq.send(message, selector)
# We are done, set the OUS state to ReadyForReview
dbdrwutils.setState(xtss, ousUID, "ReadyForReview")
ousMap = {}
for ous in ouss:
entityId = ous['entityId']
print(entityId)
ousMap[entityId] = ous
print()
ousUID = input('Please enter an OUS UID, will be processed at %s: ' %
location)
if not (ousUID in ousMap):
print("No OUS with UID='%s'" % (ousUID))
continue
ous = ousMap[ousUID]
# We are going to process this OUS, set its state and processing executive accordingly
dbdrwutils.setState(xtss, ousUID, "Processing")
dbdrwutils.setExecutive(xtss, ousUID, location)
# Launch the Pipeline Driver on Torque/Maui (pretending it listens to messages)
message = {}
message['ousUID'] = ousUID
message['recipe'] = ous["substate"]
message['progID'] = ous["progID"]
# msgTemplate = '{ "ousUID" : "%s", "recipe" : "%s", "progID" : "%s" }'
# message = msgTemplate % (ousUID, ous["substate"], ous["progID"])
torque = "pipeline.process." + location
dbdrwutils.sendMsgToSelector(message, torque, mq)
# Wait some, mainly for effect
waitTime = random.randint(3, 8)
time.sleep(waitTime)
# ts = ous['timestamp']
# print( ">>> found", ousUID, ts )
# Retrieve the list of products from the delivery status
encodedUID = dbdrwutils.encode(ousUID)
dbName = 'delivery-status'
retcode, delStatus = dbcon.findOne(dbName, encodedUID)
if retcode != 200:
raise RuntimeError("find: %s: error %d: %s" %
(dbName, retcode, delStatus))
# See if all those data products were replicated here
dataProducts = delStatus['dataProducts']
allReplicated = True
for dataProduct in dataProducts:
# print( ">>> found", dataProduct )
if (not ngas.check(dataProduct)):
allReplicated = False
break
# YES, all those data products were replicated -- this OUS can
# be delivered
if allReplicated:
dbdrwutils.setState(xtss, ousUID, "Delivered")
dbdrwutils.setSubstate(xtss, ousUID, "")
dbdrwutils.clearExecutive(xtss, ousUID)
time.sleep(5) # Pretend this actually took some time
print(">>> OUS", ousUID, "is now Delivered")
dbdrwutils.incrementalSleep(startTime)
def execute(self, params):
self._params = params
#Check parameters!
self._check(params, [
'progID', 'ousUID', 'location', 'pipelineRunDir', 'replicatedCache'
])
self._progID = params['progID']
self._ousUID = params['ousUID']
self._pipelineRunDirectory = params['pipelineRunDir']
self._replicatedCache = params['replicatedCache']
self._location = params['location']
# Copy the products directory to the replicating cache directory
# and signal that to the JAO cache
productsDir = self._findProductsDir(self._progID)
productsBasedir = os.path.basename(productsDir)
repCacheDir = os.path.join(self._replicatedCache, productsBasedir)
print(">>> PipelineDriver: Products dir name:", productsDir)
print(">>> PipelineDriver: Replicating dir name:", repCacheDir)
self._copyAndReplaceDir(productsDir, repCacheDir)
# message = '{"fileType":"productsdir", "cachedAt":"%s", "name": "%s"}' % (self._location,productsBasedir)
message = {}
message["fileType"] = "productsdir"
message["cachedAt"] = self._location
message["name"] = productsBasedir
selector = "cached.JAO"
self._mq.send(message, selector)
# Copy the weblog to the replicating cache directory
# and signal that to the JAO *and* the local cache (if
# they are not one and the same)
weblog = self._findWeblog(productsDir, self._ousUID)
print(">>> PipelineDriver: weblog: copying", weblog, "to",
self._replicatedCache)
shutil.copy(weblog, self._replicatedCache)
# message = '{"fileType":"weblog", "cachedAt":"%s", "name": "%s"}' % (self._location, os.path.basename(weblog))
message = {}
message["fileType"] = "weblog"
message["cachedAt"] = self._location
message["name"] = os.path.basename(weblog)
selector = "cached.JAO"
self._mq.send(message, selector)
if self._replicatedCache != "JAO":
selector = "cached.%s" % self._location
self._mq.send(message, selector)
# Send the XML text of the pipeline report to AQUA at JAO
# We need to BASE64-encode it because it will be wrapped in a JSON field
timestamp = self._getTimestamp(productsBasedir)
plReportFile = self._findPipelineReport(productsDir, self._ousUID)
plReport = dbdrwutils.readTextFileIntoString(plReportFile)
plReport = dbdrwutils.b64encode(plReport)
message = '''
{
"ousUID" : "%s",
"timestamp" : "%s",
"source" : "%s",
"report" : "%s",
"productsDir": "%s"
}
''' % (self._ousUID, timestamp, self._replicatedCache, plReport,
productsBasedir)
message = json.loads(message) # convert to a Python dict
selector = "pipeline.report.JAO"
self._mq.send(message, selector)
# We are done, set the OUS state to ReadyForReview
dbdrwutils.setState(self._xtss, self._ousUID, "ReadyForReview")
def start(self):
# Launch the listener in the background
print(' [*] Waiting for messages matching %s' % (self.select))
dbdrwutils.bgRun(self.mq.listen, (self.callback,))
# This is the program's text-based UI
# Loop forever:
# Show Pipeline runs awaiting review
# Ask for an OUS UID
# Lookup the most recent PL execution for that
# Print it out
# Ask for Fail, Pass, or SemiPass
# Set the OUS state accordingly
while True:
print()
print()
print('------------------------------------------')
print()
print("OUSs ready to be reviewed")
ouss = self.findReadyForReview()
if (ouss == None or len(ouss) == 0):
print("(none)")
else:
for ous in ouss:
print(ous['entityId'])
print()
ousUID = input('Please enter an OUS UID: ')
plReport = self.findMostRecentPlReport(ousUID)
if plReport == None:
print("No Pipeline executions for OUS", ousUID)
continue
# We are reviewing this OUS, set its state accordingly
dbdrwutils.setState(self.xtss, ousUID, "Reviewing")
timestamp = plReport['timestamp']
report = dbdrwutils.b64decode(plReport['encodedReport'])
productsDir = plReport['productsDir']
source = plReport['source']
print("Pipeline report for UID %s, processed %s" % (ousUID,timestamp))
print(report)
print()
print("Weblog available at: %s/weblogs/%s" % (self.weblogsBaseUrl, dbdrwutils.makeWeblogName(ousUID, timestamp)))
print()
while True:
reply = input("Enter [F]ail, [P]ass, [S]emipass, [C]ancel: ")
reply = reply[0:1].upper()
if ((reply=='F') or (reply=='P') or (reply=='S') or (reply=='C')):
break
if reply == 'C':
continue
# Set the OUS state according to the QA2 flag
self.processQA2flag(ousUID, reply)
if reply == 'F':
continue
# Tell the Product Ingestor that it should ingest those Pipeline products
selector = "ingest.JAO"
message = '{"ousUID" : "%s", "timestamp" : "%s", "productsDir" : "%s"}' % \
(ousUID, timestamp, productsDir)
message = {}
message["ousUID"] = ousUID
message["timestamp"] = timestamp
message["productsDir"] = productsDir
self.mq.send(message, selector)
# Wait some, mainly for effect
waitTime = random.randint(3,8)
time.sleep(waitTime)
# Now we can set the state of the OUS to DeliveryInProgress
dbdrwutils.setState(self.xtss, ousUID, "DeliveryInProgress")
ouss = findReadyForReview()
if (ouss == None or len(ouss) == 0):
print("(none)")
else:
for ous in ouss:
print(ous['entityId'])
print()
ousUID = input('Please enter an OUS UID: ')
plReport = findMostRecentPlReport(ousUID)
if plReport == None:
print("No Pipeline executions for OUS", ousUID)
continue
# We are reviewing this OUS, set its state accordingly
dbdrwutils.setState(xtss, ousUID, "Reviewing")
timestamp = plReport['timestamp']
report = dbdrwutils.b64decode(plReport['encodedReport'])
productsDir = plReport['productsDir']
source = plReport['source']
print("Pipeline report for UID %s, processed %s" % (ousUID, timestamp))
print(report)
print()
print("Weblog available at: %s/weblogs/%s" %
(weblogsBaseUrl, dbdrwutils.makeWeblogName(ousUID, timestamp)))
print()
while True:
reply = input("Enter [F]ail, [P]ass, [S]emipass, [C]ancel: ")
reply = reply[0:1].upper()