def tearDown(self):
os.remove(sc.workflow_history_path())
if sc.workflow_history_path() != sc.all_errors_path():
os.remove(sc.all_errors_path())
for wkf in ge.check_session(None).return_workflows():
file_name = WorkflowInfo(wkf).cache_filename('workflow_params')
os.remove(file_name)
ge.check_session(None).teardown()
def tearDown(self):
os.remove(sc.workflow_history_path())
if sc.workflow_history_path() != sc.all_errors_path():
os.remove(sc.all_errors_path())
for wkf in ge.check_session(None).return_workflows():
file_name = WorkflowInfo(wkf).cache_filename('workflow_params')
os.remove(file_name)
ge.check_session(None).teardown()
def test_steptable(self):
# This isn't particularly well written,
# but we should expect a table with
# rows corresponding to error codes 1, 2, 3 and
# columns corresponding to site names site_a, site_b, site_c
allmap = ge.check_session(None).get_allmap()
for step in self.errors:
error_table = ge.get_step_table(step)
# One by hand because weird stuff is happening while writing this test
if step == '/test1/a/1':
self.assertEqual(error_table[0][0], 100)
self.assertTrue(error_table)
for row in error_table:
self.assertTrue(row)
# Check number of rows and errors
self.assertEqual(len(allmap['errorcode']), len(error_table))
for e_index, error in enumerate(allmap['errorcode']):
# Make sure all of the sites have a column
self.assertEqual(len(allmap['sitename']), len(error_table[e_index]))
for s_index, site in enumerate(allmap['sitename']):
self.assertEqual(error_table[e_index][s_index],
self.errors[step].get(str(error), {}).get(site, 0))
def test_steptable(self):
# This isn't particularly well written,
# but we should expect a table with
# rows corresponding to error codes 1, 2, 3 and
# columns corresponding to site names site_a, site_b, site_c
allmap = ge.check_session(None).get_allmap()
for step in self.errors:
error_table = ge.get_step_table(step)
# One by hand because weird stuff is happening while writing this test
if step == '/test1/a/1':
self.assertEqual(error_table[0][0], 100)
self.assertTrue(error_table)
for row in error_table:
self.assertTrue(row)
# Check number of rows and errors
self.assertEqual(len(allmap['errorcode']), len(error_table))
for e_index, error in enumerate(allmap['errorcode']):
# Make sure all of the sites have a column
self.assertEqual(len(allmap['sitename']),
len(error_table[e_index]))
for s_index, site in enumerate(allmap['sitename']):
self.assertEqual(
error_table[e_index][s_index],
self.errors[step].get(str(error), {}).get(site, 0))
def setUp(self):
out_name = sc.all_errors_path()
if not os.path.exists(os.path.dirname(out_name)):
os.makedirs(os.path.dirname(out_name))
with open(sc.all_errors_path(), 'w') as output:
json.dump(self.errors, output)
if sc.workflow_history_path() != sc.all_errors_path():
uh.main(sc.all_errors_path())
ge.check_session(None).data_location = out_name
ge.check_session(None).setup()
for wkf in ge.check_session(None).return_workflows():
file_name = WorkflowInfo(wkf).cache_filename('workflow_params')
dirname = os.path.dirname(file_name)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(file_name, 'w') as cache:
json.dump({}, cache)
def setUp(self):
out_name = sc.all_errors_path()
if not os.path.exists(os.path.dirname(out_name)):
os.makedirs(os.path.dirname(out_name))
with open(sc.all_errors_path(), 'w') as output:
json.dump(self.errors, output)
if sc.workflow_history_path() != sc.all_errors_path():
uh.main(sc.all_errors_path())
ge.check_session(None).data_location = out_name
ge.check_session(None).setup()
for wkf in ge.check_session(None).return_workflows():
file_name = WorkflowInfo(wkf).cache_filename('workflow_params')
dirname = os.path.dirname(file_name)
if not os.path.exists(dirname):
os.makedirs(dirname)
with open(file_name, 'w') as cache:
json.dump({}, cache)
def explainerror(self, errorcode='0', workflowstep='/'):
"""Returns an explaination of the error code, along with a link returning to table
:param str errorcode: The error code to display.
:param str workflowstep: The workflow to return to from the error page.
:returns: a page dumping the error logs
:rtype: str
"""
workflow = workflowstep.split('/')[1]
if errorcode == '0' or not workflow:
return 'Need to specify error and workflow. Follow link from workflow tables.'
errs_explained = globalerrors.check_session(cherrypy.session).\
get_workflow(workflow).get_explanation(errorcode, workflowstep)
return render('explainerror.html',
error=errorcode,
explanation=errs_explained,
source=workflowstep)
def resetcache(self, prepid='', workflow=''):
"""
The function is only accessible to someone with a verified account.
Navigating to ``https://localhost:8080/resetcache``
resets the error info for the user's session.
It also clears out cached JSON files on the server.
Under normal operation, this cache is only refreshed every half hour.
:returns: a confirmation page
:rtype: str
"""
WorkflowTools.RESET_LOCK.acquire()
cherrypy.log('Cache reset by: %s' % cherrypy.request.login)
info = globalerrors.check_session(cherrypy.session)
# Force the cache reset
if info:
prepids = [prepid] if prepid else info.prepidinfos.keys()
workflows = [workflow] if workflow else \
[wf for pid in prepids for wf in info.prepidinfos[pid].get_workflows()]
for wkf in workflows:
info.get_workflow(wkf).get_errors()
info.get_workflow(wkf).reset()
if not workflow:
for pid in prepids:
info.prepidinfos[pid].reset()
info.teardown()
info.setup()
WorkflowTools.RESET_LOCK.release()
return render('complete.html')
def similarwfs(self, workflow):
"""
Gives back a list of workflows that are clustered with the queried workflow.
This is just an array of strings.
:param str workflow: The workflow to find the group.
:returns: List of similar workflows.
If the quey is not a valid workflow in the system, an empty list is returned
:rtype: JSON
"""
output = {'similar': [], 'acted': []}
self.seeworkflowlock.acquire()
try:
if workflow in \
globalerrors.check_session(cherrypy.session,
can_refresh=True).return_workflows():
clusterworkflows.CLUSTER_LOCK.acquire()
similar_wfs = clusterworkflows.\
get_clustered_group(workflow, self.clusterer, cherrypy.session)
clusterworkflows.CLUSTER_LOCK.release()
acted = [
wf for wf in manageactions.get_acted_workflows(
serverconfig.get_history_length()) if wf in similar_wfs
]
output = {'similar': sorted(list(similar_wfs)), 'acted': acted}
finally:
self.seeworkflowlock.release()
return output
def submitaction(self, workflows='', action='', **kwargs):
"""Submits the action to Unified and notifies the user that this happened
:param str workflows: is a list of workflows to apply the action to
:param str action: is the suggested action for Unified to take
:param kwargs: can include various reasons and additional datasets
:returns: a confirmation page
:rtype: str
"""
cherrypy.log('args: {0}'.format(kwargs))
if workflows == '':
return render('scolduser.html', workflow='')
if action == '':
return render('scolduser.html', workflow=workflows[0])
output = ''
self.seeworkflowlock.acquire()
try:
workflows, reasons, params = manageactions.\
submitaction(cherrypy.request.login, workflows, action, cherrypy.session,
**kwargs)
# Immediately get actions to check the sites list
check_actions = manageactions.get_actions()
blank_sites_subtask = []
sites_to_run = {}
# Loop through all workflows just submitted
for workflow in workflows:
# Check sites of recovered workflows
if check_actions[workflow]['Action'] in ['acdc', 'recovery']:
for subtask, params in check_actions[workflow][
'Parameters'].items():
# Empty sites are noted
if not params.get('sites'):
blank_sites_subtask.append('/%s/%s' %
(workflow, subtask))
sites_to_run['/%s/%s' % (workflow, subtask)] = \
globalerrors.check_session(cherrypy.session).\
get_workflow(workflow).site_to_run(subtask)
if blank_sites_subtask:
drain_statuses = {sitename: drain for sitename, _, drain in \
sitereadiness.i_site_readiness()}
output = render('picksites.html',
tasks=blank_sites_subtask,
statuses=drain_statuses,
sites_to_run=sites_to_run)
else:
output = render('actionsubmitted.html',
workflows=workflows,
action=action,
reasons=reasons,
params=params,
user=cherrypy.request.login)
finally:
self.seeworkflowlock.release()
return output
def seeworkflow(self, workflow='', issuggested=''):
"""
Located at ``https://localhost:8080/seeworkflow``,
this shows detailed tables of errors for each step in a workflow.
For the exit codes in each row, there is a link to view some of the output
of the error message for jobs having the given exit code.
This should help operators understand what the error means.
At the top of the page, there are links back for :ref:`global-view-ref`,
workflow logs, related JIRA tickets,
and ReqMgr2 information about the workflow and prep ID.
The main function of this page is to submit actions.
Note that you will need to register in order to actually submit actions.
See :ref:`new-user-ref` for more details.
Depending on which action is selected, a menu will appear
for the operator to adjust parameters for the workflows.
Under the selection of the action and parameters, there is a button
to show other workflows that are similar to the selected workflow,
according to the :ref:`clustering-ref`.
Each entry is a link to open a similar workflow view page in a new tab.
The option to submit actions will not be on this page though
(so that you can focus on the first workflow).
If you think that a workflow in the cluster should have the same actions
applied to it as the parent workflow,
then check the box next to the workflow name before submitting the action.
Finally, before submitting, you can submit reasons for your action selection.
Clicking the Add Reason button will give you an additional reason field.
Reasons submitted are stored based on the short reason you give.
You can then select past reasons from the drop down menu to save time in the future.
If you do not want to store your reason, do not fill in the Short Reason field.
The long reason will be used for logging
and communicating with the workflow requester (eventually).
:param str workflow: is the name of the workflow to look at
:param str issuggested: is a string to tell if the page
has been linked from another workflow page
:returns: the error tables page for a given workflow
:rtype: str
:raises: 404 if a workflow doesn't seem to be in assistance anymore
Resets personal cache in the meanwhile, just in case
"""
self.seeworkflowlock.acquire()
output = ''
try:
if workflow not in \
globalerrors.check_session(
cherrypy.session, can_refresh=True).return_workflows():
WorkflowTools.RESET_LOCK.acquire()
info = globalerrors.check_session(cherrypy.session)
if info:
info.teardown()
info.setup()
WorkflowTools.RESET_LOCK.release()
raise cherrypy.HTTPError(404)
workflowdata = globalerrors.see_workflow(workflow,
cherrypy.session)
drain_statuses = {sitename: drain for sitename, _, drain in \
sitereadiness.i_site_readiness()}
output = render(
'workflowtables.html',
workflowdata=workflowdata,
workflow=workflow,
issuggested=issuggested,
workflowinfo=globalerrors.check_session(
cherrypy.session).get_workflow(workflow),
readiness=globalerrors.check_session(
cherrypy.session).readiness,
drain_statuses=drain_statuses,
last_submitted=manageactions.get_datetime_submitted(workflow))
finally:
self.seeworkflowlock.release()
return output
def globalerror(self, pievar='errorcode'):
"""
This page, located at ``https://localhost:8080/globalerror``,
attempts to give an overall view of the errors that occurred
in each workflow at different sites.
The resulting view is a table of piecharts.
The rows and columns can be adjusted to contain two of the following:
- Workflow step name
- Site where error occurred
- Exit code of the error
The third variable is used to split the pie charts.
This variable inside the pie charts can be quickly changed
by submitting the form in the upper left corner of the page.
The size of the piecharts depend on the total number of errors in a given cell.
Each cell also has a tooltip, giving the total number of errors in the piechart.
The colors of the piecharts show the splitting based on the ``pievar``.
Clicking on the pie chart will show the splitting explicitly
using the :ref:`list-wfs-ref` page.
If the steps make up the rows,
the default view will show you the errors for each campaign.
Clicking on the campaign name will cause the rows to expand
to show the original workflow and all ACDCs (whether or not the ACDCs have errors).
Following the link of the workflow will bring you to :ref:`workflow-view-ref`.
Clicking anywhere else in the workflow box
will cause it to expand to show errors for each step.
:param str pievar: The variable that the pie charts are split into.
Valid values are:
- errorcode
- sitename
- stepname
:returns: the global views of errors
:rtype: str
"""
# For some reasons, we occasionally have to refresh this global errors page
errors = globalerrors.get_errors(pievar, cherrypy.session)
if pievar != 'stepname':
# This pulls out the timestamp from the workflow parameters
timestamp = lambda wkf: time.mktime(
datetime.datetime(
*(globalerrors.check_session(cherrypy.session).\
get_workflow(wkf).get_workflow_parameters()['RequestDate'])).timetuple()
)
errors = globalerrors.group_errors(
globalerrors.group_errors(errors, lambda subtask: subtask.split('/')[1],
timestamp=timestamp),
lambda workflow: globalerrors.check_session(cherrypy.session).\
get_workflow(workflow).get_prep_id()
)
# Get the names of the columns
cols = globalerrors.check_session(cherrypy.session).\
get_allmap()[globalerrors.get_row_col_names(pievar)[1]]
get_names = lambda x: [
globalerrors.TITLEMAP[name]
for name in globalerrors.get_row_col_names(x)
]
template = lambda: render(
'globalerror.html',
errors=errors,
decoder=json.dumps,
columns=cols,
pievar=pievar,
acted_workflows=manageactions.get_acted_workflows(
serverconfig.get_history_length()),
readiness=globalerrors.check_session(cherrypy.session).readiness,
get_names=get_names)
try:
return template()
# I don't remember what kind of exception this throws...
except Exception: # pylint: disable=broad-except
time.sleep(1)
return template()
