def run(self):
"""Pick a dictionary class and load the terms."""
started = datetime.now()
if self.opts.get("tier") == "PROD" and not isProdHost():
if not self.opts.get("auth"):
pw = getpw("esadmin")
if pw:
self.opts["auth"] = f"admin,{pw}"
if self.dictionary == "glossary":
loader = GlossaryLoader(**self.opts)
elif self.dictionary == "drugs":
loader = DrugLoader(**self.opts)
else:
raise Exception("no dictionary specified")
tier = loader.tier
host = loader.host
subject = f"[{tier}] Load of {self.dictionary} dictionary to {host}"
try:
loader.run()
elapsed = datetime.now() - started
message = f"Load completed in {elapsed}."
except Exception as e:
self.logger.exception("failure")
subject += " (FAILURE)"
message = f"Job failed: {e}\nSee logs for further details."
self.__notify(subject, message)
def send(self, recips, subject, message):
"""
Send an email message, trying multiple times before giving up.
We pause between attempts in order to avoid flooding the NIH
mail server with too many requests at the same time.
Note that the `recips` argument will always be a sequence
of addresses when we are sending internal messages reporting
processing to the operators, and will always be a single
string when sending the notification to an individual data
partner contact. This is why the checks which prevent sending
out email messages to outside data partners from non-production
tiers works correctly.
Pass:
recips - string (for partner contact) or list of strings (for ops)
subject - string for message's subject header
message - string for message body
Raise:
exception propagated by sendmail if too many attempts fail
"""
if isinstance(recips, str):
if not cdr.isProdHost() or self.test:
extra = "In live prod mode, would have gone to %r" % recips
if Notify.test_recips is None:
if self.recip:
Notify.test_recips = [self.recip]
else:
group = "Test Publishing Notification"
test_recips = self.get_group_email_addresses(group)
Notify.test_recips = test_recips
recips = Notify.test_recips
self.logger.info("using recips: %r", recips)
message = "<h3>%s</h3>\n%s" % (html_escape(extra), message)
else:
recips = [recips]
tries, delay = 0, self.DELAY
while True:
self.logger.debug("top of send(%r)", recips)
try:
opts = dict(subject=subject, body=message, subtype="html")
msg = cdr.EmailMessage(self.SENDER, recips, **opts)
msg.send()
msg = str(msg)
if self.log_messages:
self.logger.debug(msg)
return msg
except Exception as e:
self.logger.exception("failure sending to %r", recips)
tries += 1
if tries >= self.MAX_TRIES:
self.logger.error("bailing after %d tries", self.MAX_TRIES)
raise
self.logger.debug("pausing %s seconds", delay)
time.sleep(delay)
delay += self.DELAY
def main():
"""
Top-level entry point
"""
# Process the command-line arguments.
parser = create_parser()
opts = parser.parse_args()
# Make sure we're not doing this on the production server.
if not opts.tier and cdr.isProdHost() or opts.tier == "PROD":
parser.error("""
This program can only be used to install a filter on the development or
test server, not production.
Use CreateFilter.py to create the filter in the production database, then
use InstallFilter.py to install it in test or development with the same
title/name and (almost certainly) a different local CDR ID.
""")
# If we don't already have a session ID, make one.
if not opts.session:
password = getpass.getpass()
session = cdr.login(opts.user, password, tier=opts.tier)
error = cdr.checkErr(session)
if error:
parser.error(error)
else:
session = opts.session
# Load the document.
info = DocInfo(opts.filename, parser)
# Make sure the filter isn't already installed
info.check_unique_title(opts.tier, parser)
# All checks passed: add the document.
ctrl = dict(DocTitle=info.title.encode("utf-8"))
doc = cdr.Doc(info.xml, doctype="Filter", ctrl=ctrl, encoding="utf-8")
comment = "New filter install"
add_opts = dict(doc=str(doc), comment=comment, tier=opts.tier)
cdr_id = cdr.addDoc(session, **add_opts)
error = cdr.checkErr(cdr_id)
if error:
parser.error(error)
# Unlock the document and display its ID.
response = cdr.unlock(session, cdr_id, tier=opts.tier)
error = cdr.checkErr(response)
if error:
parser.error(error)
else:
print(cdr_id)
def alert(self, job):
"""
Send an email alert to the user to whom the translation job
is currently assigned.
"""
recip = self.UserInfo(self, job.assigned_to)
if not recip.email:
error = f"no email address found for user {recip.name}"
self.logger.error(error)
self.bail(error)
recips = [recip.email]
sender = "*****@*****.**"
body = []
subject = f"[{self.session.tier}] Translation Queue Notification"
log_message = f"mailed translation job state alert to {recip}"
if not isProdHost():
recips = getEmailList("Test Translation Queue Recips")
body.append(
f"[*** THIS IS A TEST MESSAGE ON THE {self.session.tier} TIER."
f" ON PRODUCTION IT WOULD HAVE GONE TO {recip}. ***]\n"
)
log_message = f"test alert for {recip} sent to {recips}"
if self.job.new:
body.append("A new translation job has been assigned to you.")
else:
body.append("A translation job assigned to you has a new status.")
body.append(f"Assigned by: {self.user}")
body.append(f"English summary document ID: CDR{job.english_id:d}")
body.append(f"English summary title: {job.english_title}")
if job.spanish_title:
body.append(f"Spanish summary document ID: CDR{job.spanish_id:d}")
body.append(f"Spanish summary title: {job.spanish_title}")
body.append(f"Summary audience: {job.english_audience}")
body.append(f"Job status: {self.states.map.get(job.state_id)}")
state_date = str(job.state_date)[:10]
body.append(f"Date of status transition: {state_date}")
body.append(f"Comments: {job.comments}")
opts = dict(subject=subject, body="\n".join(body))
if self.files:
opts["attachments"] = self.files
try:
message = EmailMessage(sender, recips, **opts)
message.send()
except Exception as e:
self.logger.error("sending mail: %s", e)
self.bail(f"sending mail: {e}")
self.logger.info(log_message)
class Control(Controller):
SUBTITLE = "Developers/System Administrators"
SUBMIT = None
ON_PROD = isProdHost()
def populate_form(self, page):
page.body.set("class", "admin-menu")
ol = page.B.OL()
for display, script in (
("Batch Job Status", "getBatchStatus.py"),
("Clear FileSweeper Lock File", "clear-filesweeper-lockfile.py"),
("Delete CDR Documents", "del-some-docs.py"),
("Email Logged-in Users", "MessageLoggedInUsers.py"),
("Fetch Tier Settings", "fetch-tier-settings.py"),
("Global Change Test Results", "ShowGlobalChangeTestResults.py"),
("Mailers", "Mailers.py"),
("Manage Actions", "EditActions.py"),
("Manage Configuration Files", "EditConfig.py"),
("Manage Control Values", "EditControlValues.py"),
("Manage Document Types", "EditDocTypes.py"),
("Manage DTDs", "PostDTD.py"),
("Manage Filter Sets", "EditFilterSets.py"),
("Manage Filters", "EditFilters.py"),
("Manage Glossary Servers", "glossary-servers.py"),
("Manage Groups", "EditGroups.py"),
("Manage Linking Tables", "EditLinkControl.py"),
("Manage Query Term Definitions", "EditQueryTermDefs.py"),
("Manage Scheduled Jobs", "Scheduler.py"),
("Manage Users", "EditUsers.py"),
("Manage Value Tables", "edit-value-table.py"),
("Publishing - Create Job", "Publishing.py"),
("Publishing - Fail Job", "FailBatchJob.py"),
("Re-Publishing", "Republish.py"),
("Replace CWD with Older Version", "ReplaceCWDwithVersion.py"),
("Reports", "Reports.py"),
("Restore Deleted Documents", "RestoreDeletedDocs.py"),
("Schema - Post", "post-schema.py"),
("Schema - Show", "GetSchema.py"),
("Unblock Documents", "UnblockDoc.py"),
("Unlock Media", "UnlockMedia.py"),
("Update Mapping Table", "EditExternalMap.py"),
("View Logs", "log-tail.py"),
):
ol.append(page.B.LI(page.menu_link(script, display)))
page.form.append(ol)
def run(self):
"""Run the report (top-level entry point)."""
if cdr.isProdHost():
cdrcgi.bail("Can't compare the production server to itself")
lines = []
for name in sorted(self.filter_names, key=str.lower):
if name not in self.local_filters:
lines += self.only_on(name, "PROD")
elif name not in self.prod_filters:
lines += self.only_on(name, self.tier.name)
else:
lines += self.compare(name)
if not lines:
lines = [f"Filters on {self.tier.name} and PROD are identical"]
lines = "\n".join(lines) + "\n"
stdout.buffer.write(b"Content-type: text/plain; charset=utf-8\n\n")
stdout.buffer.write(lines.encode("utf-8"))
# Setting directory and file names
# --------------------------------
PUBPATH = os.path.join('d:\\cdr', 'publishing')
# PUBPATH = os.path.join('d:\\home', 'venglisch', 'cdr', 'publishing')
TIER = cdr.Tier().name
MAX_RETRIES = 10
RETRY_MULTIPLIER = 5.0
wait = 60 # number of seconds to wait between status checks
# The performance of the publishing job has greatly improved allowing
# us to cancel a running job much sooner if it fails to finish.
# Optionally overriden below once we know the publishing subset.
# --------------------------------------------------------------------
if cdr.isProdHost():
waitTotal = 10800 # 3.0 hours
elif cdr.isDevHost():
waitTotal = 10800 # 3.0 hours
else:
waitTotal = 14400 # 4.0 hours
testMode = None
fullMode = None
session = cdr.login("cdroperator", cdr.getpw("cdroperator"))
pubSystem = 'Primary'
pubEmail = cdr.getEmailList('Operator Publishing Notification')