def __sync_logs(self):
"""
Top up our local copies of the pdq logs from the sftp server.
We're ignoring some expected errors, having to do with cygwin's
difficulty in dealing with bizarre Windows file permissions
configuration settings. If we really fail to bring down a needed
log file successfully, we'll find out when we try to read it.
"""
etc = self.tier.etc
rsa = f"{etc}/cdroperator_rsa"
ssh = f"ssh -i {rsa} -o LogLevel=error -o StrictHostKeyChecking=no"
usr = "******"
dns = "cancerinfo.nci.nih.gov"
src = "%s@%s:/sftp/sftphome/cdrstaging/logs/*" % (usr, dns)
cmd = "rsync -e \"%s\" %s ." % (ssh, src)
fix = r"%s:\cdr\bin\fix-permissions.cmd ." % cdr.WORK_DRIVE
cwd = os.getcwd()
os.chdir(self.Month.LOGDIR)
self.logger.info(cmd)
cdr.run_command(cmd)
if cdr.WORK_DRIVE:
self.logger.info(fix)
cdr.run_command(fix)
os.chdir(cwd)
def saveJobs(outputDir):
print("Saving scheduled jobs")
process = run_command(DUMP_JOBS)
with open(f"{outputDir}/scheduled-jobs.txt", "w", encoding="utf-8") as fp:
fp.write(process.stdout)
process = run_command(f"{DUMP_JOBS} --json")
with open(f"{outputDir}/scheduled-jobs.json", "w", encoding="utf-8") as fp:
fp.write(process.stdout)
def subtitle(self):
"""What we display under the banner.
Calculation of this string value actually installs the DTD
if we have one.
"""
if not hasattr(self, "_subtitle"):
self._subtitle = self.SUBTITLE
if self.file_bytes:
try:
path = self.path.replace("\\", "/")
with open(path, "wb") as fp:
fp.write(self.file_bytes)
command = rf"{FIX_PERMISSIONS} {self.path}"
result = run_command(command, merge_output=True)
if result.returncode:
args = command, result.stdout
self.logger.error("%s failed: %s", *args)
message = f"Failure fixing permissions for {path}"
self._subtitle = message
else:
self._subtitle = f"Successfully installed {path}"
except Exception as e:
self.logger.exception(path)
self._subtitle = f"Failed installing {path} (see logs)"
return self._subtitle
def push_files(self):
"""
Update the sFTP server with the content of the shadow directory.
Use `rsync` to get the the individual document files and catalogs
from the shadow directory to the sFTP server.
"""
args = self.SSH, self.USER, self.HOST, self.PATH
command = 'rsync --delete -rae "{}" full* {}@{}:{}'.format(*args)
self.logger.info("") # Blank line to format log output
self.logger.info("ssh host: %s", self.HOST)
self.logger.debug("ssh user: %s", self.USER)
self.logger.info("rsync command: %s", command)
os.chdir(self.PUB_SHADOW)
result = cdr.run_command(command)
self.logger.info("") # Blank line to format log output
self.logger.info("*** run_command output")
self.logger.info(result.stdout)
if result.stderr:
self.logger.info("*** Error:")
self.logger.info(result.stderr)
self.logger.info("finished syncing files with errors!!!")
else:
self.logger.info("finished syncing files on FTP server")
os.chdir(self.job_path)
def launch(self, script, include_pubmode=True, merge_output=False,
include_runmode=True):
"""
Execute the name Python script in a separate process and check
to make sure it succeeded. If it didn't log and report the failure
and throw an exception. All scripts take an argument indicating
that it's running live or as a test. Most also take a second
argument indicating whether this is a nightly publication job
or the larger weekly job. For the few that don't take this second
argument, pass include_pubmode=False. If the standard output and
standard error output should be merged, pass merge_output=True.
If a job ID is specified, some scripts take a third argument
to pass that ID; the form of the argument is found in the EXTRA_ARG
class property.
"""
path = "%s/%s" % (self.PUBPATH, script)
command = "python %s" % path
if include_runmode:
command += " --%s" % self.mode
if include_pubmode:
pubmode = (self.schedule == "weekly") and "export" or "interim"
command += " --%s" % pubmode
if self.job_id:
pattern = self.EXTRA_ARGS.get(script)
if pattern:
command += " " + pattern % self.job_id
self.logger.info(command)
process = cdr.run_command(command, merge_output=merge_output)
if self.failed(script, process):
self.logger.debug(process.stdout)
self.report_error(script)
def refresh_manifest(where):
"""
Top-level logic for this tool:
1. Remove the previous copy of the manifest.
2. Switch the current directory to the client files area.
3. Collect File objects for the files in this area.
4. Add a File object for the manifest file we're creating.
5. Calculate the checksums for the files.
6. Serialize the manifest to disk.
7. Adjust the permissions for the client area files/directories.
"""
try:
os.unlink(MANIFEST_PATH)
except:
pass
os.chdir(where)
files = gather_files(".")
files.append(File(os.path.join(".", cdr.MANIFEST_NAME)))
md5 = hashlib.md5()
filelist = create_filelist(sorted(files), md5)
root = etree.Element("Manifest")
root.append(create_ticket(md5))
root.append(filelist)
xml = etree.tostring(root, pretty_print=True, encoding="unicode")
write_manifest(xml)
if IS_WINDOWS:
command = f"{cdr.BASEDIR}/bin/fix-permissions.cmd {CLIENT_FILES_DIR}"
command = command.replace("/", os.path.sep)
print("fixing permissions...")
result = cdr.run_command(command, merge_output=True)
if result.returncode:
print(f"return code: {result.returncode}")
if result.stdout:
print(result.stdout)
def find(self):
"""Use the Windows find command to search for a pattern in the file."""
sys.stdout.buffer.write(self.HEADERS)
try:
cmd = f"find {self.pattern} {self.path}"
process = cdr.run_command(cmd, merge_output=True, binary=True)
sys.stdout.buffer.write(process.stdout)
except Exception as e:
sys.stdout.buffer.write(f"{cmd}\n{e}\n".encode("utf-8"))
def dir(self):
"""Display the output of the Windows dir command."""
sys.stdout.buffer.write(self.HEADERS)
try:
cmd = f"dir {self.path}"
process = cdr.run_command(cmd, merge_output=True, binary=True)
sys.stdout.buffer.write(process.stdout)
except Exception as e:
sys.stdout.buffer.write(str(e).encode("utf-8"))
def run(self):
self.logger.info("Running %s job with opts %s", self.name, self.opts)
if "command" in self.opts:
process = run_command(self.opts["command"])
self.logger.info("returncode: %d", process.returncode)
if process.stdout:
self.logger.info("stdout: %s", process.stdout)
if process.stderr:
self.logger.info("stderr: %s", process.stderr)
def get_iis_settings(self):
return {
"account": cdr.run_command("whoami").stdout.strip(),
"version": os.environ.get("SERVER_SOFTWARE"),
"web.config": {
"root": self.xmltojson(self.WEBCONFIG_ROOT),
"secure": self.xmltojson(self.WEBCONFIG_SECURE),
"glossifier": self.xmltojson(self.WEBCONFIG_GLOSSIFIER),
}
}
def fix_permissions(self):
"""Make it possible for the data partners to retrieve the files."""
args = self.SSH, self.USER, self.HOST, self.PATH
command = '{} {}@{} "chmod -R 755 {}/full*"'.format(*args)
self.logger.info("chmod command: %s", command)
result = cdr.run_command(command)
if result.stderr:
self.logger.info("*** Error:")
self.logger.info(result.stderr)
self.logger.info("finished fixing permissions with errors!!!")
else:
self.logger.info("finished fixing permissions on FTP server")
def send_mail(self, subject, message):
"""
Send email to the users who monitor publishing jobs.
If the email sending command fails, log the problem.
"""
path = "%s/PubEmail.py" % self.PUBPATH
subject = self.quote_arg(subject)
message = self.quote_arg(message)
command = "%s %s %s %s" % (cdr.PYTHON, path, subject, message)
process = cdr.run_command(command)
if process.stderr:
self.logger.error("sending email: %s", process.stderr)
def __check_dtds(self):
"""Regenerate the DTDs to reflect the new/updated schema.
Return:
string to append to the processing log display
"""
cmd = r"python d:\cdr\build\CheckDtds.py"
result = run_command(cmd, merge_output=True)
if result.returncode:
raise Exception(f"DTD check failure: {result.stdout}")
self.logger.info("DTDs updated")
return "Running CheckDtds.py ...\n" + result.stdout
def close(self):
"""Update the database row save any rejects to a new workbook."""
self.cursor.execute(self.UPDATE, self.id)
self.control.conn.commit()
if self.rejects:
path = self.new_workbook.save(Control.REVDIR).replace("/", "\\")
process = run_command(f"fix-permissions {path}")
if process.stderr:
self.control.bail(f"Unable to fix permissions for {path}",
extra=[process.stderr])
return self.new_name
return None
def __refresh_manifest(self):
"""Rebuild the client manifest file.
Return:
string to append to the processing log display
"""
cmd = r"python d:\cdr\build\RefreshManifest.py"
result = run_command(cmd, merge_output=True)
if result.returncode:
output = result.stdout
raise Exception(f"Manifest refresh failure: {output}")
self.logger.info("Manifest updated")
return "Running RefreshManifest.py ...\n" + result.stdout
def url(self):
"""Address of the new Excel workbook, if any."""
if not hasattr(self, "_url"):
self._url = None
if self.book:
directory = f"{self.session.tier.basedir}/reports"
self.book.save(directory)
path = f"{directory}/{self.book.filename}"
path = path.replace("/", "\\")
process = run_command(f"fix-permissions.cmd {path}")
if process.stderr:
self.bail(f"Failure settings permissions for {path}",
extra=[process.stderr])
self._url = f"/cdrReports/{self.book.filename}"
return self._url
def __cleanup(self, status, message):
self.log("~~In cleanup")
if self.__rtfMailerDir:
os.chdir(self.__rtfMailerDir)
command = f"{cdr.BASEDIR}/bin/fix-permissions.cmd"
command = command.replace("/", "\\")
process = cdr.run_command(command, merge_output=True)
if process.returncode:
args = self.__rtfMailerDir, process.stdout
_LOGGER.error("fixing %s permissions: %s", *args)
else:
self.log(f"fixed permissions for {self.__rtfMailerDir}")
try:
self.__updateStatus(status, message)
self.__sendMail()
if self.__session: cdr.logout(self.__session)
except:
self.log("__cleanup failed, status was '%s'" % status, tback=1)
def save(self):
"""Save the currently edited configuration file if changed."""
classes = "info center"
if self.content.strip() and self.content != self.original:
with open(self.filepath, "w", encoding="utf-8") as fp:
fp.write(self.content)
message = f"Saved new values for {self.filepath}"
command = f"{self.FIX_PERMISSIONS} {self.filepath}"
result = cdr.run_command(command, merge_output=True)
if result.returncode:
message = result.stdout or "Failure fixing permissions"
classes = "failure center"
else:
message = f"File {self.filepath} unchanged"
page = self.form_page
self.populate_form(page)
header = page.body.find("form/header")
if header is not None:
header.addnext(page.B.P(message, page.B.CLASS(classes)))
page.send()
def build_zip_file(file_names):
base_name = tempfile.mktemp()
zip_name = base_name + ".zip"
list_name = base_name + ".txt"
list_file = open(list_name, "w")
for name in file_names:
list_file.write("%s\n" % name)
list_file.close()
os.chdir(cdr.CLIENT_FILES_DIR)
Control.logger.debug("Creating %s", zip_name)
command = f"d:\\bin\\zip -@ {zip_name} < {list_name}"
process = cdr.run_command(command, merge_output=True)
if process.returncode:
msg = f"zip failure code {process.resultcode} ({result.stdout})"
Control.logger.debug(msg)
raise msg
with open(zip_name, "rb") as fp:
zip_bytes = fp.read()
Control.logger.debug("saved zip file as %r", zip_name)
if Control.logger.level != logging.DEBUG:
os.unlink(list_name)
os.unlink(zip_name)
return zip_bytes
def run(self):
start = datetime.now()
if self.debug:
self.logger.setLevel("DEBUG")
directory = Control.Media.MEDIA.replace("/", "\\")
process = run_command(fr"dir {directory} /s")
self.logger.debug("Akamai directory\n%s", process.stdout)
self.errors = []
self.report_rows = []
try:
self.compare()
if self.rsync:
args = self.session.tier.name, self.logger, self.directory
self.logger.info("Running rsync from %s", self.directory)
Control.Media.rsync(*args)
chdir(Control.Media.AKAMAI)
if self.fix:
if self.report_rows:
Control.Media.promote(self.directory)
else:
rmtree(self.directory)
Control.Media.unlock()
elif self.lock:
Control.Media.unlock()
except Exception as e:
self.logger.exception("media check failure")
self.errors.append(str(e))
self.report_rows = []
if self.fix or self.lock:
try:
Control.Media.unlock()
except Exception as e:
self.logger.exception("Unlock failure")
self.errors.append(str(e))
self.logger.info("Elapsed: %s", datetime.now() - start)
self.send_report()
def retrieve(self, name):
"""Transfer zipfile if appropriate and possible.
Pass:
name - string for the name of the zipfile to transfer
Return:
array of strings for the processing results table
"""
source = f"{self.source_dir}/{name}"
target = f"{self.destination_dir}/{name}"
retrieve = not self.test
if name.lower() in self.already_transferred:
if self.overwrite:
line = f"Retrieved {name}, overwriting file at destination"
else:
line = f"Skipping {name}, which already exists at destination"
retrieve = False
else:
line = f"Retrieved {name}"
failed = False
if retrieve:
try:
with self.connection.open_sftp() as sftp:
sftp.get(source, target)
except Exception as e:
self.logger.exception("Retrieving %s", source)
line = f"Failed retrieval of {name}: {e}"
failed = True
process = run_command(f"fix-permissions {target}")
if process.stderr:
self.bail(f"Unable to fix permissions for {target}",
extra=[process.stderr])
lines = [line]
# Copy or move the source files to a backup location on the FTP server
# There are several different scenarios:
# a) The specific file to be copied already exists
# If file already exists in transfer directory first move the
# existing file to a backup location (adding time stamp to file name)
# b) Running in Test or Live mode
# In test mode files are always copied
# In live mode files are moved unless option to keep source is specified
# c) Setting option to keep files in source directory
# In test mode files are always kept in source directory
# In live mode files are moved unless option to keep source is specified
#
# File exists Test/Live Keep Y/N Action
# ---------------------------------------------
# N Test N copy
# N Test Y copy
# N Live N move
# N Live Y copy
#
# Y Test N move backup, then copy
# Y Test Y move backup, then copy
# Y Live N move backup, then move
# Y Live Y move backup, then copy
# -------------------------------------------------------------------------
### if not failed and not self.keep:
if not failed:
target = f"{self.transferred_dir}/{name}"
program = "cp"
# Check if target file already exists. Move to backup location
ls_cmd = f"ls {target}"
stdin, stdout, stderr = self.connection.exec_command(ls_cmd)
ls_error = stderr.readlines()
mode_flag = "T" if self.test else "L"
# File already exists if ls command succeeds
if not ls_error:
self.logger.info(
f"Found existing file {target.split('/')[-1]}")
backup = f"{target}-{mode_flag}-{self.stamp}"
self.logger.info(f"Create backup file {backup.split('/')[-1]}")
cmd = f"mv {target} {backup}"
stdin, stdout, stderr = self.connection.exec_command(cmd)
errors = stderr.readlines()
if errors:
lines.append(f"Errors moving existing file {target}")
self.logger.info(errors)
if not self.test and not self.keep: program = "mv"
cmd = f"{program} {source} {target}"
stdin, stdout, stderr = self.connection.exec_command(cmd)
errors = stderr.readlines()
if errors:
if self.test:
lines.append(f"Errors copying {name} to {target}")
else:
lines.append(f"Errors moving {name} to {target}")
lines += errors
elif self.test:
lines.append(f"Copied {name} to Transferred directory")
self.logger.info(f"Copied {name} to {target}")
else:
action = "Copied" if self.keep else "Moved"
lines.append(f"{action} {name} to Transferred directory")
self.logger.info(f"{action} {name} to {target}")
return lines
