def run(self):
"""
Install the documents and perform any necessary postprocessing.
"""
if not os.path.isdir(opts.source):
self.logger.error("%s not found", opts.source)
sys.exit(1)
action = "comparing" if opts.test else "installing"
doctype = self.DOCTYPE.lower()
self.logger.info("%s %ss", action, doctype)
self.logger.info("from %s", opts.source)
changes = 0
for name in os.listdir(self.opts.source):
if name.endswith(".xml"):
xml = open(os.path.join(self.opts.source, name), "rb").read()
doc = self.Document(self, name, xml)
if doc.install():
changes += 1
if changes:
if not self.opts.test:
self.post_process()
else:
self.logger.info("%ss already up to date", doctype)
if not self.opts.test:
cdr.logout(self.session)
self.logger.info("%s installation complete", doctype)
def main():
parser = create_parser()
opts = parser.parse_args()
title = opts.title
if not title:
parser.error("empty title argument")
if not isinstance(title, unicode):
title = unicode(title.strip(), "latin-1")
if "--" in title:
parser.error("filter title cannot contain --")
if not opts.session:
password = getpass.getpass()
session = cdr.login(opts.user, password, tier="PROD")
error = cdr.checkErr(session)
if error:
parser.error(error)
else:
session = opts.session
stub = u"""\
<?xml version="1.0" ?>
<!-- Filter title: {} -->
<xsl:transform xmlns:xsl = "http://www.w3.org/1999/XSL/Transform"
xmlns:cdr = "cips.nci.nih.gov/cdr"
version = "1.0">
<xsl:output method = "xml"
encoding = "utf-8"/>
<xsl:param name = "sample-param"
select = "'default-value'"/>
<!-- Sample template -->
<xsl:template match = "@*|node()">
<xsl:copy>
<xsl:apply-templates select = "@*|node()"/>
</xsl:copy>
</xsl:template>
</xsl:transform>
""".format(escape(title)).encode("utf-8")
title = title.encode("utf-8")
ctrl = dict(DocTitle=title)
doc_opts = dict(doctype="Filter", ctrl=ctrl, encoding="utf-8")
doc = cdr.Doc(stub, **doc_opts)
cdr_id = cdr.addDoc(session, doc=str(doc), tier="PROD")
error = cdr.checkErr(cdr_id)
if error:
parser.error(error)
response = cdr.unlock(session, cdr_id, tier="PROD")
error = cdr.checkErr(response)
if error:
parser.error(error)
name = cdr_id + ".xml"
with open(name, "wb") as fp:
fp.write(stub)
print(("Created {}".format(name)))
if not opts.session:
cdr.logout(session, tier="PROD")
def __cleanup(self, status, message):
self.log("~~In cleanup")
if self.__rtfMailerDir:
os.chdir(self.__rtfMailerDir)
command = f"{cdr.BASEDIR}/bin/fix-permissions.cmd"
command = command.replace("/", "\\")
process = cdr.run_command(command, merge_output=True)
if process.returncode:
args = self.__rtfMailerDir, process.stdout
_LOGGER.error("fixing %s permissions: %s", *args)
else:
self.log(f"fixed permissions for {self.__rtfMailerDir}")
try:
self.__updateStatus(status, message)
self.__sendMail()
if self.__session: cdr.logout(self.__session)
except:
self.log("__cleanup failed, status was '%s'" % status, tback=1)
def cleanSession(cursor, session):
"""
If there is a session or cursor open, close it appropriately. If
operation fails, log failure and continue.
Pass:
session - Open CDR session or None.
cursor - Open database cursor or None.
"""
if cursor:
try:
cursor.close()
except Exception as e:
FS_LOGGER.exception("failure closing db cursor")
if session:
try:
cdr.logout(session)
except Exception as e:
FS_LOGGER.exception("failure logging out of session")
def main():
"""
Store the new version of the filter. Processing steps:
1. Parse the command-line options and arguments.
2. Load the new version of the filter from the file system.
3. Log into the CDR on the target server.
4. Find the CDR ID which matches the filter title
5. Check out the document from the target CDR server.
6. Store the new version on the target CDR server.
7. Report the number of the new version.
8. Clean up.
"""
#------------------------------------------------------------------
# 1. Parse the command-line options and arguments.
#------------------------------------------------------------------
parser = create_parser()
opts = parser.parse_args()
pub = "Y" if opts.publishable else "N"
# If no comment is specified the last comment used (from the
# all_docs table) would be stored.
# Setting the comment to something to overwrite the last comment
# -----------------------------------------------------------------
comment = opts.comment or "Replaced w/o user comment"
#------------------------------------------------------------------
# 2. Load the new version of the filter from the file system.
#------------------------------------------------------------------
with open(opts.filename) as fp:
xml = fp.read()
if "]]>" in xml:
parser.error("CdrDoc wrapper must be stripped from the file")
#------------------------------------------------------------------
# 3. Log into the CDR on the target server.
#------------------------------------------------------------------
if opts.session:
session = opts.session
else:
password = getpass.getpass()
session = cdr.login(opts.user, password, tier=opts.tier)
error_message = cdr.checkErr(session)
if error_message:
parser.error(error_message)
#------------------------------------------------------------------
# 4. Find out what the filter's document ID is.
#------------------------------------------------------------------
doc_id = get_doc_id(xml, opts.tier, session)
#------------------------------------------------------------------
# 5. Check out the document from the target CDR server.
#------------------------------------------------------------------
args = dict(checkout="Y", getObject=True, tier=opts.tier)
doc = cdr.getDoc(session, doc_id, **args)
error_message = cdr.checkErr(doc)
if error_message:
parser.error(error_message)
#------------------------------------------------------------------
# 6. Store the new version on the target CDR server.
#------------------------------------------------------------------
doc.xml = xml.encode("utf-8")
args = dict(
doc=str(doc),
checkIn="Y",
setLinks="N",
reason=comment,
comment=comment,
ver="Y",
verPublishable=pub,
tier=opts.tier
)
doc_id = cdr.repDoc(session, **args)
if not doc_id.startswith("CDR"):
parser.error(error_message)
#------------------------------------------------------------------
# 7. Report the number of the latest version.
#------------------------------------------------------------------
versions = cdr.lastVersions(session, doc_id, tier=opts.tier)
print(("Saved {} as version {}".format(doc_id, versions[0])))
#------------------------------------------------------------------
# 8. Clean up.
#------------------------------------------------------------------
if not opts.session:
cdr.logout(session, tier=opts.tier)
args = docId, lastVer, lastverDate, cwdDate
LOGGER.info("CDR%d version %d saved %s; CWD saved %s", *args)
cwdXml = getNormalizedXml(cursor, docId)
lastXml = getNormalizedXml(cursor, docId, lastVer)
if cwdXml != lastXml:
if versionChanges(session, docId):
versioned += 1
#----------------------------------------------------------------------
# Version all the document which have never been versioned at all.
#----------------------------------------------------------------------
cursor.execute("""\
SELECT d.id
FROM document d
JOIN doc_type t
ON t.id = d.doc_type
WHERE t.name = '%s'
AND d.id NOT IN (SELECT id FROM #lastver)""" % docType)
docIds = [row[0] for row in cursor.fetchall()]
args = len(docIds), docType
LOGGER.info("%d %s documents have never been versioned", *args)
for docId in docIds:
if versionChanges(session, docId):
versioned += 1
#----------------------------------------------------------------------
# Clean up and go home.
#----------------------------------------------------------------------
LOGGER.info("saved %d new versions", versioned)
cdr.logout(session)
def run(self):
"""
Transform a batch of CDR documents
The overridden `select()` and `transform()` methods are
invoked by this base class method.
"""
if not self.testing:
self.logger.info("Running in real mode, updating the database")
else:
self.logger.info("Saving test output to %r", self.output_directory)
if self.tier:
self.logger.info("Running on {}".format(self.tier))
start = datetime.now()
doc_ids = self.select()
counts = dict(processed=0, saved=0, versions=0, locked=0, errors=0)
types = {}
self.logger.info("%d documents selected", len(doc_ids))
self.logger.info("Purpose: %r", self.comment)
for doc_id in doc_ids:
self.doc = None
needs_unlock = True
try:
self.doc = self.Doc(self, doc_id)
self.logger.info("Processing %s", self.doc)
self.doc.save_changes()
if self.doc.saved:
counts["saved"] += 1
if "cwd" in self.doc.changed:
counts["versions"] += 1
if "lastv" in self.doc.changed:
counts["versions"] += 1
if "lastp" in self.doc.changed:
counts["versions"] += 1
for version_type in self.doc.saved:
types[version_type] = types.get(version_type, 0) + 1
except Job.DocumentLocked as info:
needs_unlock = False
counts["locked"] += 1
self.logger.warning(str(info))
except Exception:
self.logger.exception("Document %d", doc_id)
counts["errors"] += 1
if counts["errors"] > self.max_errors:
message = "Stopping after %d errors"
self.logger.error(message, counts["errors"])
break
if needs_unlock:
cdr.unlock(self.session, doc_id, tier=self.tier)
counts["processed"] += 1
if counts["processed"] >= self.max_docs:
message = "Stopping after processing %d documents"
self.logger.info(message, counts["processed"])
break
details = []
if types:
details = ["Specific versions saved:"]
for key in sorted(types):
details.append(" {} = {:d}".format(key, types[key]))
details.append("")
elapsed = datetime.now() - start
self.logger.info("""\
Run completed.
Docs examined = {processed:d}
Docs changed = {saved:d}
Versions changed = {versions:d}
Could not lock = {locked:d}
Errors = {errors:d}
Time = {time}
{details}""".format(details="\n".join(details), time=elapsed, **counts))
if not self.__opts.get("session"):
cdr.logout(self.session, tier=self.tier)