def main():
parser = create_parser()
opts = parser.parse_args()
title = opts.title
if not title:
parser.error("empty title argument")
if not isinstance(title, unicode):
title = unicode(title.strip(), "latin-1")
if "--" in title:
parser.error("filter title cannot contain --")
if not opts.session:
password = getpass.getpass()
session = cdr.login(opts.user, password, tier="PROD")
error = cdr.checkErr(session)
if error:
parser.error(error)
else:
session = opts.session
stub = u"""\
<?xml version="1.0" ?>
<!-- Filter title: {} -->
<xsl:transform xmlns:xsl = "http://www.w3.org/1999/XSL/Transform"
xmlns:cdr = "cips.nci.nih.gov/cdr"
version = "1.0">
<xsl:output method = "xml"
encoding = "utf-8"/>
<xsl:param name = "sample-param"
select = "'default-value'"/>
<!-- Sample template -->
<xsl:template match = "@*|node()">
<xsl:copy>
<xsl:apply-templates select = "@*|node()"/>
</xsl:copy>
</xsl:template>
</xsl:transform>
""".format(escape(title)).encode("utf-8")
title = title.encode("utf-8")
ctrl = dict(DocTitle=title)
doc_opts = dict(doctype="Filter", ctrl=ctrl, encoding="utf-8")
doc = cdr.Doc(stub, **doc_opts)
cdr_id = cdr.addDoc(session, doc=str(doc), tier="PROD")
error = cdr.checkErr(cdr_id)
if error:
parser.error(error)
response = cdr.unlock(session, cdr_id, tier="PROD")
error = cdr.checkErr(response)
if error:
parser.error(error)
name = cdr_id + ".xml"
with open(name, "wb") as fp:
fp.write(stub)
print(("Created {}".format(name)))
if not opts.session:
cdr.logout(session, tier="PROD")
def main():
"""
Top-level entry point
"""
# Process the command-line arguments.
parser = create_parser()
opts = parser.parse_args()
# Make sure we're not doing this on the production server.
if not opts.tier and cdr.isProdHost() or opts.tier == "PROD":
parser.error("""
This program can only be used to install a filter on the development or
test server, not production.
Use CreateFilter.py to create the filter in the production database, then
use InstallFilter.py to install it in test or development with the same
title/name and (almost certainly) a different local CDR ID.
""")
# If we don't already have a session ID, make one.
if not opts.session:
password = getpass.getpass()
session = cdr.login(opts.user, password, tier=opts.tier)
error = cdr.checkErr(session)
if error:
parser.error(error)
else:
session = opts.session
# Load the document.
info = DocInfo(opts.filename, parser)
# Make sure the filter isn't already installed
info.check_unique_title(opts.tier, parser)
# All checks passed: add the document.
ctrl = dict(DocTitle=info.title.encode("utf-8"))
doc = cdr.Doc(info.xml, doctype="Filter", ctrl=ctrl, encoding="utf-8")
comment = "New filter install"
add_opts = dict(doc=str(doc), comment=comment, tier=opts.tier)
cdr_id = cdr.addDoc(session, **add_opts)
error = cdr.checkErr(cdr_id)
if error:
parser.error(error)
# Unlock the document and display its ID.
response = cdr.unlock(session, cdr_id, tier=opts.tier)
error = cdr.checkErr(response)
if error:
parser.error(error)
else:
print(cdr_id)
def login(self):
"""
Create a CDR login session for adding/updating the documents.
"""
if self.opts.test:
return None
password = cdr.getpw(self.ACCOUNT)
if not password:
self.logger.error("account password not found")
sys.exit(1)
session = cdr.login(self.ACCOUNT, password)
error = cdr.checkErr(session)
if error:
self.logger.error(error)
sys.exit(1)
return session
def session(self):
"""
CDR session under which we save document versions
We use an existing session or create a new one,
depending on what has been passed to the constructor.
Return:
string for name of CDR session
"""
if not hasattr(self, "_session"):
self._session = self.__opts.get("session")
if not self._session:
user = self.__opts.get("user")
if not user:
raise ValueError("must supply session or user name")
password = getpass()
self._session = str(cdr.login(user, password, tier=self.tier))
return self._session
def main():
"""
Store the new version of the filter. Processing steps:
1. Parse the command-line options and arguments.
2. Load the new version of the filter from the file system.
3. Log into the CDR on the target server.
4. Find the CDR ID which matches the filter title
5. Check out the document from the target CDR server.
6. Store the new version on the target CDR server.
7. Report the number of the new version.
8. Clean up.
"""
#------------------------------------------------------------------
# 1. Parse the command-line options and arguments.
#------------------------------------------------------------------
parser = create_parser()
opts = parser.parse_args()
pub = "Y" if opts.publishable else "N"
# If no comment is specified the last comment used (from the
# all_docs table) would be stored.
# Setting the comment to something to overwrite the last comment
# -----------------------------------------------------------------
comment = opts.comment or "Replaced w/o user comment"
#------------------------------------------------------------------
# 2. Load the new version of the filter from the file system.
#------------------------------------------------------------------
with open(opts.filename) as fp:
xml = fp.read()
if "]]>" in xml:
parser.error("CdrDoc wrapper must be stripped from the file")
#------------------------------------------------------------------
# 3. Log into the CDR on the target server.
#------------------------------------------------------------------
if opts.session:
session = opts.session
else:
password = getpass.getpass()
session = cdr.login(opts.user, password, tier=opts.tier)
error_message = cdr.checkErr(session)
if error_message:
parser.error(error_message)
#------------------------------------------------------------------
# 4. Find out what the filter's document ID is.
#------------------------------------------------------------------
doc_id = get_doc_id(xml, opts.tier, session)
#------------------------------------------------------------------
# 5. Check out the document from the target CDR server.
#------------------------------------------------------------------
args = dict(checkout="Y", getObject=True, tier=opts.tier)
doc = cdr.getDoc(session, doc_id, **args)
error_message = cdr.checkErr(doc)
if error_message:
parser.error(error_message)
#------------------------------------------------------------------
# 6. Store the new version on the target CDR server.
#------------------------------------------------------------------
doc.xml = xml.encode("utf-8")
args = dict(
doc=str(doc),
checkIn="Y",
setLinks="N",
reason=comment,
comment=comment,
ver="Y",
verPublishable=pub,
tier=opts.tier
)
doc_id = cdr.repDoc(session, **args)
if not doc_id.startswith("CDR"):
parser.error(error_message)
#------------------------------------------------------------------
# 7. Report the number of the latest version.
#------------------------------------------------------------------
versions = cdr.lastVersions(session, doc_id, tier=opts.tier)
print(("Saved {} as version {}".format(doc_id, versions[0])))
#------------------------------------------------------------------
# 8. Clean up.
#------------------------------------------------------------------
if not opts.session:
cdr.logout(session, tier=opts.tier)
showWarnings='Y',
checkIn='Y',
verPublishable='N')
if errors:
for e in cdr.getErrors(errors, asSequence=True):
LOGGER.error(e)
return docId and True or False
#----------------------------------------------------------------------
# Processing starts here with setup.
#----------------------------------------------------------------------
if len(sys.argv) < 4:
sys.stderr.write("usage: %s uid pwd doctype\n" % sys.argv[0])
sys.exit(1)
session = cdr.login(sys.argv[1], sys.argv[2])
errors = cdr.checkErr(session)
if errors:
sys.stderr.write("login failure: %s" % errors)
sys.exit(1)
docType = sys.argv[3]
cursor = db.connect(user='******', timeout=300).cursor()
#----------------------------------------------------------------------
# Determine the last version number for each versioned document of
# the specified document type. Be sure to use the document type
# for the current working document instead of the version table,
# so we do the right thing for documents whose last version was
# saved as a different document type than the current working
# document has. The other effect of joining on the document table
# (or rather, view) is to avoid doing anything with deleted documents.
def __getCdrSession(self):
rsp = str(cdr.login("cdrmailers", cdr.getpw("cdrmailers")))
match = self.__ERR_PATTERN.search(rsp)
if match:
raise Exception("CDR login failure: %s" % match.group(1))
self.__session = rsp
logger = Logging.get_logger("deploy", console=True)
parser = ArgumentParser()
parser.add_argument("--directory", "-d", required=True)
parser.add_argument("--tier", "-t")
parser.add_argument("--session", "-s")
parser.add_argument("--group", "-g")
parser.add_argument("--name", "-n")
parser.add_argument("--verbose", "-v", action="store_true")
opts = parser.parse_args()
logger.info("installing from %s", opts.directory)
try:
if opts.session:
session = opts.session
else:
password = getpw(ACCOUNT)
session = login(ACCOUNT, password)
cursor = db.connect(user="******", tier=opts.tier).cursor()
directory = Path(opts.directory)
update_opts = dict(tier=opts.tier)
for path in directory.iterdir():
if path.is_file():
name = path.name
parts = name.split(".")
if len(parts) == 2:
name, ext = parts
if ext in EXTENSIONS:
parts = name.split("--")
if len(parts) == 2:
group, name = parts
if not opts.name or opts.name == name:
if not opts.group or opts.group == group:
def expireMeetingRecordings(self, testMode):
"""
This is a "Custom" routine that sweeps away MP3 format meeting
recordings that have passed their useful life. Implemented for
JIRA Issue OCECDR-3886.
Pass:
testMode
True = Don't actually delete any blobs, just report
False = Update docs and delete blobs.
"""
cursor = None
session = None
# Need a connection to the CDR Server
session = cdr.login('FileSweeper', cdr.getpw('FileSweeper'))
if not session:
FS_LOGGER.error("FileSweeper login to CdrServer failed")
# But no reason not to do the rest of the sweep
return
# And a read-only connection to the database
try:
conn = db.connect()
cursor = conn.cursor()
except Exception as e:
FS_LOGGER.exception("attempting DB connect")
# But continue with the sweep
cleanSession(cursor, session)
return
# Today's SQL Server date
try:
cursor.execute("SELECT GETDATE()")
now = cursor.fetchone()[0]
except Exception as e:
FS_LOGGER.exception("getting DB date")
cleanSession(cursor, session)
return
# Only want YYYY-MM-DD, not HMS
nowDate = str(now)[:10]
# Locate all Media documents linked to meeting recordings that
# are older than Oldest days.
# This is done by checking for any ADD DOCUMENT transaction in the
# audit trail for one of the qualifying documents. If any ADD was
# performed before the Oldest value, then there was a version of
# the meeting recording from before that date.
# The Media doc must also be found in one of the ...blob_usage tables.
# If not, then any blob associated with it has already been deleted.
isoFmt = "%Y-%m-%d"
earlyDate = \
datetime.datetime.fromtimestamp(self.oldSpec).strftime(isoFmt)
# DEBUG
msg = "Looking for meeting recordings older than %s"
FS_LOGGER.debug(msg, earlyDate)
qry = """
SELECT d.id, d.title
FROM document d
JOIN query_term qt
ON qt.doc_id = d.id
JOIN audit_trail at
ON at.document = d.id
JOIN action act
ON act.id = at.action
WHERE qt.path = '/Media/MediaContent/Categories/Category'
AND qt.value = 'meeting recording'
AND act.name = 'ADD DOCUMENT'
AND at.dt <= '%s'
AND (
d.id IN ( SELECT doc_id FROM doc_blob_usage )
OR
d.id IN ( SELECT doc_id FROM version_blob_usage )
)
""" % earlyDate
# Read the info into memory
try:
cursor.execute(qry)
rows = cursor.fetchall()
except Exception as e:
FS_LOGGER.exception("attempting to locate old blobs")
cleanSession(cursor, session)
return
# If there weren't any, that's normal and okay
if len(rows) == 0:
FS_LOGGER.info("No meeting recordings needed to be deleted")
cleanSession(cursor, session)
return
# Do we need to lock and load the docs for update?
checkOut = 'Y'
if testMode:
checkOut = 'N'
#-------------------------------------------------------------------
# We've got some to delete.
# For each Media document:
# Send a transaction to the CDR Server to do the following:
# Add a ProcessingStatus to the Media document to say what happened
# Delete all of the blobs.
#-------------------------------------------------------------------
for row in rows:
docId, title = row
# Fetch the original document
# We'll do this even in test mode to test the xml mods
try:
docObj = cdr.getDoc(session, docId, checkout=checkOut,
getObject=True)
except Exception as e:
FS_LOGGER.exception("attempting to fetch doc %d", docId)
cleanSession(cursor, session)
return
# Test for retrieval error, e.g., locked doc
err = cdr.checkErr(docObj)
if err:
message = "Failed getDoc for CDR ID %s: %s, continuing"
FS_LOGGER.error(message, docId, err)
continue
# Parse the xml preparatory to modifying it
mediaRoot = et.fromstring(docObj.xml)
# Create the new Comment field to record what we did
# Make it the last subelement of the Media document element
# It has to be there
comment = et.SubElement(mediaRoot, 'Comment',
audience='Internal',
user='******',
date=nowDate)
comment.text = "Removed meeting recording object after expiration"
# Back to serial XML
newXml = et.tostring(mediaRoot)
# If we're testing, just log what we would have done
if testMode:
# For log file
actionMsg = 'would delete'
else:
# Send the doc back to the database:
# Wrapped in CdrDoc wrapper
# With command to delete all blobs
actionMsg = 'deleted'
saveXml = cdr.makeCdrDoc(newXml, 'Media', docObj.id)
response = cdr.repDoc(session, doc=saveXml,
comment='Removed meeting recording blobs',
delAllBlobVersions=True, check_in=True)
# Check response
if not response[0]:
errors = cdr.getErrors(response[1], errorsExpected=True,
asSequence=False)
message = "Saving Media xml for doc %s: %s"
FS_LOGGER.error(message, docObj.id, errors)
FS_LOGGER.info("Aborting expireMeetingRecords()")
# Stop doing this, but continue rest of file sweeps.
cleanSession(cursor, session)
return
# Log results for this media recording
args = actionMsg, docId, title
msg = "FileSweeper %s blobs for cdrId: %s\n%s"
FS_LOGGER.info(msg, *args)
# Cleanup
cleanSession(cursor, session)
if arg.startswith('doc='):
docs.append(int(arg[4:]))
elif arg.startswith('job='):
jobs.append(int(arg[4:]))
elif arg.startswith('type='):
if docType:
raise Exception("only one document type may be specified")
else:
docType = arg[5:]
elif arg == "all":
all = True
elif arg == "add":
add = True
elif arg.startswith("email="):
email = arg[6:]
else:
raise Exception("invalid argument %s" % arg)
session = cdr.login(uid, pwd)
try:
republisher = CdrRepublisher(session)
jobId = republisher.republish(addNewLinkedDocuments=add,
docList=docs,
jobList=jobs,
docType=docType,
docTypeAll=all,
email=email)
cdr.logout(session)
except Exception as e:
cdr.logout(session)
raise
# -------------------------------------
logger = cdr.Logging.get_logger("RemoveProdGroups", console=True)
logger.info('RemoveProdGroups - Started')
logger.debug('Arguments: %s', opts)
# Live or test mode
# -----------------
if opts.runmode == "test":
testMode = True
else:
testMode = False
tier = opts.tier
# Log into the CDR on the target server.
# --------------------------------------
if opts.session:
session = opts.session
else:
password = getpass.getpass()
session = cdr.login(opts.user, password, tier=opts.tier)
error_message = cdr.checkErr(session)
if error_message:
parser.error(error_message)
error_count = updateGroups(session, testMode, tier)
logger.info('RemoveProdGroups - Finished')
logger.info('Missing groups: %d', error_count)
sys.exit(0)
# The performance of the publishing job has greatly improved allowing
# us to cancel a running job much sooner if it fails to finish.
# Optionally overriden below once we know the publishing subset.
# --------------------------------------------------------------------
if cdr.isProdHost():
waitTotal = 10800 # 3.0 hours
elif cdr.isDevHost():
waitTotal = 10800 # 3.0 hours
else:
waitTotal = 14400 # 4.0 hours
testMode = None
fullMode = None
session = cdr.login("cdroperator", cdr.getpw("cdroperator"))
pubSystem = 'Primary'
pubEmail = cdr.getEmailList('Operator Publishing Notification')
# ------------------------------------------------------------
# Function to parse the command line arguments
# ------------------------------------------------------------
def parseArgs(args):
# args = argv
global testMode
global fullMode
global LOGGER
def main():
#------------------------------------------------------------------
# 1. Parse the command-line options and arguments.
#------------------------------------------------------------------
op = createOptionParser()
(options, args) = op.parse_args()
if len(args) != 4:
op.error("incorrect number of arguments")
uid, pwd, idArg, filename = args
if not options.publishable:
options.publishable = 'N'
elif options.publishable == 'Y':
options.version = 'Y'
fullId, intId, fragId = cdr.exNormalize(idArg)
# If no comment is specified the last comment used (from the
# all_docs table) would be stored.
# Setting the comment to something to overwrite the last comment
# -----------------------------------------------------------------
if not options.comment:
options.comment = "Filter title changed"
#------------------------------------------------------------------
# 2. Load the new version of the filter from the file system.
#------------------------------------------------------------------
fp = open(filename, 'r')
docXml = fp.read()
fp.close()
if ']]>' in docXml:
op.error("CdrDoc wrapper must be stripped from the file")
#------------------------------------------------------------------
# 3. Log into the CDR on the target server.
#------------------------------------------------------------------
session = cdr.login(uid, pwd)
checkForProblems(session, op)
#------------------------------------------------------------------
# 4. Check out the document from the target CDR server.
#------------------------------------------------------------------
docObj = cdr.getDoc(session, fullId, checkout='Y', getObject=True)
checkForProblems(docObj, op)
#------------------------------------------------------------------
# 5. Plug in the new title for the filter.
#------------------------------------------------------------------
docObj.ctrl['DocTitle'] = getNewTitle(docXml)
#------------------------------------------------------------------
# 6. Store the new version on the target CDR server.
#------------------------------------------------------------------
doc = str(docObj)
print('Versioned: %s, Publishable: %s' %
(options.version, options.publishable))
cdrId = cdr.repDoc(session,
doc=doc,
checkIn="Y",
setLinks="N",
reason=options.comment,
comment=options.comment,
ver=options.version,
verPublishable=options.publishable)
checkForProblems(cdrId, op)
#------------------------------------------------------------------
# 7. Report the number of the latest version.
#------------------------------------------------------------------
versions = cdr.lastVersions(session, cdrId)
if options.version == "N":
print("CWD for %s updated" % cdrId)
else:
print("Latest version of %s is %d" % (cdrId, versions[0]))
print("")
print("DON'T FORGET TO CHANGE THE TITLE OF THIS FILTER ON ALL TIERS!")