def run(self):
"""Pick a dictionary class and load the terms."""
started = datetime.now()
if self.opts.get("tier") == "PROD" and not isProdHost():
if not self.opts.get("auth"):
pw = getpw("esadmin")
if pw:
self.opts["auth"] = f"admin,{pw}"
if self.dictionary == "glossary":
loader = GlossaryLoader(**self.opts)
elif self.dictionary == "drugs":
loader = DrugLoader(**self.opts)
else:
raise Exception("no dictionary specified")
tier = loader.tier
host = loader.host
subject = f"[{tier}] Load of {self.dictionary} dictionary to {host}"
try:
loader.run()
elapsed = datetime.now() - started
message = f"Load completed in {elapsed}."
except Exception as e:
self.logger.exception("failure")
subject += " (FAILURE)"
message = f"Job failed: {e}\nSee logs for further details."
self.__notify(subject, message)
def login(self):
"""
Create a CDR login session for adding/updating the documents.
"""
if self.opts.test:
return None
password = cdr.getpw(self.ACCOUNT)
if not password:
self.logger.error("account password not found")
sys.exit(1)
session = cdr.login(self.ACCOUNT, password)
error = cdr.checkErr(session)
if error:
self.logger.error(error)
sys.exit(1)
return session
def __getCdrSession(self):
rsp = str(cdr.login("cdrmailers", cdr.getpw("cdrmailers")))
match = self.__ERR_PATTERN.search(rsp)
if match:
raise Exception("CDR login failure: %s" % match.group(1))
self.__session = rsp
logger = Logging.get_logger("deploy", console=True)
parser = ArgumentParser()
parser.add_argument("--directory", "-d", required=True)
parser.add_argument("--tier", "-t")
parser.add_argument("--session", "-s")
parser.add_argument("--group", "-g")
parser.add_argument("--name", "-n")
parser.add_argument("--verbose", "-v", action="store_true")
opts = parser.parse_args()
logger.info("installing from %s", opts.directory)
try:
if opts.session:
session = opts.session
else:
password = getpw(ACCOUNT)
session = login(ACCOUNT, password)
cursor = db.connect(user="******", tier=opts.tier).cursor()
directory = Path(opts.directory)
update_opts = dict(tier=opts.tier)
for path in directory.iterdir():
if path.is_file():
name = path.name
parts = name.split(".")
if len(parts) == 2:
name, ext = parts
if ext in EXTENSIONS:
parts = name.split("--")
if len(parts) == 2:
group, name = parts
if not opts.name or opts.name == name:
def expireMeetingRecordings(self, testMode):
"""
This is a "Custom" routine that sweeps away MP3 format meeting
recordings that have passed their useful life. Implemented for
JIRA Issue OCECDR-3886.
Pass:
testMode
True = Don't actually delete any blobs, just report
False = Update docs and delete blobs.
"""
cursor = None
session = None
# Need a connection to the CDR Server
session = cdr.login('FileSweeper', cdr.getpw('FileSweeper'))
if not session:
FS_LOGGER.error("FileSweeper login to CdrServer failed")
# But no reason not to do the rest of the sweep
return
# And a read-only connection to the database
try:
conn = db.connect()
cursor = conn.cursor()
except Exception as e:
FS_LOGGER.exception("attempting DB connect")
# But continue with the sweep
cleanSession(cursor, session)
return
# Today's SQL Server date
try:
cursor.execute("SELECT GETDATE()")
now = cursor.fetchone()[0]
except Exception as e:
FS_LOGGER.exception("getting DB date")
cleanSession(cursor, session)
return
# Only want YYYY-MM-DD, not HMS
nowDate = str(now)[:10]
# Locate all Media documents linked to meeting recordings that
# are older than Oldest days.
# This is done by checking for any ADD DOCUMENT transaction in the
# audit trail for one of the qualifying documents. If any ADD was
# performed before the Oldest value, then there was a version of
# the meeting recording from before that date.
# The Media doc must also be found in one of the ...blob_usage tables.
# If not, then any blob associated with it has already been deleted.
isoFmt = "%Y-%m-%d"
earlyDate = \
datetime.datetime.fromtimestamp(self.oldSpec).strftime(isoFmt)
# DEBUG
msg = "Looking for meeting recordings older than %s"
FS_LOGGER.debug(msg, earlyDate)
qry = """
SELECT d.id, d.title
FROM document d
JOIN query_term qt
ON qt.doc_id = d.id
JOIN audit_trail at
ON at.document = d.id
JOIN action act
ON act.id = at.action
WHERE qt.path = '/Media/MediaContent/Categories/Category'
AND qt.value = 'meeting recording'
AND act.name = 'ADD DOCUMENT'
AND at.dt <= '%s'
AND (
d.id IN ( SELECT doc_id FROM doc_blob_usage )
OR
d.id IN ( SELECT doc_id FROM version_blob_usage )
)
""" % earlyDate
# Read the info into memory
try:
cursor.execute(qry)
rows = cursor.fetchall()
except Exception as e:
FS_LOGGER.exception("attempting to locate old blobs")
cleanSession(cursor, session)
return
# If there weren't any, that's normal and okay
if len(rows) == 0:
FS_LOGGER.info("No meeting recordings needed to be deleted")
cleanSession(cursor, session)
return
# Do we need to lock and load the docs for update?
checkOut = 'Y'
if testMode:
checkOut = 'N'
#-------------------------------------------------------------------
# We've got some to delete.
# For each Media document:
# Send a transaction to the CDR Server to do the following:
# Add a ProcessingStatus to the Media document to say what happened
# Delete all of the blobs.
#-------------------------------------------------------------------
for row in rows:
docId, title = row
# Fetch the original document
# We'll do this even in test mode to test the xml mods
try:
docObj = cdr.getDoc(session, docId, checkout=checkOut,
getObject=True)
except Exception as e:
FS_LOGGER.exception("attempting to fetch doc %d", docId)
cleanSession(cursor, session)
return
# Test for retrieval error, e.g., locked doc
err = cdr.checkErr(docObj)
if err:
message = "Failed getDoc for CDR ID %s: %s, continuing"
FS_LOGGER.error(message, docId, err)
continue
# Parse the xml preparatory to modifying it
mediaRoot = et.fromstring(docObj.xml)
# Create the new Comment field to record what we did
# Make it the last subelement of the Media document element
# It has to be there
comment = et.SubElement(mediaRoot, 'Comment',
audience='Internal',
user='******',
date=nowDate)
comment.text = "Removed meeting recording object after expiration"
# Back to serial XML
newXml = et.tostring(mediaRoot)
# If we're testing, just log what we would have done
if testMode:
# For log file
actionMsg = 'would delete'
else:
# Send the doc back to the database:
# Wrapped in CdrDoc wrapper
# With command to delete all blobs
actionMsg = 'deleted'
saveXml = cdr.makeCdrDoc(newXml, 'Media', docObj.id)
response = cdr.repDoc(session, doc=saveXml,
comment='Removed meeting recording blobs',
delAllBlobVersions=True, check_in=True)
# Check response
if not response[0]:
errors = cdr.getErrors(response[1], errorsExpected=True,
asSequence=False)
message = "Saving Media xml for doc %s: %s"
FS_LOGGER.error(message, docObj.id, errors)
FS_LOGGER.info("Aborting expireMeetingRecords()")
# Stop doing this, but continue rest of file sweeps.
cleanSession(cursor, session)
return
# Log results for this media recording
args = actionMsg, docId, title
msg = "FileSweeper %s blobs for cdrId: %s\n%s"
FS_LOGGER.info(msg, *args)
# Cleanup
cleanSession(cursor, session)
def session(self):
if not hasattr(self, "_session"):
opts = dict(password=cdr.getpw("pdqcontent"))
self._session = Session.create_session("pdqcontent", **opts)
return self._session
# The performance of the publishing job has greatly improved allowing
# us to cancel a running job much sooner if it fails to finish.
# Optionally overriden below once we know the publishing subset.
# --------------------------------------------------------------------
if cdr.isProdHost():
waitTotal = 10800 # 3.0 hours
elif cdr.isDevHost():
waitTotal = 10800 # 3.0 hours
else:
waitTotal = 14400 # 4.0 hours
testMode = None
fullMode = None
session = cdr.login("cdroperator", cdr.getpw("cdroperator"))
pubSystem = 'Primary'
pubEmail = cdr.getEmailList('Operator Publishing Notification')
# ------------------------------------------------------------
# Function to parse the command line arguments
# ------------------------------------------------------------
def parseArgs(args):
# args = argv
global testMode
global fullMode
global LOGGER