class MSGDBExporter(object):
"""
Export MSG DBs as SQL scripts.
Supports export to local storage and to cloud storage.
Usage:
from msg_db_exporter import MSGDBExporter
exporter = MSGDBExporter()
Public API:
exportDB(databases:List,
toCloud:Boolean,
testing:Boolean,
numChunks:Integer,
deleteOutdated:Boolean): Export a list of DBs to the cloud.
"""
# List of cloud files.
@property
def cloudFiles(self):
self._cloudFiles = self.driveService.files().list().execute()
return self._cloudFiles
@property
def driveService(self):
if self._driveService:
return self._driveService
if not self.credentialPath:
raise Exception("Credential path is required.")
storage = Storage(
'{}/google_api_credentials'.format(self.credentialPath))
self.googleAPICredentials = storage.get()
self.logger.log("Authorizing credentials.", 'info')
http = httplib2.Http()
http = self.googleAPICredentials.authorize(http)
self.logger.log("Authorized.", 'info')
self._driveService = build('drive', 'v2', http = http)
return self._driveService
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG', useColor = False)
self.timeUtil = MSGTimeUtil()
self.configer = MSGConfiger()
self.fileUtil = MSGFileUtil()
self.pythonUtil = MSGPythonUtil() # for debugging
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.notifier = SEKNotifier(connector = self.connector,
dbUtil = self.dbUtil,
user = self.configer.configOptionValue(
'Notifications', 'email_username'),
password = self.configer.configOptionValue(
'Notifications', 'email_password'),
fromaddr = self.configer.configOptionValue(
'Notifications', 'email_from_address'),
toaddr = self.configer.configOptionValue(
'Notifications', 'email_recipients'),
testing_toaddr =
self.configer.configOptionValue(
'Notifications',
'testing_email_recipients'),
smtp_server_and_port =
self.configer.configOptionValue(
'Notifications',
'smtp_server_and_port'))
# Google Drive parameters.
self.clientID = self.configer.configOptionValue('Export',
'google_api_client_id')
self.clientSecret = self.configer.configOptionValue('Export',
'google_api_client_secret')
self.oauthScope = 'https://www.googleapis.com/auth/drive'
self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
self.googleAPICredentials = ''
self.exportTempWorkPath = self.configer.configOptionValue('Export',
'db_export_work_path')
self.credentialPath = self.configer.configOptionValue('Export',
'google_api_credentials_path')
self.credentialStorage = Storage(
'{}/google_api_credentials'.format(self.credentialPath))
self._driveService = None
self._cloudFiles = None
self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
self.retryDelay = 10
self.availableFilesURL = ''
def verifyExportChecksum(self, testing = False):
"""
Verify the compressed export file using a checksum.
* Save the checksum of the original uncompressed export data.
* Extract the compressed file.
* Verify the uncompressed export data.
:param testing: When set to True, Testing Mode is used.
"""
# Get the checksum of the original file.
md5sum = self.fileUtil.md5Checksum(self.exportTempWorkPath)
self.logger.log('md5sum: {}'.format(md5sum))
def db_username(self):
return "postgres"
# return self.configer.configOptionValue('Database', 'db_username')
def db_password(self):
return self.configer.configOptionValue('Database', 'db_password')
def db_port(self):
return self.configer.configOptionValue('Database', 'db_port')
def dumpCommand(self, db = '', dumpName = ''):
"""
This method makes use of
pg_dump -s -p ${PORT}
-U ${USERNAME}
[-T ${OPTIONAL_TABLE_EXCLUSIONS}]
${DB_NAME} >
${EXPORT_TEMP_WORK_PATH}/${DUMP_TIMESTAMP}_{DB_NAME}.sql
:param db: String
:param dumpName: String
:return: String of command used to export DB.
"""
# For reference only:
# Password is passed from ~/.pgpass.
# Note that ':' and '\' characters should be escaped with '\'.
# Ref: http://www.postgresql.org/docs/9.1/static/libpq-pgpass.html
# Dump databases as the superuser. This method does not require a
# stored password when running under a root crontab.
if not db or not dumpName:
raise Exception('DB and dumpname required.')
# Process exclusions.
exclusions = self.dumpExclusionsDictionary()
excludeList = []
if db in exclusions:
excludeList = exclusions[db]
excludeString = ''
if len(excludeList) > 0 and exclusions != None:
for e in excludeList:
excludeString += """-T '"{}"' """.format(e)
return 'sudo -u postgres pg_dump -p {0} -U {1} {5} {2} > {3}/{4}' \
'.sql'.format(self.db_port(), self.db_username(), db,
self.exportTempWorkPath, dumpName, excludeString)
def dumpExclusionsDictionary(self):
"""
:param db: String of DB name for which to retrieve exclusions.
:return: Dictionary with keys as DBs and values as lists of tables to
be excluded for a given database.
"""
try:
if type(eval(self.configer.configOptionValue('Export',
'db_export_exclusions'))) == type(
{}):
return eval(self.configer.configOptionValue('Export',
'db_export_exclusions'))
else:
return None
except SyntaxError as detail:
self.logger.log(
'SyntaxError exception while getting exclusions: {}'.format(
detail))
def dumpName(self, db = ''):
"""
:param db: String
:return: String of file name used for dump file of db.
"""
if not db:
raise Exception('DB required.')
return "{}_{}".format(self.timeUtil.conciseNow(), db)
def filesToUpload(self, compressedFullPath = '', numChunks = 0,
chunkSize = 0):
"""
:param compressedFullPath: String
:param numChunks: Int
:param chunkSize: Int
:return: List of files to be uploaded according to their split
sections, if applicable.
"""
if numChunks != 0:
self.logger.log('Splitting {}'.format(compressedFullPath), 'DEBUG')
filesToUpload = self.fileUtil.splitLargeFile(
fullPath = compressedFullPath, chunkSize = chunkSize,
numChunks = numChunks)
if not filesToUpload:
raise Exception('Exception during file splitting.')
else:
self.logger.log('to upload: {}'.format(filesToUpload), 'debug')
return filesToUpload
else:
return [compressedFullPath]
def dumpResult(self, db = '', dumpName = '', fullPath = ''):
"""
:param dumpName: String of filename of dump file.
:param fullPath: String of full path to dump file.
:return: Boolean True if dump operation was successful, otherwise False.
"""
success = True
self.logger.log('fullPath: {}'.format(fullPath), 'DEBUG')
try:
# Generate the SQL script export.
# @todo check return value of dump command
self.logger.log('cmd: {}'.format(
self.dumpCommand(db = db, dumpName = dumpName)))
subprocess.check_call(
self.dumpCommand(db = db, dumpName = dumpName), shell = True)
except subprocess.CalledProcessError as error:
self.logger.log("Exception while dumping: {}".format(error))
sys.exit(-1)
return success
def exportDBs(self, databases = None, toCloud = False, localExport = True,
testing = False, chunkSize = 0, deleteOutdated = False):
"""
Export a set of DBs to local storage.
:param databases: List of database names that will be exported.
:param toCloud: Boolean if set to True, then the export will also be
copied to cloud storage.
:param localExport: Boolean when set to True the DB is exported
locally.
:param testing: Boolean flag for testing mode. (@DEPRECATED)
:param chunkSize: Integer size in bytes of chunk size used for
splitting.
:param deleteOutdated: Boolean indicating outdated files in the cloud
should be removed.
:returns: List of file IDs of uploaded files or None if there is an
error condition.
"""
# @todo separate uploading and exporting functions
noErrors = True
uploaded = []
for db in databases:
self.logger.log('Exporting {} using pg_dump.'.format(db), 'info')
dumpName = self.dumpName(db = db)
fullPath = '{}/{}.sql'.format(self.exportTempWorkPath, dumpName)
if localExport:
noErrors = self.dumpResult(db, dumpName, fullPath)
# Perform compression of the file.
self.logger.log("Compressing {} using gzip.".format(db), 'info')
self.logger.log('fullpath: {}'.format(fullPath), 'DEBUG')
gzipResult = self.fileUtil.gzipCompressFile(fullPath)
compressedFullPath = '{}{}'.format(fullPath, '.gz')
numChunks = self.numberOfChunksToUse(compressedFullPath)
# Gzip uncompress and verify by checksum is disabled until a more
# efficient, non-memory-based, uncompress is implemented.
# md5sum1 = self.fileUtil.md5Checksum(fullPath)
# self.md5Verification(compressedFullPath=compressedFullPath,
# fullPath=fullPath,md5sum1=md5sum1)
if toCloud:
# Split compressed files into a set of chunks to improve the
# reliability of uploads.
# Upload the files to the cloud.
for f in self.filesToUpload(
compressedFullPath = compressedFullPath,
numChunks = numChunks, chunkSize = chunkSize):
self.logger.log('Uploading {}.'.format(f), 'info')
fileID = self.uploadFileToCloudStorage(fullPath = f,
testing = testing,
retryCount = int(
self.configer.configOptionValue(
'Export',
'export_retry_count')))
self.logger.log('file id after upload: {}'.format(fileID))
if fileID != None:
uploaded.append(fileID)
self.logger.log('uploaded: {}'.format(uploaded),
'DEBUG')
if not self.addReaders(fileID,
self.configer.configOptionValue(
'Export',
'reader_permission_email_addresses').split(
','), retryCount = int(
self.configer.configOptionValue(
'Export',
'export_retry_count'))):
self.logger.log(
'Failed to add readers for {}.'.format(f),
'error')
self.logSuccessfulExport(*self.metadataOfFileID(fileID))
# Remove split sections if they exist.
try:
if not testing and numChunks > 1:
self.logger.log('Removing {}'.format(f))
os.remove('{}'.format(f))
except OSError as error:
self.logger.log(
'Exception while removing {}: {}.'.format(fullPath,
error))
noErrors = False
# End if toCloud.
if gzipResult:
self.moveToFinalPath(compressedFullPath = compressedFullPath)
# Remove the uncompressed file.
try:
if not testing:
self.logger.log('Removing {}'.format(fullPath))
os.remove('{}'.format(fullPath))
except OSError as error:
self.logger.log(
'Exception while removing {}: {}.'.format(fullPath, error))
noErrors = False
# End for db in databases.
if deleteOutdated:
self.deleteOutdatedFiles(datetime.timedelta(days = int(
self.configer.configOptionValue('Export',
'export_days_to_keep'))))
return uploaded if noErrors else None
def moveToFinalPath(self, compressedFullPath = ''):
"""
Move a compressed final to the final export path.
:param compressedFullPath: String for the compressed file.
:return:
"""
self.logger.log('Moving {} to final path.'.format(compressedFullPath),
'debug')
try:
shutil.move(compressedFullPath,
self.configer.configOptionValue('Export',
'db_export_final_path'))
except Exception as detail:
self.logger.log(
'Exception while moving {} to final export path: {}'.format(
compressedFullPath, detail), 'error')
def md5Verification(self, compressedFullPath = '', fullPath = '',
md5sum1 = ''):
"""
Perform md5 verification of a compressed file at compressedFullPath
where the original file is at fullPath and has md5sum1.
:param compressedFullPath: String
:param fullPath: String
:param md5sum1: String of md5sum of source file.
:return:
"""
GZIP_UNCOMPRESS_FILE = False
if GZIP_UNCOMPRESS_FILE:
# Verify the compressed file by uncompressing it and
# verifying its
# checksum against the original checksum.
self.logger.log('reading: {}'.format(compressedFullPath), 'DEBUG')
self.logger.log('writing: {}'.format(os.path.join(
self.configer.configOptionValue('Testing',
'export_test_data_path'),
os.path.splitext(os.path.basename(fullPath))[0])), 'DEBUG')
self.fileUtil.gzipUncompressFile(compressedFullPath, os.path.join(
self.configer.configOptionValue('Testing',
'export_test_data_path'),
fullPath))
VERIFY_BY_CHECKSUM = False
if VERIFY_BY_CHECKSUM:
md5sum2 = self.fileUtil.md5Checksum(fullPath)
self.logger.log("mtime: {}, md5sum2: {}".format(
time.ctime(os.path.getmtime(fullPath)), md5sum2), 'INFO')
if md5sum1 == md5sum2:
self.logger.log(
'Compressed file has been validated by checksum.', 'INFO')
else:
noErrors = False
def numberOfChunksToUse(self, fullPath):
"""
Return the number of chunks to be used by the file splitter based on
the file size of the file at fullPath.
:param fullPath: String
:returns: Int Number of chunks to create.
"""
fsize = os.path.getsize(fullPath)
self.logger.log('fullpath: {}, fsize: {}'.format(fullPath, fsize))
if (fsize >= int(self.configer.configOptionValue('Export',
'max_bytes_before_split'))):
# Note that this does not make use of the remainder in the division.
chunks = int(fsize / int(self.configer.configOptionValue('Export',
'max_bytes_before_split')))
self.logger.log('Will split with {} chunks.'.format(chunks))
return chunks
self.logger.log('Will NOT split file.', 'debug')
return 1
def uploadFileToCloudStorage(self, fullPath = '', retryCount = 0,
testing = False):
"""
Export a file to cloud storage.
:param fullPath: String of file to be exported.
:param testing: Boolean when set to True, Testing Mode is used.
:param retryCount: Int of number of times to retry the upload if
there is a failure.
:returns: String File ID on verified on upload; None if verification
fails.
"""
success = True
myFile = os.path.basename(fullPath)
self.logger.log(
'full path {}'.format(os.path.dirname(fullPath), 'DEBUG'))
self.logger.log("Uploading {}.".format(myFile))
result = {}
try:
media_body = MediaFileUpload(fullPath,
mimetype =
'application/gzip-compressed',
resumable = True)
body = {'title': myFile,
'description': 'Hawaii Smart Energy Project gzip '
'compressed DB export.',
'mimeType': 'application/gzip-compressed'}
# Result is a Files resource.
result = self.driveService.files().insert(body = body,
media_body =
media_body).execute()
except Exception as detail:
# Upload failures can result in a BadStatusLine.
self.logger.log(
"Exception while uploading {}: {}.".format(myFile, detail),
'error')
success = False
if not self.__verifyMD5Sum(fullPath, self.fileIDForFileName(myFile)):
self.logger.log('Failed MD5 checksum verification.', 'INFO')
success = False
if success:
self.logger.log('Verification by MD5 checksum succeeded.', 'INFO')
self.logger.log("Finished.")
return result['id']
if not success and retryCount <= 0:
return None
else:
time.sleep(self.retryDelay)
self.logger.log('Retrying upload of {}.'.format(fullPath),
'warning')
self.uploadFileToCloudStorage(fullPath = fullPath,
retryCount = retryCount - 1)
def __retrieveCredentials(self):
"""
Perform authorization at the server.
Credentials are loaded into the object attribute googleAPICredentials.
"""
flow = OAuth2WebServerFlow(self.clientID, self.clientSecret,
self.oauthScope, self.oauthConsent)
authorize_url = flow.step1_get_authorize_url()
print 'Go to the following link in your browser: ' + authorize_url
code = raw_input('Enter verification code: ').strip()
self.googleAPICredentials = flow.step2_exchange(code)
print "refresh_token = {}".format(
self.googleAPICredentials.refresh_token)
print "expiry = {}".format(self.googleAPICredentials.token_expiry)
def freeSpace(self):
"""
Get free space from the drive service.
:param driveService: Object for the drive service.
:returns: Int of free space (bytes B) on the drive service.
"""
aboutData = self.driveService.about().get().execute()
return int(aboutData['quotaBytesTotal']) - int(
aboutData['quotaBytesUsed']) - int(
aboutData['quotaBytesUsedInTrash'])
def deleteFile(self, fileID = ''):
"""
Delete the file with ID fileID.
:param fileID: String of a Google API file ID.
"""
if not len(fileID) > 0:
raise Exception("File ID has not been given.")
self.logger.log(
'Deleting file with file ID {} and name {}.'.format(fileID,
self.filenameForFileID(
fileID)),
'debug')
try:
# Writing the fileId arg name is required here.
self.driveService.files().delete(fileId = fileID).execute()
except errors.HttpError as error:
self.logger.log('Exception while deleting: {}'.format(error),
'error')
def deleteOutdatedFiles(self, maxAge = datetime.timedelta(weeks = 9999999)):
"""
Remove outdated files from cloud storage.
:param minAge: datetime.timedelta of the minimum age before a file is
considered outdated.
:param maxAge: datetime.timedelta of the maximum age to consider for
a file.
:returns: Int count of deleted items.
"""
# @todo Return count of actual successfully deleted files.
outdated = self.outdatedFiles(maxAge)
""":type : dict"""
for f in outdated:
self.deleteFile(f['id'])
return len(outdated)
def outdatedFiles(self,
daysBeforeOutdated = datetime.timedelta(days = 9999999)):
"""
Outdated files in the cloud where they are outdated if their age is
greater than or equal to daysBeforeOutdated.
Note: When t1 is the same day as t2, the timedelta comes back as -1.
Not sure why this isn't represented as zero. Perhaps to avoid a false
evaluation of a predicate on a tdelta.
:param daysBeforeOutdated: datetime.timedelta where the value
indicates that outdated files that have an age greater than this
parameter.
:return: Int count of deleted items.
"""
t1 = lambda x: datetime.datetime.strptime(x['createdDate'],
"%Y-%m-%dT%H:%M:%S.%fZ")
t2 = datetime.datetime.now()
return filter(lambda x: t2 - t1(x) >= daysBeforeOutdated,
self.cloudFiles['items'])
def sendNotificationOfFiles(self):
"""
Provide a notification that lists the export files along with sharing
links.
"""
pass
def sendDownloadableFiles(self):
"""
Send available files via HTTP POST.
:returns: None
"""
myPath = '{}/{}'.format(self.exportTempWorkPath,
'list-of-downloadable-files.txt')
fp = open(myPath, 'wb')
output = StringIO()
output.write(self.markdownListOfDownloadableFiles())
fp.write(self.markdownListOfDownloadableFiles())
fp.close()
headers = {'User-Agent': self.postAgent, 'Content-Type': 'text/html'}
try:
r = requests.post(self.configer.configOptionValue('Export',
'export_list_post_url'),
output.getvalue(), headers = headers)
print 'text: {}'.format(r.text)
except requests.adapters.SSLError as error:
# @todo Implement alternative verification.
self.logger.log('SSL error: {}'.format(error), 'error')
output.close()
def metadataOfFileID(self, fileID = ''):
"""
:param fileID: String of a file ID in the cloud.
:return: Tuple of metadata (name, url, timestamp, size) for a given
file ID.
"""
item = [i for i in self.cloudFiles['items'] if i['id'] == fileID][0]
return (item[u'originalFilename'], item[u'webContentLink'],
item[u'createdDate'], item[u'fileSize'])
def listOfDownloadableFiles(self):
"""
Create a list of downloadable files.
:returns: List of dicts of files that are downloadable from the cloud.
"""
files = []
for i in reversed(sorted(self.cloudFiles['items'],
key = lambda k: k['createdDate'])):
item = dict()
item['title'] = i['title']
item['webContentLink'] = i['webContentLink']
item['id'] = i['id']
item['createdDate'] = i['createdDate']
item['fileSize'] = i['fileSize']
files.append(item)
return files
def markdownListOfDownloadableFiles(self):
"""
Generate content containing a list of downloadable files in Markdown
format.
:returns: String content in Markdown format.
"""
content = "||*Name*||*Created*||*Size*||\n"
for i in self.listOfDownloadableFiles():
content += "||[`{}`]({})".format(i['title'], i['webContentLink'])
content += "||`{}`".format(i['createdDate'])
content += "||`{} B`||".format(int(i['fileSize']))
content += '\n'
# self.logger.log('content: {}'.format(content))
return content
def plaintextListOfDownloadableFiles(self):
"""
Generate content containing a list of downloadable files in plaintext
format.
:returns: String content as plaintext.
"""
content = ''
includeLink = False
for i in reversed(sorted(self.cloudFiles['items'],
key = lambda k: k['createdDate'])):
if includeLink:
content += "{}, {}, {}, {} B\n".format(i['title'],
i['webContentLink'],
i['createdDate'],
int(i['fileSize']))
else:
content += "{}, {}, {} B\n".format(i['title'], i['createdDate'],
int(i['fileSize']))
return content
def logSuccessfulExport(self, name = '', url = '', datetime = 0, size = 0):
"""
When an export has been successful, log information about the export
to the database.
The items to log include:
* filename
* URL
* timestamp
* filesize
:param name: String
:param url: String
:param datetime:
:param size: Int
:return: True if no errors occurred, else False.
"""
def exportHistoryColumns():
return ['name', 'url', 'timestamp', 'size']
timestamp = lambda \
datetime: 'to_timestamp(0)' if datetime == 0 else "timestamp " \
"'{}'".format(
datetime)
sql = 'INSERT INTO "{0}" ({1}) VALUES ({2}, {3}, {4}, {5})'.format(
self.configer.configOptionValue('Export', 'export_history_table'),
','.join(exportHistoryColumns()), "'" + name + "'", "'" + url + "'",
timestamp(datetime), size)
conn = MSGDBConnector().connectDB()
cursor = conn.cursor()
dbUtil = MSGDBUtil()
result = dbUtil.executeSQL(cursor, sql, exitOnFail = False)
conn.commit()
return result
def sendExportSummary(self, summary = ''):
"""
Send a summary of exports via email to a preconfigured list of
recipients.
:param summary: String of summary content.
:return:
"""
try:
if self.notifier.sendNotificationEmail(summary, testing = False):
self.notifier.recordNotificationEvent(
types = MSGNotificationHistoryTypes,
noticeType = MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
except Exception as detail:
self.logger.log('Exception occurred: {}'.format(detail), 'ERROR')
def currentExportSummary(self):
"""
Current summary of exports since the last summary report time.
Summaries are reported with identifier MSG_EXPORT_SUMMARY in the
NotificationHistory.
Includes:
* Number of databases exported
* Total number of files in the cloud.
* A report of available storage capacity.
* A list of available DBs.
* A link where exports can be accessed.
:return: String of summary text.
"""
availableFilesURL = self.configer.configOptionValue('Export',
'export_list_url')
lastReportDate = self.notifier.lastReportDate(
types = MSGNotificationHistoryTypes,
noticeType = MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
content = 'Cloud Export Summary:\n\n'
content += 'Last report date: {}\n'.format(lastReportDate)
# @TO BE REVIEWED: Verify time zone adjustment.
content += '{} databases have been exported since the last report ' \
'date.\n'.format(self.countOfDBExports(
lastReportDate + datetime.timedelta(
hours = 10)) if lastReportDate else self.countOfDBExports())
content += '{} B free space is available.\n'.format(self.freeSpace())
content += '\nCurrently available DBs:\n'
content += self.plaintextListOfDownloadableFiles()
content += '\n{} files can be accessed through Google Drive (' \
'https://drive.google.com) or at {}.'.format(
self.countOfCloudFiles(), availableFilesURL)
return content
def countOfDBExports(self, since = None):
"""
:param since: datetime indicating last export datetime.
:return: Int of count of exports.
"""
myDatetime = lambda x: datetime.datetime.strptime(x, '%Y-%m-%d %H:%S')
if not since:
since = myDatetime('1900-01-01 00:00')
self.logger.log(since.strftime('%Y-%m-%d %H:%M'), 'DEBUG')
sql = 'SELECT COUNT("public"."ExportHistory"."timestamp") FROM ' \
'"public"."ExportHistory" WHERE "timestamp" > \'{}\''.format(
since.strftime('%Y-%m-%d %H:%M'))
conn = MSGDBConnector().connectDB()
cursor = conn.cursor()
dbUtil = MSGDBUtil()
rows = None
if dbUtil.executeSQL(cursor, sql, exitOnFail = False):
rows = cursor.fetchall()
assert len(rows) == 1, 'Invalid return value.'
return rows[0][0]
def countOfCloudFiles(self):
"""
:param since: datetime indicating last trailing export datetime.
:return: Int of count of exports.
"""
return len(self.cloudFiles['items'])
def __verifyMD5Sum(self, localFilePath, remoteFileID):
"""
Verify that the local MD5 sum matches the MD5 sum for the remote file
corresponding to an ID.
This verifies that the uploaded file matches the local compressed
export file.
:param localFilePath: String of the full path of the local file.
:param remoteFileID: String of the cloud ID for the remote file.
:returns: Boolean True if the MD5 sums match, otherwise, False.
"""
self.logger.log('remote file ID: {}'.format(remoteFileID))
self.logger.log('local file path: {}'.format(localFilePath))
# Get the md5sum for the local file.
f = open(localFilePath, mode = 'rb')
fContent = hashlib.md5()
for buf in iter(partial(f.read, 128), b''):
fContent.update(buf)
localMD5Sum = fContent.hexdigest()
f.close()
self.logger.log('local md5: {}'.format(localMD5Sum), 'DEBUG')
def verifyFile():
# Get the MD5 sum for the remote file.
for item in self.cloudFiles['items']:
if (item['id'] == remoteFileID):
self.logger.log(
'remote md5: {}'.format(item['md5Checksum']), 'DEBUG')
if localMD5Sum == item['md5Checksum']:
return True
else:
return False
try:
if verifyFile():
return True
else:
return False
except errors.HttpError as detail:
self.logger.log('HTTP error during MD5 verification.', 'error')
time.sleep(10)
if verifyFile():
return True
else:
return False
def fileIDForFileName(self, filename):
"""
Get the file ID for the given filename.
This method supports matching multiple cloud filenames but only
returns the ID for a single matching filename.
This can then be called recursively to obtain all the file IDs for a
given filename.
:param String of the filename for which to retrieve the ID.
:returns: String of a cloud file ID or None if no match.
"""
fileIDList = filter(lambda x: x['originalFilename'] == filename,
self.cloudFiles['items'])
return fileIDList[0]['id'] if len(fileIDList) > 0 else None
def filenameForFileID(self, fileID = ''):
"""
:param fileID: String of cloud-based file ID.
:return: String of filename for a given file ID.
"""
return filter(lambda x: x['id'] == fileID, self.cloudFiles['items'])[0][
'originalFilename']
def addReaders(self, fileID = None, emailAddressList = None,
retryCount = 0):
"""
Add reader permission to an export file that has been uploaded to the
cloud for the given list of email addresses.
Email notification is suppressed by default.
:param fileID: String of the cloud file ID to be processed.
:param emailAddressList: List of email addresses.
:returns: Boolean True if successful, otherwise False.
"""
# @todo Provide support for retry count
success = True
self.logger.log('file id: {}'.format(fileID))
self.logger.log('address list: {}'.format(emailAddressList))
for addr in emailAddressList:
permission = {'value': addr, 'type': 'user', 'role': 'reader'}
if fileID:
try:
resp = self.driveService.permissions().insert(
fileId = fileID, sendNotificationEmails = False,
body = permission).execute()
self.logger.log(
'Reader permission added for {}.'.format(addr))
except errors.HttpError as error:
self.logger.log('An error occurred: {}'.format(error))
success = False
if not success and retryCount <= 0:
return False
elif success:
return True
else:
time.sleep(self.retryDelay)
self.logger.log('Retrying adding readers for ID {}.'.format(fileID),
'warning')
self.addReaders(fileID = fileID,
emailAddressList = emailAddressList,
retryCount = retryCount - 1)
class MSGDBExporterTester(unittest.TestCase):
"""
Unit tests for the MSG Cloud Exporter.
"""
def setUp(self):
self.logger = SEKLogger(__name__, 'DEBUG')
self.configer = MSGConfiger()
self.exporter = MSGDBExporter()
self.testDir = 'db_exporter_test'
self.uncompressedTestFilename = 'meco_v3_test_data.sql'
self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
self.exportTestDataPath = self.configer.configOptionValue(
'Testing', 'export_test_data_path')
self.fileUtil = MSGFileUtil()
self.fileChunks = []
self.testDataFileID = ''
self.pyUtil = MSGPythonUtil()
self.timeUtil = MSGTimeUtil()
conn = None
try:
conn = MSGDBConnector().connectDB()
except Exception as detail:
self.logger.log("Exception occurred: {}".format(detail), 'error')
exit(-1)
self.logger.log("conn = {}".format(conn), 'debug')
self.assertIsNotNone(conn)
# Create a temporary working directory.
try:
os.mkdir(self.testDir)
except OSError as detail:
self.logger.log(
'Exception during creation of temp directory: %s' % detail,
'ERROR')
def tearDown(self):
"""
Delete all test items.
"""
REMOVE_TEMPORARY_FILES = True
if REMOVE_TEMPORARY_FILES:
try:
self.logger.log(
"Removing local test files {}, {}.".format(
self.uncompressedTestFilename,
self.compressedTestFilename), 'debug')
os.remove(
os.path.join(os.getcwd(), self.testDir,
self.uncompressedTestFilename))
os.remove(
os.path.join(os.getcwd(), self.testDir,
self.compressedTestFilename))
except OSError as detail:
self.logger.log(
'Exception while removing temporary files: {}'.format(
detail), 'SILENT')
try:
os.remove(
os.path.join(os.getcwd(), self.testDir,
self.compressedTestFilename))
except OSError as detail:
self.logger.log(
'Exception while removing temporary files: {}'.format(
detail), 'SILENT')
try:
for f in self.fileChunks:
os.remove(f)
except OSError as detail:
self.logger.log(
'Exception while removing temporary files: {}'.format(
detail), 'DEBUG')
try:
os.rmdir(self.testDir)
except OSError as detail:
self.logger.log(
'Exception while removing directory: {}'.format(detail),
'ERROR')
# Keep deleting from the cloud until there are no more to delete.
def deleteFromCloud():
self.logger.log("deleting from cloud", 'debug')
try:
fileIDToDelete = self.exporter.fileIDForFileName(
self.compressedTestFilename)
if fileIDToDelete is None:
return
self.logger.log("file ID to delete: {}".format(fileIDToDelete),
'DEBUG')
self.exporter.driveService.files().delete(
fileId='{}'.format(fileIDToDelete)).execute()
deleteFromCloud()
except (TypeError, http.HttpError) as e:
self.logger.log('Delete not successful: {}'.format(e), 'DEBUG')
deleteFromCloud()
def _upload_test_data_to_cloud(self):
"""
Provide an upload of test data that can be used in other tests.
Side effect: Store the file ID as an ivar.
"""
self.logger.log("Uploading test data for caller: {}".format(
self.pyUtil.callerName()))
filePath = "{}/{}".format(self.exportTestDataPath,
self.compressedTestFilename)
self.logger.log('Uploaded {}.'.format(filePath), 'info')
uploadResult = self.exporter.uploadFileToCloudStorage(filePath)
self.logger.log('upload result: {}'.format(uploadResult))
self.testDataFileID = self.exporter.fileIDForFileName(
self.compressedTestFilename)
self.logger.log("Test file ID is {}.".format(self.testDataFileID))
def test_markdown_list_of_downloadable_files(self):
"""
Match the Markdown line entry for the uploaded file.
"""
# @REVIEWED
self._upload_test_data_to_cloud()
self.assertEquals(
len(
filter(
lambda x: self.testDataFileID in x,
self.exporter.markdownListOfDownloadableFiles().splitlines(
))), 1)
def test_get_md5_sum_from_cloud(self):
"""
Test retrieving the MD5 sum from the cloud.
"""
# @REVIEWED
self.logger.log('Testing getting the MD5 sum.', 'info')
self._upload_test_data_to_cloud()
testFileMD5 = filter(
lambda x: x['id'] == self.testDataFileID,
self.exporter.cloudFiles['items'])[0]['md5Checksum']
self.assertEquals(len(testFileMD5), 32)
self.assertTrue(re.match(r'[0-9A-Za-z]+', testFileMD5))
def test_get_file_id_for_nonexistent_file(self):
"""
Test getting a file ID for a nonexistent file.
"""
# @REVIEWED
fileIDs = self.exporter.fileIDForFileName('nonexistent_file')
self.logger.log("file ids = {}".format(fileIDs), 'info')
self.assertIsNone(fileIDs)
def test_upload_test_data(self):
"""
Upload a test data file for unit testing of DB export.
The unit test data file is a predefined set of test data stored in
the test data path of the software distribution.
"""
# @REVIEWED
self._upload_test_data_to_cloud()
self.assertGreater(len(self.testDataFileID), 0)
self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))
def test_adding_reader_permissions(self):
"""
Add reader permissions to a file that was uploaded.
"""
# @REVIEWED
self.logger.log("Testing adding reader permissions.")
self._upload_test_data_to_cloud()
email = self.configer.configOptionValue('Testing',
'tester_email_address')
service = self.exporter.driveService
try:
id_resp = service.permissions().getIdForEmail(
email=email).execute()
print id_resp
except errors.HttpError as detail:
print 'Exception while getting ID for email: {}'.format(detail)
new_permission = {'value': email, 'type': 'user', 'role': 'reader'}
try:
self.logger.log('Adding reader permission', 'INFO')
fileIDToAddTo = self.testDataFileID
# The permission dict is being output to stdout here.
resp = service.permissions().insert(fileId=fileIDToAddTo,
sendNotificationEmails=False,
body=new_permission).execute()
except errors.HttpError as detail:
self.logger.log(
'Exception while adding reader permissions: {}'.format(detail),
'error')
def permission_id(email):
try:
id_resp = service.permissions().getIdForEmail(
email=email).execute()
return id_resp['id']
except errors.HttpError as error:
self.logger.log("HTTP error: {}".format(error))
permission = {}
try:
permission = service.permissions().get(
fileId=self.testDataFileID,
permissionId=permission_id(email)).execute()
except errors.HttpError as error:
self.logger.log("HTTP error: {}".format(error))
self.assertEquals(permission['role'], 'reader')
def test_create_compressed_archived(self):
"""
* Copy test data to a temp directory (self.testDir).
* Create a checksum for test data.
* Create a gzip-compressed archive.
* Extract gzip-compressed archive.
* Create a checksum for the uncompressed data.
* Compare the checksums.
"""
# @REVIEWED
self.logger.log('Testing verification of a compressed archive.')
self.logger.log('cwd {}'.format(os.getcwd()))
fullPath = '{}'.format(
os.path.join(os.getcwd(), self.testDir,
self.uncompressedTestFilename))
shutil.copyfile(
'{}/{}'.format(self.exportTestDataPath,
self.uncompressedTestFilename), fullPath)
md5sum1 = self.fileUtil.md5Checksum(fullPath)
self.exporter.fileUtil.gzipCompressFile(fullPath)
try:
os.remove(
os.path.join(os.getcwd(), self.testDir,
self.uncompressedTestFilename))
except OSError as detail:
self.logger.log('Exception while removing: {}'.format(detail),
'ERROR')
# Extract archived data and generate checksum.
src = gzip.open('{}{}'.format(fullPath, '.gz'), "rb")
uncompressed = open(fullPath, "wb")
decoded = src.read()
uncompressed.write(decoded)
uncompressed.close()
md5sum2 = self.fileUtil.md5Checksum(fullPath)
self.assertEqual(
md5sum1, md5sum2, 'Checksums are not equal for original and new '
'decompressed archive.')
def test_export_db(self):
"""
Perform a quick test of the DB export method using Testing Mode.
This requires sudo authorization to complete.
"""
# @REVIEWED
self.logger.log('Testing exportDB using the testing DB.')
# @todo handle case where testing db does not exist.
dbs = ['test_meco']
ids = self.exporter.exportDBs(databases=dbs,
toCloud=True,
localExport=True)
self.logger.log('Count of exports: {}'.format(len(ids)))
self.assertEquals(len(ids), 1, "Count of exported files is wrong.")
map(self.exporter.deleteFile, ids)
def test_split_archive(self):
"""
Test splitting an archive into chunks.
"""
# @REVIEWED
self.logger.log('Testing archive splitting.')
fullPath = '{}/{}'.format(self.exportTestDataPath,
self.compressedTestFilename)
self.logger.log('fullpath: {}'.format(fullPath))
shutil.copyfile(
fullPath, '{}/{}'.format(self.testDir,
self.compressedTestFilename))
fullPath = '{}/{}'.format(self.testDir, self.compressedTestFilename)
self.fileChunks = self.fileUtil.splitLargeFile(fullPath=fullPath,
numChunks=3)
self.assertEquals(len(self.fileChunks), 3)
def test_get_file_size(self):
"""
Test retrieving local file sizes.
"""
# @REVIEWED
fullPath = '{}/{}'.format(self.exportTestDataPath,
self.compressedTestFilename)
fSize = self.fileUtil.fileSize(fullPath)
self.logger.log('size: {}'.format(fSize))
self.assertEqual(fSize, 12279, 'File size is correct.')
def test_upload_export_files_list(self):
"""
TBW
"""
pass
def test_checksum_after_upload(self):
"""
TBW
"""
pass
def test_dump_exclusions_dictionary(self):
"""
Verify the exclusions dictionary by its type.
:return:
"""
# @REVIEWED
exclusions = self.exporter.dumpExclusionsDictionary()
if exclusions:
self.assertEquals(type({}), type(exclusions))
def test_move_to_final(self):
"""
Test moving a file to the final destination path.
"""
# @REVIEWED
self.logger.log('Testing moving to final path {}.'.format(
self.configer.configOptionValue('Export', 'db_export_final_path')))
origCompressedFile = '{}/{}'.format(
self.configer.configOptionValue('Export', 'export_test_data_path'),
self.compressedTestFilename)
newCompressedFile = '{}/{}'.format(
self.configer.configOptionValue('Export', 'export_test_data_path'),
'temp_test_file')
shutil.copyfile(origCompressedFile, newCompressedFile)
self.exporter.moveToFinalPath(compressedFullPath=newCompressedFile)
self.assertTrue(
os.path.isfile('{}/{}'.format(
self.configer.configOptionValue('Export',
'db_export_final_path'),
'temp_test_file')))
# Remove the test file.
os.remove('{}/{}'.format(
self.configer.configOptionValue('Export', 'db_export_final_path'),
'temp_test_file'))
def test_log_successful_export(self):
"""
Test logging of export results to the export history table.
"""
# @REVIEWED
self.assertTrue(
self.exporter.logSuccessfulExport(name='test_export',
url='http://test_url',
datetime=0,
size=100))
conn = MSGDBConnector().connectDB()
cursor = conn.cursor()
dbUtil = MSGDBUtil()
self.assertTrue(
dbUtil.executeSQL(
cursor, 'select * from "ExportHistory" where '
'timestamp = '
'to_timestamp(0)'))
self.assertEqual(len(cursor.fetchall()), 1,
"There should only be one result row.")
self.assertTrue(
dbUtil.executeSQL(
cursor, 'delete from "ExportHistory" where '
'timestamp = to_timestamp(0)'))
conn.commit()
def test_metadata_of_file_id(self):
"""
Test getting the metadata for a file ID.
"""
# @REVIEWED
self._upload_test_data_to_cloud()
self.logger.log('metadata: {}'.format(
self.exporter.metadataOfFileID(self.testDataFileID)))
self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))
def test_filename_for_file_id(self):
"""
Test returning a file name given a file ID.
"""
# @REVIEWED
self._upload_test_data_to_cloud()
self.assertEquals(
self.exporter.filenameForFileID(fileID=self.testDataFileID),
self.compressedTestFilename)
def test_outdated_files(self):
# @REVIEWED
self._upload_test_data_to_cloud()
time.sleep(1)
self.logger.log("outdated:")
# For debugging:
for item in self.exporter.outdatedFiles(
daysBeforeOutdated=datetime.timedelta(days=-1)):
self.logger.log(
"name: {}, created date: {}".format(item['originalFilename'],
item['createdDate']),
'debug')
# Get all the outdated files where outdated is equal to anything
# uploaded today or later.
self.assertTrue(
self.exporter.outdatedFiles(daysBeforeOutdated=datetime.timedelta(
days=-1))[0]['id'] == self.testDataFileID)
self.logger.log('-----')
def test_delete_outdated(self):
"""
TBW
"""
pass
def test_list_of_downloadable_files(self):
"""
Test the list of downloadable files used by the available files page.
"""
# @REVIEWED
self._upload_test_data_to_cloud()
self.assertEquals(
len(
filter(lambda row: row['id'] == self.testDataFileID,
self.exporter.listOfDownloadableFiles())), 1,
"Test file not present.")
def test_count_of_db_exports(self):
count = self.exporter.countOfDBExports(EARLIEST_DATE)
self.logger.log(count, 'DEBUG')
self.assertTrue(int(count) or int(count) == int(0))
def test_count_of_cloud_files(self):
count = self.exporter.countOfCloudFiles()
self.assertTrue(int(count) or int(count) == int(0))
def test_plaintext_list_of_downloadable_files(self):
"""
This test handles content both with content links and without content
links.
"""
content = self.exporter.plaintextListOfDownloadableFiles()
self.assertRegexpMatches(
content, '\d+-\d+-\d+.*\,'
'\s+\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z\,\s+\d+\sB')
def test_last_report_date(self):
last_report = self.exporter.notifier.lastReportDate(
types=MSGNotificationHistoryTypes,
noticeType=MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
self.assertTrue(last_report is None or last_report > EARLIEST_DATE)
def test_current_export_summary(self):
self.assertRegexpMatches(
self.exporter.currentExportSummary(),
re.compile('last.*databases.*free.*currently.*accessed.*',
flags=re.IGNORECASE | re.DOTALL))
class MSGDBExporterTester(unittest.TestCase):
"""
Unit tests for the MSG Cloud Exporter.
"""
def setUp(self):
self.logger = MSGLogger(__name__, 'DEBUG')
self.configer = MSGConfiger()
self.exporter = MSGDBExporter()
self.testDir = 'db_exporter_test'
self.uncompressedTestFilename = 'meco_v3_test_data.sql'
self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
self.exportTestDataPath = self.configer.configOptionValue('Testing',
'export_test_data_path')
self.fileUtil = MSGFileUtil()
self.fileChunks = []
self.testDataFileID = ''
self.pyUtil = MSGPythonUtil()
conn = None
try:
conn = MSGDBConnector().connectDB()
except Exception as detail:
self.logger.log("Exception occurred: {}".format(detail), 'error')
exit(-1)
self.logger.log("conn = {}".format(conn), 'debug')
self.assertIsNotNone(conn)
# Create a temporary working directory.
try:
os.mkdir(self.testDir)
except OSError as detail:
self.logger.log(
'Exception during creation of temp directory: %s' % detail,
'ERROR')
def upload_test_data_to_cloud(self):
"""
Provide an upload of test data that can be used in other tests.
Side effect: Store the file ID as an ivar.
"""
self.logger.log("Uploading test data for caller: {}".format(
self.pyUtil.caller_name()))
filePath = "{}/{}".format(self.exportTestDataPath,
self.compressedTestFilename)
self.logger.log('Uploaded {}.'.format(filePath), 'info')
uploadResult = self.exporter.uploadFileToCloudStorage(filePath)
self.logger.log('upload result: {}'.format(uploadResult))
self.testDataFileID = self.exporter.fileIDForFileName(
self.compressedTestFilename)
self.logger.log("Test file ID is {}.".format(self.testDataFileID))
def test_sending_fcphase_part_0(self):
"""
/home/daniel/msg-db-dumps/2014-05-14_141223_fcphase3.sql.gz.0
"""
filesToUpload = [
'/home/daniel/msg-db-dumps/2014-05-14_141223_fcphase3.sql.gz.0',
'/home/daniel/msg-db-dumps/2014-05-14_141223_fcphase3.sql.gz.1',
'/home/daniel/msg-db-dumps/2014-05-14_141223_fcphase3.sql.gz.2',
'/home/daniel/msg-db-dumps/2014-05-14_141223_fcphase3.sql.gz.3']
for f in filesToUpload:
self.exporter.uploadFileToCloudStorage(fullPath = f,
testing = False)
def testListRemoteFiles(self):
"""
Test listing of remote files.
"""
self.logger.log('Testing listing of remote files.', 'INFO')
title = ''
id = ''
for item in self.exporter.cloudFiles['items']:
title = item['title']
id = item['id']
self.assertIsNot(title, '')
self.assertIsNot(id, '')
def testDownloadURLList(self):
"""
Test obtaining a list of downloadble URLs.
"""
self.logger.log('Testing listing of downloadable files.', 'INFO')
title = ''
id = ''
url = ''
for item in self.exporter.cloudFiles['items']:
title = item['title']
url = item['webContentLink']
id = item['id']
self.logger.log('title: %s, link: %s, id: %s' % (title, url, id))
self.assertIsNot(title, '')
self.assertIsNot(url, '')
self.assertIsNot(id, '')
def test_list_of_downloadable_files(self):
"""
Test the list of downloadable files used by the available files page.
"""
self.assertIsNotNone(self.exporter.listOfDownloadableFiles(),
'List of downloadable files is not available.')
for row in self.exporter.listOfDownloadableFiles():
print row
self.assertIsNotNone(row['id'])
self.assertIsNotNone(row['title'])
self.assertIsNotNone(row['webContentLink'])
def test_markdown_list_of_downloadable_files(self):
print self.exporter.markdownListOfDownloadableFiles()
myPath = '{}/{}'.format(
self.configer.configOptionValue('Export', 'db_export_path'),
'list-of-downloadable-files.txt')
fp = open(myPath, 'wb')
fp.write(self.exporter.markdownListOfDownloadableFiles())
fp.close()
def test_get_md5_sum_from_cloud(self):
"""
Test retrieving the MD5 sum from the cloud.
"""
# @REVIEWED
self.logger.log('Testing getting the MD5 sum.', 'info')
self.upload_test_data_to_cloud()
testFileMD5 = filter(lambda x: x['id'] == self.testDataFileID,
self.exporter.cloudFiles['items'])[0][
'md5Checksum']
self.assertEquals(len(testFileMD5), 32)
self.assertTrue(re.match(r'[0-9A-Za-z]+', testFileMD5))
def testGetFileIDsForFilename(self):
"""
Retrieve the matching file IDs for the given file name.
"""
self.logger.log('Testing getting file IDs for a filename.')
self.logger.log("Uploading test data.")
filePath = "{}/{}".format(self.exportTestDataPath,
self.compressedTestFilename)
uploadResult = self.exporter.uploadFileToCloudStorage(filePath)
self.assertTrue(uploadResult)
self.logger.log('Testing getting the file ID for a filename.')
fileIDs = self.exporter.fileIDForFileName(self.compressedTestFilename)
self.logger.log("file ids = {}".format(fileIDs), 'info')
self.assertIsNotNone(fileIDs)
def test_get_file_id_for_nonexistent_file(self):
"""
Test getting a file ID for a nonexistent file.
"""
fileIDs = self.exporter.fileIDForFileName('nonexistent_file')
self.logger.log("file ids = {}".format(fileIDs), 'info')
self.assertIsNone(fileIDs)
def test_upload_test_data(self):
"""
Upload a test data file for unit testing of DB export.
The unit test data file is a predefined set of test data stored in
the test data path of the software distribution.
"""
# @REVIEWED
self.upload_test_data_to_cloud()
self.assertGreater(len(self.testDataFileID), 0)
self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))
def test_delete_out_dated_files(self):
"""
The timestamp of an uploaded file should be set in the past to provide
the ability to test the deleting of outdated files.
"""
# return
# @TO BE REVIEWED Prevent deleting files uploaded today.
# @IMPORTANT Prevent deleting NON-testing files.
# Need to have a test file uploaded that has an explicitly set upload
# date.
self.logger.log("Test deleting outdated files.")
self.logger.log("Uploading test data.")
filePath = "%s/%s" % (
self.exportTestDataPath, self.compressedTestFilename)
uploadResult = self.exporter.uploadFileToCloudStorage(filePath)
cnt = self.exporter.deleteOutdatedFiles(
minAge = datetime.timedelta(days = 5),
maxAge = datetime.timedelta(days = 99999))
# self.assertGreater(cnt, 0)
def testAddingReaderPermissions(self):
"""
Add reader permissions to a file that was uploaded.
@todo Needs update after cloud export restoration.
"""
self.logger.log("Testing adding reader permissions.")
self.logger.log("Uploading test data.")
filePath = "%s/%s" % (
self.exportTestDataPath, self.compressedTestFilename)
uploadResult = self.exporter.uploadFileToCloudStorage(filePath)
email = self.configer.configOptionValue('Testing', 'tester_email')
service = self.exporter.driveService
try:
id_resp = service.permissions().getIdForEmail(
email = email).execute()
print id_resp
except errors.HttpError as detail:
print 'Exception while getting ID for email: %s' % detail
new_permission = {'value': email, 'type': 'user', 'role': 'reader'}
try:
self.logger.log('Adding reader permission', 'INFO')
fileIDToAddTo = self.exporter.fileIDForFileName(
self.compressedTestFilename)
# The permission dict is being output to stdout here.
resp = service.permissions().insert(fileId = fileIDToAddTo,
sendNotificationEmails = False,
body = new_permission).execute()
except errors.HttpError as detail:
self.logger.log(
'Exception while adding reader permissions: %s' % detail,
'error')
def test_create_compressed_archived(self):
"""
* Copy test data to a temp directory.
* Create a checksum for test data.
* Create a gzip-compressed archive.
* Extract gzip-compressed archive.
* Create a checksum for the uncompressed data.
* Compare the checksums.
@todo Needs update after cloud export restoration.
"""
self.logger.log('Testing verification of a compressed archive.')
self.logger.log('cwd {}'.format(os.getcwd()))
fullPath = '{}'.format(os.path.join(os.getcwd(), self.testDir,
self.uncompressedTestFilename))
shutil.copyfile('{}/{}'.format(self.exportTestDataPath,
self.uncompressedTestFilename), fullPath)
md5sum1 = self.fileUtil.md5Checksum(fullPath)
self.exporter.fileUtil.gzipCompressFile(fullPath)
try:
os.remove(os.path.join(os.getcwd(), self.testDir,
self.uncompressedTestFilename))
except OSError as detail:
self.logger.log('Exception while removing: {}'.format(detail),
'ERROR')
# Extract archived data and generate checksum.
src = gzip.open('{}{}'.format(fullPath, '.gz'), "rb")
uncompressed = open(fullPath, "wb")
decoded = src.read()
uncompressed.write(decoded)
uncompressed.close()
md5sum2 = self.fileUtil.md5Checksum(fullPath)
self.assertEqual(md5sum1, md5sum2,
'Checksums are not equal for original and new '
'decompressed archive.')
def test_export_db(self):
"""
Perform a quick test of the DB export method using Testing Mode.
@todo This needs a static test database!
"""
self.logger.log('Testing exportDB')
dbs = ['test_meco']
success = self.exporter.exportDB(databases = dbs, toCloud = True,
localExport = True, numChunks = 4)
self.logger.log('Success: {}'.format(success))
self.assertTrue(success, "Export was not successful.")
def test_split_archive(self):
"""
Test splitting an archive into chunks.
"""
# @REVIEWED
self.logger.log('Testing archive splitting.')
fullPath = '{}/{}'.format(self.exportTestDataPath,
self.compressedTestFilename)
self.logger.log('fullpath: {}'.format(fullPath))
shutil.copyfile(fullPath, '{}/{}'.format(self.testDir,
self.compressedTestFilename))
fullPath = '{}/{}'.format(self.testDir, self.compressedTestFilename)
self.fileChunks = self.fileUtil.splitLargeFile(fullPath = fullPath,
numChunks = 3)
self.assertEquals(len(self.fileChunks), 3)
def test_get_file_size(self):
"""
Test retrieving local file sizes.
"""
# @REVIEWED
fullPath = '{}/{}'.format(self.exportTestDataPath,
self.compressedTestFilename)
fSize = self.fileUtil.fileSize(fullPath)
self.logger.log('size: {}'.format(fSize))
self.assertEqual(fSize, 12279, 'File size is correct.')
def test_upload_export_files_list(self):
"""
TBW
"""
self.exporter.sendDownloadableFiles()
def test_checksum_after_upload(self):
pass
def test_dump_exclusions_dictionary(self):
"""
Verify the exclusions dictionary by its type.
:return:
"""
# @REVIEWED
exclusions = self.exporter.dumpExclusionsDictionary()
if exclusions:
self.assertEquals(type({}), type(exclusions))
def test_plaintext_downloadable_files(self):
print self.exporter.plaintextListOfDownloadableFiles()
def test_move_to_final(self):
"""
Test moving a file to the final destination path.
"""
# @REVIEWED
self.logger.log('Testing moving to final path {}.'.format(
self.configer.configOptionValue('Export', 'db_export_final_path')))
origCompressedFile = '{}/{}'.format(
self.configer.configOptionValue('Export', 'export_test_data_path'),
self.compressedTestFilename)
newCompressedFile = '{}/{}'.format(
self.configer.configOptionValue('Export', 'export_test_data_path'),
'temp_test_file')
shutil.copyfile(origCompressedFile, newCompressedFile)
self.exporter.moveToFinalPath(compressedFullPath = newCompressedFile)
self.assertTrue(os.path.isfile('{}/{}'.format(
self.configer.configOptionValue('Export', 'db_export_final_path'),
'temp_test_file')))
# Remove the test file.
os.remove('{}/{}'.format(
self.configer.configOptionValue('Export', 'db_export_final_path'),
'temp_test_file'))
def test_log_successful_export(self):
"""
Test logging of export results to the export history table.
"""
# @REVIEWED
self.assertTrue(self.exporter.logSuccessfulExport(name = 'test_export',
url =
'http://test_url',
datetime = 0,
size = 100))
conn = MSGDBConnector().connectDB()
cursor = conn.cursor()
dbUtil = MSGDBUtil()
self.assertTrue(
dbUtil.executeSQL(cursor, 'select * from "ExportHistory" where '
'timestamp = '
'to_timestamp(0)'))
self.assertEqual(len(cursor.fetchall()), 1,
"There should only be one result row.")
self.assertTrue(
dbUtil.executeSQL(cursor, 'delete from "ExportHistory" where '
'timestamp = to_timestamp(0)'))
conn.commit()
def test_metadata_of_file_id(self):
"""
Test getting the metadata for a file ID.
"""
# @REVIEWED
self.upload_test_data_to_cloud()
self.logger.log('metadata: {}'.format(
self.exporter.metadataOfFileID(self.testDataFileID)))
self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))
def test_filename_for_file_id(self):
"""
Test returning a file name given a file ID.
"""
# @REVIEWED
self.upload_test_data_to_cloud()
self.assertEquals(
self.exporter.filenameForFileID(fileID = self.testDataFileID),
self.compressedTestFilename)
def tearDown(self):
"""
Delete all test items.
"""
REMOVE_TEMPORARY_FILES = True
if REMOVE_TEMPORARY_FILES:
try:
os.remove(os.path.join(os.getcwd(), self.testDir,
self.uncompressedTestFilename))
os.remove(os.path.join(os.getcwd(), self.testDir,
self.compressedTestFilename))
except OSError as detail:
self.logger.log(
'Exception while removing temporary files: {}'.format(
detail), 'SILENT')
try:
os.remove(os.path.join(os.getcwd(), self.testDir,
self.compressedTestFilename))
except OSError as detail:
self.logger.log(
'Exception while removing temporary files: {}'.format(
detail), 'SILENT')
try:
for f in self.fileChunks:
os.remove(f)
except OSError as detail:
self.logger.log(
'Exception while removing temporary files: {}'.format(
detail), 'DEBUG')
try:
os.rmdir(self.testDir)
except OSError as detail:
self.logger.log(
'Exception while removing directory: {}'.format(detail),
'ERROR')
deleteSuccessful = True
# Keep deleting from the cloud until there is no more to delete.
while deleteSuccessful:
try:
fileIDToDelete = self.exporter.fileIDForFileName(
self.compressedTestFilename)
self.logger.log("file ID to delete: {}".format(fileIDToDelete),
'DEBUG')
self.exporter.driveService.files().delete(
fileId = '%s' % fileIDToDelete).execute()
except (TypeError, http.HttpError) as e:
self.logger.log('Delete not successful: {}'.format(e), 'SILENT')
break
class MSGDBExporterTester(unittest.TestCase):
"""
Unit tests for the MSG Cloud Exporter.
"""
def setUp(self):
self.logger = SEKLogger(__name__, 'DEBUG')
self.configer = MSGConfiger()
self.exporter = MSGDBExporter()
self.testDir = 'db_exporter_test'
self.uncompressedTestFilename = 'meco_v3_test_data.sql'
self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
self.exportTestDataPath = self.configer.configOptionValue('Testing',
'export_test_data_path')
self.fileUtil = MSGFileUtil()
self.fileChunks = []
self.testDataFileID = ''
self.pyUtil = MSGPythonUtil()
self.timeUtil = MSGTimeUtil()
conn = None
try:
conn = MSGDBConnector().connectDB()
except Exception as detail:
self.logger.log("Exception occurred: {}".format(detail), 'error')
exit(-1)
self.logger.log("conn = {}".format(conn), 'debug')
self.assertIsNotNone(conn)
# Create a temporary working directory.
try:
os.mkdir(self.testDir)
except OSError as detail:
self.logger.log(
'Exception during creation of temp directory: %s' % detail,
'ERROR')
def tearDown(self):
"""
Delete all test items.
"""
REMOVE_TEMPORARY_FILES = True
if REMOVE_TEMPORARY_FILES:
try:
self.logger.log("Removing local test files {}, {}.".format(
self.uncompressedTestFilename, self.compressedTestFilename),
'debug')
os.remove(os.path.join(os.getcwd(), self.testDir,
self.uncompressedTestFilename))
os.remove(os.path.join(os.getcwd(), self.testDir,
self.compressedTestFilename))
except OSError as detail:
self.logger.log(
'Exception while removing temporary files: {}'.format(
detail), 'SILENT')
try:
os.remove(os.path.join(os.getcwd(), self.testDir,
self.compressedTestFilename))
except OSError as detail:
self.logger.log(
'Exception while removing temporary files: {}'.format(
detail), 'SILENT')
try:
for f in self.fileChunks:
os.remove(f)
except OSError as detail:
self.logger.log(
'Exception while removing temporary files: {}'.format(
detail), 'DEBUG')
try:
os.rmdir(self.testDir)
except OSError as detail:
self.logger.log(
'Exception while removing directory: {}'.format(detail),
'ERROR')
# Keep deleting from the cloud until there are no more to delete.
def deleteFromCloud():
self.logger.log("deleting from cloud", 'debug')
try:
fileIDToDelete = self.exporter.fileIDForFileName(
self.compressedTestFilename)
if fileIDToDelete is None:
return
self.logger.log("file ID to delete: {}".format(fileIDToDelete),
'DEBUG')
self.exporter.driveService.files().delete(
fileId = '{}'.format(fileIDToDelete)).execute()
deleteFromCloud()
except (TypeError, http.HttpError) as e:
self.logger.log('Delete not successful: {}'.format(e), 'DEBUG')
deleteFromCloud()
def _upload_test_data_to_cloud(self):
"""
Provide an upload of test data that can be used in other tests.
Side effect: Store the file ID as an ivar.
"""
self.logger.log("Uploading test data for caller: {}".format(
self.pyUtil.callerName()))
filePath = "{}/{}".format(self.exportTestDataPath,
self.compressedTestFilename)
self.logger.log('Uploaded {}.'.format(filePath), 'info')
uploadResult = self.exporter.uploadFileToCloudStorage(filePath)
self.logger.log('upload result: {}'.format(uploadResult))
self.testDataFileID = self.exporter.fileIDForFileName(
self.compressedTestFilename)
self.logger.log("Test file ID is {}.".format(self.testDataFileID))
def test_markdown_list_of_downloadable_files(self):
"""
Match the Markdown line entry for the uploaded file.
"""
# @REVIEWED
self._upload_test_data_to_cloud()
self.assertEquals(len(filter(lambda x: self.testDataFileID in x,
self.exporter.markdownListOfDownloadableFiles().splitlines())),
1)
def test_get_md5_sum_from_cloud(self):
"""
Test retrieving the MD5 sum from the cloud.
"""
# @REVIEWED
self.logger.log('Testing getting the MD5 sum.', 'info')
self._upload_test_data_to_cloud()
testFileMD5 = filter(lambda x: x['id'] == self.testDataFileID,
self.exporter.cloudFiles['items'])[0][
'md5Checksum']
self.assertEquals(len(testFileMD5), 32)
self.assertTrue(re.match(r'[0-9A-Za-z]+', testFileMD5))
def test_get_file_id_for_nonexistent_file(self):
"""
Test getting a file ID for a nonexistent file.
"""
# @REVIEWED
fileIDs = self.exporter.fileIDForFileName('nonexistent_file')
self.logger.log("file ids = {}".format(fileIDs), 'info')
self.assertIsNone(fileIDs)
def test_upload_test_data(self):
"""
Upload a test data file for unit testing of DB export.
The unit test data file is a predefined set of test data stored in
the test data path of the software distribution.
"""
# @REVIEWED
self._upload_test_data_to_cloud()
self.assertGreater(len(self.testDataFileID), 0)
self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))
def test_adding_reader_permissions(self):
"""
Add reader permissions to a file that was uploaded.
"""
# @REVIEWED
self.logger.log("Testing adding reader permissions.")
self._upload_test_data_to_cloud()
email = self.configer.configOptionValue('Testing',
'tester_email_address')
service = self.exporter.driveService
try:
id_resp = service.permissions().getIdForEmail(
email = email).execute()
print id_resp
except errors.HttpError as detail:
print 'Exception while getting ID for email: {}'.format(detail)
new_permission = {'value': email, 'type': 'user', 'role': 'reader'}
try:
self.logger.log('Adding reader permission', 'INFO')
fileIDToAddTo = self.testDataFileID
# The permission dict is being output to stdout here.
resp = service.permissions().insert(fileId = fileIDToAddTo,
sendNotificationEmails = False,
body = new_permission).execute()
except errors.HttpError as detail:
self.logger.log(
'Exception while adding reader permissions: {}'.format(detail),
'error')
def permission_id(email):
try:
id_resp = service.permissions().getIdForEmail(
email = email).execute()
return id_resp['id']
except errors.HttpError as error:
self.logger.log("HTTP error: {}".format(error))
permission = {}
try:
permission = service.permissions().get(fileId = self.testDataFileID,
permissionId = permission_id(
email)).execute()
except errors.HttpError as error:
self.logger.log("HTTP error: {}".format(error))
self.assertEquals(permission['role'], 'reader')
def test_create_compressed_archived(self):
"""
* Copy test data to a temp directory (self.testDir).
* Create a checksum for test data.
* Create a gzip-compressed archive.
* Extract gzip-compressed archive.
* Create a checksum for the uncompressed data.
* Compare the checksums.
"""
# @REVIEWED
self.logger.log('Testing verification of a compressed archive.')
self.logger.log('cwd {}'.format(os.getcwd()))
fullPath = '{}'.format(os.path.join(os.getcwd(), self.testDir,
self.uncompressedTestFilename))
shutil.copyfile('{}/{}'.format(self.exportTestDataPath,
self.uncompressedTestFilename), fullPath)
md5sum1 = self.fileUtil.md5Checksum(fullPath)
self.exporter.fileUtil.gzipCompressFile(fullPath)
try:
os.remove(os.path.join(os.getcwd(), self.testDir,
self.uncompressedTestFilename))
except OSError as detail:
self.logger.log('Exception while removing: {}'.format(detail),
'ERROR')
# Extract archived data and generate checksum.
src = gzip.open('{}{}'.format(fullPath, '.gz'), "rb")
uncompressed = open(fullPath, "wb")
decoded = src.read()
uncompressed.write(decoded)
uncompressed.close()
md5sum2 = self.fileUtil.md5Checksum(fullPath)
self.assertEqual(md5sum1, md5sum2,
'Checksums are not equal for original and new '
'decompressed archive.')
def test_export_db(self):
"""
Perform a quick test of the DB export method using Testing Mode.
This requires sudo authorization to complete.
"""
# @REVIEWED
self.logger.log('Testing exportDB using the testing DB.')
# @todo handle case where testing db does not exist.
dbs = ['test_meco']
ids = self.exporter.exportDBs(databases = dbs, toCloud = True,
localExport = True)
self.logger.log('Count of exports: {}'.format(len(ids)))
self.assertEquals(len(ids), 1, "Count of exported files is wrong.")
map(self.exporter.deleteFile, ids)
def test_split_archive(self):
"""
Test splitting an archive into chunks.
"""
# @REVIEWED
self.logger.log('Testing archive splitting.')
fullPath = '{}/{}'.format(self.exportTestDataPath,
self.compressedTestFilename)
self.logger.log('fullpath: {}'.format(fullPath))
shutil.copyfile(fullPath, '{}/{}'.format(self.testDir,
self.compressedTestFilename))
fullPath = '{}/{}'.format(self.testDir, self.compressedTestFilename)
self.fileChunks = self.fileUtil.splitLargeFile(fullPath = fullPath,
numChunks = 3)
self.assertEquals(len(self.fileChunks), 3)
def test_get_file_size(self):
"""
Test retrieving local file sizes.
"""
# @REVIEWED
fullPath = '{}/{}'.format(self.exportTestDataPath,
self.compressedTestFilename)
fSize = self.fileUtil.fileSize(fullPath)
self.logger.log('size: {}'.format(fSize))
self.assertEqual(fSize, 12279, 'File size is correct.')
def test_upload_export_files_list(self):
"""
TBW
"""
pass
def test_checksum_after_upload(self):
"""
TBW
"""
pass
def test_dump_exclusions_dictionary(self):
"""
Verify the exclusions dictionary by its type.
:return:
"""
# @REVIEWED
exclusions = self.exporter.dumpExclusionsDictionary()
if exclusions:
self.assertEquals(type({}), type(exclusions))
def test_move_to_final(self):
"""
Test moving a file to the final destination path.
"""
# @REVIEWED
self.logger.log('Testing moving to final path {}.'.format(
self.configer.configOptionValue('Export', 'db_export_final_path')))
origCompressedFile = '{}/{}'.format(
self.configer.configOptionValue('Export', 'export_test_data_path'),
self.compressedTestFilename)
newCompressedFile = '{}/{}'.format(
self.configer.configOptionValue('Export', 'export_test_data_path'),
'temp_test_file')
shutil.copyfile(origCompressedFile, newCompressedFile)
self.exporter.moveToFinalPath(compressedFullPath = newCompressedFile)
self.assertTrue(os.path.isfile('{}/{}'.format(
self.configer.configOptionValue('Export', 'db_export_final_path'),
'temp_test_file')))
# Remove the test file.
os.remove('{}/{}'.format(
self.configer.configOptionValue('Export', 'db_export_final_path'),
'temp_test_file'))
def test_log_successful_export(self):
"""
Test logging of export results to the export history table.
"""
# @REVIEWED
self.assertTrue(self.exporter.logSuccessfulExport(name = 'test_export',
url =
'http://test_url',
datetime = 0,
size = 100))
conn = MSGDBConnector().connectDB()
cursor = conn.cursor()
dbUtil = MSGDBUtil()
self.assertTrue(
dbUtil.executeSQL(cursor, 'select * from "ExportHistory" where '
'timestamp = '
'to_timestamp(0)'))
self.assertEqual(len(cursor.fetchall()), 1,
"There should only be one result row.")
self.assertTrue(
dbUtil.executeSQL(cursor, 'delete from "ExportHistory" where '
'timestamp = to_timestamp(0)'))
conn.commit()
def test_metadata_of_file_id(self):
"""
Test getting the metadata for a file ID.
"""
# @REVIEWED
self._upload_test_data_to_cloud()
self.logger.log('metadata: {}'.format(
self.exporter.metadataOfFileID(self.testDataFileID)))
self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))
def test_filename_for_file_id(self):
"""
Test returning a file name given a file ID.
"""
# @REVIEWED
self._upload_test_data_to_cloud()
self.assertEquals(
self.exporter.filenameForFileID(fileID = self.testDataFileID),
self.compressedTestFilename)
def test_outdated_files(self):
# @REVIEWED
self._upload_test_data_to_cloud()
time.sleep(1)
self.logger.log("outdated:")
# For debugging:
for item in self.exporter.outdatedFiles(
daysBeforeOutdated = datetime.timedelta(
days = -1)): self.logger.log(
"name: {}, created date: {}".format(item['originalFilename'],
item['createdDate']), 'debug')
# Get all the outdated files where outdated is equal to anything
# uploaded today or later.
self.assertTrue(self.exporter.outdatedFiles(
daysBeforeOutdated = datetime.timedelta(days = -1))[0][
'id'] == self.testDataFileID)
self.logger.log('-----')
def test_delete_outdated(self):
"""
TBW
"""
pass
def test_list_of_downloadable_files(self):
"""
Test the list of downloadable files used by the available files page.
"""
# @REVIEWED
self._upload_test_data_to_cloud()
self.assertEquals(len(
filter(lambda row: row['id'] == self.testDataFileID,
self.exporter.listOfDownloadableFiles())), 1,
"Test file not present.")
def test_count_of_db_exports(self):
count = self.exporter.countOfDBExports(EARLIEST_DATE)
self.logger.log(count,'DEBUG')
self.assertTrue(int(count) or int(count) == int(0))
def test_count_of_cloud_files(self):
count = self.exporter.countOfCloudFiles()
self.assertTrue(int(count) or int(count) == int(0))
def test_plaintext_list_of_downloadable_files(self):
"""
This test handles content both with content links and without content
links.
"""
content = self.exporter.plaintextListOfDownloadableFiles()
self.assertRegexpMatches(content,
'\d+-\d+-\d+.*\,'
'\s+\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z\,\s+\d+\sB')
def test_last_report_date(self):
last_report = self.exporter.notifier.lastReportDate(
types = MSGNotificationHistoryTypes,
noticeType = MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
self.assertTrue(last_report is None or last_report > EARLIEST_DATE)
def test_current_export_summary(self):
self.assertRegexpMatches(self.exporter.currentExportSummary(),
re.compile(
'last.*databases.*free.*currently.*accessed.*',
flags = re.IGNORECASE | re.DOTALL))
class MSGDBExporter(object):
"""
Export MSG DBs as SQL scripts.
Supports export to local storage and to cloud storage.
Usage:
from msg_db_exporter import MSGDBExporter
exporter = MSGDBExporter()
Public API:
exportDB(databases:List,
toCloud:Boolean,
testing:Boolean,
numChunks:Integer,
deleteOutdated:Boolean): Export a list of DBs to the cloud.
"""
# List of cloud files.
@property
def cloudFiles(self):
self._cloudFiles = self.driveService.files().list().execute()
return self._cloudFiles
@property
def driveService(self):
if self._driveService:
return self._driveService
if not self.credentialPath:
raise Exception("Credential path is required.")
storage = Storage('{}/google_api_credentials'.format(
self.credentialPath))
self.googleAPICredentials = storage.get()
self.logger.log("Authorizing credentials.", 'info')
http = httplib2.Http()
http = self.googleAPICredentials.authorize(http)
self.logger.log("Authorized.", 'info')
self._driveService = build('drive', 'v2', http=http)
return self._driveService
def __init__(self):
"""
Constructor.
"""
self.logger = SEKLogger(__name__, 'DEBUG', useColor=False)
self.timeUtil = MSGTimeUtil()
self.configer = MSGConfiger()
self.fileUtil = MSGFileUtil()
self.pythonUtil = MSGPythonUtil() # for debugging
self.connector = MSGDBConnector()
self.conn = self.connector.connectDB()
self.cursor = self.conn.cursor()
self.dbUtil = MSGDBUtil()
self.notifier = SEKNotifier(
connector=self.connector,
dbUtil=self.dbUtil,
user=self.configer.configOptionValue('Notifications',
'email_username'),
password=self.configer.configOptionValue('Notifications',
'email_password'),
fromaddr=self.configer.configOptionValue('Notifications',
'email_from_address'),
toaddr=self.configer.configOptionValue('Notifications',
'email_recipients'),
testing_toaddr=self.configer.configOptionValue(
'Notifications', 'testing_email_recipients'),
smtp_server_and_port=self.configer.configOptionValue(
'Notifications', 'smtp_server_and_port'))
# Google Drive parameters.
self.clientID = self.configer.configOptionValue(
'Export', 'google_api_client_id')
self.clientSecret = self.configer.configOptionValue(
'Export', 'google_api_client_secret')
self.oauthScope = 'https://www.googleapis.com/auth/drive'
self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
self.googleAPICredentials = ''
self.exportTempWorkPath = self.configer.configOptionValue(
'Export', 'db_export_work_path')
self.credentialPath = self.configer.configOptionValue(
'Export', 'google_api_credentials_path')
self.credentialStorage = Storage('{}/google_api_credentials'.format(
self.credentialPath))
self._driveService = None
self._cloudFiles = None
self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
self.retryDelay = 10
self.availableFilesURL = ''
def verifyExportChecksum(self, testing=False):
"""
Verify the compressed export file using a checksum.
* Save the checksum of the original uncompressed export data.
* Extract the compressed file.
* Verify the uncompressed export data.
:param testing: When set to True, Testing Mode is used.
"""
# Get the checksum of the original file.
md5sum = self.fileUtil.md5Checksum(self.exportTempWorkPath)
self.logger.log('md5sum: {}'.format(md5sum))
def db_username(self):
return "postgres"
# return self.configer.configOptionValue('Database', 'db_username')
def db_password(self):
return self.configer.configOptionValue('Database', 'db_password')
def db_port(self):
return self.configer.configOptionValue('Database', 'db_port')
def dumpCommand(self, db='', dumpName=''):
"""
This method makes use of
pg_dump -s -p ${PORT}
-U ${USERNAME}
[-T ${OPTIONAL_TABLE_EXCLUSIONS}]
${DB_NAME} >
${EXPORT_TEMP_WORK_PATH}/${DUMP_TIMESTAMP}_{DB_NAME}.sql
:param db: String
:param dumpName: String
:return: String of command used to export DB.
"""
# For reference only:
# Password is passed from ~/.pgpass.
# Note that ':' and '\' characters should be escaped with '\'.
# Ref: http://www.postgresql.org/docs/9.1/static/libpq-pgpass.html
# Dump databases as the superuser. This method does not require a
# stored password when running under a root crontab.
if not db or not dumpName:
raise Exception('DB and dumpname required.')
# Process exclusions.
exclusions = self.dumpExclusionsDictionary()
excludeList = []
if db in exclusions:
excludeList = exclusions[db]
excludeString = ''
if len(excludeList) > 0 and exclusions != None:
for e in excludeList:
excludeString += """-T '"{}"' """.format(e)
return 'sudo -u postgres pg_dump -p {0} -U {1} {5} {2} > {3}/{4}' \
'.sql'.format(self.db_port(), self.db_username(), db,
self.exportTempWorkPath, dumpName, excludeString)
def dumpExclusionsDictionary(self):
"""
:param db: String of DB name for which to retrieve exclusions.
:return: Dictionary with keys as DBs and values as lists of tables to
be excluded for a given database.
"""
try:
if type(
eval(
self.configer.configOptionValue(
'Export', 'db_export_exclusions'))) == type({}):
return eval(
self.configer.configOptionValue('Export',
'db_export_exclusions'))
else:
return None
except SyntaxError as detail:
self.logger.log(
'SyntaxError exception while getting exclusions: {}'.format(
detail))
def dumpName(self, db=''):
"""
:param db: String
:return: String of file name used for dump file of db.
"""
if not db:
raise Exception('DB required.')
return "{}_{}".format(self.timeUtil.conciseNow(), db)
def filesToUpload(self, compressedFullPath='', numChunks=0, chunkSize=0):
"""
:param compressedFullPath: String
:param numChunks: Int
:param chunkSize: Int
:return: List of files to be uploaded according to their split
sections, if applicable.
"""
if numChunks != 0:
self.logger.log('Splitting {}'.format(compressedFullPath), 'DEBUG')
filesToUpload = self.fileUtil.splitLargeFile(
fullPath=compressedFullPath,
chunkSize=chunkSize,
numChunks=numChunks)
if not filesToUpload:
raise Exception('Exception during file splitting.')
else:
self.logger.log('to upload: {}'.format(filesToUpload), 'debug')
return filesToUpload
else:
return [compressedFullPath]
def dumpResult(self, db='', dumpName='', fullPath=''):
"""
:param dumpName: String of filename of dump file.
:param fullPath: String of full path to dump file.
:return: Boolean True if dump operation was successful, otherwise False.
"""
success = True
self.logger.log('fullPath: {}'.format(fullPath), 'DEBUG')
try:
# Generate the SQL script export.
# @todo check return value of dump command
self.logger.log('cmd: {}'.format(
self.dumpCommand(db=db, dumpName=dumpName)))
subprocess.check_call(self.dumpCommand(db=db, dumpName=dumpName),
shell=True)
except subprocess.CalledProcessError as error:
self.logger.log("Exception while dumping: {}".format(error))
sys.exit(-1)
return success
def exportDBs(self,
databases=None,
toCloud=False,
localExport=True,
testing=False,
chunkSize=0,
deleteOutdated=False):
"""
Export a set of DBs to local storage.
:param databases: List of database names that will be exported.
:param toCloud: Boolean if set to True, then the export will also be
copied to cloud storage.
:param localExport: Boolean when set to True the DB is exported
locally.
:param testing: Boolean flag for testing mode. (@DEPRECATED)
:param chunkSize: Integer size in bytes of chunk size used for
splitting.
:param deleteOutdated: Boolean indicating outdated files in the cloud
should be removed.
:returns: List of file IDs of uploaded files or None if there is an
error condition.
"""
# @todo separate uploading and exporting functions
noErrors = True
uploaded = []
for db in databases:
self.logger.log('Exporting {} using pg_dump.'.format(db), 'info')
dumpName = self.dumpName(db=db)
fullPath = '{}/{}.sql'.format(self.exportTempWorkPath, dumpName)
if localExport:
noErrors = self.dumpResult(db, dumpName, fullPath)
# Perform compression of the file.
self.logger.log("Compressing {} using gzip.".format(db), 'info')
self.logger.log('fullpath: {}'.format(fullPath), 'DEBUG')
gzipResult = self.fileUtil.gzipCompressFile(fullPath)
compressedFullPath = '{}{}'.format(fullPath, '.gz')
numChunks = self.numberOfChunksToUse(compressedFullPath)
# Gzip uncompress and verify by checksum is disabled until a more
# efficient, non-memory-based, uncompress is implemented.
# md5sum1 = self.fileUtil.md5Checksum(fullPath)
# self.md5Verification(compressedFullPath=compressedFullPath,
# fullPath=fullPath,md5sum1=md5sum1)
if toCloud:
# Split compressed files into a set of chunks to improve the
# reliability of uploads.
# Upload the files to the cloud.
for f in self.filesToUpload(
compressedFullPath=compressedFullPath,
numChunks=numChunks,
chunkSize=chunkSize):
self.logger.log('Uploading {}.'.format(f), 'info')
fileID = self.uploadFileToCloudStorage(
fullPath=f,
testing=testing,
retryCount=int(
self.configer.configOptionValue(
'Export', 'export_retry_count')))
self.logger.log('file id after upload: {}'.format(fileID))
if fileID != None:
uploaded.append(fileID)
self.logger.log('uploaded: {}'.format(uploaded),
'DEBUG')
if not self.addReaders(
fileID,
self.configer.configOptionValue(
'Export',
'reader_permission_email_addresses').split(
','),
retryCount=int(
self.configer.configOptionValue(
'Export', 'export_retry_count'))):
self.logger.log(
'Failed to add readers for {}.'.format(f),
'error')
self.logSuccessfulExport(
*self.metadataOfFileID(fileID))
# Remove split sections if they exist.
try:
if not testing and numChunks > 1:
self.logger.log('Removing {}'.format(f))
os.remove('{}'.format(f))
except OSError as error:
self.logger.log(
'Exception while removing {}: {}.'.format(
fullPath, error))
noErrors = False
# End if toCloud.
if gzipResult:
self.moveToFinalPath(compressedFullPath=compressedFullPath)
# Remove the uncompressed file.
try:
if not testing:
self.logger.log('Removing {}'.format(fullPath))
os.remove('{}'.format(fullPath))
except OSError as error:
self.logger.log('Exception while removing {}: {}.'.format(
fullPath, error))
noErrors = False
# End for db in databases.
if deleteOutdated:
self.deleteOutdatedFiles(
datetime.timedelta(days=int(
self.configer.configOptionValue('Export',
'export_days_to_keep'))))
return uploaded if noErrors else None
def moveToFinalPath(self, compressedFullPath=''):
"""
Move a compressed final to the final export path.
:param compressedFullPath: String for the compressed file.
:return:
"""
self.logger.log('Moving {} to final path.'.format(compressedFullPath),
'debug')
try:
shutil.move(
compressedFullPath,
self.configer.configOptionValue('Export',
'db_export_final_path'))
except Exception as detail:
self.logger.log(
'Exception while moving {} to final export path: {}'.format(
compressedFullPath, detail), 'error')
def md5Verification(self, compressedFullPath='', fullPath='', md5sum1=''):
"""
Perform md5 verification of a compressed file at compressedFullPath
where the original file is at fullPath and has md5sum1.
:param compressedFullPath: String
:param fullPath: String
:param md5sum1: String of md5sum of source file.
:return:
"""
GZIP_UNCOMPRESS_FILE = False
if GZIP_UNCOMPRESS_FILE:
# Verify the compressed file by uncompressing it and
# verifying its
# checksum against the original checksum.
self.logger.log('reading: {}'.format(compressedFullPath), 'DEBUG')
self.logger.log(
'writing: {}'.format(
os.path.join(
self.configer.configOptionValue(
'Testing', 'export_test_data_path'),
os.path.splitext(os.path.basename(fullPath))[0])),
'DEBUG')
self.fileUtil.gzipUncompressFile(
compressedFullPath,
os.path.join(
self.configer.configOptionValue('Testing',
'export_test_data_path'),
fullPath))
VERIFY_BY_CHECKSUM = False
if VERIFY_BY_CHECKSUM:
md5sum2 = self.fileUtil.md5Checksum(fullPath)
self.logger.log(
"mtime: {}, md5sum2: {}".format(
time.ctime(os.path.getmtime(fullPath)), md5sum2), 'INFO')
if md5sum1 == md5sum2:
self.logger.log(
'Compressed file has been validated by checksum.', 'INFO')
else:
noErrors = False
def numberOfChunksToUse(self, fullPath):
"""
Return the number of chunks to be used by the file splitter based on
the file size of the file at fullPath.
:param fullPath: String
:returns: Int Number of chunks to create.
"""
fsize = os.path.getsize(fullPath)
self.logger.log('fullpath: {}, fsize: {}'.format(fullPath, fsize))
if (fsize >= int(
self.configer.configOptionValue('Export',
'max_bytes_before_split'))):
# Note that this does not make use of the remainder in the division.
chunks = int(fsize / int(
self.configer.configOptionValue('Export',
'max_bytes_before_split')))
self.logger.log('Will split with {} chunks.'.format(chunks))
return chunks
self.logger.log('Will NOT split file.', 'debug')
return 1
def uploadFileToCloudStorage(self,
fullPath='',
retryCount=0,
testing=False):
"""
Export a file to cloud storage.
:param fullPath: String of file to be exported.
:param testing: Boolean when set to True, Testing Mode is used.
:param retryCount: Int of number of times to retry the upload if
there is a failure.
:returns: String File ID on verified on upload; None if verification
fails.
"""
success = True
myFile = os.path.basename(fullPath)
self.logger.log('full path {}'.format(os.path.dirname(fullPath),
'DEBUG'))
self.logger.log("Uploading {}.".format(myFile))
result = {}
try:
media_body = MediaFileUpload(
fullPath,
mimetype='application/gzip-compressed',
resumable=True)
body = {
'title': myFile,
'description': 'Hawaii Smart Energy Project gzip '
'compressed DB export.',
'mimeType': 'application/gzip-compressed'
}
# Result is a Files resource.
result = self.driveService.files().insert(
body=body, media_body=media_body).execute()
except Exception as detail:
# Upload failures can result in a BadStatusLine.
self.logger.log(
"Exception while uploading {}: {}.".format(myFile, detail),
'error')
success = False
if not self.__verifyMD5Sum(fullPath, self.fileIDForFileName(myFile)):
self.logger.log('Failed MD5 checksum verification.', 'INFO')
success = False
if success:
self.logger.log('Verification by MD5 checksum succeeded.', 'INFO')
self.logger.log("Finished.")
return result['id']
if not success and retryCount <= 0:
return None
else:
time.sleep(self.retryDelay)
self.logger.log('Retrying upload of {}.'.format(fullPath),
'warning')
self.uploadFileToCloudStorage(fullPath=fullPath,
retryCount=retryCount - 1)
def __retrieveCredentials(self):
"""
Perform authorization at the server.
Credentials are loaded into the object attribute googleAPICredentials.
"""
flow = OAuth2WebServerFlow(self.clientID, self.clientSecret,
self.oauthScope, self.oauthConsent)
authorize_url = flow.step1_get_authorize_url()
print 'Go to the following link in your browser: ' + authorize_url
code = raw_input('Enter verification code: ').strip()
self.googleAPICredentials = flow.step2_exchange(code)
print "refresh_token = {}".format(
self.googleAPICredentials.refresh_token)
print "expiry = {}".format(self.googleAPICredentials.token_expiry)
def freeSpace(self):
"""
Get free space from the drive service.
:param driveService: Object for the drive service.
:returns: Int of free space (bytes B) on the drive service.
"""
aboutData = self.driveService.about().get().execute()
return int(aboutData['quotaBytesTotal']) - int(
aboutData['quotaBytesUsed']) - int(
aboutData['quotaBytesUsedInTrash'])
def deleteFile(self, fileID=''):
"""
Delete the file with ID fileID.
:param fileID: String of a Google API file ID.
"""
if not len(fileID) > 0:
raise Exception("File ID has not been given.")
self.logger.log(
'Deleting file with file ID {} and name {}.'.format(
fileID, self.filenameForFileID(fileID)), 'debug')
try:
# Writing the fileId arg name is required here.
self.driveService.files().delete(fileId=fileID).execute()
except errors.HttpError as error:
self.logger.log('Exception while deleting: {}'.format(error),
'error')
def deleteOutdatedFiles(self, maxAge=datetime.timedelta(weeks=9999999)):
"""
Remove outdated files from cloud storage.
:param minAge: datetime.timedelta of the minimum age before a file is
considered outdated.
:param maxAge: datetime.timedelta of the maximum age to consider for
a file.
:returns: Int count of deleted items.
"""
# @todo Return count of actual successfully deleted files.
outdated = self.outdatedFiles(maxAge)
""":type : dict"""
for f in outdated:
self.deleteFile(f['id'])
return len(outdated)
def outdatedFiles(self,
daysBeforeOutdated=datetime.timedelta(days=9999999)):
"""
Outdated files in the cloud where they are outdated if their age is
greater than or equal to daysBeforeOutdated.
Note: When t1 is the same day as t2, the timedelta comes back as -1.
Not sure why this isn't represented as zero. Perhaps to avoid a false
evaluation of a predicate on a tdelta.
:param daysBeforeOutdated: datetime.timedelta where the value
indicates that outdated files that have an age greater than this
parameter.
:return: Int count of deleted items.
"""
t1 = lambda x: datetime.datetime.strptime(x['createdDate'],
"%Y-%m-%dT%H:%M:%S.%fZ")
t2 = datetime.datetime.now()
return filter(lambda x: t2 - t1(x) >= daysBeforeOutdated,
self.cloudFiles['items'])
def sendNotificationOfFiles(self):
"""
Provide a notification that lists the export files along with sharing
links.
"""
pass
def sendDownloadableFiles(self):
"""
Send available files via HTTP POST.
:returns: None
"""
myPath = '{}/{}'.format(self.exportTempWorkPath,
'list-of-downloadable-files.txt')
fp = open(myPath, 'wb')
output = StringIO()
output.write(self.markdownListOfDownloadableFiles())
fp.write(self.markdownListOfDownloadableFiles())
fp.close()
headers = {'User-Agent': self.postAgent, 'Content-Type': 'text/html'}
try:
r = requests.post(self.configer.configOptionValue(
'Export', 'export_list_post_url'),
output.getvalue(),
headers=headers)
print 'text: {}'.format(r.text)
except requests.adapters.SSLError as error:
# @todo Implement alternative verification.
self.logger.log('SSL error: {}'.format(error), 'error')
output.close()
def metadataOfFileID(self, fileID=''):
"""
:param fileID: String of a file ID in the cloud.
:return: Tuple of metadata (name, url, timestamp, size) for a given
file ID.
"""
item = [i for i in self.cloudFiles['items'] if i['id'] == fileID][0]
return (item[u'originalFilename'], item[u'webContentLink'],
item[u'createdDate'], item[u'fileSize'])
def listOfDownloadableFiles(self):
"""
Create a list of downloadable files.
:returns: List of dicts of files that are downloadable from the cloud.
"""
files = []
for i in reversed(
sorted(self.cloudFiles['items'],
key=lambda k: k['createdDate'])):
item = dict()
item['title'] = i['title']
item['webContentLink'] = i['webContentLink']
item['id'] = i['id']
item['createdDate'] = i['createdDate']
item['fileSize'] = i['fileSize']
files.append(item)
return files
def markdownListOfDownloadableFiles(self):
"""
Generate content containing a list of downloadable files in Markdown
format.
:returns: String content in Markdown format.
"""
content = "||*Name*||*Created*||*Size*||\n"
for i in self.listOfDownloadableFiles():
content += "||[`{}`]({})".format(i['title'], i['webContentLink'])
content += "||`{}`".format(i['createdDate'])
content += "||`{} B`||".format(int(i['fileSize']))
content += '\n'
# self.logger.log('content: {}'.format(content))
return content
def plaintextListOfDownloadableFiles(self):
"""
Generate content containing a list of downloadable files in plaintext
format.
:returns: String content as plaintext.
"""
content = ''
includeLink = False
for i in reversed(
sorted(self.cloudFiles['items'],
key=lambda k: k['createdDate'])):
if includeLink:
content += "{}, {}, {}, {} B\n".format(i['title'],
i['webContentLink'],
i['createdDate'],
int(i['fileSize']))
else:
content += "{}, {}, {} B\n".format(i['title'],
i['createdDate'],
int(i['fileSize']))
return content
def logSuccessfulExport(self, name='', url='', datetime=0, size=0):
"""
When an export has been successful, log information about the export
to the database.
The items to log include:
* filename
* URL
* timestamp
* filesize
:param name: String
:param url: String
:param datetime:
:param size: Int
:return: True if no errors occurred, else False.
"""
def exportHistoryColumns():
return ['name', 'url', 'timestamp', 'size']
timestamp = lambda \
datetime: 'to_timestamp(0)' if datetime == 0 else "timestamp " \
"'{}'".format(
datetime)
sql = 'INSERT INTO "{0}" ({1}) VALUES ({2}, {3}, {4}, {5})'.format(
self.configer.configOptionValue('Export', 'export_history_table'),
','.join(exportHistoryColumns()), "'" + name + "'",
"'" + url + "'", timestamp(datetime), size)
conn = MSGDBConnector().connectDB()
cursor = conn.cursor()
dbUtil = MSGDBUtil()
result = dbUtil.executeSQL(cursor, sql, exitOnFail=False)
conn.commit()
return result
def sendExportSummary(self, summary=''):
"""
Send a summary of exports via email to a preconfigured list of
recipients.
:param summary: String of summary content.
:return:
"""
try:
if self.notifier.sendNotificationEmail(summary, testing=False):
self.notifier.recordNotificationEvent(
types=MSGNotificationHistoryTypes,
noticeType=MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
except Exception as detail:
self.logger.log('Exception occurred: {}'.format(detail), 'ERROR')
def currentExportSummary(self):
"""
Current summary of exports since the last summary report time.
Summaries are reported with identifier MSG_EXPORT_SUMMARY in the
NotificationHistory.
Includes:
* Number of databases exported
* Total number of files in the cloud.
* A report of available storage capacity.
* A list of available DBs.
* A link where exports can be accessed.
:return: String of summary text.
"""
availableFilesURL = self.configer.configOptionValue(
'Export', 'export_list_url')
lastReportDate = self.notifier.lastReportDate(
types=MSGNotificationHistoryTypes,
noticeType=MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
content = 'Cloud Export Summary:\n\n'
content += 'Last report date: {}\n'.format(lastReportDate)
# @TO BE REVIEWED: Verify time zone adjustment.
content += '{} databases have been exported since the last report ' \
'date.\n'.format(self.countOfDBExports(
lastReportDate + datetime.timedelta(
hours = 10)) if lastReportDate else self.countOfDBExports())
content += '{} B free space is available.\n'.format(self.freeSpace())
content += '\nCurrently available DBs:\n'
content += self.plaintextListOfDownloadableFiles()
content += '\n{} files can be accessed through Google Drive (' \
'https://drive.google.com) or at {}.'.format(
self.countOfCloudFiles(), availableFilesURL)
return content
def countOfDBExports(self, since=None):
"""
:param since: datetime indicating last export datetime.
:return: Int of count of exports.
"""
myDatetime = lambda x: datetime.datetime.strptime(x, '%Y-%m-%d %H:%S')
if not since:
since = myDatetime('1900-01-01 00:00')
self.logger.log(since.strftime('%Y-%m-%d %H:%M'), 'DEBUG')
sql = 'SELECT COUNT("public"."ExportHistory"."timestamp") FROM ' \
'"public"."ExportHistory" WHERE "timestamp" > \'{}\''.format(
since.strftime('%Y-%m-%d %H:%M'))
conn = MSGDBConnector().connectDB()
cursor = conn.cursor()
dbUtil = MSGDBUtil()
rows = None
if dbUtil.executeSQL(cursor, sql, exitOnFail=False):
rows = cursor.fetchall()
assert len(rows) == 1, 'Invalid return value.'
return rows[0][0]
def countOfCloudFiles(self):
"""
:param since: datetime indicating last trailing export datetime.
:return: Int of count of exports.
"""
return len(self.cloudFiles['items'])
def __verifyMD5Sum(self, localFilePath, remoteFileID):
"""
Verify that the local MD5 sum matches the MD5 sum for the remote file
corresponding to an ID.
This verifies that the uploaded file matches the local compressed
export file.
:param localFilePath: String of the full path of the local file.
:param remoteFileID: String of the cloud ID for the remote file.
:returns: Boolean True if the MD5 sums match, otherwise, False.
"""
self.logger.log('remote file ID: {}'.format(remoteFileID))
self.logger.log('local file path: {}'.format(localFilePath))
# Get the md5sum for the local file.
f = open(localFilePath, mode='rb')
fContent = hashlib.md5()
for buf in iter(partial(f.read, 128), b''):
fContent.update(buf)
localMD5Sum = fContent.hexdigest()
f.close()
self.logger.log('local md5: {}'.format(localMD5Sum), 'DEBUG')
def verifyFile():
# Get the MD5 sum for the remote file.
for item in self.cloudFiles['items']:
if (item['id'] == remoteFileID):
self.logger.log(
'remote md5: {}'.format(item['md5Checksum']), 'DEBUG')
if localMD5Sum == item['md5Checksum']:
return True
else:
return False
try:
if verifyFile():
return True
else:
return False
except errors.HttpError as detail:
self.logger.log('HTTP error during MD5 verification.', 'error')
time.sleep(10)
if verifyFile():
return True
else:
return False
def fileIDForFileName(self, filename):
"""
Get the file ID for the given filename.
This method supports matching multiple cloud filenames but only
returns the ID for a single matching filename.
This can then be called recursively to obtain all the file IDs for a
given filename.
:param String of the filename for which to retrieve the ID.
:returns: String of a cloud file ID or None if no match.
"""
fileIDList = filter(lambda x: x['originalFilename'] == filename,
self.cloudFiles['items'])
return fileIDList[0]['id'] if len(fileIDList) > 0 else None
def filenameForFileID(self, fileID=''):
"""
:param fileID: String of cloud-based file ID.
:return: String of filename for a given file ID.
"""
return filter(lambda x: x['id'] == fileID,
self.cloudFiles['items'])[0]['originalFilename']
def addReaders(self, fileID=None, emailAddressList=None, retryCount=0):
"""
Add reader permission to an export file that has been uploaded to the
cloud for the given list of email addresses.
Email notification is suppressed by default.
:param fileID: String of the cloud file ID to be processed.
:param emailAddressList: List of email addresses.
:returns: Boolean True if successful, otherwise False.
"""
# @todo Provide support for retry count
success = True
self.logger.log('file id: {}'.format(fileID))
self.logger.log('address list: {}'.format(emailAddressList))
for addr in emailAddressList:
permission = {'value': addr, 'type': 'user', 'role': 'reader'}
if fileID:
try:
resp = self.driveService.permissions().insert(
fileId=fileID,
sendNotificationEmails=False,
body=permission).execute()
self.logger.log(
'Reader permission added for {}.'.format(addr))
except errors.HttpError as error:
self.logger.log('An error occurred: {}'.format(error))
success = False
if not success and retryCount <= 0:
return False
elif success:
return True
else:
time.sleep(self.retryDelay)
self.logger.log(
'Retrying adding readers for ID {}.'.format(fileID), 'warning')
self.addReaders(fileID=fileID,
emailAddressList=emailAddressList,
retryCount=retryCount - 1)