Esempio n. 1
0
    def setUp(self):
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.configer = MSGConfiger()
        self.exporter = MSGDBExporter()
        self.testDir = 'db_exporter_test'
        self.uncompressedTestFilename = 'meco_v3_test_data.sql'
        self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
        self.exportTestDataPath = self.configer.configOptionValue(
            'Testing', 'export_test_data_path')
        self.fileUtil = MSGFileUtil()
        self.fileChunks = []
        self.testDataFileID = ''
        self.pyUtil = MSGPythonUtil()
        self.timeUtil = MSGTimeUtil()

        conn = None
        try:
            conn = MSGDBConnector().connectDB()
        except Exception as detail:
            self.logger.log("Exception occurred: {}".format(detail), 'error')
            exit(-1)

        self.logger.log("conn = {}".format(conn), 'debug')
        self.assertIsNotNone(conn)

        # Create a temporary working directory.
        try:
            os.mkdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception during creation of temp directory: %s' % detail,
                'ERROR')
Esempio n. 2
0
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__)
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
Esempio n. 3
0
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'DEBUG', useColor=False)
        self.timeUtil = MSGTimeUtil()
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
        self.pythonUtil = MSGPythonUtil()  # for debugging
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = SEKNotifier(
            connector=self.connector,
            dbUtil=self.dbUtil,
            user=self.configer.configOptionValue('Notifications',
                                                 'email_username'),
            password=self.configer.configOptionValue('Notifications',
                                                     'email_password'),
            fromaddr=self.configer.configOptionValue('Notifications',
                                                     'email_from_address'),
            toaddr=self.configer.configOptionValue('Notifications',
                                                   'email_recipients'),
            testing_toaddr=self.configer.configOptionValue(
                'Notifications', 'testing_email_recipients'),
            smtp_server_and_port=self.configer.configOptionValue(
                'Notifications', 'smtp_server_and_port'))

        # Google Drive parameters.
        self.clientID = self.configer.configOptionValue(
            'Export', 'google_api_client_id')
        self.clientSecret = self.configer.configOptionValue(
            'Export', 'google_api_client_secret')
        self.oauthScope = 'https://www.googleapis.com/auth/drive'
        self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
        self.googleAPICredentials = ''
        self.exportTempWorkPath = self.configer.configOptionValue(
            'Export', 'db_export_work_path')

        self.credentialPath = self.configer.configOptionValue(
            'Export', 'google_api_credentials_path')
        self.credentialStorage = Storage('{}/google_api_credentials'.format(
            self.credentialPath))

        self._driveService = None
        self._cloudFiles = None
        self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
        self.retryDelay = 10
        self.availableFilesURL = ''
    def setUp(self):
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.configer = MSGConfiger()
        self.exporter = MSGDBExporter()
        self.testDir = 'db_exporter_test'
        self.uncompressedTestFilename = 'meco_v3_test_data.sql'
        self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
        self.exportTestDataPath = self.configer.configOptionValue('Testing',
                                                                  'export_test_data_path')
        self.fileUtil = MSGFileUtil()
        self.fileChunks = []
        self.testDataFileID = ''
        self.pyUtil = MSGPythonUtil()
        self.timeUtil = MSGTimeUtil()

        conn = None
        try:
            conn = MSGDBConnector().connectDB()
        except Exception as detail:
            self.logger.log("Exception occurred: {}".format(detail), 'error')
            exit(-1)

        self.logger.log("conn = {}".format(conn), 'debug')
        self.assertIsNotNone(conn)

        # Create a temporary working directory.
        try:
            os.mkdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception during creation of temp directory: %s' % detail,
                'ERROR')
    def __init__(self):
        """
        Constructor.
        """

        self.logger = MSGLogger(__name__)
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'DEBUG', useColor = False)
        self.timeUtil = MSGTimeUtil()
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
        self.pythonUtil = MSGPythonUtil()  # for debugging
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = SEKNotifier(connector = self.connector,
                                    dbUtil = self.dbUtil,
                                    user = self.configer.configOptionValue(
                                        'Notifications', 'email_username'),
                                    password = self.configer.configOptionValue(
                                        'Notifications', 'email_password'),
                                    fromaddr = self.configer.configOptionValue(
                                        'Notifications', 'email_from_address'),
                                    toaddr = self.configer.configOptionValue(
                                        'Notifications', 'email_recipients'),
                                    testing_toaddr =
                                    self.configer.configOptionValue(
                                        'Notifications',
                                        'testing_email_recipients'),
                                    smtp_server_and_port =
                                    self.configer.configOptionValue(
                                        'Notifications',
                                        'smtp_server_and_port'))

        # Google Drive parameters.
        self.clientID = self.configer.configOptionValue('Export',
                                                        'google_api_client_id')
        self.clientSecret = self.configer.configOptionValue('Export',
                                                            'google_api_client_secret')
        self.oauthScope = 'https://www.googleapis.com/auth/drive'
        self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
        self.googleAPICredentials = ''
        self.exportTempWorkPath = self.configer.configOptionValue('Export',
                                                                  'db_export_work_path')

        self.credentialPath = self.configer.configOptionValue('Export',
                                                              'google_api_credentials_path')
        self.credentialStorage = Storage(
            '{}/google_api_credentials'.format(self.credentialPath))

        self._driveService = None
        self._cloudFiles = None
        self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
        self.retryDelay = 10
        self.availableFilesURL = ''
Esempio n. 7
0
class MECODataAutoloader(object):
    """
    Provide automated loading of MECO energy data from exports in
    gzip-compressed XML source data.
    """
    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__)
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()

    def newDataExists(self):
        """
        Check the data autoload folder for the presence of new data.

        :returns: True if new data exists.
        """

        autoloadPath = self.configer.configOptionValue(
            'MECO Autoload', 'meco_autoload_new_data_path')
        if not self.fileUtil.validDirectory(autoloadPath):
            raise Exception('InvalidDirectory', '%s' % autoloadPath)

        patterns = ['*.gz']
        matchCnt = 0
        for root, dirs, filenames in os.walk(autoloadPath):
            for pat in patterns:
                for filename in fnmatch.filter(filenames, pat):
                    print filename
                    matchCnt += 1
        if matchCnt > 0:
            return True
        else:
            return False

    def loadNewData(self):
        """
        Load new data contained in the new data path.
        """

        autoloadPath = self.configer.configOptionValue(
            'MECO Autoload', 'meco_autoload_new_data_path')
        command = self.configer.configOptionValue('MECO Autoload',
                                                  'meco_autoload_command')
        os.chdir(autoloadPath)

        try:
            subprocess.check_call(command, shell=True)
        except subprocess.CalledProcessError, e:
            self.logger.log("An exception occurred: %s" % e, 'error')
class MECODataAutoloader(object):
    """
    Provide automated loading of MECO energy data from exports in gzip-compressed XML source data.
    """

    def __init__(self):
        """
        Constructor.
        """

        self.logger = MSGLogger(__name__)
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()


    def newDataExists(self):
        """
        Check the data autoload folder for the presence of new data.

        :returns: True if new data exists.
        """

        autoloadPath = self.configer.configOptionValue('MECO Autoload', 'meco_new_data_path')
        if not self.fileUtil.validDirectory(autoloadPath):
            raise Exception('InvalidDirectory', '%s' % autoloadPath)

        patterns = ['*.gz']
        matchCnt = 0
        for root, dirs, filenames in os.walk(autoloadPath):
            for pat in patterns:
                for filename in fnmatch.filter(filenames, pat):
                    print filename
                    matchCnt += 1
        if matchCnt > 0:
            return True
        else:
            return False


    def loadNewData(self):
        """
        Load new data contained in the new data path.
        """

        autoloadPath = self.configer.configOptionValue('MECO Autoload', 'meco_new_data_path')
        command = self.configer.configOptionValue('MECO Autoload', 'data_load_command')
        os.chdir(autoloadPath)

        try:
            subprocess.check_call(command, shell = True)
        except subprocess.CalledProcessError, e:
            self.logger.log("An exception occurred: %s" % e, 'error')
class MSGDBExporter(object):
    """
    Export MSG DBs as SQL scripts.

    Supports export to local storage and to cloud storage.

    Usage:

    from msg_db_exporter import MSGDBExporter
    exporter = MSGDBExporter()

    Public API:

    exportDB(databases:List, 
             toCloud:Boolean, 
             testing:Boolean,
             numChunks:Integer, 
             deleteOutdated:Boolean): Export a list of DBs to the cloud.
    """

    # List of cloud files.
    @property
    def cloudFiles(self):
        self._cloudFiles = self.driveService.files().list().execute()
        return self._cloudFiles

    @property
    def driveService(self):
        if self._driveService:
            return self._driveService

        if not self.credentialPath:
            raise Exception("Credential path is required.")
        storage = Storage(
            '{}/google_api_credentials'.format(self.credentialPath))

        self.googleAPICredentials = storage.get()

        self.logger.log("Authorizing credentials.", 'info')
        http = httplib2.Http()
        http = self.googleAPICredentials.authorize(http)

        self.logger.log("Authorized.", 'info')

        self._driveService = build('drive', 'v2', http = http)

        return self._driveService


    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'DEBUG', useColor = False)
        self.timeUtil = MSGTimeUtil()
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
        self.pythonUtil = MSGPythonUtil()  # for debugging
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = SEKNotifier(connector = self.connector,
                                    dbUtil = self.dbUtil,
                                    user = self.configer.configOptionValue(
                                        'Notifications', 'email_username'),
                                    password = self.configer.configOptionValue(
                                        'Notifications', 'email_password'),
                                    fromaddr = self.configer.configOptionValue(
                                        'Notifications', 'email_from_address'),
                                    toaddr = self.configer.configOptionValue(
                                        'Notifications', 'email_recipients'),
                                    testing_toaddr =
                                    self.configer.configOptionValue(
                                        'Notifications',
                                        'testing_email_recipients'),
                                    smtp_server_and_port =
                                    self.configer.configOptionValue(
                                        'Notifications',
                                        'smtp_server_and_port'))

        # Google Drive parameters.
        self.clientID = self.configer.configOptionValue('Export',
                                                        'google_api_client_id')
        self.clientSecret = self.configer.configOptionValue('Export',
                                                            'google_api_client_secret')
        self.oauthScope = 'https://www.googleapis.com/auth/drive'
        self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
        self.googleAPICredentials = ''
        self.exportTempWorkPath = self.configer.configOptionValue('Export',
                                                                  'db_export_work_path')

        self.credentialPath = self.configer.configOptionValue('Export',
                                                              'google_api_credentials_path')
        self.credentialStorage = Storage(
            '{}/google_api_credentials'.format(self.credentialPath))

        self._driveService = None
        self._cloudFiles = None
        self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
        self.retryDelay = 10
        self.availableFilesURL = ''


    def verifyExportChecksum(self, testing = False):
        """
        Verify the compressed export file using a checksum.

        * Save the checksum of the original uncompressed export data.
        * Extract the compressed file.
        * Verify the uncompressed export data.

        :param testing: When set to True, Testing Mode is used.
        """

        # Get the checksum of the original file.
        md5sum = self.fileUtil.md5Checksum(self.exportTempWorkPath)
        self.logger.log('md5sum: {}'.format(md5sum))


    def db_username(self):
        return "postgres"
        # return self.configer.configOptionValue('Database', 'db_username')

    def db_password(self):
        return self.configer.configOptionValue('Database', 'db_password')

    def db_port(self):
        return self.configer.configOptionValue('Database', 'db_port')


    def dumpCommand(self, db = '', dumpName = ''):
        """
        This method makes use of

        pg_dump -s -p ${PORT}
                   -U ${USERNAME}
                   [-T ${OPTIONAL_TABLE_EXCLUSIONS}]
                   ${DB_NAME} >
                   ${EXPORT_TEMP_WORK_PATH}/${DUMP_TIMESTAMP}_{DB_NAME}.sql

        :param db: String
        :param dumpName: String
        :return: String of command used to export DB.
        """

        # For reference only:
        # Password is passed from ~/.pgpass.
        # Note that ':' and '\' characters should be escaped with '\'.
        # Ref: http://www.postgresql.org/docs/9.1/static/libpq-pgpass.html

        # Dump databases as the superuser. This method does not require a
        # stored password when running under a root crontab.
        if not db or not dumpName:
            raise Exception('DB and dumpname required.')

        # Process exclusions.

        exclusions = self.dumpExclusionsDictionary()
        excludeList = []
        if db in exclusions:
            excludeList = exclusions[db]
        excludeString = ''
        if len(excludeList) > 0 and exclusions != None:
            for e in excludeList:
                excludeString += """-T '"{}"' """.format(e)

        return 'sudo -u postgres pg_dump -p {0} -U {1} {5} {2} > {3}/{4}' \
               '.sql'.format(self.db_port(), self.db_username(), db,
                             self.exportTempWorkPath, dumpName, excludeString)


    def dumpExclusionsDictionary(self):
        """
        :param db: String of DB name for which to retrieve exclusions.
        :return: Dictionary with keys as DBs and values as lists of tables to
        be excluded for a given database.
        """
        try:
            if type(eval(self.configer.configOptionValue('Export',
                                                         'db_export_exclusions'))) == type(
                    {}):
                return eval(self.configer.configOptionValue('Export',
                                                            'db_export_exclusions'))
            else:
                return None
        except SyntaxError as detail:
            self.logger.log(
                'SyntaxError exception while getting exclusions: {}'.format(
                    detail))


    def dumpName(self, db = ''):
        """
        :param db: String
        :return: String of file name used for dump file of db.
        """
        if not db:
            raise Exception('DB required.')
        return "{}_{}".format(self.timeUtil.conciseNow(), db)


    def filesToUpload(self, compressedFullPath = '', numChunks = 0,
                      chunkSize = 0):
        """
        :param compressedFullPath: String
        :param numChunks: Int
        :param chunkSize: Int
        :return: List of files to be uploaded according to their split
        sections, if applicable.
        """
        if numChunks != 0:
            self.logger.log('Splitting {}'.format(compressedFullPath), 'DEBUG')

            filesToUpload = self.fileUtil.splitLargeFile(
                fullPath = compressedFullPath, chunkSize = chunkSize,
                numChunks = numChunks)

            if not filesToUpload:
                raise Exception('Exception during file splitting.')
            else:
                self.logger.log('to upload: {}'.format(filesToUpload), 'debug')
                return filesToUpload

        else:
            return [compressedFullPath]


    def dumpResult(self, db = '', dumpName = '', fullPath = ''):
        """
        :param dumpName: String of filename of dump file.
        :param fullPath: String of full path to dump file.
        :return: Boolean True if dump operation was successful, otherwise False.
        """

        success = True

        self.logger.log('fullPath: {}'.format(fullPath), 'DEBUG')

        try:
            # Generate the SQL script export.
            # @todo check return value of dump command
            self.logger.log('cmd: {}'.format(
                self.dumpCommand(db = db, dumpName = dumpName)))
            subprocess.check_call(
                self.dumpCommand(db = db, dumpName = dumpName), shell = True)
        except subprocess.CalledProcessError as error:
            self.logger.log("Exception while dumping: {}".format(error))
            sys.exit(-1)

        return success


    def exportDBs(self, databases = None, toCloud = False, localExport = True,
                  testing = False, chunkSize = 0, deleteOutdated = False):
        """
        Export a set of DBs to local storage.

        :param databases: List of database names that will be exported.
        :param toCloud: Boolean if set to True, then the export will also be
        copied to cloud storage.
        :param localExport: Boolean when set to True the DB is exported
        locally.
        :param testing: Boolean flag for testing mode. (@DEPRECATED)
        :param chunkSize: Integer size in bytes of chunk size used for
        splitting.
        :param deleteOutdated: Boolean indicating outdated files in the cloud
        should be removed.
        :returns: List of file IDs of uploaded files or None if there is an
        error condition.
        """

        # @todo separate uploading and exporting functions

        noErrors = True
        uploaded = []

        for db in databases:
            self.logger.log('Exporting {} using pg_dump.'.format(db), 'info')

            dumpName = self.dumpName(db = db)
            fullPath = '{}/{}.sql'.format(self.exportTempWorkPath, dumpName)
            if localExport:
                noErrors = self.dumpResult(db, dumpName, fullPath)

            # Perform compression of the file.
            self.logger.log("Compressing {} using gzip.".format(db), 'info')
            self.logger.log('fullpath: {}'.format(fullPath), 'DEBUG')

            gzipResult = self.fileUtil.gzipCompressFile(fullPath)
            compressedFullPath = '{}{}'.format(fullPath, '.gz')
            numChunks = self.numberOfChunksToUse(compressedFullPath)

            # Gzip uncompress and verify by checksum is disabled until a more
            # efficient, non-memory-based, uncompress is implemented.
            # md5sum1 = self.fileUtil.md5Checksum(fullPath)
            # self.md5Verification(compressedFullPath=compressedFullPath,
            # fullPath=fullPath,md5sum1=md5sum1)

            if toCloud:
                # Split compressed files into a set of chunks to improve the
                # reliability of uploads.

                # Upload the files to the cloud.
                for f in self.filesToUpload(
                        compressedFullPath = compressedFullPath,
                        numChunks = numChunks, chunkSize = chunkSize):
                    self.logger.log('Uploading {}.'.format(f), 'info')
                    fileID = self.uploadFileToCloudStorage(fullPath = f,
                                                           testing = testing,
                                                           retryCount = int(
                                                               self.configer.configOptionValue(
                                                                   'Export',
                                                                   'export_retry_count')))

                    self.logger.log('file id after upload: {}'.format(fileID))

                    if fileID != None:
                        uploaded.append(fileID)
                        self.logger.log('uploaded: {}'.format(uploaded),
                                        'DEBUG')
                        if not self.addReaders(fileID,
                                               self.configer.configOptionValue(
                                                       'Export',
                                                       'reader_permission_email_addresses').split(
                                                       ','), retryCount = int(
                                        self.configer.configOptionValue(
                                                'Export',
                                                'export_retry_count'))):
                            self.logger.log(
                                'Failed to add readers for {}.'.format(f),
                                'error')
                        self.logSuccessfulExport(*self.metadataOfFileID(fileID))

                    # Remove split sections if they exist.
                    try:
                        if not testing and numChunks > 1:
                            self.logger.log('Removing {}'.format(f))
                            os.remove('{}'.format(f))
                    except OSError as error:
                        self.logger.log(
                            'Exception while removing {}: {}.'.format(fullPath,
                                                                      error))
                        noErrors = False

            # End if toCloud.

            if gzipResult:
                self.moveToFinalPath(compressedFullPath = compressedFullPath)

            # Remove the uncompressed file.
            try:
                if not testing:
                    self.logger.log('Removing {}'.format(fullPath))
                    os.remove('{}'.format(fullPath))
            except OSError as error:
                self.logger.log(
                    'Exception while removing {}: {}.'.format(fullPath, error))
                noErrors = False

        # End for db in databases.

        if deleteOutdated:
            self.deleteOutdatedFiles(datetime.timedelta(days = int(
                self.configer.configOptionValue('Export',
                                                'export_days_to_keep'))))

        return uploaded if noErrors else None


    def moveToFinalPath(self, compressedFullPath = ''):
        """
        Move a compressed final to the final export path.
        :param compressedFullPath: String for the compressed file.
        :return:
        """
        self.logger.log('Moving {} to final path.'.format(compressedFullPath),
                        'debug')
        try:
            shutil.move(compressedFullPath,
                        self.configer.configOptionValue('Export',
                                                        'db_export_final_path'))
        except Exception as detail:
            self.logger.log(
                'Exception while moving {} to final export path: {}'.format(
                    compressedFullPath, detail), 'error')


    def md5Verification(self, compressedFullPath = '', fullPath = '',
                        md5sum1 = ''):
        """
        Perform md5 verification of a compressed file at compressedFullPath
        where the original file is at fullPath and has md5sum1.

        :param compressedFullPath: String
        :param fullPath: String
        :param md5sum1: String of md5sum of source file.
        :return:
        """

        GZIP_UNCOMPRESS_FILE = False
        if GZIP_UNCOMPRESS_FILE:
            # Verify the compressed file by uncompressing it and
            # verifying its
            # checksum against the original checksum.
            self.logger.log('reading: {}'.format(compressedFullPath), 'DEBUG')
            self.logger.log('writing: {}'.format(os.path.join(
                self.configer.configOptionValue('Testing',
                                                'export_test_data_path'),
                os.path.splitext(os.path.basename(fullPath))[0])), 'DEBUG')

            self.fileUtil.gzipUncompressFile(compressedFullPath, os.path.join(
                self.configer.configOptionValue('Testing',
                                                'export_test_data_path'),
                fullPath))

        VERIFY_BY_CHECKSUM = False
        if VERIFY_BY_CHECKSUM:
            md5sum2 = self.fileUtil.md5Checksum(fullPath)

            self.logger.log("mtime: {}, md5sum2: {}".format(
                time.ctime(os.path.getmtime(fullPath)), md5sum2), 'INFO')

            if md5sum1 == md5sum2:
                self.logger.log(
                    'Compressed file has been validated by checksum.', 'INFO')
            else:
                noErrors = False

    def numberOfChunksToUse(self, fullPath):
        """
        Return the number of chunks to be used by the file splitter based on
        the file size of the file at fullPath.
        :param fullPath: String
        :returns: Int Number of chunks to create.
        """

        fsize = os.path.getsize(fullPath)
        self.logger.log('fullpath: {}, fsize: {}'.format(fullPath, fsize))
        if (fsize >= int(self.configer.configOptionValue('Export',
                                                         'max_bytes_before_split'))):
            # Note that this does not make use of the remainder in the division.
            chunks = int(fsize / int(self.configer.configOptionValue('Export',
                                                                     'max_bytes_before_split')))
            self.logger.log('Will split with {} chunks.'.format(chunks))
            return chunks
        self.logger.log('Will NOT split file.', 'debug')
        return 1


    def uploadFileToCloudStorage(self, fullPath = '', retryCount = 0,
                                 testing = False):
        """
        Export a file to cloud storage.

        :param fullPath: String of file to be exported.
        :param testing: Boolean when set to True, Testing Mode is used.
        :param retryCount: Int of number of times to retry the upload if
        there is a failure.
        :returns: String File ID on verified on upload; None if verification
        fails.
        """

        success = True
        myFile = os.path.basename(fullPath)

        self.logger.log(
            'full path {}'.format(os.path.dirname(fullPath), 'DEBUG'))
        self.logger.log("Uploading {}.".format(myFile))

        result = {}
        try:
            media_body = MediaFileUpload(fullPath,
                                         mimetype =
                                         'application/gzip-compressed',
                                         resumable = True)
            body = {'title': myFile,
                    'description': 'Hawaii Smart Energy Project gzip '
                                   'compressed DB export.',
                    'mimeType': 'application/gzip-compressed'}

            # Result is a Files resource.
            result = self.driveService.files().insert(body = body,
                                                      media_body =
                                                      media_body).execute()

        except Exception as detail:
            # Upload failures can result in a BadStatusLine.
            self.logger.log(
                "Exception while uploading {}: {}.".format(myFile, detail),
                'error')
            success = False

        if not self.__verifyMD5Sum(fullPath, self.fileIDForFileName(myFile)):
            self.logger.log('Failed MD5 checksum verification.', 'INFO')
            success = False

        if success:
            self.logger.log('Verification by MD5 checksum succeeded.', 'INFO')
            self.logger.log("Finished.")
            return result['id']

        if not success and retryCount <= 0:
            return None
        else:
            time.sleep(self.retryDelay)
            self.logger.log('Retrying upload of {}.'.format(fullPath),
                            'warning')
            self.uploadFileToCloudStorage(fullPath = fullPath,
                                          retryCount = retryCount - 1)


    def __retrieveCredentials(self):
        """
        Perform authorization at the server.

        Credentials are loaded into the object attribute googleAPICredentials.
        """

        flow = OAuth2WebServerFlow(self.clientID, self.clientSecret,
                                   self.oauthScope, self.oauthConsent)
        authorize_url = flow.step1_get_authorize_url()
        print 'Go to the following link in your browser: ' + authorize_url
        code = raw_input('Enter verification code: ').strip()
        self.googleAPICredentials = flow.step2_exchange(code)

        print "refresh_token = {}".format(
            self.googleAPICredentials.refresh_token)
        print "expiry = {}".format(self.googleAPICredentials.token_expiry)


    def freeSpace(self):
        """
        Get free space from the drive service.
        :param driveService: Object for the drive service.
        :returns: Int of free space (bytes B) on the drive service.
        """
        aboutData = self.driveService.about().get().execute()
        return int(aboutData['quotaBytesTotal']) - int(
            aboutData['quotaBytesUsed']) - int(
            aboutData['quotaBytesUsedInTrash'])


    def deleteFile(self, fileID = ''):
        """
        Delete the file with ID fileID.
        :param fileID: String of a Google API file ID.
        """

        if not len(fileID) > 0:
            raise Exception("File ID has not been given.")

        self.logger.log(
            'Deleting file with file ID {} and name {}.'.format(fileID,
                                                                self.filenameForFileID(
                                                                    fileID)),
            'debug')

        try:
            # Writing the fileId arg name is required here.
            self.driveService.files().delete(fileId = fileID).execute()

        except errors.HttpError as error:
            self.logger.log('Exception while deleting: {}'.format(error),
                            'error')


    def deleteOutdatedFiles(self, maxAge = datetime.timedelta(weeks = 9999999)):
        """
        Remove outdated files from cloud storage.

        :param minAge: datetime.timedelta of the minimum age before a file is
        considered outdated.
        :param maxAge: datetime.timedelta of the maximum age to consider for
        a file.
        :returns: Int count of deleted items.
        """

        # @todo Return count of actual successfully deleted files.

        outdated = self.outdatedFiles(maxAge)

        """:type : dict"""
        for f in outdated:
            self.deleteFile(f['id'])

        return len(outdated)


    def outdatedFiles(self,
                      daysBeforeOutdated = datetime.timedelta(days = 9999999)):
        """
        Outdated files in the cloud where they are outdated if their age is
        greater than or equal to daysBeforeOutdated.

        Note: When t1 is the same day as t2, the timedelta comes back as -1.
        Not sure why this isn't represented as zero. Perhaps to avoid a false
        evaluation of a predicate on a tdelta.

        :param daysBeforeOutdated: datetime.timedelta where the value
        indicates that outdated files that have an age greater than this
        parameter.
        :return: Int count of deleted items.
        """

        t1 = lambda x: datetime.datetime.strptime(x['createdDate'],
                                                  "%Y-%m-%dT%H:%M:%S.%fZ")
        t2 = datetime.datetime.now()

        return filter(lambda x: t2 - t1(x) >= daysBeforeOutdated,
                      self.cloudFiles['items'])


    def sendNotificationOfFiles(self):
        """
        Provide a notification that lists the export files along with sharing
        links.
        """

        pass


    def sendDownloadableFiles(self):
        """
        Send available files via HTTP POST.
        :returns: None
        """

        myPath = '{}/{}'.format(self.exportTempWorkPath,
                                'list-of-downloadable-files.txt')

        fp = open(myPath, 'wb')

        output = StringIO()
        output.write(self.markdownListOfDownloadableFiles())

        fp.write(self.markdownListOfDownloadableFiles())
        fp.close()

        headers = {'User-Agent': self.postAgent, 'Content-Type': 'text/html'}
        try:
            r = requests.post(self.configer.configOptionValue('Export',
                                                              'export_list_post_url'),
                              output.getvalue(), headers = headers)
            print 'text: {}'.format(r.text)
        except requests.adapters.SSLError as error:
            # @todo Implement alternative verification.
            self.logger.log('SSL error: {}'.format(error), 'error')

        output.close()


    def metadataOfFileID(self, fileID = ''):
        """
        :param fileID: String of a file ID in the cloud.
        :return: Tuple of metadata (name, url, timestamp, size) for a given
        file ID.
        """
        item = [i for i in self.cloudFiles['items'] if i['id'] == fileID][0]
        return (item[u'originalFilename'], item[u'webContentLink'],
                item[u'createdDate'], item[u'fileSize'])


    def listOfDownloadableFiles(self):
        """
        Create a list of downloadable files.
        :returns: List of dicts of files that are downloadable from the cloud.
        """

        files = []
        for i in reversed(sorted(self.cloudFiles['items'],
                                 key = lambda k: k['createdDate'])):
            item = dict()
            item['title'] = i['title']
            item['webContentLink'] = i['webContentLink']
            item['id'] = i['id']
            item['createdDate'] = i['createdDate']
            item['fileSize'] = i['fileSize']
            files.append(item)
        return files


    def markdownListOfDownloadableFiles(self):
        """
        Generate content containing a list of downloadable files in Markdown
        format.

        :returns: String content in Markdown format.
        """

        content = "||*Name*||*Created*||*Size*||\n"
        for i in self.listOfDownloadableFiles():
            content += "||[`{}`]({})".format(i['title'], i['webContentLink'])
            content += "||`{}`".format(i['createdDate'])
            content += "||`{} B`||".format(int(i['fileSize']))
            content += '\n'

        # self.logger.log('content: {}'.format(content))
        return content


    def plaintextListOfDownloadableFiles(self):
        """
        Generate content containing a list of downloadable files in plaintext
        format.

        :returns: String content as plaintext.
        """
        content = ''
        includeLink = False
        for i in reversed(sorted(self.cloudFiles['items'],
                                 key = lambda k: k['createdDate'])):
            if includeLink:
                content += "{}, {}, {}, {} B\n".format(i['title'],
                                                       i['webContentLink'],
                                                       i['createdDate'],
                                                       int(i['fileSize']))
            else:
                content += "{}, {}, {} B\n".format(i['title'], i['createdDate'],
                                                   int(i['fileSize']))

        return content


    def logSuccessfulExport(self, name = '', url = '', datetime = 0, size = 0):
        """
        When an export has been successful, log information about the export
        to the database.

        The items to log include:
        * filename
        * URL
        * timestamp
        * filesize

        :param name: String
        :param url: String
        :param datetime:
        :param size: Int
        :return: True if no errors occurred, else False.
        """

        def exportHistoryColumns():
            return ['name', 'url', 'timestamp', 'size']

        timestamp = lambda \
                datetime: 'to_timestamp(0)' if datetime == 0 else "timestamp " \
                                                                  "'{}'".format(
            datetime)

        sql = 'INSERT INTO "{0}" ({1}) VALUES ({2}, {3}, {4}, {5})'.format(
            self.configer.configOptionValue('Export', 'export_history_table'),
            ','.join(exportHistoryColumns()), "'" + name + "'", "'" + url + "'",
            timestamp(datetime), size)

        conn = MSGDBConnector().connectDB()
        cursor = conn.cursor()
        dbUtil = MSGDBUtil()
        result = dbUtil.executeSQL(cursor, sql, exitOnFail = False)
        conn.commit()
        return result


    def sendExportSummary(self, summary = ''):
        """
        Send a summary of exports via email to a preconfigured list of
        recipients.
        :param summary: String of summary content.
        :return:
        """
        try:
            if self.notifier.sendNotificationEmail(summary, testing = False):
                self.notifier.recordNotificationEvent(
                    types = MSGNotificationHistoryTypes,
                    noticeType = MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
        except Exception as detail:
            self.logger.log('Exception occurred: {}'.format(detail), 'ERROR')


    def currentExportSummary(self):
        """
        Current summary of exports since the last summary report time.

        Summaries are reported with identifier MSG_EXPORT_SUMMARY in the
        NotificationHistory.

        Includes:
        * Number of databases exported
        * Total number of files in the cloud.
        * A report of available storage capacity.
        * A list of available DBs.
        * A link where exports can be accessed.

        :return: String of summary text.
        """
        availableFilesURL = self.configer.configOptionValue('Export',
                                                            'export_list_url')
        lastReportDate = self.notifier.lastReportDate(
            types = MSGNotificationHistoryTypes,
            noticeType = MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
        content = 'Cloud Export Summary:\n\n'
        content += 'Last report date: {}\n'.format(lastReportDate)

        # @TO BE REVIEWED: Verify time zone adjustment.
        content += '{} databases have been exported since the last report ' \
                   'date.\n'.format(self.countOfDBExports(
            lastReportDate + datetime.timedelta(
                hours = 10)) if lastReportDate else self.countOfDBExports())

        content += '{} B free space is available.\n'.format(self.freeSpace())
        content += '\nCurrently available DBs:\n'
        content += self.plaintextListOfDownloadableFiles()
        content += '\n{} files can be accessed through Google Drive (' \
                   'https://drive.google.com) or at {}.'.format(
            self.countOfCloudFiles(), availableFilesURL)

        return content


    def countOfDBExports(self, since = None):
        """
        :param since: datetime indicating last export datetime.
        :return: Int of count of exports.
        """
        myDatetime = lambda x: datetime.datetime.strptime(x, '%Y-%m-%d %H:%S')
        if not since:
            since = myDatetime('1900-01-01 00:00')
        self.logger.log(since.strftime('%Y-%m-%d %H:%M'), 'DEBUG')

        sql = 'SELECT COUNT("public"."ExportHistory"."timestamp") FROM ' \
              '"public"."ExportHistory" WHERE "timestamp" > \'{}\''.format(
            since.strftime('%Y-%m-%d %H:%M'))

        conn = MSGDBConnector().connectDB()
        cursor = conn.cursor()
        dbUtil = MSGDBUtil()
        rows = None
        if dbUtil.executeSQL(cursor, sql, exitOnFail = False):
            rows = cursor.fetchall()
        assert len(rows) == 1, 'Invalid return value.'
        return rows[0][0]


    def countOfCloudFiles(self):
        """
        :param since: datetime indicating last trailing export datetime.
        :return: Int of count of exports.
        """
        return len(self.cloudFiles['items'])


    def __verifyMD5Sum(self, localFilePath, remoteFileID):
        """
        Verify that the local MD5 sum matches the MD5 sum for the remote file
        corresponding to an ID.

        This verifies that the uploaded file matches the local compressed
        export file.

        :param localFilePath: String of the full path of the local file.
        :param remoteFileID: String of the cloud ID for the remote file.
        :returns: Boolean True if the MD5 sums match, otherwise, False.
        """

        self.logger.log('remote file ID: {}'.format(remoteFileID))
        self.logger.log('local file path: {}'.format(localFilePath))

        # Get the md5sum for the local file.
        f = open(localFilePath, mode = 'rb')
        fContent = hashlib.md5()
        for buf in iter(partial(f.read, 128), b''):
            fContent.update(buf)
        localMD5Sum = fContent.hexdigest()
        f.close()

        self.logger.log('local md5: {}'.format(localMD5Sum), 'DEBUG')

        def verifyFile():
            # Get the MD5 sum for the remote file.
            for item in self.cloudFiles['items']:
                if (item['id'] == remoteFileID):
                    self.logger.log(
                        'remote md5: {}'.format(item['md5Checksum']), 'DEBUG')
                    if localMD5Sum == item['md5Checksum']:
                        return True
                    else:
                        return False

        try:
            if verifyFile():
                return True
            else:
                return False

        except errors.HttpError as detail:
            self.logger.log('HTTP error during MD5 verification.', 'error')

            time.sleep(10)

            if verifyFile():
                return True
            else:
                return False


    def fileIDForFileName(self, filename):
        """
        Get the file ID for the given filename.

        This method supports matching multiple cloud filenames but only
        returns the ID for a single matching filename.

        This can then be called recursively to obtain all the file IDs for a
        given filename.

        :param String of the filename for which to retrieve the ID.
        :returns: String of a cloud file ID or None if no match.
        """
        fileIDList = filter(lambda x: x['originalFilename'] == filename,
                            self.cloudFiles['items'])
        return fileIDList[0]['id'] if len(fileIDList) > 0 else None


    def filenameForFileID(self, fileID = ''):
        """
        :param fileID: String of cloud-based file ID.
        :return: String of filename for a given file ID.
        """
        return filter(lambda x: x['id'] == fileID, self.cloudFiles['items'])[0][
            'originalFilename']


    def addReaders(self, fileID = None, emailAddressList = None,
                   retryCount = 0):
        """
        Add reader permission to an export file that has been uploaded to the
        cloud for the given list of email addresses.

        Email notification is suppressed by default.

        :param fileID: String of the cloud file ID to be processed.
        :param emailAddressList: List of email addresses.
        :returns: Boolean True if successful, otherwise False.
        """
        # @todo Provide support for retry count
        success = True

        self.logger.log('file id: {}'.format(fileID))
        self.logger.log('address list: {}'.format(emailAddressList))

        for addr in emailAddressList:
            permission = {'value': addr, 'type': 'user', 'role': 'reader'}

            if fileID:
                try:
                    resp = self.driveService.permissions().insert(
                        fileId = fileID, sendNotificationEmails = False,
                        body = permission).execute()
                    self.logger.log(
                        'Reader permission added for {}.'.format(addr))
                except errors.HttpError as error:
                    self.logger.log('An error occurred: {}'.format(error))
                    success = False

        if not success and retryCount <= 0:
            return False
        elif success:
            return True
        else:
            time.sleep(self.retryDelay)
            self.logger.log('Retrying adding readers for ID {}.'.format(fileID),
                            'warning')
            self.addReaders(fileID = fileID,
                            emailAddressList = emailAddressList,
                            retryCount = retryCount - 1)
Esempio n. 10
0
class MSGDBExporterTester(unittest.TestCase):
    """
    Unit tests for the MSG Cloud Exporter.
    """
    def setUp(self):
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.configer = MSGConfiger()
        self.exporter = MSGDBExporter()
        self.testDir = 'db_exporter_test'
        self.uncompressedTestFilename = 'meco_v3_test_data.sql'
        self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
        self.exportTestDataPath = self.configer.configOptionValue(
            'Testing', 'export_test_data_path')
        self.fileUtil = MSGFileUtil()
        self.fileChunks = []
        self.testDataFileID = ''
        self.pyUtil = MSGPythonUtil()
        self.timeUtil = MSGTimeUtil()

        conn = None
        try:
            conn = MSGDBConnector().connectDB()
        except Exception as detail:
            self.logger.log("Exception occurred: {}".format(detail), 'error')
            exit(-1)

        self.logger.log("conn = {}".format(conn), 'debug')
        self.assertIsNotNone(conn)

        # Create a temporary working directory.
        try:
            os.mkdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception during creation of temp directory: %s' % detail,
                'ERROR')

    def tearDown(self):
        """
        Delete all test items.
        """

        REMOVE_TEMPORARY_FILES = True
        if REMOVE_TEMPORARY_FILES:
            try:
                self.logger.log(
                    "Removing local test files {}, {}.".format(
                        self.uncompressedTestFilename,
                        self.compressedTestFilename), 'debug')
                os.remove(
                    os.path.join(os.getcwd(), self.testDir,
                                 self.uncompressedTestFilename))
                os.remove(
                    os.path.join(os.getcwd(), self.testDir,
                                 self.compressedTestFilename))
            except OSError as detail:
                self.logger.log(
                    'Exception while removing temporary files: {}'.format(
                        detail), 'SILENT')
            try:
                os.remove(
                    os.path.join(os.getcwd(), self.testDir,
                                 self.compressedTestFilename))
            except OSError as detail:
                self.logger.log(
                    'Exception while removing temporary files: {}'.format(
                        detail), 'SILENT')
            try:
                for f in self.fileChunks:
                    os.remove(f)
            except OSError as detail:
                self.logger.log(
                    'Exception while removing temporary files: {}'.format(
                        detail), 'DEBUG')

        try:
            os.rmdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception while removing directory: {}'.format(detail),
                'ERROR')

        # Keep deleting from the cloud until there are no more to delete.
        def deleteFromCloud():
            self.logger.log("deleting from cloud", 'debug')
            try:
                fileIDToDelete = self.exporter.fileIDForFileName(
                    self.compressedTestFilename)
                if fileIDToDelete is None:
                    return
                self.logger.log("file ID to delete: {}".format(fileIDToDelete),
                                'DEBUG')
                self.exporter.driveService.files().delete(
                    fileId='{}'.format(fileIDToDelete)).execute()
                deleteFromCloud()
            except (TypeError, http.HttpError) as e:
                self.logger.log('Delete not successful: {}'.format(e), 'DEBUG')

        deleteFromCloud()

    def _upload_test_data_to_cloud(self):
        """
        Provide an upload of test data that can be used in other tests.

        Side effect: Store the file ID as an ivar.
        """
        self.logger.log("Uploading test data for caller: {}".format(
            self.pyUtil.callerName()))

        filePath = "{}/{}".format(self.exportTestDataPath,
                                  self.compressedTestFilename)
        self.logger.log('Uploaded {}.'.format(filePath), 'info')

        uploadResult = self.exporter.uploadFileToCloudStorage(filePath)
        self.logger.log('upload result: {}'.format(uploadResult))

        self.testDataFileID = self.exporter.fileIDForFileName(
            self.compressedTestFilename)
        self.logger.log("Test file ID is {}.".format(self.testDataFileID))

    def test_markdown_list_of_downloadable_files(self):
        """
        Match the Markdown line entry for the uploaded file.
        """
        # @REVIEWED
        self._upload_test_data_to_cloud()
        self.assertEquals(
            len(
                filter(
                    lambda x: self.testDataFileID in x,
                    self.exporter.markdownListOfDownloadableFiles().splitlines(
                    ))), 1)

    def test_get_md5_sum_from_cloud(self):
        """
        Test retrieving the MD5 sum from the cloud.
        """
        # @REVIEWED
        self.logger.log('Testing getting the MD5 sum.', 'info')
        self._upload_test_data_to_cloud()
        testFileMD5 = filter(
            lambda x: x['id'] == self.testDataFileID,
            self.exporter.cloudFiles['items'])[0]['md5Checksum']
        self.assertEquals(len(testFileMD5), 32)
        self.assertTrue(re.match(r'[0-9A-Za-z]+', testFileMD5))

    def test_get_file_id_for_nonexistent_file(self):
        """
        Test getting a file ID for a nonexistent file.
        """
        # @REVIEWED
        fileIDs = self.exporter.fileIDForFileName('nonexistent_file')
        self.logger.log("file ids = {}".format(fileIDs), 'info')
        self.assertIsNone(fileIDs)

    def test_upload_test_data(self):
        """
        Upload a test data file for unit testing of DB export.

        The unit test data file is a predefined set of test data stored in
        the test data path of the software distribution.
        """
        # @REVIEWED

        self._upload_test_data_to_cloud()
        self.assertGreater(len(self.testDataFileID), 0)
        self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))

    def test_adding_reader_permissions(self):
        """
        Add reader permissions to a file that was uploaded.
        """
        # @REVIEWED
        self.logger.log("Testing adding reader permissions.")
        self._upload_test_data_to_cloud()

        email = self.configer.configOptionValue('Testing',
                                                'tester_email_address')
        service = self.exporter.driveService
        try:
            id_resp = service.permissions().getIdForEmail(
                email=email).execute()
            print id_resp

        except errors.HttpError as detail:
            print 'Exception while getting ID for email: {}'.format(detail)

        new_permission = {'value': email, 'type': 'user', 'role': 'reader'}
        try:
            self.logger.log('Adding reader permission', 'INFO')
            fileIDToAddTo = self.testDataFileID

            # The permission dict is being output to stdout here.
            resp = service.permissions().insert(fileId=fileIDToAddTo,
                                                sendNotificationEmails=False,
                                                body=new_permission).execute()
        except errors.HttpError as detail:
            self.logger.log(
                'Exception while adding reader permissions: {}'.format(detail),
                'error')

        def permission_id(email):
            try:
                id_resp = service.permissions().getIdForEmail(
                    email=email).execute()
                return id_resp['id']
            except errors.HttpError as error:
                self.logger.log("HTTP error: {}".format(error))

        permission = {}
        try:
            permission = service.permissions().get(
                fileId=self.testDataFileID,
                permissionId=permission_id(email)).execute()
        except errors.HttpError as error:
            self.logger.log("HTTP error: {}".format(error))

        self.assertEquals(permission['role'], 'reader')

    def test_create_compressed_archived(self):
        """
        * Copy test data to a temp directory (self.testDir).
        * Create a checksum for test data.
        * Create a gzip-compressed archive.
        * Extract gzip-compressed archive.
        * Create a checksum for the uncompressed data.
        * Compare the checksums.
        """
        # @REVIEWED

        self.logger.log('Testing verification of a compressed archive.')

        self.logger.log('cwd {}'.format(os.getcwd()))
        fullPath = '{}'.format(
            os.path.join(os.getcwd(), self.testDir,
                         self.uncompressedTestFilename))
        shutil.copyfile(
            '{}/{}'.format(self.exportTestDataPath,
                           self.uncompressedTestFilename), fullPath)

        md5sum1 = self.fileUtil.md5Checksum(fullPath)

        self.exporter.fileUtil.gzipCompressFile(fullPath)

        try:
            os.remove(
                os.path.join(os.getcwd(), self.testDir,
                             self.uncompressedTestFilename))
        except OSError as detail:
            self.logger.log('Exception while removing: {}'.format(detail),
                            'ERROR')

        # Extract archived data and generate checksum.
        src = gzip.open('{}{}'.format(fullPath, '.gz'), "rb")
        uncompressed = open(fullPath, "wb")
        decoded = src.read()
        uncompressed.write(decoded)
        uncompressed.close()

        md5sum2 = self.fileUtil.md5Checksum(fullPath)

        self.assertEqual(
            md5sum1, md5sum2, 'Checksums are not equal for original and new '
            'decompressed archive.')

    def test_export_db(self):
        """
        Perform a quick test of the DB export method using Testing Mode.

        This requires sudo authorization to complete.
        """
        # @REVIEWED

        self.logger.log('Testing exportDB using the testing DB.')

        # @todo handle case where testing db does not exist.

        dbs = ['test_meco']
        ids = self.exporter.exportDBs(databases=dbs,
                                      toCloud=True,
                                      localExport=True)
        self.logger.log('Count of exports: {}'.format(len(ids)))
        self.assertEquals(len(ids), 1, "Count of exported files is wrong.")

        map(self.exporter.deleteFile, ids)

    def test_split_archive(self):
        """
        Test splitting an archive into chunks.
        """
        # @REVIEWED
        self.logger.log('Testing archive splitting.')
        fullPath = '{}/{}'.format(self.exportTestDataPath,
                                  self.compressedTestFilename)
        self.logger.log('fullpath: {}'.format(fullPath))
        shutil.copyfile(
            fullPath, '{}/{}'.format(self.testDir,
                                     self.compressedTestFilename))
        fullPath = '{}/{}'.format(self.testDir, self.compressedTestFilename)

        self.fileChunks = self.fileUtil.splitLargeFile(fullPath=fullPath,
                                                       numChunks=3)
        self.assertEquals(len(self.fileChunks), 3)

    def test_get_file_size(self):
        """
        Test retrieving local file sizes.
        """
        # @REVIEWED
        fullPath = '{}/{}'.format(self.exportTestDataPath,
                                  self.compressedTestFilename)
        fSize = self.fileUtil.fileSize(fullPath)
        self.logger.log('size: {}'.format(fSize))
        self.assertEqual(fSize, 12279, 'File size is correct.')

    def test_upload_export_files_list(self):
        """
        TBW
        """
        pass

    def test_checksum_after_upload(self):
        """
        TBW
        """
        pass

    def test_dump_exclusions_dictionary(self):
        """
        Verify the exclusions dictionary by its type.
        :return:
        """
        # @REVIEWED
        exclusions = self.exporter.dumpExclusionsDictionary()

        if exclusions:
            self.assertEquals(type({}), type(exclusions))

    def test_move_to_final(self):
        """
        Test moving a file to the final destination path.
        """
        # @REVIEWED
        self.logger.log('Testing moving to final path {}.'.format(
            self.configer.configOptionValue('Export', 'db_export_final_path')))

        origCompressedFile = '{}/{}'.format(
            self.configer.configOptionValue('Export', 'export_test_data_path'),
            self.compressedTestFilename)
        newCompressedFile = '{}/{}'.format(
            self.configer.configOptionValue('Export', 'export_test_data_path'),
            'temp_test_file')

        shutil.copyfile(origCompressedFile, newCompressedFile)

        self.exporter.moveToFinalPath(compressedFullPath=newCompressedFile)

        self.assertTrue(
            os.path.isfile('{}/{}'.format(
                self.configer.configOptionValue('Export',
                                                'db_export_final_path'),
                'temp_test_file')))

        # Remove the test file.
        os.remove('{}/{}'.format(
            self.configer.configOptionValue('Export', 'db_export_final_path'),
            'temp_test_file'))

    def test_log_successful_export(self):
        """
        Test logging of export results to the export history table.
        """
        # @REVIEWED
        self.assertTrue(
            self.exporter.logSuccessfulExport(name='test_export',
                                              url='http://test_url',
                                              datetime=0,
                                              size=100))

        conn = MSGDBConnector().connectDB()
        cursor = conn.cursor()
        dbUtil = MSGDBUtil()

        self.assertTrue(
            dbUtil.executeSQL(
                cursor, 'select * from "ExportHistory" where '
                'timestamp = '
                'to_timestamp(0)'))

        self.assertEqual(len(cursor.fetchall()), 1,
                         "There should only be one result row.")

        self.assertTrue(
            dbUtil.executeSQL(
                cursor, 'delete from "ExportHistory" where '
                'timestamp = to_timestamp(0)'))
        conn.commit()

    def test_metadata_of_file_id(self):
        """
        Test getting the metadata for a file ID.
        """
        # @REVIEWED
        self._upload_test_data_to_cloud()

        self.logger.log('metadata: {}'.format(
            self.exporter.metadataOfFileID(self.testDataFileID)))

        self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))

    def test_filename_for_file_id(self):
        """
        Test returning a file name given a file ID.
        """
        # @REVIEWED
        self._upload_test_data_to_cloud()
        self.assertEquals(
            self.exporter.filenameForFileID(fileID=self.testDataFileID),
            self.compressedTestFilename)

    def test_outdated_files(self):
        # @REVIEWED
        self._upload_test_data_to_cloud()
        time.sleep(1)
        self.logger.log("outdated:")

        # For debugging:
        for item in self.exporter.outdatedFiles(
                daysBeforeOutdated=datetime.timedelta(days=-1)):
            self.logger.log(
                "name: {}, created date: {}".format(item['originalFilename'],
                                                    item['createdDate']),
                'debug')

        # Get all the outdated files where outdated is equal to anything
        # uploaded today or later.
        self.assertTrue(
            self.exporter.outdatedFiles(daysBeforeOutdated=datetime.timedelta(
                days=-1))[0]['id'] == self.testDataFileID)

        self.logger.log('-----')

    def test_delete_outdated(self):
        """
        TBW
        """
        pass

    def test_list_of_downloadable_files(self):
        """
        Test the list of downloadable files used by the available files page.
        """
        # @REVIEWED
        self._upload_test_data_to_cloud()
        self.assertEquals(
            len(
                filter(lambda row: row['id'] == self.testDataFileID,
                       self.exporter.listOfDownloadableFiles())), 1,
            "Test file not present.")

    def test_count_of_db_exports(self):
        count = self.exporter.countOfDBExports(EARLIEST_DATE)
        self.logger.log(count, 'DEBUG')
        self.assertTrue(int(count) or int(count) == int(0))

    def test_count_of_cloud_files(self):
        count = self.exporter.countOfCloudFiles()
        self.assertTrue(int(count) or int(count) == int(0))

    def test_plaintext_list_of_downloadable_files(self):
        """
        This test handles content both with content links and without content
        links.
        """
        content = self.exporter.plaintextListOfDownloadableFiles()
        self.assertRegexpMatches(
            content, '\d+-\d+-\d+.*\,'
            '\s+\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z\,\s+\d+\sB')

    def test_last_report_date(self):
        last_report = self.exporter.notifier.lastReportDate(
            types=MSGNotificationHistoryTypes,
            noticeType=MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
        self.assertTrue(last_report is None or last_report > EARLIEST_DATE)

    def test_current_export_summary(self):
        self.assertRegexpMatches(
            self.exporter.currentExportSummary(),
            re.compile('last.*databases.*free.*currently.*accessed.*',
                       flags=re.IGNORECASE | re.DOTALL))
class MSGDBExporterTester(unittest.TestCase):
    """
    Unit tests for the MSG Cloud Exporter.
    """


    def setUp(self):
        self.logger = SEKLogger(__name__, 'DEBUG')
        self.configer = MSGConfiger()
        self.exporter = MSGDBExporter()
        self.testDir = 'db_exporter_test'
        self.uncompressedTestFilename = 'meco_v3_test_data.sql'
        self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
        self.exportTestDataPath = self.configer.configOptionValue('Testing',
                                                                  'export_test_data_path')
        self.fileUtil = MSGFileUtil()
        self.fileChunks = []
        self.testDataFileID = ''
        self.pyUtil = MSGPythonUtil()
        self.timeUtil = MSGTimeUtil()

        conn = None
        try:
            conn = MSGDBConnector().connectDB()
        except Exception as detail:
            self.logger.log("Exception occurred: {}".format(detail), 'error')
            exit(-1)

        self.logger.log("conn = {}".format(conn), 'debug')
        self.assertIsNotNone(conn)

        # Create a temporary working directory.
        try:
            os.mkdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception during creation of temp directory: %s' % detail,
                'ERROR')


    def tearDown(self):
        """
        Delete all test items.
        """

        REMOVE_TEMPORARY_FILES = True
        if REMOVE_TEMPORARY_FILES:
            try:
                self.logger.log("Removing local test files {}, {}.".format(
                    self.uncompressedTestFilename, self.compressedTestFilename),
                                'debug')
                os.remove(os.path.join(os.getcwd(), self.testDir,
                                       self.uncompressedTestFilename))
                os.remove(os.path.join(os.getcwd(), self.testDir,
                                       self.compressedTestFilename))
            except OSError as detail:
                self.logger.log(
                    'Exception while removing temporary files: {}'.format(
                        detail), 'SILENT')
            try:
                os.remove(os.path.join(os.getcwd(), self.testDir,
                                       self.compressedTestFilename))
            except OSError as detail:
                self.logger.log(
                    'Exception while removing temporary files: {}'.format(
                        detail), 'SILENT')
            try:
                for f in self.fileChunks:
                    os.remove(f)
            except OSError as detail:
                self.logger.log(
                    'Exception while removing temporary files: {}'.format(
                        detail), 'DEBUG')

        try:
            os.rmdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception while removing directory: {}'.format(detail),
                'ERROR')

        # Keep deleting from the cloud until there are no more to delete.
        def deleteFromCloud():
            self.logger.log("deleting from cloud", 'debug')
            try:
                fileIDToDelete = self.exporter.fileIDForFileName(
                    self.compressedTestFilename)
                if fileIDToDelete is None:
                    return
                self.logger.log("file ID to delete: {}".format(fileIDToDelete),
                                'DEBUG')
                self.exporter.driveService.files().delete(
                    fileId = '{}'.format(fileIDToDelete)).execute()
                deleteFromCloud()
            except (TypeError, http.HttpError) as e:
                self.logger.log('Delete not successful: {}'.format(e), 'DEBUG')


        deleteFromCloud()


    def _upload_test_data_to_cloud(self):
        """
        Provide an upload of test data that can be used in other tests.

        Side effect: Store the file ID as an ivar.
        """
        self.logger.log("Uploading test data for caller: {}".format(
            self.pyUtil.callerName()))

        filePath = "{}/{}".format(self.exportTestDataPath,
                                  self.compressedTestFilename)
        self.logger.log('Uploaded {}.'.format(filePath), 'info')

        uploadResult = self.exporter.uploadFileToCloudStorage(filePath)
        self.logger.log('upload result: {}'.format(uploadResult))

        self.testDataFileID = self.exporter.fileIDForFileName(
            self.compressedTestFilename)
        self.logger.log("Test file ID is {}.".format(self.testDataFileID))


    def test_markdown_list_of_downloadable_files(self):
        """
        Match the Markdown line entry for the uploaded file.
        """
        # @REVIEWED
        self._upload_test_data_to_cloud()
        self.assertEquals(len(filter(lambda x: self.testDataFileID in x,
                                     self.exporter.markdownListOfDownloadableFiles().splitlines())),
                          1)


    def test_get_md5_sum_from_cloud(self):
        """
        Test retrieving the MD5 sum from the cloud.
        """
        # @REVIEWED
        self.logger.log('Testing getting the MD5 sum.', 'info')
        self._upload_test_data_to_cloud()
        testFileMD5 = filter(lambda x: x['id'] == self.testDataFileID,
                             self.exporter.cloudFiles['items'])[0][
            'md5Checksum']
        self.assertEquals(len(testFileMD5), 32)
        self.assertTrue(re.match(r'[0-9A-Za-z]+', testFileMD5))


    def test_get_file_id_for_nonexistent_file(self):
        """
        Test getting a file ID for a nonexistent file.
        """
        # @REVIEWED
        fileIDs = self.exporter.fileIDForFileName('nonexistent_file')
        self.logger.log("file ids = {}".format(fileIDs), 'info')
        self.assertIsNone(fileIDs)


    def test_upload_test_data(self):
        """
        Upload a test data file for unit testing of DB export.

        The unit test data file is a predefined set of test data stored in
        the test data path of the software distribution.
        """
        # @REVIEWED

        self._upload_test_data_to_cloud()
        self.assertGreater(len(self.testDataFileID), 0)
        self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))


    def test_adding_reader_permissions(self):
        """
        Add reader permissions to a file that was uploaded.
        """
        # @REVIEWED
        self.logger.log("Testing adding reader permissions.")
        self._upload_test_data_to_cloud()

        email = self.configer.configOptionValue('Testing',
                                                'tester_email_address')
        service = self.exporter.driveService
        try:
            id_resp = service.permissions().getIdForEmail(
                email = email).execute()
            print id_resp

        except errors.HttpError as detail:
            print 'Exception while getting ID for email: {}'.format(detail)

        new_permission = {'value': email, 'type': 'user', 'role': 'reader'}
        try:
            self.logger.log('Adding reader permission', 'INFO')
            fileIDToAddTo = self.testDataFileID

            # The permission dict is being output to stdout here.
            resp = service.permissions().insert(fileId = fileIDToAddTo,
                                                sendNotificationEmails = False,
                                                body = new_permission).execute()
        except errors.HttpError as detail:
            self.logger.log(
                'Exception while adding reader permissions: {}'.format(detail),
                'error')


        def permission_id(email):
            try:
                id_resp = service.permissions().getIdForEmail(
                    email = email).execute()
                return id_resp['id']
            except errors.HttpError as error:
                self.logger.log("HTTP error: {}".format(error))


        permission = {}
        try:
            permission = service.permissions().get(fileId = self.testDataFileID,
                                                   permissionId = permission_id(
                                                       email)).execute()
        except errors.HttpError as error:
            self.logger.log("HTTP error: {}".format(error))

        self.assertEquals(permission['role'], 'reader')


    def test_create_compressed_archived(self):
        """
        * Copy test data to a temp directory (self.testDir).
        * Create a checksum for test data.
        * Create a gzip-compressed archive.
        * Extract gzip-compressed archive.
        * Create a checksum for the uncompressed data.
        * Compare the checksums.
        """
        # @REVIEWED

        self.logger.log('Testing verification of a compressed archive.')

        self.logger.log('cwd {}'.format(os.getcwd()))
        fullPath = '{}'.format(os.path.join(os.getcwd(), self.testDir,
                                            self.uncompressedTestFilename))
        shutil.copyfile('{}/{}'.format(self.exportTestDataPath,
                                       self.uncompressedTestFilename), fullPath)

        md5sum1 = self.fileUtil.md5Checksum(fullPath)

        self.exporter.fileUtil.gzipCompressFile(fullPath)

        try:
            os.remove(os.path.join(os.getcwd(), self.testDir,
                                   self.uncompressedTestFilename))
        except OSError as detail:
            self.logger.log('Exception while removing: {}'.format(detail),
                            'ERROR')

        # Extract archived data and generate checksum.
        src = gzip.open('{}{}'.format(fullPath, '.gz'), "rb")
        uncompressed = open(fullPath, "wb")
        decoded = src.read()
        uncompressed.write(decoded)
        uncompressed.close()

        md5sum2 = self.fileUtil.md5Checksum(fullPath)

        self.assertEqual(md5sum1, md5sum2,
                         'Checksums are not equal for original and new '
                         'decompressed archive.')


    def test_export_db(self):
        """
        Perform a quick test of the DB export method using Testing Mode.

        This requires sudo authorization to complete.
        """
        # @REVIEWED

        self.logger.log('Testing exportDB using the testing DB.')

        # @todo handle case where testing db does not exist.

        dbs = ['test_meco']
        ids = self.exporter.exportDBs(databases = dbs, toCloud = True,
                                      localExport = True)
        self.logger.log('Count of exports: {}'.format(len(ids)))
        self.assertEquals(len(ids), 1, "Count of exported files is wrong.")

        map(self.exporter.deleteFile, ids)


    def test_split_archive(self):
        """
        Test splitting an archive into chunks.
        """
        # @REVIEWED
        self.logger.log('Testing archive splitting.')
        fullPath = '{}/{}'.format(self.exportTestDataPath,
                                  self.compressedTestFilename)
        self.logger.log('fullpath: {}'.format(fullPath))
        shutil.copyfile(fullPath, '{}/{}'.format(self.testDir,
                                                 self.compressedTestFilename))
        fullPath = '{}/{}'.format(self.testDir, self.compressedTestFilename)

        self.fileChunks = self.fileUtil.splitLargeFile(fullPath = fullPath,
                                                       numChunks = 3)
        self.assertEquals(len(self.fileChunks), 3)


    def test_get_file_size(self):
        """
        Test retrieving local file sizes.
        """
        # @REVIEWED
        fullPath = '{}/{}'.format(self.exportTestDataPath,
                                  self.compressedTestFilename)
        fSize = self.fileUtil.fileSize(fullPath)
        self.logger.log('size: {}'.format(fSize))
        self.assertEqual(fSize, 12279, 'File size is correct.')


    def test_upload_export_files_list(self):
        """
        TBW
        """
        pass


    def test_checksum_after_upload(self):
        """
        TBW
        """
        pass


    def test_dump_exclusions_dictionary(self):
        """
        Verify the exclusions dictionary by its type.
        :return:
        """
        # @REVIEWED
        exclusions = self.exporter.dumpExclusionsDictionary()

        if exclusions:
            self.assertEquals(type({}), type(exclusions))


    def test_move_to_final(self):
        """
        Test moving a file to the final destination path.
        """
        # @REVIEWED
        self.logger.log('Testing moving to final path {}.'.format(
            self.configer.configOptionValue('Export', 'db_export_final_path')))

        origCompressedFile = '{}/{}'.format(
            self.configer.configOptionValue('Export', 'export_test_data_path'),
            self.compressedTestFilename)
        newCompressedFile = '{}/{}'.format(
            self.configer.configOptionValue('Export', 'export_test_data_path'),
            'temp_test_file')

        shutil.copyfile(origCompressedFile, newCompressedFile)

        self.exporter.moveToFinalPath(compressedFullPath = newCompressedFile)

        self.assertTrue(os.path.isfile('{}/{}'.format(
            self.configer.configOptionValue('Export', 'db_export_final_path'),
            'temp_test_file')))

        # Remove the test file.
        os.remove('{}/{}'.format(
            self.configer.configOptionValue('Export', 'db_export_final_path'),
            'temp_test_file'))


    def test_log_successful_export(self):
        """
        Test logging of export results to the export history table.
        """
        # @REVIEWED
        self.assertTrue(self.exporter.logSuccessfulExport(name = 'test_export',
                                                          url =
                                                          'http://test_url',
                                                          datetime = 0,
                                                          size = 100))

        conn = MSGDBConnector().connectDB()
        cursor = conn.cursor()
        dbUtil = MSGDBUtil()

        self.assertTrue(
            dbUtil.executeSQL(cursor, 'select * from "ExportHistory" where '
                                      'timestamp = '
                                      'to_timestamp(0)'))

        self.assertEqual(len(cursor.fetchall()), 1,
                         "There should only be one result row.")

        self.assertTrue(
            dbUtil.executeSQL(cursor, 'delete from "ExportHistory" where '
                                      'timestamp = to_timestamp(0)'))
        conn.commit()


    def test_metadata_of_file_id(self):
        """
        Test getting the metadata for a file ID.
        """
        # @REVIEWED
        self._upload_test_data_to_cloud()

        self.logger.log('metadata: {}'.format(
            self.exporter.metadataOfFileID(self.testDataFileID)))

        self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))


    def test_filename_for_file_id(self):
        """
        Test returning a file name given a file ID.
        """
        # @REVIEWED
        self._upload_test_data_to_cloud()
        self.assertEquals(
            self.exporter.filenameForFileID(fileID = self.testDataFileID),
            self.compressedTestFilename)


    def test_outdated_files(self):
        # @REVIEWED
        self._upload_test_data_to_cloud()
        time.sleep(1)
        self.logger.log("outdated:")

        # For debugging:
        for item in self.exporter.outdatedFiles(
                daysBeforeOutdated = datetime.timedelta(
                        days = -1)): self.logger.log(
            "name: {}, created date: {}".format(item['originalFilename'],
                                                item['createdDate']), 'debug')

        # Get all the outdated files where outdated is equal to anything
        # uploaded today or later.
        self.assertTrue(self.exporter.outdatedFiles(
            daysBeforeOutdated = datetime.timedelta(days = -1))[0][
                            'id'] == self.testDataFileID)

        self.logger.log('-----')


    def test_delete_outdated(self):
        """
        TBW
        """
        pass


    def test_list_of_downloadable_files(self):
        """
        Test the list of downloadable files used by the available files page.
        """
        # @REVIEWED
        self._upload_test_data_to_cloud()
        self.assertEquals(len(
            filter(lambda row: row['id'] == self.testDataFileID,
                   self.exporter.listOfDownloadableFiles())), 1,
                          "Test file not present.")


    def test_count_of_db_exports(self):
        count = self.exporter.countOfDBExports(EARLIEST_DATE)
        self.logger.log(count,'DEBUG')
        self.assertTrue(int(count) or int(count) == int(0))


    def test_count_of_cloud_files(self):
        count = self.exporter.countOfCloudFiles()
        self.assertTrue(int(count) or int(count) == int(0))


    def test_plaintext_list_of_downloadable_files(self):
        """
        This test handles content both with content links and without content
        links.
        """
        content = self.exporter.plaintextListOfDownloadableFiles()
        self.assertRegexpMatches(content,
                                 '\d+-\d+-\d+.*\,'
                                 '\s+\d+-\d+-\d+T\d+:\d+:\d+\.\d+Z\,\s+\d+\sB')


    def test_last_report_date(self):
        last_report = self.exporter.notifier.lastReportDate(
            types = MSGNotificationHistoryTypes,
            noticeType = MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
        self.assertTrue(last_report is None or last_report > EARLIEST_DATE)

    def test_current_export_summary(self):
        self.assertRegexpMatches(self.exporter.currentExportSummary(),
                                 re.compile(
                                     'last.*databases.*free.*currently.*accessed.*',
                                     flags = re.IGNORECASE | re.DOTALL))
class MSGDBExporterTester(unittest.TestCase):
    """
    Unit tests for the MSG Cloud Exporter.
    """

    def setUp(self):
        self.logger = MSGLogger(__name__, 'DEBUG')
        self.configer = MSGConfiger()
        self.exporter = MSGDBExporter()
        self.testDir = 'db_exporter_test'
        self.uncompressedTestFilename = 'meco_v3_test_data.sql'
        self.compressedTestFilename = 'meco_v3_test_data.sql.gz'
        self.exportTestDataPath = self.configer.configOptionValue('Testing',
                                                                  'export_test_data_path')
        self.fileUtil = MSGFileUtil()
        self.fileChunks = []
        self.testDataFileID = ''
        self.pyUtil = MSGPythonUtil()

        conn = None
        try:
            conn = MSGDBConnector().connectDB()
        except Exception as detail:
            self.logger.log("Exception occurred: {}".format(detail), 'error')
            exit(-1)

        self.logger.log("conn = {}".format(conn), 'debug')
        self.assertIsNotNone(conn)

        # Create a temporary working directory.
        try:
            os.mkdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception during creation of temp directory: %s' % detail,
                'ERROR')

    def upload_test_data_to_cloud(self):
        """
        Provide an upload of test data that can be used in other tests.

        Side effect: Store the file ID as an ivar.
        """
        self.logger.log("Uploading test data for caller: {}".format(
            self.pyUtil.caller_name()))

        filePath = "{}/{}".format(self.exportTestDataPath,
                                  self.compressedTestFilename)
        self.logger.log('Uploaded {}.'.format(filePath), 'info')

        uploadResult = self.exporter.uploadFileToCloudStorage(filePath)
        self.logger.log('upload result: {}'.format(uploadResult))

        self.testDataFileID = self.exporter.fileIDForFileName(
            self.compressedTestFilename)
        self.logger.log("Test file ID is {}.".format(self.testDataFileID))


    def test_sending_fcphase_part_0(self):
        """
        /home/daniel/msg-db-dumps/2014-05-14_141223_fcphase3.sql.gz.0
        """

        filesToUpload = [
            '/home/daniel/msg-db-dumps/2014-05-14_141223_fcphase3.sql.gz.0',
            '/home/daniel/msg-db-dumps/2014-05-14_141223_fcphase3.sql.gz.1',
            '/home/daniel/msg-db-dumps/2014-05-14_141223_fcphase3.sql.gz.2',
            '/home/daniel/msg-db-dumps/2014-05-14_141223_fcphase3.sql.gz.3']

        for f in filesToUpload:
            self.exporter.uploadFileToCloudStorage(fullPath = f,
                                                   testing = False)


    def testListRemoteFiles(self):
        """
        Test listing of remote files.
        """

        self.logger.log('Testing listing of remote files.', 'INFO')
        title = ''
        id = ''
        for item in self.exporter.cloudFiles['items']:
            title = item['title']
            id = item['id']
            self.assertIsNot(title, '')
            self.assertIsNot(id, '')

    def testDownloadURLList(self):
        """
        Test obtaining a list of downloadble URLs.
        """

        self.logger.log('Testing listing of downloadable files.', 'INFO')

        title = ''
        id = ''
        url = ''
        for item in self.exporter.cloudFiles['items']:
            title = item['title']
            url = item['webContentLink']
            id = item['id']
            self.logger.log('title: %s, link: %s, id: %s' % (title, url, id))
            self.assertIsNot(title, '')
            self.assertIsNot(url, '')
            self.assertIsNot(id, '')


    def test_list_of_downloadable_files(self):
        """
        Test the list of downloadable files used by the available files page.
        """
        self.assertIsNotNone(self.exporter.listOfDownloadableFiles(),
                             'List of downloadable files is not available.')
        for row in self.exporter.listOfDownloadableFiles():
            print row
            self.assertIsNotNone(row['id'])
            self.assertIsNotNone(row['title'])
            self.assertIsNotNone(row['webContentLink'])

    def test_markdown_list_of_downloadable_files(self):
        print self.exporter.markdownListOfDownloadableFiles()

        myPath = '{}/{}'.format(
            self.configer.configOptionValue('Export', 'db_export_path'),
            'list-of-downloadable-files.txt')
        fp = open(myPath, 'wb')
        fp.write(self.exporter.markdownListOfDownloadableFiles())
        fp.close()


    def test_get_md5_sum_from_cloud(self):
        """
        Test retrieving the MD5 sum from the cloud.
        """
        # @REVIEWED
        self.logger.log('Testing getting the MD5 sum.', 'info')
        self.upload_test_data_to_cloud()
        testFileMD5 = filter(lambda x: x['id'] == self.testDataFileID,
                             self.exporter.cloudFiles['items'])[0][
            'md5Checksum']
        self.assertEquals(len(testFileMD5), 32)
        self.assertTrue(re.match(r'[0-9A-Za-z]+', testFileMD5))

    def testGetFileIDsForFilename(self):
        """
        Retrieve the matching file IDs for the given file name.
        """

        self.logger.log('Testing getting file IDs for a filename.')
        self.logger.log("Uploading test data.")

        filePath = "{}/{}".format(self.exportTestDataPath,
                                  self.compressedTestFilename)

        uploadResult = self.exporter.uploadFileToCloudStorage(filePath)

        self.assertTrue(uploadResult)

        self.logger.log('Testing getting the file ID for a filename.')

        fileIDs = self.exporter.fileIDForFileName(self.compressedTestFilename)
        self.logger.log("file ids = {}".format(fileIDs), 'info')

        self.assertIsNotNone(fileIDs)

    def test_get_file_id_for_nonexistent_file(self):
        """
        Test getting a file ID for a nonexistent file.
        """

        fileIDs = self.exporter.fileIDForFileName('nonexistent_file')
        self.logger.log("file ids = {}".format(fileIDs), 'info')
        self.assertIsNone(fileIDs)


    def test_upload_test_data(self):
        """
        Upload a test data file for unit testing of DB export.

        The unit test data file is a predefined set of test data stored in
        the test data path of the software distribution.
        """
        # @REVIEWED

        self.upload_test_data_to_cloud()
        self.assertGreater(len(self.testDataFileID), 0)
        self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))

    def test_delete_out_dated_files(self):
        """
        The timestamp of an uploaded file should be set in the past to provide
        the ability to test the deleting of outdated files.
        """

        # return

        # @TO BE REVIEWED  Prevent deleting files uploaded today.
        # @IMPORTANT Prevent deleting NON-testing files.
        # Need to have a test file uploaded that has an explicitly set upload
        # date.

        self.logger.log("Test deleting outdated files.")

        self.logger.log("Uploading test data.")

        filePath = "%s/%s" % (
            self.exportTestDataPath, self.compressedTestFilename)

        uploadResult = self.exporter.uploadFileToCloudStorage(filePath)

        cnt = self.exporter.deleteOutdatedFiles(
            minAge = datetime.timedelta(days = 5),
            maxAge = datetime.timedelta(days = 99999))
        # self.assertGreater(cnt, 0)


    def testAddingReaderPermissions(self):
        """
        Add reader permissions to a file that was uploaded.

        @todo Needs update after cloud export restoration.
        """

        self.logger.log("Testing adding reader permissions.")
        self.logger.log("Uploading test data.")
        filePath = "%s/%s" % (
            self.exportTestDataPath, self.compressedTestFilename)
        uploadResult = self.exporter.uploadFileToCloudStorage(filePath)
        email = self.configer.configOptionValue('Testing', 'tester_email')
        service = self.exporter.driveService
        try:
            id_resp = service.permissions().getIdForEmail(
                email = email).execute()
            print id_resp

        except errors.HttpError as detail:
            print 'Exception while getting ID for email: %s' % detail

        new_permission = {'value': email, 'type': 'user', 'role': 'reader'}
        try:
            self.logger.log('Adding reader permission', 'INFO')
            fileIDToAddTo = self.exporter.fileIDForFileName(
                self.compressedTestFilename)

            # The permission dict is being output to stdout here.
            resp = service.permissions().insert(fileId = fileIDToAddTo,
                                                sendNotificationEmails = False,
                                                body = new_permission).execute()
        except errors.HttpError as detail:
            self.logger.log(
                'Exception while adding reader permissions: %s' % detail,
                'error')


    def test_create_compressed_archived(self):
        """
        * Copy test data to a temp directory.
        * Create a checksum for test data.
        * Create a gzip-compressed archive.
        * Extract gzip-compressed archive.
        * Create a checksum for the uncompressed data.
        * Compare the checksums.

        @todo Needs update after cloud export restoration.
        """

        self.logger.log('Testing verification of a compressed archive.')

        self.logger.log('cwd {}'.format(os.getcwd()))
        fullPath = '{}'.format(os.path.join(os.getcwd(), self.testDir,
                                            self.uncompressedTestFilename))
        shutil.copyfile('{}/{}'.format(self.exportTestDataPath,
                                       self.uncompressedTestFilename), fullPath)

        md5sum1 = self.fileUtil.md5Checksum(fullPath)

        self.exporter.fileUtil.gzipCompressFile(fullPath)

        try:
            os.remove(os.path.join(os.getcwd(), self.testDir,
                                   self.uncompressedTestFilename))
        except OSError as detail:
            self.logger.log('Exception while removing: {}'.format(detail),
                            'ERROR')

        # Extract archived data and generate checksum.
        src = gzip.open('{}{}'.format(fullPath, '.gz'), "rb")
        uncompressed = open(fullPath, "wb")
        decoded = src.read()
        uncompressed.write(decoded)
        uncompressed.close()

        md5sum2 = self.fileUtil.md5Checksum(fullPath)

        self.assertEqual(md5sum1, md5sum2,
                         'Checksums are not equal for original and new '
                         'decompressed archive.')


    def test_export_db(self):
        """
        Perform a quick test of the DB export method using Testing Mode.

        @todo This needs a static test database!
        """

        self.logger.log('Testing exportDB')
        dbs = ['test_meco']
        success = self.exporter.exportDB(databases = dbs, toCloud = True,
                                         localExport = True, numChunks = 4)
        self.logger.log('Success: {}'.format(success))
        self.assertTrue(success, "Export was not successful.")


    def test_split_archive(self):
        """
        Test splitting an archive into chunks.
        """
        # @REVIEWED
        self.logger.log('Testing archive splitting.')
        fullPath = '{}/{}'.format(self.exportTestDataPath,
                                  self.compressedTestFilename)
        self.logger.log('fullpath: {}'.format(fullPath))
        shutil.copyfile(fullPath, '{}/{}'.format(self.testDir,
                                                 self.compressedTestFilename))
        fullPath = '{}/{}'.format(self.testDir, self.compressedTestFilename)

        self.fileChunks = self.fileUtil.splitLargeFile(fullPath = fullPath,
                                                       numChunks = 3)
        self.assertEquals(len(self.fileChunks), 3)


    def test_get_file_size(self):
        """
        Test retrieving local file sizes.
        """
        # @REVIEWED
        fullPath = '{}/{}'.format(self.exportTestDataPath,
                                  self.compressedTestFilename)
        fSize = self.fileUtil.fileSize(fullPath)
        self.logger.log('size: {}'.format(fSize))
        self.assertEqual(fSize, 12279, 'File size is correct.')


    def test_upload_export_files_list(self):
        """
        TBW
        """
        self.exporter.sendDownloadableFiles()


    def test_checksum_after_upload(self):
        pass


    def test_dump_exclusions_dictionary(self):
        """
        Verify the exclusions dictionary by its type.
        :return:
        """
        # @REVIEWED
        exclusions = self.exporter.dumpExclusionsDictionary()

        if exclusions:
            self.assertEquals(type({}), type(exclusions))


    def test_plaintext_downloadable_files(self):
        print self.exporter.plaintextListOfDownloadableFiles()


    def test_move_to_final(self):
        """
        Test moving a file to the final destination path.
        """
        # @REVIEWED
        self.logger.log('Testing moving to final path {}.'.format(
            self.configer.configOptionValue('Export', 'db_export_final_path')))

        origCompressedFile = '{}/{}'.format(
            self.configer.configOptionValue('Export', 'export_test_data_path'),
            self.compressedTestFilename)
        newCompressedFile = '{}/{}'.format(
            self.configer.configOptionValue('Export', 'export_test_data_path'),
            'temp_test_file')

        shutil.copyfile(origCompressedFile, newCompressedFile)

        self.exporter.moveToFinalPath(compressedFullPath = newCompressedFile)

        self.assertTrue(os.path.isfile('{}/{}'.format(
            self.configer.configOptionValue('Export', 'db_export_final_path'),
            'temp_test_file')))

        # Remove the test file.
        os.remove('{}/{}'.format(
            self.configer.configOptionValue('Export', 'db_export_final_path'),
            'temp_test_file'))


    def test_log_successful_export(self):
        """
        Test logging of export results to the export history table.
        """
        # @REVIEWED
        self.assertTrue(self.exporter.logSuccessfulExport(name = 'test_export',
                                                          url =
                                                          'http://test_url',
                                                          datetime = 0,
                                                          size = 100))

        conn = MSGDBConnector().connectDB()
        cursor = conn.cursor()
        dbUtil = MSGDBUtil()

        self.assertTrue(
            dbUtil.executeSQL(cursor, 'select * from "ExportHistory" where '
                                      'timestamp = '
                                      'to_timestamp(0)'))

        self.assertEqual(len(cursor.fetchall()), 1,
                         "There should only be one result row.")

        self.assertTrue(
            dbUtil.executeSQL(cursor, 'delete from "ExportHistory" where '
                                      'timestamp = to_timestamp(0)'))
        conn.commit()

    def test_metadata_of_file_id(self):
        """
        Test getting the metadata for a file ID.
        """
        # @REVIEWED
        self.upload_test_data_to_cloud()

        self.logger.log('metadata: {}'.format(
            self.exporter.metadataOfFileID(self.testDataFileID)))

        self.assertTrue(re.match(r'[0-9A-Za-z]+', self.testDataFileID))

    def test_filename_for_file_id(self):
        """
        Test returning a file name given a file ID.
        """
        # @REVIEWED
        self.upload_test_data_to_cloud()
        self.assertEquals(
            self.exporter.filenameForFileID(fileID = self.testDataFileID),
            self.compressedTestFilename)


    def tearDown(self):
        """
        Delete all test items.
        """

        REMOVE_TEMPORARY_FILES = True
        if REMOVE_TEMPORARY_FILES:
            try:
                os.remove(os.path.join(os.getcwd(), self.testDir,
                                       self.uncompressedTestFilename))
                os.remove(os.path.join(os.getcwd(), self.testDir,
                                       self.compressedTestFilename))
            except OSError as detail:
                self.logger.log(
                    'Exception while removing temporary files: {}'.format(
                        detail), 'SILENT')
            try:
                os.remove(os.path.join(os.getcwd(), self.testDir,
                                       self.compressedTestFilename))
            except OSError as detail:
                self.logger.log(
                    'Exception while removing temporary files: {}'.format(
                        detail), 'SILENT')
            try:
                for f in self.fileChunks:
                    os.remove(f)
            except OSError as detail:
                self.logger.log(
                    'Exception while removing temporary files: {}'.format(
                        detail), 'DEBUG')

        try:
            os.rmdir(self.testDir)
        except OSError as detail:
            self.logger.log(
                'Exception while removing directory: {}'.format(detail),
                'ERROR')

        deleteSuccessful = True

        # Keep deleting from the cloud until there is no more to delete.
        while deleteSuccessful:
            try:
                fileIDToDelete = self.exporter.fileIDForFileName(
                    self.compressedTestFilename)
                self.logger.log("file ID to delete: {}".format(fileIDToDelete),
                                'DEBUG')
                self.exporter.driveService.files().delete(
                    fileId = '%s' % fileIDToDelete).execute()
            except (TypeError, http.HttpError) as e:
                self.logger.log('Delete not successful: {}'.format(e), 'SILENT')
                break
Esempio n. 13
0
class MSGDBExporter(object):
    """
    Export MSG DBs as SQL scripts.

    Supports export to local storage and to cloud storage.

    Usage:

    from msg_db_exporter import MSGDBExporter
    exporter = MSGDBExporter()

    Public API:

    exportDB(databases:List, 
             toCloud:Boolean, 
             testing:Boolean,
             numChunks:Integer, 
             deleteOutdated:Boolean): Export a list of DBs to the cloud.
    """

    # List of cloud files.
    @property
    def cloudFiles(self):
        self._cloudFiles = self.driveService.files().list().execute()
        return self._cloudFiles

    @property
    def driveService(self):
        if self._driveService:
            return self._driveService

        if not self.credentialPath:
            raise Exception("Credential path is required.")
        storage = Storage('{}/google_api_credentials'.format(
            self.credentialPath))

        self.googleAPICredentials = storage.get()

        self.logger.log("Authorizing credentials.", 'info')
        http = httplib2.Http()
        http = self.googleAPICredentials.authorize(http)

        self.logger.log("Authorized.", 'info')

        self._driveService = build('drive', 'v2', http=http)

        return self._driveService

    def __init__(self):
        """
        Constructor.
        """

        self.logger = SEKLogger(__name__, 'DEBUG', useColor=False)
        self.timeUtil = MSGTimeUtil()
        self.configer = MSGConfiger()
        self.fileUtil = MSGFileUtil()
        self.pythonUtil = MSGPythonUtil()  # for debugging
        self.connector = MSGDBConnector()
        self.conn = self.connector.connectDB()
        self.cursor = self.conn.cursor()
        self.dbUtil = MSGDBUtil()
        self.notifier = SEKNotifier(
            connector=self.connector,
            dbUtil=self.dbUtil,
            user=self.configer.configOptionValue('Notifications',
                                                 'email_username'),
            password=self.configer.configOptionValue('Notifications',
                                                     'email_password'),
            fromaddr=self.configer.configOptionValue('Notifications',
                                                     'email_from_address'),
            toaddr=self.configer.configOptionValue('Notifications',
                                                   'email_recipients'),
            testing_toaddr=self.configer.configOptionValue(
                'Notifications', 'testing_email_recipients'),
            smtp_server_and_port=self.configer.configOptionValue(
                'Notifications', 'smtp_server_and_port'))

        # Google Drive parameters.
        self.clientID = self.configer.configOptionValue(
            'Export', 'google_api_client_id')
        self.clientSecret = self.configer.configOptionValue(
            'Export', 'google_api_client_secret')
        self.oauthScope = 'https://www.googleapis.com/auth/drive'
        self.oauthConsent = 'urn:ietf:wg:oauth:2.0:oob'
        self.googleAPICredentials = ''
        self.exportTempWorkPath = self.configer.configOptionValue(
            'Export', 'db_export_work_path')

        self.credentialPath = self.configer.configOptionValue(
            'Export', 'google_api_credentials_path')
        self.credentialStorage = Storage('{}/google_api_credentials'.format(
            self.credentialPath))

        self._driveService = None
        self._cloudFiles = None
        self.postAgent = 'Maui Smart Grid 1.0.0 DB Exporter'
        self.retryDelay = 10
        self.availableFilesURL = ''

    def verifyExportChecksum(self, testing=False):
        """
        Verify the compressed export file using a checksum.

        * Save the checksum of the original uncompressed export data.
        * Extract the compressed file.
        * Verify the uncompressed export data.

        :param testing: When set to True, Testing Mode is used.
        """

        # Get the checksum of the original file.
        md5sum = self.fileUtil.md5Checksum(self.exportTempWorkPath)
        self.logger.log('md5sum: {}'.format(md5sum))

    def db_username(self):
        return "postgres"
        # return self.configer.configOptionValue('Database', 'db_username')

    def db_password(self):
        return self.configer.configOptionValue('Database', 'db_password')

    def db_port(self):
        return self.configer.configOptionValue('Database', 'db_port')

    def dumpCommand(self, db='', dumpName=''):
        """
        This method makes use of

        pg_dump -s -p ${PORT}
                   -U ${USERNAME}
                   [-T ${OPTIONAL_TABLE_EXCLUSIONS}]
                   ${DB_NAME} >
                   ${EXPORT_TEMP_WORK_PATH}/${DUMP_TIMESTAMP}_{DB_NAME}.sql

        :param db: String
        :param dumpName: String
        :return: String of command used to export DB.
        """

        # For reference only:
        # Password is passed from ~/.pgpass.
        # Note that ':' and '\' characters should be escaped with '\'.
        # Ref: http://www.postgresql.org/docs/9.1/static/libpq-pgpass.html

        # Dump databases as the superuser. This method does not require a
        # stored password when running under a root crontab.
        if not db or not dumpName:
            raise Exception('DB and dumpname required.')

        # Process exclusions.

        exclusions = self.dumpExclusionsDictionary()
        excludeList = []
        if db in exclusions:
            excludeList = exclusions[db]
        excludeString = ''
        if len(excludeList) > 0 and exclusions != None:
            for e in excludeList:
                excludeString += """-T '"{}"' """.format(e)

        return 'sudo -u postgres pg_dump -p {0} -U {1} {5} {2} > {3}/{4}' \
               '.sql'.format(self.db_port(), self.db_username(), db,
                             self.exportTempWorkPath, dumpName, excludeString)

    def dumpExclusionsDictionary(self):
        """
        :param db: String of DB name for which to retrieve exclusions.
        :return: Dictionary with keys as DBs and values as lists of tables to
        be excluded for a given database.
        """
        try:
            if type(
                    eval(
                        self.configer.configOptionValue(
                            'Export', 'db_export_exclusions'))) == type({}):
                return eval(
                    self.configer.configOptionValue('Export',
                                                    'db_export_exclusions'))
            else:
                return None
        except SyntaxError as detail:
            self.logger.log(
                'SyntaxError exception while getting exclusions: {}'.format(
                    detail))

    def dumpName(self, db=''):
        """
        :param db: String
        :return: String of file name used for dump file of db.
        """
        if not db:
            raise Exception('DB required.')
        return "{}_{}".format(self.timeUtil.conciseNow(), db)

    def filesToUpload(self, compressedFullPath='', numChunks=0, chunkSize=0):
        """
        :param compressedFullPath: String
        :param numChunks: Int
        :param chunkSize: Int
        :return: List of files to be uploaded according to their split
        sections, if applicable.
        """
        if numChunks != 0:
            self.logger.log('Splitting {}'.format(compressedFullPath), 'DEBUG')

            filesToUpload = self.fileUtil.splitLargeFile(
                fullPath=compressedFullPath,
                chunkSize=chunkSize,
                numChunks=numChunks)

            if not filesToUpload:
                raise Exception('Exception during file splitting.')
            else:
                self.logger.log('to upload: {}'.format(filesToUpload), 'debug')
                return filesToUpload

        else:
            return [compressedFullPath]

    def dumpResult(self, db='', dumpName='', fullPath=''):
        """
        :param dumpName: String of filename of dump file.
        :param fullPath: String of full path to dump file.
        :return: Boolean True if dump operation was successful, otherwise False.
        """

        success = True

        self.logger.log('fullPath: {}'.format(fullPath), 'DEBUG')

        try:
            # Generate the SQL script export.
            # @todo check return value of dump command
            self.logger.log('cmd: {}'.format(
                self.dumpCommand(db=db, dumpName=dumpName)))
            subprocess.check_call(self.dumpCommand(db=db, dumpName=dumpName),
                                  shell=True)
        except subprocess.CalledProcessError as error:
            self.logger.log("Exception while dumping: {}".format(error))
            sys.exit(-1)

        return success

    def exportDBs(self,
                  databases=None,
                  toCloud=False,
                  localExport=True,
                  testing=False,
                  chunkSize=0,
                  deleteOutdated=False):
        """
        Export a set of DBs to local storage.

        :param databases: List of database names that will be exported.
        :param toCloud: Boolean if set to True, then the export will also be
        copied to cloud storage.
        :param localExport: Boolean when set to True the DB is exported
        locally.
        :param testing: Boolean flag for testing mode. (@DEPRECATED)
        :param chunkSize: Integer size in bytes of chunk size used for
        splitting.
        :param deleteOutdated: Boolean indicating outdated files in the cloud
        should be removed.
        :returns: List of file IDs of uploaded files or None if there is an
        error condition.
        """

        # @todo separate uploading and exporting functions

        noErrors = True
        uploaded = []

        for db in databases:
            self.logger.log('Exporting {} using pg_dump.'.format(db), 'info')

            dumpName = self.dumpName(db=db)
            fullPath = '{}/{}.sql'.format(self.exportTempWorkPath, dumpName)
            if localExport:
                noErrors = self.dumpResult(db, dumpName, fullPath)

            # Perform compression of the file.
            self.logger.log("Compressing {} using gzip.".format(db), 'info')
            self.logger.log('fullpath: {}'.format(fullPath), 'DEBUG')

            gzipResult = self.fileUtil.gzipCompressFile(fullPath)
            compressedFullPath = '{}{}'.format(fullPath, '.gz')
            numChunks = self.numberOfChunksToUse(compressedFullPath)

            # Gzip uncompress and verify by checksum is disabled until a more
            # efficient, non-memory-based, uncompress is implemented.
            # md5sum1 = self.fileUtil.md5Checksum(fullPath)
            # self.md5Verification(compressedFullPath=compressedFullPath,
            # fullPath=fullPath,md5sum1=md5sum1)

            if toCloud:
                # Split compressed files into a set of chunks to improve the
                # reliability of uploads.

                # Upload the files to the cloud.
                for f in self.filesToUpload(
                        compressedFullPath=compressedFullPath,
                        numChunks=numChunks,
                        chunkSize=chunkSize):
                    self.logger.log('Uploading {}.'.format(f), 'info')
                    fileID = self.uploadFileToCloudStorage(
                        fullPath=f,
                        testing=testing,
                        retryCount=int(
                            self.configer.configOptionValue(
                                'Export', 'export_retry_count')))

                    self.logger.log('file id after upload: {}'.format(fileID))

                    if fileID != None:
                        uploaded.append(fileID)
                        self.logger.log('uploaded: {}'.format(uploaded),
                                        'DEBUG')
                        if not self.addReaders(
                                fileID,
                                self.configer.configOptionValue(
                                    'Export',
                                    'reader_permission_email_addresses').split(
                                        ','),
                                retryCount=int(
                                    self.configer.configOptionValue(
                                        'Export', 'export_retry_count'))):
                            self.logger.log(
                                'Failed to add readers for {}.'.format(f),
                                'error')
                        self.logSuccessfulExport(
                            *self.metadataOfFileID(fileID))

                    # Remove split sections if they exist.
                    try:
                        if not testing and numChunks > 1:
                            self.logger.log('Removing {}'.format(f))
                            os.remove('{}'.format(f))
                    except OSError as error:
                        self.logger.log(
                            'Exception while removing {}: {}.'.format(
                                fullPath, error))
                        noErrors = False

            # End if toCloud.

            if gzipResult:
                self.moveToFinalPath(compressedFullPath=compressedFullPath)

            # Remove the uncompressed file.
            try:
                if not testing:
                    self.logger.log('Removing {}'.format(fullPath))
                    os.remove('{}'.format(fullPath))
            except OSError as error:
                self.logger.log('Exception while removing {}: {}.'.format(
                    fullPath, error))
                noErrors = False

        # End for db in databases.

        if deleteOutdated:
            self.deleteOutdatedFiles(
                datetime.timedelta(days=int(
                    self.configer.configOptionValue('Export',
                                                    'export_days_to_keep'))))

        return uploaded if noErrors else None

    def moveToFinalPath(self, compressedFullPath=''):
        """
        Move a compressed final to the final export path.
        :param compressedFullPath: String for the compressed file.
        :return:
        """
        self.logger.log('Moving {} to final path.'.format(compressedFullPath),
                        'debug')
        try:
            shutil.move(
                compressedFullPath,
                self.configer.configOptionValue('Export',
                                                'db_export_final_path'))
        except Exception as detail:
            self.logger.log(
                'Exception while moving {} to final export path: {}'.format(
                    compressedFullPath, detail), 'error')

    def md5Verification(self, compressedFullPath='', fullPath='', md5sum1=''):
        """
        Perform md5 verification of a compressed file at compressedFullPath
        where the original file is at fullPath and has md5sum1.

        :param compressedFullPath: String
        :param fullPath: String
        :param md5sum1: String of md5sum of source file.
        :return:
        """

        GZIP_UNCOMPRESS_FILE = False
        if GZIP_UNCOMPRESS_FILE:
            # Verify the compressed file by uncompressing it and
            # verifying its
            # checksum against the original checksum.
            self.logger.log('reading: {}'.format(compressedFullPath), 'DEBUG')
            self.logger.log(
                'writing: {}'.format(
                    os.path.join(
                        self.configer.configOptionValue(
                            'Testing', 'export_test_data_path'),
                        os.path.splitext(os.path.basename(fullPath))[0])),
                'DEBUG')

            self.fileUtil.gzipUncompressFile(
                compressedFullPath,
                os.path.join(
                    self.configer.configOptionValue('Testing',
                                                    'export_test_data_path'),
                    fullPath))

        VERIFY_BY_CHECKSUM = False
        if VERIFY_BY_CHECKSUM:
            md5sum2 = self.fileUtil.md5Checksum(fullPath)

            self.logger.log(
                "mtime: {}, md5sum2: {}".format(
                    time.ctime(os.path.getmtime(fullPath)), md5sum2), 'INFO')

            if md5sum1 == md5sum2:
                self.logger.log(
                    'Compressed file has been validated by checksum.', 'INFO')
            else:
                noErrors = False

    def numberOfChunksToUse(self, fullPath):
        """
        Return the number of chunks to be used by the file splitter based on
        the file size of the file at fullPath.
        :param fullPath: String
        :returns: Int Number of chunks to create.
        """

        fsize = os.path.getsize(fullPath)
        self.logger.log('fullpath: {}, fsize: {}'.format(fullPath, fsize))
        if (fsize >= int(
                self.configer.configOptionValue('Export',
                                                'max_bytes_before_split'))):
            # Note that this does not make use of the remainder in the division.
            chunks = int(fsize / int(
                self.configer.configOptionValue('Export',
                                                'max_bytes_before_split')))
            self.logger.log('Will split with {} chunks.'.format(chunks))
            return chunks
        self.logger.log('Will NOT split file.', 'debug')
        return 1

    def uploadFileToCloudStorage(self,
                                 fullPath='',
                                 retryCount=0,
                                 testing=False):
        """
        Export a file to cloud storage.

        :param fullPath: String of file to be exported.
        :param testing: Boolean when set to True, Testing Mode is used.
        :param retryCount: Int of number of times to retry the upload if
        there is a failure.
        :returns: String File ID on verified on upload; None if verification
        fails.
        """

        success = True
        myFile = os.path.basename(fullPath)

        self.logger.log('full path {}'.format(os.path.dirname(fullPath),
                                              'DEBUG'))
        self.logger.log("Uploading {}.".format(myFile))

        result = {}
        try:
            media_body = MediaFileUpload(
                fullPath,
                mimetype='application/gzip-compressed',
                resumable=True)
            body = {
                'title': myFile,
                'description': 'Hawaii Smart Energy Project gzip '
                'compressed DB export.',
                'mimeType': 'application/gzip-compressed'
            }

            # Result is a Files resource.
            result = self.driveService.files().insert(
                body=body, media_body=media_body).execute()

        except Exception as detail:
            # Upload failures can result in a BadStatusLine.
            self.logger.log(
                "Exception while uploading {}: {}.".format(myFile, detail),
                'error')
            success = False

        if not self.__verifyMD5Sum(fullPath, self.fileIDForFileName(myFile)):
            self.logger.log('Failed MD5 checksum verification.', 'INFO')
            success = False

        if success:
            self.logger.log('Verification by MD5 checksum succeeded.', 'INFO')
            self.logger.log("Finished.")
            return result['id']

        if not success and retryCount <= 0:
            return None
        else:
            time.sleep(self.retryDelay)
            self.logger.log('Retrying upload of {}.'.format(fullPath),
                            'warning')
            self.uploadFileToCloudStorage(fullPath=fullPath,
                                          retryCount=retryCount - 1)

    def __retrieveCredentials(self):
        """
        Perform authorization at the server.

        Credentials are loaded into the object attribute googleAPICredentials.
        """

        flow = OAuth2WebServerFlow(self.clientID, self.clientSecret,
                                   self.oauthScope, self.oauthConsent)
        authorize_url = flow.step1_get_authorize_url()
        print 'Go to the following link in your browser: ' + authorize_url
        code = raw_input('Enter verification code: ').strip()
        self.googleAPICredentials = flow.step2_exchange(code)

        print "refresh_token = {}".format(
            self.googleAPICredentials.refresh_token)
        print "expiry = {}".format(self.googleAPICredentials.token_expiry)

    def freeSpace(self):
        """
        Get free space from the drive service.
        :param driveService: Object for the drive service.
        :returns: Int of free space (bytes B) on the drive service.
        """
        aboutData = self.driveService.about().get().execute()
        return int(aboutData['quotaBytesTotal']) - int(
            aboutData['quotaBytesUsed']) - int(
                aboutData['quotaBytesUsedInTrash'])

    def deleteFile(self, fileID=''):
        """
        Delete the file with ID fileID.
        :param fileID: String of a Google API file ID.
        """

        if not len(fileID) > 0:
            raise Exception("File ID has not been given.")

        self.logger.log(
            'Deleting file with file ID {} and name {}.'.format(
                fileID, self.filenameForFileID(fileID)), 'debug')

        try:
            # Writing the fileId arg name is required here.
            self.driveService.files().delete(fileId=fileID).execute()

        except errors.HttpError as error:
            self.logger.log('Exception while deleting: {}'.format(error),
                            'error')

    def deleteOutdatedFiles(self, maxAge=datetime.timedelta(weeks=9999999)):
        """
        Remove outdated files from cloud storage.

        :param minAge: datetime.timedelta of the minimum age before a file is
        considered outdated.
        :param maxAge: datetime.timedelta of the maximum age to consider for
        a file.
        :returns: Int count of deleted items.
        """

        # @todo Return count of actual successfully deleted files.

        outdated = self.outdatedFiles(maxAge)
        """:type : dict"""
        for f in outdated:
            self.deleteFile(f['id'])

        return len(outdated)

    def outdatedFiles(self,
                      daysBeforeOutdated=datetime.timedelta(days=9999999)):
        """
        Outdated files in the cloud where they are outdated if their age is
        greater than or equal to daysBeforeOutdated.

        Note: When t1 is the same day as t2, the timedelta comes back as -1.
        Not sure why this isn't represented as zero. Perhaps to avoid a false
        evaluation of a predicate on a tdelta.

        :param daysBeforeOutdated: datetime.timedelta where the value
        indicates that outdated files that have an age greater than this
        parameter.
        :return: Int count of deleted items.
        """

        t1 = lambda x: datetime.datetime.strptime(x['createdDate'],
                                                  "%Y-%m-%dT%H:%M:%S.%fZ")
        t2 = datetime.datetime.now()

        return filter(lambda x: t2 - t1(x) >= daysBeforeOutdated,
                      self.cloudFiles['items'])

    def sendNotificationOfFiles(self):
        """
        Provide a notification that lists the export files along with sharing
        links.
        """

        pass

    def sendDownloadableFiles(self):
        """
        Send available files via HTTP POST.
        :returns: None
        """

        myPath = '{}/{}'.format(self.exportTempWorkPath,
                                'list-of-downloadable-files.txt')

        fp = open(myPath, 'wb')

        output = StringIO()
        output.write(self.markdownListOfDownloadableFiles())

        fp.write(self.markdownListOfDownloadableFiles())
        fp.close()

        headers = {'User-Agent': self.postAgent, 'Content-Type': 'text/html'}
        try:
            r = requests.post(self.configer.configOptionValue(
                'Export', 'export_list_post_url'),
                              output.getvalue(),
                              headers=headers)
            print 'text: {}'.format(r.text)
        except requests.adapters.SSLError as error:
            # @todo Implement alternative verification.
            self.logger.log('SSL error: {}'.format(error), 'error')

        output.close()

    def metadataOfFileID(self, fileID=''):
        """
        :param fileID: String of a file ID in the cloud.
        :return: Tuple of metadata (name, url, timestamp, size) for a given
        file ID.
        """
        item = [i for i in self.cloudFiles['items'] if i['id'] == fileID][0]
        return (item[u'originalFilename'], item[u'webContentLink'],
                item[u'createdDate'], item[u'fileSize'])

    def listOfDownloadableFiles(self):
        """
        Create a list of downloadable files.
        :returns: List of dicts of files that are downloadable from the cloud.
        """

        files = []
        for i in reversed(
                sorted(self.cloudFiles['items'],
                       key=lambda k: k['createdDate'])):
            item = dict()
            item['title'] = i['title']
            item['webContentLink'] = i['webContentLink']
            item['id'] = i['id']
            item['createdDate'] = i['createdDate']
            item['fileSize'] = i['fileSize']
            files.append(item)
        return files

    def markdownListOfDownloadableFiles(self):
        """
        Generate content containing a list of downloadable files in Markdown
        format.

        :returns: String content in Markdown format.
        """

        content = "||*Name*||*Created*||*Size*||\n"
        for i in self.listOfDownloadableFiles():
            content += "||[`{}`]({})".format(i['title'], i['webContentLink'])
            content += "||`{}`".format(i['createdDate'])
            content += "||`{} B`||".format(int(i['fileSize']))
            content += '\n'

        # self.logger.log('content: {}'.format(content))
        return content

    def plaintextListOfDownloadableFiles(self):
        """
        Generate content containing a list of downloadable files in plaintext
        format.

        :returns: String content as plaintext.
        """
        content = ''
        includeLink = False
        for i in reversed(
                sorted(self.cloudFiles['items'],
                       key=lambda k: k['createdDate'])):
            if includeLink:
                content += "{}, {}, {}, {} B\n".format(i['title'],
                                                       i['webContentLink'],
                                                       i['createdDate'],
                                                       int(i['fileSize']))
            else:
                content += "{}, {}, {} B\n".format(i['title'],
                                                   i['createdDate'],
                                                   int(i['fileSize']))

        return content

    def logSuccessfulExport(self, name='', url='', datetime=0, size=0):
        """
        When an export has been successful, log information about the export
        to the database.

        The items to log include:
        * filename
        * URL
        * timestamp
        * filesize

        :param name: String
        :param url: String
        :param datetime:
        :param size: Int
        :return: True if no errors occurred, else False.
        """
        def exportHistoryColumns():
            return ['name', 'url', 'timestamp', 'size']

        timestamp = lambda \
                datetime: 'to_timestamp(0)' if datetime == 0 else "timestamp " \
                                                                  "'{}'".format(
            datetime)

        sql = 'INSERT INTO "{0}" ({1}) VALUES ({2}, {3}, {4}, {5})'.format(
            self.configer.configOptionValue('Export', 'export_history_table'),
            ','.join(exportHistoryColumns()), "'" + name + "'",
            "'" + url + "'", timestamp(datetime), size)

        conn = MSGDBConnector().connectDB()
        cursor = conn.cursor()
        dbUtil = MSGDBUtil()
        result = dbUtil.executeSQL(cursor, sql, exitOnFail=False)
        conn.commit()
        return result

    def sendExportSummary(self, summary=''):
        """
        Send a summary of exports via email to a preconfigured list of
        recipients.
        :param summary: String of summary content.
        :return:
        """
        try:
            if self.notifier.sendNotificationEmail(summary, testing=False):
                self.notifier.recordNotificationEvent(
                    types=MSGNotificationHistoryTypes,
                    noticeType=MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
        except Exception as detail:
            self.logger.log('Exception occurred: {}'.format(detail), 'ERROR')

    def currentExportSummary(self):
        """
        Current summary of exports since the last summary report time.

        Summaries are reported with identifier MSG_EXPORT_SUMMARY in the
        NotificationHistory.

        Includes:
        * Number of databases exported
        * Total number of files in the cloud.
        * A report of available storage capacity.
        * A list of available DBs.
        * A link where exports can be accessed.

        :return: String of summary text.
        """
        availableFilesURL = self.configer.configOptionValue(
            'Export', 'export_list_url')
        lastReportDate = self.notifier.lastReportDate(
            types=MSGNotificationHistoryTypes,
            noticeType=MSGNotificationHistoryTypes.MSG_EXPORT_SUMMARY)
        content = 'Cloud Export Summary:\n\n'
        content += 'Last report date: {}\n'.format(lastReportDate)

        # @TO BE REVIEWED: Verify time zone adjustment.
        content += '{} databases have been exported since the last report ' \
                   'date.\n'.format(self.countOfDBExports(
            lastReportDate + datetime.timedelta(
                hours = 10)) if lastReportDate else self.countOfDBExports())

        content += '{} B free space is available.\n'.format(self.freeSpace())
        content += '\nCurrently available DBs:\n'
        content += self.plaintextListOfDownloadableFiles()
        content += '\n{} files can be accessed through Google Drive (' \
                   'https://drive.google.com) or at {}.'.format(
            self.countOfCloudFiles(), availableFilesURL)

        return content

    def countOfDBExports(self, since=None):
        """
        :param since: datetime indicating last export datetime.
        :return: Int of count of exports.
        """
        myDatetime = lambda x: datetime.datetime.strptime(x, '%Y-%m-%d %H:%S')
        if not since:
            since = myDatetime('1900-01-01 00:00')
        self.logger.log(since.strftime('%Y-%m-%d %H:%M'), 'DEBUG')

        sql = 'SELECT COUNT("public"."ExportHistory"."timestamp") FROM ' \
              '"public"."ExportHistory" WHERE "timestamp" > \'{}\''.format(
            since.strftime('%Y-%m-%d %H:%M'))

        conn = MSGDBConnector().connectDB()
        cursor = conn.cursor()
        dbUtil = MSGDBUtil()
        rows = None
        if dbUtil.executeSQL(cursor, sql, exitOnFail=False):
            rows = cursor.fetchall()
        assert len(rows) == 1, 'Invalid return value.'
        return rows[0][0]

    def countOfCloudFiles(self):
        """
        :param since: datetime indicating last trailing export datetime.
        :return: Int of count of exports.
        """
        return len(self.cloudFiles['items'])

    def __verifyMD5Sum(self, localFilePath, remoteFileID):
        """
        Verify that the local MD5 sum matches the MD5 sum for the remote file
        corresponding to an ID.

        This verifies that the uploaded file matches the local compressed
        export file.

        :param localFilePath: String of the full path of the local file.
        :param remoteFileID: String of the cloud ID for the remote file.
        :returns: Boolean True if the MD5 sums match, otherwise, False.
        """

        self.logger.log('remote file ID: {}'.format(remoteFileID))
        self.logger.log('local file path: {}'.format(localFilePath))

        # Get the md5sum for the local file.
        f = open(localFilePath, mode='rb')
        fContent = hashlib.md5()
        for buf in iter(partial(f.read, 128), b''):
            fContent.update(buf)
        localMD5Sum = fContent.hexdigest()
        f.close()

        self.logger.log('local md5: {}'.format(localMD5Sum), 'DEBUG')

        def verifyFile():
            # Get the MD5 sum for the remote file.
            for item in self.cloudFiles['items']:
                if (item['id'] == remoteFileID):
                    self.logger.log(
                        'remote md5: {}'.format(item['md5Checksum']), 'DEBUG')
                    if localMD5Sum == item['md5Checksum']:
                        return True
                    else:
                        return False

        try:
            if verifyFile():
                return True
            else:
                return False

        except errors.HttpError as detail:
            self.logger.log('HTTP error during MD5 verification.', 'error')

            time.sleep(10)

            if verifyFile():
                return True
            else:
                return False

    def fileIDForFileName(self, filename):
        """
        Get the file ID for the given filename.

        This method supports matching multiple cloud filenames but only
        returns the ID for a single matching filename.

        This can then be called recursively to obtain all the file IDs for a
        given filename.

        :param String of the filename for which to retrieve the ID.
        :returns: String of a cloud file ID or None if no match.
        """
        fileIDList = filter(lambda x: x['originalFilename'] == filename,
                            self.cloudFiles['items'])
        return fileIDList[0]['id'] if len(fileIDList) > 0 else None

    def filenameForFileID(self, fileID=''):
        """
        :param fileID: String of cloud-based file ID.
        :return: String of filename for a given file ID.
        """
        return filter(lambda x: x['id'] == fileID,
                      self.cloudFiles['items'])[0]['originalFilename']

    def addReaders(self, fileID=None, emailAddressList=None, retryCount=0):
        """
        Add reader permission to an export file that has been uploaded to the
        cloud for the given list of email addresses.

        Email notification is suppressed by default.

        :param fileID: String of the cloud file ID to be processed.
        :param emailAddressList: List of email addresses.
        :returns: Boolean True if successful, otherwise False.
        """
        # @todo Provide support for retry count
        success = True

        self.logger.log('file id: {}'.format(fileID))
        self.logger.log('address list: {}'.format(emailAddressList))

        for addr in emailAddressList:
            permission = {'value': addr, 'type': 'user', 'role': 'reader'}

            if fileID:
                try:
                    resp = self.driveService.permissions().insert(
                        fileId=fileID,
                        sendNotificationEmails=False,
                        body=permission).execute()
                    self.logger.log(
                        'Reader permission added for {}.'.format(addr))
                except errors.HttpError as error:
                    self.logger.log('An error occurred: {}'.format(error))
                    success = False

        if not success and retryCount <= 0:
            return False
        elif success:
            return True
        else:
            time.sleep(self.retryDelay)
            self.logger.log(
                'Retrying adding readers for ID {}.'.format(fileID), 'warning')
            self.addReaders(fileID=fileID,
                            emailAddressList=emailAddressList,
                            retryCount=retryCount - 1)