def baselinePackagesInDatabase():
"""Called by the installer to load all content on server into the DB."""
dbClient = None
try:
globalSettings = utils.loadSettings(os.path.join(env.configPath, 'globalSettings.json'))
logger = utils.setupLogger('Packages', env, globalSettings['fileContainingCoreLogSettings'])
dbClient = getDbConnection(logger)
## Work through each package system type containing packages to load
for system,context in validPackageSystems.items():
systemName = context.get('name')
systemPath = context.get('path')
logger.info('Working on {} packages: {}'.format(systemName, systemPath))
if os.path.isdir(systemPath):
packageList = os.listdir(systemPath)
for packageName in packageList:
thisPath = os.path.join(systemPath, packageName)
if not os.path.isdir(thisPath):
continue
loadPackageIntoDB(logger, packageName, systemName, ['content', systemName], dbClient, thisPath)
except:
stacktrace = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
logger.error('Exception in baselinePackagesInDatabase: {}'.format(stacktrace))
with suppress(Exception):
dbClient.session.close()
dbClient.close()
## end baselinePackagesInDatabase
return
def updatePackage(pkgName, pkgSystem='contentGathering', pkgPath=None, forceUpdate=True):
globalSettings = utils.loadSettings(os.path.join(env.configPath, 'globalSettings.json'))
logger = utils.setupLogger('Packages', env, globalSettings['fileContainingCoreLogSettings'])
dbClient = getDbConnection(logger)
try:
## Initialize directories for this work
(packageBasePath, newPackagePath, oldPackagePath) = initializePaths(logger, pkgName)
## Check the target system for this package
if pkgSystem.lower() not in validPackageSystems:
raise EnvironmentError('Content management expecting package for a valid system {}, but received unknown type: {}.'.format(validPackageSystems, pkgType))
packageSystemName = validPackageSystems[pkgSystem.lower()]['name']
packageSystemPath = validPackageSystems[pkgSystem.lower()]['path']
## Default the new package location to the exploded dir on the server
newPackagePath = os.path.join(packageSystemPath, pkgName)
if pkgPath is not None:
## And reset that location if one was designated in the call
newPackagePath = pkgPath
## If package is in the database already, extract into side-by-side path
pkgExists = getContentPackage(logger, dbClient, oldPackagePath, pkgName, stripString='content,{},{}'.format(packageSystemName, pkgName))
if pkgExists:
## Compare the files with filecmp/difflib and present differences...
changes = []
comparePackageVersions(logger, pkgName, oldPackagePath, newPackagePath, changes)
if len(changes) <= 0:
logger.info('No changes found; package {} remains unchanged.'.format(pkgName))
else:
logger.info('Changes found in package {}, with the following files: {}'.format(pkgName, str(changes)))
if not forceUpdate:
logger.info('Leaving package unchanged because the forceUpdate flag was not set.')
else:
logger.info('Overwriting previous version...')
loadPackageIntoDB(logger, pkgName, packageSystemName, ['content', packageSystemName], dbClient, newPackagePath)
else:
## First time load of the package into the database
logger.error('Attempting to update a package that did not previously exist in the database. Please compress and then add the package first.')
print('Attempting to update a package that did not previously exist in the database. Please compress and then add the package first.')
## Cleanup
logger.info('Finished content management work on package {}; cleaning up.'.format(pkgName))
except:
stacktrace = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
logger.error('Exception in updatePackage: {}'.format(str(stacktrace)))
## end updatePackage
return
def main():
"""Entry point for the database configuration utility.
Usage::
$ python rebuildDataSchema.py
"""
try:
## Setup requested log handlers
logEntity = 'Database'
logger = utils.setupLogger(logEntity, env, 'logSettingsCore.json')
logger.info('Starting rebuildDataSchema utility.')
## Attempt connection
dbClient = DatabaseClient(logger)
if dbClient is None:
raise SystemError(
'Failed to connect to database; unable to rebuild data schema.'
)
print('\nDatabase connection successful.')
logger.debug('Database connection successful')
## Start the work
createTables(dbClient, logger)
## Close the connection
dbClient.close()
logger.info('Exiting rebuildDataSchema utility.')
except:
stacktrace = traceback.format_exception(sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
## The basic print is here for a console message in case we weren't
## able to use the logging mechanism before encountering the failure.
print('Failure in rebuildDataSchema.main: {}'.format(stacktrace))
try:
logger.debug(
'Failure in rebuildDataSchema.main: {}'.format(stacktrace))
except:
pass
## end main
return
def main():
"""Entry point for this utility.
Usage::
$ python createProtocolEntry.py
"""
try:
## Setup requested log handlers
logEntity = 'Protocols'
logger = utils.setupLogger(logEntity, env, 'logSettingsCore.json')
logger.info('Starting manageProtocols.')
## Connect to database
dbClient = DatabaseClient(logger)
if dbClient is None:
raise SystemError('Failed to connect to database; unable to initialize tables.')
## Get list of valid realms
realms = []
getRealms(dbClient, logger, realms)
## Create and insert a new credential
createProtocolEntry(dbClient, logger, realms)
## Cleanup
dbClient.session.remove()
dbClient.close()
logger.info('Exiting configureDatabase utility.')
except:
stacktrace = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
## The basic print is here for a console message in case we weren't
## able to use the logging mechanism before encountering the failure.
print('Failure in configureDatabase: {}'.format(stacktrace))
try:
logger.debug('Failure in configureDatabase: {}'.format(stacktrace))
except:
pass
## end main
return
def listPackages():
globalSettings = utils.loadSettings(os.path.join(env.configPath, 'globalSettings.json'))
logger = utils.setupLogger('Packages', env, globalSettings['fileContainingCoreLogSettings'])
dbClient = getDbConnection(logger)
## Cleanup this package result; prep work for comparison operation
dbClass = platformSchema.ContentPackage
packages = dbClient.session.query(dbClass).order_by(dbClass.system.desc(),dbClass.name.desc()).all()
jsonReport = {}
for package in packages:
packageSystem = package.system
packageName = package.name
packageDate = package.time_created
if packageSystem not in jsonReport:
jsonReport[packageSystem] = []
jsonReport[packageSystem].append({ 'name' : packageName, 'created' : packageDate })
dbClient.session.commit()
## And finally report
strReport = json.dumps(jsonReport, default=utils.customJsonDumpsConverter, indent=4)
logger.debug('Valid packages: \n{}'.format(str(strReport)))
print('Valid packages: \n{}'.format(str(strReport)))
## end listPackages
return
def removePackage(packageName):
globalSettings = utils.loadSettings(os.path.join(env.configPath, 'globalSettings.json'))
logger = utils.setupLogger('Packages', env, globalSettings['fileContainingCoreLogSettings'])
dbClient = getDbConnection(logger)
try:
logger.info('Searching for package in database: {}'.format(packageName))
dbClass = platformSchema.ContentPackage
package = dbClient.session.query(dbClass).filter(dbClass.name == packageName).first()
if package is None:
logger.info('Package {} not found in database... nothing to do.'.format(packageName))
else:
logger.info('Removing package files...'.format(packageName))
dbClient.session.commit()
## Remove all files first
removeFiles(logger, dbClient, packageName)
## And now remove the package
package = dbClient.session.query(dbClass).filter(dbClass.name == packageName).first()
packageType = package.system
dbClient.session.delete(package)
dbClient.session.commit()
## Remove from filesystem
packageSystemPath = validPackageSystems[pkgSystem.lower()]['path']
packagePath = os.path.join(packageSystemPath, packageName)
logger.info('Removing files from server directory: {}...'.format(packagePath))
utils.cleanDirectory(logger, packagePath)
logger.info('Package {} removed.'.format(packageName))
except:
stacktrace = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
logger.error('Exception in removePackage: {}'.format(stacktrace))
with suppress(Exception):
dbClient.session.close()
dbClient.close()
## end removePackage
return
def validatePackage(packageName, compressedFile, pkgSystem='contentGathering', forceUpdate=False):
"""Entry function.
Arguments:
compressedFile (str) : fully qualified package to be deployed
forceUpdate (bool) : whether to force update if package already exists
pkgSystem (str) : package type (target system/service to deploy into)
"""
globalSettings = utils.loadSettings(os.path.join(env.configPath, 'globalSettings.json'))
logger = utils.setupLogger('Packages', env, globalSettings['fileContainingCoreLogSettings'])
dbClient = getDbConnection(logger)
try:
## Check the extension
(pkgName, pkgExtension) = compressedFile.split('.')
if pkgExtension.lower() != 'zip' and pkgExtension.lower() != 'tar':
raise EnvironmentError('Content management expecting package in either ZIP or TAR format; unable to work with this format: {}.'.format(pkgExtension))
## Initialize directories for this work
(packageBasePath, newPackagePath, oldPackagePath) = initializePaths(logger, packageName)
## Check the target system for this package
if pkgSystem.lower() not in validPackageSystems:
raise EnvironmentError('Content management expecting package for a valid system {}, but received unknown type: {}.'.format(validPackageSystems, pkgType))
packageSystemName = validPackageSystems[pkgSystem.lower()]['name']
packageSystemPath = validPackageSystems[pkgSystem.lower()]['path']
## Extract contents into a temp runtime directory
extractContents(logger, compressedFile, packageName, pkgExtension, packageBasePath, newPackagePath)
## If package is in the database already, extract into side-by-side path
pkgExists = getContentPackage(logger, dbClient, oldPackagePath, packageName, stripString='content,{},{}'.format(packageSystemName, packageName))
if pkgExists:
## Compare the files with filecmp/difflib and present differences...
changes = []
comparePackageVersions(logger, packageName, oldPackagePath, newPackagePath, changes)
if len(changes) <= 0:
logger.info('No changes found; package {} remains unchanged.'.format(packageName))
else:
logger.info('Changes found in package {}, with the following files: {}'.format(packageName, str(changes)))
if not forceUpdate:
logger.info('Leaving package unchanged because the forceUpdate flag was not set.')
print('Leaving package unchanged because the forceUpdate flag was not set.')
else:
logger.info('Overwriting previous version...')
print('Overwriting previous version...')
loadPackageIntoDB(logger, packageName, packageSystemName, ['content', packageSystemName], dbClient, newPackagePath)
else:
## First time load of the package into the database
logger.info('Attempting to load new package into database...')
loadPackageIntoDB(logger, packageName, packageSystemName, ['content', packageSystemName], dbClient, newPackagePath)
## Cleanup
logger.info('Finished content management work on package {}; cleaning up.'.format(packageName))
utils.cleanDirectory(logger, packageBasePath)
except:
stacktrace = traceback.format_exception(sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2])
logger.error('Exception in validatePackage: {}'.format(str(stacktrace)))
with suppress(Exception):
dbClient.session.close()
dbClient.close()
## end validatePackage
return
def setUp(self):
(self.rootLogger, self._buffer) = setupLogger()
self.rootLogger.setLevel(logging.DEBUG)
def main():
"""Entry point for the database configuration utility.
Usage::
$ python configureDatabase.py
"""
try:
## Setup requested log handlers
logEntity = 'Database'
logger = utils.setupLogger(logEntity, env, 'logSettingsCore.json')
logger.info('Starting configureDatabase utility.')
## Using an externally provided library to create and/or update a local
## config file for database parameters; defined in globalSettings and
## located in '<install_path>/external/'.
globalSettings = utils.loadSettings(
os.path.join(env.configPath, 'globalSettings.json'))
externalLibrary = utils.loadExternalLibrary('externalDatabaseLibrary',
env)
dbClient = None
while dbClient is None:
try:
## Get settings from user (new or updated) and save to file
externalLibrary.updateSettingsFile(globalSettings, logger)
## Attempt connection with the new or updated settings
dbClient = DatabaseClient(logger)
## In case we fall through the cracks without an exception...
if dbClient is None:
## The settings didn't work; request a new set
print(
'Failed to connect to database with provided settings; try again...'
)
logger.debug(
'Failed to connect to database with provided settings; try again...'
)
except exc.OperationalError:
## Intentionally catch database connection errors
print('\nException in configureDatabase: {}'.format(
str(sys.exc_info()[1])))
logger.error('Exception in configureDatabase: {}'.format(
str(sys.exc_info()[1])))
## The settings didn't work; request a new set
print(
'\nFailed to connect to database with provided settings; try again...'
)
logger.debug(
'Failed to connect to database with provided settings; try again...'
)
print('\nDatabase connection successful\n')
logger.debug('Database connection successful')
## Close the connection
dbClient.close()
logger.info('Exiting configureDatabase utility.')
except:
stacktrace = traceback.format_exception(sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
print('Failure in configureDatabase.main: {}'.format(stacktrace))
with suppress(Exception):
logger.debug('Failure in configureDatabase: {}'.format(stacktrace))
## end main
return
def powerShell():
import sys
import traceback
import os
import re
try:
## Add openContentPlatform directories onto the sys path
thisPath = os.path.dirname(os.path.abspath(__file__))
basePath = os.path.abspath(os.path.join(thisPath, '..'))
if basePath not in sys.path:
sys.path.append(basePath)
import env
env.addLibPath()
env.addDatabasePath()
env.addExternalPath()
## Setup requested log handlers
globalSettings = utils.loadSettings(
os.path.join(env.configPath, 'globalSettings.json'))
logEntity = 'Protocols'
logger = utils.setupLogger(logEntity, env, 'logSettingsCore.json')
logger.info('Starting protocolWrapperPowershell...')
import twisted.logger
logFiles = utils.setupLogFile(
'JobDetail',
env,
globalSettings['fileContainingContentGatheringLogSettings'],
directoryName='client')
logObserver = utils.setupObservers(
logFiles, 'JobDetail', env,
globalSettings['fileContainingContentGatheringLogSettings'])
logger = twisted.logger.Logger(observer=logObserver,
namespace='JobDetail')
from remoteRuntime import Runtime
runtime = Runtime(logger, env, 'TestPkg', 'TestJob', 'endpoint', {},
None, {}, None, {}, None)
## Manual creation of a protocol via protocolHandler
externalProtocolHandler = utils.loadExternalLibrary(
'externalProtocolHandler', env, globalSettings)
protocolHandler = externalProtocolHandler.ProtocolHandler(
None, globalSettings, env, logger)
protocolType = 'ProtocolPowerShell'
protocolData = {'user': '******', 'password': '******'}
protocolHandler.createManual(runtime, protocolType, protocolData)
protocol = externalProtocolHandler.getProtocolObject(runtime, 1)
print('protocol to use: {}'.format(protocol))
print('protocols: {}'.format(
externalProtocolHandler.getProtocolObjects(runtime)))
endpoint = '192.168.1.100'
client = PowerShell(runtime, logger, endpoint, 1, protocol)
client.open()
osAttrDict = {}
queryOperatingSystem(client, logger, osAttrDict)
logger.debug('osAttrDict: {osAttrDict!r}', osAttrDict=osAttrDict)
client.close()
except:
stacktrace = traceback.format_exception(sys.exc_info()[0],
sys.exc_info()[1],
sys.exc_info()[2])
msg = str(sys.exc_info()[1])
## Cleanup message when we know what it is:
## "<x_wmi: The RPC server is unavailable. (-2147023174, 'The RPC server is unavailable. ', (0, None, 'The RPC server is unavailable. ', None, None, -2147023174), None)>"
if re.search(
'The client cannot connect to the destination specified in the request',
msg, re.I):
## Remove the rest of the fluff
msg = 'The client cannot connect to the destination specified in the request. Verify that the service on the destination is running and is accepting requests.'
logger.debug('Main Exception: {exception!r}', exception=msg)
else:
logger.debug('Main Exception: {exception!r}', exception=stacktrace)
__date__ = '30/07/2012'
__copyright__ = ('Copyright 2012, Australia Indonesia Facility for '
'Disaster Reduction')
import os
import sys
import logging
from urllib2 import URLError
from zipfile import BadZipfile
from ftp_client import FtpClient
from sftp_client import SFtpClient
from utils import setupLogger, dataDir, is_event_id
from shake_event import ShakeEvent
# Loading from package __init__ not working in this context so manually doing
setupLogger()
LOGGER = logging.getLogger('InaSAFE')
def processEvent(theEventId=None, theLocale='en'):
"""Launcher that actually runs the event processing.
:param theEventId: The event id to process. If None the latest event will
be downloaded and processed.
:param theLocale:
"""
myPopulationPath = os.path.join(
dataDir(),
'exposure',
'IDN_mosaic',
'popmap10_all.tif')
import os
import json
import time
from contextlib import suppress
from confluent_kafka import KafkaError
thisPath = os.path.dirname(os.path.abspath(__file__))
basePath = os.path.abspath(os.path.join(thisPath, '..', 'framework'))
if basePath not in sys.path:
sys.path.append(basePath)
import env
env.addLibPath()
import utils
logger = utils.setupLogger('TestService',
env,
'logSettingsServices.json',
directoryName='service')
logger.info('Starting apiKafkaTest...')
def getKafkaConsumer(globalSettings):
"""Simple wrapper to get the parameters and connect the kafka consumer."""
kafkaEndpoint = globalSettings['kafkaEndpoint']
useCertsWithKafka = globalSettings['useCertificatesWithKafka']
kafkaCaRootFile = os.path.join(env.configPath,
globalSettings['kafkaCaRootFile'])
kafkaCertFile = os.path.join(env.configPath,
globalSettings['kafkaCertificateFile'])
kafkaKeyFile = os.path.join(env.configPath, globalSettings['kafkaKeyFile'])
kafkaTopic = globalSettings['kafkaTopic']
## Connect the consumer
def setUp(self):
(self.rootLogger, self._buffer) = setupLogger()
import os
import sys
import argparse
import utils
from EventRecord import EventRecord
import importlib
parser = argparse.ArgumentParser()
parser.add_argument(
"-log",
"--log",
default="info",
help=("Provide logging level. "
"Example --log debug', default='warning'"),
)
log = utils.setupLogger(parser)
class MainWindow(QtWidgets.QMainWindow):
def __init__(self, *args, **kwargs):
super(MainWindow, self).__init__(*args, **kwargs)
# Load the UI Page
uic.loadUi('ui.ui', self)
print(self.__dict__.keys())
self.recordingButton.clicked.connect(self.recordingButtonPressed)
self.runButton.clicked.connect(self.runButtonPressed)
def recordingButtonPressed(self):
log.debug("recordingButtonPressed clicked")
from gettext import gettext as _
import comparetext
from feed import Feed
from filter import Filter
import logging
import settings
import utils
# setup gettext
gettext.textdomain('feedfilter')
feedfile = settings.read_argv()
settings.read_settings()
utils.setupLogger()
# read and parse filterfiles
wordfilter = Filter()
wordfilter.read_filterlist('./blackwordlist.txt')
wordfilter.read_filterlist(settings.sitename)
# Parse feed
feed = Feed(feedfile)
# For now we use the language without any regional variants
lang = feed.get_lang().split('-')[0]
wordlists = {}
for child in feed:
title = feed.get_title(child)
summary = feed.get_description(child)
description="Upload TMDB TV show data to S3",
add_help=True
)
parser.add_argument("path_config", type=str,
help="Path to configuration file with API credentials")
args = parser.parse_args()
# Create the required directories if not exits
if not createDir(dirLogs):
sys.exit('The directory "{}" could not be created'.format(dirLogs))
if not createDir(dirTweet):
sys.exit('The directory "{}" could not be created'.format(dirTweet))
# Setup the logger
logName = date.today().strftime("%Y-%m-%d") + '-tweet-upload.log'
setupLogger(dirLogs, logName)
# Get datetime interval for the past hour
dt_save, datetime_start, datetime_end = getPastHourInterval()
# Connect to the database
conn = connectToDB(dbPath)
if conn is None:
logging.error('Error while connecting to the database')
sys.exit(1)
# Get the tweet data streamed in the past hour
df = retrieveDataFromDB(conn, datetime_start, datetime_end)
# Close the database connection
try:
def __init__(self):
(self.rootLogger, self.buffer) = setupLogger()