def testDatabaseInstantiation(self):
sample1 = {
'database_hostname': 'A',
'database_port': 'B',
'database_name': 'C',
'database_username': '******',
'database_password': '******',
}
d = db.Database(sample1)
assert d.dsn == 'host=A port=B dbname=C user=D password=E', 'dsn not created correctly'
assert type(d.logger) == type(
util.FakeLogger()), 'should have a %s but got %s instead' % (type(
util.FakeLogger()), type(d.logger))
d = db.Database(sample1, 1)
assert d.logger == 1, 'logger pass as a parameter was not saved, got %s instead' % d.logger
sample1 = {
'database_hostname': 'A',
'database_port': 'B',
'database_name': 'C',
'database_username': '******',
'database_password': '******',
'logger': 2
}
d = db.Database(sample1)
assert d.dsn == 'host=A port=B dbname=C user=D password=E', 'dsn not created correctly'
assert d.logger == 2, 'logger passed with dictionary was not saved, got %s instead' % d.logger
d = db.Database(sample1, 1)
assert d.dsn == 'host=A port=B dbname=C user=D password=E', 'dsn not created correctly'
assert d.logger == 1, 'logger passed with dictionary was not overridden by logger passed as a parameter, got %s instead' % d.logger
def __init__(self, *args, **kwargs):
"""
Store the config and create a connection to the database.
Keyword arguments:
config -- Configuration of the application.
"""
self.context = kwargs.get("config")
if hasattr(self.context, 'database'):
# XXX this should be replaced with connection_context instead
self.context.database[
'databaseHost'] = self.context.database.database_host
self.context.database[
'databasePort'] = self.context.database.database_port
self.context.database[
'databaseName'] = self.context.database.database_name
self.context.database[
'databaseUserName'] = self.context.database.database_user
self.context.database[
'databasePassword'] = self.context.database.database_password
self.database = db.Database(self.context.database)
else:
# the old middleware
self.database = db.Database(self.context)
self.connection = None
def __init__(self, configContext):
super(EmailSender, self).__init__(configContext)
self.database = db.Database(configContext)
self.config = configContext
self.email_form = form.Form(form.Textbox('campaign_id', form.notnull),
form.Textbox('status', form.notnull))
def setUp(self):
"""Create a configuration context and a database connection. """
self.config = ConfigurationManager.newConfiguration(
configurationModule=commonconfig,
applicationName="PostgreSQL Tests")
self.database = db.Database(self.config)
self.connection = self.database.connection()
def setUpClass(cls):
"""Create a configuration context and a database connection.
This will create (and later destroy) one connection per test
case (aka. test class).
"""
cls.config = cls.get_standard_config()
cls.database = db.Database(cls.config)
cls.connection = cls.database.connection()
def __init__(self, config):
"""
Set the DB and the pool up and store the config.
"""
super(JsonServiceBase, self).__init__(config)
try:
self.database = db.Database(config)
self.crashStoragePool = cs.CrashStoragePool(config,
storageClass=config.hbaseStorageClass)
except (AttributeError, KeyError):
util.reportExceptionAndContinue(logger)
def setUp(self):
"""Create a configuration context and a database connection. """
self.config = ConfigurationManager.newConfiguration(
configurationModule=commonconfig,
applicationName="PostgreSQL Tests")
try:
self.database = db.Database(self.config)
except (AttributeError, KeyError):
raise
self.connection = self.database.connection()
def __init__(self, configContext):
super(EmailCampaignCreate, self).__init__(configContext)
self.database = db.Database(configContext)
self.config = configContext
self.email_form = form.Form(
form.Textbox('product', form.notnull),
form.Textbox('versions', form.notnull),
form.Textbox('signature', form.notnull),
form.Textbox('subject', form.notnull),
form.Textarea('body', form.notnull),
form.Textbox('start_date', form.notnull),
form.Textbox('end_date', form.notnull),
form.Textbox('author', form.notnull))
def export_uuids(path, numberofdays):
"""Export crash report uuids from a PostgreSQL database to a CSV file
path - Directory where the csv file will be created.
numberofdays - Number of days of crash reports to retrieve, before the most
recent crash date.
"""
database = db.Database(config)
connection = database.connection()
cur = connection.cursor()
# steps
# 1. pull all distinct dates
sql = """
SELECT DISTINCT to_char(date_processed, 'YYYY-MM-DD') as day
FROM reports
ORDER BY day DESC
"""
if numberofdays:
sql = "%s LIMIT %s" % (sql, numberofdays)
print 'Calculating dates... '
days = db.execute(cur, sql)
days_list = []
for day in days:
days_list.append(day[0])
store_filename = 'uuids.csv'
store_filename = os.path.normpath('%s/%s' % (path, store_filename))
store_file = open(store_filename, 'w')
store = csv.writer(store_file, delimiter=',', quotechar='"')
print 'Store file created: %s' % store_filename
for day in days_list:
date_from = dtu.datetimeFromISOdateString(day)
date_to = date_from + datetime.timedelta(1)
sql = "SELECT uuid FROM reports WHERE date_processed BETWEEN %s AND %s"
print 'Getting crash reports for day %s' % date_from.date()
crashes_list = db.execute(cur, sql, (date_from, date_to))
for crash in crashes_list:
store.writerow(crash)
store_file.close()
connection.close()
return store_filename
def __init__(self, *args, **kwargs):
"""
Store the config and create a connection to the database.
Keyword arguments:
config -- Configuration of the application.
"""
self.context = kwargs.get("config")
try:
self.database = db.Database(self.context)
except (AttributeError, KeyError):
util.reportExceptionAndContinue(logger)
self.connection = None
def __init__(self, config):
"""
Set the DB and the pool up and store the config.
"""
super(JsonServiceBase, self).__init__(config)
try:
self.database = db.Database(config)
self.crashStoragePool = cs.CrashStoragePool(
config,
storageClass=config.hbaseStorageClass
)
except (AttributeError, KeyError), x:
self.config.logger.error(
str(x),
exc_info=True
)
def __init__(self, context):
super(ProductVersionCache, self).__init__()
self.config = context
self.cache = {}
sql = """
select
product_name as product,
version_string as version,
product_version_id as id
from
product_info
"""
self.database = db.Database(self.config)
connection = self.database.connection()
cursor = connection.cursor()
for product, version, id in db.execute(cursor, sql):
self.cache[(product, version)] = id
connection.close()
def __init__(self, configContext):
super(EmailCampaign, self).__init__(configContext)
self.database = db.Database(configContext)
def export(path, numberofdays=0):
"""Export crash reports from a PostgreSQL database.
path - Directory where the dump file will be created.
numberofdays - Number of days of crash reports to retrieve, before the most
recent crash date.
"""
database = db.Database(config)
connection = database.connection()
cur = connection.cursor()
crash_files = []
fields_list = ("client_crash_date",
"date_processed",
"uuid",
"product",
"version",
"build",
"signature",
"url",
"install_age",
"last_crash",
"uptime",
"cpu_name",
"cpu_info",
"reason",
"address",
"os_name",
"os_version",
"email",
"build_date",
"user_id",
"started_datetime",
"completed_datetime",
"success",
"truncated",
"processor_notes",
"user_comments",
"app_notes",
"distributor",
"distributor_version",
"topmost_filenames",
"addons_checked",
"flash_version",
"hangid",
"process_type",
"release_channel")
# steps
# 1. pull all distinct dates
sql = """
SELECT DISTINCT to_char(date_processed, 'YYYY-MM-DD') as day
FROM reports
ORDER BY day DESC
"""
if numberofdays:
sql = "%s LIMIT %s" % (sql, numberofdays)
print 'Calculating dates... '
days = db.execute(cur, sql)
days_list = []
for day in days:
days_list.append(day[0])
#~ days_list = [
#~ '2012-03-04T00:00:00+00:00'
#~ ]
store_filename = 'dump.json'
store_filename = os.path.normpath('%s/%s' % (path, store_filename))
store = open(store_filename, 'w')
print 'Store file created: %s' % store_filename
indexes_filename = 'indexes.txt'
indexes_filename = os.path.normpath('%s/%s' % (path, indexes_filename))
indexes = open(indexes_filename, 'w')
print 'Indexes file created: %s' % indexes_filename
for day in days_list:
date_from = dtu.datetimeFromISOdateString(day)
date_to = date_from + datetime.timedelta(1)
datestr = date_from.strftime('%y%m%d')
es_index = 'socorro_%s' % datestr
es_type = 'crash_reports'
action_line = '{"index":{"_index":"%s","_type":"%s"}}\n' % (
es_index, es_type)
indexes.write('%s\n' % es_index)
# 2. for each date, pull all crashes of the day
day_sql = " ".join(("SELECT %s" % ", ".join(fields_list),
"FROM reports",
"WHERE date_processed BETWEEN %s AND %s"))
print 'Getting crash reports for day %s' % date_from.date()
crashes_list = db.execute(cur, day_sql, (date_from, date_to))
for crash in crashes_list:
# 3. for each crash report
json_crash = dict(zip(fields_list, crash))
# stringify datetime fields
for i in json_crash:
if isinstance(json_crash[i], datetime.datetime):
json_crash[i] = dtu.date_to_string(json_crash[i])
store.write(action_line)
store.write('%s\n' % json.dumps(json_crash))
store.close()
crash_files.append(store_filename)
indexes.close()
crash_files.append(indexes_filename)
connection.close()
return generate_dump(crash_files, path)
def setUp(self):
"""Create a configuration context and a database connection. """
self.config = self.get_standard_config()
self.database = db.Database(self.config)
self.connection = self.database.connection()
os.makedirs(logfileDir)
except OSError, x:
if errno.EEXIST != x.errno: raise
f = open(me.config.logFilePathname, 'w')
f.close()
fileLog = logging.FileHandler(me.logFilePathname, 'a')
fileLog.setLevel(logging.DEBUG)
fileLogFormatter = logging.Formatter(
me.config.get('logFileLineFormatString',
'%(asctime)s %(levelname)s - %(message)s'))
fileLog.setFormatter(fileLogFormatter)
me.logger = logging.getLogger('cron_test')
me.logger.setLevel(logging.DEBUG)
me.logger.addHandler(fileLog)
me.database = sdatabase.Database(me.config)
#me.dsn = "host=%(databaseHost)s dbname=%(databaseName)s user=%(databaseUserName)s password=%(databasePassword)s" % (me.config)
class TestNamedCursor(TestCase):
def setUp(self):
global me
if not me:
createMe()
self.testDB = TestDB()
self.testDB.removeDB(me.config, me.logger)
self.testDB.createDB(me.config, me.logger)
#self.connection = psycopg2.connect(me.dsn)
self.connection = me.database.connection()
def tearDown(self):
def __init__(self, configContext):
super(EmailSubscription, self).__init__(configContext)
self.database = db.Database(configContext)
self.email_form = form.Form(
form.Textbox('token', form.notnull),
form.Textbox('status', form.notnull))
