def setUp(self):
fd, self.filename = tempfile.mkstemp()
os.close(fd)
self.sqluri = 'sqlite:///%s' % self.filename
self.db = Database(database=self.sqluri)
self._today = datetime.date.today()
self._last_week = self._today - datetime.timedelta(weeks=1)
self._yesterday = self._today - datetime.timedelta(days=1)
def setUp(self):
fd, self.filename = tempfile.mkstemp()
os.close(fd)
self.sqluri = 'sqlite:///%s' % self.filename
self.db = Database(database=self.sqluri)
self._today = datetime.date.today()
self._last_week = self._today - datetime.timedelta(weeks=1)
self._yesterday = self._today - datetime.timedelta(days=1)
class TestDatabase(TestCase):
def setUp(self):
fd, self.filename = tempfile.mkstemp()
os.close(fd)
self.sqluri = 'sqlite:///%s' % self.filename
self.db = Database(database=self.sqluri)
self._today = datetime.date.today()
self._last_week = self._today - datetime.timedelta(weeks=1)
self._yesterday = self._today - datetime.timedelta(days=1)
def tearDown(self):
if os.path.exists(self.filename):
os.remove(self.filename)
def test_inject_no_defaults(self):
self.assertRaises(KeyError, self.db.inject,
[('test', dict(_type='foo', key='value'))])
self.assertRaises(KeyError, self.db.inject,
[('test', dict(key='value', _date=self._today))])
def test_inject(self):
self.db.inject([
('test', dict(_type='foo', key='value', _date=self._last_week)),
])
query = self.db.session.query(Record)
results = query.all()
self.assertEquals(results[0].date, self._last_week)
self.assertEquals(results[0].type, 'foo')
self.assertEquals(results[0].source_id, 'test')
self.assertEquals(results[0].value, '{"key": "value"}')
def test_clear(self):
self.db.inject([
('s1', dict(_type='foo', key='1', _date=self._yesterday)),
('s1', dict(_type='bar', key='2', _date=self._today)),
('s1', dict(_type='bar', key='3', _date=self._today)),
('s2', dict(_type='baz', key='4', _date=self._today)),
('s2', dict(_type='baz', key='5', _date=self._today)),
])
removed = self.db.clear(self._yesterday, self._yesterday, ['s2'])
self.assertEqual(removed, 0)
removed = self.db.clear(self._today, self._today, ['s1'])
self.assertEqual(removed, 2)
removed = self.db.clear(self._yesterday, self._today, ['s1', 's2'])
self.assertEqual(removed, 3)
def extract(config,
start_date,
end_date,
sequence=None,
batch_size=None,
force=False,
purge_only=False,
retries=3):
"""Reads the configuration file and does the job.
"""
defaults = {'here': os.path.abspath(os.path.dirname(config))}
parser = ConfigParser(defaults=defaults)
parser.read(config)
try:
batch_size = parser.get('monolith', 'batch_size')
except NoOptionError:
# using the default value
if batch_size is None:
batch_size = 100
# creating the sequence
sequence = Sequence(parser, sequence)
# load the database
try:
monolith_db = parser.get('monolith', 'database')
except NoOptionError: # pragma: no cover
# BBB support old history config name
try:
monolith_db = parser.get('monolith', 'history')
except NoOptionError:
raise ValueError("You need a database option")
database = Database(database=monolith_db)
# run the engine
engine = Engine(sequence,
database,
batch_size=batch_size,
force=force,
retries=retries)
return engine.run(start_date, end_date, purge_only)
class TestDatabase(TestCase):
def setUp(self):
fd, self.filename = tempfile.mkstemp()
os.close(fd)
self.sqluri = 'sqlite:///%s' % self.filename
self.db = Database(database=self.sqluri)
self._today = datetime.date.today()
self._last_week = self._today - datetime.timedelta(weeks=1)
self._yesterday = self._today - datetime.timedelta(days=1)
def tearDown(self):
if os.path.exists(self.filename):
os.remove(self.filename)
def test_inject_no_defaults(self):
self.assertRaises(KeyError, self.db.inject,
[('test', dict(_type='foo', key='value'))])
self.assertRaises(KeyError, self.db.inject,
[('test', dict(key='value', _date=self._today))])
def test_inject(self):
self.db.inject([
('test', dict(_type='foo', key='value', _date=self._last_week)),
])
query = self.db.session.query(Record)
results = query.all()
self.assertEquals(results[0].date, self._last_week)
self.assertEquals(results[0].type, 'foo')
self.assertEquals(results[0].source_id, 'test')
self.assertEquals(results[0].value, '{"key": "value"}')
def test_clear(self):
self.db.inject([
('s1', dict(_type='foo', key='1', _date=self._yesterday)),
('s1', dict(_type='bar', key='2', _date=self._today)),
('s1', dict(_type='bar', key='3', _date=self._today)),
('s2', dict(_type='baz', key='4', _date=self._today)),
('s2', dict(_type='baz', key='5', _date=self._today)),
])
removed = self.db.clear(self._yesterday, self._yesterday, ['s2'])
self.assertEqual(removed, 0)
removed = self.db.clear(self._today, self._today, ['s1'])
self.assertEqual(removed, 2)
removed = self.db.clear(self._yesterday, self._today, ['s1', 's2'])
self.assertEqual(removed, 3)
