def test_discover_logfiles_in_s3(self):
# bucket must exist
self.s3.create_bucket(Bucket='bucket')
self.s3.put_object(Bucket='bucket', Key='logs/inst1/f1')
self.s3.put_object(Bucket='bucket', Key='logs/inst1/f2')
# now some logfile for another instance
self.s3.put_object(Bucket='bucket', Key='logs/other/f1')
logfiles = {
S3LogFile('f1', '', '', size=0),
S3LogFile('f2', '', '', size=0)
}
logfilehandler = LogFileHandler(RDSInstance('inst1'), 'bucket', 'logs')
self.assertSetEqual(logfiles, logfilehandler.discover_logfiles_in_s3())
def discover_logfiles_in_s3(self):
files = set()
for (filename, size) in s3.get_files(self.dst_bucket, self.dst_prefix_instance):
if len(filename) > len(self.dst_prefix_instance) + 1:
name = filename[len(self.dst_prefix_instance) + 1:]
log_file = S3LogFile(name, self.dst_bucket,
self.dst_prefix_instance, size=size)
files.add(log_file)
return files
def test_not_equals__differ_size(self):
log_file = S3LogFile('foo', 'bucket', 'prefix')
rds_file = RdsLogFile('foo', 'instance')
log_file.size = 1
rds_file.size = 2
self.assertTrue(log_file != rds_file)
self.assertTrue(rds_file != log_file)
self.assertFalse(log_file == rds_file)
self.assertFalse(rds_file == log_file)
def test_sync_one_file_with_differs_in_size(self):
logger.info('[test] redownloading file ...')
rds_log_dog = RDSLogDog(self.config)
rds_log_dog.process_instance(self.rds_instance)
# choose a random file to reduce filesize in s3 to trigger redownload
logfile = utils.get_one_random_logfile(self.rds_instance.name)
lf = S3LogFile(logfile, self.bucket_name,
"{}/{}".format(self.dst_prefix, self.rds_instance.name))
logger.debug('truncate size of {}'.format(lf.name))
lf.fetch_size()
size = lf.size
s3.write_data_to_object(lf.bucket, lf.get_dst_key(), 'foooooo')
lf.fetch_size()
logger.debug('new size of {} is {}'.format(lf.name, lf.size))
self.assertNotEqual(size, lf.size)
logger.debug('reprocessing instance ...')
rds_log_dog.process_instance(self.rds_instance)
lf.fetch_size()
self.assertEqual(size, lf.size)
def get_s3logfile(self, name):
return S3LogFile(name, self.dst_bucket, self.dst_prefix_instance)
def test_logfiles_to_copy_size_diff(self):
src = {RdsLogFile('foo', '', size=123)}
dst = {S3LogFile('foo', '', '', size=0)}
self.assertEqual({LogFile('foo', size=123)},
LogFileHandler.logfiles_to_copy(src, dst))
def test_logfiles_to_copy_new_files_on_src_old_on_dst(self):
src = {RdsLogFile('foo', ''), RdsLogFile('bar', '')}
dst = {S3LogFile('foo', '', ''), S3LogFile('xyz', '', '')}
self.assertEqual({LogFile('bar')},
LogFileHandler.logfiles_to_copy(src, dst))
def test_logfiles_to_copy_empty_src(self):
src = set()
dst = set()
self.assertEqual(set(), LogFileHandler.logfiles_to_copy(src, dst))
dst = {S3LogFile('foo', '', '')}
self.assertEqual(set(), LogFileHandler.logfiles_to_copy(src, dst))
def test_not_equals__wo_size(self):
log_file = S3LogFile('foo', 'bucket', 'prefix')
rds_file = RdsLogFile('bar', 'instance')
self.assertNotEqual(log_file, rds_file)
def test_equals__w_size(self):
log_file = S3LogFile('foo', 'bucket', 'prefix')
rds_file = RdsLogFile('foo', 'instance')
log_file.size = 1
rds_file.size = 1
self.assertEqual(log_file, rds_file)
def test_get_dst_key(self):
log_file = S3LogFile('name', 'bucket', 'prefix')
self.assertEqual('prefix/name', log_file.get_dst_key())
def test_fetch_size_s3__calls(self, s3_get_size, rds_get_size):
log_file = S3LogFile('foo', 'bucket', 'prefix')
log_file.fetch_size()
self.assertTrue(s3_get_size.called)
self.assertFalse(rds_get_size.called)
def test_fetch_size_s3(self, get_size):
log_file = S3LogFile('foo', 'bucket', 'prefix')
get_size.return_value = 23
log_file.fetch_size()
get_size.assert_called_with('bucket', log_file.get_dst_key())
self.assertEqual(23, log_file.size)
def test_s3_write(self, copy):
log_file = S3LogFile('name', 'bucket', 'prefix')
with NamedTemporaryFile() as tempfile:
log_file.write(tempfile)
copy.assert_called_with('bucket', 'prefix/name', tempfile)
def test_s3_log_file_init(self):
log_file = S3LogFile('foo', 'bucket', 'prefix')
self.assertEqual('foo', log_file.name)
self.assertEqual('prefix', log_file.prefix)
self.assertEqual('bucket', log_file.bucket)