def test_discover_logfiles_in_s3(self): # bucket must exist self.s3.create_bucket(Bucket='bucket') self.s3.put_object(Bucket='bucket', Key='logs/inst1/f1') self.s3.put_object(Bucket='bucket', Key='logs/inst1/f2') # now some logfile for another instance self.s3.put_object(Bucket='bucket', Key='logs/other/f1') logfiles = { S3LogFile('f1', '', '', size=0), S3LogFile('f2', '', '', size=0) } logfilehandler = LogFileHandler(RDSInstance('inst1'), 'bucket', 'logs') self.assertSetEqual(logfiles, logfilehandler.discover_logfiles_in_s3())
def discover_logfiles_in_s3(self): files = set() for (filename, size) in s3.get_files(self.dst_bucket, self.dst_prefix_instance): if len(filename) > len(self.dst_prefix_instance) + 1: name = filename[len(self.dst_prefix_instance) + 1:] log_file = S3LogFile(name, self.dst_bucket, self.dst_prefix_instance, size=size) files.add(log_file) return files
def test_not_equals__differ_size(self): log_file = S3LogFile('foo', 'bucket', 'prefix') rds_file = RdsLogFile('foo', 'instance') log_file.size = 1 rds_file.size = 2 self.assertTrue(log_file != rds_file) self.assertTrue(rds_file != log_file) self.assertFalse(log_file == rds_file) self.assertFalse(rds_file == log_file)
def test_sync_one_file_with_differs_in_size(self): logger.info('[test] redownloading file ...') rds_log_dog = RDSLogDog(self.config) rds_log_dog.process_instance(self.rds_instance) # choose a random file to reduce filesize in s3 to trigger redownload logfile = utils.get_one_random_logfile(self.rds_instance.name) lf = S3LogFile(logfile, self.bucket_name, "{}/{}".format(self.dst_prefix, self.rds_instance.name)) logger.debug('truncate size of {}'.format(lf.name)) lf.fetch_size() size = lf.size s3.write_data_to_object(lf.bucket, lf.get_dst_key(), 'foooooo') lf.fetch_size() logger.debug('new size of {} is {}'.format(lf.name, lf.size)) self.assertNotEqual(size, lf.size) logger.debug('reprocessing instance ...') rds_log_dog.process_instance(self.rds_instance) lf.fetch_size() self.assertEqual(size, lf.size)
def get_s3logfile(self, name): return S3LogFile(name, self.dst_bucket, self.dst_prefix_instance)
def test_logfiles_to_copy_size_diff(self): src = {RdsLogFile('foo', '', size=123)} dst = {S3LogFile('foo', '', '', size=0)} self.assertEqual({LogFile('foo', size=123)}, LogFileHandler.logfiles_to_copy(src, dst))
def test_logfiles_to_copy_new_files_on_src_old_on_dst(self): src = {RdsLogFile('foo', ''), RdsLogFile('bar', '')} dst = {S3LogFile('foo', '', ''), S3LogFile('xyz', '', '')} self.assertEqual({LogFile('bar')}, LogFileHandler.logfiles_to_copy(src, dst))
def test_logfiles_to_copy_empty_src(self): src = set() dst = set() self.assertEqual(set(), LogFileHandler.logfiles_to_copy(src, dst)) dst = {S3LogFile('foo', '', '')} self.assertEqual(set(), LogFileHandler.logfiles_to_copy(src, dst))
def test_not_equals__wo_size(self): log_file = S3LogFile('foo', 'bucket', 'prefix') rds_file = RdsLogFile('bar', 'instance') self.assertNotEqual(log_file, rds_file)
def test_equals__w_size(self): log_file = S3LogFile('foo', 'bucket', 'prefix') rds_file = RdsLogFile('foo', 'instance') log_file.size = 1 rds_file.size = 1 self.assertEqual(log_file, rds_file)
def test_get_dst_key(self): log_file = S3LogFile('name', 'bucket', 'prefix') self.assertEqual('prefix/name', log_file.get_dst_key())
def test_fetch_size_s3__calls(self, s3_get_size, rds_get_size): log_file = S3LogFile('foo', 'bucket', 'prefix') log_file.fetch_size() self.assertTrue(s3_get_size.called) self.assertFalse(rds_get_size.called)
def test_fetch_size_s3(self, get_size): log_file = S3LogFile('foo', 'bucket', 'prefix') get_size.return_value = 23 log_file.fetch_size() get_size.assert_called_with('bucket', log_file.get_dst_key()) self.assertEqual(23, log_file.size)
def test_s3_write(self, copy): log_file = S3LogFile('name', 'bucket', 'prefix') with NamedTemporaryFile() as tempfile: log_file.write(tempfile) copy.assert_called_with('bucket', 'prefix/name', tempfile)
def test_s3_log_file_init(self): log_file = S3LogFile('foo', 'bucket', 'prefix') self.assertEqual('foo', log_file.name) self.assertEqual('prefix', log_file.prefix) self.assertEqual('bucket', log_file.bucket)