def test_run(self): project = Mock() project.name = "test_project" project.user = "******" filename = '/tmp/' + str(uuid.uuid4()) shutil.copyfile(self.data_dir + '/interval15_1k.csv', filename) process_file_mock = Mock() reduce_interval_mock = Mock() chunk = Chunk( file=filename, logger=Mock(), compress=None, interval=15, process_file=process_file_mock, reduce_interval=reduce_interval_mock, project=project) chunk._chain_tasks = Mock() chunk.chunk_size = 100 # reduce chunk size for testing chunk.run() # events = Event \ # .objects(user=project.user, project=project.name) \ # .allow_filtering().all() # print(events[0]) ok_(not os.path.exists(filename), 'Original file should be deleted') for i in range(1, 11): filename_part = '{0}-part{1:03d}'.format(filename, i) ok_( os.path.exists(filename_part), 'File part {0} does not exists'.format(i)) eq_( self._file_len(filename_part), chunk.chunk_size+1, 'File part {0} does fit in {1} size'.format(i, Chunk.chunk_size)) os.remove(filename_part) eq_(1, chunk._chain_tasks.call_count)
def test_init(self): project = Mock() project.name = "test_project" project.user = "******" chunk = Chunk(file='file', compress=None, interval=15, notexistingproperty='value', project=project) eq_(chunk.file, 'file') eq_(chunk.compress, None) eq_(chunk.project, project) eq_(chunk.interval, 15) assert_raises(AttributeError, getattr, chunk, 'notexistingproperty')
def test_run(self): project = Mock() project.name = "test_project" project.user = "******" filename = '/tmp/' + str(uuid.uuid4()) shutil.copyfile(self.data_dir + '/interval15_1k.csv', filename) process_file_mock = Mock() reduce_interval_mock = Mock() chunk = Chunk(file=filename, logger=Mock(), compress=None, interval=15, process_file=process_file_mock, reduce_interval=reduce_interval_mock, project=project) chunk._chain_tasks = Mock() chunk.chunk_size = 100 # reduce chunk size for testing chunk.run() # events = Event \ # .objects(user=project.user, project=project.name) \ # .allow_filtering().all() # print(events[0]) ok_(not os.path.exists(filename), 'Original file should be deleted') for i in range(1, 11): filename_part = '{0}-part{1:03d}'.format(filename, i) ok_(os.path.exists(filename_part), 'File part {0} does not exists'.format(i)) eq_( self._file_len(filename_part), chunk.chunk_size + 1, 'File part {0} does fit in {1} size'.format( i, Chunk.chunk_size)) os.remove(filename_part) eq_(1, chunk._chain_tasks.call_count)
def chunk(*args, **kwargs): kwargs['logger'] = get_task_logger('tasks.' + __name__) kwargs['process_file'] = filetodb kwargs['simple_stats'] = simple_stats chunker = Chunk(*args, **kwargs) return chunker.run()