def test_basic(self): database: TestDatabase = TestDatabase() table1: TestTable = database.create_table("table1") table1.add_entry("0/123.400000-13/1-1/2-0.00000-2-suffix.new", "G H I\nJ K L\n") entry1: TestEntry = table1.add_entry("0/123.400000-13/1-1/1-0.0000-2-suffix.new", "A B C\nD E F\n") log = database.create_table("log") params = { "bucket": table1.name, "chunk_size": 20, "file": "combine_file", "format": "new_line", "log": log.name, "name": "combine", "ranges": False, "sort": False, "output_format": "new_line", "timeout": 60, } database.params = params event = tutils.create_event(database, table1.name, entry1.key) context = tutils.create_context(params) combine_files.main(event, context) entries: List[TestEntry] = database.get_entries(table1.name) self.assertEqual(len(entries), 3) combined_entry = entries[-1] self.assertEqual(combined_entry.key, "1/123.400000-13/1-1/1-0.000000-1-suffix.new") self.assertEqual(combined_entry.get_content().decode("utf-8"), "A B C\nD E F\nG H I\nJ K L\n")
def test_run(self): params = { "file": "application", "log": "log", "name": "util", "test": True, "timeout": 60, } s3 = TestDatabase() bucket1 = s3.create_table("bucket1") log = s3.create_table("log") bucket1.add_entry("0/123.400000-13/1-1/1-1-0-suffix.txt", "") bucket1.add_entry("1/123.400000-13/1-1/1-1-0-suffix.txt", "") bucket1.add_entry("0/123.400000-13/1-1/2-1-1-suffix.txt", "") bucket1.add_entry("1/123.400000-13/1-1/2-1-1-suffix.txt", "") log.add_entry("0/123.400000-13/1-1/3-1-1-suffix.log", "") log.add_entry("1/123.400000-13/1-1/3-1-1-suffix.log", "") event = tutils.create_event(s3, bucket1.name, "0/123.4-13/1-1/1-1-0-suffix.txt", params) context = tutils.create_context(params) # Call on entry that doesn't have a log entry # func = MagicMock() # util.handle(event, context, func) # self.assertTrue(func.called) # Call on entry that does have a log entry func = MagicMock() event["Records"][0]["s3"]["bucket"]["name"] = bucket1.name event["Records"][0]["s3"]["object"][ "key"] = "0/123.400000-13/1-1/3-1-1-suffix.txt" util.handle(event, context, func) self.assertFalse(func.called)
def test_basic(self): s3: TestDatabase = TestDatabase() s3.create_table("log") table1: TestTable = s3.create_table("table1") entry1: TestEntry = table1.add_entry( "0/123.400000-13/1-1/1-1-1-suffix.blast", """target_name: 1 query_name: 1 optimal_alignment_score: 540 suboptimal_alignment_score: 9 target_name: 1 query_name: 1 optimal_alignment_score: 300 suboptimal_alignment_score: 112 target_name: 1 query_name: 1 optimal_alignment_score: 193 suboptimal_alignment_score: 48""") pivots: List[Dict[str, Any]] = [] increment = 300000 for i in range(3): start = i * increment pivots.append({"range": [start, start + increment], "bin": i + 1}) params = { "bucket": table1.name, "execute": 0, "file": "sort", "format": "blast", "identifier": "score", "log": "log", "name": "sort", "pivots": pivots, "s3": s3, "storage_class": "STANDARD", "timeout": 60, } event = tutils.create_event(s3, table1.name, entry1.key, params) context = tutils.create_context(params) sort.handler(event, context) objs = s3.get_entries(table1.name) self.assertEqual(len(objs), 4) objs = sorted(objs, key=lambda obj: obj.key) self.assertEqual( objs[1].get_content().decode("utf-8"), """target_name: 1 query_name: 1 optimal_alignment_score: 193 suboptimal_alignment_score: 48""") self.assertEqual( objs[2].get_content().decode("utf-8"), """target_name: 1 query_name: 1 optimal_alignment_score: 300 suboptimal_alignment_score: 112 target_name: 1 query_name: 1 optimal_alignment_score: 540 suboptimal_alignment_score: 9""") self.assertEqual(objs[3].get_content().decode("utf-8"), "")
def test_offsets(self): pivots = [] increment = 300000 for i in range(3): start = i * increment pivots.append({"range": [start, start + increment], "bin": i + 1}) params = { "bucket": "table1", "file": "sort", "identifier": "score", "input_format": "blast", "log": "log", "name": "sort", "pivots": pivots, "timeout": 60, } s3 = TestDatabase(params) s3.create_table(params["log"]) table1 = s3.create_table(params["bucket"]) entry1 = table1.add_entry( "0/123.400000-13/1-1/1-0.0000-1-suffix.blast", """target_name: 1 query_name: 1 optimal_alignment_score: 540 suboptimal_alignment_score: 9 target_name: 1 query_name: 1 optimal_alignment_score: 300 suboptimal_alignment_score: 112 target_name: 1 query_name: 1 optimal_alignment_score: 193 suboptimal_alignment_score: 48""") event = tutils.create_event(s3, table1.name, entry1.key) context = tutils.create_context(params) sort.main(event, context) entries = sorted(s3.get_entries(table1.name, "1/"), key=lambda e: e.key) self.assertEqual(len(entries), 3) self.assertEqual( entries[0].get_content().decode("utf-8"), """target_name: 1 query_name: 1 optimal_alignment_score: 193 suboptimal_alignment_score: 48""") self.assertEqual( entries[1].get_content().decode("utf-8"), """target_name: 1 query_name: 1 optimal_alignment_score: 300 suboptimal_alignment_score: 112 target_name: 1 query_name: 1 optimal_alignment_score: 540 suboptimal_alignment_score: 9""") self.assertEqual(entries[2].get_content().decode("utf-8"), "")
def test_offsets(self): database: TestDatabase = TestDatabase() log: TestTable = database.create_table("log") table1: TestTable = database.create_table("table1") entry1: TestEntry = table1.add_entry( "0/123.400000-13/1-1/1-1-1-suffix.blast", """target_name: 1 query_name: 1 optimal_alignment_score: 540 suboptimal_alignment_score: 9 target_name: 1 query_name: 1 optimal_alignment_score: 2 suboptimal_alignment_score: 9 target_name: 1 query_name: 1 optimal_alignment_score: 300 suboptimal_alignment_score: 112 target_name: 1 query_name: 1 optimal_alignment_score: 290 suboptimal_alignment_score: 321 target_name: 1 query_name: 1 optimal_alignment_score: 193 suboptimal_alignment_score: 48""") params = { "bucket": table1.name, "execute": 0, "file": "sort", "format": "blast", "identifier": "score", "log": log.name, "name": "sort", "num_bins": 2, "database": database, "storage_class": "STANDARD", "timeout": 60, } event = tutils.create_event(database, table1.name, entry1.key, params) context = tutils.create_context(params) pivot_file.handler(event, context) entries = database.get_entries(table1.name) self.assertEqual(len(entries), 2) self.assertEqual( entries[1].get_content().decode("utf-8"), "{0:s}\n{1:s}\n2009.0\t290321.0\t540010.0".format( table1.name, entry1.key))
def test_basic(self): database = TestDatabase() table1 = database.create_table("table1") log = database.create_table("log") entry1 = table1.add_entry( "0/123.400000-13/1-1/1-1-0-suffix.new_line", "A B C\nD E F\nG H I\nJ K L\nM N O\nP Q R\n") input_format = util.parse_file_name(entry1.key) output_format = dict(input_format) output_format["prefix"] = 1 params = { "execute": 0, "file": "split_file", "format": "new_line", "log": log.name, "name": "split", "output_function": "an-output-function", "ranges": False, "split_size": 20, "timeout": 60, } event = tutils.create_event(database, table1.name, entry1.key, params) context = tutils.create_context(params) split_file.handler(event, context) invoke1 = get_invoke("an-output-function", table1.name, entry1.key, prefix=1, offsets=[0, 19], file_id=1, num_files=2) invoke2 = get_invoke("an-output-function", table1.name, entry1.key, prefix=1, offsets=[20, 35], file_id=2, num_files=2) expected_invokes = [invoke1, invoke2] self.check_payload_equality(expected_invokes, database.payloads)
def test_basic(self): params = { "bucket": "table1", "file": "sort", "input_format": "blast", "identifier": "score", "log": "log", "name": "sort", "num_pivot_bins": 2, "timeout": 60, } database: TestDatabase = TestDatabase(params) log: TestTable = database.create_table(params["log"]) table1: TestTable = database.create_table(params["bucket"]) entry1: TestEntry = table1.add_entry("0/123.400000-13/1-1/1-0.0000-1-suffix.blast", """target_name: 1 query_name: 1 optimal_alignment_score: 540 suboptimal_alignment_score: 9 target_name: 1 query_name: 1 optimal_alignment_score: 2 suboptimal_alignment_score: 9 target_name: 1 query_name: 1 optimal_alignment_score: 300 suboptimal_alignment_score: 112 target_name: 1 query_name: 1 optimal_alignment_score: 290 suboptimal_alignment_score: 321 target_name: 1 query_name: 1 optimal_alignment_score: 193 suboptimal_alignment_score: 48""") event = tutils.create_event(database, table1.name, entry1.key) context = tutils.create_context(params) pivot_file.main(event, context) entries = database.get_entries(table1.name) self.assertEqual(len(entries), 2) self.assertEqual(entries[1].get_content().decode("utf-8"), "{0:s}\n{1:s}\n2009.0\t290321.0\t540010.0".format(table1.name, entry1.key))