def test_basic(self):
s3: TestDatabase = TestDatabase()
s3.create_table("log")
table1: TestTable = s3.create_table("table1")
entry1: TestEntry = table1.add_entry(
"0/123.400000-13/1-1/1-1-1-suffix.blast", """target_name: 1
query_name: 1
optimal_alignment_score: 540 suboptimal_alignment_score: 9
target_name: 1
query_name: 1
optimal_alignment_score: 300 suboptimal_alignment_score: 112
target_name: 1
query_name: 1
optimal_alignment_score: 193 suboptimal_alignment_score: 48""")
pivots: List[Dict[str, Any]] = []
increment = 300000
for i in range(3):
start = i * increment
pivots.append({"range": [start, start + increment], "bin": i + 1})
params = {
"bucket": table1.name,
"execute": 0,
"file": "sort",
"format": "blast",
"identifier": "score",
"log": "log",
"name": "sort",
"pivots": pivots,
"s3": s3,
"storage_class": "STANDARD",
"timeout": 60,
}
event = tutils.create_event(s3, table1.name, entry1.key, params)
context = tutils.create_context(params)
sort.handler(event, context)
objs = s3.get_entries(table1.name)
self.assertEqual(len(objs), 4)
objs = sorted(objs, key=lambda obj: obj.key)
self.assertEqual(
objs[1].get_content().decode("utf-8"), """target_name: 1
query_name: 1
optimal_alignment_score: 193 suboptimal_alignment_score: 48""")
self.assertEqual(
objs[2].get_content().decode("utf-8"), """target_name: 1
query_name: 1
optimal_alignment_score: 300 suboptimal_alignment_score: 112
target_name: 1
query_name: 1
optimal_alignment_score: 540 suboptimal_alignment_score: 9""")
self.assertEqual(objs[3].get_content().decode("utf-8"), "")
def test_run(self):
params = {
"file": "application",
"log": "log",
"name": "util",
"test": True,
"timeout": 60,
}
s3 = TestDatabase()
bucket1 = s3.create_table("bucket1")
log = s3.create_table("log")
bucket1.add_entry("0/123.400000-13/1-1/1-1-0-suffix.txt", "")
bucket1.add_entry("1/123.400000-13/1-1/1-1-0-suffix.txt", "")
bucket1.add_entry("0/123.400000-13/1-1/2-1-1-suffix.txt", "")
bucket1.add_entry("1/123.400000-13/1-1/2-1-1-suffix.txt", "")
log.add_entry("0/123.400000-13/1-1/3-1-1-suffix.log", "")
log.add_entry("1/123.400000-13/1-1/3-1-1-suffix.log", "")
event = tutils.create_event(s3, bucket1.name,
"0/123.4-13/1-1/1-1-0-suffix.txt", params)
context = tutils.create_context(params)
# Call on entry that doesn't have a log entry
# func = MagicMock()
# util.handle(event, context, func)
# self.assertTrue(func.called)
# Call on entry that does have a log entry
func = MagicMock()
event["Records"][0]["s3"]["bucket"]["name"] = bucket1.name
event["Records"][0]["s3"]["object"][
"key"] = "0/123.400000-13/1-1/3-1-1-suffix.txt"
util.handle(event, context, func)
self.assertFalse(func.called)
def test_basic(self):
database: TestDatabase = TestDatabase()
table1: TestTable = database.create_table("table1")
table1.add_entry("0/123.400000-13/1-1/2-0.00000-2-suffix.new", "G H I\nJ K L\n")
entry1: TestEntry = table1.add_entry("0/123.400000-13/1-1/1-0.0000-2-suffix.new", "A B C\nD E F\n")
log = database.create_table("log")
params = {
"bucket": table1.name,
"chunk_size": 20,
"file": "combine_file",
"format": "new_line",
"log": log.name,
"name": "combine",
"ranges": False,
"sort": False,
"output_format": "new_line",
"timeout": 60,
}
database.params = params
event = tutils.create_event(database, table1.name, entry1.key)
context = tutils.create_context(params)
combine_files.main(event, context)
entries: List[TestEntry] = database.get_entries(table1.name)
self.assertEqual(len(entries), 3)
combined_entry = entries[-1]
self.assertEqual(combined_entry.key, "1/123.400000-13/1-1/1-0.000000-1-suffix.new")
self.assertEqual(combined_entry.get_content().decode("utf-8"), "A B C\nD E F\nG H I\nJ K L\n")
def test_offsets(self):
pivots = []
increment = 300000
for i in range(3):
start = i * increment
pivots.append({"range": [start, start + increment], "bin": i + 1})
params = {
"bucket": "table1",
"file": "sort",
"identifier": "score",
"input_format": "blast",
"log": "log",
"name": "sort",
"pivots": pivots,
"timeout": 60,
}
s3 = TestDatabase(params)
s3.create_table(params["log"])
table1 = s3.create_table(params["bucket"])
entry1 = table1.add_entry(
"0/123.400000-13/1-1/1-0.0000-1-suffix.blast", """target_name: 1
query_name: 1
optimal_alignment_score: 540 suboptimal_alignment_score: 9
target_name: 1
query_name: 1
optimal_alignment_score: 300 suboptimal_alignment_score: 112
target_name: 1
query_name: 1
optimal_alignment_score: 193 suboptimal_alignment_score: 48""")
event = tutils.create_event(s3, table1.name, entry1.key)
context = tutils.create_context(params)
sort.main(event, context)
entries = sorted(s3.get_entries(table1.name, "1/"),
key=lambda e: e.key)
self.assertEqual(len(entries), 3)
self.assertEqual(
entries[0].get_content().decode("utf-8"), """target_name: 1
query_name: 1
optimal_alignment_score: 193 suboptimal_alignment_score: 48""")
self.assertEqual(
entries[1].get_content().decode("utf-8"), """target_name: 1
query_name: 1
optimal_alignment_score: 300 suboptimal_alignment_score: 112
target_name: 1
query_name: 1
optimal_alignment_score: 540 suboptimal_alignment_score: 9""")
self.assertEqual(entries[2].get_content().decode("utf-8"), "")
def test_offsets(self):
database: TestDatabase = TestDatabase()
log: TestTable = database.create_table("log")
table1: TestTable = database.create_table("table1")
entry1: TestEntry = table1.add_entry(
"0/123.400000-13/1-1/1-1-1-suffix.blast", """target_name: 1
query_name: 1
optimal_alignment_score: 540 suboptimal_alignment_score: 9
target_name: 1
query_name: 1
optimal_alignment_score: 2 suboptimal_alignment_score: 9
target_name: 1
query_name: 1
optimal_alignment_score: 300 suboptimal_alignment_score: 112
target_name: 1
query_name: 1
optimal_alignment_score: 290 suboptimal_alignment_score: 321
target_name: 1
query_name: 1
optimal_alignment_score: 193 suboptimal_alignment_score: 48""")
params = {
"bucket": table1.name,
"execute": 0,
"file": "sort",
"format": "blast",
"identifier": "score",
"log": log.name,
"name": "sort",
"num_bins": 2,
"database": database,
"storage_class": "STANDARD",
"timeout": 60,
}
event = tutils.create_event(database, table1.name, entry1.key, params)
context = tutils.create_context(params)
pivot_file.handler(event, context)
entries = database.get_entries(table1.name)
self.assertEqual(len(entries), 2)
self.assertEqual(
entries[1].get_content().decode("utf-8"),
"{0:s}\n{1:s}\n2009.0\t290321.0\t540010.0".format(
table1.name, entry1.key))
def test_basic(self):
database = TestDatabase()
table1 = database.create_table("table1")
log = database.create_table("log")
entry1 = table1.add_entry(
"0/123.400000-13/1-1/1-1-0-suffix.new_line",
"A B C\nD E F\nG H I\nJ K L\nM N O\nP Q R\n")
input_format = util.parse_file_name(entry1.key)
output_format = dict(input_format)
output_format["prefix"] = 1
params = {
"execute": 0,
"file": "split_file",
"format": "new_line",
"log": log.name,
"name": "split",
"output_function": "an-output-function",
"ranges": False,
"split_size": 20,
"timeout": 60,
}
event = tutils.create_event(database, table1.name, entry1.key, params)
context = tutils.create_context(params)
split_file.handler(event, context)
invoke1 = get_invoke("an-output-function",
table1.name,
entry1.key,
prefix=1,
offsets=[0, 19],
file_id=1,
num_files=2)
invoke2 = get_invoke("an-output-function",
table1.name,
entry1.key,
prefix=1,
offsets=[20, 35],
file_id=2,
num_files=2)
expected_invokes = [invoke1, invoke2]
self.check_payload_equality(expected_invokes, database.payloads)
def test_basic(self):
params = {
"bucket": "table1",
"file": "sort",
"input_format": "blast",
"identifier": "score",
"log": "log",
"name": "sort",
"num_pivot_bins": 2,
"timeout": 60,
}
database: TestDatabase = TestDatabase(params)
log: TestTable = database.create_table(params["log"])
table1: TestTable = database.create_table(params["bucket"])
entry1: TestEntry = table1.add_entry("0/123.400000-13/1-1/1-0.0000-1-suffix.blast",
"""target_name: 1
query_name: 1
optimal_alignment_score: 540 suboptimal_alignment_score: 9
target_name: 1
query_name: 1
optimal_alignment_score: 2 suboptimal_alignment_score: 9
target_name: 1
query_name: 1
optimal_alignment_score: 300 suboptimal_alignment_score: 112
target_name: 1
query_name: 1
optimal_alignment_score: 290 suboptimal_alignment_score: 321
target_name: 1
query_name: 1
optimal_alignment_score: 193 suboptimal_alignment_score: 48""")
event = tutils.create_event(database, table1.name, entry1.key)
context = tutils.create_context(params)
pivot_file.main(event, context)
entries = database.get_entries(table1.name)
self.assertEqual(len(entries), 2)
self.assertEqual(entries[1].get_content().decode("utf-8"), "{0:s}\n{1:s}\n2009.0\t290321.0\t540010.0".format(table1.name, entry1.key))
def __trigger__(self, payload: Dict[str, Any]):
stage: int = get_stage(payload)
if stage > self.stage:
self.stage = stage
if self.stage == len(self.pipeline):
self.task_queue.put(payload)
return
function_name: str = self.pipeline[stage]["name"]
function_params: Dict[str, Any] = {
**self.params,
**self.pipeline[stage],
**self.functions[function_name]
}
function_module = self.__import_function__("lambdas." +
function_params["file"])
self.database.params = function_params
event = tutils.create_event_from_payload(self.database, payload)
context = tutils.create_context({"timeout": 60})
function_module.main(event, context)
def __trigger__(self, payload: Dict[str, Any]):
stage: int = self.__get_stage__(payload)
if stage == len(self.pipeline):
return
function_name: str = self.pipeline[stage]["name"]
if stage > self.stage:
print("Starting stage {0:d}: {1:s}".format(stage + 1, function_name))
self.stage = stage
function_params: Dict[str, Any] = {**self.params, **self.pipeline[stage], **self.functions[function_name]}
if "format" in function_params:
self.__import_format__(function_params["format"])
if "application" in function_params:
self.__import_application__(function_params["application"])
if function_params["file"] == "map":
self.__import_format__("pivot")
function_module = self.__import_function__(function_params["file"])
self.database.params = function_params
event = tutils.create_event_from_payload(self.database, payload)
context = tutils.create_context({"timeout": 60})
function_module.handler(event, context)
