def setUp(self) -> None:
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)token', 'Author', 'User', '(?i)kms'],
targets=[
'arn:aws:sqs:us-west-2:\d+',
'us-west-2.queue.amazonaws.com/\d+', '3A\d+', '2F\d+', ':\d+:',
'\"sg-.*\"', '\"subnet-.*\"', '\"vpc-.*\"'
])
with self.vcr.use_cassette('AWSExportIO_setUp.yml'):
ground = FQFile(s3_settings=upl_test_s3,
read_settings=upl_test_read,
byte_range="0-1000",
qc_scale="phred")
async_file = FQFile(s3_settings=s3_set3,
read_settings=read_set3,
byte_range="0-1000",
qc_scale="phred")
self.sync_two_reads = FQSyncTwoReads((ground, async_file))
self.sync_two_reads.two_read_sync()
self.bus_file_adt_atac = BUSFileAdtAtac(self.sync_two_reads)
self.bus_file_adt_atac.bus_file_stream()
adt_atac_bus_header = BUSHeaderAdtAtac()
self.header = adt_atac_bus_header.output_header()
self.native_bus_tools = BUSTools(
bus_header=self.header,
bus_records=self.bus_file_adt_atac.bit_records)
self.native_bus_tools.run_pipe([self.native_bus_tools.bus_sort()])
def setUp(self) -> None:
settings = {'profile_name': 'gvaihir'}
self.sqs_interface = SQSInterface(conf, 'unit-test', **settings)
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)X-Amz', 'Author', 'User'],
targets=['us-west-2:\d+:unit', 'com/\d+/unit', '2F\d+%2F'])
def setUp(self) -> None:
self.sqs_interface = SQSInterface(conf, 'unit_test')
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)token', 'Author', 'User'],
targets=[
'arn:aws:sqs:us-west-2:\d+',
'us-west-2.queue.amazonaws.com/\d+', '3A\d+', '2F\d+', ':\d+:',
'\"sg-.*\"', '\"subnet-.*\"', '\"vpc-.*\"'
])
def setUp(self) -> None:
settings = {'profile_name': 'gvaihir'}
self.lambda_interface = LambdaInterface(conf, 'unittests', **settings)
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)token', 'Author', 'User'],
targets=[
'arn:aws:sqs:us-west-2:\d+',
'us-west-2.queue.amazonaws.com/\d+', '3A\d+', '2F\d+', ':\d+:',
'\"sg-.*\"', '\"subnet-.*\"', '\"vpc-.*\"'
])
def test_blind_byte_range(self):
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)token', 'Author', 'User', '(?i)kms'],
targets=[
'arn:aws:sqs:us-west-2:\d+',
'us-west-2.queue.amazonaws.com/\d+', '3A\d+', '2F\d+', ':\d+:',
'\"sg-.*\"', '\"subnet-.*\"', '\"vpc-.*\"'
])
with self.vcr.use_cassette('blind_byte_range.yml'):
blind_range = blind_byte_range(s3_set)
next(blind_range)
lol = next(blind_range)
self.assertEqual(lol[0], int(1e9 - 18))
class TestBUSFileAdtAtac(TestCase):
def setUp(self) -> None:
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)token', 'Author', 'User', '(?i)kms'],
targets=[
'arn:aws:sqs:us-west-2:\d+',
'us-west-2.queue.amazonaws.com/\d+', '3A\d+', '2F\d+', ':\d+:',
'\"sg-.*\"', '\"subnet-.*\"', '\"vpc-.*\"'
])
with self.vcr.use_cassette('BUSFileAdtAtac_setUp.yml'):
ground = FQFile(s3_settings=upl_test_s3,
read_settings=upl_test_read,
byte_range='0-1000',
qc_scale="phred")
async_file = FQFile(s3_settings=s3_set3,
read_settings=read_set3,
byte_range='0-1000',
qc_scale="phred")
self.sync_two_reads = FQSyncTwoReads((ground, async_file))
self.sync_two_reads.two_read_sync()
self.bus_file_adt_atac = BUSFileAdtAtac(self.sync_two_reads)
def test_bus_file_stream_adt_atac(self):
self.bus_file_adt_atac.bus_file_stream()
bit_rec = BitRecord()
bc = bit_rec.dna_to_twobit('TCGTCGGCAGCGTCAGCTGGA')
umi = bit_rec.dna_to_twobit('CCTTTAAG')
seq = bit_rec.dna_to_twobit('GCTAT')
pack = struct.pack('<QQLLLL', bc, umi, seq, 1, 0, 0)
lol = next(self.bus_file_adt_atac.bit_records)
self.assertEqual(lol, pack)
class TestAWSExportIO(TestCase):
def setUp(self) -> None:
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)token', 'Author', 'User', '(?i)kms'],
targets=[
'arn:aws:sqs:us-west-2:\d+',
'us-west-2.queue.amazonaws.com/\d+', '3A\d+', '2F\d+', ':\d+:',
'\"sg-.*\"', '\"subnet-.*\"', '\"vpc-.*\"'
])
with self.vcr.use_cassette('AWSExportIO_setUp.yml'):
ground = FQFile(s3_settings=upl_test_s3,
read_settings=upl_test_read,
byte_range="0-1000",
qc_scale="phred")
async_file = FQFile(s3_settings=s3_set3,
read_settings=read_set3,
byte_range="0-1000",
qc_scale="phred")
self.sync_two_reads = FQSyncTwoReads((ground, async_file))
self.sync_two_reads.two_read_sync()
self.bus_file_adt_atac = BUSFileAdtAtac(self.sync_two_reads)
self.bus_file_adt_atac.bus_file_stream()
adt_atac_bus_header = BUSHeaderAdtAtac()
self.header = adt_atac_bus_header.output_header()
self.native_bus_tools = BUSTools(
bus_header=self.header,
bus_records=self.bus_file_adt_atac.bit_records)
self.native_bus_tools.run_pipe([self.native_bus_tools.bus_sort()])
def test_bus_tools_export(self):
with self.vcr.use_cassette('AWSExportIO_bus_tools_export.yml'):
bt_export = BUSToolsExport(s3_settings=s3_set2)
bt_export.processed_bus_upload_s3(
byte_seq=self.native_bus_tools.processed_bus_file)
lol = S3Interface(test_s3_set.bucket,
test_s3_set.object_key,
profile_name="gvaihir")
header = BUSHeaderAdtAtac()
h = header.output_header()
test_h = lol.get_bytes_s3(0, 44).read()
self.assertEqual(h, test_h)
def test_block_byte_export(self):
with self.vcr.use_cassette('AWSExportIO_block_byte_export.yml'):
handle = BlocksIO(io_s3_set, '0-101000')
handle.get_object_part()
self.block_split: BlockSplit = BlockSplit(handle)
lol = BlockByte(self.block_split)
lol.byte_blocks()
bb_export = BlockByteExport(s3_settings=io_s3_set,
misc_id='0-101000')
bb_export.block_range_upload_s3(byte_seq=lol.byte_block_gen)
kk = S3Interface(
io_s3_set.bucket,
'anton/scito/mock/fastq/TEST_FASTQ.BLOCK_BYTE.0-101000',
profile_name="gvaihir")
test_kk = kk.get_bytes_s3(0, 15).read()
self.assertEqual(struct.unpack('<QQ', test_kk), (0, 7331))
class TestLambdaInterface(TestCase):
def setUp(self) -> None:
settings = {'profile_name': 'gvaihir'}
self.lambda_interface = LambdaInterface(conf, 'unittests', **settings)
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)token', 'Author', 'User'],
targets=[
'arn:aws:sqs:us-west-2:\d+',
'us-west-2.queue.amazonaws.com/\d+', '3A\d+', '2F\d+', ':\d+:',
'\"sg-.*\"', '\"subnet-.*\"', '\"vpc-.*\"'
])
def test_function_exists(self):
with self.vcr.use_cassette('LambdaInterface_function_exists.yml'):
try:
existence = self.lambda_interface.function_exists()
except:
existence = False
self.assertTrue(isinstance(existence, bool))
self.assertFalse(existence)
def test_destroy(self):
settings = {'profile_name': 'gvaihir'}
self.lambda_interface = LambdaInterface(conf, 'genomics-Unit-test',
**settings)
with self.vcr.use_cassette('LambdaInterface_destroy.yml'):
self.assertTrue(self.lambda_interface.function_exists())
self.lambda_interface.destroy()
#time.sleep(15)
self.assertFalse(self.lambda_interface.function_exists())
def setUp(self) -> None:
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)token', 'Author', 'User', '(?i)kms'],
targets=[
'arn:aws:sqs:us-west-2:\d+',
'us-west-2.queue.amazonaws.com/\d+', '3A\d+', '2F\d+', ':\d+:',
'\"sg-.*\"', '\"subnet-.*\"', '\"vpc-.*\"'
])
with self.vcr.use_cassette('BUSFileAdtAtac_setUp.yml'):
ground = FQFile(s3_settings=upl_test_s3,
read_settings=upl_test_read,
byte_range='0-1000',
qc_scale="phred")
async_file = FQFile(s3_settings=s3_set3,
read_settings=read_set3,
byte_range='0-1000',
qc_scale="phred")
self.sync_two_reads = FQSyncTwoReads((ground, async_file))
self.sync_two_reads.two_read_sync()
self.bus_file_adt_atac = BUSFileAdtAtac(self.sync_two_reads)
class TestECBuild(TestCase):
def setUp(self) -> None:
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(attributes=['(?i)X-Amz', 'Author', 'User'],
targets=['arn:aws:sqs:us-west-2:\d+'])
self.ec_build = ECBuild(config=conf)
def test_prepare_maps(self):
with self.vcr.use_cassette('ECBuild_prepare_maps.yml'):
self.ec_build.prepare_maps(outdir=os.path.join(curr_dir, "fixtures/maps"))
ec = []
with open(os.path.join(curr_dir, "fixtures/maps", "ec_map.tsv"), 'r') as ec_map:
for line in ec_map:
ec.append(line.strip().split('\t'))
self.assertEqual(len(ec), 60)
self.assertEqual(ec[0], ['627', '0'])
genes = []
with open(os.path.join(curr_dir, "fixtures/maps", "gene_map.tsv"), 'r') as gene_map:
for line in gene_map:
genes.append(line.strip().split('\t'))
self.assertEqual(len(genes), 60)
self.assertEqual(genes[0], ['CD45', 'CD45'])
def test__serve_features(self):
with self.vcr.use_cassette('ECBuild__serve_features.yml'):
feat_generator = self.ec_build._serve_features()
test = next(feat_generator)
self.assertTrue(isinstance(test, str))
self.assertEqual(test, 'GCTAT\tCD45')
def test__construct_maps(self):
with self.vcr.use_cassette('ECBuild__construct_maps.yml'):
maps = self.ec_build._construct_maps(sep='\t')
entries = [next(x) for x in maps]
[self.assertTrue(isinstance(x, str)) for x in entries]
self.assertEqual(entries, ['627\t0\n', 'CD45\n', 'CD45\tCD45\n'])
def test__is_dna(self):
self.assertTrue(self.ec_build._is_dna('ANGNANTNTANTACACAAGCAC'))
self.assertTrue(self.ec_build._is_dna('actgn'))
self.assertFalse(self.ec_build._is_dna(''))
self.assertFalse(self.ec_build._is_dna('JFKSDJFA'))
class TestSQSInterface(TestCase):
def setUp(self) -> None:
settings = {'profile_name': 'gvaihir'}
self.sqs_interface = SQSInterface(conf, 'unit-test', **settings)
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)X-Amz', 'Author', 'User'],
targets=['us-west-2:\d+:unit', 'com/\d+/unit', '2F\d+%2F'])
def test_queue_exists(self):
with self.vcr.use_cassette('SQSInterface_queue_exists.yml'):
create_queues(self.sqs_interface)
self.assertTrue(self.sqs_interface.queue_exists(dead_letter=True))
self.assertFalse(
self.sqs_interface.queue_exists(dead_letter=False))
def test_messages_pending(self):
msg_body = 'Hello World'
with self.vcr.use_cassette('SQSInterface_messages_pending.yml'):
create_queues(self.sqs_interface)
self.assertFalse(
self.sqs_interface.messages_pending(dead_letter=True))
active_sqs = self.sqs_interface.sqs.get_queue_by_name(
QueueName=self.sqs_interface.dead_letter_name)
active_sqs.send_message(MessageBody=msg_body)
self.assertTrue(
self.sqs_interface.messages_pending(dead_letter=True))
msgs = []
for message in active_sqs.receive_messages():
msgs.append(message.body)
message.delete()
self.assertFalse(
self.sqs_interface.messages_pending(dead_letter=True))
def test_destroy(self):
settings = {'profile_name': 'gvaihir'}
self.sqs_interface = SQSInterface(conf, 'unit_test', **settings)
with self.vcr.use_cassette('SQSInterface_destroy.yml'):
self.assertTrue(self.sqs_interface.queue_exists())
self.sqs_interface.destroy()
#time.sleep(15)
self.assertFalse(self.sqs_interface.queue_exists())
class Test(TestCase):
def setUp(self) -> None:
self.sqs_interface = SQSInterface(conf, 'unit_test')
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(
attributes=['(?i)token', 'Author', 'User'],
targets=[
'arn:aws:sqs:us-west-2:\d+',
'us-west-2.queue.amazonaws.com/\d+', '3A\d+', '2F\d+', ':\d+:',
'\"sg-.*\"', '\"subnet-.*\"', '\"vpc-.*\"'
])
def test_prep_queues(self):
with self.vcr.use_cassette('architecture_utils_prep_queues.yml'):
main_queue = prep_queues(conf, 'unit_test')
main_sqs_exists = self.sqs_interface.queue_exists(
dead_letter=False)
dead_letter_sqs_exists = self.sqs_interface.queue_exists(
dead_letter=True)
# no need to destroy a queue, it's only in fixtures and does not exist in AWS
self.assertTrue(main_sqs_exists)
self.assertTrue(dead_letter_sqs_exists)
# No fixtures
def test_build_lambda(self):
lambda_conf = 'anton/scito/scito_count/lambda_settings/true_split_settings_TEST.json'
lambda_interface = LambdaInterface(conf, 'genomics-Unit-test')
self.assertFalse(lambda_interface.function_exists())
main_queue = prep_queues(conf, 'unit_test')
func_response, event_map_response = build_lambda(
config=conf,
lambda_name='genomics-Unit-test',
lambda_settings=lambda_conf,
event_source_func=settings_event_source_true_split_lambda,
sqs_queue=main_queue)
self.assertTrue(lambda_interface.function_exists())
def setUp(self) -> None:
self.ufixtures = UfixVcr(os.path.join(curr_dir, 'fixtures/cassettes'))
self.vcr = self.ufixtures.sanitize(attributes=['(?i)X-Amz', 'Author', 'User'],
targets=['arn:aws:sqs:us-west-2:\d+'])
self.ec_build = ECBuild(config=conf)