def test_commands(self):
epidb = DeepBlueClient(address="localhost", port=31415)
# explicitly no initialization. The command should always work.
res, cmds = epidb.commands()
self.assertSuccess(res, cmds)
self.assertEqual(len(cmds), 96)
self.assertEqual(cmds["add_experiment"]["parameters"][0],
['name', 'string', False, 'experiment name'])
self.assertEqual(cmds["add_experiment"]["parameters"][1],
['genome', 'string', False, 'the target genome'])
self.assertEqual(cmds["add_experiment"]["parameters"][2], [
'epigenetic_mark', 'string', False,
'epigenetic mark of the experiment'
])
self.assertEqual(cmds["add_experiment"]["parameters"][3],
['sample', 'string', False, 'id of the used sample'])
self.assertEqual(cmds["add_experiment"]["parameters"][4], [
'technique', 'string', False, 'technique used by this experiment'
])
self.assertEqual(cmds["add_experiment"]["parameters"][5],
['project', 'string', False, 'the project name'])
self.assertEqual(
cmds["add_experiment"]["parameters"][6],
['description', 'string', False, 'description of the experiment'])
self.assertEqual(cmds["add_experiment"]["parameters"][7],
['data', 'string', False, 'the BED formated data'])
self.assertEqual(
cmds["add_experiment"]["parameters"][8],
['format', 'string', False, 'format of the provided data'])
self.assertEqual(
cmds["add_experiment"]["parameters"][9],
['extra_metadata', 'struct', False, 'additional metadata'])
self.assertEqual(cmds["add_experiment"]["parameters"][10],
['user_key', 'string', False, 'users token key'])
def init_base(self, epidb=None):
if not epidb:
epidb = DeepBlueClient(address="localhost", port=31415)
self.init(epidb)
for genome in data.GENOMES:
self.insert_genome(epidb, genome)
for bsource in data.BIOSOURCES:
self.insert_biosource(epidb, bsource)
for tech in data.TECHNIQUES:
self.insert_technique(epidb, tech)
for epimark in data.EPIGENETIC_MARKS:
self.insert_epigenetic_mark(epidb, epimark)
for project in data.PROJECTS:
self.insert_project(epidb, project)
for sample in data.SAMPLES:
self.insert_samples(epidb, sample)
for column in data.COLUMNS:
self.insert_column(epidb, column)
res, cid = epidb.create_column_type_category("STRAND", "Region strand: +, -, .", ["+", "-", "."], self.admin_key)
self.assertSuccess(res, cid)
def test_set_project_public_by_user_no_permission(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
(s, user) = epidb.add_user("NAME", "EMAIL", "INSTITUTE",
self.admin_key)
(s, user_ass) = epidb.add_user("NAME ASS", "EMAIL ASS", "INSTITUTE",
self.admin_key)
s, tmp_user = epidb.modify_user_admin(user[0], "permission_level",
"INCLUDE_EXPERIMENTS",
self.admin_key)
s, tmp_user = epidb.modify_user_admin(user_ass[0], "permission_level",
"INCLUDE_EXPERIMENTS",
self.admin_key)
s, project = epidb.add_project("USER's PROJECT", "COOL", user[1])
(s, (user_add,
project_add)) = epidb.add_user_to_project(user_ass[0], project,
True, user[1])
self.assertSuccess(s, (user, project))
self.assertEqual(user_add, user_ass[0])
self.assertEqual(project, project_add)
(s, user_two) = epidb.add_user("ANOTHER NAME", "ANOTHER EMAIL",
"INSTITUTE", self.admin_key)
(s, status) = epidb.add_user_to_project(user_two[0], project, True,
user_ass[1])
self.assertEqual(
status,
"107100:You are not the project 'p3' owner and neither an administrator."
)
def test_extend(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
(s, query_id) = epidb.tiling_regions(100000000, "hg18", "chr1",
self.admin_key)
(s, r_id) = epidb.get_regions(query_id, "CHROMOSOME,START,END",
self.admin_key)
regions = self.get_regions_request(r_id)
self.assertEqual(regions,
"chr1\t0\t100000000\nchr1\t100000000\t200000000")
(s, f_id) = epidb.extend(query_id, 10000000, "FORWARD", False,
self.admin_key)
(s, r_id) = epidb.get_regions(f_id, "CHROMOSOME,START,END",
self.admin_key)
regions = self.get_regions_request(r_id)
self.assertEqual(regions,
"chr1\t0\t110000000\nchr1\t100000000\t210000000")
(s, f_id) = epidb.extend(query_id, 10000000, "BACKWARD", False,
self.admin_key)
(s, r_id) = epidb.get_regions(f_id, "CHROMOSOME,START,END",
self.admin_key)
regions = self.get_regions_request(r_id)
self.assertEqual(regions,
"chr1\t0\t100000000\nchr1\t90000000\t200000000")
(s, f_id) = epidb.extend(query_id, 10000000, "BOTH", False,
self.admin_key)
(s, r_id) = epidb.get_regions(f_id, "CHROMOSOME,START,END",
self.admin_key)
regions = self.get_regions_request(r_id)
self.assertEqual(regions,
"chr1\t0\t110000000\nchr1\t90000000\t210000000")
def test_include_invalid_regions(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
data = """chr1\t1\t100000000000000000
chr10\t666\t66610000
chrY\t12345\t1234567"""
(s, a) = epidb.add_annotation("test annotation", "hg19", "testing",
data, "CHROMOSOME,START,END",
{"HI": "HOW ARE YOU?"}, self.admin_key)
self.assertFailure(s, a)
self.assertEquals(
a,
"Error while reading the BED file. Line: 0. - '100000000000000000 is not a valid end position'"
)
data = """chr1\t2147483647\t2147483648"""
(s, a) = epidb.add_annotation("test annotation", "hg19", "testing",
data, "CHROMOSOME,START,END",
{"HI": "HOW ARE YOU?"}, self.admin_key)
self.assertFailure(s, a)
self.assertEquals(
a,
"Invalid region: 2147483647 - 2147483648. It is beyond the length of the chromosome chr1 ."
)
def test_category_fail(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
sample_id = self.sample_ids[0]
format = ",".join([
"CHROMOSOME",
"START",
"END",
"NAME",
"SCORE",
"STRAND_X",
"SIGNAL_VALUE",
"P_VALUE",
"Q_VALUE",
"PEAK"
])
res = epidb.create_column_type_category("STRAND_X", "strand of the region", ["X", "-"], self.admin_key)
self.assertSuccess(res)
regions_data = helpers.load_bed("hg19_chr1_1")
res, msg = epidb.add_experiment("test_exp_fail2", "hg19", "Methylation", sample_id, "tech1", "ENCODE",
"desc1", regions_data, format, None, self.admin_key)
self.assertFailure(res, msg)
self.assertTrue("STRAND_X" in msg)
def test_wrong_password(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
s, key = epidb.user_auth(settings.EPIDB_TEST_ADMIN[1],
"wrong_password")
self.assertFailure(s, key)
def test_calculated_get_region(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
sample_id = self.sample_ids[0]
# adding two experiments with the same data should work
res = epidb.add_experiment(
"test_exp1", "hg19", "Methylation", sample_id, "tech1", "ENCODE",
"desc1", "", "wig",
{"__local_file__": "../tests/data/wig/scores1.wig"},
self.admin_key)
self.assertSuccess(res)
res, qid1 = epidb.select_regions("test_exp1", "hg19", None, None, None,
None, None, None, None,
self.admin_key)
self.assertSuccess(res, qid1)
(s, req) = epidb.get_regions(
qid1,
"CHROMOSOME,START,END,VALUE,@CALCULATED(return math.log(value_of('VALUE'))),@CALCULATED(em = value_of('@EPIGENETIC_MARK') if em == 'Methylation' then return 'it is methylation!' else return 'it is not methylation' end)",
self.admin_key)
self.assertSuccess(s, req)
regions_1 = self.get_regions_request(req)
r0 = regions_1.split('\n')[0].split('\t')[3]
r1 = regions_1.split('\n')[0].split('\t')[4]
r2 = regions_1.split('\n')[0].split('\t')[5]
self.assertEqual(r0, '8.1235')
self.assertEqual(r1, '2.094756')
self.assertEqual(r2, 'it is methylation!')
def test_error_maximum_number_of_instructions(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
sample_id = self.sample_ids[0]
# adding two experiments with the same data should work
res = epidb.add_experiment(
"test_exp1", "hg19", "Methylation", sample_id, "tech1", "ENCODE",
"desc1", "", "wig",
{"__local_file__": "../tests/data/wig/scores1.wig"},
self.admin_key)
self.assertSuccess(res)
res, qid1 = epidb.select_regions("test_exp1", "hg19", None, None, None,
None, None, None, None,
self.admin_key)
self.assertSuccess(res, qid1)
# missing math. before log
(s, req) = epidb.get_regions(
qid1,
"CHROMOSOME,START,END,VALUE,@CALCULATED(while 1 do math.log(value_of('VALUE')) end return 'never')",
self.admin_key)
self.assertSuccess(s, req)
msg = self.get_regions_request_error(req)
self.assertEqual(
msg, 'The maximum number of instructions has been reached')
def test_annotation_full_cpg_islands(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
cpg_island = ",".join([
"CHROMOSOME", "START", "END", "NAME", "LENGTH", "NUM_CPG",
"NUM_GC", "PER_CPG", "PER_CG", "OBS_EXP"
])
with open("data/cpgIslandExtFull.txt", 'r') as f:
file_data = f.read()
regions_count = len(file_data.split("\n"))
(res, a_1) = epidb.add_annotation("Cpg Islands", "hg19",
"Complete CpG islands",
file_data, cpg_island, None,
self.admin_key)
self.assertSuccess(res, a_1)
res, qid_1 = epidb.select_annotations("Cpg Islands", "hg19", None,
None, None, self.admin_key)
self.assertSuccess(res, qid_1)
(s, req) = epidb.count_regions(qid_1, self.admin_key)
count = self.count_request(req)
self.assertEqual(regions_count, count)
def test_wrong_column_creation(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
sample_id = self.sample_ids[0]
# adding two experiments with the same data should work
res = epidb.add_experiment(
"test_exp1", "hg19", "Methylation", sample_id, "tech1", "ENCODE",
"desc1", "", "wig",
{"__local_file__": "../tests/data/wig/scores1.wig"},
self.admin_key)
self.assertSuccess(res)
res, qid1 = epidb.select_regions("test_exp1", "hg19", None, None, None,
None, None, None, None,
self.admin_key)
self.assertSuccess(res, qid1)
# It is missing a ' before the EM
res = epidb.create_column_type_calculated(
"calculated", "description",
"return EM and Name: - '.. value_of('@EPIGENETIC_MARK') .. ' - ' .. value_of('@NAME')",
self.admin_key)
self.assertFailure(res)
self.assertEqual(
res[1],
'[string "function row_value()..."]:2: \'<name>\' expected near \'-\''
)
def test_bed_graph_files(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
sample_id = self.sample_ids[0]
files = ["reference_example", "test1", "bigwig"]
for filename in files:
wig_data = helpers.load_bedgraph(filename)
res = epidb.add_experiment(filename, "hg19", "Methylation",
sample_id, "tech1", "ENCODE", "desc1",
wig_data, "bedgraph", None,
self.admin_key)
self.assertSuccess(res)
(s, q) = epidb.select_regions(files, "hg19", None, None, None, None,
None, None, None, self.admin_key)
(s, req) = epidb.count_regions(q, self.admin_key)
self.assertSuccess(s, req)
count = self.count_request(req)
# 3997106 // grep -v # *.bg | grep -v browser | grep -v track | wc -l
self.assertEqual(3997106, count)
def test_uninitialized_system(self):
epidb = DeepBlueClient(address="localhost", port=31415)
# uninitialized system should fail
res, msg = epidb.add_user("user1", "*****@*****.**", "test",
"somekey")
self.assertFailure(res, msg)
def test_genome_empty(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init(epidb)
hg18_genome_info = None
with open("data/genomes/hg18", 'r') as f:
hg18_genome_info = f.read().replace(",", "")
mm9_genome_info = None
with open("data/genomes/mm9", 'r') as f:
mm9_genome_info = f.read().replace(",", "")
res = epidb.add_genome("hg18", "Human genome 18", hg18_genome_info,
self.admin_key)
self.assertSuccess(res)
res = epidb.add_genome("mm9", "Mouse genome 9", mm9_genome_info,
self.admin_key)
self.assertSuccess(res)
(res, genomes) = epidb.list_genomes(self.admin_key)
self.assertEqual(len(genomes), 2)
genome_names = [x[1] for x in genomes]
self.assertTrue("hg18" in genome_names)
self.assertTrue("mm9" in genome_names)
def test_multiple_genomes_2(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base()
sample_id = self.sample_ids[0]
format = "CHROMOSOME,START,END"
self.insert_experiment(epidb, "hg18_chr1_1", sample_id)
self.insert_experiment(epidb, "hg19_chr1_1", sample_id)
self.insert_experiment(epidb, "hg19_chr1_2", sample_id)
# retrieve every paired combination
combinations = [(["hg18_chr1_1", "hg19_chr1_2"], "multiple_genomes"),
(["hg18_chr1_1", "hg19_chr1_1"], "multiple_genomes_2"),
(["hg19_chr1_1",
"hg19_chr1_2"], "multiple_experiments")]
for (experiments, result_regions) in combinations:
res, qid = epidb.select_regions(experiments, ["hg18", "hg19"],
None, None, None, None, None, None,
None, self.admin_key)
self.assertSuccess(res, qid)
res, req = epidb.get_regions(qid, format, self.admin_key)
self.assertSuccess(res, req)
regions = self.get_regions_request(req)
regions_expected = helpers.get_result(result_regions)
self.assertEqual(regions, regions_expected)
def test_multiple_genomes(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base()
sample_id = self.sample_ids[0]
format = "CHROMOSOME,START,END"
self.insert_experiment(epidb, "hg18_chr1_1", sample_id)
self.insert_experiment(epidb, "hg19_chr1_2", sample_id)
multiple_genomes_regions = helpers.get_result("multiple_genomes")
# there was a bug in internal merge procedure triggered by the order of genomes
for genomes in [["hg19", "hg18"], ["hg18", "hg19"]]:
res, qid = epidb.select_regions(["hg18_chr1_1", "hg19_chr1_2"],
genomes, None, None, None, None,
None, None, None, self.admin_key)
self.assertSuccess(res, qid)
res, req = epidb.get_regions(qid, format, self.admin_key)
self.assertSuccess(res, req)
regions = self.get_regions_request(req)
self.assertEqual(regions, multiple_genomes_regions)
def test_malformed_format(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base()
self.insert_experiment(epidb, "hg19_chr1_1")
# test various bad format strings that should fail
bad_formats = [
",",
"chr,start,,",
"chr,start:0:,end",
"chr,start:0:foo,end",
"CHROMOSOME,START,,",
"CHROMOSOME,START:0:,end",
"CHROMOSOME,START:0:foo,end",
]
res, qid = epidb.select_regions("hg19_chr1_1", "hg19", None, None,
None, None, None, None, None,
self.admin_key)
self.assertSuccess(res, qid)
for fmt in bad_formats:
res, req = epidb.get_regions(qid, fmt, self.admin_key)
self.assertFailure(res, req)
def test_double_experiment_same_user_fail(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
uid, user_key = self.insert_user(epidb, "test_user")
s, tmp_user = epidb.modify_user_admin(uid, "permission_level",
"INCLUDE_EXPERIMENTS",
self.admin_key)
self.assertSuccess(s)
sample_id = self.sample_ids[0]
regions_data = helpers.load_bed("hg19_chr1_1")
format = data_info.EXPERIMENTS["hg19_chr1_1"]["format"]
# adding the same experiments with different users should work
exp = ("test_exp1", "hg19", "Methylation", sample_id, "tech1",
"ENCODE", "desc1", regions_data, format, None)
res = epidb.add_experiment(*(exp + (self.admin_key, )))
self.assertSuccess(res)
res = epidb.add_experiment(*(exp + (user_key, )))
self.assertFailure(res)
self.assertEqual(
res[1],
"102001:The experiment name 'test_exp1' is already being used.")
def test_empty_info(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init(epidb)
(s, rs) = epidb.info("", self.admin_key)
self.assertEquals(rs, [])
(s, rs) = epidb.info([], self.admin_key)
self.assertEquals(rs, [])
def init(self, epidb=None):
if not epidb:
epidb = DeepBlueClient(address="localhost", port=31415)
res, key = epidb.init_system(*settings.EPIDB_TEST_ADMIN)
self.assertSuccess(res, key)
self.admin_key = key
def test_duplicate_column_type(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init(epidb)
res = epidb.create_column_type_simple("name2", "description", "string", self.admin_key)
self.assertSuccess(res)
res = epidb.create_column_type_simple("name2", "description", "integer", self.admin_key)
self.assertFailure(res)
def test_echo(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init(epidb)
self.assertEquals(epidb.echo(self.admin_key),
['okay', 'DeepBlue (1.18.11) says hi to test_admin'])
self.assertEquals(epidb.echo("invalid"),
['okay', 'DeepBlue (1.18.11) says hi to a Stranger'])
self.assertEquals(epidb.echo("anonymous_key"),
['okay', 'DeepBlue (1.18.11) says hi to anonymous'])
def test_column_complex_types(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init(epidb)
res = epidb.create_column_type_range("score", "description", 0.0, 1.0, self.admin_key)
self.assertSuccess(res)
strand = ["+", "-"]
res = epidb.create_column_type_category("STRAND", "description", strand, self.admin_key)
self.assertSuccess(res)
def test_nonexisting_genome(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
sequence = "BLABLABLA"
sequence = sequence + "\n" + sequence
res = epidb.upload_chromosome("hg_nothing", "chr19", sequence,
self.admin_key)
self.assertFailure(res)
def test_list_annotations2(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init(epidb)
res = epidb.add_genome("GRCh38", "GRCh38",
"chr1 1000000\nchr2 2000000", self.admin_key)
self.assertSuccess(res)
status, anns = epidb.list_annotations("GRCh38", self.admin_key)
self.assertEqual(anns, [['a1', 'Chromosomes size for GRCh38']])
def test_invalid_annotation(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base(epidb)
res, qid = epidb.select_annotations("Cpg Islands", "hg19", None, None,
None, self.admin_key)
self.assertFailure(res, qid)
self.assertEqual(
qid,
"102000:Unable to find the annotation 'Cpg Islands' in the genome hg19."
)
def test_basic_column_types(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init(epidb)
res = epidb.create_column_type_simple("string_column", "description", "string", self.admin_key)
self.assertSuccess(res)
res = epidb.create_column_type_simple("integer_column", "description", "integer", self.admin_key)
self.assertSuccess(res)
res = epidb.create_column_type_simple("double_column", "description", "double", self.admin_key)
self.assertSuccess(res)
def test_init_system(self):
epidb = DeepBlueClient(address="localhost", port=31415)
res, admin_key = epidb.init_system(*EPIDB_TEST_ADMIN)
self.assertSuccess(res, admin_key)
# check if system is able to receive commands
res, key = epidb.add_user("user1", "*****@*****.**", "test",
admin_key)
self.assertSuccess(res, key)
def test_full_genome(self):
epidb = DeepBlueClient(address="localhost", port=31415)
self.init_base()
res, qid = epidb.tiling_regions(1000000, "hg19", None, self.admin_key)
self.assertSuccess(res, qid)
res, req = epidb.get_regions(qid, "CHROMOSOME,START,END", self.admin_key)
self.assertSuccess(res, req)
regions = self.get_regions_request(req)
def test_types(self):
epidb = DeepBlueClient(address="localhost", port=31415)
in_vals = ("foo", 1, 123.2234, False, [1, 2, "test"], {
"foo": "bar",
"baz": 1
})
res, out_vals = epidb.test_types(*in_vals)
self.assertSuccess(res, out_vals)
self.assertEqual(list(in_vals), out_vals)
