def test_commands(self): epidb = DeepBlueClient(address="localhost", port=31415) # explicitly no initialization. The command should always work. res, cmds = epidb.commands() self.assertSuccess(res, cmds) self.assertEqual(len(cmds), 96) self.assertEqual(cmds["add_experiment"]["parameters"][0], ['name', 'string', False, 'experiment name']) self.assertEqual(cmds["add_experiment"]["parameters"][1], ['genome', 'string', False, 'the target genome']) self.assertEqual(cmds["add_experiment"]["parameters"][2], [ 'epigenetic_mark', 'string', False, 'epigenetic mark of the experiment' ]) self.assertEqual(cmds["add_experiment"]["parameters"][3], ['sample', 'string', False, 'id of the used sample']) self.assertEqual(cmds["add_experiment"]["parameters"][4], [ 'technique', 'string', False, 'technique used by this experiment' ]) self.assertEqual(cmds["add_experiment"]["parameters"][5], ['project', 'string', False, 'the project name']) self.assertEqual( cmds["add_experiment"]["parameters"][6], ['description', 'string', False, 'description of the experiment']) self.assertEqual(cmds["add_experiment"]["parameters"][7], ['data', 'string', False, 'the BED formated data']) self.assertEqual( cmds["add_experiment"]["parameters"][8], ['format', 'string', False, 'format of the provided data']) self.assertEqual( cmds["add_experiment"]["parameters"][9], ['extra_metadata', 'struct', False, 'additional metadata']) self.assertEqual(cmds["add_experiment"]["parameters"][10], ['user_key', 'string', False, 'users token key'])
def init_base(self, epidb=None): if not epidb: epidb = DeepBlueClient(address="localhost", port=31415) self.init(epidb) for genome in data.GENOMES: self.insert_genome(epidb, genome) for bsource in data.BIOSOURCES: self.insert_biosource(epidb, bsource) for tech in data.TECHNIQUES: self.insert_technique(epidb, tech) for epimark in data.EPIGENETIC_MARKS: self.insert_epigenetic_mark(epidb, epimark) for project in data.PROJECTS: self.insert_project(epidb, project) for sample in data.SAMPLES: self.insert_samples(epidb, sample) for column in data.COLUMNS: self.insert_column(epidb, column) res, cid = epidb.create_column_type_category("STRAND", "Region strand: +, -, .", ["+", "-", "."], self.admin_key) self.assertSuccess(res, cid)
def test_set_project_public_by_user_no_permission(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) (s, user) = epidb.add_user("NAME", "EMAIL", "INSTITUTE", self.admin_key) (s, user_ass) = epidb.add_user("NAME ASS", "EMAIL ASS", "INSTITUTE", self.admin_key) s, tmp_user = epidb.modify_user_admin(user[0], "permission_level", "INCLUDE_EXPERIMENTS", self.admin_key) s, tmp_user = epidb.modify_user_admin(user_ass[0], "permission_level", "INCLUDE_EXPERIMENTS", self.admin_key) s, project = epidb.add_project("USER's PROJECT", "COOL", user[1]) (s, (user_add, project_add)) = epidb.add_user_to_project(user_ass[0], project, True, user[1]) self.assertSuccess(s, (user, project)) self.assertEqual(user_add, user_ass[0]) self.assertEqual(project, project_add) (s, user_two) = epidb.add_user("ANOTHER NAME", "ANOTHER EMAIL", "INSTITUTE", self.admin_key) (s, status) = epidb.add_user_to_project(user_two[0], project, True, user_ass[1]) self.assertEqual( status, "107100:You are not the project 'p3' owner and neither an administrator." )
def test_extend(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) (s, query_id) = epidb.tiling_regions(100000000, "hg18", "chr1", self.admin_key) (s, r_id) = epidb.get_regions(query_id, "CHROMOSOME,START,END", self.admin_key) regions = self.get_regions_request(r_id) self.assertEqual(regions, "chr1\t0\t100000000\nchr1\t100000000\t200000000") (s, f_id) = epidb.extend(query_id, 10000000, "FORWARD", False, self.admin_key) (s, r_id) = epidb.get_regions(f_id, "CHROMOSOME,START,END", self.admin_key) regions = self.get_regions_request(r_id) self.assertEqual(regions, "chr1\t0\t110000000\nchr1\t100000000\t210000000") (s, f_id) = epidb.extend(query_id, 10000000, "BACKWARD", False, self.admin_key) (s, r_id) = epidb.get_regions(f_id, "CHROMOSOME,START,END", self.admin_key) regions = self.get_regions_request(r_id) self.assertEqual(regions, "chr1\t0\t100000000\nchr1\t90000000\t200000000") (s, f_id) = epidb.extend(query_id, 10000000, "BOTH", False, self.admin_key) (s, r_id) = epidb.get_regions(f_id, "CHROMOSOME,START,END", self.admin_key) regions = self.get_regions_request(r_id) self.assertEqual(regions, "chr1\t0\t110000000\nchr1\t90000000\t210000000")
def test_include_invalid_regions(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) data = """chr1\t1\t100000000000000000 chr10\t666\t66610000 chrY\t12345\t1234567""" (s, a) = epidb.add_annotation("test annotation", "hg19", "testing", data, "CHROMOSOME,START,END", {"HI": "HOW ARE YOU?"}, self.admin_key) self.assertFailure(s, a) self.assertEquals( a, "Error while reading the BED file. Line: 0. - '100000000000000000 is not a valid end position'" ) data = """chr1\t2147483647\t2147483648""" (s, a) = epidb.add_annotation("test annotation", "hg19", "testing", data, "CHROMOSOME,START,END", {"HI": "HOW ARE YOU?"}, self.admin_key) self.assertFailure(s, a) self.assertEquals( a, "Invalid region: 2147483647 - 2147483648. It is beyond the length of the chromosome chr1 ." )
def test_category_fail(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) sample_id = self.sample_ids[0] format = ",".join([ "CHROMOSOME", "START", "END", "NAME", "SCORE", "STRAND_X", "SIGNAL_VALUE", "P_VALUE", "Q_VALUE", "PEAK" ]) res = epidb.create_column_type_category("STRAND_X", "strand of the region", ["X", "-"], self.admin_key) self.assertSuccess(res) regions_data = helpers.load_bed("hg19_chr1_1") res, msg = epidb.add_experiment("test_exp_fail2", "hg19", "Methylation", sample_id, "tech1", "ENCODE", "desc1", regions_data, format, None, self.admin_key) self.assertFailure(res, msg) self.assertTrue("STRAND_X" in msg)
def test_wrong_password(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) s, key = epidb.user_auth(settings.EPIDB_TEST_ADMIN[1], "wrong_password") self.assertFailure(s, key)
def test_calculated_get_region(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) sample_id = self.sample_ids[0] # adding two experiments with the same data should work res = epidb.add_experiment( "test_exp1", "hg19", "Methylation", sample_id, "tech1", "ENCODE", "desc1", "", "wig", {"__local_file__": "../tests/data/wig/scores1.wig"}, self.admin_key) self.assertSuccess(res) res, qid1 = epidb.select_regions("test_exp1", "hg19", None, None, None, None, None, None, None, self.admin_key) self.assertSuccess(res, qid1) (s, req) = epidb.get_regions( qid1, "CHROMOSOME,START,END,VALUE,@CALCULATED(return math.log(value_of('VALUE'))),@CALCULATED(em = value_of('@EPIGENETIC_MARK') if em == 'Methylation' then return 'it is methylation!' else return 'it is not methylation' end)", self.admin_key) self.assertSuccess(s, req) regions_1 = self.get_regions_request(req) r0 = regions_1.split('\n')[0].split('\t')[3] r1 = regions_1.split('\n')[0].split('\t')[4] r2 = regions_1.split('\n')[0].split('\t')[5] self.assertEqual(r0, '8.1235') self.assertEqual(r1, '2.094756') self.assertEqual(r2, 'it is methylation!')
def test_error_maximum_number_of_instructions(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) sample_id = self.sample_ids[0] # adding two experiments with the same data should work res = epidb.add_experiment( "test_exp1", "hg19", "Methylation", sample_id, "tech1", "ENCODE", "desc1", "", "wig", {"__local_file__": "../tests/data/wig/scores1.wig"}, self.admin_key) self.assertSuccess(res) res, qid1 = epidb.select_regions("test_exp1", "hg19", None, None, None, None, None, None, None, self.admin_key) self.assertSuccess(res, qid1) # missing math. before log (s, req) = epidb.get_regions( qid1, "CHROMOSOME,START,END,VALUE,@CALCULATED(while 1 do math.log(value_of('VALUE')) end return 'never')", self.admin_key) self.assertSuccess(s, req) msg = self.get_regions_request_error(req) self.assertEqual( msg, 'The maximum number of instructions has been reached')
def test_annotation_full_cpg_islands(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) cpg_island = ",".join([ "CHROMOSOME", "START", "END", "NAME", "LENGTH", "NUM_CPG", "NUM_GC", "PER_CPG", "PER_CG", "OBS_EXP" ]) with open("data/cpgIslandExtFull.txt", 'r') as f: file_data = f.read() regions_count = len(file_data.split("\n")) (res, a_1) = epidb.add_annotation("Cpg Islands", "hg19", "Complete CpG islands", file_data, cpg_island, None, self.admin_key) self.assertSuccess(res, a_1) res, qid_1 = epidb.select_annotations("Cpg Islands", "hg19", None, None, None, self.admin_key) self.assertSuccess(res, qid_1) (s, req) = epidb.count_regions(qid_1, self.admin_key) count = self.count_request(req) self.assertEqual(regions_count, count)
def test_wrong_column_creation(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) sample_id = self.sample_ids[0] # adding two experiments with the same data should work res = epidb.add_experiment( "test_exp1", "hg19", "Methylation", sample_id, "tech1", "ENCODE", "desc1", "", "wig", {"__local_file__": "../tests/data/wig/scores1.wig"}, self.admin_key) self.assertSuccess(res) res, qid1 = epidb.select_regions("test_exp1", "hg19", None, None, None, None, None, None, None, self.admin_key) self.assertSuccess(res, qid1) # It is missing a ' before the EM res = epidb.create_column_type_calculated( "calculated", "description", "return EM and Name: - '.. value_of('@EPIGENETIC_MARK') .. ' - ' .. value_of('@NAME')", self.admin_key) self.assertFailure(res) self.assertEqual( res[1], '[string "function row_value()..."]:2: \'<name>\' expected near \'-\'' )
def test_bed_graph_files(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) sample_id = self.sample_ids[0] files = ["reference_example", "test1", "bigwig"] for filename in files: wig_data = helpers.load_bedgraph(filename) res = epidb.add_experiment(filename, "hg19", "Methylation", sample_id, "tech1", "ENCODE", "desc1", wig_data, "bedgraph", None, self.admin_key) self.assertSuccess(res) (s, q) = epidb.select_regions(files, "hg19", None, None, None, None, None, None, None, self.admin_key) (s, req) = epidb.count_regions(q, self.admin_key) self.assertSuccess(s, req) count = self.count_request(req) # 3997106 // grep -v # *.bg | grep -v browser | grep -v track | wc -l self.assertEqual(3997106, count)
def test_uninitialized_system(self): epidb = DeepBlueClient(address="localhost", port=31415) # uninitialized system should fail res, msg = epidb.add_user("user1", "*****@*****.**", "test", "somekey") self.assertFailure(res, msg)
def test_genome_empty(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init(epidb) hg18_genome_info = None with open("data/genomes/hg18", 'r') as f: hg18_genome_info = f.read().replace(",", "") mm9_genome_info = None with open("data/genomes/mm9", 'r') as f: mm9_genome_info = f.read().replace(",", "") res = epidb.add_genome("hg18", "Human genome 18", hg18_genome_info, self.admin_key) self.assertSuccess(res) res = epidb.add_genome("mm9", "Mouse genome 9", mm9_genome_info, self.admin_key) self.assertSuccess(res) (res, genomes) = epidb.list_genomes(self.admin_key) self.assertEqual(len(genomes), 2) genome_names = [x[1] for x in genomes] self.assertTrue("hg18" in genome_names) self.assertTrue("mm9" in genome_names)
def test_multiple_genomes_2(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base() sample_id = self.sample_ids[0] format = "CHROMOSOME,START,END" self.insert_experiment(epidb, "hg18_chr1_1", sample_id) self.insert_experiment(epidb, "hg19_chr1_1", sample_id) self.insert_experiment(epidb, "hg19_chr1_2", sample_id) # retrieve every paired combination combinations = [(["hg18_chr1_1", "hg19_chr1_2"], "multiple_genomes"), (["hg18_chr1_1", "hg19_chr1_1"], "multiple_genomes_2"), (["hg19_chr1_1", "hg19_chr1_2"], "multiple_experiments")] for (experiments, result_regions) in combinations: res, qid = epidb.select_regions(experiments, ["hg18", "hg19"], None, None, None, None, None, None, None, self.admin_key) self.assertSuccess(res, qid) res, req = epidb.get_regions(qid, format, self.admin_key) self.assertSuccess(res, req) regions = self.get_regions_request(req) regions_expected = helpers.get_result(result_regions) self.assertEqual(regions, regions_expected)
def test_multiple_genomes(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base() sample_id = self.sample_ids[0] format = "CHROMOSOME,START,END" self.insert_experiment(epidb, "hg18_chr1_1", sample_id) self.insert_experiment(epidb, "hg19_chr1_2", sample_id) multiple_genomes_regions = helpers.get_result("multiple_genomes") # there was a bug in internal merge procedure triggered by the order of genomes for genomes in [["hg19", "hg18"], ["hg18", "hg19"]]: res, qid = epidb.select_regions(["hg18_chr1_1", "hg19_chr1_2"], genomes, None, None, None, None, None, None, None, self.admin_key) self.assertSuccess(res, qid) res, req = epidb.get_regions(qid, format, self.admin_key) self.assertSuccess(res, req) regions = self.get_regions_request(req) self.assertEqual(regions, multiple_genomes_regions)
def test_malformed_format(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base() self.insert_experiment(epidb, "hg19_chr1_1") # test various bad format strings that should fail bad_formats = [ ",", "chr,start,,", "chr,start:0:,end", "chr,start:0:foo,end", "CHROMOSOME,START,,", "CHROMOSOME,START:0:,end", "CHROMOSOME,START:0:foo,end", ] res, qid = epidb.select_regions("hg19_chr1_1", "hg19", None, None, None, None, None, None, None, self.admin_key) self.assertSuccess(res, qid) for fmt in bad_formats: res, req = epidb.get_regions(qid, fmt, self.admin_key) self.assertFailure(res, req)
def test_double_experiment_same_user_fail(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) uid, user_key = self.insert_user(epidb, "test_user") s, tmp_user = epidb.modify_user_admin(uid, "permission_level", "INCLUDE_EXPERIMENTS", self.admin_key) self.assertSuccess(s) sample_id = self.sample_ids[0] regions_data = helpers.load_bed("hg19_chr1_1") format = data_info.EXPERIMENTS["hg19_chr1_1"]["format"] # adding the same experiments with different users should work exp = ("test_exp1", "hg19", "Methylation", sample_id, "tech1", "ENCODE", "desc1", regions_data, format, None) res = epidb.add_experiment(*(exp + (self.admin_key, ))) self.assertSuccess(res) res = epidb.add_experiment(*(exp + (user_key, ))) self.assertFailure(res) self.assertEqual( res[1], "102001:The experiment name 'test_exp1' is already being used.")
def test_empty_info(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init(epidb) (s, rs) = epidb.info("", self.admin_key) self.assertEquals(rs, []) (s, rs) = epidb.info([], self.admin_key) self.assertEquals(rs, [])
def init(self, epidb=None): if not epidb: epidb = DeepBlueClient(address="localhost", port=31415) res, key = epidb.init_system(*settings.EPIDB_TEST_ADMIN) self.assertSuccess(res, key) self.admin_key = key
def test_duplicate_column_type(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init(epidb) res = epidb.create_column_type_simple("name2", "description", "string", self.admin_key) self.assertSuccess(res) res = epidb.create_column_type_simple("name2", "description", "integer", self.admin_key) self.assertFailure(res)
def test_echo(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init(epidb) self.assertEquals(epidb.echo(self.admin_key), ['okay', 'DeepBlue (1.18.11) says hi to test_admin']) self.assertEquals(epidb.echo("invalid"), ['okay', 'DeepBlue (1.18.11) says hi to a Stranger']) self.assertEquals(epidb.echo("anonymous_key"), ['okay', 'DeepBlue (1.18.11) says hi to anonymous'])
def test_column_complex_types(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init(epidb) res = epidb.create_column_type_range("score", "description", 0.0, 1.0, self.admin_key) self.assertSuccess(res) strand = ["+", "-"] res = epidb.create_column_type_category("STRAND", "description", strand, self.admin_key) self.assertSuccess(res)
def test_nonexisting_genome(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) sequence = "BLABLABLA" sequence = sequence + "\n" + sequence res = epidb.upload_chromosome("hg_nothing", "chr19", sequence, self.admin_key) self.assertFailure(res)
def test_list_annotations2(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init(epidb) res = epidb.add_genome("GRCh38", "GRCh38", "chr1 1000000\nchr2 2000000", self.admin_key) self.assertSuccess(res) status, anns = epidb.list_annotations("GRCh38", self.admin_key) self.assertEqual(anns, [['a1', 'Chromosomes size for GRCh38']])
def test_invalid_annotation(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base(epidb) res, qid = epidb.select_annotations("Cpg Islands", "hg19", None, None, None, self.admin_key) self.assertFailure(res, qid) self.assertEqual( qid, "102000:Unable to find the annotation 'Cpg Islands' in the genome hg19." )
def test_basic_column_types(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init(epidb) res = epidb.create_column_type_simple("string_column", "description", "string", self.admin_key) self.assertSuccess(res) res = epidb.create_column_type_simple("integer_column", "description", "integer", self.admin_key) self.assertSuccess(res) res = epidb.create_column_type_simple("double_column", "description", "double", self.admin_key) self.assertSuccess(res)
def test_init_system(self): epidb = DeepBlueClient(address="localhost", port=31415) res, admin_key = epidb.init_system(*EPIDB_TEST_ADMIN) self.assertSuccess(res, admin_key) # check if system is able to receive commands res, key = epidb.add_user("user1", "*****@*****.**", "test", admin_key) self.assertSuccess(res, key)
def test_full_genome(self): epidb = DeepBlueClient(address="localhost", port=31415) self.init_base() res, qid = epidb.tiling_regions(1000000, "hg19", None, self.admin_key) self.assertSuccess(res, qid) res, req = epidb.get_regions(qid, "CHROMOSOME,START,END", self.admin_key) self.assertSuccess(res, req) regions = self.get_regions_request(req)
def test_types(self): epidb = DeepBlueClient(address="localhost", port=31415) in_vals = ("foo", 1, 123.2234, False, [1, 2, "test"], { "foo": "bar", "baz": 1 }) res, out_vals = epidb.test_types(*in_vals) self.assertSuccess(res, out_vals) self.assertEqual(list(in_vals), out_vals)