def setUp(self): self.experiment1 = Experiment(orig_study_id=1) self.experiment2 = Experiment(orig_study_id=2) self.experiment3 = Experiment(orig_study_id=3) self.subject1 = Subject(orig_subject_id=1) self.subject2 = Subject(orig_subject_id=2) self.sample1 = Sample(orig_sample_id=1) self.sample2 = Sample(orig_sample_id=2) self.sample3 = Sample(orig_sample_id=3) self.sample4 = Sample(orig_sample_id=4) # TODO: Delete? # Set up relationships self.subject1._samples = {self.sample1, self.sample2} self.subject2._samples = {self.sample3} self.sample1._subject = self.subject1 self.sample2._subject = self.subject1 self.sample3._subject = self.subject2 self.experiment1._samples = {self.sample1, self.sample2} self.experiment1._subjects = {self.subject1} self.experiment2._samples = {self.sample1} self.experiment2._subjects = {self.subject1} self.experiment3._samples = {self.sample1, self.sample3} self.experiment3._subjects = {self.subject1, self.subject2} self.subject1._experiments = { self.experiment1, self.experiment2, self.experiment3 } self.subject2._experiments = {self.experiment3} self.sample1._experiments = { self.experiment1, self.experiment2, self.experiment3 } self.sample2._experiments = {self.experiment1} self.sample3._experiments = {self.experiment3}
def setUp(self): self.experiment1 = Experiment() self.subject1 = Subject(orig_subject_id=1) self.subject2 = Subject(orig_subject_id=2) self.sample1 = Sample(orig_sample_id=1) self.sample2 = Sample(orig_sample_id=2) self.sample3 = Sample(orig_sample_id=3) self.sample4 = Sample(orig_sample_id=4) # Set up relationships self.subject1._samples = {self.sample1, self.sample2} self.subject2._samples = {self.sample3} self.sample1._subject = self.subject1 self.sample2._subject = self.subject1 self.sample3._subject = self.subject2
def fetch_data(device, date, step): ss = Sample.all() ss.filter("device =", device) ss.filter("ts >=", date) ss.filter("ts <", date + timedelta(days=1)) ss.order("ts") x = []; y = [] mn = datetime(date.year, date.month, date.day) # subsample or the request will be too large last = -1 for i, s in enumerate(ss): minute = (s.ts - mn).seconds // 60 if minute > last: # logger.info("ts %s: %s", i, s.ts) x.append(minute) y.append(s.value) last = ((minute // step) + 1) * step - 1 # guard against an empty range if x and x[0] < x[-1]: return x, y else: return [], []
def testHaplotyperPath(self): Grid.useGrid = False expOutFile = "../testFiles/output/testPool/out.testPoolSL2.40ch11_22900-24100.BEAGLE.PL.phased.gz" TestHaplotyper.testPool.vcf = {} TestHaplotyper.sample = Sample.Sample(TestHaplotyper.testPool, "testLib") TestHaplotyper.testPool.addSample(TestHaplotyper.sample) TestHaplotyper.sample.bam = BamFile.BamFile(TestHaplotyper.testPool, TestHaplotyper.sample, TestHaplotyper.inputBam, sortedBam=True, headerLine=True, duplicates=False, mdTag=True, index=True) TestHaplotyper.haplotyper.callHaplotypes() createdOutFile = TestHaplotyper.testPool.beagleFiles.values( )[0].phasedFile self.assertEqual( os.path.abspath(createdOutFile), os.path.abspath(expOutFile), os.path.abspath(createdOutFile) + " not is " + os.path.abspath(expOutFile)) self.checkNoOfSnps(expOutFile)
def parse(fileName): record = SeqIO.parse(fileName, "genbank") rec = next(record) ##print(rec.annotations['topology']) intronPositions = [] # there is only one record for f in rec.features: if f.type.lower() == 'insert' or 'insert' in [ l.lower() for l in f.qualifiers['label'] ]: ##print("Insert sequence begins at {} and ends at {}".format(f.location.start,f.location.end)) insertPosition = (f.location.start.position, f.location.end.position) if f.type.lower() == 'intron' or 'intron' in [ l.lower() for l in f.qualifiers['label'] ]: ##print("Intron sequence begins at {} and ends at {}".format(f.location.start,f.location.end)) intronPositions.append( (f.location.start.position, f.location.end.position)) sample = Sample(rec.seq, insertPosition, intronPositions, rec.annotations['topology'] == "circular") insertPrimers = findPossiblePrimers(sample) #print(insertPrimers) primers = findCompatiblePlasmidPrimerPair(sample, insertPrimers[0], False) if primers == None: primers = findCompatiblePlasmidPrimerPair(sample, insertPrimers[1], True) #website uses primers object print(primers) return primers
def parse_samples(samples, move): for sample_node in samples: sample = Sample() sample.move = move for child in sample_node.iterchildren(): tag = normalize_tag(child.tag) value = child.text if tag == 'events': sample.events = parse_json(child) elif tag == 'satellites': sample.satellites = parse_json(child) elif tag == 'apps_data': sample.apps_data = parse_json(child) else: set_attr(sample, tag, value) yield sample
def parse_sample(row, attr_map, site_attrs_map=None, time_attr=None): """Parse a row of an indexable collection into a Sample. Parameters ---------- row : indexable collection A row to be parsed into a Sample. attr_map : dict A dictionary mapping indexes for the indexable collection `row` to attribute names of the Sample object. """ sample = Sample() # print(row) for index, attr in attr_map.items(): setattr(sample, attr, row[index]) if site_attrs_map: sample.sampling_site = parse_sample_site(row, site_attrs_map) if time_attr: sample.sampling_time = parse_sample_time(row, time_attr) return sample
def get_sample(samp_id, study_id, session): try: sample, experiment = session.query(Sample, Experiment)\ .filter(Sample.sample_id == samp_id)\ .filter(Experiment.study_id == study_id)\ .one() return sample except NoResultFound: # TODO A sample should always be present in database, before this # method is called, but for testing purposes, we create an empty # Sample object. return Sample()
def insert_pause(samples, insert_pause_idx, move, pause_type): if (insert_pause_idx <= 0): return stop_sample = samples[insert_pause_idx - 1] start_sample = samples[insert_pause_idx] pause_duration = start_sample.time - stop_sample.time pause_distance = distance((radian_to_degree(stop_sample.latitude), radian_to_degree(stop_sample.longitude)), (radian_to_degree(start_sample.latitude), radian_to_degree(start_sample.longitude))).meters # Introduce start of pause sample pause_sample = Sample() pause_sample.move = move pause_sample.utc = stop_sample.utc pause_sample.time = stop_sample.time stop_sample.utc -= timedelta(microseconds=1) # Cut off 1ms from last recorded sample in order to introduce the new pause sample and keep time order stop_sample.time -= timedelta(microseconds=1) pause_sample.events = {"pause": {"state": "True", "type": str(pause_type), "duration": str(pause_duration), "distance": str(pause_distance), }} samples.insert(insert_pause_idx, pause_sample) # Duplicate last element # Introduce end of pause sample pause_sample = Sample() pause_sample.move = move pause_sample.utc = start_sample.utc pause_sample.time = start_sample.time start_sample.utc += timedelta(microseconds=1) # Add 1ms to the first recorded sample in order to introduce the new pause sample and keep time order start_sample.time += timedelta(microseconds=1) pause_sample.events = {"pause": {"state": "False", "duration": "0", "distance": "0", "type": str(pause_type) }} samples.insert(insert_pause_idx + 1, pause_sample)
def setUp(self): for delFile in os.listdir("../testFiles/output/"): file_path = os.path.join("../testFiles/output/", delFile) if os.path.isdir(file_path): shutil.rmtree(file_path) else: os.unlink(file_path) TestMapper.testPool = Pool.Pool("testPool", "../testFiles/output/") Program.config.setPath("refGenome", "../testFiles/input/smallRefGenome.fa") TestMapper.sample = Sample.Sample(TestMapper.testPool, "testLib") TestMapper.testPool.addSample(TestMapper.sample) TestMapper.mapper = Mapper.Mapper()
def update_sample_record(data): """ Update a vardb record Data should be an object of class SampleView """ session = controller.connect_to_database() record = session.query(Sample).filter_by(id=data["id"]).one() record = Sample() record.sample = data["sample"] record.panel = data["panel"] record.sample_taken = data["sample_taken"] record.genotyping = data["genotyping"] record.variant_calling = data["variant_calling"] record.qc_status = data["qc_status"] record.qc_report = data["qc_report"] record.coverage = data["coverage"] session.commit() session.close()
def setUp(self): for delFile in os.listdir("../testFiles/output/"): file_path = os.path.join("../testFiles/output/", delFile) if os.path.isdir(file_path): shutil.rmtree(file_path) else: os.unlink(file_path) TestDecompressor.testPool = Pool.Pool("testPool", "../testFiles/output/") TestDecompressor.sample = Sample.Sample(TestDecompressor.testPool, "testLib") TestDecompressor.testPool.addSample(TestDecompressor.sample) TestDecompressor.sample.setForwardFq(TestDecompressor.gzFile) TestDecompressor.sample.setReversedFq(TestDecompressor.refGzFile) TestDecompressor.decompressor = Decompressor.Decompressor()
def add_sample_record(data, export=False): """ Data should be an object of class SamplView """ sample = Sample() sample.sample = data["sample"] sample.panel = data["panel"] sample.sample_taken = data["sample_taken"] sample.genotyping = data["genotyping"] sample.variant_calling = data["variant_calling"] sample.qc_status = data["qc_status"] sample.qc_report = data["qc_report"] sample.coverage = data["coverage"] if export: session = controller.connect_to_export_file() else: session = controller.connect_to_database() session.add(sample) session.commit() session.close()
def testHaplotyperFullPathGrid(self): expOutFile = "../testFiles/output/testPool/SL2.40ch11_22900-24100_testPool_SL2.40ch11_22900-24100.vcf" gzFile = "../testFiles/input/test.fq.gz" refGzFile = "../testFiles/input/revTest.fq.gz" TestHaplotyper.testPool.vcf = {} TestHaplotyper.sample = Sample.Sample(TestHaplotyper.testPool, "testLib") TestHaplotyper.testPool.addSample(TestHaplotyper.sample) TestHaplotyper.sample.setForwardFq(gzFile) TestHaplotyper.sample.setReversedFq(refGzFile) TestHaplotyper.sample.reversedFq.forward = False Haplotyper.executeBeagleCluster(TestHaplotyper.testPool) # createdOutFile = TestHaplotyper.testPool.vcf[TestHaplotyper.chrIndex].fileName # self.assertEqual(os.path.abspath(createdOutFile),os.path.abspath(expOutFile) , os.path.abspath(createdOutFile) + " not is " + os.path.abspath(expOutFile)) #Check if the file contains exactly one snp self.checkNoOfSnps(expOutFile)
def testSamtoolsMultiple(self): #add an extra sample to the pool TestSnvCaller.sample2 = Sample.Sample(TestSnvCaller.testPool, "testLib2") TestSnvCaller.sample2.bam = BamFile.BamFile(TestSnvCaller.testPool, TestSnvCaller.sample2, TestSnvCaller.inputBam) TestSnvCaller.testPool.addSample(TestSnvCaller.sample2) #Execute and check execution output SamtoolsMpileup.SamtoolsMpileup(TestSnvCaller.testPool).callSnvs() outputFile = TestSnvCaller.testPool.vcf[None].fileName self.assertEqual( os.path.abspath(outputFile), os.path.abspath(TestSnvCaller.expVcfFile), os.path.abspath(outputFile) + " not is " + os.path.abspath(TestSnvCaller.expVcfFile)) #Check if the file contains exactly one snp self.checkNoOfSnps(TestSnvCaller.expVcfFile)
def testHaplotyperPathGrid(self): TestHaplotyper.testPool.vcf = {} TestHaplotyper.sample = Sample.Sample(TestHaplotyper.testPool, "testLib") TestHaplotyper.testPool.addSample(TestHaplotyper.sample) TestHaplotyper.sample.bam = BamFile.BamFile(TestHaplotyper.testPool, TestHaplotyper.sample, TestHaplotyper.inputBam, sortedBam=True, headerLine=True, duplicates=False, mdTag=True, index=True) Haplotyper.executeBeagleCluster(TestHaplotyper.testPool) # self.assertEqual(os.path.abspath(createdOutFile),os.path.abspath(expOutFile) , os.path.abspath(createdOutFile) + " not is " + os.path.abspath(expOutFile)) self.checkNoOfSnps( "../testFiles/output/testPool/SL2.40ch11_22900-24100_testPool_SL2.40ch11_22900-24100.vcf" )
def setUp(self): for handler in logging.getLogger().handlers: handler.close() for delFile in os.listdir("../testFiles/output/"): file_path = os.path.join("../testFiles/output/", delFile) if os.path.isdir(file_path): shutil.rmtree(file_path) else: os.unlink(file_path) TestSnvCaller.testPool = Pool.Pool("testPool", "../testFiles/output/") Program.config.setPath("refGenome", "../testFiles/input/smallRefGenome.fa") TestSnvCaller.sample = Sample.Sample(TestSnvCaller.testPool, "testLib") TestSnvCaller.testPool.addSample(TestSnvCaller.sample) TestSnvCaller.sample.bam = BamFile.BamFile(TestSnvCaller.testPool, TestSnvCaller.sample, TestSnvCaller.inputBam, sortedBam=True, headerLine=True, duplicates=False, mdTag=True, index=True)
class SampleParserTest(unittest.TestCase): sample_test_file = './data/test_data/samp_metadata/sample1.txt' row = OrderedDict([ ('sample_name', '317.F10'), ('age', '22'), ('age_unit', 'years'), ('altitude', '0'), ('anatomical_body_site', 'FMA:Palm'), ('anonymized_name', 'F10'), ('body_habitat', 'UBERON:skin'), ('body_product', 'UBERON:sebum'), ('body_site', 'UBERON:zone of skin of hand'), ('collection_date', '11/12/2006'), ('country', 'GAZ:United States of America'), ('depth', '0'), ('description', 'human skin metagenome'), ('dna_extracted', 'true'), ('dominant_hand', ''), ('elevation', '1591.99'), ('env_biome', 'ENVO:human-associated habitat'), ('env_feature', 'ENVO:human-associated habitat'), ('host_common_name', 'human'), ('host_subject_id', 'F1'), ('host_taxid', '9606'), ('latitude', '40'), ('longitude', '-105'), ('palm_size', ''), ('physical_specimen_remaining', 'false'), ('public', 'true'), ('qiita_study_id', '317'), ('sample_type', 'XXQIITAXX'), ('sex', 'female'), ('time_since_last_wash', '0'), ('title', 'The influence of sex handedness and washing on the diversity of hand surface bacteriaS1_V160' ) ]) dayfirst_dict = {'collection_date': False} # TODO Update details of source (when necessary) source1 = Source(name='qiita', type_='Database (Public)', url='https://qiita.ucsd.edu/study/description/0') experiment1 = Experiment(source=source1, orig_study_id='317') subject1 = Subject( source=source1, orig_study_id='317', orig_subject_id='F1', sex='female', country='United States of America', race=None, csection=None, disease=None, dob=None, ) subject2 = Subject( source=source1, orig_study_id='317', orig_subject_id='F2', sex='female', country='United States of America', race=None, csection=None, disease=None, dob=None, ) sampling_site = SamplingSite( uberon_habitat_term='UBERON:skin', uberon_product_term='UBERON:sebum', uberon_site_term='UBERON:zone of skin of hand', env_biom_term='ENVO:human-associated habitat', env_feature_term='ENVO:human-associated habitat') sampling_time = Time(timestamp=datetime.datetime(2006, 11, 12), uncertainty=None, date=datetime.date(2006, 11, 12), time=None, year=2006, month=11, day=12, hour=None, minute=None, second=None, season='autumn') sample1 = Sample(source=source1, orig_study_id='317', orig_subject_id='F1', orig_sample_id='317.F10', age_units=ureg.years, age=22.0, latitude=40.0, longitude=-105.0, elevation=1591.99, height_units=ureg.metres, height=None, weight_units=ureg.kilograms, weight=None, bmi=None, sample_date=datetime.date(2006, 11, 12), sample_time=None, sampling_site=sampling_site, sampling_time=sampling_time) sample2 = Sample(source=source1, orig_study_id='317', orig_subject_id='F1', orig_sample_id='317.F12', age_units=ureg.years, age=22.0, latitude=40.0, longitude=-105.0, elevation=1591.99, height_units=ureg.metres, height=None, weight_units=ureg.kilograms, weight=None, bmi=None, sample_date=datetime.date(2006, 11, 12), sample_time=None, sampling_site=sampling_site, sampling_time=sampling_time) sample3 = Sample(source=source1, orig_study_id='317', orig_subject_id='F2', orig_sample_id='317.F20', age_units=ureg.years, age=None, latitude=40.0, longitude=-105.0, elevation=1591.99, height_units=ureg.metres, height=None, weight_units=ureg.kilograms, weight=None, bmi=None, sample_date=datetime.date(2006, 11, 12), sample_time=None, sampling_site=sampling_site, sampling_time=sampling_time) # Not necessary to establish these relationships for purpose of # test_parse_objects: sample1._subject = subject1 sample2._subject = subject1 sample3._subject = subject2 subject1._samples = {sample1, sample2} subject2._samples = {sample3} experiment1._subjects = {subject1, subject2} experiment1._samples = {sample1, sample2, sample3} def test_parse_objects(self): experiment_ids = parse_objects(self.sample_test_file) self.assertIn('317', experiment_ids) experiment = experiment_ids['317'] self.assertEqual(self.experiment1, experiment) self.assertIn(self.subject1, experiment.subjects) self.assertIn(self.subject2, experiment.subjects) self.assertIn(self.sample1, experiment.samples) self.assertIn(self.sample2, experiment.samples) self.assertIn(self.sample3, experiment.samples) # TODO: We will have to test without the source keyword at some point. def test_parse_sample(self): self.maxDiff = None blacklist_attrs = [ '_sa_instance_state', 'source', 'counts', '_experiments', '_subject', '_preparations' ] sample = parse_sample(self.row, self.dayfirst_dict, source=self.source1) sample_attrs = set((key, value) for key, value in sample.__dict__.items() if key not in blacklist_attrs) expected_attrs = set((key, value) for key, value in self.sample1.__dict__.items() if key not in blacklist_attrs) self.assertEqual(sample_attrs, expected_attrs) self.assertEqual(sample.source, self.source1) self.assertEqual(sample.counts, self.sample1.counts) # When sample is parsed, it is not yet associated with subject/experiments self.assertEqual(sample._subject, None) self.assertEqual(sample._experiments, set()) self.assertEqual(sample._preparations, set()) def test_parse_subject(self): self.maxDiff = None blacklist_attrs = [ '_sa_instance_state', 'source', 'counts', 'perturbation_facts', '_experiments', '_samples', '_perturbations' ] subject = parse_subject(self.row, source=self.source1) subject_attrs = set((key, value) for key, value in subject.__dict__.items() if key not in blacklist_attrs) expected_attrs = set((key, value) for key, value in self.subject1.__dict__.items() if key not in blacklist_attrs) self.assertEqual(subject_attrs, expected_attrs) self.assertEqual(subject.source, self.source1) self.assertEqual(subject.counts, self.subject1.counts) self.assertEqual(subject.perturbation_facts, self.subject1.perturbation_facts) # When subject is parsed, it is not yet associated with samples/experiments self.assertEqual(subject._experiments, set()) self.assertEqual(subject._samples, set()) self.assertEqual(subject._perturbations, set()) def test_parse_processing(self): self.maxDiff = None processing1 = Processing(parent=None, parameter_values='{}', orig_prep_id='577', orig_proc_id='2593') processing2 = Processing(parent=processing1, parameter_values='{' '"barcode_type":"golay_12",' '"command":"Split libraries (QIIMEq2 1.9.1)",' '"disable_bc_correction":"False",' '"disable_primers":"False",' '"generated on":"2016-01-14 17:01",' '"input_data":"2593",' '"max_ambig":"6",' '"max_barcode_errors":"1.5",' '"max_homopolymer":"6",' '"max_primer_mismatch":"0",' '"max_seq_len":"1000",' '"min_qual_score":"25",' '"min_seq_len":"200",' '"qual_score_window":"0",' '"reverse_primer_mismatches":"0",' '"reverse_primers":"disable",' '"trim_seq_length":"False",' '"truncate_ambi_bases":"False"' '}', orig_prep_id='577', orig_proc_id='310') processing3 = Processing( parent=processing2, parameter_values='{' '"command":"Pick closed-reference OTUs (QIIMEq2 1.9.1)",' '"generated on":"2015-06-30 14:06",' '"input_data":"310",' '"reference-seq":"/databases/gg/13_8/rep_set/97_otus.fasta",' '"reference-tax":"/databases/gg/13_8/taxonomy/97_otu_taxonomy.txt",' '"similarity":"0.97",' '"sortmerna_coverage":"0.97",' '"sortmerna_e_value":"1",' '"sortmerna_max_pos":"10000",' '"threads":"1"' '}', orig_prep_id='577', orig_proc_id='2594') expected_processings = { '2593': processing1, '310': processing2, '2594': processing3 } processings = parse_processings('./data/test_data/proc1.json') # TODO: Implement workflows and parents as mocks? blacklist_attrs = ['_sa_instance_state', 'workflows', 'parent'] for proc_id, processing in processings.items(): self.assertIn(proc_id, expected_processings) processing_attrs = set( (key, value) for key, value in processing.__dict__.items() if key not in blacklist_attrs) expected_attrs = set( (key, value) for key, value in expected_processings[proc_id].__dict__.items() if key not in blacklist_attrs) self.assertEqual(processing_attrs, expected_attrs)
def run_test(input_images, value=None): # Initialize Test Debugger if DEBUG: if os.path.exists(constants.TEST_DEBUG_PATH): shutil.rmtree(constants.TEST_DEBUG_PATH) os.mkdir(constants.TEST_DEBUG_PATH) # If test program was run with no arguments, each image inside # 'input' folder is tested if input_images == []: input_images = os.listdir(INPUT_IMAGES_PATH) # Choose algorithm algorithm = algorithms.FreeComet(False, False) #algorithm = algorithms.OpenComet() comet_statistics_list = [] head_statistics_list = [] for image_name in input_images: print("Input Image: " + image_name + "\n") # [1] Get input image input_image_path = os.path.join(INPUT_IMAGES_PATH, image_name) grayscale_input_image, original_input_image = utils.read_image( input_image_path, True) if DEBUG: path = __create_debug_path(image_name) utils.save_image(original_input_image, path) # [2] Execute algorithm. Returns a list of lists of contours sample = Sample(image_name, original_input_image) comets_contours_list = algorithm.execute(sample, value) # Build Comet objects comet_list = __build_comets(comets_contours_list, sample) # [3] Get algorithm execution output masks heads_output_mask = numpy.zeros(shape=(grayscale_input_image.shape), dtype=numpy.uint8) comets_output_mask = numpy.zeros(shape=(grayscale_input_image.shape), dtype=numpy.uint8) for comet in comet_list: if comet._get_comet_contour() is not None: utils.draw_contours(comets_output_mask, [comet._get_comet_contour()]) else: utils.draw_contours(comets_output_mask, [comet._get_head_contour()]) utils.draw_contours(heads_output_mask, [comet._get_head_contour()]) utils.save_image(comets_output_mask, "1.png") utils.save_image(heads_output_mask, "2.png") # [4] Get expected output masks (input image has to be previously manually segmented) # Expected output heads mask _, expected_heads_image = utils.read_image( os.path.join(OUTPUT_HEADS_PATH, image_name), True) expected_heads_mask = utils.get_red_from_image(expected_heads_image) expected_heads_mask = utils.to_binary_image( utils.to_gray_image(expected_heads_mask), 1) # Expected output comets mask _, expected_comets_image = utils.read_image( os.path.join(OUTPUT_COMETS_PATH, image_name), True) expected_comets_mask = utils.get_red_from_image(expected_comets_image) expected_comets_mask = utils.to_binary_image( utils.to_gray_image(expected_comets_mask), 1) # [5] Input image test results comet_statistics = __intersection_over_union(comets_output_mask, expected_comets_mask, original_input_image, "COMETS TEST", "Comets", image_name) comet_statistics_list.append(comet_statistics) head_statistics = __intersection_over_union(heads_output_mask, expected_heads_mask, original_input_image, "HEADS TEST", "Heads", image_name) head_statistics_list.append(head_statistics) # Final average test results return __test_results(comet_statistics_list, head_statistics_list)
def parse_sample(row): sample = Sample() return sample
def main(): #Path to config file _, config_path = deepcopy(sys.argv) with open(config_path, "r") as f: config = json.load(f) data_dir = config["data_dir"] if not os.path.isdir(data_dir): os.mkdir(data_dir) specs = config["specifications"] with open(os.path.join(data_dir, "config_copy.json"), 'w', newline='') as f: f.write(json.dumps(config)) vol_dat = [] run_num = 0 for run_vals in specs: concentrations = run_vals["concentrations"] sample_volume = run_vals["sample_volume"] sample_diffusive_const = run_vals["sample_diffusive_const"] num_timesteps = run_vals["Number of timesteps"] molecular_radius = run_vals["Molecular Radius"] min_droplet_vol = run_vals["Min Droplet Volume"] num_droplets = [] means = [] for c in concentrations: print("Started run for concentration " + str(c) + "uM", flush=True) sample = Sample(sample_volume, sample_diffusive_const, c, molecular_radius, min_droplet_vol) aggs = sample.simulate(num_timesteps) volumes = [] for agg in aggs: if agg.is_droplet(): volumes.append(agg.volume()) vol_dat += [volumes] if len(volumes) == 0: num_droplets.append(0) means.append(0) else: nobs, minmax, mean, variance, skewness, kurtosis = sp.describe( volumes) num_droplets.append(nobs) means.append(mean) print("Finished run for concentration " + str(c) + "uM", flush=True) run_num += 1 run_dir = os.path.join(data_dir, "run" + str(run_num)) os.mkdir(run_dir) with open(os.path.join(run_dir, "volumes_dat.csv"), 'w', newline='') as csvfile: writer = csv.writer(csvfile, delimiter=',') writer.writerows(vol_dat) plt.rc('font', family='serif') fig = plt.figure(figsize=(10, 10)) ax = fig.add_subplot(1, 1, 1) for item in (ax.get_xticklabels() + ax.get_yticklabels()): item.set_fontsize(20) for item in ([ax.title, ax.xaxis.label, ax.yaxis.label]): item.set_fontsize(30) ax.plot(concentrations, num_droplets, "sb:") ax.set_title("Number of Droplets") ax.set_xlabel("Concentration (uM)") ax.set_ylabel("Number of Droplets") plt.savefig(os.path.join(run_dir, 'num_droplets.png'), bbox_inches='tight') fig = plt.figure(figsize=(10, 10)) ax = fig.add_subplot(1, 1, 1) for item in (ax.get_xticklabels() + ax.get_yticklabels()): item.set_fontsize(20) for item in ([ax.title, ax.xaxis.label, ax.yaxis.label]): item.set_fontsize(30) ax.plot(concentrations, means, "sb:") ax.set_title("Mean Droplet Volume") ax.set_xlabel("Concentration (uM)") ax.set_ylabel("Mean Droplet Volume (um^3)") plt.savefig(os.path.join(run_dir, 'mean_droplet_volume.png'), bbox_inches='tight')
class ModelRemoveTest(unittest.TestCase): def setUp(self): self.experiment1 = Experiment(orig_study_id=1) self.experiment2 = Experiment(orig_study_id=2) self.experiment3 = Experiment(orig_study_id=3) self.subject1 = Subject(orig_subject_id=1) self.subject2 = Subject(orig_subject_id=2) self.sample1 = Sample(orig_sample_id=1) self.sample2 = Sample(orig_sample_id=2) self.sample3 = Sample(orig_sample_id=3) self.sample4 = Sample(orig_sample_id=4) # TODO: Delete? # Set up relationships self.subject1._samples = {self.sample1, self.sample2} self.subject2._samples = {self.sample3} self.sample1._subject = self.subject1 self.sample2._subject = self.subject1 self.sample3._subject = self.subject2 self.experiment1._samples = {self.sample1, self.sample2} self.experiment1._subjects = {self.subject1} self.experiment2._samples = {self.sample1} self.experiment2._subjects = {self.subject1} self.experiment3._samples = {self.sample1, self.sample3} self.experiment3._subjects = {self.subject1, self.subject2} self.subject1._experiments = { self.experiment1, self.experiment2, self.experiment3 } self.subject2._experiments = {self.experiment3} self.sample1._experiments = { self.experiment1, self.experiment2, self.experiment3 } self.sample2._experiments = {self.experiment1} self.sample3._experiments = {self.experiment3} def tearDown(self): del self.experiment1 del self.experiment2 del self.experiment3 del self.subject1 del self.subject2 del self.sample1 del self.sample2 del self.sample3 del self.sample4 def test_remove_subject_from_experiment(self): # Attributes that should change expected_subject1_experiments = {self.experiment2, self.experiment3} expected_experiment1_subjects = set() expected_experiment1_samples = set() expected_sample1_experiments = {self.experiment2, self.experiment3} expected_sample2_experiments = set() # Attributes that should not be affected expected_subject2_experiments = self.subject2.experiments expected_experiment2_subjects = self.experiment2.subjects expected_experiment3_subjects = self.experiment3.subjects expected_experiment2_samples = self.experiment2.samples expected_experiment3_samples = self.experiment3.samples expected_sample3_experiments = self.sample3.experiments # Call the function self.experiment1.remove_subject(self.subject1) # Check self.assertEqual(self.subject1.experiments, expected_subject1_experiments) self.assertEqual(self.experiment1.subjects, expected_experiment1_subjects) self.assertEqual(self.experiment1.samples, expected_experiment1_samples) self.assertEqual(self.sample1.experiments, expected_sample1_experiments) self.assertEqual(self.sample2.experiments, expected_sample2_experiments) self.assertEqual(self.subject2.experiments, expected_subject2_experiments) self.assertEqual(self.experiment2.subjects, expected_experiment2_subjects) self.assertEqual(self.experiment3.subjects, expected_experiment3_subjects) self.assertEqual(self.experiment2.samples, expected_experiment2_samples) self.assertEqual(self.experiment3.samples, expected_experiment3_samples) self.assertEqual(self.sample3.experiments, expected_sample3_experiments) def test_remove_experiment_from_subject(self): # Attributes that should change expected_subject1_experiments = {self.experiment2, self.experiment3} expected_experiment1_subjects = set() expected_experiment1_samples = set() expected_sample1_experiments = {self.experiment2, self.experiment3} expected_sample2_experiments = set() # Attributes that should not be affected expected_subject2_experiments = self.subject2.experiments expected_experiment2_subjects = self.experiment2.subjects expected_experiment3_subjects = self.experiment3.subjects expected_experiment2_samples = self.experiment2.samples expected_experiment3_samples = self.experiment3.samples expected_sample3_experiments = self.sample3.experiments # Call the function self.subject1.remove_experiment(self.experiment1) # Check self.assertEqual(self.subject1.experiments, expected_subject1_experiments) self.assertEqual(self.experiment1.subjects, expected_experiment1_subjects) self.assertEqual(self.experiment1.samples, expected_experiment1_samples) self.assertEqual(self.sample1.experiments, expected_sample1_experiments) self.assertEqual(self.sample2.experiments, expected_sample2_experiments) self.assertEqual(self.subject2.experiments, expected_subject2_experiments) self.assertEqual(self.experiment2.subjects, expected_experiment2_subjects) self.assertEqual(self.experiment3.subjects, expected_experiment3_subjects) self.assertEqual(self.experiment2.samples, expected_experiment2_samples) self.assertEqual(self.experiment3.samples, expected_experiment3_samples) self.assertEqual(self.sample3.experiments, expected_sample3_experiments) def test_remove_sample_from_experiment(self): # Attributes that should change expected_experiment1_samples = {self.sample2} expected_sample1_experiments = {self.experiment2, self.experiment3} # Attributes that should not be affected expected_subject1_experiments = { self.experiment1, self.experiment2, self.experiment3 } expected_subject2_experiments = self.subject2.experiments expected_experiment1_subjects = {self.subject1} expected_experiment2_subjects = self.experiment2.subjects expected_experiment3_subjects = self.experiment3.subjects expected_experiment2_samples = self.experiment2.samples expected_experiment3_samples = self.experiment3.samples expected_sample2_experiments = {self.experiment1} expected_sample3_experiments = self.sample3.experiments # Call the function self.experiment1.remove_sample(self.sample1) # Check self.assertEqual(self.subject1.experiments, expected_subject1_experiments) self.assertEqual(self.experiment1.subjects, expected_experiment1_subjects) self.assertEqual(self.experiment1.samples, expected_experiment1_samples) self.assertEqual(self.sample1.experiments, expected_sample1_experiments) self.assertEqual(self.sample2.experiments, expected_sample2_experiments) self.assertEqual(self.subject2.experiments, expected_subject2_experiments) self.assertEqual(self.experiment2.subjects, expected_experiment2_subjects) self.assertEqual(self.experiment3.subjects, expected_experiment3_subjects) self.assertEqual(self.experiment2.samples, expected_experiment2_samples) self.assertEqual(self.experiment3.samples, expected_experiment3_samples) self.assertEqual(self.sample3.experiments, expected_sample3_experiments) def test_remove_experiment_from_sample(self): # Attributes that should change expected_experiment1_samples = {self.sample2} expected_sample1_experiments = {self.experiment2, self.experiment3} # Attributes that should not be affected expected_subject1_experiments = { self.experiment1, self.experiment2, self.experiment3 } expected_subject2_experiments = self.subject2.experiments expected_experiment1_subjects = {self.subject1} expected_experiment2_subjects = self.experiment2.subjects expected_experiment3_subjects = self.experiment3.subjects expected_experiment2_samples = self.experiment2.samples expected_experiment3_samples = self.experiment3.samples expected_sample2_experiments = {self.experiment1} expected_sample3_experiments = self.sample3.experiments # Call the function self.sample1.remove_experiment(self.experiment1) # Check self.assertEqual(self.subject1.experiments, expected_subject1_experiments) self.assertEqual(self.experiment1.subjects, expected_experiment1_subjects) self.assertEqual(self.experiment1.samples, expected_experiment1_samples) self.assertEqual(self.sample1.experiments, expected_sample1_experiments) self.assertEqual(self.sample2.experiments, expected_sample2_experiments) self.assertEqual(self.subject2.experiments, expected_subject2_experiments) self.assertEqual(self.experiment2.subjects, expected_experiment2_subjects) self.assertEqual(self.experiment3.subjects, expected_experiment3_subjects) self.assertEqual(self.experiment2.samples, expected_experiment2_samples) self.assertEqual(self.experiment3.samples, expected_experiment3_samples) self.assertEqual(self.sample3.experiments, expected_sample3_experiments) def test_remove_sample_from_subject(self): # Attributes that should change expected_subject1_samples = {self.sample2} expected_experiment1_samples = {self.sample2} expected_sample1_subject = None expected_sample1_experiments = set() expected_experiment3_samples = {self.sample3} expected_experiment3_subjects = {self.subject2} expected_experiment2_samples = set() expected_experiment2_subjects = set() # Attributes that should not be affected expected_subject1_experiments = self.subject1.experiments expected_subject2_experiments = self.subject2.experiments expected_experiment1_subjects = self.experiment1.subjects expected_sample2_experiments = self.sample2.experiments expected_sample3_experiments = self.sample3.experiments # Call the function self.subject1.remove_sample(self.sample1) # Check self.assertEqual(self.subject1.samples, expected_subject1_samples) self.assertEqual(self.sample1.subject, expected_sample1_subject) self.assertEqual(self.subject1.experiments, expected_subject1_experiments) self.assertEqual(self.experiment1.subjects, expected_experiment1_subjects) self.assertEqual(self.experiment1.samples, expected_experiment1_samples) self.assertEqual(self.sample1.experiments, expected_sample1_experiments) self.assertEqual(self.sample2.experiments, expected_sample2_experiments) self.assertEqual(self.subject2.experiments, expected_subject2_experiments) self.assertEqual(self.experiment2.subjects, expected_experiment2_subjects) self.assertEqual(self.experiment3.subjects, expected_experiment3_subjects) self.assertEqual(self.experiment2.samples, expected_experiment2_samples) self.assertEqual(self.experiment3.samples, expected_experiment3_samples) self.assertEqual(self.sample3.experiments, expected_sample3_experiments) def test_remove_subject_from_sample(self): # Attributes that should change expected_subject1_samples = {self.sample2} expected_experiment1_samples = {self.sample2} expected_sample1_subject = None expected_sample1_experiments = set() expected_experiment3_samples = {self.sample3} expected_experiment3_subjects = {self.subject2} expected_experiment2_samples = set() expected_experiment2_subjects = set() # Attributes that should not be affected expected_subject1_experiments = self.subject1.experiments expected_subject2_experiments = self.subject2.experiments expected_experiment1_subjects = self.experiment1.subjects expected_sample2_experiments = self.sample2.experiments expected_sample3_experiments = self.sample3.experiments # Call the function del self.sample1.subject # TODO What about self.sample1.subject = None? # Check self.assertEqual(self.subject1.samples, expected_subject1_samples) self.assertEqual(self.sample1.subject, expected_sample1_subject) self.assertEqual(self.subject1.experiments, expected_subject1_experiments) self.assertEqual(self.experiment1.subjects, expected_experiment1_subjects) self.assertEqual(self.experiment1.samples, expected_experiment1_samples) self.assertEqual(self.sample1.experiments, expected_sample1_experiments) self.assertEqual(self.sample2.experiments, expected_sample2_experiments) self.assertEqual(self.subject2.experiments, expected_subject2_experiments) self.assertEqual(self.experiment2.subjects, expected_experiment2_subjects) self.assertEqual(self.experiment3.subjects, expected_experiment3_subjects) self.assertEqual(self.experiment2.samples, expected_experiment2_samples) self.assertEqual(self.experiment3.samples, expected_experiment3_samples) self.assertEqual(self.sample3.experiments, expected_sample3_experiments)
def parse_samples(tree, move, gpx_namespace, import_options): all_samples = [] tracks = tree.iterchildren(tag=gpx_namespace + GPX_TRK) for track in tracks: track_samples = [] track_segments = track.iterchildren(tag=gpx_namespace + GPX_TRKSEG) for track_segment in track_segments: segment_samples = [] track_points = track_segment.iterchildren(tag=gpx_namespace + GPX_TRKPT) for track_point in track_points: sample = Sample() sample.move = move # GPS position / altitude sample.latitude = degree_to_radian(float(track_point.attrib[GPX_TRKPT_ATTRIB_LATITUDE])) sample.longitude = degree_to_radian(float(track_point.attrib[GPX_TRKPT_ATTRIB_LONGITUDE])) sample.sample_type = GPX_SAMPLE_TYPE if hasattr(track_point, GPX_TRKPT_ATTRIB_ELEVATION): sample.gps_altitude = float(track_point.ele) sample.altitude = int(round(sample.gps_altitude)) # Time / UTC sample.utc = dateutil.parser.parse(str(track_point.time)) # Option flags pause_detected = False # Track segment samples if len(segment_samples) > 0: # Accumulate time delta to previous sample to get the total duration time_delta = sample.utc - segment_samples[-1].utc sample.time = segment_samples[-1].time + time_delta # Accumulate distance to previous sample distance_delta = vincenty((radian_to_degree(sample.latitude), radian_to_degree(sample.longitude)), (radian_to_degree(segment_samples[-1].latitude), radian_to_degree(segment_samples[-1].longitude))).meters sample.distance = segment_samples[-1].distance + distance_delta if time_delta > timedelta(0): sample.speed = distance_delta / time_delta.total_seconds() else: sample.speed = 0 # Option: Pause detection based on time delta threshold if GPX_IMPORT_OPTION_PAUSE_DETECTION in import_options and time_delta > import_options[GPX_IMPORT_OPTION_PAUSE_DETECTION]: pause_detected = True sample.distance = segment_samples[-1].distance sample.speed = 0 # Track segment -> Track (multiple track segments contained) elif len(track_samples) > 0: sample.time = track_samples[-1].time + (sample.utc - track_samples[-1].utc) # Time diff to last sample of the previous track segment sample.distance = track_samples[-1].distance sample.speed = 0 # Track -> Full GPX (multiple tracks contained) elif len(all_samples) > 0: sample.time = all_samples[-1].time + (sample.utc - all_samples[-1].utc) # Time diff to last sample of the previous track sample.distance = all_samples[-1].distance sample.speed = 0 # First sample else: sample.time = timedelta(0) sample.distance = 0 sample.speed = 0 parse_sample_extensions(sample, track_point) segment_samples.append(sample) # Finally insert a found pause based on time delta threshold if pause_detected: insert_pause(segment_samples, len(segment_samples) - 1, move, pause_type=GPX_IMPORT_PAUSE_TYPE_PAUSE_DETECTION) # end for track_points # Insert an pause event between every track segment insert_pause_idx = len(track_samples) track_samples.extend(segment_samples) insert_pause(track_samples, insert_pause_idx, move, pause_type=GPX_TRKSEG) # end for track_segments # Insert an pause event between every track insert_pause_idx = len(all_samples) all_samples.extend(track_samples) insert_pause(all_samples, insert_pause_idx, move, pause_type=GPX_TRK) # end for tracks return all_samples
def insert_pause(samples, insert_pause_idx, move, pause_type): if (insert_pause_idx <= 0): return stop_sample = samples[insert_pause_idx - 1] start_sample = samples[insert_pause_idx] pause_duration = start_sample.time - stop_sample.time pause_distance = vincenty((radian_to_degree(stop_sample.latitude), radian_to_degree(stop_sample.longitude)), (radian_to_degree(start_sample.latitude), radian_to_degree(start_sample.longitude))).meters # Introduce start of pause sample pause_sample = Sample() pause_sample.move = move pause_sample.utc = stop_sample.utc pause_sample.time = stop_sample.time stop_sample.utc -= timedelta(microseconds=1) # Cut off 1ms from last recorded sample in order to introduce the new pause sample and keep time order stop_sample.time -= timedelta(microseconds=1) pause_sample.events = {"pause": {"state": "True", "type": str(pause_type), "duration": str(pause_duration), "distance": str(pause_distance), }} samples.insert(insert_pause_idx, pause_sample) # Duplicate last element # Introduce end of pause sample pause_sample = Sample() pause_sample.move = move pause_sample.utc = start_sample.utc pause_sample.time = start_sample.time start_sample.utc += timedelta(microseconds=1) # Add 1ms to the first recorded sample in order to introduce the new pause sample and keep time order start_sample.time += timedelta(microseconds=1) pause_sample.events = {"pause": {"state": "False", "duration": "0", "distance": "0", "type": str(pause_type) }} samples.insert(insert_pause_idx + 1, pause_sample)
def parse_sample(row, dayfirst_dict, source=None): """Parse a row into a Sample. Parameters ---------- row : dict-like Object whose keys are column headings and values are the row values. dayfirst_dict : dict Dictionary whose keys are names of columns containing date/time data and values are boolean indicating whether the dates in that column should be interpreted as having a day as the first component (True) or a month or year as the first component (False). source : model.Source Source for the returned sample. Returns ------- Sample """ sample = Sample() sample.age_units = get_age_units(row, ureg.years) sample.age = get_age(row) sample.latitude = get_latitude(row) sample.longitude = get_longitude(row) sample.elevation = get_elevation(row) sample.height_units = get_height_units(row, ureg.metres) sample.height = get_height(row) sample.weight_units = get_weight_units(row, ureg.kilograms) sample.weight = get_weight(row) sample.bmi = get_bmi(row) (d, t) = get_collection_datetime(row, dayfirst_dict) sample.sample_date = d sample.sample_time = t sample.sampling_time = parse_sampling_time(sample) sample.sampling_site = parse_sampling_site(row) # Initialize equality attrs if not source: sample.source = parse_source(row) elif isinstance(source, Source): sample.source = source else: raise TypeError(f'Given source was not of type {type(Source())!r}.') sample.orig_study_id = get_study_id(row) sample.orig_subject_id = get_subject_id(row) sample.orig_sample_id = get_sample_id(row) return sample
def parse_samples(tree, move, gpx_namespace, import_options): all_samples = [] tracks = tree.iterchildren(tag=gpx_namespace + GPX_TRK) for track in tracks: track_samples = [] track_segments = track.iterchildren(tag=gpx_namespace + GPX_TRKSEG) for track_segment in track_segments: segment_samples = [] track_points = track_segment.iterchildren(tag=gpx_namespace + GPX_TRKPT) for track_point in track_points: sample = Sample() sample.move = move # GPS position / altitude sample.latitude = degree_to_radian(float(track_point.attrib[GPX_TRKPT_ATTRIB_LATITUDE])) sample.longitude = degree_to_radian(float(track_point.attrib[GPX_TRKPT_ATTRIB_LONGITUDE])) sample.sample_type = GPX_SAMPLE_TYPE if hasattr(track_point, GPX_TRKPT_ATTRIB_ELEVATION): sample.gps_altitude = float(track_point.ele) sample.altitude = int(round(sample.gps_altitude)) # Time / UTC sample.utc = dateutil.parser.parse(str(track_point.time)) # Option flags pause_detected = False # Track segment samples if len(segment_samples) > 0: # Accumulate time delta to previous sample to get the total duration time_delta = sample.utc - segment_samples[-1].utc sample.time = segment_samples[-1].time + time_delta # Accumulate distance to previous sample distance_delta = distance((radian_to_degree(sample.latitude), radian_to_degree(sample.longitude)), (radian_to_degree(segment_samples[-1].latitude), radian_to_degree(segment_samples[-1].longitude))).meters sample.distance = segment_samples[-1].distance + distance_delta if time_delta > timedelta(0): sample.speed = distance_delta / time_delta.total_seconds() else: sample.speed = 0 # Option: Pause detection based on time delta threshold if GPX_IMPORT_OPTION_PAUSE_DETECTION in import_options and time_delta > import_options[GPX_IMPORT_OPTION_PAUSE_DETECTION]: pause_detected = True sample.distance = segment_samples[-1].distance sample.speed = 0 # Track segment -> Track (multiple track segments contained) elif len(track_samples) > 0: sample.time = track_samples[-1].time + (sample.utc - track_samples[-1].utc) # Time diff to last sample of the previous track segment sample.distance = track_samples[-1].distance sample.speed = 0 # Track -> Full GPX (multiple tracks contained) elif len(all_samples) > 0: sample.time = all_samples[-1].time + (sample.utc - all_samples[-1].utc) # Time diff to last sample of the previous track sample.distance = all_samples[-1].distance sample.speed = 0 # First sample else: sample.time = timedelta(0) sample.distance = 0 sample.speed = 0 parse_sample_extensions(sample, track_point) segment_samples.append(sample) # Finally insert a found pause based on time delta threshold if pause_detected: insert_pause(segment_samples, len(segment_samples) - 1, move, pause_type=GPX_IMPORT_PAUSE_TYPE_PAUSE_DETECTION) # end for track_points # Insert an pause event between every track segment insert_pause_idx = len(track_samples) track_samples.extend(segment_samples) insert_pause(track_samples, insert_pause_idx, move, pause_type=GPX_TRKSEG) # end for track_segments # Insert an pause event between every track insert_pause_idx = len(all_samples) all_samples.extend(track_samples) insert_pause(all_samples, insert_pause_idx, move, pause_type=GPX_TRK) # end for tracks return all_samples
def strava_import(current_user, activity_id): client = get_strava_client(current_user) activity = client.get_activity(activity_id=activity_id) stream_types = ['time', 'distance', 'latlng', 'temp', 'heartrate', 'velocity_smooth', 'altitude'] streams = client.get_activity_streams(activity_id, types=stream_types) device_ids = [device_id for device_id, in db.session.query(func.distinct(Move.device_id)) .join(User) .join(Device) .filter(Device.name != gpx_import.GPX_DEVICE_NAME) .filter(Move.user == current_user).all()] assert len(device_ids) == 1 device_id = device_ids[0] device = db.session.query(Device).filter_by(id = device_id).one(); activity_string = map_type(activity.type) result = db.session.query(Move.activity_type).filter(Move.activity == activity_string).first() if result: activity_type, = result else: activity_type = None move = Move() move.user = current_user move.duration = activity.elapsed_time move.ascent = float(activity.total_elevation_gain) move.speed_avg = float(activity.average_speed) move.hr_avg = heart_rate(activity.average_heartrate) move.temperature_avg = celcius_to_kelvin(activity.average_temp) move.device = device move.date_time = activity.start_date_local move.activity = activity_string move.activity_type = activity_type move.distance = float(activity.distance) move.import_date_time = datetime.now() move.import_module = __name__ move.strava_activity_id = activity_id move.public = False move.source = "Strava activity id=%d; external_id='%s'" % (activity_id, activity.external_id) lengths = set([len(streams[stream].data) for stream in streams]) assert len(lengths) == 1 length, = lengths move.speed_max = move.speed_avg all_samples = [] for i in range(0, length): time = timedelta(seconds=streams['time'].data[i]) distance = float(streams['distance'].data[i]) if 'heartrate' in streams: hr = float(streams['heartrate'].data[i]) else: hr = None if 'latlng' in streams: lat, lng = streams['latlng'].data[i] else: lat = None lng = None if 'altitude' in streams: altitude = float(streams['altitude'].data[i]) else: altitude = None if 'velocity_smooth' in streams: speed = float(streams['velocity_smooth'].data[i]) else: speed = None if 'temp' in streams: temperature = celcius_to_kelvin(streams['temp'].data[i]) else: temperature = None sample = Sample() sample.sample_type = SAMPLE_TYPE sample.move = move sample.time = time sample.utc = (activity.start_date + time).replace(tzinfo=None) sample.distance = distance sample.latitude = degree_to_radian(lat) sample.longitude = degree_to_radian(lng) sample.hr = heart_rate(hr) sample.temperature = temperature sample.speed = speed sample.altitude = altitude move.speed_max = max(move.speed_max, speed) all_samples.append(sample) derive_move_infos_from_samples(move, all_samples) db.session.add(move) db.session.commit() return move
def strava_import(current_user, activity_id): client = get_strava_client(current_user) activity = client.get_activity(activity_id=activity_id) stream_types = [ 'time', 'distance', 'latlng', 'temp', 'heartrate', 'velocity_smooth', 'altitude' ] streams = client.get_activity_streams(activity_id, types=stream_types) activity_string = map_type(activity.type) result = db.session.query( Move.activity_type).filter(Move.activity == activity_string).first() if result: activity_type, = result else: activity_type = None device = find_device(current_user) move = Move() move.user = current_user move.duration = activity.elapsed_time move.ascent = float(activity.total_elevation_gain) move.speed_avg = float(activity.average_speed) move.hr_avg = heart_rate(activity.average_heartrate) move.temperature_avg = celcius_to_kelvin(activity.average_temp) move.device = device move.date_time = activity.start_date_local move.activity = activity_string move.activity_type = activity_type move.distance = float(activity.distance) move.import_date_time = datetime.now() move.import_module = __name__ move.strava_activity_id = activity_id move.public = False move.source = "Strava activity id=%d; external_id='%s'" % ( activity_id, activity.external_id) if streams: lengths = set([len(streams[stream].data) for stream in streams]) assert len(lengths) == 1 length, = lengths else: length = 0 move.speed_max = move.speed_avg all_samples = [] for i in range(0, length): time = timedelta(seconds=streams['time'].data[i]) distance = float(streams['distance'].data[i]) if 'heartrate' in streams: hr = float(streams['heartrate'].data[i]) else: hr = None if 'latlng' in streams: lat, lng = streams['latlng'].data[i] else: lat = None lng = None if 'altitude' in streams: altitude = float(streams['altitude'].data[i]) else: altitude = None if 'velocity_smooth' in streams: speed = float(streams['velocity_smooth'].data[i]) else: speed = None if 'temp' in streams: temperature = celcius_to_kelvin(streams['temp'].data[i]) else: temperature = None sample = Sample() sample.sample_type = SAMPLE_TYPE sample.move = move sample.time = time sample.utc = (activity.start_date + time).replace(tzinfo=None) sample.distance = distance sample.latitude = degree_to_radian(lat) sample.longitude = degree_to_radian(lng) sample.hr = heart_rate(hr) sample.temperature = temperature sample.speed = speed sample.altitude = altitude move.speed_max = max(move.speed_max, speed) all_samples.append(sample) derive_move_infos_from_samples(move, all_samples) db.session.add(move) db.session.flush() postprocess_move(move) db.session.commit() return move
class ModelAddTest(unittest.TestCase): def setUp(self): self.experiment1 = Experiment() self.subject1 = Subject(orig_subject_id=1) self.subject2 = Subject(orig_subject_id=2) self.sample1 = Sample(orig_sample_id=1) self.sample2 = Sample(orig_sample_id=2) self.sample3 = Sample(orig_sample_id=3) self.sample4 = Sample(orig_sample_id=4) # Set up relationships self.subject1._samples = {self.sample1, self.sample2} self.subject2._samples = {self.sample3} self.sample1._subject = self.subject1 self.sample2._subject = self.subject1 self.sample3._subject = self.subject2 def tearDown(self): del self.experiment1 del self.subject1 del self.subject2 del self.sample1 del self.sample2 del self.sample3 del self.sample4 def test_add_subject_to_experiment(self): # Attributes that should change expected_subject_experiments = {self.experiment1} expected_experiment_subjects = {self.subject1} expected_experiment_samples = {self.sample1, self.sample2} expected_sample_experiments = {self.experiment1} # Call the function self.experiment1.add_subject(self.subject1) # Check self.assertEqual(self.subject1.experiments, expected_subject_experiments) self.assertEqual(self.subject2.experiments, set()) self.assertEqual(self.experiment1.subjects, expected_experiment_subjects) self.assertEqual(self.experiment1.samples, expected_experiment_samples) self.assertEqual(self.sample1.experiments, expected_sample_experiments) self.assertEqual(self.sample2.experiments, expected_sample_experiments) def test_add_experiment_to_subject(self): # Attributes that should change expected_subject_experiments = {self.experiment1} expected_experiment_subjects = {self.subject1} expected_experiment_samples = {self.sample1, self.sample2} expected_sample_experiments = {self.experiment1} # Call the function self.subject1.add_experiment(self.experiment1) # Check self.assertEqual(self.subject1.experiments, expected_subject_experiments) self.assertEqual(self.subject2.experiments, set()) self.assertEqual(self.experiment1.subjects, expected_experiment_subjects) self.assertEqual(self.experiment1.samples, expected_experiment_samples) self.assertEqual(self.sample1.experiments, expected_sample_experiments) self.assertEqual(self.sample2.experiments, expected_sample_experiments) def test_add_sample_to_experiment(self): # Attributes that should change expected_subject_experiments = {self.experiment1} expected_experiment_subjects = {self.subject1} expected_experiment_samples = {self.sample1} expected_sample_experiments = {self.experiment1} # Call the function self.experiment1.add_sample(self.sample1) # Check self.assertEqual(self.subject1.experiments, expected_subject_experiments) self.assertEqual(self.subject2.experiments, set()) self.assertEqual(self.experiment1.subjects, expected_experiment_subjects) self.assertEqual(self.experiment1.samples, expected_experiment_samples) self.assertEqual(self.sample1.experiments, expected_sample_experiments) self.assertEqual(self.sample2.experiments, set()) def test_add_experiment_to_sample(self): # Attributes that should change expected_subject_experiments = {self.experiment1} expected_experiment_subjects = {self.subject1} expected_experiment_samples = {self.sample1} expected_sample_experiments = {self.experiment1} # Call the function self.sample1.add_experiment(self.experiment1) # Check self.assertEqual(self.subject1.experiments, expected_subject_experiments) self.assertEqual(self.subject2.experiments, set()) self.assertEqual(self.experiment1.subjects, expected_experiment_subjects) self.assertEqual(self.experiment1.samples, expected_experiment_samples) self.assertEqual(self.sample1.experiments, expected_sample_experiments) self.assertEqual(self.sample2.experiments, set()) # Where the subject is not linked to any experiment def test_add_sample_to_subject(self): # Attributes that should change expected_sample_subject = self.subject1 expected_subject_samples = {self.sample1, self.sample2, self.sample4} # Call the function self.subject1.add_sample(self.sample4) self.assertEqual(self.subject1.samples, expected_subject_samples) self.assertEqual(self.sample4.subject, expected_sample_subject) # Attributes that should not be affected self.assertEqual(self.subject2.samples, {self.sample3}) self.assertEqual(self.subject1.experiments, set()) self.assertEqual(self.subject2.experiments, set()) self.assertEqual(self.experiment1.subjects, set()) self.assertEqual(self.experiment1.samples, set()) self.assertEqual(self.sample1.experiments, set()) self.assertEqual(self.sample2.experiments, set()) self.assertEqual(self.sample1.subject, self.subject1) self.assertEqual(self.sample3.subject, self.subject2) # Where the subject is not linked to any experiment def test_add_subject_to_sample(self): # Attributes that should change expected_sample_subject = self.subject1 expected_subject_samples = {self.sample1, self.sample2, self.sample4} # Call the function (perform assignment) self.sample4.subject = self.subject1 self.assertEqual(self.subject1.samples, expected_subject_samples) self.assertEqual(self.sample4.subject, expected_sample_subject) # Attributes that should not be affected self.assertEqual(self.subject2.samples, {self.sample3}) self.assertEqual(self.subject1.experiments, set()) self.assertEqual(self.subject2.experiments, set()) self.assertEqual(self.experiment1.subjects, set()) self.assertEqual(self.experiment1.samples, set()) self.assertEqual(self.sample1.experiments, set()) self.assertEqual(self.sample2.experiments, set()) self.assertEqual(self.sample1.subject, self.subject1) self.assertEqual(self.sample3.subject, self.subject2)
def parse_samples(tree, move, gpx_namespace): all_samples = [] tracks = tree.iterchildren(tag=gpx_namespace + GPX_TRK) for track in tracks: track_samples = [] track_segments = track.iterchildren(tag=gpx_namespace + GPX_TRKSEG) for track_segment in track_segments: segment_samples = [] track_points = track_segment.iterchildren(tag=gpx_namespace + GPX_TRKPT) for track_point in track_points: sample = Sample() sample.move = move # GPS position / altitude sample.latitude = degree_to_radian(float(track_point.attrib[GPX_TRKPT_ATTRIB_LATITUDE])) sample.longitude = degree_to_radian(float(track_point.attrib[GPX_TRKPT_ATTRIB_LONGITUDE])) sample.sample_type = "gps-base" if hasattr(track_point, GPX_TRKPT_ATTRIB_ELEVATION): sample.gps_altitude = float(track_point.ele) sample.altitude = int(round(sample.gps_altitude)) # Time / UTC sample.utc = dateutil.parser.parse(str(track_point.time)) if len(segment_samples) > 0: # Accumulate time delta to previous sample to get the total duration time_delta = sample.utc - segment_samples[-1].utc sample.time = segment_samples[-1].time + time_delta # Accumulate distance to previous sample distance_delta = vincenty( (radian_to_degree(sample.latitude), radian_to_degree(sample.longitude)), ( radian_to_degree(segment_samples[-1].latitude), radian_to_degree(segment_samples[-1].longitude), ), ).meters sample.distance = segment_samples[-1].distance + distance_delta if time_delta > timedelta(0): sample.speed = distance_delta / time_delta.total_seconds() else: sample.speed = 0 elif len(track_samples) > 0: sample.time = track_samples[-1].time sample.distance = track_samples[-1].distance sample.speed = track_samples[-1].speed elif len(all_samples) > 0: sample.time = all_samples[-1].time sample.distance = all_samples[-1].distance sample.speed = all_samples[-1].speed else: sample.time = timedelta(0) sample.distance = 0 sample.speed = 0 parse_sample_extensions(sample, track_point) segment_samples.append(sample) # Insert an pause event between every tracksegment insert_pause(track_samples, segment_samples) track_samples.extend(segment_samples) # Insert an pause event between every track insert_pause(all_samples, track_samples) all_samples.extend(track_samples) return all_samples
def findFastqFiles(self, directory, inFormat): """The method findFastqFiles finds all fastq files recursively in a directory, from each directory with fastq files a sample is created. :param directory: the directory where the user hid his fastq files :type directory: str -- path to the directory """ fastqFiles = [] for fileName in os.listdir(directory): fileName = directory + "/" + fileName if os.path.isdir(fileName): self.findFastqFiles(fileName, inFormat) else: if inFormat == "bam": if fileName.endswith(".bam") or fileName.endswith( ".bam.gz"): newSamp = Sample.Sample( self.pool, os.path.basename(os.path.splitext(fileName)[0])) self.samples.append(newSamp) newSamp.bam = BamFile.BamFile(self.pool, newSamp, fileName, sortedBam=True, headerLine=True, duplicates=False, mdTag=True, index=True) self.pool.addSample(newSamp) elif inFormat == "fq": if fileName.endswith(".fq") or fileName.endswith(".fq.gz"): fastqFiles.append(fileName) elif inFormat == "vcf": if fileName.endswith(".vcf") or fileName.endswith( ".vcf.gz"): if len(os.listdir(directory)) == 1: chrom = None else: chrom = self.getChromosomeFromVcf(fileName) self.pool.vcf[chrom] = VcfFile.VcfFile(self.pool, fileName, bcf=False, filtered=True, phased=True, chrom=chrom) elif fileName.endswith(".bcf") or fileName.endswith( ".bcf.gz"): chrom = Tools.getChromosomeOfFile( Program.config.getPath("refGenome"), fileName) self.pool.vcf[chrom] = VcfFile.VcfFile(self.pool, fileName, bcf=True, filtered=True, phased=True, chrom=chrom) if inFormat == "bam" or inFormat == "vcf": return if len(fastqFiles) > 0: #create a library name from the file name libName = os.path.basename(fastqFiles[0]) if libName.endswith("_1.fq") or libName.endswith("_2.fq"): libName = libName[:-5] if libName.endswith("_1.fq.gz") or libName.endswith("_2.fq.gz"): libName = libName[:-8] else: libName = libName[:-3] #create the sample sample = Sample.Sample(self.pool, libName) self.pool.addSample(sample) #add the fastq files to the sample if len(fastqFiles) == 1: sample.setForwardFq(fastqFiles[0]) elif len(fastqFiles) == 2: sample.setForwardFq(fastqFiles[0]) sample.setReversedFq(fastqFiles[1]) elif len(fastqFiles) > 2: if fastqFiles[0].endswith("_1.fq"): suffix = "_1.fq" elif fastqFiles[0].endswith("_1.fq.gz"): suffix = "_1.fq.gz" else: print( "WARNING: files do not end with _1.fq or _1.fq.gz or _2.fq or _2.fq.gz, using all files in one directory as 1 sample with only forward reads" ) suffix = fastqFiles[0][-3:] #create a list of forward fastq files and one of reversed fastq files forward = [] reversedFastq = [] for fastqFile in fastqFiles: if fastqFile.endswith(suffix): forward.append(fastqFile) else: reversedFastq.append(fastqFile) #Convert files to fastqFile objects for i in range(len(forward)): forward[i] = FastqFile.FastqFile( self.pool, sample, forward[i]) for i in range(len(reversedFastq)): reversedFastq[i] = FastqFile.FastqFile( self.pool, sample, reversedFastq[i], forward=False) #add the fastq files to the sample sample.forwardFq = forward sample.reversedFq = reversedFastq