def case_function(samples, obj): sam = Samples() # object required to plot num_bin_x, num_bin_y = (bin_x.get(), bin_y.get()) mi_text.set(mi_text.get() + '\nCalculating...') not_selected = True if plot_sample.get(): sam.plot_sample(samples, num_bin_x, num_bin_y) not_selected = False if plot_compare.get(): obj.plot_compare2(num_bin_x, num_bin_y, noise.get(), step.get(), significance.get()) not_selected = False if plot_proposal.get(): sam.plot_propose(samples, num_bin_x, step.get()) not_selected = False if plot_partition.get(): sam.plot_partition(samples, num_bin_x, num_bin_y) not_selected = False if plot_converge_in_samples.get(): converge_in_samples(num_bin_x, num_bin_y, obj) not_selected = False if samples_check.get(): samples_grid(num_bin_x, num_bin_y) not_selected = False if bin_selection.get(): bin_selection_validation(sigma=significance.get(), obj=obj) not_selected = False if cal_measures.get(): calculate_measures(samples, num_bin_x, num_bin_y) not_selected = False if not_selected: mi_text.set('Select a plot or calculation')
def relate_sample( self, sample, seq_request_issue ): ''' @param sample - The name of the sample to relate to the sequence request @param seq_request_issue - Issue json of the sequence request to relate the sample to ''' from samples import Samples s = Samples( self.url, self.key ) created, sample_issue = s.get_or_create( sample ) # Set the relation try: self.set_relation( seq_request_issue, 'blocks', sample_issue ) except self.RelationExists as e: # Already created pass
def __init__(self, json_path, out_dir): # Attributes # self.out_dir = out_dir self.json_path = json_path # Parse # self.info = load_json_path(self.json_path) # Basic # self.account = self.info['uppmax_id'] self.run_num = self.info['run_num'] self.run_label = self.info['run_id'] self.project_short_name = self.info['project'] self.project_long_name = self.info['project_name'] self.fwd_name = self.info['forward_reads'] self.rev_name = self.info['reverse_reads'] # Own attributes # self.num = self.info['pool_num'] self.label = self.info['pool_id'] self.short_name = self.info['pool'] self.long_name = self.info['pool_name'] self.id_name = "run%03d-pool%02d" % (self.run_num, self.num) # Special # self.samples = Samples(self) self.primers = TwoPrimers(self) # Second init # self.loaded = False
def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.setup_layout() self.setup_waveform() self.setup_instructions() self.layout.nextRow() self.setup_spectrograph() self.layout.nextRow() self.setup_editor() self.setup_index() self.samples = Samples(self.waveform.data_available, self.spectrograph.data_available) self.copy_func_to_samples() self.copy_func_to_editor_and_display() self.setup_pyaudio() self.stream.start_stream()
def on_sample_chosen(self): size = self.sample_radio_id.get() logging.info('Loading sample of size ' + str(size)) self.grid_size, self.pipe_ends = Samples.get_puzzle(str(size)) self.on_grid_size_changed() self.grid_manager.load_maze(self.grid_size, self.pipe_ends) self.ready_for_run = True self.init_run()
def case_function(samples, obj): sam = Samples() print(samples) numBinx, numBiny = Binx.get(), Biny.get() #numBinx = 30 #numBiny = 30 print(numBinx, numBiny) print(numBinx) x, y = samples[:, 0], samples[:, 1] MI = mutual_info(x, y, numBinx, numBiny) ex = entropy(x, numBinx) ey = entropy(y, numBiny) R = norm_MI(x, y, numBinx, numBiny) rxy = pearson_corr(x, y) exy = entropyx_y(x, y, numBinx, numBiny) eyx = entropyx_y(y, x, numBinx, numBiny) MIE = MI_Entropy(x, y, numBinx, numBiny) Ixy = propuesta_Ixy(x, y, numBinx, numBiny) Iyx = propuesta_Iyx(x, y, numBinx, numBiny) Ixy2 = propuesta2_Ixy(x, y, numBinx, numBiny) Iyx2 = propuesta2_Iyx(x, y, numBinx, numBiny) PMD = propuesta_mutual_dependency(x, y, numBinx, numBiny) PMD2 = propuesta2_mutual_dependency(x, y, numBinx, numBiny) # d_cor = d_corr(x,y) # Mic = MIC(x,y) mitexto.set( 'Entropy of x: ' + str(ex) + '\n' + 'Entropy of y: ' + str(ey) + '\n' + 'Mutual information: ' + str(MI) + '\n' + 'Mutual info with entropy: ' + str(MIE) + '\n' + '\n' + '(Max=1)Normalized Mutual info: ' + str(R) + '\n' + '(Max=1)Pearson Correlation: ' + str(rxy) + '\n' + # '(Max=1)Distance Correlation: ' + str(d_cor) + '\n' + # '(Max=1)MIC: ' + str(Mic) + '\n' + '(Max=1)Mutual Dependency: ' + str(PMD) + '\n' + '(Max=1)Mutual Dependency2: ' + str(PMD2) + '\n' + '\n' + 'Entropy of X|Y = ' + str(exy) + '\n' + 'Entropy of Y|X = ' + str(eyx) + '\n' + '(Max=1)Information in Y of X: ' + str(Ixy) + '\n' + '(Max=1)Information in X of Y: ' + str(Iyx) + '\n' + '(Max=1)Information2 in Y of X: ' + str(Ixy2) + '\n' + '(Max=1)Information2 in X of Y: ' + str(Iyx2)) if plot_sample.get(): sam.plot_sample(samples, numBinx, numBiny, step.get()) if plot_compare.get(): obj.plot_compare2(numBinx, numBiny, noise.get(), step.get()) if plot_propuse.get(): sam.plot_propose(samples, numBinx) if plot_partition.get(): sam.plot_partition(samples, numBinx, numBiny) if plot_converge_in_samples.get(): Converge_in_samples(numBinx, numBiny, obj) if samples_check.get(): samples_grid(numBinx, numBiny)
def __init__(self, json_path, out_dir): # Attributes # self.out_dir = out_dir self.json_path = json_path # Parse # with open(json_path) as handle: self.info = json.load(handle) # Basic # self.account = self.info['uppmax_id'] self.run_num = self.info['run_num'] self.run_label = self.info['run_id'] self.project_short_name = self.info['project'] self.project_long_name = self.info['project_name'] # Own attributes # self.num = self.info['pool_num'] self.label = self.info['pool_id'] self.short_name = self.info['pool'] self.long_name = self.info['pool_name'] self.id_name = "run%03d-pool%02d" % (self.run_num, self.num) # Special # self.samples = Samples(self) self.primers = TwoPrimers(self) # Second init # self.loaded = False
def find_by_shebang(cls, data): """ Public: Look up Languages by shebang line. data - Array of tokens or String data to analyze. Examples Language.find_by_shebang("#!/bin/bash\ndate;") # => [#<Language name="Bash">] Returns the matching Language """ interpreter = Samples.interpreter_from_shebang(data) return cls.interpreter_index.get(interpreter)
def save_wav(self) -> None: # Save waveform duration, ok = QtGui.QInputDialog.getInt(self, "Seconds of Audio:", "Seconds:", 1, 0, MAX_VAL, STEP_VAL) if ok: dialog = QtGui.QFileDialog() path = dialog.getSaveFileName(self, 'Save File', os.getenv('HOME'), 'WAV (*.wav)') if path[0] != '': samples = Samples() expression = copy.deepcopy(self.expression) samples.set_expression(expression) samples.gen_write_16(path[0], duration)
def plot_select(): ind = list1.curselection() if list1.curselection() != (): if ind[0] == 0: sam = Samples(tipo=0) samples = sam.get_sin(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 1: sam = Samples(tipo=1) samples = sam.get_square(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 2: sam = Samples(tipo=2) samples = sam.get_blur(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 3: sam = Samples(tipo=3) samples = sam.get_cuadratic(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 4: sam = Samples(tipo=4) samples = sam.get_diagonal_line(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 5: sam = Samples(tipo=5) samples = sam.get_horizontal_line(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 6: sam = Samples(tipo=6) samples = sam.get_vertical_line(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 7: sam = Samples(tipo=7) samples = sam.get_x(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 8: sam = Samples(tipo=8) samples = sam.get_circle(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 9: sam = Samples(tipo=9) samples = sam.get_curve_x(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 10: sam = Samples(tipo=10) samples = sam.get_diagonal_line2(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 11: sam = Samples(tipo=11) samples = sam.get_dependent(numberSam.get()) case_function(samples, sam) elif ind[0] == 12: sam = Samples(tipo=12) samples = sam.get_independent(numberSam.get()) case_function(samples, sam) elif ind[0] == 13: sam = Samples(tipo=13) samples = sam.get_corr(numberSam.get(), noise.get()) case_function(samples, sam) elif ind[0] == 14: sam = Samples(tipo=14) samples = sam.get_file() case_function(samples, sam) else: mitexto.set('Error') else: mitexto.set('Select a type')
class Pool(object): """An illumina MID is called here a 'pool'.""" all_paths = """ /samples/ /groups/ /graphs/ /fastqc/ /logs/ /quality_reads/ /quality_reads/fractions/ /info.json """ def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.id_name) def __str__(self): return self.id_name def __iter__(self): return iter(self.children) def __len__(self): return self.count def __getitem__(self, key): return self.samples[key] @property def seq_len(self): return len(self.fwd.first_read) def __init__(self, json_path, out_dir): # Attributes # self.out_dir = out_dir self.json_path = json_path # Parse # with open(json_path) as handle: self.info = json.load(handle) # Basic # self.account = self.info['uppmax_id'] self.run_num = self.info['run_num'] self.run_label = self.info['run_id'] self.project_short_name = self.info['project'] self.project_long_name = self.info['project_name'] # Own attributes # self.num = self.info['pool_num'] self.label = self.info['pool_id'] self.short_name = self.info['pool'] self.long_name = self.info['pool_name'] self.id_name = "run%03d-pool%02d" % (self.run_num, self.num) # Special # self.samples = Samples(self) self.primers = TwoPrimers(self) # Second init # self.loaded = False def load(self): """A second __init__ that is delayed, solves some circular references""" # Automatic paths # self.base_dir = self.out_dir + self.id_name + '/' self.p = AutoPaths(self.base_dir, self.all_paths) # Make an alias to the json # self.p.info_json.link_from(self.json_path, safe=True) # Children # self.samples.load() # Raw file pairs # self.fwd_path = home + "ILLUMITAG/INBOX/%s/%s/%s" % (self.run.label, self.label, self.info['forward_reads']) self.rev_path = home + "ILLUMITAG/INBOX/%s/%s/%s" % (self.run.label, self.label, self.info['reverse_reads']) self.fwd = FASTQ(self.fwd_path) self.rev = FASTQ(self.rev_path) self.fastq = PairedFASTQ(self.fwd.path, self.rev.path, self) # Barcode length # self.bar_len = self.samples.bar_len # Make Outcomes # self.no_barcodes = NoBarcode(self) self.one_barcodes = OneBarcode(self) self.same_barcodes = SameBarcode(self) self.bad_barcodes = BadBarcode(self) self.good_barcodes = GoodBarcode(self) self.outcomes = (self.good_barcodes, self.no_barcodes, self.one_barcodes, self.same_barcodes, self.bad_barcodes) self.children = self.outcomes # The good reads # self.quality_reads = QualityReads(self.good_barcodes.assembled.good_primers.len_filtered, self) self.fractions = Fractions(self) # Runner # self.runner = PoolRunner(self) # Graphs # self.graphs = [getattr(pool_plots, cls_name)(self) for cls_name in pool_plots.__all__] # Loaded # self.loaded = True # Return self for convenience # return self @property def first(self): return self.children[0] @property def count(self): if not self.loaded: self.load() return self.fastq.count @property def avg_quality(self): if not self.loaded: self.load() return self.fastq.avg_quality def __call__(self, *args, **kwargs): if not self.loaded: self.load() self.runner.run(*args, **kwargs) def run_slurm(self, *args, **kwargs): if not self.loaded: self.load() return self.runner.run_slurm(*args, **kwargs) def pool_fastqc(self): """Run fastqc on the all the sequences of the pool""" if not self.loaded: self.load() self.fastq.fastqc(self.p.fastqc_dir) def create_outcomes(self): """Sort the sequences depending on their barcode status""" if not self.loaded: self.load() for o in self.outcomes: o.create() for r in self.fastq.parse_barcodes(): if len(r.matches) == 0: self.no_barcodes.add_pair(r) elif len(r.matches) == 1: self.one_barcodes.add_pair(r) elif r.matches[0].set == r.matches[1].set: self.same_barcodes.add_pair(r) elif r.matches[0].sample is not r.matches[1].sample: self.bad_barcodes.add_pair(r) else: self.good_barcodes.add_pair(r) for o in self.outcomes: o.close() def create_samples(self): """Sort the sequences in different files according to their barcode number (if they have one)""" if not self.loaded: self.load() for sample in self.samples: sample.create() for r in tqdm(self.quality_reads.untrimmed.parse_barcodes()): r.first.sample.add_read(r.read) for sample in self.samples: sample.close() def check_fastq_version(self): """Let's make sure we are dealing with the Sanger encoding""" for o in self.outcomes: assert fastqident.detect_encoding(o.fwd_path) == 'sanger' assert fastqident.detect_encoding(o.rev_path) == 'sanger' def make_pool_plots(self): """Call graphs that are in pool_plots.py""" if not self.loaded: self.load() for graph in self.graphs: graph.plot() @property_cached def loss_statistics(self): """This should be moved to the reporting file""" class MessageStat(object): def __init__(self, msg, value): self.msg = msg self.value = value class LossStatistics(object): def __init__(self, pool): self.pool = pool def __iter__(self): return iter((self.outcome, self.assembly, self.primers, self.n_filter, self.qual_filter, self.len_filter)) @property def outcome(self): return MessageStat("Good barcodes are only %f%% of total", (100*len(self.pool.good_barcodes)/self.pool.count)) @property def assembly(self): return MessageStat("Assembled is only %f%% of good barcodes", (100*len(self.pool.good_barcodes.assembled)/len(self.pool.good_barcodes))) @property def primers(self): return MessageStat("Good primers is only %f%% of assembled", (100*len(self.pool.good_barcodes.assembled.good_primers)/len(self.pool.good_barcodes.assembled))) @property def n_filter(self): return MessageStat("N filtered is only %f%% of good primers", (100*len(self.pool.good_barcodes.assembled.good_primers.n_filtered)/len(self.pool.good_barcodes.assembled.good_primers))) @property def qual_filter(self): return MessageStat("Qual filtered is only %f%% of N filtered", (100*len(self.pool.good_barcodes.assembled.good_primers.qual_filtered)/len(self.pool.good_barcodes.assembled.good_primers.n_filtered))) @property def len_filter(self): return MessageStat("Length filter is only %f%% of qual filtered", (100*len(self.pool.good_barcodes.assembled.good_primers.len_filtered)/len(self.pool.good_barcodes.assembled.good_primers.qual_filtered))) return LossStatistics(self) @property def json(self): """Regenerate the JSON string for every sample""" return ',\n\n'.join(s.json for s in self.samples)
wall_nb = max(start_walls, target_walls) score += wall_nb * 10 # increase score if the distance between the start and target is high distance = self.shortest_path(start, target, self.original_id(i)) if distance > self.grid_size / 2: score += distance * 2 if score > best_score: best_pipe = i best_score = score logging.debug("Choosing next pipe {0}".format( self.original_id(best_pipe))) # flip the next pipe with the one we want to process if best_pipe != self.curr_pipe: original_best_pipe_id = self.pipes_mapping[best_pipe] self.pipes_mapping[best_pipe] = self.pipes_mapping[self.curr_pipe] self.pipes_mapping[self.curr_pipe] = original_best_pipe_id if __name__ == "__main__": setup_logging() (size, pipes) = Samples.get_puzzle("12") engine = ShortestPathEngine(size, pipes) while not engine.solved: for next_move in engine.next_moves(): logging.debug(next_move) logging.info(engine.display())
class Madulator(pg.GraphicsView): function_index = 1 generator = Generator(function_index) expression = generator.random_function() def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) self.setup_layout() self.setup_waveform() self.setup_instructions() self.layout.nextRow() self.setup_spectrograph() self.layout.nextRow() self.setup_editor() self.setup_index() self.samples = Samples(self.waveform.data_available, self.spectrograph.data_available) self.copy_func_to_samples() self.copy_func_to_editor_and_display() self.setup_pyaudio() self.stream.start_stream() def setup_pyaudio(self) -> None: self.pa = pa.PyAudio() self.stream = self.pa.open( format=pa.paUInt8, channels=1, rate=BITRATE, output=True, stream_callback=self.samples.pyaudio_callback) def keyPressEvent(self, ev: QtGui.QKeyEvent) -> None: key = ev.key() if key == QtCore.Qt.Key.Key_Escape: # Stop stream and terminate all self.terminate_program() elif key == QtCore.Qt.Key.Key_Space: # Pause/Resume audio stream self.pause_resume() elif key == QtCore.Qt.Key.Key_R: # Restart playback self.restart_stream() elif key == QtCore.Qt.Key.Key_W: # Save waveform self.save_wav() elif key == QtCore.Qt.Key.Key_S: # Save and download a function self.save_func() elif key == QtCore.Qt.Key.Key_L: # Load a function from computer self.load_func() elif key == QtCore.Qt.Key.Key_BracketLeft: # Index through older randomized functions self.previous_index() elif key == QtCore.Qt.Key.Key_BracketRight: # Index through newer randomized functions self.next_index() elif key == QtCore.Qt.Key.Key_I: # Go to a certain function index self.get_index() elif key == QtCore.Qt.Key.Key_V: # Change expression into a Value entered by user self.change_to_value() elif key == QtCore.Qt.Key.Key_Comma: self.samples.decrease_playback_speed() self.update_index_speed_text() elif key == QtCore.Qt.Key.Key_Period: self.samples.increase_playback_speed() self.update_index_speed_text() elif key == QtCore.Qt.Key.Key_Equal: self.samples.reset_playback_speed() self.update_index_speed_text() else: # Change expression as dictated by user is_editor_key = self.editor.new_key(ev.key()) if is_editor_key: self.restart_stream() else: self.update_editor_info() def update_index_speed_text(self): self.index_text.setText( "Function index: {:d} | Playback speed: {:.2f}".format( self.function_index, self.samples.get_playback_speed())) # Key press events def terminate_program(self) -> None: # Stop stream and terminate all if self.stream.is_active(): self.stream.stop_stream() self.stream.close() self.pa.terminate() QtCore.QCoreApplication.quit() def pause_resume(self) -> None: if self.stream.is_active(): self.stream.stop_stream() else: self.stream.start_stream() def save_wav(self) -> None: # Save waveform duration, ok = QtGui.QInputDialog.getInt(self, "Seconds of Audio:", "Seconds:", 1, 0, MAX_VAL, STEP_VAL) if ok: dialog = QtGui.QFileDialog() path = dialog.getSaveFileName(self, 'Save File', os.getenv('HOME'), 'WAV (*.wav)') if path[0] != '': samples = Samples() expression = copy.deepcopy(self.expression) samples.set_expression(expression) samples.gen_write_16(path[0], duration) def save_func(self) -> None: # Save and download a function dialog = QtGui.QFileDialog() path = dialog.getSaveFileName(self, 'Save File', 'save/', 'MAD (*.mad)') if path[0] != '': with open(path[0], 'wb') as out_file: exp = self.samples.get_expression() pickle.dump(exp, out_file) def load_func(self) -> None: # Load a function from computer if self.stream.is_active(): self.stream.stop_stream() dialog = QtGui.QFileDialog() path = dialog.getOpenFileName(self, 'Open File', 'save/', "MAD (*.mad)") if path[0] != '': with open(path[0], 'rb') as in_file: exp = pickle.load(in_file) self.expression = exp # Pass a copy to samples self.copy_func_to_samples() self.editor = Editor(exp) # Pass a copy of the expression to editor and display self.copy_func_to_editor_and_display() self.samples.reset_playback_speed() self.update_index_speed_text() self.stream.start_stream() def restart_stream(self) -> None: # Stop stream and get reset function if self.stream.is_active(): self.stream.stop_stream() expression = self.editor.get_function() self.expression = expression # Pass a copy to samples, start stream, and display self.copy_func_to_samples() self.stream.start_stream() self.update_editor_info() def previous_index(self) -> None: if self.function_index > 1: self.function_index = self.function_index - 1 self.update_function_from_index() self.samples.reset_playback_speed() self.update_index_speed_text() def next_index(self) -> None: self.function_index = self.function_index + 1 self.update_function_from_index() self.samples.reset_playback_speed() self.update_index_speed_text() def get_index(self) -> None: val, ok = QtGui.QInputDialog.getInt(self, "Input Index:", "Index:", 1, 1, 2**30, 1) if ok: self.function_index = val self.index_text.setText("Random function index: " + str(self.function_index)) self.generator = Generator(self.function_index) self.expression = self.generator.random_function() self.copy_func_to_samples() self.samples.reset_playback_speed() self.copy_func_to_editor_and_display() self.update_index_speed_text() def change_to_value(self) -> None: # Change expression into a Value entered by user val = self.get_number() if val != -1: self.editor.create_value(val) self.restart_stream() self.update_editor_info() def update_function_from_index(self) -> None: self.generator = Generator(self.function_index) self.expression = self.generator.random_function() self.copy_func_to_samples() self.copy_func_to_editor_and_display() def update_editor_info(self) -> None: selection = self.editor.get_selection() expression = self.editor.get_function() self.editor_text.setText(expression.html_tree(selection)) def copy_func_to_samples(self) -> None: expression = copy.deepcopy(self.expression) self.samples.set_expression(expression) def copy_func_to_editor_and_display(self) -> None: function = copy.deepcopy(self.expression) self.editor.set_function(function) self.update_editor_info() def get_number(self) -> int: val, ok = QtGui.QInputDialog.getInt(self, "Input Value:", "Value:", DEFAULT_VAL, MIN_VAL, MAX_VAL, STEP_VAL) if ok: return val return -1 def setup_layout(self) -> None: self.layout = pg.GraphicsLayout(border=(100, 100, 100)) self.setCentralItem(self.layout) self.show() self.setWindowTitle('MADulator') self.resize(1024, 720) def setup_waveform(self) -> None: self.waveform = Waveform() self.layout.addItem(self.waveform) def setup_spectrograph(self) -> None: self.spectrograph = SpectrogramWidget(BITRATE) self.layout.addItem(self.spectrograph) def setup_instructions(self) -> None: text = ''' <h1>MADulator</h1> <p><strong>Program keys</strong></p> <ul> <li>[<span style='color:white'>space</span>] pause or resume playback</li> <li>[<span style='color:white'>r</span>] restart playback</li> <li>[<span style='color:white'>[</span>] [<span style='color:white'>]</span>] prev/next random function</li> <li>[<span style='color:white'>i</span>] goto random function index</li> <li>[<span style='color:white'>,</span>] [<span style='color:white'>.</span>] decrease/increase playback speed 10%</li> <li>[<span style='color:white'>=</span>] restore playback speed to normal</li> <li>[<span style='color:white'>s</span>] save function to file</li> <li>[<span style='color:white'>l</span>] load function from file</li> <li>[<span style='color:white'>w</span>] save audio as .WAV file</li> <li>[<span style='color:white'>ESC</span>] exit program</li> </ul> <p><strong>Function Editor keys</strong></p> <ul> <li>[<span style='color:white'>↑</span>] Editor: navigate to parent node</li> <li>[<span style='color:white'>←</span>] [<span style='color:white'>→</span>] Editor: navigate to left/right child</li> <li>[<span style='color:white'>v</span>] replace expression with value (integer)</li> <li>[<span style='color:white'>t</span>] replace expression with variable</li> <li>[<span style='color:white'>+</span>] replace expression with addition</li> <li>[<span style='color:white'>-</span>] replace expression with subtraction</li> <li>[<span style='color:white'>*</span>] replace expression with multiplication</li> <li>[<span style='color:white'>/</span>] replace expression with integer division</li> <li>[<span style='color:white'>%</span>] replace expression with modulo</li> <li>[<span style='color:white'>&</span>] replace expression with bitwise AND</li> <li>[<span style='color:white'>|</span>] replace expression with bitwise OR</li> <li>[<span style='color:white'>^</span>] replace expression with bitwise XOR</li> <li>[<span style='color:white'><</span>] [<span style='color:white'>></span>] replace expression with shift left/right</li> </ul> <p><small>Developed by Marko, Angelic, and Daniel</small></p> ''' self.layout.addLabel(text, rowspan=2) def setup_editor(self) -> None: function = copy.deepcopy(self.expression) self.editor = Editor(function) self.editor_text = pg.LabelItem(name='Editor') self.layout.addItem(self.editor_text) self.update_editor_info() def setup_index(self) -> None: self.index_text = pg.LabelItem(name='Index') self.layout.addItem(self.index_text) self.index_text.setText( "Function index: {:d} | Playback speed: 1.00".format( self.function_index))
def command_parser(): ''' THIS PART CAN PARSER THE COMMAND PARAMETERS. Value 0 is off, 1 is open. ''' try: source_path= "" save_path = "" # The default value of data_deal model. iamges_deal = 0 size_x = 28 size_y = 28 channel = 3 # samples model parameters samples = 0 no_replace = 0 replace = 0 number = 10000 # train and test the model train_test = 0 dataset = 'mnist.pkl.gz' config_path= "../config-example" n_y = 10 m1 = 0 m2 = 0 opts, args = getopt.getopt(sys.argv[1:], "ho:", \ ["help", \ "images_deal", \ "images_path=", \ "size_x=", \ "size_y=" , \ "channel=", \ "samples", \ "no_replace=", \ "replace=", \ "source_path=" , \ "number=", \ "save_path=", \ "train_test", \ "dataset=", \ "config_path=", \ "n_y=" , \ "m1=", \ "m2=" ]) #print opts,args for x,y in opts: if x in ("-h","--help"): print "-----------------------------------------------------------" print "ensemb: command line brew" print "usage : python ensemb.py <command> <args>" print "---------------------------------------------------------------------" print "images_deal : deal the image to pickle file" print "---------- ensemb.py --images_deal " print "---------- --source_path=/home/gd/ensemble-nn/data/image" print "---------- --size_x=28" print "---------- --size_y=28 " print "---------- --channel=3" print "---------------------------------------------------------------------" print "samples : sample the samples from the datasets" print "---------- ensemb.py --samples " print "---------- --no_replace=1" print "---------- --replace=0 " print "---------- --source_path=./data/mnist.pkl.gz" print "---------- --number=30000" print "---------- --save_path=./data/mnist-sample-replace.pkl" print "---------------------------------------------------------------------" print "train_test : train and test the model" print "---------- ensemb.py --train_test " print "---------- --dataset =/home/gd/ensemble-nn/data/mnist.pkl.gz" print "---------- --config_path=./config-example " print "---------- --n_y =10" print "---------- --m1 = 1" print "---------------------------------------------------------------------" # MODEL IMAGES_DEAL if x in ("--images_deal"): print "Deal the iamge .." iamges_deal = 1 if x in ("--source_path"): images_path = y if x in ("--size_x"): size_x = int(y) if x in ("--size_y"): size_y = int(y) if x in ("--channel"): channel = int(y) # MODEL SAMPLES if x in ("--samples"): print "sampling .." samples = 1 if x in ("--no_replace"): no_replace = int(y) if x in ("--replace"): replace = int(y) if x in ("--source_path"): source_path = y if x in ("--number"): number = int(y) if x in ("--save_path"): save_path = y # TRAIN AND TEST A MODEL if x in ("--train_test"): print "train and test .." train_test = 1 if x in ("--dataset"): dataset = y if x in ("--config_path"): config_path = y if x in ("--n_y"): n_y = int(y) if x in ("--m1"): m1 = int(y) if x in ("--m2"): m2 = int(y) if iamges_deal == 1: source_path="/home/gd/ensemble-nn/data/image" DealDate_deal(image_path=source_path,size_x=size_x,size_y = size_y, channel =channel) if samples == 1: source_path="/home/gd/ensemble-nn/data/mnist.pkl.gz" save_path="/home/gd/ensemble-nn/data/mnist-sample-replace.pkl" if no_replace == 0 and replace == 0: print "no_replace or replace ?" if no_replace == 1 and replace == 1: print "no_replace or replace ?" if no_replace == 1: instance = Samples() instance.no_replace_sample(source_path, number, save_path) if replace == 1: instance = Samples.Samples() instance.replace_sample(source_path, number, save_path) if train_test == 1: if m1 == 0 and m2 == 0: print "adaboost-m1 or adaboost-m2 ?" if m1 == 1 and m2 == 1: print "adaboost-m1 or adaboost-m2 ?" if m1 == 1 and m2 == 0: instance = Controller() instance.m1_controller(dirctory = config_path, dataset = dataset,n_y = n_y) if m1 == 0 and m2 == 1: instance = Controller() instance.m2_controller(dirctory = config_path, dataset = dataset,n_y = n_y) sys.exit() except Exception as err: print err
DIR: 'data/facedata', HEIGHT: 68, WIDTH: 61, LABEL: 2, PIXELS: None }, DIGIT: { DIR: 'data/digitdata', HEIGHT: 20, WIDTH: 29, LABEL: 10, PIXELS: None } } samples = Samples(map.get(inp).get(DIR)) dataClassifier = DataClassifier( map.get(inp).get(HEIGHT), map.get(inp).get(WIDTH), map.get(inp).get(LABEL), map.get(inp).get(PIXELS)) perceptronClassifier = PerceptronClassifier(dataClassifier.FEATURES, dataClassifier.LABELS) samples.readFiles() dataset = 0 featureValueListForAllTrainingImages, actualLabelForTrainingList = dataClassifier.extractFeatures( samples.train_lines_itr, samples.train_labelsLines_itr) TOTALDATASET = len(featureValueListForAllTrainingImages)
HEIGHT: 68, WIDTH: 61, LABEL: 2, PIXELS: None }, DIGIT: { DIR: 'data/digitdata', HEIGHT: 20, WIDTH: 29, LABEL: 10, PIXELS: None } } dataType = map.get(inp) samples = Samples(dataType.get(DIR)) dataClassifier = DataClassifier(dataType.get(HEIGHT), dataType.get(WIDTH), dataType.get(LABEL), dataType.get(PIXELS), gridSize) samples.readFiles() ''' Extracting Features from the Training Data ''' featureValueListForAllTrainingImages, actualLabelForTrainingList = \ dataClassifier.extractFeatures(samples.train_lines_itr, samples.train_labelsLines_itr) runErrorList = [] runTimeList = [] runDataSetIncrements = [] for _ in range(0, 3):
def command_parser(): ''' THIS PART CAN PARSER THE COMMAND PARAMETERS. Value 0 is off, 1 is open. ''' try: source_path = "" save_path = "" # The default value of data_deal model. iamges_deal = 0 size_x = 28 size_y = 28 channel = 3 # samples model parameters samples = 0 no_replace = 0 replace = 0 number = 10000 # train and test the model train_test = 0 dataset = 'mnist.pkl.gz' config_path = "../config-example" n_y = 10 m1 = 0 m2 = 0 opts, args = getopt.getopt(sys.argv[1:], "ho:", \ ["help", \ "images_deal", \ "images_path=", \ "size_x=", \ "size_y=" , \ "channel=", \ "samples", \ "no_replace=", \ "replace=", \ "source_path=" , \ "number=", \ "save_path=", \ "train_test", \ "dataset=", \ "config_path=", \ "n_y=" , \ "m1=", \ "m2=" ]) #print opts,args for x, y in opts: if x in ("-h", "--help"): print "-----------------------------------------------------------" print "ensemb: command line brew" print "usage : python ensemb.py <command> <args>" print "---------------------------------------------------------------------" print "images_deal : deal the image to pickle file" print "---------- ensemb.py --images_deal " print "---------- --source_path=/home/gd/ensemble-nn/data/image" print "---------- --size_x=28" print "---------- --size_y=28 " print "---------- --channel=3" print "---------------------------------------------------------------------" print "samples : sample the samples from the datasets" print "---------- ensemb.py --samples " print "---------- --no_replace=1" print "---------- --replace=0 " print "---------- --source_path=./data/mnist.pkl.gz" print "---------- --number=30000" print "---------- --save_path=./data/mnist-sample-replace.pkl" print "---------------------------------------------------------------------" print "train_test : train and test the model" print "---------- ensemb.py --train_test " print "---------- --dataset =/home/gd/ensemble-nn/data/mnist.pkl.gz" print "---------- --config_path=./config-example " print "---------- --n_y =10" print "---------- --m1 = 1" print "---------------------------------------------------------------------" # MODEL IMAGES_DEAL if x in ("--images_deal"): print "Deal the iamge .." iamges_deal = 1 if x in ("--source_path"): images_path = y if x in ("--size_x"): size_x = int(y) if x in ("--size_y"): size_y = int(y) if x in ("--channel"): channel = int(y) # MODEL SAMPLES if x in ("--samples"): print "sampling .." samples = 1 if x in ("--no_replace"): no_replace = int(y) if x in ("--replace"): replace = int(y) if x in ("--source_path"): source_path = y if x in ("--number"): number = int(y) if x in ("--save_path"): save_path = y # TRAIN AND TEST A MODEL if x in ("--train_test"): print "train and test .." train_test = 1 if x in ("--dataset"): dataset = y if x in ("--config_path"): config_path = y if x in ("--n_y"): n_y = int(y) if x in ("--m1"): m1 = int(y) if x in ("--m2"): m2 = int(y) if iamges_deal == 1: source_path = "/home/gd/ensemble-nn/data/image" DealDate_deal(image_path=source_path, size_x=size_x, size_y=size_y, channel=channel) if samples == 1: source_path = "/home/gd/ensemble-nn/data/mnist.pkl.gz" save_path = "/home/gd/ensemble-nn/data/mnist-sample-replace.pkl" if no_replace == 0 and replace == 0: print "no_replace or replace ?" if no_replace == 1 and replace == 1: print "no_replace or replace ?" if no_replace == 1: instance = Samples() instance.no_replace_sample(source_path, number, save_path) if replace == 1: instance = Samples.Samples() instance.replace_sample(source_path, number, save_path) if train_test == 1: if m1 == 0 and m2 == 0: print "adaboost-m1 or adaboost-m2 ?" if m1 == 1 and m2 == 1: print "adaboost-m1 or adaboost-m2 ?" if m1 == 1 and m2 == 0: instance = Controller() instance.m1_controller(dirctory=config_path, dataset=dataset, n_y=n_y) if m1 == 0 and m2 == 1: instance = Controller() instance.m2_controller(dirctory=config_path, dataset=dataset, n_y=n_y) sys.exit() except Exception as err: print err
class Pool(object): """An illumina MID is called here a 'pool'.""" all_paths = """ /samples/ /groups/ /graphs/ /fastqc/ /logs/ /quality_reads/ /quality_reads/fractions/ /info.json """ kind = 'pool' def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.id_name) def __str__(self): return self.id_name def __iter__(self): return iter(self.samples) def __len__(self): return self.count def __getitem__(self, key): return self.samples[key] @property def seq_len(self): return len(self.fwd.first_read) def __init__(self, json_path, out_dir): # Attributes # self.out_dir = out_dir self.json_path = json_path # Parse # self.info = load_json_path(self.json_path) # Basic # self.account = self.info['uppmax_id'] self.run_num = self.info['run_num'] self.run_label = self.info['run_id'] self.project_short_name = self.info['project'] self.project_long_name = self.info['project_name'] self.fwd_name = self.info['forward_reads'] self.rev_name = self.info['reverse_reads'] # Own attributes # self.num = self.info['pool_num'] self.label = self.info['pool_id'] self.short_name = self.info['pool'] self.long_name = self.info['pool_name'] self.id_name = "run%03d-pool%02d" % (self.run_num, self.num) # Special # self.samples = Samples(self) self.primers = TwoPrimers(self) # Second init # self.loaded = False def load(self): """A second __init__ that is delayed, solves some circular references""" # Raw file pairs # self.fwd_path = home + "/proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.fwd_name) self.rev_path = home + "/proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.rev_name) self.fwd = FASTQ(self.fwd_path) self.rev = FASTQ(self.rev_path) self.fastq = BarcodedPairedFASTQ(self.fwd.path, self.rev.path, self.samples) # Check we can load it # if not os.access('/proj/%s' % self.account, os.R_OK): raise Exception("You don't have access to the project %s" % self.account) if not self.fwd.exists: raise Exception("No file at '%s'" % self.fwd) if not self.rev.exists: raise Exception("No file at '%s'" % self.rev) # Automatic paths # self.base_dir = self.out_dir + self.id_name + '/' self.p = AutoPaths(self.base_dir, self.all_paths) # Make an alias to the json # self.p.info_json.link_from(self.json_path, safe=True) # Children # self.samples.load() # Check there are 50 # assert len(self.samples.children) == 50 # Barcode length # self.bar_len = self.samples.bar_len # Make Outcomes # self.no_barcodes = NoBarcode(self) self.one_barcodes = OneBarcode(self) self.same_barcodes = SameBarcode(self) self.bad_barcodes = BadBarcode(self) self.good_barcodes = GoodBarcode(self) self.outcomes = (self.good_barcodes, self.no_barcodes, self.one_barcodes, self.same_barcodes, self.bad_barcodes) self.children = self.outcomes # The good reads # self.quality_reads = QualityReads(self.good_barcodes.assembled.good_primers.len_filtered, self) self.fractions = Fractions(self) # Runner # self.runner = PoolRunner(self) # Graphs # self.graphs = [getattr(pool_plots, cls_name)(self) for cls_name in pool_plots.__all__] # Loaded # self.loaded = True # Return self for convenience # return self @property def first(self): return self.children[0] @property def count(self): if not self.loaded: self.load() return self.fastq.count @property def avg_quality(self): if not self.loaded: self.load() return self.fastq.avg_quality def __call__(self, *args, **kwargs): if not self.loaded: self.load() self.runner.run(*args, **kwargs) def run_slurm(self, *args, **kwargs): if not self.loaded: self.load() return self.runner.run_slurm(*args, **kwargs) def pool_fastqc(self): """Run fastqc on the all the sequences of the pool""" if not self.loaded: self.load() self.fastq.fastqc(self.p.fastqc_dir) def create_outcomes(self): """Sort the sequences depending on their barcode status""" print "Seperating sequences according to barcodes" if not self.loaded: self.load() for o in self.outcomes: o.create() for r in self.fastq.parse_barcodes(): if len(r.matches) == 0: self.no_barcodes.add_pair( (r.fwd,r.rev)) elif len(r.matches) == 1: self.one_barcodes.add_pair( (r.fwd,r.rev)) elif r.matches[0].set == r.matches[1].set: self.same_barcodes.add_pair( (r.fwd,r.rev)) elif r.matches[0].sample is not r.matches[1].sample: self.bad_barcodes.add_pair( (r.fwd,r.rev)) else: self.good_barcodes.add_pair( (r.fwd,r.rev)) for o in self.outcomes: o.close() def create_samples(self): """Sort the sequences in different files according to their barcode number (if they have one)""" if not self.loaded: self.load() for sample in self.samples: sample.create() for r in tqdm(self.quality_reads.untrimmed.parse_barcodes()): r.first.sample.add_seq(r.read) for sample in self.samples: sample.close() def create_raw_samples(self): """Sort the sequences in different files according to their barcode before any other quality filtering such as read joining.""" if not self.loaded: self.load() for sample in self.samples: sample.raw.create() for r in tqdm(self.good_barcodes.parse_barcodes()): r.first.sample.raw.add_pair(r) for sample in self.samples: sample.raw.close() def check_fastq_version(self): """Let's make sure we are dealing with the Sanger encoding""" pass def make_pool_plots(self): """Call graphs that are in pool_plots.py""" if not self.loaded: self.load() for graph in self.graphs: graph.plot() @property_cached def loss_statistics(self): """This should be moved to the reporting file""" if not self.loaded: self.load() class MessageStat(object): def __str__(self): return self.msg % self.value def __init__(self, msg, value): self.msg = msg self.value = value class LossStatistics(object): def __init__(self, pool): self.pool = pool def __str__(self): return '\n'.join(map(str, self)) def __iter__(self): return iter((self.outcome, self.assembly, self.primers, self.n_filter, self.qual_filter, self.len_filter)) @property def outcome(self): return MessageStat("Good barcodes are only %f%% of total", (100*len(self.pool.good_barcodes)/self.pool.count)) @property def assembly(self): return MessageStat("Assembled is only %f%% of good barcodes", (100*len(self.pool.good_barcodes.assembled)/len(self.pool.good_barcodes))) @property def primers(self): return MessageStat("Good primers is only %f%% of assembled", (100*len(self.pool.good_barcodes.assembled.good_primers)/len(self.pool.good_barcodes.assembled))) @property def n_filter(self): return MessageStat("N filtered is only %f%% of good primers", (100*len(self.pool.good_barcodes.assembled.good_primers.n_filtered)/len(self.pool.good_barcodes.assembled.good_primers))) @property def qual_filter(self): return MessageStat("Qual filtered is only %f%% of N filtered", (100*len(self.pool.good_barcodes.assembled.good_primers.qual_filtered)/len(self.pool.good_barcodes.assembled.good_primers.n_filtered))) @property def len_filter(self): return MessageStat("Length filter is only %f%% of qual filtered", (100*len(self.pool.good_barcodes.assembled.good_primers.len_filtered)/len(self.pool.good_barcodes.assembled.good_primers.qual_filtered))) return LossStatistics(self) @property def json(self): """Regenerate the JSON string for every sample""" return ',\n\n'.join(s.json for s in self.samples)