def case_function(samples, obj):
sam = Samples() # object required to plot
num_bin_x, num_bin_y = (bin_x.get(), bin_y.get())
mi_text.set(mi_text.get() + '\nCalculating...')
not_selected = True
if plot_sample.get():
sam.plot_sample(samples, num_bin_x, num_bin_y)
not_selected = False
if plot_compare.get():
obj.plot_compare2(num_bin_x, num_bin_y, noise.get(), step.get(), significance.get())
not_selected = False
if plot_proposal.get():
sam.plot_propose(samples, num_bin_x, step.get())
not_selected = False
if plot_partition.get():
sam.plot_partition(samples, num_bin_x, num_bin_y)
not_selected = False
if plot_converge_in_samples.get():
converge_in_samples(num_bin_x, num_bin_y, obj)
not_selected = False
if samples_check.get():
samples_grid(num_bin_x, num_bin_y)
not_selected = False
if bin_selection.get():
bin_selection_validation(sigma=significance.get(), obj=obj)
not_selected = False
if cal_measures.get():
calculate_measures(samples, num_bin_x, num_bin_y)
not_selected = False
if not_selected:
mi_text.set('Select a plot or calculation')
def relate_sample( self, sample, seq_request_issue ):
'''
@param sample - The name of the sample to relate to the sequence request
@param seq_request_issue - Issue json of the sequence request to relate the sample to
'''
from samples import Samples
s = Samples( self.url, self.key )
created, sample_issue = s.get_or_create( sample )
# Set the relation
try:
self.set_relation( seq_request_issue, 'blocks', sample_issue )
except self.RelationExists as e:
# Already created
pass
def __init__(self, json_path, out_dir):
# Attributes #
self.out_dir = out_dir
self.json_path = json_path
# Parse #
self.info = load_json_path(self.json_path)
# Basic #
self.account = self.info['uppmax_id']
self.run_num = self.info['run_num']
self.run_label = self.info['run_id']
self.project_short_name = self.info['project']
self.project_long_name = self.info['project_name']
self.fwd_name = self.info['forward_reads']
self.rev_name = self.info['reverse_reads']
# Own attributes #
self.num = self.info['pool_num']
self.label = self.info['pool_id']
self.short_name = self.info['pool']
self.long_name = self.info['pool_name']
self.id_name = "run%03d-pool%02d" % (self.run_num, self.num)
# Special #
self.samples = Samples(self)
self.primers = TwoPrimers(self)
# Second init #
self.loaded = False
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.setup_layout()
self.setup_waveform()
self.setup_instructions()
self.layout.nextRow()
self.setup_spectrograph()
self.layout.nextRow()
self.setup_editor()
self.setup_index()
self.samples = Samples(self.waveform.data_available,
self.spectrograph.data_available)
self.copy_func_to_samples()
self.copy_func_to_editor_and_display()
self.setup_pyaudio()
self.stream.start_stream()
def on_sample_chosen(self):
size = self.sample_radio_id.get()
logging.info('Loading sample of size ' + str(size))
self.grid_size, self.pipe_ends = Samples.get_puzzle(str(size))
self.on_grid_size_changed()
self.grid_manager.load_maze(self.grid_size, self.pipe_ends)
self.ready_for_run = True
self.init_run()
def case_function(samples, obj):
sam = Samples()
print(samples)
numBinx, numBiny = Binx.get(), Biny.get()
#numBinx = 30
#numBiny = 30
print(numBinx, numBiny)
print(numBinx)
x, y = samples[:, 0], samples[:, 1]
MI = mutual_info(x, y, numBinx, numBiny)
ex = entropy(x, numBinx)
ey = entropy(y, numBiny)
R = norm_MI(x, y, numBinx, numBiny)
rxy = pearson_corr(x, y)
exy = entropyx_y(x, y, numBinx, numBiny)
eyx = entropyx_y(y, x, numBinx, numBiny)
MIE = MI_Entropy(x, y, numBinx, numBiny)
Ixy = propuesta_Ixy(x, y, numBinx, numBiny)
Iyx = propuesta_Iyx(x, y, numBinx, numBiny)
Ixy2 = propuesta2_Ixy(x, y, numBinx, numBiny)
Iyx2 = propuesta2_Iyx(x, y, numBinx, numBiny)
PMD = propuesta_mutual_dependency(x, y, numBinx, numBiny)
PMD2 = propuesta2_mutual_dependency(x, y, numBinx, numBiny)
# d_cor = d_corr(x,y)
# Mic = MIC(x,y)
mitexto.set(
'Entropy of x: ' + str(ex) + '\n' + 'Entropy of y: ' + str(ey) + '\n' +
'Mutual information: ' + str(MI) + '\n' +
'Mutual info with entropy: ' + str(MIE) + '\n' + '\n' +
'(Max=1)Normalized Mutual info: ' + str(R) + '\n' +
'(Max=1)Pearson Correlation: ' + str(rxy) + '\n' +
# '(Max=1)Distance Correlation: ' + str(d_cor) + '\n' +
# '(Max=1)MIC: ' + str(Mic) + '\n' +
'(Max=1)Mutual Dependency: ' + str(PMD) + '\n' +
'(Max=1)Mutual Dependency2: ' + str(PMD2) + '\n' + '\n' +
'Entropy of X|Y = ' + str(exy) + '\n' + 'Entropy of Y|X = ' +
str(eyx) + '\n' + '(Max=1)Information in Y of X: ' + str(Ixy) + '\n' +
'(Max=1)Information in X of Y: ' + str(Iyx) + '\n' +
'(Max=1)Information2 in Y of X: ' + str(Ixy2) + '\n' +
'(Max=1)Information2 in X of Y: ' + str(Iyx2))
if plot_sample.get():
sam.plot_sample(samples, numBinx, numBiny, step.get())
if plot_compare.get():
obj.plot_compare2(numBinx, numBiny, noise.get(), step.get())
if plot_propuse.get():
sam.plot_propose(samples, numBinx)
if plot_partition.get():
sam.plot_partition(samples, numBinx, numBiny)
if plot_converge_in_samples.get():
Converge_in_samples(numBinx, numBiny, obj)
if samples_check.get():
samples_grid(numBinx, numBiny)
def __init__(self, json_path, out_dir):
# Attributes #
self.out_dir = out_dir
self.json_path = json_path
# Parse #
with open(json_path) as handle: self.info = json.load(handle)
# Basic #
self.account = self.info['uppmax_id']
self.run_num = self.info['run_num']
self.run_label = self.info['run_id']
self.project_short_name = self.info['project']
self.project_long_name = self.info['project_name']
# Own attributes #
self.num = self.info['pool_num']
self.label = self.info['pool_id']
self.short_name = self.info['pool']
self.long_name = self.info['pool_name']
self.id_name = "run%03d-pool%02d" % (self.run_num, self.num)
# Special #
self.samples = Samples(self)
self.primers = TwoPrimers(self)
# Second init #
self.loaded = False
def find_by_shebang(cls, data):
"""
Public: Look up Languages by shebang line.
data - Array of tokens or String data to analyze.
Examples
Language.find_by_shebang("#!/bin/bash\ndate;")
# => [#<Language name="Bash">]
Returns the matching Language
"""
interpreter = Samples.interpreter_from_shebang(data)
return cls.interpreter_index.get(interpreter)
def save_wav(self) -> None:
# Save waveform
duration, ok = QtGui.QInputDialog.getInt(self, "Seconds of Audio:",
"Seconds:", 1, 0, MAX_VAL,
STEP_VAL)
if ok:
dialog = QtGui.QFileDialog()
path = dialog.getSaveFileName(self, 'Save File', os.getenv('HOME'),
'WAV (*.wav)')
if path[0] != '':
samples = Samples()
expression = copy.deepcopy(self.expression)
samples.set_expression(expression)
samples.gen_write_16(path[0], duration)
def plot_select():
ind = list1.curselection()
if list1.curselection() != ():
if ind[0] == 0:
sam = Samples(tipo=0)
samples = sam.get_sin(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 1:
sam = Samples(tipo=1)
samples = sam.get_square(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 2:
sam = Samples(tipo=2)
samples = sam.get_blur(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 3:
sam = Samples(tipo=3)
samples = sam.get_cuadratic(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 4:
sam = Samples(tipo=4)
samples = sam.get_diagonal_line(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 5:
sam = Samples(tipo=5)
samples = sam.get_horizontal_line(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 6:
sam = Samples(tipo=6)
samples = sam.get_vertical_line(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 7:
sam = Samples(tipo=7)
samples = sam.get_x(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 8:
sam = Samples(tipo=8)
samples = sam.get_circle(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 9:
sam = Samples(tipo=9)
samples = sam.get_curve_x(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 10:
sam = Samples(tipo=10)
samples = sam.get_diagonal_line2(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 11:
sam = Samples(tipo=11)
samples = sam.get_dependent(numberSam.get())
case_function(samples, sam)
elif ind[0] == 12:
sam = Samples(tipo=12)
samples = sam.get_independent(numberSam.get())
case_function(samples, sam)
elif ind[0] == 13:
sam = Samples(tipo=13)
samples = sam.get_corr(numberSam.get(), noise.get())
case_function(samples, sam)
elif ind[0] == 14:
sam = Samples(tipo=14)
samples = sam.get_file()
case_function(samples, sam)
else:
mitexto.set('Error')
else:
mitexto.set('Select a type')
class Pool(object):
"""An illumina MID is called here a 'pool'."""
all_paths = """
/samples/
/groups/
/graphs/
/fastqc/
/logs/
/quality_reads/
/quality_reads/fractions/
/info.json
"""
def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.id_name)
def __str__(self): return self.id_name
def __iter__(self): return iter(self.children)
def __len__(self): return self.count
def __getitem__(self, key): return self.samples[key]
@property
def seq_len(self): return len(self.fwd.first_read)
def __init__(self, json_path, out_dir):
# Attributes #
self.out_dir = out_dir
self.json_path = json_path
# Parse #
with open(json_path) as handle: self.info = json.load(handle)
# Basic #
self.account = self.info['uppmax_id']
self.run_num = self.info['run_num']
self.run_label = self.info['run_id']
self.project_short_name = self.info['project']
self.project_long_name = self.info['project_name']
# Own attributes #
self.num = self.info['pool_num']
self.label = self.info['pool_id']
self.short_name = self.info['pool']
self.long_name = self.info['pool_name']
self.id_name = "run%03d-pool%02d" % (self.run_num, self.num)
# Special #
self.samples = Samples(self)
self.primers = TwoPrimers(self)
# Second init #
self.loaded = False
def load(self):
"""A second __init__ that is delayed, solves some circular references"""
# Automatic paths #
self.base_dir = self.out_dir + self.id_name + '/'
self.p = AutoPaths(self.base_dir, self.all_paths)
# Make an alias to the json #
self.p.info_json.link_from(self.json_path, safe=True)
# Children #
self.samples.load()
# Raw file pairs #
self.fwd_path = home + "ILLUMITAG/INBOX/%s/%s/%s" % (self.run.label, self.label, self.info['forward_reads'])
self.rev_path = home + "ILLUMITAG/INBOX/%s/%s/%s" % (self.run.label, self.label, self.info['reverse_reads'])
self.fwd = FASTQ(self.fwd_path)
self.rev = FASTQ(self.rev_path)
self.fastq = PairedFASTQ(self.fwd.path, self.rev.path, self)
# Barcode length #
self.bar_len = self.samples.bar_len
# Make Outcomes #
self.no_barcodes = NoBarcode(self)
self.one_barcodes = OneBarcode(self)
self.same_barcodes = SameBarcode(self)
self.bad_barcodes = BadBarcode(self)
self.good_barcodes = GoodBarcode(self)
self.outcomes = (self.good_barcodes, self.no_barcodes, self.one_barcodes, self.same_barcodes, self.bad_barcodes)
self.children = self.outcomes
# The good reads #
self.quality_reads = QualityReads(self.good_barcodes.assembled.good_primers.len_filtered, self)
self.fractions = Fractions(self)
# Runner #
self.runner = PoolRunner(self)
# Graphs #
self.graphs = [getattr(pool_plots, cls_name)(self) for cls_name in pool_plots.__all__]
# Loaded #
self.loaded = True
# Return self for convenience #
return self
@property
def first(self): return self.children[0]
@property
def count(self):
if not self.loaded: self.load()
return self.fastq.count
@property
def avg_quality(self):
if not self.loaded: self.load()
return self.fastq.avg_quality
def __call__(self, *args, **kwargs):
if not self.loaded: self.load()
self.runner.run(*args, **kwargs)
def run_slurm(self, *args, **kwargs):
if not self.loaded: self.load()
return self.runner.run_slurm(*args, **kwargs)
def pool_fastqc(self):
"""Run fastqc on the all the sequences of the pool"""
if not self.loaded: self.load()
self.fastq.fastqc(self.p.fastqc_dir)
def create_outcomes(self):
"""Sort the sequences depending on their barcode status"""
if not self.loaded: self.load()
for o in self.outcomes: o.create()
for r in self.fastq.parse_barcodes():
if len(r.matches) == 0: self.no_barcodes.add_pair(r)
elif len(r.matches) == 1: self.one_barcodes.add_pair(r)
elif r.matches[0].set == r.matches[1].set: self.same_barcodes.add_pair(r)
elif r.matches[0].sample is not r.matches[1].sample: self.bad_barcodes.add_pair(r)
else: self.good_barcodes.add_pair(r)
for o in self.outcomes: o.close()
def create_samples(self):
"""Sort the sequences in different files according to their barcode number
(if they have one)"""
if not self.loaded: self.load()
for sample in self.samples: sample.create()
for r in tqdm(self.quality_reads.untrimmed.parse_barcodes()):
r.first.sample.add_read(r.read)
for sample in self.samples: sample.close()
def check_fastq_version(self):
"""Let's make sure we are dealing with the Sanger encoding"""
for o in self.outcomes:
assert fastqident.detect_encoding(o.fwd_path) == 'sanger'
assert fastqident.detect_encoding(o.rev_path) == 'sanger'
def make_pool_plots(self):
"""Call graphs that are in pool_plots.py"""
if not self.loaded: self.load()
for graph in self.graphs: graph.plot()
@property_cached
def loss_statistics(self):
"""This should be moved to the reporting file"""
class MessageStat(object):
def __init__(self, msg, value):
self.msg = msg
self.value = value
class LossStatistics(object):
def __init__(self, pool): self.pool = pool
def __iter__(self): return iter((self.outcome, self.assembly, self.primers, self.n_filter, self.qual_filter, self.len_filter))
@property
def outcome(self): return MessageStat("Good barcodes are only %f%% of total",
(100*len(self.pool.good_barcodes)/self.pool.count))
@property
def assembly(self): return MessageStat("Assembled is only %f%% of good barcodes",
(100*len(self.pool.good_barcodes.assembled)/len(self.pool.good_barcodes)))
@property
def primers(self): return MessageStat("Good primers is only %f%% of assembled",
(100*len(self.pool.good_barcodes.assembled.good_primers)/len(self.pool.good_barcodes.assembled)))
@property
def n_filter(self): return MessageStat("N filtered is only %f%% of good primers",
(100*len(self.pool.good_barcodes.assembled.good_primers.n_filtered)/len(self.pool.good_barcodes.assembled.good_primers)))
@property
def qual_filter(self): return MessageStat("Qual filtered is only %f%% of N filtered",
(100*len(self.pool.good_barcodes.assembled.good_primers.qual_filtered)/len(self.pool.good_barcodes.assembled.good_primers.n_filtered)))
@property
def len_filter(self): return MessageStat("Length filter is only %f%% of qual filtered",
(100*len(self.pool.good_barcodes.assembled.good_primers.len_filtered)/len(self.pool.good_barcodes.assembled.good_primers.qual_filtered)))
return LossStatistics(self)
@property
def json(self):
"""Regenerate the JSON string for every sample"""
return ',\n\n'.join(s.json for s in self.samples)
wall_nb = max(start_walls, target_walls)
score += wall_nb * 10
# increase score if the distance between the start and target is high
distance = self.shortest_path(start, target, self.original_id(i))
if distance > self.grid_size / 2:
score += distance * 2
if score > best_score:
best_pipe = i
best_score = score
logging.debug("Choosing next pipe {0}".format(
self.original_id(best_pipe)))
# flip the next pipe with the one we want to process
if best_pipe != self.curr_pipe:
original_best_pipe_id = self.pipes_mapping[best_pipe]
self.pipes_mapping[best_pipe] = self.pipes_mapping[self.curr_pipe]
self.pipes_mapping[self.curr_pipe] = original_best_pipe_id
if __name__ == "__main__":
setup_logging()
(size, pipes) = Samples.get_puzzle("12")
engine = ShortestPathEngine(size, pipes)
while not engine.solved:
for next_move in engine.next_moves():
logging.debug(next_move)
logging.info(engine.display())
class Madulator(pg.GraphicsView):
function_index = 1
generator = Generator(function_index)
expression = generator.random_function()
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.setup_layout()
self.setup_waveform()
self.setup_instructions()
self.layout.nextRow()
self.setup_spectrograph()
self.layout.nextRow()
self.setup_editor()
self.setup_index()
self.samples = Samples(self.waveform.data_available,
self.spectrograph.data_available)
self.copy_func_to_samples()
self.copy_func_to_editor_and_display()
self.setup_pyaudio()
self.stream.start_stream()
def setup_pyaudio(self) -> None:
self.pa = pa.PyAudio()
self.stream = self.pa.open(
format=pa.paUInt8,
channels=1,
rate=BITRATE,
output=True,
stream_callback=self.samples.pyaudio_callback)
def keyPressEvent(self, ev: QtGui.QKeyEvent) -> None:
key = ev.key()
if key == QtCore.Qt.Key.Key_Escape:
# Stop stream and terminate all
self.terminate_program()
elif key == QtCore.Qt.Key.Key_Space:
# Pause/Resume audio stream
self.pause_resume()
elif key == QtCore.Qt.Key.Key_R:
# Restart playback
self.restart_stream()
elif key == QtCore.Qt.Key.Key_W:
# Save waveform
self.save_wav()
elif key == QtCore.Qt.Key.Key_S:
# Save and download a function
self.save_func()
elif key == QtCore.Qt.Key.Key_L:
# Load a function from computer
self.load_func()
elif key == QtCore.Qt.Key.Key_BracketLeft:
# Index through older randomized functions
self.previous_index()
elif key == QtCore.Qt.Key.Key_BracketRight:
# Index through newer randomized functions
self.next_index()
elif key == QtCore.Qt.Key.Key_I:
# Go to a certain function index
self.get_index()
elif key == QtCore.Qt.Key.Key_V:
# Change expression into a Value entered by user
self.change_to_value()
elif key == QtCore.Qt.Key.Key_Comma:
self.samples.decrease_playback_speed()
self.update_index_speed_text()
elif key == QtCore.Qt.Key.Key_Period:
self.samples.increase_playback_speed()
self.update_index_speed_text()
elif key == QtCore.Qt.Key.Key_Equal:
self.samples.reset_playback_speed()
self.update_index_speed_text()
else:
# Change expression as dictated by user
is_editor_key = self.editor.new_key(ev.key())
if is_editor_key:
self.restart_stream()
else:
self.update_editor_info()
def update_index_speed_text(self):
self.index_text.setText(
"Function index: {:d} | Playback speed: {:.2f}".format(
self.function_index, self.samples.get_playback_speed()))
# Key press events
def terminate_program(self) -> None:
# Stop stream and terminate all
if self.stream.is_active():
self.stream.stop_stream()
self.stream.close()
self.pa.terminate()
QtCore.QCoreApplication.quit()
def pause_resume(self) -> None:
if self.stream.is_active():
self.stream.stop_stream()
else:
self.stream.start_stream()
def save_wav(self) -> None:
# Save waveform
duration, ok = QtGui.QInputDialog.getInt(self, "Seconds of Audio:",
"Seconds:", 1, 0, MAX_VAL,
STEP_VAL)
if ok:
dialog = QtGui.QFileDialog()
path = dialog.getSaveFileName(self, 'Save File', os.getenv('HOME'),
'WAV (*.wav)')
if path[0] != '':
samples = Samples()
expression = copy.deepcopy(self.expression)
samples.set_expression(expression)
samples.gen_write_16(path[0], duration)
def save_func(self) -> None:
# Save and download a function
dialog = QtGui.QFileDialog()
path = dialog.getSaveFileName(self, 'Save File', 'save/',
'MAD (*.mad)')
if path[0] != '':
with open(path[0], 'wb') as out_file:
exp = self.samples.get_expression()
pickle.dump(exp, out_file)
def load_func(self) -> None:
# Load a function from computer
if self.stream.is_active():
self.stream.stop_stream()
dialog = QtGui.QFileDialog()
path = dialog.getOpenFileName(self, 'Open File', 'save/',
"MAD (*.mad)")
if path[0] != '':
with open(path[0], 'rb') as in_file:
exp = pickle.load(in_file)
self.expression = exp
# Pass a copy to samples
self.copy_func_to_samples()
self.editor = Editor(exp)
# Pass a copy of the expression to editor and display
self.copy_func_to_editor_and_display()
self.samples.reset_playback_speed()
self.update_index_speed_text()
self.stream.start_stream()
def restart_stream(self) -> None:
# Stop stream and get reset function
if self.stream.is_active():
self.stream.stop_stream()
expression = self.editor.get_function()
self.expression = expression
# Pass a copy to samples, start stream, and display
self.copy_func_to_samples()
self.stream.start_stream()
self.update_editor_info()
def previous_index(self) -> None:
if self.function_index > 1:
self.function_index = self.function_index - 1
self.update_function_from_index()
self.samples.reset_playback_speed()
self.update_index_speed_text()
def next_index(self) -> None:
self.function_index = self.function_index + 1
self.update_function_from_index()
self.samples.reset_playback_speed()
self.update_index_speed_text()
def get_index(self) -> None:
val, ok = QtGui.QInputDialog.getInt(self, "Input Index:", "Index:", 1,
1, 2**30, 1)
if ok:
self.function_index = val
self.index_text.setText("Random function index: " +
str(self.function_index))
self.generator = Generator(self.function_index)
self.expression = self.generator.random_function()
self.copy_func_to_samples()
self.samples.reset_playback_speed()
self.copy_func_to_editor_and_display()
self.update_index_speed_text()
def change_to_value(self) -> None:
# Change expression into a Value entered by user
val = self.get_number()
if val != -1:
self.editor.create_value(val)
self.restart_stream()
self.update_editor_info()
def update_function_from_index(self) -> None:
self.generator = Generator(self.function_index)
self.expression = self.generator.random_function()
self.copy_func_to_samples()
self.copy_func_to_editor_and_display()
def update_editor_info(self) -> None:
selection = self.editor.get_selection()
expression = self.editor.get_function()
self.editor_text.setText(expression.html_tree(selection))
def copy_func_to_samples(self) -> None:
expression = copy.deepcopy(self.expression)
self.samples.set_expression(expression)
def copy_func_to_editor_and_display(self) -> None:
function = copy.deepcopy(self.expression)
self.editor.set_function(function)
self.update_editor_info()
def get_number(self) -> int:
val, ok = QtGui.QInputDialog.getInt(self, "Input Value:", "Value:",
DEFAULT_VAL, MIN_VAL, MAX_VAL,
STEP_VAL)
if ok:
return val
return -1
def setup_layout(self) -> None:
self.layout = pg.GraphicsLayout(border=(100, 100, 100))
self.setCentralItem(self.layout)
self.show()
self.setWindowTitle('MADulator')
self.resize(1024, 720)
def setup_waveform(self) -> None:
self.waveform = Waveform()
self.layout.addItem(self.waveform)
def setup_spectrograph(self) -> None:
self.spectrograph = SpectrogramWidget(BITRATE)
self.layout.addItem(self.spectrograph)
def setup_instructions(self) -> None:
text = '''
<h1>MADulator</h1>
<p><strong>Program keys</strong></p>
<ul>
<li>[<span style='color:white'>space</span>] pause or resume playback</li>
<li>[<span style='color:white'>r</span>] restart playback</li>
<li>[<span style='color:white'>[</span>] [<span style='color:white'>]</span>] prev/next random function</li>
<li>[<span style='color:white'>i</span>] goto random function index</li>
<li>[<span style='color:white'>,</span>] [<span style='color:white'>.</span>] decrease/increase playback speed 10%</li>
<li>[<span style='color:white'>=</span>] restore playback speed to normal</li>
<li>[<span style='color:white'>s</span>] save function to file</li>
<li>[<span style='color:white'>l</span>] load function from file</li>
<li>[<span style='color:white'>w</span>] save audio as .WAV file</li>
<li>[<span style='color:white'>ESC</span>] exit program</li>
</ul>
<p><strong>Function Editor keys</strong></p>
<ul>
<li>[<span style='color:white'>↑</span>] Editor: navigate to parent node</li>
<li>[<span style='color:white'>←</span>] [<span style='color:white'>→</span>] Editor: navigate to left/right child</li>
<li>[<span style='color:white'>v</span>] replace expression with value (integer)</li>
<li>[<span style='color:white'>t</span>] replace expression with variable</li>
<li>[<span style='color:white'>+</span>] replace expression with addition</li>
<li>[<span style='color:white'>-</span>] replace expression with subtraction</li>
<li>[<span style='color:white'>*</span>] replace expression with multiplication</li>
<li>[<span style='color:white'>/</span>] replace expression with integer division</li>
<li>[<span style='color:white'>%</span>] replace expression with modulo</li>
<li>[<span style='color:white'>&</span>] replace expression with bitwise AND</li>
<li>[<span style='color:white'>|</span>] replace expression with bitwise OR</li>
<li>[<span style='color:white'>^</span>] replace expression with bitwise XOR</li>
<li>[<span style='color:white'><</span>] [<span style='color:white'>></span>] replace expression with shift left/right</li>
</ul>
<p><small>Developed by Marko, Angelic, and Daniel</small></p>
'''
self.layout.addLabel(text, rowspan=2)
def setup_editor(self) -> None:
function = copy.deepcopy(self.expression)
self.editor = Editor(function)
self.editor_text = pg.LabelItem(name='Editor')
self.layout.addItem(self.editor_text)
self.update_editor_info()
def setup_index(self) -> None:
self.index_text = pg.LabelItem(name='Index')
self.layout.addItem(self.index_text)
self.index_text.setText(
"Function index: {:d} | Playback speed: 1.00".format(
self.function_index))
def command_parser():
'''
THIS PART CAN PARSER THE COMMAND PARAMETERS.
Value 0 is off, 1 is open.
'''
try:
source_path= ""
save_path = ""
# The default value of data_deal model.
iamges_deal = 0
size_x = 28
size_y = 28
channel = 3
# samples model parameters
samples = 0
no_replace = 0
replace = 0
number = 10000
# train and test the model
train_test = 0
dataset = 'mnist.pkl.gz'
config_path= "../config-example"
n_y = 10
m1 = 0
m2 = 0
opts, args = getopt.getopt(sys.argv[1:], "ho:", \
["help", \
"images_deal", \
"images_path=", \
"size_x=", \
"size_y=" , \
"channel=", \
"samples", \
"no_replace=", \
"replace=", \
"source_path=" , \
"number=", \
"save_path=", \
"train_test", \
"dataset=", \
"config_path=", \
"n_y=" , \
"m1=", \
"m2="
])
#print opts,args
for x,y in opts:
if x in ("-h","--help"):
print "-----------------------------------------------------------"
print "ensemb: command line brew"
print "usage : python ensemb.py <command> <args>"
print "---------------------------------------------------------------------"
print "images_deal : deal the image to pickle file"
print "---------- ensemb.py --images_deal "
print "---------- --source_path=/home/gd/ensemble-nn/data/image"
print "---------- --size_x=28"
print "---------- --size_y=28 "
print "---------- --channel=3"
print "---------------------------------------------------------------------"
print "samples : sample the samples from the datasets"
print "---------- ensemb.py --samples "
print "---------- --no_replace=1"
print "---------- --replace=0 "
print "---------- --source_path=./data/mnist.pkl.gz"
print "---------- --number=30000"
print "---------- --save_path=./data/mnist-sample-replace.pkl"
print "---------------------------------------------------------------------"
print "train_test : train and test the model"
print "---------- ensemb.py --train_test "
print "---------- --dataset =/home/gd/ensemble-nn/data/mnist.pkl.gz"
print "---------- --config_path=./config-example "
print "---------- --n_y =10"
print "---------- --m1 = 1"
print "---------------------------------------------------------------------"
# MODEL IMAGES_DEAL
if x in ("--images_deal"):
print "Deal the iamge .."
iamges_deal = 1
if x in ("--source_path"):
images_path = y
if x in ("--size_x"):
size_x = int(y)
if x in ("--size_y"):
size_y = int(y)
if x in ("--channel"):
channel = int(y)
# MODEL SAMPLES
if x in ("--samples"):
print "sampling .."
samples = 1
if x in ("--no_replace"):
no_replace = int(y)
if x in ("--replace"):
replace = int(y)
if x in ("--source_path"):
source_path = y
if x in ("--number"):
number = int(y)
if x in ("--save_path"):
save_path = y
# TRAIN AND TEST A MODEL
if x in ("--train_test"):
print "train and test .."
train_test = 1
if x in ("--dataset"):
dataset = y
if x in ("--config_path"):
config_path = y
if x in ("--n_y"):
n_y = int(y)
if x in ("--m1"):
m1 = int(y)
if x in ("--m2"):
m2 = int(y)
if iamges_deal == 1:
source_path="/home/gd/ensemble-nn/data/image"
DealDate_deal(image_path=source_path,size_x=size_x,size_y = size_y, channel =channel)
if samples == 1:
source_path="/home/gd/ensemble-nn/data/mnist.pkl.gz"
save_path="/home/gd/ensemble-nn/data/mnist-sample-replace.pkl"
if no_replace == 0 and replace == 0:
print "no_replace or replace ?"
if no_replace == 1 and replace == 1:
print "no_replace or replace ?"
if no_replace == 1:
instance = Samples()
instance.no_replace_sample(source_path, number, save_path)
if replace == 1:
instance = Samples.Samples()
instance.replace_sample(source_path, number, save_path)
if train_test == 1:
if m1 == 0 and m2 == 0:
print "adaboost-m1 or adaboost-m2 ?"
if m1 == 1 and m2 == 1:
print "adaboost-m1 or adaboost-m2 ?"
if m1 == 1 and m2 == 0:
instance = Controller()
instance.m1_controller(dirctory = config_path, dataset = dataset,n_y = n_y)
if m1 == 0 and m2 == 1:
instance = Controller()
instance.m2_controller(dirctory = config_path, dataset = dataset,n_y = n_y)
sys.exit()
except Exception as err:
print err
DIR: 'data/facedata',
HEIGHT: 68,
WIDTH: 61,
LABEL: 2,
PIXELS: None
},
DIGIT: {
DIR: 'data/digitdata',
HEIGHT: 20,
WIDTH: 29,
LABEL: 10,
PIXELS: None
}
}
samples = Samples(map.get(inp).get(DIR))
dataClassifier = DataClassifier(
map.get(inp).get(HEIGHT),
map.get(inp).get(WIDTH),
map.get(inp).get(LABEL),
map.get(inp).get(PIXELS))
perceptronClassifier = PerceptronClassifier(dataClassifier.FEATURES,
dataClassifier.LABELS)
samples.readFiles()
dataset = 0
featureValueListForAllTrainingImages, actualLabelForTrainingList = dataClassifier.extractFeatures(
samples.train_lines_itr, samples.train_labelsLines_itr)
TOTALDATASET = len(featureValueListForAllTrainingImages)
HEIGHT: 68,
WIDTH: 61,
LABEL: 2,
PIXELS: None
},
DIGIT: {
DIR: 'data/digitdata',
HEIGHT: 20,
WIDTH: 29,
LABEL: 10,
PIXELS: None
}
}
dataType = map.get(inp)
samples = Samples(dataType.get(DIR))
dataClassifier = DataClassifier(dataType.get(HEIGHT), dataType.get(WIDTH),
dataType.get(LABEL), dataType.get(PIXELS),
gridSize)
samples.readFiles()
''' Extracting Features from the Training Data '''
featureValueListForAllTrainingImages, actualLabelForTrainingList = \
dataClassifier.extractFeatures(samples.train_lines_itr, samples.train_labelsLines_itr)
runErrorList = []
runTimeList = []
runDataSetIncrements = []
for _ in range(0, 3):
def command_parser():
'''
THIS PART CAN PARSER THE COMMAND PARAMETERS.
Value 0 is off, 1 is open.
'''
try:
source_path = ""
save_path = ""
# The default value of data_deal model.
iamges_deal = 0
size_x = 28
size_y = 28
channel = 3
# samples model parameters
samples = 0
no_replace = 0
replace = 0
number = 10000
# train and test the model
train_test = 0
dataset = 'mnist.pkl.gz'
config_path = "../config-example"
n_y = 10
m1 = 0
m2 = 0
opts, args = getopt.getopt(sys.argv[1:], "ho:", \
["help", \
"images_deal", \
"images_path=", \
"size_x=", \
"size_y=" , \
"channel=", \
"samples", \
"no_replace=", \
"replace=", \
"source_path=" , \
"number=", \
"save_path=", \
"train_test", \
"dataset=", \
"config_path=", \
"n_y=" , \
"m1=", \
"m2="
])
#print opts,args
for x, y in opts:
if x in ("-h", "--help"):
print "-----------------------------------------------------------"
print "ensemb: command line brew"
print "usage : python ensemb.py <command> <args>"
print "---------------------------------------------------------------------"
print "images_deal : deal the image to pickle file"
print "---------- ensemb.py --images_deal "
print "---------- --source_path=/home/gd/ensemble-nn/data/image"
print "---------- --size_x=28"
print "---------- --size_y=28 "
print "---------- --channel=3"
print "---------------------------------------------------------------------"
print "samples : sample the samples from the datasets"
print "---------- ensemb.py --samples "
print "---------- --no_replace=1"
print "---------- --replace=0 "
print "---------- --source_path=./data/mnist.pkl.gz"
print "---------- --number=30000"
print "---------- --save_path=./data/mnist-sample-replace.pkl"
print "---------------------------------------------------------------------"
print "train_test : train and test the model"
print "---------- ensemb.py --train_test "
print "---------- --dataset =/home/gd/ensemble-nn/data/mnist.pkl.gz"
print "---------- --config_path=./config-example "
print "---------- --n_y =10"
print "---------- --m1 = 1"
print "---------------------------------------------------------------------"
# MODEL IMAGES_DEAL
if x in ("--images_deal"):
print "Deal the iamge .."
iamges_deal = 1
if x in ("--source_path"):
images_path = y
if x in ("--size_x"):
size_x = int(y)
if x in ("--size_y"):
size_y = int(y)
if x in ("--channel"):
channel = int(y)
# MODEL SAMPLES
if x in ("--samples"):
print "sampling .."
samples = 1
if x in ("--no_replace"):
no_replace = int(y)
if x in ("--replace"):
replace = int(y)
if x in ("--source_path"):
source_path = y
if x in ("--number"):
number = int(y)
if x in ("--save_path"):
save_path = y
# TRAIN AND TEST A MODEL
if x in ("--train_test"):
print "train and test .."
train_test = 1
if x in ("--dataset"):
dataset = y
if x in ("--config_path"):
config_path = y
if x in ("--n_y"):
n_y = int(y)
if x in ("--m1"):
m1 = int(y)
if x in ("--m2"):
m2 = int(y)
if iamges_deal == 1:
source_path = "/home/gd/ensemble-nn/data/image"
DealDate_deal(image_path=source_path,
size_x=size_x,
size_y=size_y,
channel=channel)
if samples == 1:
source_path = "/home/gd/ensemble-nn/data/mnist.pkl.gz"
save_path = "/home/gd/ensemble-nn/data/mnist-sample-replace.pkl"
if no_replace == 0 and replace == 0:
print "no_replace or replace ?"
if no_replace == 1 and replace == 1:
print "no_replace or replace ?"
if no_replace == 1:
instance = Samples()
instance.no_replace_sample(source_path, number, save_path)
if replace == 1:
instance = Samples.Samples()
instance.replace_sample(source_path, number, save_path)
if train_test == 1:
if m1 == 0 and m2 == 0:
print "adaboost-m1 or adaboost-m2 ?"
if m1 == 1 and m2 == 1:
print "adaboost-m1 or adaboost-m2 ?"
if m1 == 1 and m2 == 0:
instance = Controller()
instance.m1_controller(dirctory=config_path,
dataset=dataset,
n_y=n_y)
if m1 == 0 and m2 == 1:
instance = Controller()
instance.m2_controller(dirctory=config_path,
dataset=dataset,
n_y=n_y)
sys.exit()
except Exception as err:
print err
class Pool(object):
"""An illumina MID is called here a 'pool'."""
all_paths = """
/samples/
/groups/
/graphs/
/fastqc/
/logs/
/quality_reads/
/quality_reads/fractions/
/info.json
"""
kind = 'pool'
def __repr__(self): return '<%s object "%s">' % (self.__class__.__name__, self.id_name)
def __str__(self): return self.id_name
def __iter__(self): return iter(self.samples)
def __len__(self): return self.count
def __getitem__(self, key): return self.samples[key]
@property
def seq_len(self): return len(self.fwd.first_read)
def __init__(self, json_path, out_dir):
# Attributes #
self.out_dir = out_dir
self.json_path = json_path
# Parse #
self.info = load_json_path(self.json_path)
# Basic #
self.account = self.info['uppmax_id']
self.run_num = self.info['run_num']
self.run_label = self.info['run_id']
self.project_short_name = self.info['project']
self.project_long_name = self.info['project_name']
self.fwd_name = self.info['forward_reads']
self.rev_name = self.info['reverse_reads']
# Own attributes #
self.num = self.info['pool_num']
self.label = self.info['pool_id']
self.short_name = self.info['pool']
self.long_name = self.info['pool_name']
self.id_name = "run%03d-pool%02d" % (self.run_num, self.num)
# Special #
self.samples = Samples(self)
self.primers = TwoPrimers(self)
# Second init #
self.loaded = False
def load(self):
"""A second __init__ that is delayed, solves some circular references"""
# Raw file pairs #
self.fwd_path = home + "/proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.fwd_name)
self.rev_path = home + "/proj/%s/INBOX/%s/%s/%s" % (self.account, self.run_label, self.label, self.rev_name)
self.fwd = FASTQ(self.fwd_path)
self.rev = FASTQ(self.rev_path)
self.fastq = BarcodedPairedFASTQ(self.fwd.path, self.rev.path, self.samples)
# Check we can load it #
if not os.access('/proj/%s' % self.account, os.R_OK):
raise Exception("You don't have access to the project %s" % self.account)
if not self.fwd.exists: raise Exception("No file at '%s'" % self.fwd)
if not self.rev.exists: raise Exception("No file at '%s'" % self.rev)
# Automatic paths #
self.base_dir = self.out_dir + self.id_name + '/'
self.p = AutoPaths(self.base_dir, self.all_paths)
# Make an alias to the json #
self.p.info_json.link_from(self.json_path, safe=True)
# Children #
self.samples.load()
# Check there are 50 #
assert len(self.samples.children) == 50
# Barcode length #
self.bar_len = self.samples.bar_len
# Make Outcomes #
self.no_barcodes = NoBarcode(self)
self.one_barcodes = OneBarcode(self)
self.same_barcodes = SameBarcode(self)
self.bad_barcodes = BadBarcode(self)
self.good_barcodes = GoodBarcode(self)
self.outcomes = (self.good_barcodes, self.no_barcodes, self.one_barcodes, self.same_barcodes, self.bad_barcodes)
self.children = self.outcomes
# The good reads #
self.quality_reads = QualityReads(self.good_barcodes.assembled.good_primers.len_filtered, self)
self.fractions = Fractions(self)
# Runner #
self.runner = PoolRunner(self)
# Graphs #
self.graphs = [getattr(pool_plots, cls_name)(self) for cls_name in pool_plots.__all__]
# Loaded #
self.loaded = True
# Return self for convenience #
return self
@property
def first(self): return self.children[0]
@property
def count(self):
if not self.loaded: self.load()
return self.fastq.count
@property
def avg_quality(self):
if not self.loaded: self.load()
return self.fastq.avg_quality
def __call__(self, *args, **kwargs):
if not self.loaded: self.load()
self.runner.run(*args, **kwargs)
def run_slurm(self, *args, **kwargs):
if not self.loaded: self.load()
return self.runner.run_slurm(*args, **kwargs)
def pool_fastqc(self):
"""Run fastqc on the all the sequences of the pool"""
if not self.loaded: self.load()
self.fastq.fastqc(self.p.fastqc_dir)
def create_outcomes(self):
"""Sort the sequences depending on their barcode status"""
print "Seperating sequences according to barcodes"
if not self.loaded: self.load()
for o in self.outcomes: o.create()
for r in self.fastq.parse_barcodes():
if len(r.matches) == 0: self.no_barcodes.add_pair( (r.fwd,r.rev))
elif len(r.matches) == 1: self.one_barcodes.add_pair( (r.fwd,r.rev))
elif r.matches[0].set == r.matches[1].set: self.same_barcodes.add_pair( (r.fwd,r.rev))
elif r.matches[0].sample is not r.matches[1].sample: self.bad_barcodes.add_pair( (r.fwd,r.rev))
else: self.good_barcodes.add_pair( (r.fwd,r.rev))
for o in self.outcomes: o.close()
def create_samples(self):
"""Sort the sequences in different files according to their barcode number
(if they have one)"""
if not self.loaded: self.load()
for sample in self.samples: sample.create()
for r in tqdm(self.quality_reads.untrimmed.parse_barcodes()): r.first.sample.add_seq(r.read)
for sample in self.samples: sample.close()
def create_raw_samples(self):
"""Sort the sequences in different files according to their barcode
before any other quality filtering such as read joining."""
if not self.loaded: self.load()
for sample in self.samples: sample.raw.create()
for r in tqdm(self.good_barcodes.parse_barcodes()): r.first.sample.raw.add_pair(r)
for sample in self.samples: sample.raw.close()
def check_fastq_version(self):
"""Let's make sure we are dealing with the Sanger encoding"""
pass
def make_pool_plots(self):
"""Call graphs that are in pool_plots.py"""
if not self.loaded: self.load()
for graph in self.graphs: graph.plot()
@property_cached
def loss_statistics(self):
"""This should be moved to the reporting file"""
if not self.loaded: self.load()
class MessageStat(object):
def __str__(self): return self.msg % self.value
def __init__(self, msg, value):
self.msg = msg
self.value = value
class LossStatistics(object):
def __init__(self, pool): self.pool = pool
def __str__(self): return '\n'.join(map(str, self))
def __iter__(self): return iter((self.outcome, self.assembly, self.primers, self.n_filter, self.qual_filter, self.len_filter))
@property
def outcome(self): return MessageStat("Good barcodes are only %f%% of total",
(100*len(self.pool.good_barcodes)/self.pool.count))
@property
def assembly(self): return MessageStat("Assembled is only %f%% of good barcodes",
(100*len(self.pool.good_barcodes.assembled)/len(self.pool.good_barcodes)))
@property
def primers(self): return MessageStat("Good primers is only %f%% of assembled",
(100*len(self.pool.good_barcodes.assembled.good_primers)/len(self.pool.good_barcodes.assembled)))
@property
def n_filter(self): return MessageStat("N filtered is only %f%% of good primers",
(100*len(self.pool.good_barcodes.assembled.good_primers.n_filtered)/len(self.pool.good_barcodes.assembled.good_primers)))
@property
def qual_filter(self): return MessageStat("Qual filtered is only %f%% of N filtered",
(100*len(self.pool.good_barcodes.assembled.good_primers.qual_filtered)/len(self.pool.good_barcodes.assembled.good_primers.n_filtered)))
@property
def len_filter(self): return MessageStat("Length filter is only %f%% of qual filtered",
(100*len(self.pool.good_barcodes.assembled.good_primers.len_filtered)/len(self.pool.good_barcodes.assembled.good_primers.qual_filtered)))
return LossStatistics(self)
@property
def json(self):
"""Regenerate the JSON string for every sample"""
return ',\n\n'.join(s.json for s in self.samples)
