def step(self):
"""Runs CDN for one generation - must be called after fitness evaluation"""
self.pareto_population.update_pareto_front()
self.module_population.step(self)
for blueprint_individual in self.blueprint_population.individuals:
blueprint_individual.reset_number_of_module_species(
self.module_population.get_num_species(),
self.generation_number)
self.blueprint_population.step(self)
if Config.evolve_data_augmentations:
self.da_population.step(self)
for blueprint_individual in self.blueprint_population.individuals:
blueprint_individual.end_step(self)
for module_individual in self.module_population.individuals:
module_individual.end_step() # this also sets fitness to zero
DataManager.save_generation_state(self)
print(
'Module species distribution:', ', '.join(
[str(len(spc)) for spc in self.module_population.species]))
def create_cleaned():
# Clean up old data.
out_dir = './input/train_cleaned'
if os.path.exists(out_dir):
shutil.rmtree(out_dir)
os.makedirs(out_dir)
# Group by patient id.
patient_classes, imgs, imgs_mask = DataManager.read_train_images()
pid_data_dict = defaultdict(list)
for i, pid in enumerate(patient_classes):
pid_data_dict[pid].append((imgs[i], imgs_mask[i]))
imgs_cleaned = []
imgs_masks_cleaned = []
for pid in pid_data_dict:
imgs, masks = zip(*pid_data_dict[pid])
filtered_imgs, filtered_masks = _filter_inconsistent(imgs, masks)
print("Discarded {} from patient {}".format(
len(imgs) - len(filtered_imgs), pid))
imgs_cleaned.extend(filtered_imgs)
imgs_masks_cleaned.extend(filtered_masks)
imgs = np.array(imgs_cleaned)
imgs_mask = np.array(imgs_masks_cleaned)
print("Creating cleaned train dataset: {} items".format(len(imgs)))
mask_labels = [
1 if np.count_nonzero(mask) > 0 else 0 for mask in imgs_mask
]
DataManager.save_train_val_split(imgs,
imgs_mask,
"cleaned",
stratify=mask_labels)
def test_tags(self):
manager = DataManager(WordnetDataSource(),
['edible_fruit.n.01', 'wheeled_vehicle.n.01'])
assert manager.tag_index('edible_fruit.n.01') == 0
assert manager.tag_index('wheeled_vehicle.n.01') == 1
assert manager.tag(0) == 'edible_fruit.n.01'
assert manager.tag(1) == 'wheeled_vehicle.n.01'
def load_data(args):
dm = DataManager(args['type_learning'], args['name_data'])
if args['type_learning'][:2] == 'RL':
return dm.get_env()
elif args['type_learning'][:2] == 'SL':
return dm
def _setup_data_mgr(self):
log.info("_setup_data_mgr")
#
if not "data" in self.config:
log.error("no 'data' section in config")
return None
config_data = self.config["data"]
#
data_mgr = DataManager()
if not data_mgr.initialize(self.directory, config_data,
self.size_total, self.size_chunk):
return None
return data_mgr
def train(resume=False):
print('Loading data...')
X_train, X_val, y_train, y_val = DataManager.load_train_val_data("cleaned")
# X_train, y_train = filter_mask_presence(X_train, y_train)
# X_val, y_val = filter_mask_presence(X_train, y_train)
print('Creating and compiling model...')
model = build_model()
if resume:
model.load_weights('./results/net.hdf5')
model_checkpoint = ModelCheckpoint('./results/net.hdf5', monitor='val_loss', save_best_only=True)
print('Training on model')
model.summary()
batch_size = 64
nb_epoch = 200
train_generator = CustomDataGenerator(X_train, y_train, transform, batch_size)
# Use fixed samples instead to visualize histograms. There is currently a bug that prevents it
# when a val generator is used.
# Not aug val samples to keep the eval consistent.
val_generator = CustomDataGenerator(X_val, y_val, lambda x, y: transform(x, y, augment=False), batch_size)
model.fit_generator(train_generator, validation_data=val_generator, nb_val_samples=X_val.shape[0],
samples_per_epoch=X_train.shape[0], nb_epoch=nb_epoch, verbose=2,
callbacks=[model_checkpoint, reduce_lr, tb], max_q_size=1000)
def __init__(self, _stock_num, _target_month, _budget=50000000):
self.account = _AccountEnv(_budget)
self.stock_num = _stock_num
self.DataManager = DataManager.DataManager(self.stock_num)
self.day_list = self.DataManager.get_daylist_in_month(_target_month)
self.day_idx = 0
self.account = ForeignAccount()
class Managers:
def __init__(self, app, WSGI_PATH_PREFIX):
self.app = app
# self.cmpmgr = CompressionManager( app, compresslevel=9 )
self.DataManager = DataManager(app)
self.Engine = Engine(
{"postgres_db": self.DataManager.get_pg_connection_string()})
mngrs = {
# 'Compression': self.cmpmgr,
'DataManager': self.DataManager,
'Engine': self.Engine
}
self.app.config.setdefault('Managers', mngrs)
self.app.config['Managers'] = mngrs
print(' Registered Application Managers')
print(
'------------------------------------------------------------------'
)
def convert(self, data):
if isinstance(data, basestring):
return str(data)
elif isinstance(data, collections.Mapping):
return dict(map(self.convert, data.iteritems()))
elif isinstance(data, collections.Iterable):
return type(data)(map(self.convert, data))
else:
return data
def get_next_log_file_name(log_file_name=None):
if log_file_name is None:
log_file_name = "log"
file_exists_already = os.path.isfile(
os.path.join(DataManager.get_Logs_folder(), log_file_name + ".txt"))
if file_exists_already:
counter = 1
while file_exists_already:
file_exists_already = os.path.isfile(
os.path.join(DataManager.get_Logs_folder(),
log_file_name + "_" + repr(counter) + ".txt"))
counter += 1
counter -= 1
log_file_name = log_file_name + "_" + repr(counter)
return log_file_name + ".txt"
def __init__( self, nets=None, parent=None, width=3 ):
Frame.__init__( self, parent )
#data structure
self.netManager = NetManager()
self.dataManager = DataManager()
self.scheduler = Scheduler()
self.hosts = []
# UI
self.top = self.winfo_toplevel()
self.top.title( 'Mininet节点调度子系统' )
self.createMenuBar()
self.menubar = self.createFramBar()
self.createCfram()
cleanUpScreens()
self.pack( expand=True, fill='both' )
def run_binary_classification(datasource, tag1, tag2, verbose=True):
"""
Trains a binary classifier to distinguish between TaggedPhraseDataSource
phrases tagged with tag1 and phrases tagged with tag2.
This returns the accuracy of the binary classifier on the test
partition.
"""
vectorizer = lambda p: w2v.get_word_vector(p)
phrase_recognizer = lambda p: vectorizer(p) is not None
dmanager = DataManager(datasource, [tag1, tag2], vectorizer,
phrase_recognizer)
classifier = SimpleClassifier(300, 100, 2)
net = train_net(classifier,
dmanager,
batch_size=32,
n_epochs=30,
learning_rate=0.001,
verbose=False)
acc, misclassified = evaluate(net, dmanager, 'test')
if verbose:
for tag in sorted(dmanager.tags):
print('{} phrases are tagged with "{}".'.format(
dmanager.num_phrases[tag], tag))
print('\nERRORS:')
for (phrase, guessed, actual) in sorted(misclassified):
print('"{}" classified as "{}"\n actually: "{}".'.format(
phrase, guessed, actual))
print("\nOverall test accuracy = {:.2f}".format(acc))
return acc
def plot_tree_with_graphvis(self,
title="",
graph=None,
nodes_plotted=None,
file="temp",
view=False):
file = os.path.join(DataManager.get_Graphs_folder(), file)
if graph is None:
graph = graphviz.Digraph(comment=title)
if nodes_plotted is None:
nodes_plotted = set()
else:
if self in nodes_plotted:
return
nodes_plotted.add(self)
prefix = 'INPUT\n' if self.is_input_node() else (
"OUTPUT\n" if self.is_output_node() else '')
graph.node(self.traversal_id, (prefix + self.get_layer_type_name()),
style="filled",
fillcolor=self.get_plot_colour(include_shape=False))
for child in self.children:
child.plot_tree_with_graphvis(graph=graph,
nodes_plotted=nodes_plotted)
graph.edge(self.traversal_id, child.traversal_id)
if self.is_input_node():
graph.render(file, view=view)
def plot_tree_with_graphvis(self,
title="",
file="temp_g",
view=False,
graph=None,
return_graph_obj=False,
node_prefix=""):
file = os.path.join(DataManager.get_Graphs_folder(), file)
if graph is None:
graph = graphviz.Digraph(comment=title)
for node in self._nodes.values():
graph.node(node_prefix + str(node.id),
node.get_node_name(),
style="filled",
fillcolor="white")
for c in self._connections.values():
if not c.enabled():
continue
graph.edge(node_prefix + repr(c.from_node),
node_prefix + repr(c.to_node))
graph.render(file, view=view)
if return_graph_obj:
return graph
def __init__(self, app, WSGI_PATH_PREFIX):
self.app = app
# self.cmpmgr = CompressionManager( app, compresslevel=9 )
self.DataManager = DataManager(app)
self.Engine = Engine(
{"postgres_db": self.DataManager.get_pg_connection_string()})
mngrs = {
# 'Compression': self.cmpmgr,
'DataManager': self.DataManager,
'Engine': self.Engine
}
self.app.config.setdefault('Managers', mngrs)
self.app.config['Managers'] = mngrs
print(' Registered Application Managers')
print(
'------------------------------------------------------------------'
)
def load_vcf(self, fname=None) :
if not fname : fname = str(QtGui.QFileDialog.getOpenFileName(self, 'Open VCF file', self.lastopeneddir))
if not fname : return False
self.data = DataManager(self, fname)
self.setWindowTitle("Allele frequency viewer : %s" %os.path.basename(fname))
self.chromosome_combobox.addItems(self.data.get_chromosomes())
self.lastopeneddir = os.path.dirname(fname)
self.show_plot()
return True
def __init__(self, _stock_num):
self.DataManager = DataManager.DataManager(_stock_num)
self.name = DataUtils.get_stock_num_to_name(_stock_num)
self.num = _stock_num
self.queue_sell = asyncio.Queue()
self.stock_df = self.DataManager.get_dataframe()
self.count = 0
self.amount_foreign = 0
self.amount_agency = 0
def build_all(
self,
resource_limit=None
) -> Tuple[List[Process], ResourceManager, DataManager]:
models = self.create_process_model()
process_list = [x.id for x in models]
rm = ResourceManager(
self.create_resources(resource_limit=resource_limit))
dm = DataManager(self.create_data(), process_list=process_list)
return models, rm, dm
def plot_all_representatives(self):
graph = None
for spec in self.species:
graph = spec.representative.plot_tree_with_graphvis(
graph=graph,
return_graph_obj=True,
view=False,
node_prefix=repr(self.species.index(spec)) + "_")
file = os.path.join(DataManager.get_Graphs_folder(), "reps")
graph.render(file, view=True)
def get_all_run_names():
runs = set()
for subdir, dirs, files in os.walk(
os.path.join(DataManager.get_data_folder(), "runs")):
sub = subdir.split("runs")[1][1:].split("\\")[0].split("/")[0]
if sub == "":
continue
runs.add(sub)
return runs
def get_all_results_files_in_folder(dataset,folder):
files = set()
for subdir, dirs, files in os.walk(os.path.join(DataManager.get_results_folder(), dataset,"fully_train",folder)):
sub = subdir.split(folder)[1][1:].split("\\")[0].split("/")[0]
# print(sub)
if sub == "":
continue
files.add(sub)
return files
def get_all_results_folders(dataset):
folders = set()
# print("getting results from:",os.path.join(DataManager.get_results_folder(), dataset,"fully_train"))
for subdir, dirs, files in os.walk(os.path.join(DataManager.get_results_folder(), dataset,"fully_train")):
sub = subdir.split("fully_train")[1][1:].split("\\")[0].split("/")[0]
if sub == "":
continue
# print("result folder:",sub)
folders.add(sub)
return folders
def plot_best_graphs(run_name, top_num=1, view=True):
# try:
generation = DataManager.load_generation_state(run_name)
# except Exception as e:
# print(e)
# return
best_graphs = generation.pareto_population.get_highest_accuracy(
num=top_num, check_set=generation.pareto_population.best_members)
if top_num == 1:
best_graphs = [best_graphs]
for graph in best_graphs:
sample, _ = DataLoader.sample_data(Config.get_device(),
dataset=graph.dataset)
model = create_nn(graph, sample, feature_multiplier=1)
file_name = run_name + "_best" + repr(best_graphs.index(graph))
graph.plot_tree_with_graphvis(
view=view,
file=os.path.join(DataManager.get_data_folder(), "runs", run_name,
"Graphs", file_name))
def log_new_generation(accuracies,
generation_number,
second_objective_values=None,
third_objective_values=None,
write_summaries=False):
global generations
global log_file
generations.append(
GenerationData(accuracies, generation_number, second_objective_values,
third_objective_values))
if not os.path.exists(DataManager.get_Logs_folder()):
os.makedirs(DataManager.get_Logs_folder())
with open(os.path.join(DataManager.get_Logs_folder(), log_file),
"a+") as f:
if write_summaries:
f.write(generations[-1].get_summary() + "\n")
else:
f.write(generations[-1].get_data() + "\n")
def examine_generator():
X_train, X_val, y_train, y_val = DataManager.load_train_val_data("all")
from generator import CustomDataGenerator
from train import transform, filter_mask_presence
X_val, y_val = filter_mask_presence(X_val, y_val)
generator = CustomDataGenerator(X_val, y_val, lambda x, y: transform(x, y, augment=True), 32)
imgs, outs = generator.next()
for i in range(len(imgs)):
cv2.imshow("image", imgs[i, 0])
cv2.imshow("mask", outs['main_output'][i, 0])
cv2.waitKey(0)
def load_data(composed_transforms: transforms.Compose, split: str) -> DataLoader:
"""
Loads the data given the config.dataset
:param composed_transforms: the augmentations to apply to the dataset
:param split: either train or test, dataset returned also depends on config.fully_train -
'train' will return 42500 images for evolution, but 50000 for fully training.
'test' will return 7500 images for evolution, but 10000 for fully training.
Note: the validation/train split does not try balance the classes of data, it just takes the first n for the train
set and the remaining data goes to the validation set
"""
if split.lower() not in ['train', 'test']:
raise ValueError('Parameter split can be one of train, test or validation, but received: ' + str(split))
# when to load train set
train: bool = True if split == 'train' or (split == 'test' and not config.fully_train) else False
dataset_args = {
'root': DataManager.get_datasets_folder(),
'train': train,
'download': config.download_dataset,
'transform': composed_transforms
}
if config.dataset == 'mnist':
dataset = MNIST(**dataset_args)
elif config.dataset == 'cifar10':
dataset = CIFAR10(**dataset_args)
elif config.dataset == 'cifar100':
dataset = CIFAR100(**dataset_args)
elif config.dataset == 'fashionMnist':
dataset = FashionMNIST(**dataset_args)
elif config.dataset == 'custom':
dataset = get_generic_dataset(composed_transforms, train)
else:
raise ValueError('config.dataset can be one of mnist, cifar10 or custom, but received: ' + str(config.dataset))
if train and not config.fully_train:
# Splitting the train set into a train and valid set
train_size = int(len(dataset) * (1 - config.validation_split))
if split == 'train':
dataset = Subset(dataset, range(train_size))
elif split == 'test':
dataset = Subset(dataset, range(train_size, len(dataset)))
else:
raise Exception("")
# print(split, 'set size in', 'FT' if config.fully_train else 'evo', len(dataset))
# TODO: test num workers and pin memory
return DataLoader(dataset, batch_size=config.batch_size, shuffle=True, num_workers=0, pin_memory=False)
def __init__(self, app, WSGI_PATH_PREFIX):
self.app = app
# self.cmpmgr = CompressionManager( app, compresslevel=9 )
self.DataManager = DataManager()
mngrs = {
# 'Compression': self.cmpmgr,
'DataManager': self.DataManager
}
self.app.config.setdefault('Managers', mngrs)
self.app.config['Managers'] = mngrs
print(' Registered Application Managers')
print(
'------------------------------------------------------------------'
)
def __init__(self, _stock_num, _target_month, _global_actor,
_global_critic):
self.global_actor = _global_actor
self.global_critic = _global_critic
self.local_actor, self.local_critic = BaseModel.BaseModel(
64, 1).build_model()
self.local_actor._make_predict_function()
self.local_critic._make_predict_function()
self.local_actor.set_weights(self.global_actor.get_weights())
self.local_critic.set_weights(self.global_critic.get_weights())
self.EnvForeign = EnvForeign.ForeignEnv(_stock_num, _target_month)
dataManager = DataManager.DataManager(_stock_num)
self.day_list = dataManager.get_daylist_in_month(_target_month)
self.dataset_list = dataManager.get_dataset_in_month(_target_month)
def __init__(self):
QtGui.QMainWindow.__init__(self)
self.setAttribute(QtCore.Qt.WA_DeleteOnClose)
self.setWindowTitle("Structure Viewer")
self.data_manager = DataManager(self)
self.color_switch = None
self.lastopenfile = None
self.initUI()
self.main_widget.setFocus()
self.setCentralWidget(self.main_widget)
self.showMaximized()
self.update_plot()
def print_max_accuracies(dataset):
for run in get_all_results_folders(dataset):
run_name = run
if run_name in name_overrides:
run_name = name_overrides[run_name]
print(run_name)
for result_file in get_all_results_files_in_folder(dataset,run):
file_path = os.path.join(DataManager.get_results_folder(), dataset,"fully_train",run, result_file)
# print(result_file,file_path)
with open(file_path) as file:
lines = file.readlines()
accuracies = get_accuracies(lines)
# print(accuracies)
max_acc = max(accuracies)
config_name = result_file.replace(".txt", "")
print("\t", config_name, "max acc:", max_acc)
def load_data(composed_transforms: transforms.Compose,
split: str) -> DataLoader:
"""Loads the data given the config.dataset"""
if split not in ['train', 'test', 'validation']:
raise ValueError(
'Parameter split can be one of train, test or validation, but received: '
+ str(split))
train: bool = True if split == 'train' or split == 'validation' else False
dataset_args = {
'root': DataManager.get_datasets_folder(),
'train': train,
'download': True,
'transform': composed_transforms
}
if config.dataset == 'mnist':
dataset = MNIST(**dataset_args)
elif config.dataset == 'cifar10':
dataset = CIFAR10(**dataset_args)
elif config.dataset == 'custom':
dataset = get_generic_dataset(composed_transforms, train)
else:
raise ValueError(
'config.dataset can be one of mnist, cifar10 or custom, but received: '
+ str(config.dataset))
if train:
# Splitting the train set into a train and valid set
train_size = int(len(dataset) * (1 - config.validation_split))
validation_size = len(dataset) - train_size
train, valid = random_split(dataset, [train_size, validation_size])
if split == 'train':
dataset = train
else:
dataset = valid
# TODO: test num workers and pin memory
return DataLoader(dataset,
batch_size=config.batch_size,
shuffle=True,
num_workers=0,
pin_memory=False)
def load_date_from_log_file(run_name, iteration=0, summary=False):
global generations
generations = []
filename = os.path.join(
DataManager.get_Logs_folder(run_name),
"log" + (("_" + repr(iteration)) if iteration > 0 else ""))
log = open(filename + ".txt")
for gen in log:
gen_number = int(gen.split("{")[0].split(":")[1])
gen = gen.split("{")[1].split("}")[0]
objectives = gen.split("|")
o = 0
accuracies = second = third = None
for objective in objectives:
if summary:
name = objective.split("~")[0]
vals = objective.split("~")[1].split(";")
max = None
av = None
for val in vals:
if "max" in val.split(":")[0]:
max = float(val.split(":")[1])
if "average" in val.split(":")[0]:
av = float(val.split(":")[1])
else:
name = objective.split(":")[0]
vals = objective.split(":")[1]
if "accuracy" in name:
if o > 0:
print(
"warning, accuracy not the first objective in log",
filename)
return
accuracies = ast.literal_eval(vals)
elif o == 1:
second = ast.literal_eval(vals)
elif o == 2:
third = ast.literal_eval(vals)
else:
raise Exception("too many objectives in log", filename, o,
name)
o += 1
generations.append(
GenerationData(accuracies, gen_number, second, third))
def generate_submission():
# Load test images and preprocess for conv net.
print('Loading and processing test images')
imgs_test = DataManager.load_test_data()
total = imgs_test.shape[0]
imgs = np.ndarray(
(total, 1, DataManager.IMG_TARGET_ROWS, DataManager.IMG_TARGET_ROWS),
dtype=np.uint8)
i = 0
for img in imgs_test:
imgs[i] = preprocess(img)
i += 1
print('Loading network')
model = build_model()
model.load_weights('./results/net.hdf5')
print('Generating predictions')
masks, has_masks = model.predict(imgs, verbose=1)
ids = []
rles = []
for i in range(total):
# Zero out masks when there is no-nerve pred.
if has_masks[i, 0] < 0.5:
masks[i, 0] *= 0.
mask = post_process_mask(masks[i, 0])
rle = run_length_enc(mask)
rles.append(rle)
ids.append(i + 1)
if i % 100 == 0:
print('{}/{}'.format(i, total))
first_row = 'img,pixels'
file_name = 'results/submission_{}.csv'.format(str(datetime.now()))
with open(file_name, 'w+') as f:
f.write(first_row + '\n')
for i in range(total):
s = str(ids[i]) + ',' + rles[i]
f.write(s + '\n')
# Output file
target = "out.csv"
# Row containing the problem body
DATA_COL = 32
# Categories to be checked using ConceptNet
conceptnet_categories = ['car','animal','sport','object','food','subject','place']
# Categories to be checked using Stanford NER
NER_categories = ['location','time', 'person', 'organization' ,'money', 'percent','date']
# Categories based on given keywords
keyword_categories = [('geometry',['square','circle','rectangle','triangle','angle','quadrant'])]
# Categories based on values from other columns (limited only to categories in this program)
columnval_categories = [('real_world_reference',is_real_world,['car','animal','sport','object','food','subject','place','location','person','organization','money'])]
# Specify column in source data containing questions
dm = DataManager(source, target, DATA_COL, write_mode='a', batch = 100, pool_size = 8)
# Add categories into header
dm.add_header(conceptnet_categories,header_type = "conceptnet")
dm.add_header(NER_categories,header_type = "NER")
dm.add_header(keyword_categories,header_type = "keyword")
dm.add_header(columnval_categories,header_type = "columnval")
# Measure duration of running the program
start_time = time.time()
dm.process()
print time.time() - start_time, "seconds"
class ApplicationWindow(QtGui.QMainWindow):
def __init__(self):
QtGui.QMainWindow.__init__(self)
self.setAttribute(QtCore.Qt.WA_DeleteOnClose)
self.setWindowTitle("Allele frequency viewer")
self.data = None
self.genes = {}
self.lastopeneddir = os.path.expanduser("~")
self.initUI()
self.init_menu()
self.initRegionViewer()
self.main_widget.setFocus()
self.setCentralWidget(self.main_widget)
self.showMaximized()
self.tab_names = []
#self.test()
def test(self) :
vcf = "/Volumes/CORSAIR/PoolsTeo_jan2016/vsYJM326/320_1.rg.dedup.realigned.raw.info.var.vcf"
#vcf = "/media/jevanni/CORSAIR/PoolsTeo_jan2016/vsYJM326/320_1.rg.dedup.realigned.raw.info.var.vcf"
fastafile = "./Anno/yjm326.fasta"
gff3fname = "./Anno/sace_new.gff"
self.load_vcf(vcf)
self.load_annotation(fastafile, gff3fname)
def update_completer_gene(self) :
model = QtGui.QStringListModel()
model.setStringList(self.genes.keys())
self.completer.setModel(model)
def initUI(self) :
self.main_widget = QtGui.QWidget(self)
self.layout = QtGui.QVBoxLayout(self.main_widget)
self.layout_top_right = QtGui.QHBoxLayout()
self.layout_top = QtGui.QHBoxLayout()
# Search bar with auto completer
model = QtGui.QStringListModel()
model.setStringList(self.genes.keys())
self.completer = QtGui.QCompleter()
self.completer.setModel(model)
self.search_bar = QtGui.QLineEdit("Search gene")
self.search_bar.keyPressEvent = self.search_bar_key_event
self.search_bar.mousePressEvent = self.search_bar_mouse_event
self.search_bar.setCompleter(self.completer)
self.af_bar = QtGui.QLineEdit("Set af bar")
self.af_bar.keyPressEvent = self.af_bar_key_event
self.af_bar.mousePressEvent = self.af_bar_mouse_event
self.chromosome_combobox = QtGui.QComboBox()
self.start_bar = QtGui.QLineEdit("Start coordinate")
self.start_bar.keyPressEvent = self.start_bar_key_event
self.start_bar.mousePressEvent = self.start_bar_mouse_event
self.end_bar = QtGui.QLineEdit("End coordinate")
self.end_bar.keyPressEvent = self.end_bar_key_event
self.end_bar.mousePressEvent = self.end_bar_mouse_event
self.layout_top_right.addWidget(self.chromosome_combobox)
self.layout_top_right.addWidget(self.start_bar)
self.layout_top_right.addWidget(self.end_bar)
self.layout_top.addWidget(self.search_bar)
self.layout_top.addWidget(self.af_bar)
self.layout_top.addLayout(self.layout_top_right)
self.tab_widget = QtGui.QTabWidget(self)
self.tab_widget.currentChanged.connect(self.change_tab)
self.statut_bar = QtGui.QStatusBar(self)
self.layout.addLayout(self.layout_top)
self.layout.addWidget(self.tab_widget)
self.layout.addWidget(self.statut_bar)
def init_menu(self) :
self.file_menu = QtGui.QMenu("&Files", self)
self.file_menu.addAction("&Open files", self.load_annotation, QtCore.Qt.CTRL + QtCore.Qt.Key_O)
self.menuBar().addMenu(self.file_menu)
def initRegionViewer(self) :
screen = QtGui.QDesktopWidget()
second_screen = screen.screenGeometry(0)
self.regionViewer = RegionViewer(self)
self.regionViewer.move(second_screen.left(), second_screen.top())
def load_vcf(self, fname=None) :
if not fname : fname = str(QtGui.QFileDialog.getOpenFileName(self, 'Open VCF file', self.lastopeneddir))
if not fname : return False
self.data = DataManager(self, fname)
self.setWindowTitle("Allele frequency viewer : %s" %os.path.basename(fname))
self.chromosome_combobox.addItems(self.data.get_chromosomes())
self.lastopeneddir = os.path.dirname(fname)
self.show_plot()
return True
def load_annotation(self, fasta=None, annofname=None) :
loaded = True
if not self.data : loaded = self.load_vcf()
if not loaded : return
if not fasta : fasta = str(QtGui.QFileDialog.getOpenFileName(self, 'Open Fasta file', self.lastopeneddir))
if not annofname : annofname = str(QtGui.QFileDialog.getOpenFileName(self, 'Open annotation file', self.lastopeneddir))
if not fasta or not annofname : return
self.annotations = get_feature_gff3(annofname, fasta, self.data)
self.genes = self.annotations.get_dict_parents()
self.update_completer_gene()
self.lastopeneddir = os.path.dirname(annofname)
def show_plot(self, chromosome=None) :
name = chromosome if chromosome else "Genome"
if name not in self.tab_names :
canva = self.data.get_canva(chromosome=chromosome)
self.tab_widget.addTab(canva, name)
self.tab_names.append(name)
self.tab_widget.setCurrentIndex(self.tab_names.index(name))
self.get_current_tabwidget().update_info()
def get_current_tabwidget(self) :
return self.tab_widget.currentWidget()
def update_statut_whole(self, chromosome=None) :
info = self.data.get_statut(chromosome=chromosome)
def search_bar_mouse_event(self, event) :
if event.button() == 1 and self.search_bar.text() == "Search gene" :
self.search_bar.setText("")
def search_bar_key_event(self, event) :
if event.text() == '\r' : self.zone_gene(self.search_bar.text())
else : QtGui.QLineEdit.keyPressEvent(self.search_bar, event)
def zone_gene(self, gene_name) :
annotation = self.genes.get(str(gene_name), None)
if not annotation : return
chromosome, start, end = annotation["seqname"], annotation["start"], annotation["end"]
self.show_plot(chromosome=chromosome)
self.get_current_tabwidget().draw_region(start, end)
self.get_current_tabwidget().send_info()
def af_bar_key_event(self, event) :
if event.text() == '\r' : self.set_bar(self.af_bar.text())
else : QtGui.QLineEdit.keyPressEvent(self.af_bar, event)
def af_bar_mouse_event(self, event) :
if event.button() == 1 and self.af_bar.text() == "Set af bar" :
self.af_bar.setText("")
def start_bar_mouse_event(self, event) :
if event.button() == 1 and self.start_bar.text() == "Start coordinate" :
self.start_bar.setText("")
def end_bar_mouse_event(self, event) :
if event.button() == 1 and self.end_bar.text() == "End coordinate" :
self.end_bar.setText("")
def start_bar_key_event(self, event) :
if event.text() == '\r' : self.set_coordinate_manualy()
else : QtGui.QLineEdit.keyPressEvent(self.start_bar, event)
def end_bar_key_event(self, event) :
if event.text() == '\r' : self.set_coordinate_manualy()
else : QtGui.QLineEdit.keyPressEvent(self.end_bar, event)
def set_coordinate_manualy(self) :
chromosome = str(self.chromosome_combobox.currentText())
start = str(self.start_bar.text())
end = str(self.end_bar.text())
try :
start, end = int(start), int(end)
except ValueError :
return
self.show_plot(chromosome=chromosome)
self.get_current_tabwidget().draw_region(start, end)
def set_bar(self, value) :
self.get_current_tabwidget().remove_line()
try : value = float(value)
except ValueError : return
if value > 1 or value < 0 : return
self.get_current_tabwidget().draw_bar(value)
def change_tab(self) :
self.get_current_tabwidget().update_info()
def closeEvent(self, ce) :
if self.regionViewer :
self.regionViewer.close()
class ApplicationWindow(QtGui.QMainWindow):
def __init__(self):
QtGui.QMainWindow.__init__(self)
self.setAttribute(QtCore.Qt.WA_DeleteOnClose)
self.setWindowTitle("Structure Viewer")
self.data_manager = DataManager(self)
self.color_switch = None
self.lastopenfile = None
self.initUI()
self.main_widget.setFocus()
self.setCentralWidget(self.main_widget)
self.showMaximized()
self.update_plot()
def initUI(self) :
self.main_widget = QtGui.QWidget(self)
self.main_widget.move(10, 20)
self.main_widget.resize(960, 730)
self.layout = QtGui.QVBoxLayout(self.main_widget)
self.initMenu()
self.error_dialog_load = QtGui.QErrorMessage()
def initMenu(self) :
self.file_menu = QtGui.QMenu("&Options", self)
self.file_menu.addAction("&Open file", self.open_file, QtCore.Qt.CTRL + QtCore.Qt.Key_O)
self.file_menu.addAction("&Save fig", self.save_file, QtCore.Qt.CTRL + QtCore.Qt.Key_S)
self.file_menu.addAction('&Graph settings', self.switch_color, QtCore.Qt.CTRL + QtCore.Qt.Key_G)
self.action_border = QtGui.QAction("Border", self)
self.action_labels = QtGui.QAction("Show all labels", self)
self.action_border.changed.connect(self.update_plot)
self.action_border.setCheckable(True)
self.action_labels.changed.connect(self.update_plot)
self.action_labels.setCheckable(True)
self.file_menu.addAction(self.action_border)
self.file_menu.addAction(self.action_labels)
self.menuBar().addMenu(self.file_menu)
def open_file(self) :
if self.lastopenfile :
fnames = QtGui.QFileDialog.getOpenFileNames(self, 'Open file', dirname(self.lastopenfile))
else :
fnames = QtGui.QFileDialog.getOpenFileNames(self, 'Open file')
for fname in fnames :
try :
self.load_file(fname)
except SVInternalException as error :
self.error_dialog_load.showMessage(str(fname) + " : " + str(error))
return
self.lastopenfile = str(fname)
self.update_plot()
def load_file(self, fname) :
self.data_manager.add_structure_fname(fname)
def update_plot(self) :
# Generate an error, I don't really know why
border = self.is_border_checked()
show_all_labels = self.is_all_labels_checked()
self.canva = self.data_manager.get_canva(edge=border, show_all_labels=show_all_labels)
if not self.canva : return
for i in reversed(range(self.layout.count())):
if isinstance(self.layout.itemAt(i), QtGui.QWidgetItem) :
self.layout.itemAt(i).widget().setParent(None)
self.canva.setParent(self.main_widget)
self.canva.setFocus()
self.layout.addWidget(self.canva)
self.show()
def is_border_checked(self) :
return self.action_border.isChecked()
def is_all_labels_checked(self) :
return self.action_labels.isChecked()
def popup(self, data, row, column) :
self.menu = QtGui.QMenu(self)
go_up_action = QtGui.QAction("Go up", self)
go_up_action.triggered.connect(partial(self.change_row, data, column, 1))
go_down_action = QtGui.QAction("Go down", self)
go_down_action.triggered.connect(partial(self.change_row, data, column, -1))
sort_action = QtGui.QAction("Sort", self)
sort_action.triggered.connect(partial(self.sort_pop, data, column))
self.menu.addAction(go_up_action)
self.menu.addAction(go_down_action)
self.menu.addAction(sort_action)
self.menu.popup(QtGui.QCursor.pos())
def switch_color(self) :
self.color_switch = SwitchColorWindow(self)
def sort_pop(self, data, column) :
data.reorder_pop(column)
self.data_manager.uniform_names(data)
self.update_plot()
def change_row(self, data, column, level) :
if data.reorder_column(column, level) :
self.update_plot()
def save_file(self) :
if self.lastopenfile :
fname = QtGui.QFileDialog.getSaveFileName(self, 'Open file', dirname(self.lastopenfile))
else :
fname = QtGui.QFileDialog.getSaveFileName(self, 'Open file')
self.data_manager.save(str(fname))
def closeEvent(self, ce) :
if self.color_switch :
self.color_switch.terminate()
self.close()