Esempio n. 1
0
    def step(self):
        """Runs CDN for one generation - must be called after fitness evaluation"""
        self.pareto_population.update_pareto_front()

        self.module_population.step(self)
        for blueprint_individual in self.blueprint_population.individuals:
            blueprint_individual.reset_number_of_module_species(
                self.module_population.get_num_species(),
                self.generation_number)

        self.blueprint_population.step(self)

        if Config.evolve_data_augmentations:
            self.da_population.step(self)

        for blueprint_individual in self.blueprint_population.individuals:
            blueprint_individual.end_step(self)

        for module_individual in self.module_population.individuals:
            module_individual.end_step()  # this also sets fitness to zero

        DataManager.save_generation_state(self)

        print(
            'Module species distribution:', ', '.join(
                [str(len(spc)) for spc in self.module_population.species]))
Esempio n. 2
0
def create_cleaned():
    # Clean up old data.
    out_dir = './input/train_cleaned'
    if os.path.exists(out_dir):
        shutil.rmtree(out_dir)
    os.makedirs(out_dir)

    # Group by patient id.
    patient_classes, imgs, imgs_mask = DataManager.read_train_images()
    pid_data_dict = defaultdict(list)
    for i, pid in enumerate(patient_classes):
        pid_data_dict[pid].append((imgs[i], imgs_mask[i]))

    imgs_cleaned = []
    imgs_masks_cleaned = []
    for pid in pid_data_dict:
        imgs, masks = zip(*pid_data_dict[pid])
        filtered_imgs, filtered_masks = _filter_inconsistent(imgs, masks)
        print("Discarded {} from patient {}".format(
            len(imgs) - len(filtered_imgs), pid))
        imgs_cleaned.extend(filtered_imgs)
        imgs_masks_cleaned.extend(filtered_masks)

    imgs = np.array(imgs_cleaned)
    imgs_mask = np.array(imgs_masks_cleaned)
    print("Creating cleaned train dataset: {} items".format(len(imgs)))
    mask_labels = [
        1 if np.count_nonzero(mask) > 0 else 0 for mask in imgs_mask
    ]
    DataManager.save_train_val_split(imgs,
                                     imgs_mask,
                                     "cleaned",
                                     stratify=mask_labels)
 def test_tags(self):
     manager = DataManager(WordnetDataSource(),
                           ['edible_fruit.n.01', 'wheeled_vehicle.n.01'])
     assert manager.tag_index('edible_fruit.n.01') == 0
     assert manager.tag_index('wheeled_vehicle.n.01') == 1
     assert manager.tag(0) == 'edible_fruit.n.01'
     assert manager.tag(1) == 'wheeled_vehicle.n.01'
Esempio n. 4
0
def load_data(args):
    dm = DataManager(args['type_learning'], args['name_data'])

    if args['type_learning'][:2] == 'RL':
        return dm.get_env()
    elif args['type_learning'][:2] == 'SL':
        return dm
Esempio n. 5
0
 def _setup_data_mgr(self):
     log.info("_setup_data_mgr")
     #
     if not "data" in self.config:
         log.error("no 'data' section in config")
         return None
     config_data = self.config["data"]
     #
     data_mgr = DataManager()
     if not data_mgr.initialize(self.directory, config_data,
                                self.size_total, self.size_chunk):
         return None
     return data_mgr
def train(resume=False):
    print('Loading data...')
    X_train, X_val, y_train, y_val = DataManager.load_train_val_data("cleaned")
    # X_train, y_train = filter_mask_presence(X_train, y_train)
    # X_val, y_val = filter_mask_presence(X_train, y_train)

    print('Creating and compiling model...')
    model = build_model()
    if resume:
        model.load_weights('./results/net.hdf5')
    model_checkpoint = ModelCheckpoint('./results/net.hdf5', monitor='val_loss', save_best_only=True)

    print('Training on model')
    model.summary()
    batch_size = 64
    nb_epoch = 200

    train_generator = CustomDataGenerator(X_train, y_train, transform, batch_size)

    # Use fixed samples instead to visualize histograms. There is currently a bug that prevents it
    # when a val generator is used.
    # Not aug val samples to keep the eval consistent.
    val_generator = CustomDataGenerator(X_val, y_val, lambda x, y: transform(x, y, augment=False), batch_size)

    model.fit_generator(train_generator, validation_data=val_generator, nb_val_samples=X_val.shape[0],
                        samples_per_epoch=X_train.shape[0], nb_epoch=nb_epoch, verbose=2,
                        callbacks=[model_checkpoint, reduce_lr, tb], max_q_size=1000)
Esempio n. 7
0
 def __init__(self, _stock_num, _target_month, _budget=50000000):
     self.account = _AccountEnv(_budget)
     self.stock_num = _stock_num
     self.DataManager = DataManager.DataManager(self.stock_num)
     self.day_list = self.DataManager.get_daylist_in_month(_target_month)
     self.day_idx = 0
     self.account = ForeignAccount()
Esempio n. 8
0
class Managers:
    def __init__(self, app, WSGI_PATH_PREFIX):
        self.app = app
        # self.cmpmgr = CompressionManager( app, compresslevel=9 )
        self.DataManager = DataManager(app)
        self.Engine = Engine(
            {"postgres_db": self.DataManager.get_pg_connection_string()})
        mngrs = {
            # 'Compression': self.cmpmgr,
            'DataManager': self.DataManager,
            'Engine': self.Engine
        }
        self.app.config.setdefault('Managers', mngrs)
        self.app.config['Managers'] = mngrs
        print('             Registered Application Managers')
        print(
            '------------------------------------------------------------------'
        )

    def convert(self, data):
        if isinstance(data, basestring):
            return str(data)
        elif isinstance(data, collections.Mapping):
            return dict(map(self.convert, data.iteritems()))
        elif isinstance(data, collections.Iterable):
            return type(data)(map(self.convert, data))
        else:
            return data
Esempio n. 9
0
def get_next_log_file_name(log_file_name=None):
    if log_file_name is None:
        log_file_name = "log"

    file_exists_already = os.path.isfile(
        os.path.join(DataManager.get_Logs_folder(), log_file_name + ".txt"))
    if file_exists_already:
        counter = 1
        while file_exists_already:
            file_exists_already = os.path.isfile(
                os.path.join(DataManager.get_Logs_folder(),
                             log_file_name + "_" + repr(counter) + ".txt"))
            counter += 1
        counter -= 1
        log_file_name = log_file_name + "_" + repr(counter)
    return log_file_name + ".txt"
Esempio n. 10
0
    def __init__( self, nets=None, parent=None, width=3 ):
        Frame.__init__( self, parent )

        #data structure
        self.netManager = NetManager()
        self.dataManager = DataManager()
        self.scheduler = Scheduler()
        self.hosts = []
        # UI
        self.top = self.winfo_toplevel()
        self.top.title( 'Mininet节点调度子系统' )
        self.createMenuBar()
        self.menubar = self.createFramBar()
        self.createCfram()
        cleanUpScreens()
        self.pack( expand=True, fill='both' )
def run_binary_classification(datasource, tag1, tag2, verbose=True):
    """
    Trains a binary classifier to distinguish between TaggedPhraseDataSource
    phrases tagged with tag1 and phrases tagged with tag2.
    
    This returns the accuracy of the binary classifier on the test
    partition.
    
    """
    vectorizer = lambda p: w2v.get_word_vector(p)
    phrase_recognizer = lambda p: vectorizer(p) is not None
    dmanager = DataManager(datasource, [tag1, tag2], vectorizer,
                           phrase_recognizer)
    classifier = SimpleClassifier(300, 100, 2)
    net = train_net(classifier,
                    dmanager,
                    batch_size=32,
                    n_epochs=30,
                    learning_rate=0.001,
                    verbose=False)
    acc, misclassified = evaluate(net, dmanager, 'test')
    if verbose:
        for tag in sorted(dmanager.tags):
            print('{} phrases are tagged with "{}".'.format(
                dmanager.num_phrases[tag], tag))
        print('\nERRORS:')
        for (phrase, guessed, actual) in sorted(misclassified):
            print('"{}" classified as "{}"\n  actually: "{}".'.format(
                phrase, guessed, actual))
        print("\nOverall test accuracy = {:.2f}".format(acc))
    return acc
Esempio n. 12
0
    def plot_tree_with_graphvis(self,
                                title="",
                                graph=None,
                                nodes_plotted=None,
                                file="temp",
                                view=False):
        file = os.path.join(DataManager.get_Graphs_folder(), file)

        if graph is None:
            graph = graphviz.Digraph(comment=title)

        if nodes_plotted is None:
            nodes_plotted = set()
        else:
            if self in nodes_plotted:
                return

        nodes_plotted.add(self)

        prefix = 'INPUT\n' if self.is_input_node() else (
            "OUTPUT\n" if self.is_output_node() else '')
        graph.node(self.traversal_id, (prefix + self.get_layer_type_name()),
                   style="filled",
                   fillcolor=self.get_plot_colour(include_shape=False))
        for child in self.children:
            child.plot_tree_with_graphvis(graph=graph,
                                          nodes_plotted=nodes_plotted)
            graph.edge(self.traversal_id, child.traversal_id)

        if self.is_input_node():
            graph.render(file, view=view)
Esempio n. 13
0
    def plot_tree_with_graphvis(self,
                                title="",
                                file="temp_g",
                                view=False,
                                graph=None,
                                return_graph_obj=False,
                                node_prefix=""):

        file = os.path.join(DataManager.get_Graphs_folder(), file)

        if graph is None:
            graph = graphviz.Digraph(comment=title)

        for node in self._nodes.values():
            graph.node(node_prefix + str(node.id),
                       node.get_node_name(),
                       style="filled",
                       fillcolor="white")

        for c in self._connections.values():
            if not c.enabled():
                continue
            graph.edge(node_prefix + repr(c.from_node),
                       node_prefix + repr(c.to_node))

        graph.render(file, view=view)
        if return_graph_obj:
            return graph
Esempio n. 14
0
 def __init__(self, app, WSGI_PATH_PREFIX):
     self.app = app
     # self.cmpmgr = CompressionManager( app, compresslevel=9 )
     self.DataManager = DataManager(app)
     self.Engine = Engine(
         {"postgres_db": self.DataManager.get_pg_connection_string()})
     mngrs = {
         # 'Compression': self.cmpmgr,
         'DataManager': self.DataManager,
         'Engine': self.Engine
     }
     self.app.config.setdefault('Managers', mngrs)
     self.app.config['Managers'] = mngrs
     print('             Registered Application Managers')
     print(
         '------------------------------------------------------------------'
     )
Esempio n. 15
0
	def load_vcf(self, fname=None) :
		if not fname : fname = str(QtGui.QFileDialog.getOpenFileName(self, 'Open VCF file', self.lastopeneddir))
		if not fname : return False
		self.data = DataManager(self, fname)
		self.setWindowTitle("Allele frequency viewer : %s" %os.path.basename(fname))
		self.chromosome_combobox.addItems(self.data.get_chromosomes())
		self.lastopeneddir = os.path.dirname(fname)
		self.show_plot()
		return True
Esempio n. 16
0
 def __init__(self, _stock_num):
     self.DataManager = DataManager.DataManager(_stock_num)
     self.name = DataUtils.get_stock_num_to_name(_stock_num)
     self.num = _stock_num
     self.queue_sell = asyncio.Queue()
     self.stock_df = self.DataManager.get_dataframe()
     self.count = 0
     self.amount_foreign = 0
     self.amount_agency = 0
Esempio n. 17
0
 def build_all(
     self,
     resource_limit=None
 ) -> Tuple[List[Process], ResourceManager, DataManager]:
     models = self.create_process_model()
     process_list = [x.id for x in models]
     rm = ResourceManager(
         self.create_resources(resource_limit=resource_limit))
     dm = DataManager(self.create_data(), process_list=process_list)
     return models, rm, dm
Esempio n. 18
0
 def plot_all_representatives(self):
     graph = None
     for spec in self.species:
         graph = spec.representative.plot_tree_with_graphvis(
             graph=graph,
             return_graph_obj=True,
             view=False,
             node_prefix=repr(self.species.index(spec)) + "_")
     file = os.path.join(DataManager.get_Graphs_folder(), "reps")
     graph.render(file, view=True)
Esempio n. 19
0
def get_all_run_names():
    runs = set()
    for subdir, dirs, files in os.walk(
            os.path.join(DataManager.get_data_folder(), "runs")):
        sub = subdir.split("runs")[1][1:].split("\\")[0].split("/")[0]
        if sub == "":
            continue
        runs.add(sub)

    return runs
def get_all_results_files_in_folder(dataset,folder):
    files = set()
    for subdir, dirs, files in os.walk(os.path.join(DataManager.get_results_folder(), dataset,"fully_train",folder)):
        sub = subdir.split(folder)[1][1:].split("\\")[0].split("/")[0]
        # print(sub)
        if sub == "":
            continue
        files.add(sub)

    return files
def get_all_results_folders(dataset):
    folders = set()
    # print("getting results from:",os.path.join(DataManager.get_results_folder(), dataset,"fully_train"))
    for subdir, dirs, files in os.walk(os.path.join(DataManager.get_results_folder(), dataset,"fully_train")):
        sub = subdir.split("fully_train")[1][1:].split("\\")[0].split("/")[0]
        if sub == "":
            continue
        # print("result folder:",sub)
        folders.add(sub)

    return folders
Esempio n. 22
0
def plot_best_graphs(run_name, top_num=1, view=True):
    # try:
    generation = DataManager.load_generation_state(run_name)
    # except Exception as e:
    #     print(e)
    #     return

    best_graphs = generation.pareto_population.get_highest_accuracy(
        num=top_num, check_set=generation.pareto_population.best_members)
    if top_num == 1:
        best_graphs = [best_graphs]
    for graph in best_graphs:
        sample, _ = DataLoader.sample_data(Config.get_device(),
                                           dataset=graph.dataset)
        model = create_nn(graph, sample, feature_multiplier=1)
        file_name = run_name + "_best" + repr(best_graphs.index(graph))
        graph.plot_tree_with_graphvis(
            view=view,
            file=os.path.join(DataManager.get_data_folder(), "runs", run_name,
                              "Graphs", file_name))
Esempio n. 23
0
def log_new_generation(accuracies,
                       generation_number,
                       second_objective_values=None,
                       third_objective_values=None,
                       write_summaries=False):
    global generations
    global log_file

    generations.append(
        GenerationData(accuracies, generation_number, second_objective_values,
                       third_objective_values))

    if not os.path.exists(DataManager.get_Logs_folder()):
        os.makedirs(DataManager.get_Logs_folder())

    with open(os.path.join(DataManager.get_Logs_folder(), log_file),
              "a+") as f:
        if write_summaries:
            f.write(generations[-1].get_summary() + "\n")
        else:
            f.write(generations[-1].get_data() + "\n")
Esempio n. 24
0
def examine_generator():
    X_train, X_val, y_train, y_val = DataManager.load_train_val_data("all")
    from generator import CustomDataGenerator
    from train import transform, filter_mask_presence

    X_val, y_val = filter_mask_presence(X_val, y_val)
    generator = CustomDataGenerator(X_val, y_val, lambda x, y: transform(x, y, augment=True), 32)
    imgs, outs = generator.next()
    for i in range(len(imgs)):
        cv2.imshow("image", imgs[i, 0])
        cv2.imshow("mask", outs['main_output'][i, 0])
        cv2.waitKey(0)
Esempio n. 25
0
def load_data(composed_transforms: transforms.Compose, split: str) -> DataLoader:
    """
    Loads the data given the config.dataset

    :param composed_transforms: the augmentations to apply to the dataset
    :param split: either train or test, dataset returned also depends on config.fully_train -
    'train' will return 42500 images for evolution, but 50000 for fully training.
    'test' will return 7500 images for evolution, but 10000 for fully training.

    Note: the validation/train split does not try balance the classes of data, it just takes the first n for the train
    set and the remaining data goes to the validation set
    """
    if split.lower() not in ['train', 'test']:
        raise ValueError('Parameter split can be one of train, test or validation, but received: ' + str(split))

    # when to load train set
    train: bool = True if split == 'train' or (split == 'test' and not config.fully_train) else False

    dataset_args = {
        'root': DataManager.get_datasets_folder(),
        'train': train,
        'download': config.download_dataset,
        'transform': composed_transforms
    }

    if config.dataset == 'mnist':
        dataset = MNIST(**dataset_args)
    elif config.dataset == 'cifar10':
        dataset = CIFAR10(**dataset_args)
    elif config.dataset == 'cifar100':
        dataset = CIFAR100(**dataset_args)
    elif config.dataset == 'fashionMnist':
        dataset = FashionMNIST(**dataset_args)
    elif config.dataset == 'custom':
        dataset = get_generic_dataset(composed_transforms, train)
    else:
        raise ValueError('config.dataset can be one of mnist, cifar10 or custom, but received: ' + str(config.dataset))

    if train and not config.fully_train:
        # Splitting the train set into a train and valid set
        train_size = int(len(dataset) * (1 - config.validation_split))
        if split == 'train':
            dataset = Subset(dataset, range(train_size))
        elif split == 'test':
            dataset = Subset(dataset, range(train_size, len(dataset)))
        else:
            raise Exception("")

    # print(split, 'set size in', 'FT' if config.fully_train else 'evo', len(dataset))

    # TODO: test num workers and pin memory
    return DataLoader(dataset, batch_size=config.batch_size, shuffle=True, num_workers=0, pin_memory=False)
Esempio n. 26
0
 def __init__(self, app, WSGI_PATH_PREFIX):
     self.app = app
     # self.cmpmgr = CompressionManager( app, compresslevel=9 )
     self.DataManager = DataManager()
     mngrs = {
         # 'Compression': self.cmpmgr,
         'DataManager': self.DataManager
     }
     self.app.config.setdefault('Managers', mngrs)
     self.app.config['Managers'] = mngrs
     print('             Registered Application Managers')
     print(
         '------------------------------------------------------------------'
     )
Esempio n. 27
0
 def __init__(self, _stock_num, _target_month, _global_actor,
              _global_critic):
     self.global_actor = _global_actor
     self.global_critic = _global_critic
     self.local_actor, self.local_critic = BaseModel.BaseModel(
         64, 1).build_model()
     self.local_actor._make_predict_function()
     self.local_critic._make_predict_function()
     self.local_actor.set_weights(self.global_actor.get_weights())
     self.local_critic.set_weights(self.global_critic.get_weights())
     self.EnvForeign = EnvForeign.ForeignEnv(_stock_num, _target_month)
     dataManager = DataManager.DataManager(_stock_num)
     self.day_list = dataManager.get_daylist_in_month(_target_month)
     self.dataset_list = dataManager.get_dataset_in_month(_target_month)
Esempio n. 28
0
	def __init__(self):
		QtGui.QMainWindow.__init__(self)
		self.setAttribute(QtCore.Qt.WA_DeleteOnClose)
		self.setWindowTitle("Structure Viewer")

		self.data_manager = DataManager(self)

		self.color_switch = None
		self.lastopenfile = None

		self.initUI()
		self.main_widget.setFocus()
		self.setCentralWidget(self.main_widget)
		self.showMaximized()

		self.update_plot()
def print_max_accuracies(dataset):
    for run in get_all_results_folders(dataset):
        run_name = run
        if run_name in name_overrides:
            run_name = name_overrides[run_name]
        print(run_name)

        for result_file in get_all_results_files_in_folder(dataset,run):
            file_path = os.path.join(DataManager.get_results_folder(), dataset,"fully_train",run, result_file)
            # print(result_file,file_path)
            with open(file_path) as file:
                lines = file.readlines()
                accuracies = get_accuracies(lines)
                # print(accuracies)
                max_acc = max(accuracies)
                config_name = result_file.replace(".txt", "")
                print("\t", config_name, "max acc:", max_acc)
Esempio n. 30
0
def load_data(composed_transforms: transforms.Compose,
              split: str) -> DataLoader:
    """Loads the data given the config.dataset"""
    if split not in ['train', 'test', 'validation']:
        raise ValueError(
            'Parameter split can be one of train, test or validation, but received: '
            + str(split))

    train: bool = True if split == 'train' or split == 'validation' else False

    dataset_args = {
        'root': DataManager.get_datasets_folder(),
        'train': train,
        'download': True,
        'transform': composed_transforms
    }

    if config.dataset == 'mnist':
        dataset = MNIST(**dataset_args)
    elif config.dataset == 'cifar10':
        dataset = CIFAR10(**dataset_args)
    elif config.dataset == 'custom':
        dataset = get_generic_dataset(composed_transforms, train)
    else:
        raise ValueError(
            'config.dataset can be one of mnist, cifar10 or custom, but received: '
            + str(config.dataset))

    if train:
        # Splitting the train set into a train and valid set
        train_size = int(len(dataset) * (1 - config.validation_split))
        validation_size = len(dataset) - train_size

        train, valid = random_split(dataset, [train_size, validation_size])
        if split == 'train':
            dataset = train
        else:
            dataset = valid

    # TODO: test num workers and pin memory
    return DataLoader(dataset,
                      batch_size=config.batch_size,
                      shuffle=True,
                      num_workers=0,
                      pin_memory=False)
Esempio n. 31
0
def load_date_from_log_file(run_name, iteration=0, summary=False):
    global generations
    generations = []
    filename = os.path.join(
        DataManager.get_Logs_folder(run_name),
        "log" + (("_" + repr(iteration)) if iteration > 0 else ""))
    log = open(filename + ".txt")
    for gen in log:
        gen_number = int(gen.split("{")[0].split(":")[1])
        gen = gen.split("{")[1].split("}")[0]
        objectives = gen.split("|")
        o = 0
        accuracies = second = third = None
        for objective in objectives:
            if summary:
                name = objective.split("~")[0]
                vals = objective.split("~")[1].split(";")

                max = None
                av = None
                for val in vals:
                    if "max" in val.split(":")[0]:
                        max = float(val.split(":")[1])
                    if "average" in val.split(":")[0]:
                        av = float(val.split(":")[1])
            else:
                name = objective.split(":")[0]
                vals = objective.split(":")[1]
                if "accuracy" in name:
                    if o > 0:
                        print(
                            "warning, accuracy not the first objective in log",
                            filename)
                        return
                    accuracies = ast.literal_eval(vals)
                elif o == 1:
                    second = ast.literal_eval(vals)
                elif o == 2:
                    third = ast.literal_eval(vals)
                else:
                    raise Exception("too many objectives in log", filename, o,
                                    name)
            o += 1
        generations.append(
            GenerationData(accuracies, gen_number, second, third))
def generate_submission():
    # Load test images and preprocess for conv net.
    print('Loading and processing test images')
    imgs_test = DataManager.load_test_data()
    total = imgs_test.shape[0]
    imgs = np.ndarray(
        (total, 1, DataManager.IMG_TARGET_ROWS, DataManager.IMG_TARGET_ROWS),
        dtype=np.uint8)
    i = 0
    for img in imgs_test:
        imgs[i] = preprocess(img)
        i += 1

    print('Loading network')
    model = build_model()
    model.load_weights('./results/net.hdf5')

    print('Generating predictions')
    masks, has_masks = model.predict(imgs, verbose=1)

    ids = []
    rles = []
    for i in range(total):
        # Zero out masks when there is no-nerve pred.
        if has_masks[i, 0] < 0.5:
            masks[i, 0] *= 0.

        mask = post_process_mask(masks[i, 0])
        rle = run_length_enc(mask)
        rles.append(rle)
        ids.append(i + 1)

        if i % 100 == 0:
            print('{}/{}'.format(i, total))

    first_row = 'img,pixels'
    file_name = 'results/submission_{}.csv'.format(str(datetime.now()))

    with open(file_name, 'w+') as f:
        f.write(first_row + '\n')
        for i in range(total):
            s = str(ids[i]) + ',' + rles[i]
            f.write(s + '\n')
Esempio n. 33
0
	# Output file
	target = "out.csv"

	# Row containing the problem body
	DATA_COL = 32

	# Categories to be checked using ConceptNet
	conceptnet_categories = ['car','animal','sport','object','food','subject','place']

	# Categories to be checked using Stanford NER
	NER_categories = ['location','time', 'person', 'organization' ,'money', 'percent','date']

	# Categories based on given keywords
	keyword_categories = [('geometry',['square','circle','rectangle','triangle','angle','quadrant'])]

	# Categories based on values from other columns (limited only to categories in this program)
	columnval_categories = [('real_world_reference',is_real_world,['car','animal','sport','object','food','subject','place','location','person','organization','money'])]

	# Specify column in source data containing questions
	dm = DataManager(source, target, DATA_COL, write_mode='a', batch = 100, pool_size = 8)

	# Add categories into header
	dm.add_header(conceptnet_categories,header_type = "conceptnet")
	dm.add_header(NER_categories,header_type = "NER")
	dm.add_header(keyword_categories,header_type = "keyword")
	dm.add_header(columnval_categories,header_type = "columnval")

	# Measure duration of running the program
	start_time = time.time()
	dm.process()
	print time.time() - start_time, "seconds"
Esempio n. 34
0
class ApplicationWindow(QtGui.QMainWindow):
	def __init__(self):
		QtGui.QMainWindow.__init__(self)
		self.setAttribute(QtCore.Qt.WA_DeleteOnClose)
		self.setWindowTitle("Allele frequency viewer")

		self.data = None
		self.genes = {}
		self.lastopeneddir = os.path.expanduser("~")

		self.initUI()
		self.init_menu()
		self.initRegionViewer()
		self.main_widget.setFocus()
		self.setCentralWidget(self.main_widget)
		self.showMaximized()

		self.tab_names = []
		#self.test()

	def test(self) :
		vcf = "/Volumes/CORSAIR/PoolsTeo_jan2016/vsYJM326/320_1.rg.dedup.realigned.raw.info.var.vcf"
		#vcf = "/media/jevanni/CORSAIR/PoolsTeo_jan2016/vsYJM326/320_1.rg.dedup.realigned.raw.info.var.vcf"
		fastafile = "./Anno/yjm326.fasta"
		gff3fname = "./Anno/sace_new.gff"
		self.load_vcf(vcf)
		self.load_annotation(fastafile, gff3fname) 

	def update_completer_gene(self) :
		model = QtGui.QStringListModel()
		model.setStringList(self.genes.keys())
		self.completer.setModel(model)

	def initUI(self) :
		self.main_widget = QtGui.QWidget(self)
		self.layout = QtGui.QVBoxLayout(self.main_widget)
		self.layout_top_right = QtGui.QHBoxLayout()
		self.layout_top = QtGui.QHBoxLayout()

		# Search bar with auto completer
		model = QtGui.QStringListModel()
		model.setStringList(self.genes.keys())
		self.completer = QtGui.QCompleter()
		self.completer.setModel(model)
		self.search_bar = QtGui.QLineEdit("Search gene")
		self.search_bar.keyPressEvent = self.search_bar_key_event
		self.search_bar.mousePressEvent = self.search_bar_mouse_event
		self.search_bar.setCompleter(self.completer)

		self.af_bar = QtGui.QLineEdit("Set af bar")
		self.af_bar.keyPressEvent = self.af_bar_key_event
		self.af_bar.mousePressEvent = self.af_bar_mouse_event
		
		self.chromosome_combobox = QtGui.QComboBox()
		self.start_bar = QtGui.QLineEdit("Start coordinate")
		self.start_bar.keyPressEvent = self.start_bar_key_event
		self.start_bar.mousePressEvent = self.start_bar_mouse_event
		self.end_bar = QtGui.QLineEdit("End coordinate")
		self.end_bar.keyPressEvent = self.end_bar_key_event
		self.end_bar.mousePressEvent = self.end_bar_mouse_event
		self.layout_top_right.addWidget(self.chromosome_combobox)
		self.layout_top_right.addWidget(self.start_bar)
		self.layout_top_right.addWidget(self.end_bar)
	
		self.layout_top.addWidget(self.search_bar)
		self.layout_top.addWidget(self.af_bar)
		self.layout_top.addLayout(self.layout_top_right)

		self.tab_widget = QtGui.QTabWidget(self)
		self.tab_widget.currentChanged.connect(self.change_tab)
		self.statut_bar = QtGui.QStatusBar(self)
		self.layout.addLayout(self.layout_top)
		self.layout.addWidget(self.tab_widget)
		self.layout.addWidget(self.statut_bar)

	def init_menu(self) :
		self.file_menu = QtGui.QMenu("&Files", self)
		self.file_menu.addAction("&Open files", self.load_annotation, QtCore.Qt.CTRL + QtCore.Qt.Key_O)
		self.menuBar().addMenu(self.file_menu)

	def initRegionViewer(self) :
		screen = QtGui.QDesktopWidget()
		second_screen = screen.screenGeometry(0)
		self.regionViewer = RegionViewer(self)
		self.regionViewer.move(second_screen.left(), second_screen.top())

	def load_vcf(self, fname=None) :
		if not fname : fname = str(QtGui.QFileDialog.getOpenFileName(self, 'Open VCF file', self.lastopeneddir))
		if not fname : return False
		self.data = DataManager(self, fname)
		self.setWindowTitle("Allele frequency viewer : %s" %os.path.basename(fname))
		self.chromosome_combobox.addItems(self.data.get_chromosomes())
		self.lastopeneddir = os.path.dirname(fname)
		self.show_plot()
		return True

	def load_annotation(self, fasta=None, annofname=None) :
		loaded = True
		if not self.data : loaded = self.load_vcf()
		if not loaded : return
		if not fasta : fasta = str(QtGui.QFileDialog.getOpenFileName(self, 'Open Fasta file', self.lastopeneddir))
		if not annofname : annofname = str(QtGui.QFileDialog.getOpenFileName(self, 'Open annotation file', self.lastopeneddir))

		if not fasta or not annofname : return

		self.annotations = get_feature_gff3(annofname, fasta, self.data)
		self.genes = self.annotations.get_dict_parents()
		self.update_completer_gene()
		self.lastopeneddir = os.path.dirname(annofname)

	def show_plot(self, chromosome=None) :
		name = chromosome if chromosome else "Genome"
		if name not in self.tab_names :
			canva = self.data.get_canva(chromosome=chromosome)
			self.tab_widget.addTab(canva, name)
			self.tab_names.append(name)

		self.tab_widget.setCurrentIndex(self.tab_names.index(name))
		self.get_current_tabwidget().update_info()

	def get_current_tabwidget(self) :
		return self.tab_widget.currentWidget()

	def update_statut_whole(self, chromosome=None) :
		info = self.data.get_statut(chromosome=chromosome)

	def search_bar_mouse_event(self, event) :
		if event.button() == 1 and self.search_bar.text() == "Search gene" :
			self.search_bar.setText("")

	def search_bar_key_event(self, event) :
		if event.text() == '\r' : self.zone_gene(self.search_bar.text())
		else : QtGui.QLineEdit.keyPressEvent(self.search_bar, event)

	def zone_gene(self, gene_name) :
		annotation = self.genes.get(str(gene_name), None)
		if not annotation : return

		chromosome, start, end = annotation["seqname"], annotation["start"], annotation["end"]
		self.show_plot(chromosome=chromosome)
		self.get_current_tabwidget().draw_region(start, end)
		self.get_current_tabwidget().send_info()

	def af_bar_key_event(self, event) :
		if event.text() == '\r' : self.set_bar(self.af_bar.text())
		else : QtGui.QLineEdit.keyPressEvent(self.af_bar, event)

	def af_bar_mouse_event(self, event) :
		if event.button() == 1 and self.af_bar.text() == "Set af bar" :
			self.af_bar.setText("")

	def start_bar_mouse_event(self, event) :
		if event.button() == 1 and self.start_bar.text() == "Start coordinate" :
			self.start_bar.setText("")

	def end_bar_mouse_event(self, event) :
		if event.button() == 1 and self.end_bar.text() == "End coordinate" :
			self.end_bar.setText("")

	def start_bar_key_event(self, event) :
		if event.text() == '\r' : self.set_coordinate_manualy()
		else : QtGui.QLineEdit.keyPressEvent(self.start_bar, event)

	def end_bar_key_event(self, event) :
		if event.text() == '\r' : self.set_coordinate_manualy()
		else : QtGui.QLineEdit.keyPressEvent(self.end_bar, event)

	def set_coordinate_manualy(self) :
		chromosome = str(self.chromosome_combobox.currentText())
		start = str(self.start_bar.text())
		end = str(self.end_bar.text())
		try :
			start, end = int(start), int(end)
		except ValueError :
			return

		self.show_plot(chromosome=chromosome)
		self.get_current_tabwidget().draw_region(start, end)

	def set_bar(self, value) :
		self.get_current_tabwidget().remove_line()
		try : value = float(value)
		except ValueError : return
		if value > 1 or value < 0 : return
		self.get_current_tabwidget().draw_bar(value)

	def change_tab(self) :
		self.get_current_tabwidget().update_info()

	def closeEvent(self, ce) :
		if self.regionViewer :
			self.regionViewer.close()
Esempio n. 35
0
class ApplicationWindow(QtGui.QMainWindow):
	def __init__(self):
		QtGui.QMainWindow.__init__(self)
		self.setAttribute(QtCore.Qt.WA_DeleteOnClose)
		self.setWindowTitle("Structure Viewer")

		self.data_manager = DataManager(self)

		self.color_switch = None
		self.lastopenfile = None

		self.initUI()
		self.main_widget.setFocus()
		self.setCentralWidget(self.main_widget)
		self.showMaximized()

		self.update_plot()

	def initUI(self) :
		self.main_widget = QtGui.QWidget(self)
		self.main_widget.move(10, 20)
		self.main_widget.resize(960, 730)
		self.layout = QtGui.QVBoxLayout(self.main_widget)
		self.initMenu()

		self.error_dialog_load = QtGui.QErrorMessage()

	def initMenu(self) :
		self.file_menu = QtGui.QMenu("&Options", self)
		self.file_menu.addAction("&Open file", self.open_file, QtCore.Qt.CTRL + QtCore.Qt.Key_O)
		self.file_menu.addAction("&Save fig", self.save_file, QtCore.Qt.CTRL + QtCore.Qt.Key_S)
		self.file_menu.addAction('&Graph settings', self.switch_color, QtCore.Qt.CTRL + QtCore.Qt.Key_G)

		self.action_border = QtGui.QAction("Border", self)
		self.action_labels = QtGui.QAction("Show all labels", self)

		self.action_border.changed.connect(self.update_plot)
		self.action_border.setCheckable(True)

		self.action_labels.changed.connect(self.update_plot)
		self.action_labels.setCheckable(True)

		self.file_menu.addAction(self.action_border)
		self.file_menu.addAction(self.action_labels)
		self.menuBar().addMenu(self.file_menu)

	def open_file(self) :
		if self.lastopenfile :
			fnames = QtGui.QFileDialog.getOpenFileNames(self, 'Open file', dirname(self.lastopenfile))
		else :
			fnames = QtGui.QFileDialog.getOpenFileNames(self, 'Open file')

		for fname in fnames :
			try :
				self.load_file(fname)
			except SVInternalException as error :
				self.error_dialog_load.showMessage(str(fname) + " : " + str(error))
				return

		self.lastopenfile = str(fname)
		self.update_plot()

	def load_file(self, fname) :
		self.data_manager.add_structure_fname(fname)

	def update_plot(self) :
		# Generate an error, I don't really know why
		border = self.is_border_checked()
		show_all_labels = self.is_all_labels_checked()
		self.canva = self.data_manager.get_canva(edge=border, show_all_labels=show_all_labels)
		if not self.canva : return
		for i in reversed(range(self.layout.count())):
				if isinstance(self.layout.itemAt(i), QtGui.QWidgetItem) :
					self.layout.itemAt(i).widget().setParent(None)

		self.canva.setParent(self.main_widget)
		self.canva.setFocus()
		self.layout.addWidget(self.canva)
		self.show()

	def is_border_checked(self) :
		return self.action_border.isChecked()

	def is_all_labels_checked(self) :
		return self.action_labels.isChecked()

	def popup(self, data, row, column) :
		self.menu = QtGui.QMenu(self)
		go_up_action = QtGui.QAction("Go up", self)
		go_up_action.triggered.connect(partial(self.change_row, data, column, 1))
		go_down_action = QtGui.QAction("Go down", self)
		go_down_action.triggered.connect(partial(self.change_row, data, column, -1))
		sort_action = QtGui.QAction("Sort", self)
		sort_action.triggered.connect(partial(self.sort_pop, data, column))
		self.menu.addAction(go_up_action)
		self.menu.addAction(go_down_action)
		self.menu.addAction(sort_action)
		self.menu.popup(QtGui.QCursor.pos())

	def switch_color(self) :
		self.color_switch = SwitchColorWindow(self)

	def sort_pop(self, data, column) :
		data.reorder_pop(column)
		self.data_manager.uniform_names(data)
		self.update_plot()

	def change_row(self, data, column, level) :
		if data.reorder_column(column, level) :
			self.update_plot()

	def save_file(self) :
		if self.lastopenfile :
			fname = QtGui.QFileDialog.getSaveFileName(self, 'Open file', dirname(self.lastopenfile))
		else :
			fname = QtGui.QFileDialog.getSaveFileName(self, 'Open file')

		self.data_manager.save(str(fname))

	def closeEvent(self, ce) :
		if self.color_switch :
			self.color_switch.terminate()
		self.close()