def plot_runs_mean(runs_results): graph_name = shared.getConst("graph_name") max_duration = shared.getConst("max_duration") nb_runs, generations, mean_fit_mins, mean_fit_avg, mean_duration_mins, mean_duration_maxs = runs_results fig, ax1 = plt.subplots() fig.suptitle(f"Mean results over {nb_runs} runs, graph: '{graph_name}', duration constraint: {max_duration}") line1 = ax1.plot(generations, mean_fit_mins, "b-", label="Minimum Cost") line3 = ax1.plot(generations, mean_fit_avg, "g-", label= "Average Cost") ax1.set_xlabel("Generation") ax1.set_ylabel("Cost", color="b") for tl in ax1.get_yticklabels(): tl.set_color("b") ax2 = ax1.twinx() line2 = ax2.plot(generations, mean_duration_mins, "r-", label="Minimum Duration") line4 = ax2.plot(generations, mean_duration_maxs, "y-", label="Maximum Duration") ax2.set_ylabel("Duration", color="r") for tl in ax2.get_yticklabels(): tl.set_color("r") lns = line1 + line2 + line3 + line4 labs = [l.get_label() for l in lns] ax1.legend(lns, labs, loc="upper right") OUTPUT_DIR = os.environ.get("AZ_BATCH_TASK_DIR", ".") OUTPUT_FILE = os.path.join(OUTPUT_DIR, f"{graph_name}_{nb_runs}_runs.png") plt.savefig(OUTPUT_FILE)
def extractComProFea(compro): com,pro = compro comFea = sh.getConst('krDict')[com] proFea = sh.getConst('aacDict')[pro] fea = np.append(comFea,proFea) fea = fea.tolist() return fea
def _cluster(params): cls = None method = sh.getConst('method') if method=='kmedoid': assert False # from kmedoid import kmedsoid # cls = kmedoid elif method=='dbscan': from sklearn.cluster import DBSCAN cls = DBSCAN(eps=params['eps'],min_samples=params['min_samples'], metric='precomputed') else: assert False, 'FATAL: unknown cluster method' ## mat = sh.getConst('mat') labels = cls.fit_predict(mat) nLabels = len(set(labels)) ## sil = None; cal = None if (nLabels >= 2)and(nLabels <= len(labels)-1): sil = met.silhouette_score(mat,labels,'precomputed') cal = met.calinski_harabaz_score(mat,labels) perf = dict(silhouette_score=sil,calinski_harabaz_score=cal) return (labels,perf)
def extractComProFea(compro): com, pro = compro comFea = sh.getConst('krDict')[com] proFea = sh.getConst('aacDict')[pro] fea = np.append(comFea, proFea) fea = fea.tolist() return fea
def funcUseSharedConstant(): # Tries on a mutable and an immutable object assert shared.getConst('myVar') == { 1: 'Example 1', 2: 'Example 2', 3: 'Example 3', } assert shared.getConst('secondVar') == "Hello World!" return True
def genetic_algo(): # Shared constants graph_name = shared.getConst("graph_name") graph = shared.getConst("graph") max_duration = shared.getConst("max_duration") # Extra toolbox registers toolbox.register("population_guess", initPopulation, list, toolbox.individual_guess) toolbox.register("evaluate", evaluate, graph, max_duration=max_duration) # Creating the population pop = toolbox.population_guess() # Creating a logbook for recording statistics logbook = tools.Logbook() # To the heart of the genetic algorithm for g in range(NGEN): # Select the next generation individuals offspring = toolbox.select(pop, len(pop)) # Clone the selected individuals offspring = [toolbox.clone(ind) for ind in offspring] # Apply crossover on the offspring for child1, child2 in zip(offspring[::2], offspring[1::2]): if random.random() < CXPB: toolbox.mate(child1, child2) del child1.fitness.values del child2.fitness.values # Apply mutation on the offspring for mutant in offspring: if random.random() < MUTPB: toolbox.mutate(mutant) del mutant.fitness.values # Evaluate the individuals with an invalid fitness invalid_ind = [ind for ind in offspring if not ind.fitness.valid] fitnesses = toolbox.map(toolbox.evaluate, invalid_ind) for ind, fit in zip(invalid_ind, fitnesses): ind.fitness.values = fit # The population is entirely replaced by the offspring pop[:] = offspring # Recording statistics record = mstats.compile(pop) logbook.record(gen=g, evals=len(invalid_ind), **record) gen = logbook.select("gen") fit_mins = logbook.chapters["cost"].select("min") duration_mins = logbook.chapters["duration"].select("min") duration_maxs = logbook.chapters["duration"].select("max") fit_avg = logbook.chapters["cost"].select("avg") return gen, fit_mins, fit_avg, duration_mins, duration_maxs
def funcUseDeletedSharedConstant(): # Tries to retrieve a deleted variable, should timeout shared.deleteConst('myVar') shared.deleteConst('secondVar') timeout = 0.01 assert shared.getConst('myVar', timeout=timeout) is None assert shared.getConst('secondVar', timeout=timeout) is None try: # You should not be able to reset a deleted variable shared.setConst(myVar=42) return False except TypeError: pass return True
def evaluate(compile_valtree, compile_condtree, individual: DoubleTreeBasedIndividual): compiled_conditions = [ compile_condtree(cond_tree) for cond_tree in individual.cond_trees ] # compiled_conditions = [compile_condtree(cond_tree) for cond_tree in individual["CONDITION_TREES"]] compiled_values = [ compile_valtree(val_tree) for val_tree in individual.value_trees ] # compiled_values = [compile_valtree(val_tree) for val_tree in individual["VALUE_TREES"]] results = [] num_of_solved = 0 # run with Bomb only # nonogram_solved = utils.load_solved_bomb_nonogram_from_file() # nonogram_unsolved = utils.load_unsolved_bomb_nonogram_from_file() # results.append(evaluate_single_nonogram(compiled_conditions, compiled_values, nonogram_solved, nonogram_unsolved)) # run on all solved nonograms train_nonograms_sh = shared.getConst('train_nonograms') for nonogram_unsolved, nonogram_solved in train_nonograms_sh: result = round( evaluate_single_nonogram(compiled_conditions, compiled_values, nonogram_solved, nonogram_unsolved), 4) results.append(result) if result == 5: num_of_solved += 1 if print_individual_fitness: print("Fitness:", results, round(results, 4)) # print('-------------------') # if num_of_solved > 0: # print("Solved:", num_of_solved, "Nonograms") distance = _calc_distance(results) return distance,
def maxTreeDepthDivide(rootValue, currentDepth=0, parallelLevel=2): """Finds a tree node that represents rootValue and computes the max depth of this tree branch. This function will emit new futures until currentDepth=parallelLevel""" thisRoot = shared.getConst('myTree').search(rootValue) if currentDepth >= parallelLevel: return thisRoot.maxDepth(currentDepth) else: # Base case if not any([thisRoot.left, thisRoot.right]): return currentDepth if not all([thisRoot.left, thisRoot.right]): return thisRoot.maxDepth(currentDepth) # Parallel recursion return max( futures.map( maxTreeDepthDivide, [ thisRoot.left.payload, thisRoot.right.payload, ], cycle([currentDepth + 1]), cycle([parallelLevel]), ) )
def maxTreeDepthDivide(rootValue, currentDepth=0, parallelLevel=2): """Finds a tree node that represents rootValue and computes the max depth of this tree branch. This function will emit new futures until currentDepth=parallelLevel""" thisRoot = shared.getConst('myTree').search(rootValue) if currentDepth >= parallelLevel: return thisRoot.maxDepth(currentDepth) else: # Base case if not any([thisRoot.left, thisRoot.right]): return currentDepth if not all([thisRoot.left, thisRoot.right]): return thisRoot.maxDepth(currentDepth) # Parallel recursion return max( futures.map( maxTreeDepthDivide, [ thisRoot.left.payload, thisRoot.right.payload, ], cycle([currentDepth + 1]), cycle([parallelLevel]), ))
def process_doc(filepath): print "filepath", filepath journal_title, article_subjects, article_kwds, article_title, article_abstract, article_contents = get_fields(filepath) doc_id = filepath.strip().split("/")[-1].split(".")[0] s = "<DOC>\n" s += "<DOCNO> "+ doc_id +" </DOCNO>\n " if journal_title: s += "<JTITLE> "+ journal_title +" </JTITLE>\n " if article_subjects: for sub in article_subjects: if sub: s += "<SUB> "+ sub +" </SUB>\n " if article_kwds: for kwd in article_kwds: if kwd: s += "<KWD> "+ kwd +" </KWD>\n " if article_title: s += "<ATITLE> "+ article_title +" </ATITLE>\n " if article_abstract: s += "<ABSTRACT> "+ article_abstract +" </ABSTRACT>\n " s += "<TEXT> "+ article_contents +" </TEXT>\n" s += "</DOC>\n" destination = shared.getConst('destination') with codecs.open(destination + "/" + doc_id, encoding="utf-8", mode="w") as f: f.write(s) return True
def assess_reaction(rxn, reactionSystems, tolerance, data): """ Returns whether the reaction is important or not in the reactions. It iterates over the reaction systems, and loads the concentration profile of each reaction system into memory. It iterates over a number of samples in profile and evaluates the importance of the reaction at every sample. """ logging.debug('Assessing reaction {}'.format(rxn)) reactions = shared.getConst('reactions') # read in the intermediate state variables for datum, reactionSystem in zip(data, reactionSystems): T, P = reactionSystem.T.value_si, reactionSystem.P.value_si species_names, profile = datum # take N evenly spaced indices from the table with simulation results: """ Number of time steps between start and end time of the batch reactor simulation at which the importance of reactions should be evaluated. The more timesteps, the less chance we have to remove an important reactions, but the more simulations need to be carried out. """ timesteps = len(profile) / 2 logging.debug('Evaluating the importance of a reaction at {} time samples.'.format(timesteps)) assert timesteps <= len(profile) indices = map(int, np.linspace(0, len(profile)-1, num = timesteps)) for index in indices: assert profile[index] is not None timepoint, coreSpeciesConcentrations = profile[index] coreSpeciesConcentrations = {key: float(value) for (key, value) in zip(species_names, coreSpeciesConcentrations)} # print 'Species concentrations at {}: {}'.format(timepoint, reactionSystem.coreSpeciesConcentrations) for species_i in rxn.reactants: if isImportant(rxn, species_i, reactions, 'reactant', tolerance, T, P, coreSpeciesConcentrations): return True #only continue if the reaction is not important yet. for species_i in rxn.products: if isImportant(rxn, species_i, reactions, 'product', tolerance, T, P, coreSpeciesConcentrations): return True return False
def _calc_max_possible_fitness(): train_nonograms = shared.getConst('train_nonograms') compares = [ _compare_to_solution(solved, solved) for unsolved, solved in train_nonograms ] res = np.mean(compares) print('max possible fitness is:', res) return res
def getValue(words): """Computes the sum of the values of the words.""" value = 0 for word in words: for letter in word: # shared.getConst will evaluate to the dictionary broadcasted by # the root Future value += shared.getConst("lettersValue")[letter] return value
def getValue(words): """Computes the sum of the values of the words.""" value = 0 for word in words: for letter in word: # shared.getConst will evaluate to the dictionary broadcasted by # the root Future value += shared.getConst('lettersValue')[letter] return value
def generate(size, pmin, pmax, smin, smax): part = creator.Particle(random.uniform(pmin, pmax) for _ in range(size)) part.speed = [random.uniform(smin, smax) for _ in range(size)] part.smin = smin part.smax = smax part.pmax = pmax part.pmin = pmin factory = getattr(parameterSetsAdaption, shared.getConst('adaptionParams')) myInt = int(shared.getConst('parameterListIndex')) if (shared.getConst('parameterListIndex') is not None): factory = factory[myInt] vfile_name = shared.getConst('vesselInputFile') grp_name = shared.getConst('vessel_grp') factory['adaption'].update( vesselFileName=vfile_name, vesselGroupName=grp_name, ) part.adaptionParameters = factory return part
def myFunc(parameter): """This function will be executed on the remote host even if it was not available at launch.""" print('Hello World from {0}!'.format(scoop.worker)) # It is possible to get a constant anywhere print(shared.getConst('myVar')[2]) # Parameters are handled as usual return parameter + 1
def initPopulation(pcls, ind_init): """ pcls : class of the population (i.e. a toolbox attribute of an individual) ind_init : function to initialize individuals return : a list of individuals """ graph = shared.getConst("graph") contents = init_generation(NB_POP, MAX_MACH, graph) return pcls(ind_init(c) for c in contents)
def get(key): """ Searches for the shared variable to retrieve identified by the parameter key. """ try: data = shared.getConst(key) return data except KeyError, e: logging.error('An object with the key {} could not be found.'.format(key)) raise e
def get(key): """ Searches for the shared variable to retrieve identified by the parameter key. """ try: data = shared.getConst(key) return data except KeyError, e: logging.error( 'An object with the key {} could not be found.'.format(key)) raise e
def get(key): """ Searches for the shared variable to retrieve identified by the parameter key. """ try: data = shared.getConst(key, timeout=1e-9) return data except NameError: """ Name error will be caught when the SCOOP library is not imported properly. """ logger.debug('SCOOP not loaded. Not retrieving the shared object with key {}'.format(key))
def broadcast(obj, key): """ Broadcasts the object across the workers using the key parameter as the key. """ kwargs = {key : obj} try: if shared.getConst(key): logger.debug('An object with the key {} was already broadcasted.'.format(key)) else: shared.setConst(**kwargs) except NameError, e: """ Name error will be caught when the SCOOP library is not imported properly. """ logger.debug('SCOOP not loaded. Not broadcasting the object {}'.format(obj))
def funcBroadcast(): """ Broadcast the data with the given key, and retrieve it again by querying the key again. """ data = 'foo' key = 'bar' broadcast(data, key) try: assert data == shared.getConst(key) except AssertionError: return False return True
def configureScoop(): global CONFIGURED if CONFIGURED or not SCOOP_IS_RUNNING: return global MYSYSTEM, N_RANDOM_PROBLEMS, RANDOM_SEED, USE_RECAST, BASELINE, OUTPUT_FOLDER MYSYSTEM = shared.getConst('MYSYSTEM') N_RANDOM_PROBLEMS = shared.getConst('N_RANDOM_PROBLEMS') RANDOM_SEED = shared.getConst('RANDOM_SEED') USE_RECAST = shared.getConst('USE_RECAST') BASELINE = shared.getConst('BASELINE') OUTPUT_FOLDER = shared.getConst('OUTPUT_FOLDER') # finish configureEvaluator() CONFIGURED = True
def start_experiment(self): train_nonograms = shared.getConst('train_nonograms') nonogram_names = [ unsolved.title for unsolved, solved in train_nonograms ] print('running experiment on', train_size, 'nonograms. names:', nonogram_names) # max_possible_fitness = _calc_max_possible_fitness() start = time.time() # mu = len(self.pop) # lambda_ = len(self.pop) # pop, log = algorithms.eaMuPlusLambda(self.pop, self.toolbox, mu, lambda_, prob_crossover_global, prob_mutate_global, num_gen, # halloffame=self.hof, verbose=True, stats=self.stats) pop, log = eaSimple_new(self.pop, self.toolbox, prob_crossover_global, prob_mutate_global, num_gen, halloffame=self.hof, verbose=True, stats=self.stats) end = time.time() return pop, log, self.hof, self.stats, end - start
def process(sub_parameters): V = shared.getConst("V") sequence = eval("lambda x:{}".format(V['sequence'])) weight_function = eval(V['weight_function']) extrema = V['extrema'] i_ip = V['i_ip'] i_i = V['i_i'] sub_values = {V['subs_symbols'][a]:b[0] for a,b in sub_parameters.iteritems()} subs_choice = matrix_map(V['choice'],lambda x,i,j:lambdify(V['pop_symbols'],x.subs(sub_values))) pops = [a.copy() for a in V['origional_pops']] for i in range(V['iterations']): for p in range(len(pops)): flattened_pop = [a[0,0] for a in pops[p]] weighted_choice = numpy.matrix(matrix_map(subs_choice,lambda x,i,j:x(*flattened_pop))) extrema_eigen_pairs = [eigen(weighted_choice*e,i_ip,i_i) for e in extrema] random.shuffle(extrema_eigen_pairs) extrema_eigen_pairs = sorted(extrema_eigen_pairs, key=operator.itemgetter(0), reverse=True) growths = [a[0] for a in extrema_eigen_pairs] weights = [weight_function(index,growths) for index in range(len(growths))] weights = [a/sum(weights) for a in weights] z = sequence(i) pops[p] = pops[p]*(1-z) + sum([weights[index]*extrema_eigen_pairs[index][1] for index in range(len(growths))])*z return {"parameters":sub_parameters,"pops":[[ppp.sum() for ppp in p] for p in pops]}
def funcUseSharedFunction(): assert shared.getConst('myRemoteFunc')(5) == 5 * 5 assert shared.getConst('myRemoteFunc')(25) == 25 * 25 return True
def assess_reaction(rxn, reactionSystems, tolerance, data): """ Returns whether the reaction is important or not in the reactions. It iterates over the reaction systems, and loads the concentration profile of each reaction system into memory. It iterates over a number of samples in profile and evaluates the importance of the reaction at every sample. """ logging.debug('Assessing reaction {}'.format(rxn)) reactions = shared.getConst('reactions') # read in the intermediate state variables for datum, reactionSystem in zip(data, reactionSystems): T, P = reactionSystem.T.value_si, reactionSystem.P.value_si species_names, profile = datum # take N evenly spaced indices from the table with simulation results: """ Number of time steps between start and end time of the batch reactor simulation at which the importance of reactions should be evaluated. The more timesteps, the less chance we have to remove an important reactions, but the more simulations need to be carried out. """ timesteps = len(profile) / 2 logging.debug( 'Evaluating the importance of a reaction at {} time samples.'. format(timesteps)) assert timesteps <= len(profile) indices = map(int, np.linspace(0, len(profile) - 1, num=timesteps)) for index in indices: assert profile[index] is not None timepoint, coreSpeciesConcentrations = profile[index] coreSpeciesConcentrations = { key: float(value) for (key, value) in zip(species_names, coreSpeciesConcentrations) } # print 'Species concentrations at {}: {}'.format(timepoint, reactionSystem.coreSpeciesConcentrations) for species_i in rxn.reactants: if isImportant(rxn, species_i, reactions, 'reactant', tolerance, T, P, coreSpeciesConcentrations): return True #only continue if the reaction is not important yet. for species_i in rxn.products: if isImportant(rxn, species_i, reactions, 'product', tolerance, T, P, coreSpeciesConcentrations): return True return False
def ensembleSmote(xydev): xdevf,ydev = xydev sm = SMOTE(kind='svm',random_state=sh.getConst('smoteSeed')) xdevfr,ydevr = sm.fit_sample(xdevf,ydev) return (xdevfr,ydevr)
def myParallelFunc(inV): myV = shared.getConst("myValue") return inV + myV