def initialize(): global annotator, answer_catalog, analysis annotator = Annotator() answer_catalog = Answers() analysis = Analysis()
from knn_gridsearch import KNNModel from analysis import Analysis from sklearn.ensemble import RandomForestClassifier #predict Arsenal away wins #pick out feature most important to predicting away wins league_id=1729 away_team_api_id=9825 sqlite_file = '/Users/schumanzhang/Desktop/machine_learning_projects/soccer_dataset/raw_data/database.sqlite' try: processed_data = pd.read_csv('dataframe.csv', index_col=0) except IOError: analysis = Analysis(league_id, away_team_api_id, sqlite_file) raw_data = analysis.retrieve_raw_data() basic_info_list = analysis.basic_data_info() print(basic_info_list) analysis.addInitAttributes() analysis.addPlayerSkillTotals() analysis.addPlayerHeightTotals() analysis.parse_selected_tags('shoton', ['away_shoton', 'home_shoton']) analysis.parse_selected_tags('shotoff', ['away_shotoff', 'home_shotoff']) analysis.parse_selected_tags('foulcommit', ['away_fouls', 'home_fouls']) analysis.parse_selected_tags('card', ['away_cards', 'home_cards']) analysis.parse_selected_tags('cross', ['away_crosses', 'home_crosses']) analysis.parse_selected_tags('corner', ['away_corners', 'home_corners']) analysis.parse_selected_tags('possession', ['away_possession', 'home_possession'])
def test_main_flow(self): raw_dir = "." since_date = "2006/01/01" to_date = "2008/12/31" filling_types = {"10-K", "10-Q"} # case-sensitive by default, or re.IGNORECASE for case-insensive match flags = 0 workbook = xlsxwriter.Workbook('acq_asset_report.xlsx') red_format = workbook.add_format({ 'font_color': 'red', 'bold': 1, 'underline': 1, 'font_size': 12, }) # read the cik/targetname between row11~20 from the firms spreadsheet # the default specs are overwitten! TestMainFlow.specs = utils.read_specs("./firms.xlsx", 1, 100) for spec in TestMainFlow.specs: cik = spec[0] target_name = spec[1] golden_doc = spec[2] index_file = os.path.join(raw_dir, cik, "download.idx") clean_name = ''.join(e for e in target_name if e.isalnum()) # A company could acquire lots of small company during a period worksheet = workbook.add_worksheet(f"{cik}-{clean_name}") worksheet.write_string('A1', "Target Name:") worksheet.write_string('B1', target_name) worksheet.set_column('A:A', 35) worksheet.set_column('B:B', 150) utils.logger.info(f'scanning acquisition asset information about {target_name} in {cik} docs...') if os.path.exists(index_file) is False: try: company = Company(cik) utils.logger.info(f"\tdownloading {'/'.join(filling_types)} documents for cik:{cik}...") index_file = company.download_documents(since_date, to_date, filling_types, raw_dir) except Exception as e: utils.logger.error(f"Failed to download documents for cik:{cik}...\n{utils.traceback.format_exc()}") continue # grep the target name in case-sensitive way. or set flags = re.IGNORECASE for a case-insensitive search ana = Analysis(target_name, index_file) docs = ana.locate_target_documents(flags) doc_found = "" row = 1 # parse the html and locate the acquisition paragraph for filename, dict in docs.items(): row = row + 1 if doc_found != "": worksheet.write_url(f"A{row}", dict['FILLING_URL'], string=filename) continue utils.logger.info(f'\tfound {target_name} in {filename}, analyzing...') try: info = ana.extract_assets(filename, flags) except Exception as e: utils.logger.error(f"Failed in analyzing document:{filename}...\n{utils.traceback.format_exc()}") continue if info is None: worksheet.write_url(f"A{row}", dict['FILLING_URL'], string=filename) continue else: doc_found = os.path.basename(filename) worksheet.write_url(f"A{row}", dict['FILLING_URL'], red_format, string=filename) worksheet.write_string(f"B{row}", info) utils.logger.info(f"!!!!located acquisition asset report!!!!") utils.logger.info(f"\tCIK:{cik}\tTarget:{target_name}\tDate:{dict['FILLING_DATE']}\t{dict['FILLING_TYPE']}:{filename}") utils.logger.info(f"\tFilling Details:{dict['FILLING_URL']}") utils.logger.info(f"{info}\n") continue #self.assertEqual(golden_doc, doc_found) #if golden_doc != doc_found: #utils.logger.error(f"Expecting initial report: {golden_doc} for {cik} {target_name}, but it locates: {doc_found}") workbook.close()
pressures = [0.1, 1,10,25,50] nozzles = [100000, 300000, 400000, 500000, 800000,900000] angles = [5,10,15,20,30,35,40,50,60] Results = [] counter = 0 for distance in distances: for diameter in diameters: for pressure in pressures: sim = gasPhaseSimulation(sample_nozzle_distance = distance, pressure = pressure, source_diameter = diameter, initial_KE = 1100, loss_function = loss_function) sim.simulateMany(5000, 'start finish') results = sim.start_finish A = Analysis(results, parameters = sim.parameters) for nozzle in nozzles: nozzle_diameter = nozzle for angle in angles: A.pathHistogram(bins=200, x_limits=(290000,350000), accept_angle=angle, nozzle_diameter=nozzle_diameter) A.areasUnderProfiles() params = A.parameters p = A.profiles a = A.areas avg = A.getAveragePathLength() params['avg_path'] = avg Results += [params.copy()] print('counter : ' + str(counter)) counter+=1
== date.replace(hour=0, minute=0, second=0)): return False # The strategy needs to be active. if strategy["action"] == "hold": return False # We need to know the stock price. if not strategy["price_at"] or not strategy["price_eod"]: return False return True if __name__ == "__main__": analysis = Analysis(logs_to_cloud=False) trading = Trading(logs_to_cloud=False) twitter = Twitter(logs_to_cloud=False) # Look up the metadata for the tweets. tweets = twitter.get_tweets(SINCE_TWEET_ID) events = [] for tweet in tqdm(tweets): event = {} timestamp_str = tweet["created_at"] timestamp = trading.utc_to_market_time( datetime.strptime(timestamp_str, "%a %b %d %H:%M:%S +0000 %Y")) text = tweet["text"] event["timestamp"] = timestamp
def main(): f = open('try_3.txt', 'w') g = open('accs.txt', 'w') g.close() task = MarioTask("testbed", initMarioMode=2) task.env.initMarioMode = 2 task.env.levelDifficulty = 1 results = [] names = [] iterations = 20 rounds = 30 learning_samples = 33 eval_samples = 10 # iterations = 5 # rounds = 2 # learning_samples = 3 # eval_samples = 2 if args['noisy']: prefix = '-noisy-sup-eval' dire = './training_data_noisy/' agent = NoisySupervise(IT, useKMM=False) else: prefix = '-sup-eval' dire = './training_data/' agent = Supervise(IT, useKMM=False) if args['linear']: agent.learner.linear = True prefix = 'svc' + prefix else: agent.learner.linear = False prefix = 'dt' + prefix exp = EpisodicExperiment(task, agent) E = Evaluator(agent, exp) sl_data, sup_data, acc, loss, js, test_acc = E.eval( rounds=rounds, iterations=iterations, learning_samples=learning_samples, eval_samples=eval_samples, prefix=prefix, directory=dire) np.save('./data/' + prefix + '-sl_data.npy', sl_data) np.save('./data/' + prefix + '-acc.npy', acc) np.save('./data/' + prefix + '-loss.npy', loss) np.save('./data/' + prefix + '-js.npy', js) np.save('./data/' + prefix + '-test_acc.npy', test_acc) analysis = Analysis() analysis.get_perf(sl_data, range(iterations)) analysis.plot(names=['Supervised Learning'], label='Reward', filename='./results/' + prefix + '-return_plots.eps') #, ylims=[0, 1600]) acc_a = Analysis() acc_a.get_perf(acc, range(iterations)) acc_a.plot(names=['Supervised Learning Acc.'], label='Accuracy', filename='./results/' + prefix + '-acc_plots.eps', ylims=[0, 1]) loss_a = Analysis() loss_a.get_perf(loss, range(iterations)) loss_a.plot(names=['Supervised Learning loss'], label='Loss', filename='./results/' + prefix + '-loss_plots.eps', ylims=[0, 1]) js_a = Analysis() js_a.get_perf(js, range(iterations)) js_a.plot(names=['Supervised Learning'], label='J()', filename='./results/' + prefix + '-js_plots.eps') test_acc_a = Analysis() test_acc_a.get_perf(test_acc, range(iterations)) test_acc_a.plot(names=['Supervised Learning Acc.'], label='Test Accuracy', filename='./results/' + prefix + '-test_acc_plots.eps', ylims=[0, 1]) #agent.saveModel() print "finished"
from algorithm import Algorithm from analysis import Analysis if __name__ == '__main__': tool = Analysis('results_algorithm/') name = 'RL' algorithm = Algorithm(name, param=100) runtime = algorithm.execute(network_path='networks/', sub_filename='sub-wm.txt', req_num=1000) tool.save_evaluations(algorithm.evaluation, '%s.txt' % name) print(runtime)
def main(): f = open('try_3.txt', 'w') g = open('accs.txt', 'w') g.close() task = MarioTask("testbed", initMarioMode=2) task.env.initMarioMode = 2 task.env.levelDifficulty = 1 results = [] names = [] with open('type.txt', 'w') as f: f.write('svc') # # #test dagger # iterations = 1 # rounds = 1 iterations = 50 rounds = 15 #agent = Dagger(IT,useKMM = False) #exp = EpisodicExperiment(task, agent) #T = Tester(agent,exp) #dagger_results = T.test(rounds = rounds,iterations = iterations) #dagger_data = dagger_results[-1] #dagger_results = dagger_results[:-1] #results.append(dagger_results) #names.append('dagger') #pickle.dump(results,open('results.p','wb')) #agent = Dagger(IT, useKMM=False) #exp = EpisodicExperiment(task, agent) #T = Tester(agent, exp) #dagger_data, _, acc = T.test(rounds = rounds, iterations = iterations) agent = Supervise(IT, useKMM=False) exp = EpisodicExperiment(task, agent) T = Tester(agent, exp) prefix = 'svc-sup-change' sl_data, sup_data, acc = T.test(rounds=rounds, iterations=iterations, prefix=prefix) np.save('./data/svc-sup-change-sup_data.npy', sup_data) np.save('./data/svc-sup-change-sl_data.npy', sl_data) np.save('./data/svc-sup-change-acc.npy', acc) # IPython.embed() analysis = Analysis() analysis.get_perf(sup_data, range(iterations)) analysis.get_perf(sl_data, range(iterations)) analysis.plot(names=['Supervisor', 'Supervised Learning'], label='Reward', filename='./results/svc-sup-change-return_plots.eps' ) #, ylims=[0, 1600]) acc_a = Analysis() acc_a.get_perf(acc, range(iterations)) acc_a.plot(names=['Supervised Learning Acc.'], label='Accuracy', filename='./results/svc-sup-change-acc_plots.eps') """ agent = Dagger(IT,useKMM = False) exp = EpisodicExperiment(task, agent) T = Tester(agent,exp) dagger_data, _, acc = T.test(rounds = rounds, iterations = iterations) np.save('./data/dagger_data.npy', dagger_data) np.save('./data/acc.npy', acc) IPython.embed() analysis = Analysis() analysis.get_perf(dagger_data, range(iterations)) analysis.plot(names=['DAgger'], label='Reward', filename='./results/return_plots.eps') acc_a = Analysis() acc_a.get_perf(acc, range(iterations)) acc_a.plot(names=['DAgger Acc.'], label='Accuracy', filename='./results/acc_plots.eps') """ #agent = Supervise(IT,useKMM = False) #exp = EpisodicExperiment(task, agent) #T = Tester(agent,exp) #supervise_results = T.test(rounds = rounds, iterations = iterations) #supervise_data = supervise_results[-1] #supervise_results = supervise_results[:-1] #results.append(supervise_results) #names.append('supervise') #pickle.dump(results,open('results.p','wb')) #IPython.embed() #analysis = Analysis() #analysis.get_perf(supervise_data, results[1][5]) #analysis.get_perf(dagger_data, results[0][5]) #analysis.plot(names=['Supervise', 'DAgger'], label='Reward', filename='./return_plot.eps')#, ylims=[-1, 0]) # agent = Sheath(IT,useKMM = False,sigma = 1.0) # exp = EpisodicExperiment(task, agent) # T = Tester(agent,exp) # dagger_results = T.test(rounds = 10,iterations = 35) # results.append(dagger_results) # names.append('sheath_1') # pickle.dump(results,open('results.p','wb')) # agent = Sheath(IT,useKMM = False,sigma = 1e-1) # exp = EpisodicExperiment(task, agent) # T = Tester(agent,exp) # dagger_results = T.test(rounds = 10,iterations = 35) # results.append(dagger_results) # names.append('sheath_1') # pickle.dump(results,open('results.p','wb')) # agent = Sheath(IT,useKMM = False,sigma = 0.5) # exp = EpisodicExperiment(task, agent) # T = Tester(agent,exp) # dagger_results = T.test(rounds = 10,iterations = 35) # results.append(dagger_results) # names.append('sheath_1') # pickle.dump(results,open('results.p','wb')) # agent = Sheath(IT,useKMM = False,sigma = 1e-1) # exp = EpisodicExperiment(task, agent) # T = Tester(agent,exp) # dagger_results = T.test(rounds = 4,iterations = 35) # results.append(dagger_results) # names.append('sheath_1') # agent = Sheath(IT,useKMM = False,sigma = 1e-2) # exp = EpisodicExperiment(task, agent) # T = Tester(agent,exp) # dagger_results = T.test(rounds = 4,iterations = 35) # results.append(dagger_results) # names.append('sheath_1') # # # # # #test big ahude # agent = Ahude(IT,f,gamma = 1e-2,labelState = True, useKMM = True) # exp = EpisodicExperiment(task, agent) # T = Tester(agent,exp) # ahude_big_results = T.test(rounds = 3) # results.append(ahude_big_results) # names.append('ahude_1e-1') # pickle.dump(results,open('results.p','wb')) # # # # # #test med ahude # agent = Ahude(IT,f,gamma = 1e-2,labelState = False,useKMM = True) # exp = EpisodicExperiment(task, agent) # T = Tester(agent,exp) # ahude_med_results = T.test(rounds = 3) # results.append(ahude_med_results) # names.append('ahude_1e-2') # # # # # # # # # #test small ahude # agent = Ahude(IT,f,gamma = 1e-3) # exp = EpisodicExperiment(task, agent) # T = Tester(agent,exp) # ahude_small_results = T.test() # results.append(ahude_small_results) # names.append('ahude_1e-3') # pickle.dump(results,open('results.p','wb')) #plt.figure(1) #for i in range(len(results)): # plt.plot(results[i][5],results[i][1]) #plt.legend(names,loc='upper left') # plt.figure(2) # for i in range(len(results)): # plt.plot(results[i][0]) # plt.legend(names,loc='upper left') # plt.figure(3) # for i in range(0,len(results)): # plt.plot(results[i][3]) # plt.legend(names,loc='upper left') plt.show() # IPython.embed() f.close() #agent.saveModel() print "finished"
def setUp(self): self.analysis = Analysis() self.analysis.load_har_file(har_filename_for_tests) pass
from analysis import Analysis from models import * analysis = Analysis('weather', normalize=True) analysis.load_data() # Move this into a function to allow for quickly switching between predictors and features # Or just leave it since will do our own feature selection? predictor = 'actual_mean_temp' features = [predictor] analysis.set_feature_predictor_columns(features, predictor) analysis.train_test_split() NUM_DAYS = 5 NUM_FEATURES = len(analysis.features) from keras.models import Sequential from keras import optimizers from keras import layers mask_value = 0 nn = Sequential() nn.add( layers.Masking(mask_value=mask_value, input_shape=(NUM_DAYS, NUM_FEATURES))) nn.add( layers.LSTM(50, activation='tanh', input_shape=(NUM_DAYS, NUM_FEATURES),
'realm': realm, 'frequency': frequency, 'variable': variable, 'ensemble_element': ensemble_element } # Two pressure levels are already registered, you can choose to work at # 850 hPa to seek polar jets, or at 250 hPa for both of them. # If you want to study another pressure level, be sure to adjust the "plev" # variable to have the right data in the pickle file. if pressure == 85000: plev = 1 if pressure == 25000: plev = 4 analysis = Analysis(input_options) analysis.get_files(only_files=True) print len(analysis.files), "files will be analyzed" data_list = [] # Files are loaded here... for fichier in range(0, len( analysis.files)): #Loop over all the data files available analysis.mf = Dataset(analysis.files[fichier]) variable_data = np.asarray(analysis.mf.variables[analysis.variable][:], dtype=np.float32)
def run_simulation(scenario : dict, t_step : float = 1) -> dict: """ Runs the simulation process. """ t0 = time.time() # Initial setup scenario['timesteps'] = [] scenario['debug'] = [] n_all = len(scenario['persons']) # Divide persons to compartments healthy = {} sick = {} healed = {} for id_code, person in scenario['persons'].items(): if person.sick_now: sick[id_code] = person elif person.healed: healed[id_code] = person else: healthy[id_code] = person # Initialize callback status for cb in scenario['callbacks']: cb['done'] = False # Run simulation sim_time = 0 healthy_idx = 0 while sim_time < scenario['sim_hours']: n_healthy = len(healthy) healthy_list = list(healthy.values()) # Update persons to_healed = [] to_sick = [] for id_code, person in sick.items(): person.update_infection(sim_time) if person.healed: to_healed.append(id_code) continue if person.expose_others: if n_healthy == 0: continue # Calculate number of healthy persons to expose n = person.n_exposed n *= float(n_healthy) / float(n_all) frac = n - int(n) n = int(n) if np.random.rand() < frac: n += 1 # Expose persons for _ in range(n): healthy_idx += 1 if healthy_idx >= n_healthy: healthy_idx = 0 exp_person = healthy_list[healthy_idx] new_infection = exp_person.expose( infected_by=person.id_code, virus=scenario['virus'], timestamp=sim_time ) # If exposure caused infection, store the contact information if new_infection: person.report_infected_others(id_code=exp_person.id_code) to_sick.append(exp_person.id_code) for id_code in to_healed: healed[id_code] = sick.pop(id_code) for id_code in to_sick: sick[id_code] = healthy.pop(id_code) scenario['timesteps'].append(sim_time) sim_time += t_step # Run timed callbacks for cb in scenario['callbacks']: if (cb['trigger'] < sim_time) and (cb['done'] == False): cb['function'](scenario) cb['done'] = True # Analyze the data analysis = Analysis() scenario = analysis.run_analysis(scenario) # Progress monitoring t1 = time.time() print('Scenario {} done in {:0.1f} s.'.format(scenario['label'], t1 - t0)) return scenario
from analysis import Analysis from algorithm import Algorithm # 5 0.37612838515546637 1525.0753908157349 683.8999999999988 # 10 0.3711133400200602 1613.1240510940552 684.0999999999992 # 15 0.3510531594784353 761.4128797054291 647.3999999999991 # 20 0.3781344032096289 755.6144239902496 674.599999999999 # if __name__ == '__main__': # # tool = Analysis('results_epoch/') # epoch = 3 # algorithm = Algorithm(name='RLQ', param=epoch) # runtime = algorithm.execute(network_path='networks/', # sub_filename='sub-ts.txt', # req_num=1000) # tool.save_evaluations(algorithm.evaluation, '%s.txt' % epoch) # qos_loss=algorithm.evaluation.total_loss/algorithm.evaluation.total_accepted # tool.save_epoch(epoch, algorithm.evaluation.acc_ratio, runtime, qos_loss) if __name__ == '__main__': tool = Analysis('results_epoch/') for i in range(5): epoch = (i + 1) * 10 algorithm = Algorithm(name='RLQ', param=epoch) runtime = algorithm.execute(network_path='networks/', sub_filename='sub-ts.txt', req_num=1000) tool.save_evaluations(algorithm.evaluation, '%s.txt' % epoch) qos_loss = algorithm.evaluation.total_loss / algorithm.evaluation.total_accepted
ylabel="Data Values", color="k", ) """ For every dataset we now create an `Analysis` class using it and use `Emcee` to fit it with a `Gaussian`. """ for dataset_name in dataset_names: """Load the dataset from the `autofit_workspace/dataset` folder.""" dataset_path = path.join("dataset", "example_1d", dataset_name) data = af.util.numpy_array_from_json( file_path=path.join(dataset_path, "data.json")) noise_map = af.util.numpy_array_from_json( file_path=path.join(dataset_path, "noise_map.json")) """Create the `DynestyStatic` `NonLinearSearch` and use it to fit the data.""" analysis = Analysis(data=data, noise_map=noise_map) emcee = af.DynestyStatic( path_prefix=path.join("howtofit", "chapter_graphical_models"), name="tutorial_1_global_model", unique_tag=dataset_name, ) print( f"Emcee has begun running, checkout \n" f"autofit_workspace/output/howtofit/chapter_graphica_models/tutorial_1_global_model/{dataset_name} for live \n" f"output of the results. This Jupyter notebook cell with progress once Emcee has completed, this could take a \n" f"few minutes!") emcee.fit(model=model, analysis=analysis) """
def analysis(): return Analysis(logs_to_cloud=False)
def transformation(self, sent, ant, justify=False): """ Transformation step in the simplification process. This is a recursive method that receives two parameters: @param sent: sentence to be simplified. @param ant: previous sentence. If sent = ant, then no simplification should be performed. @param justify: controls cases where sentence order is inverted and should invert the entire recursion. @return: the simplified sentences. """ def remove_all(aux, item): """ Remove all incidences of a node in a graph (needed for graphs with cycles). @param aux: auxiliary of parse structure @param item: node to be removed """ for a in aux.keys(): for d in aux[a].keys(): if item in aux[a][d]: aux[a][d].remove(item) def recover_punct(final, s): """ Recover the punctuation of the sentence (needed because the dependency parser does not keep punctuation. @param final: the final dictionary with the words in order @param s: the tokenised sentence with the punctuation marks @return the final dictionary with the punctuation marks """ char_list = "``\'\'" ant = 0 for k in sorted(final.keys()): if int(k) - ant == 2: if s[k - 2] in string.punctuation + char_list: final[k - 1] = s[k - 2] elif s[k - 2] == "-RRB-": final[k - 1] = ")" elif s[k - 2] == "-LRB-": final[k - 1] = "(" if int(k) - ant == 3: if s[k - 2] in string.punctuation + char_list and s[ k - 3] in string.punctuation + char_list: final[k - 1] = s[k - 2] final[k - 2] = s[k - 3] ant = k return final def build(root, dep, aux, words, final, yes_root=True, previous=None): """ Creates a dictionary with the words of a simplified clause, following the sentence order. This is a recursive method that navigates through the dependency tree. @param root: the root node in the dependency tree @param dep: the dependencies of the root node @param aux: the auxiliary parser output @param words: auxiliary parsed words @param final: dictionary with the positions and words @param yes_root: flag to define whether or not the root node should be included @param previous: list of nodes visited """ ## controls recursion if previous == None: previous = [] if root in previous: return previous.append(root) ## for cases where the rule does not include the root node if yes_root: final[root] = words[root - 1][0] previous.append(root) for k in dep.keys(): for i in dep[k]: if i in aux.keys(): deps = aux[i] ## needed for breaking loops -- solved by the recursion condition #for d in deps.keys(): # if i in deps[d]: # deps[d].remove(i) build(i, deps, aux, words, final, previous=previous) final[i] = words[i - 1][0] def conjoint_clauses(aux, words, root, deps_root, ant, _type, rel): """ Simplify conjoint clauses @param aux: auxiliary parser output @param words: auxiliary words and POS tags structure @param root: root node in the dependency tree @param deps_root: dependencies of the root node @param ant: previous sentence (for recursion purposes) @param _type: list of markers found in the sentence that can indicate conjoint clauses @param rel: parser relation between the main and the dependent clause (can be 'advcl' or 'conj') @return: a flag that indicates whether or not the sentence was simplified and the result sentence (if flag = False, ant is returned) """ ## split the clauses others = deps_root[rel] pos = 0 s1 = s2 = "" v_tense = "" for o in others: flag = True if o not in aux: flag = False continue deps_other = aux[o] ## check the marker position ('when' is advmod, while others are mark) if 'advcl' in rel: if 'mark' in deps_other.keys(): mark = deps_other['mark'][0] mark_name = words[mark - 1][0].lower() elif 'advmod' in deps_other.keys(): mark = deps_other['advmod'][0] mark_name = words[mark - 1][0].lower() else: flag = False #needed for broken cases continue else: if 'cc' in deps_root.keys() and 'conj' in rel: conj = deps_root[rel][0] if 'vm' in words[conj - 1][1][ 'PartOfSpeech'] and 'vm' in words[root - 1][1][ 'PartOfSpeech']: #needed for broken cases like 'Care and support you won't have to pay towards' mark = deps_root['cc'][0] mark_name = words[mark - 1][0].lower() else: flag = False continue else: flag = False continue ## hack for simpatico use cases if mark_name == "and" and words[mark - 2][0].lower( ) == "care" and words[mark][0].lower() == "support": flag = False continue ## dealing with cases without subject if 'nsubj' not in deps_other and 'nsubj' in deps_root: deps_other['nsubj'] = deps_root['nsubj'] elif 'nsubj' not in deps_other and 'nsubjpass' in deps_root: deps_other['nsubj'] = deps_root['nsubjpass'] elif 'nsubj' not in deps_other and 'nsubj' not in deps_root: flag = False continue ## check if the marker is in the list of selected markers if mark_name in _type: ## check if verbs have objects tag_list = ('advcl', 'xcomp', 'acomp', 'amod', 'appos', 'cc', 'ccomp', 'dep', 'dobj', 'iobj', 'nwe', 'pcomp', 'pobj', 'prepc', 'rcmod', 'ucomp', 'nmod', 'auxpass', 'advmod', 'prep') #if not any([t in tag_list for t in deps_root.keys()]): #return False, ant # flag = False # continue #elif not any([t in tag_list for t in deps_other.keys()]): #return False, ant # flag = False # continue #if (len(deps_root) < 2 or len(deps_other) < 2): # return False, ant ## delete marker and relation from the graph if 'advcl' in rel: if 'mark' in deps_other.keys(): del deps_other['mark'][0] elif 'advmod' in deps_other.keys(): del deps_other['advmod'][0] else: del deps_root['cc'][0] #del deps_root[rel][pos] #pos+=1 deps_root[rel].remove(o) ## for cases with time markers -- This + modal + happen modal = None if 'aux' in deps_root and mark_name in self.time: modal_pos = deps_root['aux'][0] modal = words[modal_pos - 1][0] ## for cases either..or with the modal verb attached to the main clause if 'aux' in deps_root and mark_name in self.condition2: deps_other['aux'] = deps_root[u'aux'] ## built the sentence again final_root = {} build(root, deps_root, aux, words, final_root) final_deps = {} build(o, deps_other, aux, words, final_deps) ## TODO: remove this part from here --> move to another module: self.generation root_tag = words[root - 1][1]['PartOfSpeech'] justify = True #if ((root > o) and (mark_name in self.time and mark>1)) or (mark_name == 'because' and mark > 1): if (root > o) or (mark_name == 'because' and mark > 1): if (mark_name in self.time and mark == 1): sentence1, sentence2 = self.generation.print_sentence( final_root, final_deps, root_tag, mark_name, mark, modal) else: sentence1, sentence2 = self.generation.print_sentence( final_deps, final_root, root_tag, mark_name, mark, modal) else: sentence1, sentence2 = self.generation.print_sentence( final_root, final_deps, root_tag, mark_name, mark, modal) s1 = self.transformation(sentence1, ant, justify) s2 = self.transformation(sentence2, ant) flag = True else: flag = False continue if flag: return flag, s1 + " " + s2 else: return flag, ant def relative_clauses(aux, words, root, deps_root, ant, rel): """ Simplify relative clauses @param aux: auxiliary parser output @param words: auxiliary words and POS tags structure @param root: root node in the dependency tree @param deps_root: dependencies of the root node @param ant: previous sentence (for recursion purposes) @param rel: parser relation between the main and the dependent clause (can be 'nsubj' or 'dobj') @return: a flag that indicates whether or not the sentence was simplified and the result sentence (if flag = False, ant is returned) """ subj = deps_root[rel][0] if subj not in aux.keys(): return False, ant deps_subj = aux[subj] if 'acl:relcl' in deps_subj.keys() or 'rcmod' in deps_subj.keys(): if 'acl:relcl' in deps_subj.keys(): relc = deps_subj['acl:relcl'][0] type_rc = 'acl:relcl' else: relc = deps_subj['rcmod'][0] type_rc = 'rcmod' deps_relc = aux[relc] subj_rel = "" if 'nsubj' in deps_relc.keys(): subj_rel = 'nsubj' elif 'nsubjpass' in deps_relc.keys(): subj_rel = 'nsubjpass' else: subj_rel = 'mark' if 'ref' in deps_subj: to_remove = deps_subj['ref'][0] mark = words[deps_subj['ref'][0] - 1][0].lower() elif subj_rel in deps_relc: to_remove = deps_relc[subj_rel][0] mark = words[deps_relc[subj_rel][0] - 1][0].lower() if mark in self.relpron: deps_relc[subj_rel][0] = subj remove_all(aux, to_remove) elif 'dobj' in deps_relc: ## needed for cases where the subject of the relative clause is the object obj = deps_relc['dobj'][0] if 'poss' in aux[obj]: mod = aux[obj]['poss'][0] aux_words = list(words[mod - 1]) aux_words[0] = words[subj - 1][0] + '\'s' words[mod - 1] = tuple(aux_words) aux[mod] = aux[subj] else: return False, ant else: return False, ant #for borken cases - " There are some situations where it is particularly important that you get financial information and advice that is independent of us." del aux[subj][type_rc] if 'punct' in deps_subj.keys(): del aux[subj]['punct'] if 'punct' in deps_relc.keys(): del aux[relc]['punct'] final_root = {} build(root, deps_root, aux, words, final_root) final_relc = {} build(relc, deps_relc, aux, words, final_relc) if justify: sentence2, sentence1 = self.generation.print_sentence( final_root, final_relc) else: sentence1, sentence2 = self.generation.print_sentence( final_root, final_relc) s1 = self.transformation(sentence1, ant, justify) s2 = self.transformation(sentence2, ant) return True, s1 + " " + s2 else: return False, ant def appositive_phrases(aux, words, root, deps_root, ant): """ Simplify appositive phrases @param aux: auxiliary parser output @param words: auxiliary words and POS tags structure @param root: root node in the dependency tree @param deps_root: dependencies of the root node @param ant: previous sentence (for recursion purposes) @return: a flag that indicates whether or not the sentence was simplified and the result sentence (if flag = False, ant is returned) """ ## apposition needs to have a subject -- same subject of the mais sentence. if 'nsubj' in deps_root.keys(): subj = deps_root['nsubj'][0] subj_word = words[subj - 1][0] if subj not in aux: return False, ant deps_subj = aux[subj] v_tense = words[root - 1][1]['PartOfSpeech'] n_num = words[subj - 1][1]['PartOfSpeech'] if 'amod' in deps_subj: ## bug -- this generates several mistakes... mod = deps_subj['amod'][0] if mod in aux: deps_mod = aux[mod] else: deps_mod = {} del aux[subj]['amod'] deps_subj = aux[subj] ## Treat simple cases such as 'general rule' #if 'JJ' in words[mod-1][1]['PartOfSpeech'] and len(deps_mod.keys()) == 0: if 'JJ' in words[ mod - 1][1]['PartOfSpeech'] and 'punct' not in deps_subj: return False, ant elif 'appos' in deps_subj: mod = deps_subj['appos'][0] if mod in aux: deps_mod = aux[mod] else: deps_mod = {} del aux[subj]['appos'] deps_subj = aux[subj] else: return False, ant if 'punct' in deps_subj.keys(): del deps_subj['punct'] final_root = {} build(root, deps_root, aux, words, final_root) final_appos = {} build(mod, deps_mod, aux, words, final_appos) final_subj = {} build(subj, deps_subj, aux, words, final_subj) if len(final_appos.keys()) < 2: return False, ant sentence1, sentence2 = self.generation.print_sentence_appos( final_root, final_appos, final_subj, v_tense, n_num, subj_word) s1 = self.transformation(sentence1, ant) s2 = self.transformation(sentence2, ant) return True, s1 + " " + s2 else: return False, ant def passive_voice(aux, words, root, deps_root, ant): """ Simplify sentence from passive to active voice. @param aux: auxiliary parser output @param words: auxiliary words and POS tags structure @param root: root node in the dependency tree @param deps_root: dependencies of the root node @param ant: previous sentence (for recursion purposes) @return: a flag that indicates whether or not the sentence was simplified and the result sentence (if flag = False, ant is returned) """ if 'auxpass' in deps_root.keys(): if 'nmod:por' in deps_root.keys(): if 'nsubjpass' not in deps_root: return False, ant subj = deps_root['nsubjpass'][0] if subj in aux: deps_subj = aux[subj] else: deps_subj = {} root_tense = words[root - 1][1]['PartOfSpeech'] aux_tense = words[deps_root['auxpass'][0] - 1][1]['PartOfSpeech'] v_aux = None if 'v' in aux_tense and 'aux' in deps_root.keys(): aux_tense = words[deps_root['aux'][0] - 1][1]['PartOfSpeech'] v_aux = words[deps_root['aux'][0] - 1][0] del deps_root['aux'] del deps_root['auxpass'] del deps_root['nsubjpass'] if len(deps_root['nmod:por']) > 1: mod = deps_root['nmod:por'][1] mod2 = deps_root['nmod:por'][0] deps_mod = aux[mod] deps_mod2 = aux[mod2] if 'case' in deps_mod: if words[deps_mod[u'case'][0] - 1][0].lower() != 'por': return False, ant del deps_mod['case'] del deps_root['nmod:por'] subj_tag = words[mod - 1][1]['PartOfSpeech'] subj_word = words[mod - 1][0] final_subj = {} build(mod, deps_mod, aux, words, final_subj) final_obj = {} build(subj, deps_subj, aux, words, final_obj) final_mod2 = {} build(mod2, deps_mod2, aux, words, final_mod2) final_root = {} build(root, deps_root, aux, words, final_root, False) sentence1 = self.generation.print_sentence_voice( final_subj, final_obj, words[root - 1][0], root_tense, v_aux, aux_tense, subj_tag, subj_word, final_mod2, final_root) s1 = self.transformation(sentence1, ant) return True, s1 elif 'case' in deps_mod2: if words[deps_mod2['case'][0] - 1][0].lower() != 'por': return False, ant del deps_mod2['case'] del deps_root['nmod:por'] subj_tag = words[mod2 - 1][1]['PartOfSpeech'] subj_word = words[mod2 - 1][0] final_subj = {} build(mod2, deps_mod2, aux, words, final_subj) final_obj = {} build(subj, deps_subj, aux, words, final_obj) final_mod2 = {} build(mod, deps_mod, aux, words, final_mod2) final_root = {} build(root, deps_root, aux, words, final_root, False) sentence1 = self.generation.print_sentence_voice( final_subj, final_obj, words[root - 1][0], root_tense, v_aux, aux_tense, subj_tag, subj_word, final_mod2, final_root) s1 = self.transformation(sentence1, ant) return True, s1 else: return False, ant else: mod = deps_root['nmod:por'][0] deps_mod = aux[mod] if 'case' in deps_mod: if words[deps_mod['case'][0] - 1][0].lower() != 'por': return False, ant del deps_mod['case'] del deps_root['nmod:por'] subj_tag = words[mod - 1][1]['PartOfSpeech'] subj_word = words[mod - 1][0] final_subj = {} build(mod, deps_mod, aux, words, final_subj) final_obj = {} build(subj, deps_subj, aux, words, final_obj) final_root = {} build(root, deps_root, aux, words, final_root, False) sentence1 = self.generation.print_sentence_voice( final_subj, final_obj, words[root - 1][0], root_tense, v_aux, aux_tense, subj_tag, subj_word, final_root) s1 = self.transformation(sentence1, ant) return True, s1 else: return False, ant else: return False, ant else: return False, ant ## MAIN OF TRANSFORMATION ## control recursion: check whether there is no simplification to be done if sent == ant: return sent flag = False ant = sent ## parser try: parsed = self.parser.process(sent) except AssertionError: return ant ## data structure for the words and POS words = parsed['words'] ## data structure for the dependency parser dict_dep = self.parser.transform(parsed) ## check whether or not the sentence has a root node if 0 not in dict_dep: return ant root = dict_dep[0]['root'][0] ## check for root dependencies if root not in dict_dep: return ant deps_root = dict_dep[root] ## get tokens sent_tok = [] for w in words: sent_tok.append(w[0]) ## dealing with questions ## TODO: improve this control with parser information. if sent_tok[0].lower() in ("what", "where", "when", "whose", "who", "which", "whom", "whatever", "whatsoever", "whichever", "whoever", "whosoever", "whomever", "whomsoever", "whoseever", "whereever") and sent_tok[-1] == "?": return ant ## deal with apposition flag, simpl = appositive_phrases(dict_dep, words, root, deps_root, ant) if flag: return simpl ## analyse whether or not a sentence has simplification clues (in this case, discourse markers or relative pronouns) a = Analysis(sent_tok, self.cc, self.relpron) flag_cc, type_cc = a.analyse_cc() ## if sentence has a marker that requires attention if flag_cc: ## sorting according to the order of the relations rel = {} for k in deps_root.keys(): if 'conj' in k or 'advcl' in k: others = sorted(deps_root[k], reverse=True) cnt = 0 for o in others: deps_root[k + str(cnt)] = [] deps_root[k + str(cnt)].append(o) rel[k + str(cnt)] = deps_root[k][0] cnt += 1 del deps_root[k] sorted_rel = sorted(rel.items(), key=operator.itemgetter(1)) for k in sorted_rel: flag, simpl = conjoint_clauses(dict_dep, words, root, deps_root, ant, type_cc, k[0]) if flag: return simpl flag_rc, type_rc = a.analyse_rc() ## if sentence has a relative pronoun if flag_rc: ## check where is the dependency of the relative clause if 'nsubj' in deps_root: flag, simpl = relative_clauses(dict_dep, words, root, deps_root, ant, 'nsubj') if flag: return simpl elif 'dobj' in deps_root: flag, simpl = relative_clauses(dict_dep, words, root, deps_root, ant, 'dobj') if flag: return simpl ## deal with passive voice flag, simpl = passive_voice(dict_dep, words, root, deps_root, ant) if flag: return simpl ## return the original sentence if no simplification was done if flag == False: return ant
def get_tweet_text(tweet_id): """Looks up the text for a single tweet.""" tweet = get_tweet(tweet_id) analysis = Analysis(logs_to_cloud=False) return analysis.get_expanded_text(tweet)
def checkup(symbols): latest_date = utils.latest_date_str() q = Quote() td = TechData() a = Analysis() swidth = [] swidth_pb = [] fwidth = [] fwidth_pb = [] fwidth_roc_pb = [] rwb = [] macd_hist_pb = [] swidth_roc_pb = [] latest_timestamp = None for sym in symbols: print '\n' print 'Checking up', sym, '......' df = q.intraday(sym) if latest_timestamp is None: latest_timestamp = df.index[-1] td.macd(df) td.ma(df) td.guppy(df) a.guppy_alignment(df) a.guppy_roc(df) tmp_df = df[[ 'swidth', 'swidth_pb', 'fwidth', 'fwidth_pb', 'fwidth_roc_pb', 'rwb', 'macd_hist_pb', 'swidth_roc_pb' ]] print '++++++++', sym, '+++++++++' print tmp_df.tail(5) swidth.append(df['swidth'][-1]) swidth_pb.append(df['swidth_pb'][-1]) fwidth.append(df['fwidth'][-1]) fwidth_pb.append(df['fwidth_pb'][-1]) fwidth_roc_pb.append(df['fwidth_roc_pb'][-1]) rwb.append(df['rwb'][-1]) macd_hist_pb.append(df['macd_hist_pb'][-1]) swidth_roc_pb.append(df['swidth_roc_pb'][-1]) total_df = pd.DataFrame(index=symbols) total_df['swidth'] = swidth total_df['swidth_pb'] = swidth_pb total_df['fwidth'] = fwidth total_df['fwidth_pb'] = fwidth_pb total_df['fwidth_roc_pb'] = fwidth_roc_pb total_df['rwb'] = rwb total_df['macd_hist_pb'] = macd_hist_pb total_df['swidth_roc_pb'] = swidth_roc_pb print '\n' print '---------------------------------', latest_timestamp, '---------------------------------' print total_df
def __init__(self): self.crawl = Crawl() self.pipe = Pipeline() self.analysis = Analysis()
""" res = [] for match_id in match_ids: url = 'https://api.opendota.com/api/matches/%s' % str(match_id) r = requests.get(url) r_str = r.content.decode() r_json = json.loads(r_str) res.append(r_json) print("players" in r_json) time.sleep(1) pickle.dump(res, open("match_details_3p.pkl", "wb")) return res if __name__ == '__main__': config = BaseConfig() analysis = Analysis() # request_url = construct_request_url(config.account_ids) # print(request_url) # match_ids = get_match_ids(request_url) # print(len(match_ids)) # match_detail = get_matches_detail(match_ids) match_detail = pickle.load(open("match_details_5p.pkl", "rb")) total_odds = analysis.total_odds(config.account_ids[0], match_detail, verbose=True) stat = analysis.hero_odds(config.account_ids, match_detail, verbose=True)
def train(self, training_set): loss_average = [] iteration = 0 start = time.time() # 训练开始 while iteration < self.num_epoch: values = [] print("Iteration %s" % iteration) # 每轮训练开始前,都需要重置底层网络和相关的强化学习环境 sub_copy = copy.deepcopy(self.sub) env = NodeEnv(self.sub) # 创建存储参数梯度的缓冲器 grad_buffer = self.sess.run(self.tvars) # 初始化为0 for ix, grad in enumerate(grad_buffer): grad_buffer[ix] = grad * 0 # 记录已经处理的虚拟网络请求数量 counter = 0 for req in training_set: # 当前待映射的虚拟网络请求ID req_id = req.graph['id'] print("\nHandling req%s..." % req_id) if req.graph['type'] == 0: print("\tIt's a newly arrived request, try to map it...") counter += 1 sub_copy.total_arrived = counter # 向环境传入当前的待映射虚拟网络 env.set_vnr(req) # 获得底层网络的状态 observation = env.reset() node_map = {} xs, acts = [], [] for vn_id in range(req.number_of_nodes()): x = np.reshape(observation, [1, observation.shape[0], observation.shape[1], 1]) sn_id = self.choose_action(observation, sub_copy, req.nodes[vn_id]['cpu'], acts) if sn_id == -1: break else: # 输入的环境信息添加到xs列表中 xs.append(x) # 将选择的动作添加到acts列表中 acts.append(sn_id) # 执行一次action,获取返回的四个数据 observation, _, done, info = env.step(sn_id) node_map.update({vn_id: sn_id}) # end for,即一个VNR的全部节点映射全部尝试完毕 if len(node_map) == req.number_of_nodes(): link_map = Network.cut_then_find_path(sub_copy, req, node_map) reward = Evaluation.revenue_to_cost_ratio(req, link_map) if reward != -1: epx = np.vstack(xs) epy = np.eye(self.n_actions)[acts] # 返回损失函数值 loss_value = self.sess.run(self.loss, feed_dict={self.tf_obs: epx, self.input_y: epy}) print("Success! The loss value is: %s" % loss_value) values.append(loss_value) # 返回求解梯度 tf_grad = self.sess.run(self.newGrads, feed_dict={self.tf_obs: epx, self.input_y: epy}) # 将获得的梯度累加到gradBuffer中 for ix, grad in enumerate(tf_grad): grad_buffer[ix] += grad grad_buffer[0] *= reward grad_buffer[1] *= reward # 分配资源 Network.allocate(sub_copy, req, node_map, link_map) else: print("Failure!") # 当实验次数达到batch size整倍数,累积的梯度更新一次参数 if counter % self.batch_size == 0: self.sess.run(self.update_grads, feed_dict={self.kernel_grad: grad_buffer[0], self.biases_grad: grad_buffer[1]}) # 清空gradBuffer for ix, grad in enumerate(grad_buffer): grad_buffer[ix] = grad * 0 if req.graph['type'] == 1: # 收回该请求占用的资源 Network.recover(sub_copy, req) env.set_sub(sub_copy) loss_average.append(np.mean(values)) iteration = iteration + 1 end = (time.time() - start) / 3600 tool = Analysis('results_loss/') tool.save_loss(end, self.num_epoch, loss_average,"RLN3")
def test(self, rounds=20, iterations=35, learning_samples=1, eval_samples=1, prefix=''): num_ask_help = np.zeros(iterations) num_mismatch = np.zeros(iterations) avg_data = np.zeros(iterations) avg_distance = np.zeros(iterations) avg_precision = np.zeros(iterations) avg_human_input = np.zeros(iterations) num_ask_help_r = np.zeros((rounds, iterations)) num_mismatch_r = np.zeros((rounds, iterations)) avg_data_r = np.zeros((rounds, iterations)) avg_distance_r = np.zeros((rounds, iterations)) avg_precision_r = np.zeros((rounds, iterations)) avg_human_input_r = np.zeros((rounds, iterations)) sup_data_r = np.zeros((rounds, iterations)) acc_data = np.zeros((rounds, iterations)) test_acc_data = np.zeros((rounds, iterations)) avg_losses_r = np.zeros((rounds, iterations)) avg_js_r = np.zeros((rounds, iterations)) # self.exp.task.env.changeLevel() for r in range(rounds): print "Trial: " + str(r) self.agent.initialTraining = True self.exp.doEpisodes(1) self.agent.newModel() self.agent.saveModel() self.agent.initialTraining = False self.agent.loadModel() self.agent.reset() distances = np.zeros(iterations) mis_match = np.zeros(iterations) data = np.zeros(iterations) losses = np.zeros(iterations) js = np.zeros(iterations) sup_data = np.zeros(iterations) num_help = np.zeros(iterations) precision = np.zeros(iterations) human_input = np.zeros(iterations) acc = np.zeros(iterations) test_acc = np.zeros(iterations) for t in range(iterations): rewards, loss, j, sup_rewards, = self.exp.doEpisodes( 1, learning_samples, eval_samples) rewards = np.mean(rewards, axis=0) loss = np.mean(loss, axis=0) j = np.mean(j, axis=0) data[t] = rewards[-1] losses[t] = loss[-1] js[t] = j[-1] # data[t] = rewards[0][-1] # taking from the first sample if self.agent._name == 'supervise': sup_rewards = np.mean(sup_rewards, axis=0) sup_data[t] = sup_rewards[-1] #sup_data[t] = sup_rewards[0][-1] # taking from the first sample #self.agent.updateModel() acc[t] = self.agent.learner.accs test_acc[t] = self.agent.learner.test_accs # if(self.agent._getName() == 'Ahude'): # num_help[t] = self.agent.getNumHelp() # mis_match[t] = self.agent.getMismatch() # self.agent.off = True # rewards = self.exp.doEpisodes(1) # self.agent.off = False # size = len(rewards[0]) size = len(rewards) #distances[t] = rewards[0][size-1] distances[t] = rewards[size - 1] precision[t] = self.agent.learner.getPrecision() human_input[t] = self.agent.getNumHumanInput() self.agent.reset() # if(self.agent._getName() == 'Ahude'): # num_ask_help = num_ask_help + num_help # num_mismatch = num_mismatch + mis_match avg_data_r[r, :] = data avg_losses_r[r, :] = losses avg_js_r[r, :] = js if self.agent._name == 'supervise': sup_data_r[r, :] = sup_data acc_data[r, :] = acc test_acc_data[r, :] = test_acc # plot single trial if self.agent._name == 'supervise': np.save('./data/' + prefix + 'loss_round' + str(r) + '.npy', losses) np.save( './data/' + prefix + 'sl_reward_round' + str(r) + '.npy', data) np.save( './data/' + prefix + 'sup_reward_round' + str(r) + '.npy', sup_data) np.save('./data/' + prefix + 'acc_round' + str(r) + '.npy', acc) np.save( './data/' + prefix + 'test_acc_round' + str(r) + '.npy', test_acc) np.save('./data/' + prefix + 'js_round' + str(r) + '.npy', js) a = Analysis() a.get_perf(np.array([sup_data]), range(iterations)) a.get_perf(np.array([data]), range(iterations)) a.plot(names=['Supervisor', 'Supervised Learning'], label='Rewards', filename='./results/' + prefix + 'return_plot' + str(r) + '.eps') a = Analysis() a.get_perf(np.array([losses]), range(iterations)) a.plot(names=['Supervised Learning'], label='Loss', filename='./results/' + prefix + 'loss_plot' + str(r) + '.eps', ylims=[0, 1]) a = Analysis() a.get_perf(np.array([js]), range(iterations)) a.plot(names=['Supervised Learning'], label='J()', filename='./results/' + prefix + 'js_plot' + str(r) + '.eps') elif self.agent._name == 'dagger': np.save('./data/' + prefix + 'loss_round' + str(r) + '.npy', losses) np.save( './data/' + prefix + 'dagger_reward_round' + str(r) + '.npy', data) np.save('./data/' + prefix + 'acc_round' + str(r) + '.npy', acc) np.save( './data/' + prefix + 'test_acc_round' + str(r) + '.npy', test_acc) np.save('./data/' + prefix + 'js_round' + str(r) + '.npy', js) a = Analysis() a.get_perf(np.array([data]), range(iterations)) a.plot(names=['DAgger'], label='Reward', filename='./results/' + prefix + 'return_plot' + str(r) + '.eps') a = Analysis() a.get_perf(np.array([losses]), range(iterations)) a.plot(names=['DAgger'], label='Loss', filename='./results/' + prefix + 'loss_plot' + str(r) + '.eps', ylims=[0, 1]) a = Analysis() a.get_perf(np.array([js]), range(iterations)) a.plot(names=['DAgger'], label='J()', filename='./results/' + prefix + 'js_plot' + str(r) + '.eps') avg_data = data + avg_data avg_distance = distances + avg_distance avg_precision = precision + avg_precision avg_human_input = avg_human_input + human_input # self.exp.task.env.changeLevel() num_ask_help = num_ask_help / rounds num_mismatch = num_mismatch / rounds avg_data = avg_data / rounds avg_distance = avg_distance / rounds avg_precision = avg_precision / rounds avg_human_input = avg_human_input / rounds self.exp.task.env.setLevelBack() #print avg_distance if self.agent._name == 'supervise': return avg_data_r, sup_data_r, acc_data, avg_losses_r, avg_js_r, test_acc_data else: return avg_data_r, None, acc_data, avg_losses_r, avg_js_r, test_acc_data
version = packageVersion+'('+version+')' if logOut: self.log.out("# "+intro+toolName+" [version: " + version + "]") return version ############ command line testing ############ if __name__ == '__main__': ''' Command-line testing - WARNING: out of date ''' print "======== begin '" + sys.argv[0] + "' test ========" e3 = Analysis('/hive/users/tdreszer/galaxy/galaxy-dist/tools/encode/settingsE3.txt', analysisId='command-line test') samInterim = e3.registerInterimOutput('sam',e3.dir + 'rep1Interim.sam') bamTarget = e3.registerTargetOutput( 'bam',e3.dir + 'rep1Target.bam') ls = LogicalStep(e3,stepName='test Logical Step') sai1 = ls.declareGarbageFile('read1','sai') sai2 = ls.declareGarbageFile('read2','sai') sam = ls.declareInterimFile('sam') bam = ls.declareAnalysisFile('bam') ls.run() # should create temp dir, logFile etc. # fake some outputs e3.getCmdOut('echo sai1 > '+ sai1,logResult=True, log=ls.log) e3.getCmdOut('echo sam > '+ sam,logResult=True, log=ls.log) e3.getCmdOut('echo bam > '+ bam,logResult=True, log=ls.log) # try failure:
def get_session_traffic_summary(username, session_id): result = get_response_model() try: user = app.coordination.get_user(username) session = app.coordination.get_session(session_id) if not user_has_access_to_session(user, session): raise Exception("Unauthorised access") if session["intercept_mode"] != "decrypted": raise Exception( "The analysis of non-HAR files (such as the pcap files generated in Raw and Metadata modes) is not currently supported." ) result["returned"] = { "name": session["name"] if "name" in session else None, "intercept_mode": session["intercept_mode"], "created": session["created"].astimezone().isoformat(), "ended": None if not "ended" in session else session["ended"].astimezone().isoformat(), "entry_node_name": session["resources"]["entry_node"]["name"], "exit_node_name": session["resources"]["exit_node"]["name"], "analysis_generated": False, "hosts": [], "request_headers": [], "request_queries": [], "request_cookies": [], "postdata_form_params": [], "postdata_json_attributes": [] } fileinfo = get_session_data_fileinfo(session_id) if os.path.exists(fileinfo["fullpath"]): if app.coordination_config["WebAPI"][ "analytics_exec_mode"] == "dask": analysis = Analysis_Dask( app.coordination_config["WebAPI"] ["analytics_exec_dask_config_scheduler"]) analysis.load_har_file(fileinfo["fullpath"]) analysis_tables = analysis.get_all_summary_tables() result["returned"] = {**result["returned"], **analysis_tables} result["returned"]["analysis_generated"] = True else: analysis = Analysis() analysis.load_har_file(fileinfo["fullpath"]) result["returned"]["hosts"] = analysis.extract_har_hosts_count( ) result["returned"][ "request_headers"] = analysis.extract_har_request_headers( ) result["returned"][ "request_queries"] = analysis.extract_har_request_queries( ) result["returned"][ "request_cookies"] = analysis.extract_har_request_cookies( ) result["returned"][ "postdata_form_params"] = analysis.extract_har_request_postdata_form_params( ) result["returned"][ "postdata_json_attributes"] = analysis.extract_har_request_postdata_json_attributes( ) except Exception as ex: result["errorMessage"] = str(ex) return jsonify(result)
# get data training_data = datasets.MNISTData(validation=False, batch_size=32) validation_data = datasets.MNISTData(validation=True, batch_size=32) # define model nin = training_data.nin nout = training_data.nout model = Classifier(models.ConvNet(nin, 10, nout)) # Set up an optimizer optimizer = chainer.optimizers.Adam() optimizer.setup(model) optimizer.add_hook(chainer.optimizer.WeightDecay(1e-5)) # instantiate learning algorithm ann = supervised_learning.FeedforwardLearner(optimizer) # run the optimization ann.optimize(training_data, validation_data=validation_data, epochs=100) # plot loss and throughput ann.report('results/tmp') # create analysis object ana = Analysis(ann.model, fname='results/tmp') # handle sequential data; deal with classifier analysis separately # analyse data ana.classification_analysis(validation_data.X, validation_data.T)
from analysis import Analysis from algorithm import Algorithm if __name__ == '__main__': tool = Analysis('results_epoch_new/') for i in range(20): epoch = (i + 1) * 10 algorithm = Algorithm(name='ML', param=epoch) runtime = algorithm.execute(network_path='networks/', sub_filename='sub-wm.txt', req_num=2000) tool.save_evaluations(algorithm.evaluation, '%s.txt' % epoch) tool.save_epoch(epoch, algorithm.evaluation.acc_ratio, runtime)
def main(): from optparse import OptionParser usage = "usage: %prog [options] arg" parser = OptionParser(usage) parser.add_option("-c", "--code", dest="stockcode", help="stock code") parser.add_option("-n", "--name", dest="stockname", help="stock name") parser.add_option("-s", "--stock", dest="stocktxt", help="read scraping stock names from text file") parser.add_option("-r", "--readfile", dest="csvfile", help="read stock data from csv file") parser.add_option("-u", "--update", help="update csvfile (overwirte)", action="store_true", dest="update") parser.add_option("-d", "--date", dest="startdate", help="specify start date as '2014-10-01'") parser.add_option("-y", "--days", dest="days", help="plot days as '240', specify 0 for all days") parser.add_option("-a", "--axis", dest="axis", help="setting y-axis limit (1 or 2, default 2)") parser.add_option("-p", "--complexity", dest="complexity", help="complexity of chart (1-3, default 3)") (options, args) = parser.parse_args() if len(args) != 0: parser.error("incorrect number of arguments") if options.days is None: options.days = 240 if options.axis is None: options.axis = 2 if options.complexity is None: options.complexity = 3 if options.stocktxt: return read_csv(filename=options.stocktxt, start=options.startdate, days=int(options.days), update=options.update, axis=int(options.axis), complexity=int(options.complexity)) else: analysis = Analysis(code=options.stockcode, name=options.stockname, fullname=options.stockname, start=options.startdate, days=int(options.days), csvfile=options.csvfile, update=options.update, axis=int(options.axis), complexity=int(options.complexity)) return analysis.run()
def run(self, path): self.path = path data = pandas.read_csv(self.path) data.dropna(inplace=True) Analysis().boxplot(data)
from analysis import Analysis from word_gen import WordGen extended = [ 'é', 'è', 'ê', 'ë', 'à', 'â', 'ä', 'î', 'ï', 'ù', 'û', 'ü', 'ç', 'ô', 'ö', 'œ' ] A = Analysis("liste_francais.txt", extended) A.run() B = Analysis("words.txt", [], encoding="utf-8") B.run() wordA = WordGen(A) wordB = WordGen(B) for j in range(500): print(wordA.generate_word(7), " | ", wordB.generate_word(7))
print( "I hope you know what you're doing! Voting the previous proposal..." ) approved_counts, rejected_counts, vote_list = 0, 0, [] for i in range(a.num_players): choice = sanitised_input( "Player " + str(i) + ", approve (1) or reject (0) mission? ", int, 0, 1) if choice == 1: approved_counts += 1 else: rejected_counts += 1 vote_list.append(choice) a.force_vote(approved_counts, rejected_counts, vote_list) elif (user_input == "analyze" or user_input == "ana"): ana = Analysis(a) ana.start_analysis() ana.analyze() print(ana.player_values) elif (user_input == "anamore"): ana = Analysis(a) ana.start_analysis() ana.analyze(15) print(ana.player_values) else: print("Invalid command.") else: print("Invalid command, must guess Merlin.") if a.game_state == 0: print("Minions have won!") else: