def initialize():
global annotator, answer_catalog, analysis
annotator = Annotator()
answer_catalog = Answers()
analysis = Analysis()
from knn_gridsearch import KNNModel
from analysis import Analysis
from sklearn.ensemble import RandomForestClassifier
#predict Arsenal away wins
#pick out feature most important to predicting away wins
league_id=1729
away_team_api_id=9825
sqlite_file = '/Users/schumanzhang/Desktop/machine_learning_projects/soccer_dataset/raw_data/database.sqlite'
try:
processed_data = pd.read_csv('dataframe.csv', index_col=0)
except IOError:
analysis = Analysis(league_id, away_team_api_id, sqlite_file)
raw_data = analysis.retrieve_raw_data()
basic_info_list = analysis.basic_data_info()
print(basic_info_list)
analysis.addInitAttributes()
analysis.addPlayerSkillTotals()
analysis.addPlayerHeightTotals()
analysis.parse_selected_tags('shoton', ['away_shoton', 'home_shoton'])
analysis.parse_selected_tags('shotoff', ['away_shotoff', 'home_shotoff'])
analysis.parse_selected_tags('foulcommit', ['away_fouls', 'home_fouls'])
analysis.parse_selected_tags('card', ['away_cards', 'home_cards'])
analysis.parse_selected_tags('cross', ['away_crosses', 'home_crosses'])
analysis.parse_selected_tags('corner', ['away_corners', 'home_corners'])
analysis.parse_selected_tags('possession', ['away_possession', 'home_possession'])
def test_main_flow(self):
raw_dir = "."
since_date = "2006/01/01"
to_date = "2008/12/31"
filling_types = {"10-K", "10-Q"}
# case-sensitive by default, or re.IGNORECASE for case-insensive match
flags = 0
workbook = xlsxwriter.Workbook('acq_asset_report.xlsx')
red_format = workbook.add_format({
'font_color': 'red',
'bold': 1,
'underline': 1,
'font_size': 12,
})
# read the cik/targetname between row11~20 from the firms spreadsheet
# the default specs are overwitten!
TestMainFlow.specs = utils.read_specs("./firms.xlsx", 1, 100)
for spec in TestMainFlow.specs:
cik = spec[0]
target_name = spec[1]
golden_doc = spec[2]
index_file = os.path.join(raw_dir, cik, "download.idx")
clean_name = ''.join(e for e in target_name if e.isalnum())
# A company could acquire lots of small company during a period
worksheet = workbook.add_worksheet(f"{cik}-{clean_name}")
worksheet.write_string('A1', "Target Name:")
worksheet.write_string('B1', target_name)
worksheet.set_column('A:A', 35)
worksheet.set_column('B:B', 150)
utils.logger.info(f'scanning acquisition asset information about {target_name} in {cik} docs...')
if os.path.exists(index_file) is False:
try:
company = Company(cik)
utils.logger.info(f"\tdownloading {'/'.join(filling_types)} documents for cik:{cik}...")
index_file = company.download_documents(since_date, to_date, filling_types, raw_dir)
except Exception as e:
utils.logger.error(f"Failed to download documents for cik:{cik}...\n{utils.traceback.format_exc()}")
continue
# grep the target name in case-sensitive way. or set flags = re.IGNORECASE for a case-insensitive search
ana = Analysis(target_name, index_file)
docs = ana.locate_target_documents(flags)
doc_found = ""
row = 1
# parse the html and locate the acquisition paragraph
for filename, dict in docs.items():
row = row + 1
if doc_found != "":
worksheet.write_url(f"A{row}", dict['FILLING_URL'], string=filename)
continue
utils.logger.info(f'\tfound {target_name} in {filename}, analyzing...')
try:
info = ana.extract_assets(filename, flags)
except Exception as e:
utils.logger.error(f"Failed in analyzing document:{filename}...\n{utils.traceback.format_exc()}")
continue
if info is None:
worksheet.write_url(f"A{row}", dict['FILLING_URL'], string=filename)
continue
else:
doc_found = os.path.basename(filename)
worksheet.write_url(f"A{row}", dict['FILLING_URL'], red_format, string=filename)
worksheet.write_string(f"B{row}", info)
utils.logger.info(f"!!!!located acquisition asset report!!!!")
utils.logger.info(f"\tCIK:{cik}\tTarget:{target_name}\tDate:{dict['FILLING_DATE']}\t{dict['FILLING_TYPE']}:{filename}")
utils.logger.info(f"\tFilling Details:{dict['FILLING_URL']}")
utils.logger.info(f"{info}\n")
continue
#self.assertEqual(golden_doc, doc_found)
#if golden_doc != doc_found:
#utils.logger.error(f"Expecting initial report: {golden_doc} for {cik} {target_name}, but it locates: {doc_found}")
workbook.close()
pressures = [0.1, 1,10,25,50]
nozzles = [100000, 300000, 400000, 500000, 800000,900000]
angles = [5,10,15,20,30,35,40,50,60]
Results = []
counter = 0
for distance in distances:
for diameter in diameters:
for pressure in pressures:
sim = gasPhaseSimulation(sample_nozzle_distance = distance,
pressure = pressure,
source_diameter = diameter,
initial_KE = 1100,
loss_function = loss_function)
sim.simulateMany(5000, 'start finish')
results = sim.start_finish
A = Analysis(results, parameters = sim.parameters)
for nozzle in nozzles:
nozzle_diameter = nozzle
for angle in angles:
A.pathHistogram(bins=200, x_limits=(290000,350000),
accept_angle=angle, nozzle_diameter=nozzle_diameter)
A.areasUnderProfiles()
params = A.parameters
p = A.profiles
a = A.areas
avg = A.getAveragePathLength()
params['avg_path'] = avg
Results += [params.copy()]
print('counter : ' + str(counter))
counter+=1
== date.replace(hour=0, minute=0, second=0)):
return False
# The strategy needs to be active.
if strategy["action"] == "hold":
return False
# We need to know the stock price.
if not strategy["price_at"] or not strategy["price_eod"]:
return False
return True
if __name__ == "__main__":
analysis = Analysis(logs_to_cloud=False)
trading = Trading(logs_to_cloud=False)
twitter = Twitter(logs_to_cloud=False)
# Look up the metadata for the tweets.
tweets = twitter.get_tweets(SINCE_TWEET_ID)
events = []
for tweet in tqdm(tweets):
event = {}
timestamp_str = tweet["created_at"]
timestamp = trading.utc_to_market_time(
datetime.strptime(timestamp_str, "%a %b %d %H:%M:%S +0000 %Y"))
text = tweet["text"]
event["timestamp"] = timestamp
def main():
f = open('try_3.txt', 'w')
g = open('accs.txt', 'w')
g.close()
task = MarioTask("testbed", initMarioMode=2)
task.env.initMarioMode = 2
task.env.levelDifficulty = 1
results = []
names = []
iterations = 20
rounds = 30
learning_samples = 33
eval_samples = 10
# iterations = 5
# rounds = 2
# learning_samples = 3
# eval_samples = 2
if args['noisy']:
prefix = '-noisy-sup-eval'
dire = './training_data_noisy/'
agent = NoisySupervise(IT, useKMM=False)
else:
prefix = '-sup-eval'
dire = './training_data/'
agent = Supervise(IT, useKMM=False)
if args['linear']:
agent.learner.linear = True
prefix = 'svc' + prefix
else:
agent.learner.linear = False
prefix = 'dt' + prefix
exp = EpisodicExperiment(task, agent)
E = Evaluator(agent, exp)
sl_data, sup_data, acc, loss, js, test_acc = E.eval(
rounds=rounds,
iterations=iterations,
learning_samples=learning_samples,
eval_samples=eval_samples,
prefix=prefix,
directory=dire)
np.save('./data/' + prefix + '-sl_data.npy', sl_data)
np.save('./data/' + prefix + '-acc.npy', acc)
np.save('./data/' + prefix + '-loss.npy', loss)
np.save('./data/' + prefix + '-js.npy', js)
np.save('./data/' + prefix + '-test_acc.npy', test_acc)
analysis = Analysis()
analysis.get_perf(sl_data, range(iterations))
analysis.plot(names=['Supervised Learning'],
label='Reward',
filename='./results/' + prefix +
'-return_plots.eps') #, ylims=[0, 1600])
acc_a = Analysis()
acc_a.get_perf(acc, range(iterations))
acc_a.plot(names=['Supervised Learning Acc.'],
label='Accuracy',
filename='./results/' + prefix + '-acc_plots.eps',
ylims=[0, 1])
loss_a = Analysis()
loss_a.get_perf(loss, range(iterations))
loss_a.plot(names=['Supervised Learning loss'],
label='Loss',
filename='./results/' + prefix + '-loss_plots.eps',
ylims=[0, 1])
js_a = Analysis()
js_a.get_perf(js, range(iterations))
js_a.plot(names=['Supervised Learning'],
label='J()',
filename='./results/' + prefix + '-js_plots.eps')
test_acc_a = Analysis()
test_acc_a.get_perf(test_acc, range(iterations))
test_acc_a.plot(names=['Supervised Learning Acc.'],
label='Test Accuracy',
filename='./results/' + prefix + '-test_acc_plots.eps',
ylims=[0, 1])
#agent.saveModel()
print "finished"
from algorithm import Algorithm
from analysis import Analysis
if __name__ == '__main__':
tool = Analysis('results_algorithm/')
name = 'RL'
algorithm = Algorithm(name, param=100)
runtime = algorithm.execute(network_path='networks/',
sub_filename='sub-wm.txt',
req_num=1000)
tool.save_evaluations(algorithm.evaluation, '%s.txt' % name)
print(runtime)
def main():
f = open('try_3.txt', 'w')
g = open('accs.txt', 'w')
g.close()
task = MarioTask("testbed", initMarioMode=2)
task.env.initMarioMode = 2
task.env.levelDifficulty = 1
results = []
names = []
with open('type.txt', 'w') as f:
f.write('svc')
# # #test dagger
# iterations = 1
# rounds = 1
iterations = 50
rounds = 15
#agent = Dagger(IT,useKMM = False)
#exp = EpisodicExperiment(task, agent)
#T = Tester(agent,exp)
#dagger_results = T.test(rounds = rounds,iterations = iterations)
#dagger_data = dagger_results[-1]
#dagger_results = dagger_results[:-1]
#results.append(dagger_results)
#names.append('dagger')
#pickle.dump(results,open('results.p','wb'))
#agent = Dagger(IT, useKMM=False)
#exp = EpisodicExperiment(task, agent)
#T = Tester(agent, exp)
#dagger_data, _, acc = T.test(rounds = rounds, iterations = iterations)
agent = Supervise(IT, useKMM=False)
exp = EpisodicExperiment(task, agent)
T = Tester(agent, exp)
prefix = 'svc-sup-change'
sl_data, sup_data, acc = T.test(rounds=rounds,
iterations=iterations,
prefix=prefix)
np.save('./data/svc-sup-change-sup_data.npy', sup_data)
np.save('./data/svc-sup-change-sl_data.npy', sl_data)
np.save('./data/svc-sup-change-acc.npy', acc)
# IPython.embed()
analysis = Analysis()
analysis.get_perf(sup_data, range(iterations))
analysis.get_perf(sl_data, range(iterations))
analysis.plot(names=['Supervisor', 'Supervised Learning'],
label='Reward',
filename='./results/svc-sup-change-return_plots.eps'
) #, ylims=[0, 1600])
acc_a = Analysis()
acc_a.get_perf(acc, range(iterations))
acc_a.plot(names=['Supervised Learning Acc.'],
label='Accuracy',
filename='./results/svc-sup-change-acc_plots.eps')
"""
agent = Dagger(IT,useKMM = False)
exp = EpisodicExperiment(task, agent)
T = Tester(agent,exp)
dagger_data, _, acc = T.test(rounds = rounds, iterations = iterations)
np.save('./data/dagger_data.npy', dagger_data)
np.save('./data/acc.npy', acc)
IPython.embed()
analysis = Analysis()
analysis.get_perf(dagger_data, range(iterations))
analysis.plot(names=['DAgger'], label='Reward', filename='./results/return_plots.eps')
acc_a = Analysis()
acc_a.get_perf(acc, range(iterations))
acc_a.plot(names=['DAgger Acc.'], label='Accuracy', filename='./results/acc_plots.eps')
"""
#agent = Supervise(IT,useKMM = False)
#exp = EpisodicExperiment(task, agent)
#T = Tester(agent,exp)
#supervise_results = T.test(rounds = rounds, iterations = iterations)
#supervise_data = supervise_results[-1]
#supervise_results = supervise_results[:-1]
#results.append(supervise_results)
#names.append('supervise')
#pickle.dump(results,open('results.p','wb'))
#IPython.embed()
#analysis = Analysis()
#analysis.get_perf(supervise_data, results[1][5])
#analysis.get_perf(dagger_data, results[0][5])
#analysis.plot(names=['Supervise', 'DAgger'], label='Reward', filename='./return_plot.eps')#, ylims=[-1, 0])
# agent = Sheath(IT,useKMM = False,sigma = 1.0)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# dagger_results = T.test(rounds = 10,iterations = 35)
# results.append(dagger_results)
# names.append('sheath_1')
# pickle.dump(results,open('results.p','wb'))
# agent = Sheath(IT,useKMM = False,sigma = 1e-1)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# dagger_results = T.test(rounds = 10,iterations = 35)
# results.append(dagger_results)
# names.append('sheath_1')
# pickle.dump(results,open('results.p','wb'))
# agent = Sheath(IT,useKMM = False,sigma = 0.5)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# dagger_results = T.test(rounds = 10,iterations = 35)
# results.append(dagger_results)
# names.append('sheath_1')
# pickle.dump(results,open('results.p','wb'))
# agent = Sheath(IT,useKMM = False,sigma = 1e-1)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# dagger_results = T.test(rounds = 4,iterations = 35)
# results.append(dagger_results)
# names.append('sheath_1')
# agent = Sheath(IT,useKMM = False,sigma = 1e-2)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# dagger_results = T.test(rounds = 4,iterations = 35)
# results.append(dagger_results)
# names.append('sheath_1')
# # # # # #test big ahude
# agent = Ahude(IT,f,gamma = 1e-2,labelState = True, useKMM = True)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# ahude_big_results = T.test(rounds = 3)
# results.append(ahude_big_results)
# names.append('ahude_1e-1')
# pickle.dump(results,open('results.p','wb'))
# # # # # #test med ahude
# agent = Ahude(IT,f,gamma = 1e-2,labelState = False,useKMM = True)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# ahude_med_results = T.test(rounds = 3)
# results.append(ahude_med_results)
# names.append('ahude_1e-2')
# # #
# # # # # # #test small ahude
# agent = Ahude(IT,f,gamma = 1e-3)
# exp = EpisodicExperiment(task, agent)
# T = Tester(agent,exp)
# ahude_small_results = T.test()
# results.append(ahude_small_results)
# names.append('ahude_1e-3')
# pickle.dump(results,open('results.p','wb'))
#plt.figure(1)
#for i in range(len(results)):
# plt.plot(results[i][5],results[i][1])
#plt.legend(names,loc='upper left')
# plt.figure(2)
# for i in range(len(results)):
# plt.plot(results[i][0])
# plt.legend(names,loc='upper left')
# plt.figure(3)
# for i in range(0,len(results)):
# plt.plot(results[i][3])
# plt.legend(names,loc='upper left')
plt.show()
# IPython.embed()
f.close()
#agent.saveModel()
print "finished"
def setUp(self):
self.analysis = Analysis()
self.analysis.load_har_file(har_filename_for_tests)
pass
from analysis import Analysis
from models import *
analysis = Analysis('weather', normalize=True)
analysis.load_data()
# Move this into a function to allow for quickly switching between predictors and features
# Or just leave it since will do our own feature selection?
predictor = 'actual_mean_temp'
features = [predictor]
analysis.set_feature_predictor_columns(features, predictor)
analysis.train_test_split()
NUM_DAYS = 5
NUM_FEATURES = len(analysis.features)
from keras.models import Sequential
from keras import optimizers
from keras import layers
mask_value = 0
nn = Sequential()
nn.add(
layers.Masking(mask_value=mask_value,
input_shape=(NUM_DAYS, NUM_FEATURES)))
nn.add(
layers.LSTM(50,
activation='tanh',
input_shape=(NUM_DAYS, NUM_FEATURES),
'realm': realm,
'frequency': frequency,
'variable': variable,
'ensemble_element': ensemble_element
}
# Two pressure levels are already registered, you can choose to work at
# 850 hPa to seek polar jets, or at 250 hPa for both of them.
# If you want to study another pressure level, be sure to adjust the "plev"
# variable to have the right data in the pickle file.
if pressure == 85000:
plev = 1
if pressure == 25000:
plev = 4
analysis = Analysis(input_options)
analysis.get_files(only_files=True)
print len(analysis.files), "files will be analyzed"
data_list = []
# Files are loaded here...
for fichier in range(0, len(
analysis.files)): #Loop over all the data files available
analysis.mf = Dataset(analysis.files[fichier])
variable_data = np.asarray(analysis.mf.variables[analysis.variable][:],
dtype=np.float32)
def run_simulation(scenario : dict, t_step : float = 1) -> dict:
"""
Runs the simulation process.
"""
t0 = time.time()
# Initial setup
scenario['timesteps'] = []
scenario['debug'] = []
n_all = len(scenario['persons'])
# Divide persons to compartments
healthy = {}
sick = {}
healed = {}
for id_code, person in scenario['persons'].items():
if person.sick_now:
sick[id_code] = person
elif person.healed:
healed[id_code] = person
else:
healthy[id_code] = person
# Initialize callback status
for cb in scenario['callbacks']:
cb['done'] = False
# Run simulation
sim_time = 0
healthy_idx = 0
while sim_time < scenario['sim_hours']:
n_healthy = len(healthy)
healthy_list = list(healthy.values())
# Update persons
to_healed = []
to_sick = []
for id_code, person in sick.items():
person.update_infection(sim_time)
if person.healed:
to_healed.append(id_code)
continue
if person.expose_others:
if n_healthy == 0:
continue
# Calculate number of healthy persons to expose
n = person.n_exposed
n *= float(n_healthy) / float(n_all)
frac = n - int(n)
n = int(n)
if np.random.rand() < frac:
n += 1
# Expose persons
for _ in range(n):
healthy_idx += 1
if healthy_idx >= n_healthy:
healthy_idx = 0
exp_person = healthy_list[healthy_idx]
new_infection = exp_person.expose(
infected_by=person.id_code,
virus=scenario['virus'],
timestamp=sim_time
)
# If exposure caused infection, store the contact information
if new_infection:
person.report_infected_others(id_code=exp_person.id_code)
to_sick.append(exp_person.id_code)
for id_code in to_healed:
healed[id_code] = sick.pop(id_code)
for id_code in to_sick:
sick[id_code] = healthy.pop(id_code)
scenario['timesteps'].append(sim_time)
sim_time += t_step
# Run timed callbacks
for cb in scenario['callbacks']:
if (cb['trigger'] < sim_time) and (cb['done'] == False):
cb['function'](scenario)
cb['done'] = True
# Analyze the data
analysis = Analysis()
scenario = analysis.run_analysis(scenario)
# Progress monitoring
t1 = time.time()
print('Scenario {} done in {:0.1f} s.'.format(scenario['label'], t1 - t0))
return scenario
from analysis import Analysis
from algorithm import Algorithm
# 5 0.37612838515546637 1525.0753908157349 683.8999999999988
# 10 0.3711133400200602 1613.1240510940552 684.0999999999992
# 15 0.3510531594784353 761.4128797054291 647.3999999999991
# 20 0.3781344032096289 755.6144239902496 674.599999999999
# if __name__ == '__main__':
#
# tool = Analysis('results_epoch/')
# epoch = 3
# algorithm = Algorithm(name='RLQ', param=epoch)
# runtime = algorithm.execute(network_path='networks/',
# sub_filename='sub-ts.txt',
# req_num=1000)
# tool.save_evaluations(algorithm.evaluation, '%s.txt' % epoch)
# qos_loss=algorithm.evaluation.total_loss/algorithm.evaluation.total_accepted
# tool.save_epoch(epoch, algorithm.evaluation.acc_ratio, runtime, qos_loss)
if __name__ == '__main__':
tool = Analysis('results_epoch/')
for i in range(5):
epoch = (i + 1) * 10
algorithm = Algorithm(name='RLQ', param=epoch)
runtime = algorithm.execute(network_path='networks/',
sub_filename='sub-ts.txt',
req_num=1000)
tool.save_evaluations(algorithm.evaluation, '%s.txt' % epoch)
qos_loss = algorithm.evaluation.total_loss / algorithm.evaluation.total_accepted
ylabel="Data Values",
color="k",
)
"""
For every dataset we now create an `Analysis` class using it and use `Emcee` to fit it with a `Gaussian`.
"""
for dataset_name in dataset_names:
"""Load the dataset from the `autofit_workspace/dataset` folder."""
dataset_path = path.join("dataset", "example_1d", dataset_name)
data = af.util.numpy_array_from_json(
file_path=path.join(dataset_path, "data.json"))
noise_map = af.util.numpy_array_from_json(
file_path=path.join(dataset_path, "noise_map.json"))
"""Create the `DynestyStatic` `NonLinearSearch` and use it to fit the data."""
analysis = Analysis(data=data, noise_map=noise_map)
emcee = af.DynestyStatic(
path_prefix=path.join("howtofit", "chapter_graphical_models"),
name="tutorial_1_global_model",
unique_tag=dataset_name,
)
print(
f"Emcee has begun running, checkout \n"
f"autofit_workspace/output/howtofit/chapter_graphica_models/tutorial_1_global_model/{dataset_name} for live \n"
f"output of the results. This Jupyter notebook cell with progress once Emcee has completed, this could take a \n"
f"few minutes!")
emcee.fit(model=model, analysis=analysis)
"""
def analysis():
return Analysis(logs_to_cloud=False)
def transformation(self, sent, ant, justify=False):
"""
Transformation step in the simplification process.
This is a recursive method that receives two parameters:
@param sent: sentence to be simplified.
@param ant: previous sentence. If sent = ant, then no simplification should be performed.
@param justify: controls cases where sentence order is inverted and should invert the entire recursion.
@return: the simplified sentences.
"""
def remove_all(aux, item):
"""
Remove all incidences of a node in a graph (needed for graphs with cycles).
@param aux: auxiliary of parse structure
@param item: node to be removed
"""
for a in aux.keys():
for d in aux[a].keys():
if item in aux[a][d]:
aux[a][d].remove(item)
def recover_punct(final, s):
"""
Recover the punctuation of the sentence (needed because the dependency parser does not keep punctuation.
@param final: the final dictionary with the words in order
@param s: the tokenised sentence with the punctuation marks
@return the final dictionary with the punctuation marks
"""
char_list = "``\'\'"
ant = 0
for k in sorted(final.keys()):
if int(k) - ant == 2:
if s[k - 2] in string.punctuation + char_list:
final[k - 1] = s[k - 2]
elif s[k - 2] == "-RRB-":
final[k - 1] = ")"
elif s[k - 2] == "-LRB-":
final[k - 1] = "("
if int(k) - ant == 3:
if s[k - 2] in string.punctuation + char_list and s[
k - 3] in string.punctuation + char_list:
final[k - 1] = s[k - 2]
final[k - 2] = s[k - 3]
ant = k
return final
def build(root, dep, aux, words, final, yes_root=True, previous=None):
"""
Creates a dictionary with the words of a simplified clause, following the sentence order.
This is a recursive method that navigates through the dependency tree.
@param root: the root node in the dependency tree
@param dep: the dependencies of the root node
@param aux: the auxiliary parser output
@param words: auxiliary parsed words
@param final: dictionary with the positions and words
@param yes_root: flag to define whether or not the root node should be included
@param previous: list of nodes visited
"""
## controls recursion
if previous == None:
previous = []
if root in previous:
return
previous.append(root)
## for cases where the rule does not include the root node
if yes_root:
final[root] = words[root - 1][0]
previous.append(root)
for k in dep.keys():
for i in dep[k]:
if i in aux.keys():
deps = aux[i]
## needed for breaking loops -- solved by the recursion condition
#for d in deps.keys():
# if i in deps[d]:
# deps[d].remove(i)
build(i, deps, aux, words, final, previous=previous)
final[i] = words[i - 1][0]
def conjoint_clauses(aux, words, root, deps_root, ant, _type, rel):
"""
Simplify conjoint clauses
@param aux: auxiliary parser output
@param words: auxiliary words and POS tags structure
@param root: root node in the dependency tree
@param deps_root: dependencies of the root node
@param ant: previous sentence (for recursion purposes)
@param _type: list of markers found in the sentence that can indicate conjoint clauses
@param rel: parser relation between the main and the dependent clause (can be 'advcl' or 'conj')
@return: a flag that indicates whether or not the sentence was simplified and the result sentence (if flag = False, ant is returned)
"""
## split the clauses
others = deps_root[rel]
pos = 0
s1 = s2 = ""
v_tense = ""
for o in others:
flag = True
if o not in aux:
flag = False
continue
deps_other = aux[o]
## check the marker position ('when' is advmod, while others are mark)
if 'advcl' in rel:
if 'mark' in deps_other.keys():
mark = deps_other['mark'][0]
mark_name = words[mark - 1][0].lower()
elif 'advmod' in deps_other.keys():
mark = deps_other['advmod'][0]
mark_name = words[mark - 1][0].lower()
else:
flag = False #needed for broken cases
continue
else:
if 'cc' in deps_root.keys() and 'conj' in rel:
conj = deps_root[rel][0]
if 'vm' in words[conj - 1][1][
'PartOfSpeech'] and 'vm' in words[root - 1][1][
'PartOfSpeech']: #needed for broken cases like 'Care and support you won't have to pay towards'
mark = deps_root['cc'][0]
mark_name = words[mark - 1][0].lower()
else:
flag = False
continue
else:
flag = False
continue
## hack for simpatico use cases
if mark_name == "and" and words[mark - 2][0].lower(
) == "care" and words[mark][0].lower() == "support":
flag = False
continue
## dealing with cases without subject
if 'nsubj' not in deps_other and 'nsubj' in deps_root:
deps_other['nsubj'] = deps_root['nsubj']
elif 'nsubj' not in deps_other and 'nsubjpass' in deps_root:
deps_other['nsubj'] = deps_root['nsubjpass']
elif 'nsubj' not in deps_other and 'nsubj' not in deps_root:
flag = False
continue
## check if the marker is in the list of selected markers
if mark_name in _type:
## check if verbs have objects
tag_list = ('advcl', 'xcomp', 'acomp', 'amod', 'appos',
'cc', 'ccomp', 'dep', 'dobj', 'iobj', 'nwe',
'pcomp', 'pobj', 'prepc', 'rcmod', 'ucomp',
'nmod', 'auxpass', 'advmod', 'prep')
#if not any([t in tag_list for t in deps_root.keys()]):
#return False, ant
# flag = False
# continue
#elif not any([t in tag_list for t in deps_other.keys()]):
#return False, ant
# flag = False
# continue
#if (len(deps_root) < 2 or len(deps_other) < 2):
# return False, ant
## delete marker and relation from the graph
if 'advcl' in rel:
if 'mark' in deps_other.keys():
del deps_other['mark'][0]
elif 'advmod' in deps_other.keys():
del deps_other['advmod'][0]
else:
del deps_root['cc'][0]
#del deps_root[rel][pos]
#pos+=1
deps_root[rel].remove(o)
## for cases with time markers -- This + modal + happen
modal = None
if 'aux' in deps_root and mark_name in self.time:
modal_pos = deps_root['aux'][0]
modal = words[modal_pos - 1][0]
## for cases either..or with the modal verb attached to the main clause
if 'aux' in deps_root and mark_name in self.condition2:
deps_other['aux'] = deps_root[u'aux']
## built the sentence again
final_root = {}
build(root, deps_root, aux, words, final_root)
final_deps = {}
build(o, deps_other, aux, words, final_deps)
## TODO: remove this part from here --> move to another module: self.generation
root_tag = words[root - 1][1]['PartOfSpeech']
justify = True
#if ((root > o) and (mark_name in self.time and mark>1)) or (mark_name == 'because' and mark > 1):
if (root > o) or (mark_name == 'because' and mark > 1):
if (mark_name in self.time and mark == 1):
sentence1, sentence2 = self.generation.print_sentence(
final_root, final_deps, root_tag, mark_name,
mark, modal)
else:
sentence1, sentence2 = self.generation.print_sentence(
final_deps, final_root, root_tag, mark_name,
mark, modal)
else:
sentence1, sentence2 = self.generation.print_sentence(
final_root, final_deps, root_tag, mark_name, mark,
modal)
s1 = self.transformation(sentence1, ant, justify)
s2 = self.transformation(sentence2, ant)
flag = True
else:
flag = False
continue
if flag:
return flag, s1 + " " + s2
else:
return flag, ant
def relative_clauses(aux, words, root, deps_root, ant, rel):
"""
Simplify relative clauses
@param aux: auxiliary parser output
@param words: auxiliary words and POS tags structure
@param root: root node in the dependency tree
@param deps_root: dependencies of the root node
@param ant: previous sentence (for recursion purposes)
@param rel: parser relation between the main and the dependent clause (can be 'nsubj' or 'dobj')
@return: a flag that indicates whether or not the sentence was simplified and the result sentence (if flag = False, ant is returned)
"""
subj = deps_root[rel][0]
if subj not in aux.keys():
return False, ant
deps_subj = aux[subj]
if 'acl:relcl' in deps_subj.keys() or 'rcmod' in deps_subj.keys():
if 'acl:relcl' in deps_subj.keys():
relc = deps_subj['acl:relcl'][0]
type_rc = 'acl:relcl'
else:
relc = deps_subj['rcmod'][0]
type_rc = 'rcmod'
deps_relc = aux[relc]
subj_rel = ""
if 'nsubj' in deps_relc.keys():
subj_rel = 'nsubj'
elif 'nsubjpass' in deps_relc.keys():
subj_rel = 'nsubjpass'
else:
subj_rel = 'mark'
if 'ref' in deps_subj:
to_remove = deps_subj['ref'][0]
mark = words[deps_subj['ref'][0] - 1][0].lower()
elif subj_rel in deps_relc:
to_remove = deps_relc[subj_rel][0]
mark = words[deps_relc[subj_rel][0] - 1][0].lower()
if mark in self.relpron:
deps_relc[subj_rel][0] = subj
remove_all(aux, to_remove)
elif 'dobj' in deps_relc: ## needed for cases where the subject of the relative clause is the object
obj = deps_relc['dobj'][0]
if 'poss' in aux[obj]:
mod = aux[obj]['poss'][0]
aux_words = list(words[mod - 1])
aux_words[0] = words[subj - 1][0] + '\'s'
words[mod - 1] = tuple(aux_words)
aux[mod] = aux[subj]
else:
return False, ant
else:
return False, ant #for borken cases - " There are some situations where it is particularly important that you get financial information and advice that is independent of us."
del aux[subj][type_rc]
if 'punct' in deps_subj.keys():
del aux[subj]['punct']
if 'punct' in deps_relc.keys():
del aux[relc]['punct']
final_root = {}
build(root, deps_root, aux, words, final_root)
final_relc = {}
build(relc, deps_relc, aux, words, final_relc)
if justify:
sentence2, sentence1 = self.generation.print_sentence(
final_root, final_relc)
else:
sentence1, sentence2 = self.generation.print_sentence(
final_root, final_relc)
s1 = self.transformation(sentence1, ant, justify)
s2 = self.transformation(sentence2, ant)
return True, s1 + " " + s2
else:
return False, ant
def appositive_phrases(aux, words, root, deps_root, ant):
"""
Simplify appositive phrases
@param aux: auxiliary parser output
@param words: auxiliary words and POS tags structure
@param root: root node in the dependency tree
@param deps_root: dependencies of the root node
@param ant: previous sentence (for recursion purposes)
@return: a flag that indicates whether or not the sentence was simplified and the result sentence (if flag = False, ant is returned)
"""
## apposition needs to have a subject -- same subject of the mais sentence.
if 'nsubj' in deps_root.keys():
subj = deps_root['nsubj'][0]
subj_word = words[subj - 1][0]
if subj not in aux:
return False, ant
deps_subj = aux[subj]
v_tense = words[root - 1][1]['PartOfSpeech']
n_num = words[subj - 1][1]['PartOfSpeech']
if 'amod' in deps_subj: ## bug -- this generates several mistakes...
mod = deps_subj['amod'][0]
if mod in aux:
deps_mod = aux[mod]
else:
deps_mod = {}
del aux[subj]['amod']
deps_subj = aux[subj]
## Treat simple cases such as 'general rule'
#if 'JJ' in words[mod-1][1]['PartOfSpeech'] and len(deps_mod.keys()) == 0:
if 'JJ' in words[
mod -
1][1]['PartOfSpeech'] and 'punct' not in deps_subj:
return False, ant
elif 'appos' in deps_subj:
mod = deps_subj['appos'][0]
if mod in aux:
deps_mod = aux[mod]
else:
deps_mod = {}
del aux[subj]['appos']
deps_subj = aux[subj]
else:
return False, ant
if 'punct' in deps_subj.keys():
del deps_subj['punct']
final_root = {}
build(root, deps_root, aux, words, final_root)
final_appos = {}
build(mod, deps_mod, aux, words, final_appos)
final_subj = {}
build(subj, deps_subj, aux, words, final_subj)
if len(final_appos.keys()) < 2:
return False, ant
sentence1, sentence2 = self.generation.print_sentence_appos(
final_root, final_appos, final_subj, v_tense, n_num,
subj_word)
s1 = self.transformation(sentence1, ant)
s2 = self.transformation(sentence2, ant)
return True, s1 + " " + s2
else:
return False, ant
def passive_voice(aux, words, root, deps_root, ant):
"""
Simplify sentence from passive to active voice.
@param aux: auxiliary parser output
@param words: auxiliary words and POS tags structure
@param root: root node in the dependency tree
@param deps_root: dependencies of the root node
@param ant: previous sentence (for recursion purposes)
@return: a flag that indicates whether or not the sentence was simplified and the result sentence (if flag = False, ant is returned)
"""
if 'auxpass' in deps_root.keys():
if 'nmod:por' in deps_root.keys():
if 'nsubjpass' not in deps_root:
return False, ant
subj = deps_root['nsubjpass'][0]
if subj in aux:
deps_subj = aux[subj]
else:
deps_subj = {}
root_tense = words[root - 1][1]['PartOfSpeech']
aux_tense = words[deps_root['auxpass'][0] -
1][1]['PartOfSpeech']
v_aux = None
if 'v' in aux_tense and 'aux' in deps_root.keys():
aux_tense = words[deps_root['aux'][0] -
1][1]['PartOfSpeech']
v_aux = words[deps_root['aux'][0] - 1][0]
del deps_root['aux']
del deps_root['auxpass']
del deps_root['nsubjpass']
if len(deps_root['nmod:por']) > 1:
mod = deps_root['nmod:por'][1]
mod2 = deps_root['nmod:por'][0]
deps_mod = aux[mod]
deps_mod2 = aux[mod2]
if 'case' in deps_mod:
if words[deps_mod[u'case'][0] -
1][0].lower() != 'por':
return False, ant
del deps_mod['case']
del deps_root['nmod:por']
subj_tag = words[mod - 1][1]['PartOfSpeech']
subj_word = words[mod - 1][0]
final_subj = {}
build(mod, deps_mod, aux, words, final_subj)
final_obj = {}
build(subj, deps_subj, aux, words, final_obj)
final_mod2 = {}
build(mod2, deps_mod2, aux, words, final_mod2)
final_root = {}
build(root, deps_root, aux, words, final_root,
False)
sentence1 = self.generation.print_sentence_voice(
final_subj, final_obj, words[root - 1][0],
root_tense, v_aux, aux_tense, subj_tag,
subj_word, final_mod2, final_root)
s1 = self.transformation(sentence1, ant)
return True, s1
elif 'case' in deps_mod2:
if words[deps_mod2['case'][0] -
1][0].lower() != 'por':
return False, ant
del deps_mod2['case']
del deps_root['nmod:por']
subj_tag = words[mod2 - 1][1]['PartOfSpeech']
subj_word = words[mod2 - 1][0]
final_subj = {}
build(mod2, deps_mod2, aux, words, final_subj)
final_obj = {}
build(subj, deps_subj, aux, words, final_obj)
final_mod2 = {}
build(mod, deps_mod, aux, words, final_mod2)
final_root = {}
build(root, deps_root, aux, words, final_root,
False)
sentence1 = self.generation.print_sentence_voice(
final_subj, final_obj, words[root - 1][0],
root_tense, v_aux, aux_tense, subj_tag,
subj_word, final_mod2, final_root)
s1 = self.transformation(sentence1, ant)
return True, s1
else:
return False, ant
else:
mod = deps_root['nmod:por'][0]
deps_mod = aux[mod]
if 'case' in deps_mod:
if words[deps_mod['case'][0] -
1][0].lower() != 'por':
return False, ant
del deps_mod['case']
del deps_root['nmod:por']
subj_tag = words[mod - 1][1]['PartOfSpeech']
subj_word = words[mod - 1][0]
final_subj = {}
build(mod, deps_mod, aux, words, final_subj)
final_obj = {}
build(subj, deps_subj, aux, words, final_obj)
final_root = {}
build(root, deps_root, aux, words, final_root,
False)
sentence1 = self.generation.print_sentence_voice(
final_subj, final_obj, words[root - 1][0],
root_tense, v_aux, aux_tense, subj_tag,
subj_word, final_root)
s1 = self.transformation(sentence1, ant)
return True, s1
else:
return False, ant
else:
return False, ant
else:
return False, ant
## MAIN OF TRANSFORMATION
## control recursion: check whether there is no simplification to be done
if sent == ant:
return sent
flag = False
ant = sent
## parser
try:
parsed = self.parser.process(sent)
except AssertionError:
return ant
## data structure for the words and POS
words = parsed['words']
## data structure for the dependency parser
dict_dep = self.parser.transform(parsed)
## check whether or not the sentence has a root node
if 0 not in dict_dep:
return ant
root = dict_dep[0]['root'][0]
## check for root dependencies
if root not in dict_dep:
return ant
deps_root = dict_dep[root]
## get tokens
sent_tok = []
for w in words:
sent_tok.append(w[0])
## dealing with questions
## TODO: improve this control with parser information.
if sent_tok[0].lower() in ("what", "where", "when", "whose", "who",
"which", "whom", "whatever", "whatsoever",
"whichever", "whoever", "whosoever",
"whomever", "whomsoever", "whoseever",
"whereever") and sent_tok[-1] == "?":
return ant
## deal with apposition
flag, simpl = appositive_phrases(dict_dep, words, root, deps_root, ant)
if flag:
return simpl
## analyse whether or not a sentence has simplification clues (in this case, discourse markers or relative pronouns)
a = Analysis(sent_tok, self.cc, self.relpron)
flag_cc, type_cc = a.analyse_cc()
## if sentence has a marker that requires attention
if flag_cc:
## sorting according to the order of the relations
rel = {}
for k in deps_root.keys():
if 'conj' in k or 'advcl' in k:
others = sorted(deps_root[k], reverse=True)
cnt = 0
for o in others:
deps_root[k + str(cnt)] = []
deps_root[k + str(cnt)].append(o)
rel[k + str(cnt)] = deps_root[k][0]
cnt += 1
del deps_root[k]
sorted_rel = sorted(rel.items(), key=operator.itemgetter(1))
for k in sorted_rel:
flag, simpl = conjoint_clauses(dict_dep, words, root,
deps_root, ant, type_cc, k[0])
if flag:
return simpl
flag_rc, type_rc = a.analyse_rc()
## if sentence has a relative pronoun
if flag_rc:
## check where is the dependency of the relative clause
if 'nsubj' in deps_root:
flag, simpl = relative_clauses(dict_dep, words, root,
deps_root, ant, 'nsubj')
if flag:
return simpl
elif 'dobj' in deps_root:
flag, simpl = relative_clauses(dict_dep, words, root,
deps_root, ant, 'dobj')
if flag:
return simpl
## deal with passive voice
flag, simpl = passive_voice(dict_dep, words, root, deps_root, ant)
if flag:
return simpl
## return the original sentence if no simplification was done
if flag == False:
return ant
def get_tweet_text(tweet_id):
"""Looks up the text for a single tweet."""
tweet = get_tweet(tweet_id)
analysis = Analysis(logs_to_cloud=False)
return analysis.get_expanded_text(tweet)
def checkup(symbols):
latest_date = utils.latest_date_str()
q = Quote()
td = TechData()
a = Analysis()
swidth = []
swidth_pb = []
fwidth = []
fwidth_pb = []
fwidth_roc_pb = []
rwb = []
macd_hist_pb = []
swidth_roc_pb = []
latest_timestamp = None
for sym in symbols:
print '\n'
print 'Checking up', sym, '......'
df = q.intraday(sym)
if latest_timestamp is None:
latest_timestamp = df.index[-1]
td.macd(df)
td.ma(df)
td.guppy(df)
a.guppy_alignment(df)
a.guppy_roc(df)
tmp_df = df[[
'swidth', 'swidth_pb', 'fwidth', 'fwidth_pb', 'fwidth_roc_pb',
'rwb', 'macd_hist_pb', 'swidth_roc_pb'
]]
print '++++++++', sym, '+++++++++'
print tmp_df.tail(5)
swidth.append(df['swidth'][-1])
swidth_pb.append(df['swidth_pb'][-1])
fwidth.append(df['fwidth'][-1])
fwidth_pb.append(df['fwidth_pb'][-1])
fwidth_roc_pb.append(df['fwidth_roc_pb'][-1])
rwb.append(df['rwb'][-1])
macd_hist_pb.append(df['macd_hist_pb'][-1])
swidth_roc_pb.append(df['swidth_roc_pb'][-1])
total_df = pd.DataFrame(index=symbols)
total_df['swidth'] = swidth
total_df['swidth_pb'] = swidth_pb
total_df['fwidth'] = fwidth
total_df['fwidth_pb'] = fwidth_pb
total_df['fwidth_roc_pb'] = fwidth_roc_pb
total_df['rwb'] = rwb
total_df['macd_hist_pb'] = macd_hist_pb
total_df['swidth_roc_pb'] = swidth_roc_pb
print '\n'
print '---------------------------------', latest_timestamp, '---------------------------------'
print total_df
def __init__(self):
self.crawl = Crawl()
self.pipe = Pipeline()
self.analysis = Analysis()
"""
res = []
for match_id in match_ids:
url = 'https://api.opendota.com/api/matches/%s' % str(match_id)
r = requests.get(url)
r_str = r.content.decode()
r_json = json.loads(r_str)
res.append(r_json)
print("players" in r_json)
time.sleep(1)
pickle.dump(res, open("match_details_3p.pkl", "wb"))
return res
if __name__ == '__main__':
config = BaseConfig()
analysis = Analysis()
# request_url = construct_request_url(config.account_ids)
# print(request_url)
# match_ids = get_match_ids(request_url)
# print(len(match_ids))
# match_detail = get_matches_detail(match_ids)
match_detail = pickle.load(open("match_details_5p.pkl", "rb"))
total_odds = analysis.total_odds(config.account_ids[0],
match_detail,
verbose=True)
stat = analysis.hero_odds(config.account_ids, match_detail, verbose=True)
def train(self, training_set):
loss_average = []
iteration = 0
start = time.time()
# 训练开始
while iteration < self.num_epoch:
values = []
print("Iteration %s" % iteration)
# 每轮训练开始前，都需要重置底层网络和相关的强化学习环境
sub_copy = copy.deepcopy(self.sub)
env = NodeEnv(self.sub)
# 创建存储参数梯度的缓冲器
grad_buffer = self.sess.run(self.tvars)
# 初始化为0
for ix, grad in enumerate(grad_buffer):
grad_buffer[ix] = grad * 0
# 记录已经处理的虚拟网络请求数量
counter = 0
for req in training_set:
# 当前待映射的虚拟网络请求ID
req_id = req.graph['id']
print("\nHandling req%s..." % req_id)
if req.graph['type'] == 0:
print("\tIt's a newly arrived request, try to map it...")
counter += 1
sub_copy.total_arrived = counter
# 向环境传入当前的待映射虚拟网络
env.set_vnr(req)
# 获得底层网络的状态
observation = env.reset()
node_map = {}
xs, acts = [], []
for vn_id in range(req.number_of_nodes()):
x = np.reshape(observation, [1, observation.shape[0], observation.shape[1], 1])
sn_id = self.choose_action(observation, sub_copy, req.nodes[vn_id]['cpu'], acts)
if sn_id == -1:
break
else:
# 输入的环境信息添加到xs列表中
xs.append(x)
# 将选择的动作添加到acts列表中
acts.append(sn_id)
# 执行一次action，获取返回的四个数据
observation, _, done, info = env.step(sn_id)
node_map.update({vn_id: sn_id})
# end for,即一个VNR的全部节点映射全部尝试完毕
if len(node_map) == req.number_of_nodes():
link_map = Network.cut_then_find_path(sub_copy, req, node_map)
reward = Evaluation.revenue_to_cost_ratio(req, link_map)
if reward != -1:
epx = np.vstack(xs)
epy = np.eye(self.n_actions)[acts]
# 返回损失函数值
loss_value = self.sess.run(self.loss,
feed_dict={self.tf_obs: epx,
self.input_y: epy})
print("Success! The loss value is: %s" % loss_value)
values.append(loss_value)
# 返回求解梯度
tf_grad = self.sess.run(self.newGrads,
feed_dict={self.tf_obs: epx,
self.input_y: epy})
# 将获得的梯度累加到gradBuffer中
for ix, grad in enumerate(tf_grad):
grad_buffer[ix] += grad
grad_buffer[0] *= reward
grad_buffer[1] *= reward
# 分配资源
Network.allocate(sub_copy, req, node_map, link_map)
else:
print("Failure!")
# 当实验次数达到batch size整倍数，累积的梯度更新一次参数
if counter % self.batch_size == 0:
self.sess.run(self.update_grads,
feed_dict={self.kernel_grad: grad_buffer[0],
self.biases_grad: grad_buffer[1]})
# 清空gradBuffer
for ix, grad in enumerate(grad_buffer):
grad_buffer[ix] = grad * 0
if req.graph['type'] == 1:
# 收回该请求占用的资源
Network.recover(sub_copy, req)
env.set_sub(sub_copy)
loss_average.append(np.mean(values))
iteration = iteration + 1
end = (time.time() - start) / 3600
tool = Analysis('results_loss/')
tool.save_loss(end, self.num_epoch, loss_average,"RLN3")
def test(self,
rounds=20,
iterations=35,
learning_samples=1,
eval_samples=1,
prefix=''):
num_ask_help = np.zeros(iterations)
num_mismatch = np.zeros(iterations)
avg_data = np.zeros(iterations)
avg_distance = np.zeros(iterations)
avg_precision = np.zeros(iterations)
avg_human_input = np.zeros(iterations)
num_ask_help_r = np.zeros((rounds, iterations))
num_mismatch_r = np.zeros((rounds, iterations))
avg_data_r = np.zeros((rounds, iterations))
avg_distance_r = np.zeros((rounds, iterations))
avg_precision_r = np.zeros((rounds, iterations))
avg_human_input_r = np.zeros((rounds, iterations))
sup_data_r = np.zeros((rounds, iterations))
acc_data = np.zeros((rounds, iterations))
test_acc_data = np.zeros((rounds, iterations))
avg_losses_r = np.zeros((rounds, iterations))
avg_js_r = np.zeros((rounds, iterations))
# self.exp.task.env.changeLevel()
for r in range(rounds):
print "Trial: " + str(r)
self.agent.initialTraining = True
self.exp.doEpisodes(1)
self.agent.newModel()
self.agent.saveModel()
self.agent.initialTraining = False
self.agent.loadModel()
self.agent.reset()
distances = np.zeros(iterations)
mis_match = np.zeros(iterations)
data = np.zeros(iterations)
losses = np.zeros(iterations)
js = np.zeros(iterations)
sup_data = np.zeros(iterations)
num_help = np.zeros(iterations)
precision = np.zeros(iterations)
human_input = np.zeros(iterations)
acc = np.zeros(iterations)
test_acc = np.zeros(iterations)
for t in range(iterations):
rewards, loss, j, sup_rewards, = self.exp.doEpisodes(
1, learning_samples, eval_samples)
rewards = np.mean(rewards, axis=0)
loss = np.mean(loss, axis=0)
j = np.mean(j, axis=0)
data[t] = rewards[-1]
losses[t] = loss[-1]
js[t] = j[-1]
# data[t] = rewards[0][-1] # taking from the first sample
if self.agent._name == 'supervise':
sup_rewards = np.mean(sup_rewards, axis=0)
sup_data[t] = sup_rewards[-1]
#sup_data[t] = sup_rewards[0][-1] # taking from the first sample
#self.agent.updateModel()
acc[t] = self.agent.learner.accs
test_acc[t] = self.agent.learner.test_accs
# if(self.agent._getName() == 'Ahude'):
# num_help[t] = self.agent.getNumHelp()
# mis_match[t] = self.agent.getMismatch()
# self.agent.off = True
# rewards = self.exp.doEpisodes(1)
# self.agent.off = False
# size = len(rewards[0])
size = len(rewards)
#distances[t] = rewards[0][size-1]
distances[t] = rewards[size - 1]
precision[t] = self.agent.learner.getPrecision()
human_input[t] = self.agent.getNumHumanInput()
self.agent.reset()
# if(self.agent._getName() == 'Ahude'):
# num_ask_help = num_ask_help + num_help
# num_mismatch = num_mismatch + mis_match
avg_data_r[r, :] = data
avg_losses_r[r, :] = losses
avg_js_r[r, :] = js
if self.agent._name == 'supervise':
sup_data_r[r, :] = sup_data
acc_data[r, :] = acc
test_acc_data[r, :] = test_acc
# plot single trial
if self.agent._name == 'supervise':
np.save('./data/' + prefix + 'loss_round' + str(r) + '.npy',
losses)
np.save(
'./data/' + prefix + 'sl_reward_round' + str(r) + '.npy',
data)
np.save(
'./data/' + prefix + 'sup_reward_round' + str(r) + '.npy',
sup_data)
np.save('./data/' + prefix + 'acc_round' + str(r) + '.npy',
acc)
np.save(
'./data/' + prefix + 'test_acc_round' + str(r) + '.npy',
test_acc)
np.save('./data/' + prefix + 'js_round' + str(r) + '.npy', js)
a = Analysis()
a.get_perf(np.array([sup_data]), range(iterations))
a.get_perf(np.array([data]), range(iterations))
a.plot(names=['Supervisor', 'Supervised Learning'],
label='Rewards',
filename='./results/' + prefix + 'return_plot' +
str(r) + '.eps')
a = Analysis()
a.get_perf(np.array([losses]), range(iterations))
a.plot(names=['Supervised Learning'],
label='Loss',
filename='./results/' + prefix + 'loss_plot' + str(r) +
'.eps',
ylims=[0, 1])
a = Analysis()
a.get_perf(np.array([js]), range(iterations))
a.plot(names=['Supervised Learning'],
label='J()',
filename='./results/' + prefix + 'js_plot' + str(r) +
'.eps')
elif self.agent._name == 'dagger':
np.save('./data/' + prefix + 'loss_round' + str(r) + '.npy',
losses)
np.save(
'./data/' + prefix + 'dagger_reward_round' + str(r) +
'.npy', data)
np.save('./data/' + prefix + 'acc_round' + str(r) + '.npy',
acc)
np.save(
'./data/' + prefix + 'test_acc_round' + str(r) + '.npy',
test_acc)
np.save('./data/' + prefix + 'js_round' + str(r) + '.npy', js)
a = Analysis()
a.get_perf(np.array([data]), range(iterations))
a.plot(names=['DAgger'],
label='Reward',
filename='./results/' + prefix + 'return_plot' +
str(r) + '.eps')
a = Analysis()
a.get_perf(np.array([losses]), range(iterations))
a.plot(names=['DAgger'],
label='Loss',
filename='./results/' + prefix + 'loss_plot' + str(r) +
'.eps',
ylims=[0, 1])
a = Analysis()
a.get_perf(np.array([js]), range(iterations))
a.plot(names=['DAgger'],
label='J()',
filename='./results/' + prefix + 'js_plot' + str(r) +
'.eps')
avg_data = data + avg_data
avg_distance = distances + avg_distance
avg_precision = precision + avg_precision
avg_human_input = avg_human_input + human_input
# self.exp.task.env.changeLevel()
num_ask_help = num_ask_help / rounds
num_mismatch = num_mismatch / rounds
avg_data = avg_data / rounds
avg_distance = avg_distance / rounds
avg_precision = avg_precision / rounds
avg_human_input = avg_human_input / rounds
self.exp.task.env.setLevelBack()
#print avg_distance
if self.agent._name == 'supervise':
return avg_data_r, sup_data_r, acc_data, avg_losses_r, avg_js_r, test_acc_data
else:
return avg_data_r, None, acc_data, avg_losses_r, avg_js_r, test_acc_data
version = packageVersion+'('+version+')'
if logOut:
self.log.out("# "+intro+toolName+" [version: " + version + "]")
return version
############ command line testing ############
if __name__ == '__main__':
'''
Command-line testing - WARNING: out of date
'''
print "======== begin '" + sys.argv[0] + "' test ========"
e3 = Analysis('/hive/users/tdreszer/galaxy/galaxy-dist/tools/encode/settingsE3.txt',
analysisId='command-line test')
samInterim = e3.registerInterimOutput('sam',e3.dir + 'rep1Interim.sam')
bamTarget = e3.registerTargetOutput( 'bam',e3.dir + 'rep1Target.bam')
ls = LogicalStep(e3,stepName='test Logical Step')
sai1 = ls.declareGarbageFile('read1','sai')
sai2 = ls.declareGarbageFile('read2','sai')
sam = ls.declareInterimFile('sam')
bam = ls.declareAnalysisFile('bam')
ls.run() # should create temp dir, logFile etc.
# fake some outputs
e3.getCmdOut('echo sai1 > '+ sai1,logResult=True, log=ls.log)
e3.getCmdOut('echo sam > '+ sam,logResult=True, log=ls.log)
e3.getCmdOut('echo bam > '+ bam,logResult=True, log=ls.log)
# try failure:
def get_session_traffic_summary(username, session_id):
result = get_response_model()
try:
user = app.coordination.get_user(username)
session = app.coordination.get_session(session_id)
if not user_has_access_to_session(user, session):
raise Exception("Unauthorised access")
if session["intercept_mode"] != "decrypted":
raise Exception(
"The analysis of non-HAR files (such as the pcap files generated in Raw and Metadata modes) is not currently supported."
)
result["returned"] = {
"name":
session["name"] if "name" in session else None,
"intercept_mode":
session["intercept_mode"],
"created":
session["created"].astimezone().isoformat(),
"ended":
None if not "ended" in session else
session["ended"].astimezone().isoformat(),
"entry_node_name":
session["resources"]["entry_node"]["name"],
"exit_node_name":
session["resources"]["exit_node"]["name"],
"analysis_generated":
False,
"hosts": [],
"request_headers": [],
"request_queries": [],
"request_cookies": [],
"postdata_form_params": [],
"postdata_json_attributes": []
}
fileinfo = get_session_data_fileinfo(session_id)
if os.path.exists(fileinfo["fullpath"]):
if app.coordination_config["WebAPI"][
"analytics_exec_mode"] == "dask":
analysis = Analysis_Dask(
app.coordination_config["WebAPI"]
["analytics_exec_dask_config_scheduler"])
analysis.load_har_file(fileinfo["fullpath"])
analysis_tables = analysis.get_all_summary_tables()
result["returned"] = {**result["returned"], **analysis_tables}
result["returned"]["analysis_generated"] = True
else:
analysis = Analysis()
analysis.load_har_file(fileinfo["fullpath"])
result["returned"]["hosts"] = analysis.extract_har_hosts_count(
)
result["returned"][
"request_headers"] = analysis.extract_har_request_headers(
)
result["returned"][
"request_queries"] = analysis.extract_har_request_queries(
)
result["returned"][
"request_cookies"] = analysis.extract_har_request_cookies(
)
result["returned"][
"postdata_form_params"] = analysis.extract_har_request_postdata_form_params(
)
result["returned"][
"postdata_json_attributes"] = analysis.extract_har_request_postdata_json_attributes(
)
except Exception as ex:
result["errorMessage"] = str(ex)
return jsonify(result)
# get data
training_data = datasets.MNISTData(validation=False, batch_size=32)
validation_data = datasets.MNISTData(validation=True, batch_size=32)
# define model
nin = training_data.nin
nout = training_data.nout
model = Classifier(models.ConvNet(nin, 10, nout))
# Set up an optimizer
optimizer = chainer.optimizers.Adam()
optimizer.setup(model)
optimizer.add_hook(chainer.optimizer.WeightDecay(1e-5))
# instantiate learning algorithm
ann = supervised_learning.FeedforwardLearner(optimizer)
# run the optimization
ann.optimize(training_data, validation_data=validation_data, epochs=100)
# plot loss and throughput
ann.report('results/tmp')
# create analysis object
ana = Analysis(ann.model, fname='results/tmp')
# handle sequential data; deal with classifier analysis separately
# analyse data
ana.classification_analysis(validation_data.X, validation_data.T)
from analysis import Analysis
from algorithm import Algorithm
if __name__ == '__main__':
tool = Analysis('results_epoch_new/')
for i in range(20):
epoch = (i + 1) * 10
algorithm = Algorithm(name='ML', param=epoch)
runtime = algorithm.execute(network_path='networks/',
sub_filename='sub-wm.txt',
req_num=2000)
tool.save_evaluations(algorithm.evaluation, '%s.txt' % epoch)
tool.save_epoch(epoch, algorithm.evaluation.acc_ratio, runtime)
def main():
from optparse import OptionParser
usage = "usage: %prog [options] arg"
parser = OptionParser(usage)
parser.add_option("-c", "--code", dest="stockcode", help="stock code")
parser.add_option("-n", "--name", dest="stockname", help="stock name")
parser.add_option("-s",
"--stock",
dest="stocktxt",
help="read scraping stock names from text file")
parser.add_option("-r",
"--readfile",
dest="csvfile",
help="read stock data from csv file")
parser.add_option("-u",
"--update",
help="update csvfile (overwirte)",
action="store_true",
dest="update")
parser.add_option("-d",
"--date",
dest="startdate",
help="specify start date as '2014-10-01'")
parser.add_option("-y",
"--days",
dest="days",
help="plot days as '240', specify 0 for all days")
parser.add_option("-a",
"--axis",
dest="axis",
help="setting y-axis limit (1 or 2, default 2)")
parser.add_option("-p",
"--complexity",
dest="complexity",
help="complexity of chart (1-3, default 3)")
(options, args) = parser.parse_args()
if len(args) != 0:
parser.error("incorrect number of arguments")
if options.days is None:
options.days = 240
if options.axis is None:
options.axis = 2
if options.complexity is None:
options.complexity = 3
if options.stocktxt:
return read_csv(filename=options.stocktxt,
start=options.startdate,
days=int(options.days),
update=options.update,
axis=int(options.axis),
complexity=int(options.complexity))
else:
analysis = Analysis(code=options.stockcode,
name=options.stockname,
fullname=options.stockname,
start=options.startdate,
days=int(options.days),
csvfile=options.csvfile,
update=options.update,
axis=int(options.axis),
complexity=int(options.complexity))
return analysis.run()
def run(self, path):
self.path = path
data = pandas.read_csv(self.path)
data.dropna(inplace=True)
Analysis().boxplot(data)
from analysis import Analysis
from word_gen import WordGen
extended = [
'é', 'è', 'ê', 'ë', 'à', 'â', 'ä', 'î', 'ï', 'ù', 'û', 'ü', 'ç', 'ô', 'ö',
'œ'
]
A = Analysis("liste_francais.txt", extended)
A.run()
B = Analysis("words.txt", [], encoding="utf-8")
B.run()
wordA = WordGen(A)
wordB = WordGen(B)
for j in range(500):
print(wordA.generate_word(7), " | ", wordB.generate_word(7))
print(
"I hope you know what you're doing! Voting the previous proposal..."
)
approved_counts, rejected_counts, vote_list = 0, 0, []
for i in range(a.num_players):
choice = sanitised_input(
"Player " + str(i) +
", approve (1) or reject (0) mission? ", int, 0, 1)
if choice == 1:
approved_counts += 1
else:
rejected_counts += 1
vote_list.append(choice)
a.force_vote(approved_counts, rejected_counts, vote_list)
elif (user_input == "analyze" or user_input == "ana"):
ana = Analysis(a)
ana.start_analysis()
ana.analyze()
print(ana.player_values)
elif (user_input == "anamore"):
ana = Analysis(a)
ana.start_analysis()
ana.analyze(15)
print(ana.player_values)
else:
print("Invalid command.")
else:
print("Invalid command, must guess Merlin.")
if a.game_state == 0:
print("Minions have won!")
else:
