def calculate(self):
#get input data from various combo boxes
start_day = int(self.day1.currentText())
start_month = int(self.month1.currentText())
start_year = int(self.year1.currentText())
end_day = int(self.day2.currentText())
end_month = int(self.month2.currentText())
end_year = int(self.year2.currentText())
#make list of companies to be compared
list_companies = []
company_1 = str(self.comp1.currentText())
company_2 = str(self.comp2.currentText())
list_companies.append(company_1)
list_companies.append(company_2)
#call the 'compare' function from the imported file 'comparison.py'
comparison.compare(start_day, start_month, start_year, end_day, end_month, end_year, list_companies)
#get pixmaps from saved files
normalized_returns = QtGui.QPixmap('normalized_returns.jpg')
scatter1v2 = QtGui.QPixmap('scatter1v2.jpg')
#set the frames for labels & display pixmaps
self.normalized_label.setFrameStyle(QtGui.QFrame.Panel)
self.normalized_label.setPixmap(normalized_returns)
self.scatter_label.setFrameStyle(QtGui.QFrame.Panel)
self.scatter_label.setPixmap(scatter1v2)
def calculate(self):
#get input data from various combo boxes
start_day = int(self.day1.currentText())
start_month = int(self.month1.currentText())
start_year = int(self.year1.currentText())
end_day = int(self.day2.currentText())
end_month = int(self.month2.currentText())
end_year = int(self.year2.currentText())
#make list of companies to be compared
list_companies = []
company_1 = str(self.comp1.currentText())
company_2 = str(self.comp2.currentText())
list_companies.append(company_1)
list_companies.append(company_2)
#call the 'compare' function from the imported file 'comparison.py'
comparison.compare(start_day, start_month, start_year, end_day,
end_month, end_year, list_companies)
#get pixmaps from saved files
normalized_returns = QtGui.QPixmap('normalized_returns.jpg')
scatter1v2 = QtGui.QPixmap('scatter1v2.jpg')
#set the frames for labels & display pixmaps
self.normalized_label.setFrameStyle(QtGui.QFrame.Panel)
self.normalized_label.setPixmap(normalized_returns)
self.scatter_label.setFrameStyle(QtGui.QFrame.Panel)
self.scatter_label.setPixmap(scatter1v2)
def main():
# Read data from titles.txt
with open(TITLES_DIR, 'r') as file:
data = file.read()
# Each newline is a new movie
movies = data.split('\n')
# Split before the slash
movies = [x.split(' /')[0] for x in movies]
# Remove lines prefixed with * and blank lines
movies = [mov for mov in movies if not '*' in mov and mov.strip() != '']
try:
# Load the cache
with open(CACHE_DIR, 'r') as file:
cache = json.load(file)
except (json.decoder.JSONDecodeError, FileNotFoundError):
# Create cache if it doesn't exist
with open(CACHE_DIR, 'w'):
pass
cache = {'movies': []}
for movie in movies:
# Skip cached movies
cached_movies = []
for keys in [x.keys() for x in cache['movies']]:
for key in keys:
cached_movies.append(key)
if movie in cached_movies:
for mov in cache['movies']:
try:
name = mov[movie]['name']
break
except KeyError:
continue
name = movie
print(f'Cast found for {name}')
continue
# Find cast for uncached movies
cast = scrape(movie)
if cast:
cache['movies'].append(cast)
# Push changes to cache
with open(CACHE_DIR, 'w') as file:
json.dump(cache, file)
# Find common actors in movies
compare(movies, cache)
def get_wiki_descr(self):
wiki_html = self.wiki_html
city_desc_array = wiki_html.find_all('p')[:5]
try:
city_description = [
item.text for item in city_desc_array
if compare(self.city, item.text)
][0]
except IndexError:
city_description = ''
try:
city_description_sec_paragraph = [
item.text for item in city_desc_array
if compare(self.city, item.text)
][1]
except IndexError:
city_description_sec_paragraph = ''
if len(city_description) < 1000 and city_description != '':
city_description = (city_description +
city_description_sec_paragraph)
self.wiki_descr = '{0}{1}{2}{3}'.format(
city_description[:-1], '\n', '\t' *
16, '(из википедии)\n') if city_description != '' else ''
sig3 += (2 * np.sqrt(3)) * np.sin(2 * np.pi * freq5 * time) #alpha
sig3 += (2 * np.sqrt(3)) * np.sin(2 * np.pi * freq8 * time) #theta
sig3 += (2 * np.sqrt(1)) * np.sin(2 * np.pi * freq10 * time) #delta
sig3 += np.random.normal(scale=np.sqrt(noise_power),
size=time.shape) #add noise
# construction of signal 4
sig4 = (2 * np.sqrt(3)) * np.sin(2 * np.pi * freq4 * time) #beta
sig4 += (2 * np.sqrt(1.5)) * np.sin(2 * np.pi * freq6 * time) #alpha
sig4 += (2 * np.sqrt(3)) * np.sin(2 * np.pi * freq7 * time) #theta
sig4 += (2 * np.sqrt(5)) * np.sin(2 * np.pi * freq10 * time) #delta
sig4 += np.random.normal(scale=np.sqrt(noise_power),
size=time.shape) #add noise
#freqBase, PSD_base, PSD_impact, sumsBase, sumsImpact, relDiff = comparison.ft_compare(sig1, sig4, 0.006)
#comparison.ft_plot(freqBase, PSD_base, PSD_impact, sumsBase, sumsImpact)
#freq, C_base, C_impact, sumsBase, sumsImpact, relDiff = comparison.xcoh_compare(sig1, sig3, sig2, sig4, 0.006)
#comparison.xcoh_plot(freq, C_base, C_impact, sumsBase, sumsImpact)
# construction of signal 5
sig5 = (2 * np.sqrt(3)) * np.sin(2 * np.pi * freq4 * time) #beta
sig5 += (2 * np.sqrt(1.5)) * np.sin(2 * np.pi * freq6 * time) #alpha
sig5 += (2 * np.sqrt(3)) * np.sin(2 * np.pi * freq7 * time) #theta
sig5 += (2 * np.sqrt(5)) * np.sin(2 * np.pi * freq10 * time) #delta
sig5 += np.random.normal(scale=np.sqrt(noise_power),
size=time.shape) #add noise
sig5 += 150 #DC constant
comparison.compare(sig1, sig3, sig2, sig4, 0.006)
print('liberty')
def central_agent(net_weights_qs, net_gradients_qs, stats_qs):
logger = log.getLogger(name="central_agent", level=pm.LOG_MODE)
logger.info("Start central agent...")
if not pm.RANDOMNESS:
np.random.seed(pm.np_seed)
tf.set_random_seed(pm.tf_seed)
config = tf.ConfigProto()
config.allow_soft_placement=False
config.gpu_options.allow_growth = True
tb_logger = tb_log.Logger(pm.SUMMARY_DIR)
log_config(tb_logger)
with tf.Session(config=config) as sess:
policy_net = network.PolicyNetwork(sess, "policy_net", pm.TRAINING_MODE, logger)
if pm.VALUE_NET:
value_net = network.ValueNetwork(sess, "value_net", pm.TRAINING_MODE, logger)
logger.info("Create the policy network, with "+str(policy_net.get_num_weights())+" parameters")
sess.run(tf.global_variables_initializer())
tb_logger.add_graph(sess.graph)
tb_logger.flush()
policy_tf_saver = tf.train.Saver(max_to_keep=pm.MAX_NUM_CHECKPOINTS, var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='policy_net'))
if pm.POLICY_NN_MODEL is not None:
policy_tf_saver.restore(sess, pm.POLICY_NN_MODEL)
logger.info("Policy model "+pm.POLICY_NN_MODEL+" is restored.")
if pm.VALUE_NET:
value_tf_saver = tf.train.Saver(max_to_keep=pm.MAX_NUM_CHECKPOINTS, var_list=tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope='value_net'))
if pm.VALUE_NN_MODEL is not None:
value_tf_saver.restore(sess, pm.VALUE_NN_MODEL)
logger.info("Value model " + pm.VALUE_NN_MODEL + " is restored.")
step = 1
start_t = time.time()
if pm.VAL_ON_MASTER:
validation_traces = [] # validation traces
tags_prefix = ["DRF: ", "SRTF: ", "FIFO: ", "Tetris: ", "Optimus: "]
for i in range(pm.VAL_DATASET):
validation_traces.append(trace.Trace(None).get_trace())
stats = comparison.compare(copy.deepcopy(validation_traces), logger) # deep copy to avoid changes to validation_traces
if not pm.SKIP_FIRST_VAL:
stats.append(test(policy_net, copy.deepcopy(validation_traces), logger, step=0, tb_logger=tb_logger))
tags_prefix.append("Init_NN: ")
f = open(LOG_DIR + "baselines.txt", 'w')
for i in range(len(stats)):
jct, makespan, reward = stats[i]
value = tags_prefix[i] + " JCT: " + str(jct) + " Makespan: " + str(makespan) + " Reward: " + str(reward) + "\n"
f.write(value)
tb_logger.add_text(tag=tags_prefix[i], value=value, step=step)
f.close()
tb_logger.flush()
logger.info("Finish validation for heuristics and initialized NN.")
while step <= pm.TOT_NUM_STEPS:
# send updated parameters to agents
policy_weights = policy_net.get_weights()
if pm.VALUE_NET:
value_weights = value_net.get_weights()
for i in range(pm.NUM_AGENTS):
net_weights_qs[i].put((policy_weights, value_weights))
else:
for i in range(pm.NUM_AGENTS):
net_weights_qs[i].put(policy_weights)
# display speed
if step % 1 == 0:
elaps_t = time.time() - start_t
speed = step / elaps_t
logger.info("Central agent: Step " + str(
step) + " Speed " + '%.3f' % speed + " batches/sec" + " Time " + '%.3f' % elaps_t + " seconds")
# statistics
if pm.TRAINING_MODE == "RL":
policy_net.anneal_entropy_weight(step)
tb_logger.add_scalar(tag="Entropy Weight", value=policy_net.entropy_weight, step=step)
if pm.EPSILON_GREEDY:
eps = 2 / (1 + np.exp(step / pm.ANNEALING_TEMPERATURE)) * 0.6
tb_logger.add_scalar(tag="Epsilon Greedy", value=eps, step=step)
collect_stats(stats_qs, tb_logger, step)
if not pm.FIX_LEARNING_RATE:
if step in pm.ADJUST_LR_STEPS:
policy_net.lr /= 2
if pm.VALUE_NET:
value_net.lr /= 2
logger.info("Learning rate is decreased to " + str(policy_net.lr) + " at step " + str(step))
if step < pm.STEP_TRAIN_CRITIC_NET: # set policy net lr to 0 to train critic net only
policy_net.lr = 0.0
if step % pm.DISP_INTERVAL == 0:
tb_logger.add_scalar(tag="Learning rate", value=policy_net.lr, step=step)
# save model
if step % pm.CHECKPOINT_INTERVAL == 0:
name_prefix = ""
if pm.TRAINING_MODE == "SL":
name_prefix += "sl_"
else:
name_prefix += "rl_"
if pm.PS_WORKER:
name_prefix += "ps_worker_"
else:
name_prefix += "worker_"
model_name = pm.MODEL_DIR + "policy_" + name_prefix + str(step) + ".ckpt"
path = policy_tf_saver.save(sess, model_name)
logger.info("Policy model saved: " + path)
if pm.VALUE_NET and pm.SAVE_VALUE_MODEL:
model_name = pm.MODEL_DIR + "value_" + name_prefix + str(step) + ".ckpt"
path = value_tf_saver.save(sess, model_name)
logger.info("Value model saved: " + path)
# validation
if pm.VAL_ON_MASTER and step % pm.VAL_INTERVAL == 0:
test(policy_net, copy.deepcopy(validation_traces), logger, step, tb_logger)
# poll and update parameters
poll_ids = set([i for i in range(pm.NUM_AGENTS)])
avg_policy_grads = []
avg_value_grads = []
while True:
for i in poll_ids.copy():
try:
if pm.VALUE_NET:
policy_gradients, value_gradients = net_gradients_qs[i].get(False)
else:
policy_gradients = net_gradients_qs[i].get(False)
poll_ids.remove(i)
if len(avg_policy_grads) == 0:
avg_policy_grads = policy_gradients
else:
for j in range(len(avg_policy_grads)):
avg_policy_grads[j] += policy_gradients[j]
if pm.VALUE_NET:
if len(avg_value_grads) == 0:
avg_value_grads = value_gradients
else:
for j in range(len(avg_value_grads)):
avg_value_grads[j] += value_gradients[j]
except:
continue
if len(poll_ids) == 0:
break
for i in range(0, len(avg_policy_grads)):
avg_policy_grads[i] = avg_policy_grads[i] / pm.NUM_AGENTS
policy_net.apply_gradients(avg_policy_grads)
if pm.VALUE_NET:
for i in range(0, len(avg_value_grads)):
avg_value_grads[i] = avg_value_grads[i] / pm.NUM_AGENTS
value_net.apply_gradients(avg_value_grads)
# visualize gradients and weights
if step % pm.VISUAL_GW_INTERVAL == 0 and pm.EXPERIMENT_NAME is None:
assert len(policy_weights) == len(avg_policy_grads)
for i in range(0,len(policy_weights),10):
tb_logger.add_histogram(tag="Policy weights " + str(i), value=policy_weights[i], step=step)
tb_logger.add_histogram(tag="Policy gradients " + str(i), value=avg_policy_grads[i], step=step)
if pm.VALUE_NET:
assert len(value_weights) == len(avg_value_grads)
for i in range(0,len(value_weights),10):
tb_logger.add_histogram(tag="Value weights " + str(i), value=value_weights[i], step=step)
tb_logger.add_histogram(tag="Value gradients " + str(i), value=avg_value_grads[i], step=step)
step += 1
logger.info("Training ends...")
if pm.VALUE_NET:
for i in range(pm.NUM_AGENTS):
net_weights_qs[i].put(("exit", "exit"))
else:
for i in range(pm.NUM_AGENTS):
net_weights_qs[i].put("exit")
# os.system("sudo pkill -9 python")
exit(0)
def main(): scraper.get_tables() comparison.compare()
for file in files:
f = open(f"./tests/{file}", "r")
text = f.read()
file_name = Path(file).with_suffix('')
print(f"Processing {file}")
tokens = create_tokens(text)
tokens_iterator = peekable(tokens)
tree = parse_tokens(tokens_iterator)
result = generate(tree)
result_file = open(f"./results/{file_name}.s", "w")
result_file.write(result)
result_file.close()
system(f'gcc ./results/{file_name}.s -o ./compiled/{file_name}.c')
system(f'gcc -w ./tests/{file} -o ./compiledWithGCC/{file}')
print("GCC start")
subprocess.call(['sh', './scripts/echoGCCresults.sh'])
print("self-compiled start")
subprocess.call(['sh', './scripts/echoResults.sh'])
compare()
def plotData(i):
global n
global time_step
global window
global previousNumberValues
global currentNumberValues
global currentIndex
global numDiff
global impactIndex
global accX
global accY
global isConky
global elec1Impact
global elec2Impact
global impactDataRecorded
global subPSD11
global subPSD12
global subPSD13
global subPSD21
global subPSD22
global subPSD23
global subC1
global subC2
global subC3
global reportFigPSD1
global reportFigPSD2
global reportFigC
global isRecBaseline
global sig1
if (getData()):
print(str(max(accX)) + " " + str(max(accY)))
# Impact Checking
if ((isConky != True) and (impactDataRecorded != True) and isRecBaseline):
impactIndex = comparison.isImpact(accX, accY)
print("impact check!")
if (impactIndex >= 0):
impactText = mainFig.text(0.5, 0.5, "Impact Detected!", fontsize=40, ha='center',
bbox={'facecolor': 'red', 'alpha': 0.8, 'pad': 10})
print("done impact fcn")
isConky = True
if (isConky):
if(currentIndex - impactIndex > recordTime):
elec1Impact = elec1[impactIndex:(impactIndex + recordTime)]
elec2Impact = elec2[impactIndex:(impactIndex + recordTime)]
comparison.compare(elec1Baseline, elec2Baseline, elec1Impact, list(sig1[impactIndex:(impactIndex+recordTime)]), 0.00675, reportFigPSD1, subPSD11, subPSD12, subPSD13, reportFigPSD2, subPSD21, subPSD22, subPSD23, reportFigC, subC1, subC2, subC3)
print('Recorded impact!')
print(elec1Impact)
print(elec2Impact)
isConky = False
impactDataRecorded = True
# Data plotting
if (i % 2 == 0): # Only graph data every 3 iterations
currentNumberValues = len(elec1)
numDiff = currentNumberValues - previousNumberValues
currentIndex = currentNumberValues - 1
# Get real time values
timeValues = list(range(previousNumberValues, currentNumberValues))
for i in range(0, len(timeValues)):
timeValues[i] = timeValues[i] * time_step
timeValues = list(timeValues)
# Plot elec1 values
a1.clear()
a1.plot(timeValues,elec1[previousNumberValues:currentNumberValues],
"r")
a1.set_xlabel("Time")
a1.set_ylabel("Magnitude")
title="EEG Input 1"
a1.set_title(title)
fs = 166.67
N = currentNumberValues
time = np.arange(N) / fs
freq1 = 40
freq2 = 45
freq3 = 23
freq4 = 20
freq5 = 10
freq6 = 12
freq7 = 5
freq8 = 7
freq9 = 2
freq10 = 3
noise_power = 0.5 * fs / 2
sig1 = (2 * np.sqrt(2)) * np.sin(2 * np.pi * freq1 * time) # gamma
sig1 += (2 * np.sqrt(3)) * np.sin(2 * np.pi * freq3 * time) # beta
sig1 += (2 * np.sqrt(3)) * np.sin(2 * np.pi * freq4 * time) # beta
sig1 += (2 * np.sqrt(1.5)) * np.sin(2 * np.pi * freq6 * time) # alpha
sig1 += (2 * np.sqrt(2)) * np.sin(2 * np.pi * freq5 * time) # alpha
sig1 += (2 * np.sqrt(3)) * np.sin(2 * np.pi * freq7 * time) # theta
sig1 += (2 * np.sqrt(5)) * np.sin(2 * np.pi * freq10 * time) # delta
sig1 += np.random.normal(scale=np.sqrt(noise_power), size=time.shape)
sig1 += 106
# Plot elec2 values
a2.clear()
a2.plot(timeValues, list(sig1[previousNumberValues:currentNumberValues]),
"b")
a2.set_xlabel("Time")
a2.set_ylabel("Magnitude")
title="EEG Input 2"
a2.set_title(title)
previousNumberValues = currentNumberValues
return
train_correct = correct[0:int(0.7 * len(correct))]
test_correct = correct[int(0.7 * len(correct) + 1):len(correct)]
# Training phase
print("\n\nModel Training started\n\n")
match_list_noisy_channel_train = noisy_channel_evaluation(train,
config.word_dict,
type_of_data='train')
match_list_n_gram_train = ngram_evaluation(train)
match_list_unsupervised_rules_train = unsupervised_rules_evaluation(train)
# Training models comparison
print("\n\nTraining Comparison\n\n")
compare(match_list_noisy_channel_train,
match_list_n_gram_train,
match_list_unsupervised_rules_train,
list=train,
correct=train_correct,
plot_title="Training")
# Testing phase 1
match_list_noisy_channel_test = noisy_channel_evaluation(test,
config.word_dict,
type_of_data='test')
match_list_n_gram_test = ngram_evaluation(test)
match_list_unsupervised_rules_test = unsupervised_rules_evaluation(test)
compare(match_list_noisy_channel_test,
match_list_n_gram_test,
match_list_unsupervised_rules_test,
correct=test_correct,
list=test,
