def begin_scraping():
start = input("""Select the desired option: \n
1. Start Scraping \n""")
start = int(start)
# other_start = if you want to scrape and explore other options, press 2:
if start == 1:
model.get_data()
def on_click(self, event):
global party
global raw
global sort_ascending
if self.demand == "dem" or self.demand == "gop":
#key step: let global variable changable according to the demand, the button you clicked every time
party = self.demand
data = model.get_data(party=party,
raw=raw,
sort_ascending=sort_ascending)
self.chart.set_values(data)
if self.demand == True or self.demand == False:
if self.text == "Up" or self.text == "Down":
sort_ascending = self.demand
data = model.get_data(party=party,
raw=raw,
sort_ascending=sort_ascending)
self.chart.set_values(data)
else:
raw = self.demand
data = model.get_data(party=party,
raw=raw,
sort_ascending=sort_ascending)
if raw == True:
self.chart.max_val = 10000000
self.chart.set_values(data)
if raw == False:
self.chart.max_val = 100
self.chart.set_values(data)
def on_click(self, event):
self.switch_color()
if model.PARTY == 'dem':
data = model.get_data(party='gop',
raw=model.RAW,
sort_ascending=model.SORT_ASC)
else:
data = model.get_data(party='dem',
raw=model.RAW,
sort_ascending=model.SORT_ASC)
self.chart.set_values(data)
def main():
model = get_model(5, 188, 0.065185, 0.001032)
train_ds, test_ds = get_data()
# need to fit once so get_info() works
# TODO: Fix get_info() so I don't need to do this
model.fit(test_ds, epochs=1)
blockchain = BlockChain(model, train_ds, coal=30)
blockchain.model.evaluate(test_ds)
num = 0
while blockchain.mineable():
num += 1
blockchain.mine(Block(blockchain.get_info(), num))
blockchain.model.evaluate(test_ds)
# modify chain so it fails verification - just to test it
# blockchain.chain[-2].data = "sneaky Bitch!"
# blockchain.mine(blockchain.chain[-2])
for block in blockchain.chain:
print(block)
print('valid chain: {}'.format(blockchain.is_valid()))
def on_click(self, event):
global partySelection
partySelection = self.party
data = model.get_data(party=partySelection,
raw=rawSelection,
sort_ascending=sort_ascendingSelection)
self.chart.set_values(data, self.chart.max_val)
def on_click(self, event):
self.switch_color()
data = model.get_data(party=model.PARTY,
raw=model.RAW,
sort_ascending=not model.SORT_ASC)
self.chart.set_values(data)
def main(test):
t0 = time.time()
global normal_folder
global anomalous_folders
if test:
print("Test Dataset Mode")
normal_folder = [f"test_{r}" for r in normal_folder]
anomalous_folders = [f"test_{r}" for r in anomalous_folders]
X_train, X_test, y_train, y_test = get_data(
normal_folders=normal_folder,
anomalous_folders=anomalous_folders,
agent_map_folder=base_agent_map_folder,
)
t1 = time.time()
print("Time to get_data =", t1 - t0)
print(
"Dataset stats: ",
"\n",
f"X_train: {X_train.shape}, X_test: {X_test.shape}, y_train: {y_train.shape}, y_test: {y_test.shape}, ",
)
model, score = build_randomforest_model(
normal_folders=normal_folder,
anomalous_folders=anomalous_folders,
agent_map_folder=base_agent_map_folder,
data=(X_train, X_test, y_train, y_test),
)
print("RandomForest Results:", score)
def handle_event(self, event):
if event.type == pygame.MOUSEBUTTONDOWN:
(x, y) = pygame.mouse.get_pos()
if x >= 1000 and x <= 1150 and y >= 100 and y <= 140:
self.party = 'dem'
clickedbuttons[0] = 'dem'
if x >= 1000 and x <= 1150 and y > 140 and y <= 180:
self.party = 'gop'
clickedbuttons[0] = 'gop'
if x >= 1000 and x <= 1150 and y >= 300 and y <= 340:
self.ascend = True
clickedbuttons[1] = 'up'
if x >= 1000 and x <= 1150 and y > 340 and y <= 380:
self.ascend = False
clickedbuttons[1] = 'down'
if x >= 1000 and x <= 1150 and y >= 500 and y <= 540:
self.raw = True
clickedbuttons[2] = 'raw'
if x >= 1000 and x <= 1150 and y > 540 and y <= 580:
self.raw = False
clickedbuttons[2] = '%'
data = model.get_data(party=self.party,
raw=self.raw,
sort_ascending=self.ascend,
year=2016)
self.chart.set_values(data)
def test_get_data():
X, y = model.get_data()
assert isinstance(X, list)
assert isinstance(y, list)
assert len(X) == len(y) > 0
assert len(set(y)) == 2
assert all(isinstance(x, str) for x in X)
assert all(isinstance(i, int) for i in y)
def on_click(self, event):
self.switch_color()
if model.RAW:
data = model.get_data(party=model.PARTY,
raw=not model.RAW,
sort_ascending=model.SORT_ASC)
self.chart.set_values(data)
self.chart.set_decimal(True)
self.chart.set_max_val(1)
else:
data = model.get_data(party=model.PARTY,
raw=not model.RAW,
sort_ascending=model.SORT_ASC)
self.chart.set_values(data)
self.chart.set_decimal(False)
self.chart.set_max_val(10000000)
def _action_roster(cid, user, content, update, context):
"""
Show sleepy members
"""
content = content.split(b'\\n')[1:]
# check UTC battle
utcnow = datetime.datetime.utcnow()
time_list = []
for t in settings.BATTLES:
time_list.append(
(utcnow - datetime.datetime(year=utcnow.year,
month=utcnow.month,
day=utcnow.day,
hour=int(t[0:2]),
minute=int(t[3:5]))).total_seconds() /
60.0)
delta = int(model.get_data("BATTLE_TIME_DELTA_MINUTES", 20))
time_list = [t > -delta and t < 0 for t in time_list]
print(" Roster", time_list, delta)
# call to the battle
if True in time_list:
i = 0
tmp = ""
header = u'\U0001F4E2 The battle is coming\n'
for player in content:
if b' [\\u2694] ' not in player and b' [\\U0001f6e1] ' not in player and b' [\\U0001f4a4] ' not in player:
cw_name = b' '.join(player.split(b' ')[3:]).decode()
user = model.user_by_cw_name(cw_name)
if user:
tmp += "@{0} ".format(user.username)
i += 1
if not i % 3:
i = 0
try:
if tmp:
context.bot.send_message(
chat_id=cid,
text=header + tmp,
parse_mode=telegram.ParseMode.HTML,
disable_web_page_preview=True)
except Exception as e:
utils._admin_error(context,
"_action_roster: attention",
user=user,
error=str(e))
tmp = ""
if tmp:
try:
context.bot.send_message(chat_id=cid,
text=header + tmp,
parse_mode=telegram.ParseMode.HTML,
disable_web_page_preview=True)
except Exception as e:
utils._admin_error(context,
"_action_roster: attention",
user=user,
error=str(e))
def getData(self, params):
"""
Get DF from model per sport. Process data and return DF ready for plotting.
"""
file = params["sport_type"]
df = model.get_data(file)
df["distance"] = df["distance"] / 1000
df["date"] = df["date"].map(lambda x: x.date())
return df
def main():
pygame.init()
screen = pygame.display.set_mode((1200, 800))
pygame.display.set_caption("Election Data Viewer")
pygame.display.update()
data = model.get_data()
bc = view.BarChart(rect=pygame.Rect(0, 0,
screen.get_width() - 200,
screen.get_height()),
plot_area_width_ratio=0.9,
plot_area_height_ratio=0.8,
values=data,
ticks=5,
max_val=10000000)
bc.set_rotate(True)
bc.set_decimal(False)
button1 = DataChangeButton1(["dem", "gop"],
pygame.Rect(screen.get_width() - 200, 70, 150,
60), bc)
button2 = DataChangeButton2(["up", "down"],
pygame.Rect(screen.get_width() - 200, 170, 150,
60), bc)
button3 = DataChangeButton3(["raw", "%"],
pygame.Rect(screen.get_width() - 200, 270, 150,
60), bc)
# display loop
done = False
while not done:
screen.fill(view.BLACK)
for event in pygame.event.get():
if event.type == pygame.QUIT:
done = True
else:
button1.handle_event(event)
button2.handle_event(event)
button3.handle_event(event)
bc.draw(screen)
button1.draw(screen)
button2.draw(screen)
button3.draw(screen)
pygame.display.update()
def on_click(self, event):
global rawSelection
rawSelection = self.raw
if rawSelection == True:
maxValNow = maxValGlobal
else:
maxValNow = 100
data = model.get_data(party=partySelection,
raw=rawSelection,
sort_ascending=sort_ascendingSelection)
self.chart.set_values(data, maxValNow)
def predict():
content = request.get_json()
# {"input":"1,2,3,4,5","model":"SA"}
x = get_data(content["input"])
output = None
if content["model"] == "SA":
output = model_SA.predict(x)
elif content["model"] == "BKNET":
output = model_BKNET.predict(x)
if output is not None:
output = ",".join(output.astype(str))
return jsonify({"message": "OK", "data": output})
else:
return jsonify({"message": "Wrong argument"})
def check_outdata_crafting_list():
while True:
time.sleep(int(model.get_data("CRAFT_OUTDATE_INTERVAL_HOURS", 8*3600)))
cid=model.get_data("CRAFTING_ROOM_CHAT_ID", None)
print("[TIMER] check_outdata_crafting_list:", cid)
if cid:
today=datetime.datetime.today()
try:
for u in model.users():
crafting=json.loads(u.crafting)
if crafting and len(crafting.keys()):
# validate the date
t=datetime.datetime.fromisoformat(crafting["datetime"])
tmp=(datetime.datetime.today()-t).total_seconds()/(24.0*3600.0)
if tmp>int(model.get_data("CRAFT_OUTDATE_INTERVAL_DAYS", 3)):
BOT.send_message(chat_id=cid,
text=u'@{0}, your crafting data is outdated. Please, forward it here...'.format(html.escape(u.username)),
parse_mode=telegram.ParseMode.HTML,
disable_web_page_preview=True)
except Exception as e:
class Context:
bot=BOT
context=Context()
utils._admin_error(context, "timer.check_outdata_crafting_list", error=str(e))
def elite(update, context):
"""
Mention the elite
"""
cid=update.message.chat.id
# validate user
if not utils._check_user(context, update.effective_user.id, cid):
return
# telegram user
user=model.User(id=update.effective_user.id,
username=update.effective_user.username,
full_name=update.effective_user.full_name,
link=update.effective_user.link,
is_bot=update.effective_user.is_bot)
# mention users
users=model.users()
i=0
tmp=""
max_cw_level=0
for u in users:
if u.cw_level>max_cw_level:
max_cw_level=u.cw_level
for u in users:
if u.id==str(update.effective_user.id) or u.cw_level<max_cw_level-model.get_data("MENTION_ELITE_DELTA", 15):
continue
tmp+="@{0} ".format(u.username)
i+=1
if not i%3:
i=0
try:
context.bot.send_message(chat_id=cid,
text=u"\U0000270A Sir! Yes Sir!\n"+tmp,
parse_mode=telegram.ParseMode.HTML,
disable_web_page_preview=True)
except Exception as e:
utils._admin_error(context, "/elite", user=user, error=str(e))
tmp=""
if tmp:
try:
context.bot.send_message(chat_id=cid,
text=u"\U0000270A Sir! Yes Sir!\n"+tmp,
parse_mode=telegram.ParseMode.HTML,
disable_web_page_preview=True)
except Exception as e:
utils._admin_error(context, "/elite", user=user, error=str(e))
def page_info(request):
data = []
res = []
page = get_data(request.GET['page'])
if (page == None):
return HttpResponse(json.dumps(''), mimetype="application/json")
print page
if (page['product'].count > 0):
for obj in page['product']:
if (obj != ''):
res += [{'image': obj[0], 'url': obj[1], 'float': obj[2]}]
desc = []
for obj in page['desc']:
if (obj != ''):
desc += [{'desc': obj[0]}]
data = {"product": res, 'desc': desc}
return HttpResponse(json.dumps(data), mimetype="application/json")
def dispatch_request(self, searchValue, term):
# do rendering before some model coding
"""
Terms to search : 1. Dow fall
2. Dow rise
3. nasdaq fall
4. nasdaq rise
5. sp500 fall
6. sp500 rise
"""
term_copy = term.lower()
data_points = get_data(int(searchValue), term_copy)
return render_template('reader.html',
searchValue=searchValue,
term=term,
data_points=data_points)
def get_data(self, request, component, metric):
session = self.__SessionMaker()
# See if it is in the database
row = session.query(Data).filter(Data.component == component).filter(Data.metric == metric).all()
if len(row) > 0:
model = row[0]
else:
request.setResponseCode(400)
request.redirect("/")
return ""
ret, should_save = model.get_data()
if should_save:
session.merge(model)
session.commit()
return str(ret)
def page_info(request):
data = []
res = []
page = get_data(request.GET['page'])
if(page == None):
return HttpResponse(json.dumps(''), mimetype="application/json")
print page
if(page['product'].count > 0):
for obj in page['product']:
if(obj != ''):
res += [{
'image': obj[0],
'url': obj[1],
'float': obj[2]
}]
desc = []
for obj in page['desc']:
if(obj != ''):
desc += [{
'desc': obj[0]
}]
data = {"product": res, 'desc': desc}
return HttpResponse(json.dumps(data), mimetype="application/json")
def test_model_performance():
if not os.path.exists(constants.MODEL_FILE):
warnings.warn(
f"{constants.MODEL_FILE} not found, skipping test_model_performance()"
)
return
if not os.path.exists(constants.TOKENIZER_PICKLE):
warnings.warn(
f"{constants.TOKENIZER_PICKLE} not found, skipping test_model_performance()"
)
return
m = model.get_model()
tokenizer = model.get_tokenizer()
texts, labels = model.get_data()
X = tokenizer.texts_to_sequences(texts)
X = tf.keras.preprocessing.sequence.pad_sequences(X, maxlen=16)
y = tf.keras.utils.to_categorical(labels)
accuracy = m.evaluate(X, y)[1]
assert accuracy > .9
activityid = 17972
username = "******"
if __name__ == '__main__':
# Event data
tables = model.get_metadata(activityid)
event_details = model.process_event(tables[0])
pprint(event_details)
# Pairs
standings = []
if len(tables) > 1:
for table in tables[1:]:
standings += model.process_standings(table)
# pprint(standings)
# Board data
data = model.get_data(activityid, username)
data_tables = model.get_data_table(data)
tables = model.extract_data(data_tables)
# pprint(f'{len(tables)} tables') # 5
# tables[0] contains info about the data
# tables[1] is the first header 'Overzicht Resultaten'
# tables[2] is the results table
# tables[3] is the second header 'Details per Spel'
# tables[4] is the details table
# 2020-10-03: Decided against using the results table
# Reason: gathering the same data from the details will
# have benefit of adding the lead.
# results = model.process_results(tables[2])
# pprint(results)
boards = model.process_details(tables[4], username)
import numpy
import model
avg = numpy.mean(model.get_data()[:], axis=0)
model.draw_grid(avg / 255.).save('avg.png')
sort_ascending=self.ascend,
year=2016)
self.chart.set_values(data)
pygame.init()
screen = pygame.display.set_mode((1200, 800))
pygame.display.set_caption("Election Data Viewer")
pygame.display.update()
screen_rect = screen.get_rect()
bc = BarChart(rect=screen.get_rect(),
values=model.get_data(party='dem',
raw=True,
sort_ascending=True,
year=2016),
ticks=4)
button1 = DataChangeButton("dem", pygame.Rect(1000, 100, 150, 40), bc)
button2 = DataChangeButton("gop", pygame.Rect(1000, 140, 150, 40), bc)
button3 = DataChangeButton("up", pygame.Rect(1000, 300, 150, 40), bc)
button4 = DataChangeButton("down", pygame.Rect(1000, 340, 150, 40), bc)
button5 = DataChangeButton("raw", pygame.Rect(1000, 500, 150, 40), bc)
button6 = DataChangeButton("%", pygame.Rect(1000, 540, 150, 40), bc)
allbuttons = [button1, button2, button3, button4, button5, button6]
# display loop
done = False
while not done:
screen = pygame.display.set_mode((1200, 800))
pygame.display.set_caption("Election Data Viewer")
pygame.display.update()
BLACK = (0, 0, 0)
WHITE = (255, 255, 255)
GRAY = (127, 127, 127)
RED = (255, 0, 0)
GREEN = (0, 255, 0)
BLUE = (0, 0, 255)
YELLOW = (255, 255, 0)
MAGENTA = (255, 0, 255)
CYAN = (0, 255, 255)
data = model.get_data()
bc1 = view.BarChart(rect=pygame.Rect(0, 50, 950, 700),
ticks=5,
max_val=10000000,
plot_area_width_ratio=0.95,
values=data)
party = "dem"
max_val = 10000000
raw = True
sort_ascending = True
class OneButton(Button):
def __init__(self, text, rect, chart, demand):
import random
import numpy
import model
data = model.get_data()
n = data.shape[0]
alphabet = numpy.zeros((62, 64, 64))
for z in xrange(62):
alphabet[z] = random.choice(data)[z] * 1.0/255
model.draw_grid(alphabet).save('alphabet.png')
import os
import sys
import model
import utils
import transfer
import numpy as np
if __name__ == '__main__':
if sys.argv[2] == 'dry':
os.chdir(os.path.dirname(sys.argv[1]))
train = utils.load_data_to_memory(os.path.join(sys.argv[1], 'train'))
test = utils.load_data_to_memory(os.path.join(sys.argv[1], 'test'))
train_x = model.get_data(train, 224)
train_y = model.get_labels(train)
test_x = model.get_data(test, 224)
test_y = model.get_labels(test)
model_ = model.get_MobileNet_v2((224, 224, 1))
train_x = np.expand_dims(train_x, -1)
test_x = np.expand_dims(test_x, -1)
model_ = model.build(train_x, train_y, test_x, test_y, model_)
elif sys.argv[2] == 'transfer':
fer_dataset = '~/Downloads/fer2013/fer2013.csv'
os.chdir(os.path.dirname(sys.argv[1]))
train = utils.load_data_to_memory(os.path.join(sys.argv[1], 'train'))
test = utils.load_data_to_memory(os.path.join(sys.argv[1], 'test'))
train_x = model.get_data(train, 48)
train_y = model.get_labels(train)
test_x = model.get_data(test, 48)
import tensorflow as tf
from tensorflow import keras
import numpy as np
import model
import utils
import cv2
test_dir = '/Users/ezra/Downloads/Face_data_split/test'
test = utils.load_data_to_memory(test_dir)
test_x = model.get_data(test, 48)
test_y = model.get_labels(test)
model_ = keras.models.load_model('model.h5')
test_x = np.expand_dims(test_x, -1)
model_.compile(optimizer='adam',
loss='categorical_crossentropy',
metrics=['accuracy'])
# running evaluation of the test set
model_.evaluate(test_x, test_y)
# producing the test set images
test_x = tf.cast(test_x, tf.float32)
predictions = model_.predict(test_x)
key_ = dict()
key_['0'] = 'neutral'
key_['1'] = 'anger'
key_['2'] = 'surprise'
key_['3'] = 'smile'
key_['4'] = 'sad'
import numpy as np
import model as md
x_input, y_input, n, k, wh = md.get_data()
model = md.Model(n, k, wh)
print(n, k, wh)
#model.try_load()
x_train, y_train, x_test, y_test = md.load_data(x_input, y_input)
for lr in [0.1, 0.01, 0.001]:
model.train(x_train,
y_train,
x_test,
y_test,
batch_size=100,
learning_late=lr)
model.model_test(x_test, y_test)
import pygame
import model #imports model.py file in this directory!!
# all import statements are importing a file or folder that
# exists somewhere !!! That this program has access to!
import view
pygame.init()
surface = pygame.display.set_mode((800,600))
font = pygame.font.Font(None, 20)
pygame.display.set_caption("Scatterplot")
data = model.get_data('data.csv')
scatter = view.ScatterPlot(data, surface, font)
loop = True
while loop:
for event in pygame.event.get():
if event.type == pygame.QUIT:
loop = False
scatter.draw()
pygame.display.update()
pygame.quit()
quit()
done = False
while not done:
screen.fill(view.BLACK)
for event in pygame.event.get():
if event.type == pygame.QUIT:
done = True
else:
button.handle_event(event)
if button.chosen == True:
b1c = view.RED
b2c = view.GRAY
party = "dem"
data = model.get_data(party=party,
raw=raw,
sort_ascending=sort_ascending)
bc = view.BarChart(pygame.Rect(50, 50, 1000, 600),
values=data,
ticks=5,
max_val=mv)
button_gop.chosen = False
#print("DEM Selected")
button_gop.handle_event(event)
if button_gop.chosen == True:
b2c = view.RED
b1c = view.GRAY
party = "gop"
data = model.get_data(party=party,
raw=raw,
def trends():
return render_template('trends.html',
num_fake=model.get_data("FAKE"),
num_real=model.get_data("REAL"),
num_clickbait=model.get_data("CLICKBAIT"),
num_notclickbait=model.get_data("NOT CLICKBAIT"))
from redis import Redis
import time, sys
from model import get_data, predict, to_target_names
import numpy as np
# settings
wait_seconds = 3
# Tell RQ what Redis connection to use
redis_conn = Redis()
q = Queue(connection=redis_conn) # no args implies the default queue
# Take the input from the user or the test dataset.
if len(sys.argv) == 5:
X_predict = [[float(i) for i in sys.argv[1:]]]
else:
_, X_test, _, y_test = get_data()
X_predict = X_test[:3]
# submit the job
job = q.enqueue(predict, X_predict)
print('Predicting on:')
print(X_predict)
print('Checking the job immediately we get:')
# check the result before they are complete
print(to_target_names(job.result)) # => None
# Now, wait a while, until the worker is finished and check again
print(f"After waiting for {wait_seconds} seconds we get:")
time.sleep(wait_seconds)
print(to_target_names(job.result)) # => [0, 1, 2] or something
