def begin_scraping(): start = input("""Select the desired option: \n 1. Start Scraping \n""") start = int(start) # other_start = if you want to scrape and explore other options, press 2: if start == 1: model.get_data()
def on_click(self, event): global party global raw global sort_ascending if self.demand == "dem" or self.demand == "gop": #key step: let global variable changable according to the demand, the button you clicked every time party = self.demand data = model.get_data(party=party, raw=raw, sort_ascending=sort_ascending) self.chart.set_values(data) if self.demand == True or self.demand == False: if self.text == "Up" or self.text == "Down": sort_ascending = self.demand data = model.get_data(party=party, raw=raw, sort_ascending=sort_ascending) self.chart.set_values(data) else: raw = self.demand data = model.get_data(party=party, raw=raw, sort_ascending=sort_ascending) if raw == True: self.chart.max_val = 10000000 self.chart.set_values(data) if raw == False: self.chart.max_val = 100 self.chart.set_values(data)
def on_click(self, event): self.switch_color() if model.PARTY == 'dem': data = model.get_data(party='gop', raw=model.RAW, sort_ascending=model.SORT_ASC) else: data = model.get_data(party='dem', raw=model.RAW, sort_ascending=model.SORT_ASC) self.chart.set_values(data)
def main(): model = get_model(5, 188, 0.065185, 0.001032) train_ds, test_ds = get_data() # need to fit once so get_info() works # TODO: Fix get_info() so I don't need to do this model.fit(test_ds, epochs=1) blockchain = BlockChain(model, train_ds, coal=30) blockchain.model.evaluate(test_ds) num = 0 while blockchain.mineable(): num += 1 blockchain.mine(Block(blockchain.get_info(), num)) blockchain.model.evaluate(test_ds) # modify chain so it fails verification - just to test it # blockchain.chain[-2].data = "sneaky Bitch!" # blockchain.mine(blockchain.chain[-2]) for block in blockchain.chain: print(block) print('valid chain: {}'.format(blockchain.is_valid()))
def on_click(self, event): global partySelection partySelection = self.party data = model.get_data(party=partySelection, raw=rawSelection, sort_ascending=sort_ascendingSelection) self.chart.set_values(data, self.chart.max_val)
def on_click(self, event): self.switch_color() data = model.get_data(party=model.PARTY, raw=model.RAW, sort_ascending=not model.SORT_ASC) self.chart.set_values(data)
def main(test): t0 = time.time() global normal_folder global anomalous_folders if test: print("Test Dataset Mode") normal_folder = [f"test_{r}" for r in normal_folder] anomalous_folders = [f"test_{r}" for r in anomalous_folders] X_train, X_test, y_train, y_test = get_data( normal_folders=normal_folder, anomalous_folders=anomalous_folders, agent_map_folder=base_agent_map_folder, ) t1 = time.time() print("Time to get_data =", t1 - t0) print( "Dataset stats: ", "\n", f"X_train: {X_train.shape}, X_test: {X_test.shape}, y_train: {y_train.shape}, y_test: {y_test.shape}, ", ) model, score = build_randomforest_model( normal_folders=normal_folder, anomalous_folders=anomalous_folders, agent_map_folder=base_agent_map_folder, data=(X_train, X_test, y_train, y_test), ) print("RandomForest Results:", score)
def handle_event(self, event): if event.type == pygame.MOUSEBUTTONDOWN: (x, y) = pygame.mouse.get_pos() if x >= 1000 and x <= 1150 and y >= 100 and y <= 140: self.party = 'dem' clickedbuttons[0] = 'dem' if x >= 1000 and x <= 1150 and y > 140 and y <= 180: self.party = 'gop' clickedbuttons[0] = 'gop' if x >= 1000 and x <= 1150 and y >= 300 and y <= 340: self.ascend = True clickedbuttons[1] = 'up' if x >= 1000 and x <= 1150 and y > 340 and y <= 380: self.ascend = False clickedbuttons[1] = 'down' if x >= 1000 and x <= 1150 and y >= 500 and y <= 540: self.raw = True clickedbuttons[2] = 'raw' if x >= 1000 and x <= 1150 and y > 540 and y <= 580: self.raw = False clickedbuttons[2] = '%' data = model.get_data(party=self.party, raw=self.raw, sort_ascending=self.ascend, year=2016) self.chart.set_values(data)
def test_get_data(): X, y = model.get_data() assert isinstance(X, list) assert isinstance(y, list) assert len(X) == len(y) > 0 assert len(set(y)) == 2 assert all(isinstance(x, str) for x in X) assert all(isinstance(i, int) for i in y)
def on_click(self, event): self.switch_color() if model.RAW: data = model.get_data(party=model.PARTY, raw=not model.RAW, sort_ascending=model.SORT_ASC) self.chart.set_values(data) self.chart.set_decimal(True) self.chart.set_max_val(1) else: data = model.get_data(party=model.PARTY, raw=not model.RAW, sort_ascending=model.SORT_ASC) self.chart.set_values(data) self.chart.set_decimal(False) self.chart.set_max_val(10000000)
def _action_roster(cid, user, content, update, context): """ Show sleepy members """ content = content.split(b'\\n')[1:] # check UTC battle utcnow = datetime.datetime.utcnow() time_list = [] for t in settings.BATTLES: time_list.append( (utcnow - datetime.datetime(year=utcnow.year, month=utcnow.month, day=utcnow.day, hour=int(t[0:2]), minute=int(t[3:5]))).total_seconds() / 60.0) delta = int(model.get_data("BATTLE_TIME_DELTA_MINUTES", 20)) time_list = [t > -delta and t < 0 for t in time_list] print(" Roster", time_list, delta) # call to the battle if True in time_list: i = 0 tmp = "" header = u'\U0001F4E2 The battle is coming\n' for player in content: if b' [\\u2694] ' not in player and b' [\\U0001f6e1] ' not in player and b' [\\U0001f4a4] ' not in player: cw_name = b' '.join(player.split(b' ')[3:]).decode() user = model.user_by_cw_name(cw_name) if user: tmp += "@{0} ".format(user.username) i += 1 if not i % 3: i = 0 try: if tmp: context.bot.send_message( chat_id=cid, text=header + tmp, parse_mode=telegram.ParseMode.HTML, disable_web_page_preview=True) except Exception as e: utils._admin_error(context, "_action_roster: attention", user=user, error=str(e)) tmp = "" if tmp: try: context.bot.send_message(chat_id=cid, text=header + tmp, parse_mode=telegram.ParseMode.HTML, disable_web_page_preview=True) except Exception as e: utils._admin_error(context, "_action_roster: attention", user=user, error=str(e))
def getData(self, params): """ Get DF from model per sport. Process data and return DF ready for plotting. """ file = params["sport_type"] df = model.get_data(file) df["distance"] = df["distance"] / 1000 df["date"] = df["date"].map(lambda x: x.date()) return df
def main(): pygame.init() screen = pygame.display.set_mode((1200, 800)) pygame.display.set_caption("Election Data Viewer") pygame.display.update() data = model.get_data() bc = view.BarChart(rect=pygame.Rect(0, 0, screen.get_width() - 200, screen.get_height()), plot_area_width_ratio=0.9, plot_area_height_ratio=0.8, values=data, ticks=5, max_val=10000000) bc.set_rotate(True) bc.set_decimal(False) button1 = DataChangeButton1(["dem", "gop"], pygame.Rect(screen.get_width() - 200, 70, 150, 60), bc) button2 = DataChangeButton2(["up", "down"], pygame.Rect(screen.get_width() - 200, 170, 150, 60), bc) button3 = DataChangeButton3(["raw", "%"], pygame.Rect(screen.get_width() - 200, 270, 150, 60), bc) # display loop done = False while not done: screen.fill(view.BLACK) for event in pygame.event.get(): if event.type == pygame.QUIT: done = True else: button1.handle_event(event) button2.handle_event(event) button3.handle_event(event) bc.draw(screen) button1.draw(screen) button2.draw(screen) button3.draw(screen) pygame.display.update()
def on_click(self, event): global rawSelection rawSelection = self.raw if rawSelection == True: maxValNow = maxValGlobal else: maxValNow = 100 data = model.get_data(party=partySelection, raw=rawSelection, sort_ascending=sort_ascendingSelection) self.chart.set_values(data, maxValNow)
def predict(): content = request.get_json() # {"input":"1,2,3,4,5","model":"SA"} x = get_data(content["input"]) output = None if content["model"] == "SA": output = model_SA.predict(x) elif content["model"] == "BKNET": output = model_BKNET.predict(x) if output is not None: output = ",".join(output.astype(str)) return jsonify({"message": "OK", "data": output}) else: return jsonify({"message": "Wrong argument"})
def check_outdata_crafting_list(): while True: time.sleep(int(model.get_data("CRAFT_OUTDATE_INTERVAL_HOURS", 8*3600))) cid=model.get_data("CRAFTING_ROOM_CHAT_ID", None) print("[TIMER] check_outdata_crafting_list:", cid) if cid: today=datetime.datetime.today() try: for u in model.users(): crafting=json.loads(u.crafting) if crafting and len(crafting.keys()): # validate the date t=datetime.datetime.fromisoformat(crafting["datetime"]) tmp=(datetime.datetime.today()-t).total_seconds()/(24.0*3600.0) if tmp>int(model.get_data("CRAFT_OUTDATE_INTERVAL_DAYS", 3)): BOT.send_message(chat_id=cid, text=u'@{0}, your crafting data is outdated. Please, forward it here...'.format(html.escape(u.username)), parse_mode=telegram.ParseMode.HTML, disable_web_page_preview=True) except Exception as e: class Context: bot=BOT context=Context() utils._admin_error(context, "timer.check_outdata_crafting_list", error=str(e))
def elite(update, context): """ Mention the elite """ cid=update.message.chat.id # validate user if not utils._check_user(context, update.effective_user.id, cid): return # telegram user user=model.User(id=update.effective_user.id, username=update.effective_user.username, full_name=update.effective_user.full_name, link=update.effective_user.link, is_bot=update.effective_user.is_bot) # mention users users=model.users() i=0 tmp="" max_cw_level=0 for u in users: if u.cw_level>max_cw_level: max_cw_level=u.cw_level for u in users: if u.id==str(update.effective_user.id) or u.cw_level<max_cw_level-model.get_data("MENTION_ELITE_DELTA", 15): continue tmp+="@{0} ".format(u.username) i+=1 if not i%3: i=0 try: context.bot.send_message(chat_id=cid, text=u"\U0000270A Sir! Yes Sir!\n"+tmp, parse_mode=telegram.ParseMode.HTML, disable_web_page_preview=True) except Exception as e: utils._admin_error(context, "/elite", user=user, error=str(e)) tmp="" if tmp: try: context.bot.send_message(chat_id=cid, text=u"\U0000270A Sir! Yes Sir!\n"+tmp, parse_mode=telegram.ParseMode.HTML, disable_web_page_preview=True) except Exception as e: utils._admin_error(context, "/elite", user=user, error=str(e))
def page_info(request): data = [] res = [] page = get_data(request.GET['page']) if (page == None): return HttpResponse(json.dumps(''), mimetype="application/json") print page if (page['product'].count > 0): for obj in page['product']: if (obj != ''): res += [{'image': obj[0], 'url': obj[1], 'float': obj[2]}] desc = [] for obj in page['desc']: if (obj != ''): desc += [{'desc': obj[0]}] data = {"product": res, 'desc': desc} return HttpResponse(json.dumps(data), mimetype="application/json")
def dispatch_request(self, searchValue, term): # do rendering before some model coding """ Terms to search : 1. Dow fall 2. Dow rise 3. nasdaq fall 4. nasdaq rise 5. sp500 fall 6. sp500 rise """ term_copy = term.lower() data_points = get_data(int(searchValue), term_copy) return render_template('reader.html', searchValue=searchValue, term=term, data_points=data_points)
def get_data(self, request, component, metric): session = self.__SessionMaker() # See if it is in the database row = session.query(Data).filter(Data.component == component).filter(Data.metric == metric).all() if len(row) > 0: model = row[0] else: request.setResponseCode(400) request.redirect("/") return "" ret, should_save = model.get_data() if should_save: session.merge(model) session.commit() return str(ret)
def page_info(request): data = [] res = [] page = get_data(request.GET['page']) if(page == None): return HttpResponse(json.dumps(''), mimetype="application/json") print page if(page['product'].count > 0): for obj in page['product']: if(obj != ''): res += [{ 'image': obj[0], 'url': obj[1], 'float': obj[2] }] desc = [] for obj in page['desc']: if(obj != ''): desc += [{ 'desc': obj[0] }] data = {"product": res, 'desc': desc} return HttpResponse(json.dumps(data), mimetype="application/json")
def test_model_performance(): if not os.path.exists(constants.MODEL_FILE): warnings.warn( f"{constants.MODEL_FILE} not found, skipping test_model_performance()" ) return if not os.path.exists(constants.TOKENIZER_PICKLE): warnings.warn( f"{constants.TOKENIZER_PICKLE} not found, skipping test_model_performance()" ) return m = model.get_model() tokenizer = model.get_tokenizer() texts, labels = model.get_data() X = tokenizer.texts_to_sequences(texts) X = tf.keras.preprocessing.sequence.pad_sequences(X, maxlen=16) y = tf.keras.utils.to_categorical(labels) accuracy = m.evaluate(X, y)[1] assert accuracy > .9
activityid = 17972 username = "******" if __name__ == '__main__': # Event data tables = model.get_metadata(activityid) event_details = model.process_event(tables[0]) pprint(event_details) # Pairs standings = [] if len(tables) > 1: for table in tables[1:]: standings += model.process_standings(table) # pprint(standings) # Board data data = model.get_data(activityid, username) data_tables = model.get_data_table(data) tables = model.extract_data(data_tables) # pprint(f'{len(tables)} tables') # 5 # tables[0] contains info about the data # tables[1] is the first header 'Overzicht Resultaten' # tables[2] is the results table # tables[3] is the second header 'Details per Spel' # tables[4] is the details table # 2020-10-03: Decided against using the results table # Reason: gathering the same data from the details will # have benefit of adding the lead. # results = model.process_results(tables[2]) # pprint(results) boards = model.process_details(tables[4], username)
import numpy import model avg = numpy.mean(model.get_data()[:], axis=0) model.draw_grid(avg / 255.).save('avg.png')
sort_ascending=self.ascend, year=2016) self.chart.set_values(data) pygame.init() screen = pygame.display.set_mode((1200, 800)) pygame.display.set_caption("Election Data Viewer") pygame.display.update() screen_rect = screen.get_rect() bc = BarChart(rect=screen.get_rect(), values=model.get_data(party='dem', raw=True, sort_ascending=True, year=2016), ticks=4) button1 = DataChangeButton("dem", pygame.Rect(1000, 100, 150, 40), bc) button2 = DataChangeButton("gop", pygame.Rect(1000, 140, 150, 40), bc) button3 = DataChangeButton("up", pygame.Rect(1000, 300, 150, 40), bc) button4 = DataChangeButton("down", pygame.Rect(1000, 340, 150, 40), bc) button5 = DataChangeButton("raw", pygame.Rect(1000, 500, 150, 40), bc) button6 = DataChangeButton("%", pygame.Rect(1000, 540, 150, 40), bc) allbuttons = [button1, button2, button3, button4, button5, button6] # display loop done = False while not done:
screen = pygame.display.set_mode((1200, 800)) pygame.display.set_caption("Election Data Viewer") pygame.display.update() BLACK = (0, 0, 0) WHITE = (255, 255, 255) GRAY = (127, 127, 127) RED = (255, 0, 0) GREEN = (0, 255, 0) BLUE = (0, 0, 255) YELLOW = (255, 255, 0) MAGENTA = (255, 0, 255) CYAN = (0, 255, 255) data = model.get_data() bc1 = view.BarChart(rect=pygame.Rect(0, 50, 950, 700), ticks=5, max_val=10000000, plot_area_width_ratio=0.95, values=data) party = "dem" max_val = 10000000 raw = True sort_ascending = True class OneButton(Button): def __init__(self, text, rect, chart, demand):
import random import numpy import model data = model.get_data() n = data.shape[0] alphabet = numpy.zeros((62, 64, 64)) for z in xrange(62): alphabet[z] = random.choice(data)[z] * 1.0/255 model.draw_grid(alphabet).save('alphabet.png')
import os import sys import model import utils import transfer import numpy as np if __name__ == '__main__': if sys.argv[2] == 'dry': os.chdir(os.path.dirname(sys.argv[1])) train = utils.load_data_to_memory(os.path.join(sys.argv[1], 'train')) test = utils.load_data_to_memory(os.path.join(sys.argv[1], 'test')) train_x = model.get_data(train, 224) train_y = model.get_labels(train) test_x = model.get_data(test, 224) test_y = model.get_labels(test) model_ = model.get_MobileNet_v2((224, 224, 1)) train_x = np.expand_dims(train_x, -1) test_x = np.expand_dims(test_x, -1) model_ = model.build(train_x, train_y, test_x, test_y, model_) elif sys.argv[2] == 'transfer': fer_dataset = '~/Downloads/fer2013/fer2013.csv' os.chdir(os.path.dirname(sys.argv[1])) train = utils.load_data_to_memory(os.path.join(sys.argv[1], 'train')) test = utils.load_data_to_memory(os.path.join(sys.argv[1], 'test')) train_x = model.get_data(train, 48) train_y = model.get_labels(train) test_x = model.get_data(test, 48)
import tensorflow as tf from tensorflow import keras import numpy as np import model import utils import cv2 test_dir = '/Users/ezra/Downloads/Face_data_split/test' test = utils.load_data_to_memory(test_dir) test_x = model.get_data(test, 48) test_y = model.get_labels(test) model_ = keras.models.load_model('model.h5') test_x = np.expand_dims(test_x, -1) model_.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy']) # running evaluation of the test set model_.evaluate(test_x, test_y) # producing the test set images test_x = tf.cast(test_x, tf.float32) predictions = model_.predict(test_x) key_ = dict() key_['0'] = 'neutral' key_['1'] = 'anger' key_['2'] = 'surprise' key_['3'] = 'smile' key_['4'] = 'sad'
import numpy as np import model as md x_input, y_input, n, k, wh = md.get_data() model = md.Model(n, k, wh) print(n, k, wh) #model.try_load() x_train, y_train, x_test, y_test = md.load_data(x_input, y_input) for lr in [0.1, 0.01, 0.001]: model.train(x_train, y_train, x_test, y_test, batch_size=100, learning_late=lr) model.model_test(x_test, y_test)
import pygame import model #imports model.py file in this directory!! # all import statements are importing a file or folder that # exists somewhere !!! That this program has access to! import view pygame.init() surface = pygame.display.set_mode((800,600)) font = pygame.font.Font(None, 20) pygame.display.set_caption("Scatterplot") data = model.get_data('data.csv') scatter = view.ScatterPlot(data, surface, font) loop = True while loop: for event in pygame.event.get(): if event.type == pygame.QUIT: loop = False scatter.draw() pygame.display.update() pygame.quit() quit()
done = False while not done: screen.fill(view.BLACK) for event in pygame.event.get(): if event.type == pygame.QUIT: done = True else: button.handle_event(event) if button.chosen == True: b1c = view.RED b2c = view.GRAY party = "dem" data = model.get_data(party=party, raw=raw, sort_ascending=sort_ascending) bc = view.BarChart(pygame.Rect(50, 50, 1000, 600), values=data, ticks=5, max_val=mv) button_gop.chosen = False #print("DEM Selected") button_gop.handle_event(event) if button_gop.chosen == True: b2c = view.RED b1c = view.GRAY party = "gop" data = model.get_data(party=party, raw=raw,
def trends(): return render_template('trends.html', num_fake=model.get_data("FAKE"), num_real=model.get_data("REAL"), num_clickbait=model.get_data("CLICKBAIT"), num_notclickbait=model.get_data("NOT CLICKBAIT"))
from redis import Redis import time, sys from model import get_data, predict, to_target_names import numpy as np # settings wait_seconds = 3 # Tell RQ what Redis connection to use redis_conn = Redis() q = Queue(connection=redis_conn) # no args implies the default queue # Take the input from the user or the test dataset. if len(sys.argv) == 5: X_predict = [[float(i) for i in sys.argv[1:]]] else: _, X_test, _, y_test = get_data() X_predict = X_test[:3] # submit the job job = q.enqueue(predict, X_predict) print('Predicting on:') print(X_predict) print('Checking the job immediately we get:') # check the result before they are complete print(to_target_names(job.result)) # => None # Now, wait a while, until the worker is finished and check again print(f"After waiting for {wait_seconds} seconds we get:") time.sleep(wait_seconds) print(to_target_names(job.result)) # => [0, 1, 2] or something