def part_b(theta): # Use vectorization to remove this and the corresponding map? def find_x2(x, t=theta): """ Find x2 from equation of the line. > theta @ X = 0 > t0 + t1 x1 + t2 x2 = 0 > t2 x2 = -t0 -t1 x1 > x2 = (-t0 -t1 x1) / t2 """ return (-t[0] - t[1] * x[1]) / t[2] colors = ["r" if cls else "b" for cls in y] yy = list(map(find_x2, X)) plt.scatter(X[:, 1], X[:, 2], c=colors) line, = plt.plot(X[:, 1], yy, 'g', label="Decision Boundary") cls0 = mpatches.Patch(color='blue', label='Class 0') cls1 = mpatches.Patch(color='red', label='Class 1') plt.xlabel(r'Feature 0 ($X_0)$') plt.ylabel(r'Feature 1 ($X_1)$') plt.legend(handles=[cls0, cls1, line]) save(plt, "q3_b.png") plt.show()
def shop_list(url, page): """商家列表""" # proxies = xdaili_proxy() # session = requests.Session() # response = json.loads(session.get(url, headers=HEADERS, proxies=proxies, timeout=TIMEOUT).text) # response = json.loads(requests.get(url, headers=HEADERS, timeout=TIMEOUT).text) request_payload = REQUEST_PAYLOAD.copy() request_payload["offset"] = page <= 0 ? 0 : (page - 1) * request_payload["limit"] response = requests.post(url, data=json.dumps(request_payload), headers=HEADERS, cookies=COOKIES) list_data = None for i in range(3): try: list_data = json.loads(response.text) break except Exception as e: logging.warning(e) time.sleep(random.randint(1,3)) if list_data is None: logging.warning("can not parse data") return try: print(list_data) infos = parse_json(list_data) for info in infos["data"]: data = parse_json(info) print(data, sep='\n') try: save(data) except Exception as err: logging.warning("data save fail: {}".format(e)) return infos["total"] except Exception as e: logging.warning(" Response status code: {}, Requests was found, no target data was obtained!".format(response['code']))
def part_c(phi, mu, sigma): """Plot the linear decision boundary.""" sigma_i = LA.inv(sigma) # Parameters of the line equation: AX - B = 0 A = 2 * (mu[0] - mu[1]).T @ sigma_i B = ((mu[0].T @ sigma_i @ mu[0]) - (mu[1].T @ sigma_i @ mu[1]) - 2 * np.log((1 / phi) - 1)) # Plot data points X0, X1 = X[:, 0], X[:, 1] plt.scatter(X0, X1, c=colors) # Plot line X1 = (B - A[0] * X0) / (A[1]) line, = plt.plot(X0, X1, "g", label="Decision Boundary") cls0 = mpatches.Patch(color='blue', label="Alaska") cls1 = mpatches.Patch(color='red', label="Canada") plt.xlabel(r'Feature 0 ($X_0)$') plt.ylabel(r'Feature 1 ($X_1)$') plt.legend(handles=[cls0, cls1, line]) save(plt, "q4_c.png") plt.show() return (X0, X1)
def part_d(Jt=None, show=False): T0, T1 = np.mgrid[0:2:50j, -1:1:50j] mesh = np.c_[T0.flatten(), T1.flatten()] J_values = ( np.array([J(point) for point in mesh]) .reshape(T0.shape) ) plt.ion() plt.contour(T0, T1, J_values, 25, colors="k") plt.xlabel(r'$\theta_0$', labelpad=10) plt.ylabel(r'$\theta_1$', labelpad=10) if show: plt.show() if Jt is not None: # To draw line with points # plt.plot(Jt[:, 0], Jt[:, 1], linestyle='-', # color='r', marker='o', markersize=2.5) for jt in Jt: plt.plot([jt[0]], [jt[1]], linestyle='-', color='r', marker='o', markersize=2.5) if show: plt.pause(0.02) if show: save(plt, "q1_d_0.001.png") plt.close() return plt
def part_c(): for tau in [0.1, 0.3, 2, 10]: print("Tau: ", tau) plt = weighted_regression(tau) save(plt, "q2_b_%.1f.png" % tau) plt.close() plt.show()
def main(args): if len(args) < 2: Log.w("Arguments Error") return path = args[0] iteration = int(args[1]) Log.i("Training file: %s" % path) Log.i("----------------") sentences = c.readconllfile(path) featdict, weights = train(sentences, iteration) c.save(featdict, weights)
def part_e(): for eta in [0.001, 0.005, 0.009, 0.013, 0.017, 0.021, 0.025]: print("\n --- \n Eta: %.2f \n --- \n" % eta) theta, J_trace = part_a(eta) plt = part_d(J_trace) plt.title(r"$Contours (\eta=$" + str(eta) + ")") save(plt, "q1_e_%.3f.png" % eta) plt.show() plt.close()
def part_e(phi, mu, sigma, line): """Decision Boundar for GDA with different covariance matrices.""" # Inverse and Determinant of sigma_0/1 sigma_i = [LA.inv(sig) for sig in sigma] sigma_d = [LA.det(sig) for sig in sigma] # Parameters for equation of the decision boundary: X'AX + BX + C = 0 # Compare these with the straight line equations A = (sigma_i[0] - sigma_i[1]) B = -2 * ((mu[0].T @ sigma_i[0]) - (mu[1].T @ sigma_i[1])) C = ((mu[0].T @ sigma_i[0] @ mu[0]) - (mu[1].T @ sigma_i[1] @ mu[1]) - 2 * np.log(((1 / phi) - 1) * (sigma_d[1] / sigma_d[0]))) # Mesh of points p, q = np.mgrid[-2.5:2.5:50j, -3:3:50j] M = np.c_[p.flatten(), q.flatten()] def bdry(x): return x.T @ A @ x + B @ x + C # The quadratic decision boundary quad = np.array([bdry(m) for m in M]).reshape(p.shape) # Plot data points X0, X1 = X[:, 0], X[:, 1] plt.scatter(X0, X1, c=colors) plt.contour(p, q, quad, [0], colors="y") line, = plt.plot(line[0], line[1], color="g", label="Linear Decision Boundary") # If line were calculated via vectorized method! # plt.contour(p, q, line, [0], colors="g") cls0 = mpatches.Patch(color='blue', label="Alaska") cls1 = mpatches.Patch(color='red', label="Canada") qdb = mlines.Line2D(color='yellow', label="Quadratic Decision Boundary", xdata=[], ydata=[]) plt.xlabel(r'Feature 0 ($X_0)$') plt.ylabel(r'Feature 1 ($X_1)$') plt.legend(handles=[cls0, cls1, qdb, line]) save(plt, "q4_d.png") plt.show()
def part_b(): """Plot data with proper colors.""" plt.scatter(X[:, 0], X[:, 1], c=colors) cls0 = mpatches.Patch(color='blue', label="Alaska") cls1 = mpatches.Patch(color='red', label="Canada") plt.xlabel(r'Feature 0 ($X_0)$') plt.ylabel(r'Feature 1 ($X_1)$') plt.legend(handles=[cls0, cls1]) save(plt, "q4_b.png") plt.show()
def part_b(theta): yy = list(map(lambda x: theta @ x, X)) data, = plt.plot(X[:, 1], y, 'rx', label="Data") line, = plt.plot(X[:, 1], yy, 'b', label="Hypothesis") plt.xlabel("Acidity of wine (normalised)") plt.ylabel("Density of wine") plt.legend(handles=[data, line]) save(plt, "q1_b.png") plt.show()
def part_a(): theta = LA.inv(X.T @ X) @ X.T @ y print("Parameters:", theta) yy = list(map(lambda x: theta @ x, X)) data, = plt.plot(X[:, 1], y, 'rx', label="Data") line, = plt.plot(X[:, 1], yy, 'b', label="Hypothesis (unweighted)") plt.xlabel("X") plt.ylabel("Y") plt.legend(handles=[data, line]) save(plt, "q2_a.png") plt.show()
def main(base_url, page): """主函数""" # 添加_token参数 GET_PARAM["_token"] = encrypt_token() GET_PARAM['page'] = str(page) url = base_url + urlencode(GET_PARAM) # proxies = xdaili_proxy() # session = requests.Session() # response = json.loads(session.get(url, headers=HEADERS, proxies=proxies, timeout=TIMEOUT).text) response = json.loads(requests.get(url, headers=HEADERS, timeout=TIMEOUT).text) try: infos = response['data']['poiInfos'] for info in infos: data = parse_json(info) print(data, sep='\n') save(data) except Exception as e: logging.warning(" Response status code: {}, Requests was found, no target data was obtained!".format(response['code'])) _ = e
def main(base_url, page): """主函数""" # 添加_token参数 GET_PARAM["_token"] = encrypt_token() GET_PARAM['page'] = str(page) url = base_url + urlencode(GET_PARAM) # proxies = xdaili_proxy() # session = requests.Session() # response = json.loads(session.get(url, headers=HEADERS, proxies=proxies, timeout=TIMEOUT).text) response = json.loads(requests.get(url, headers=HEADERS, timeout=TIMEOUT).text) try: infos = response['data']['poiInfos'] for info in infos: data = parse_json(info) print(data, sep='\n') save(data) except Exception as e: a=0 print(response)
def main(output_name, mode="points_div_opportunities", pick_rand_topn=1): for ds_name in dsets: if ds_name in output_name: break assert ds_name in output_name rest_name = osp.basename(output_name)[len(ds_name) + 1:-len(".out")] data = load(ds_name) prev_output = load_output(output_name) output = construct_empty_output(prev_output, data) lib_id_to_output_id = dict( zip([d["index"] for d in output], range(len(output)))) if mode == "points_div_opportunities": book_score_func = books_by_points_div_opportunities else: assert mode == "opportunities" book_score_func = books_by_opportunities est_score = 0 # books_left_data = init_books_left_data( # output, scores=data["S"], lib_data=data["libs"] # ) while True: books_left_data, nbr_empty_slots = get_books_left_data( output, data["S"], data["libs"]) print(nbr_empty_slots, len(books_left_data)) if books_left_data is None or len(books_left_data) == 0: break sel_books = sorted(books_left_data, key=book_score_func)[-pick_rand_topn:] np.random.shuffle(sel_books) sel_book = sel_books[0] est_score += data["S"][sel_book["id"]] output[lib_id_to_output_id[sel_book["put_lib_idx"]]]["ids"].append( sel_book["id"]) print(est_score) save( output, method_name="greedy_best_books_%s_pick_top_%d_from_%s" % (mode, pick_rand_topn, rest_name), ds_name=ds_name, ) return est_score
def main(base_url, page,cateId,originUrl): """主函数""" # 添加_token参数 GET_PARAM['cateId'] = str(cateId) GET_PARAM["originUrl"]=originUrl SIGN_PARAM = "areaId={}&cateId={}&cityName={}&dinnerCountAttrId={}&optimusCode={}&originUrl={}&page={}&partner={}&platform={}&riskLevel={}&sort={}&userId={}&uuid={}".format( GET_PARAM["areaId"], GET_PARAM["cateId"], GET_PARAM["cityName"], GET_PARAM["dinnerCountAttrId"], GET_PARAM["optimusCode"], GET_PARAM["originUrl"], GET_PARAM["page"], GET_PARAM["partner"], GET_PARAM["platform"], GET_PARAM["riskLevel"], GET_PARAM["sort"], GET_PARAM["userId"], GET_PARAM["uuid"] ) GET_PARAM["_token"] = encrypt_token(SIGN_PARAM) GET_PARAM['page'] = str(page) url = base_url + urlencode(GET_PARAM) # proxies = xdaili_proxy() # session = requests.Session() # response = json.loads(session.get(url, headers=HEADERS, proxies=proxies, timeout=TIMEOUT).text) response = json.loads(requests.get(url, headers=HEADERS, timeout=TIMEOUT).text) try: infos = response['data']['poiInfos'] for info in infos: data = parse_json(info) data['city'] = base_url.split('//')[-1].split('.')[0] data['cateId'] = GET_PARAM['cateId'] print(data, sep='\n') save(data) except Exception as e: logging.warning(" Response status code: {}, Requests was found, no target data was obtained!".format(response['code'])) _ = e
def part_c(Jt=None): # Create a mesh using numpy awesome-sauce! T0, T1 = np.mgrid[0:2:50j, -1:1:50j] mesh = np.c_[T0.flatten(), T1.flatten()] # Compute J_values for the grid J_values = ( np.array([J(point) for point in mesh]) .reshape(50, 50) ) plt.ion() fig = plt.figure() ax = fig.add_subplot(111, projection='3d') ax.plot_surface(T0, T1, J_values, cmap=cm.RdBu_r) ax.set_xlabel(r'$\theta_0$', labelpad=10) ax.set_ylabel(r'$\theta_1$', labelpad=10) ax.set_zlabel(r'$J(\theta)$', labelpad=10) plt.show() if Jt is not None: # To draw line between points # ax.plot(Jt[:, 0], Jt[:, 1], Jt[:, 2], linestyle='-', # color='r', marker='o', markersize=2.5) for jt in Jt: ax.plot([jt[0]], [jt[1]], [jt[2]], linestyle='-', color='r', marker='o', markersize=2.5) plt.pause(0.2) save(plt, "q1_a.png")
def generate(weeks_, data_, path_=None): """ Generate and optionally save the image :param int weeks_: Number of weeks to paint :param list data_: List of tuples (day, amount) :param str path_: Image file path :return: Either the data or None :rtype: bytes or None """ # Process arguments weeks = min(max(weeks_, 1), 52) data = {} weekly_values = [[], [], [], [], [], [], []] today = (datetime.date.today().toordinal() - 1) // 7 * 7 starting_day = today - (weeks - 1) * 7 for date, _amount in data_: if date < starting_day: continue amount = _amount / 10 if _amount % 10 else _amount // 10 data[date] = amount weekly_values[date % 7].append(amount) non_empty_weeks = set(i // 7 for i in data) total_average = sum(data.values()) / len(data) try: color_step = 255 / max(abs(i - total_average) for i in data.values()) except ZeroDivisionError: color_step = 0 # Create image image = common.pg_surface(SIZE) image.fill(BACKGROUND_COLOR) HEIGHT = ((SIZE[1] - HEADER_HEIGHT - PREFOOTER_HEIGHT - FOOTER_HEIGHT) // len(non_empty_weeks)) WEEK_HEADER_HEIGHT = HEIGHT // 4 HEIGHT -= WEEK_HEADER_HEIGHT # Paint header header_font = common.fit_font(common.FONT_MONO, "0", (WIDTH // 4, HEADER_HEIGHT)) for wDay in range(7): x = wDay * WIDTH width = WIDTH if wDay == 6: width += EXTRA_WIDTH color = common.GRAY[215 if wDay & 1 else 225] common.blit_text(image, header_font, (x, 0), common.WEEK_DAYS[wDay], common.WHITE, color, size=(width, HEADER_HEIGHT), anchor="") # Blit every day non_empty_week = -1 iterator = iter(range(starting_day, starting_day + weeks * 7)) for day in iterator: if not day % 7: # Check for empty weeks if day // 7 in non_empty_weeks: non_empty_week += 1 else: for _ in range(6): next(iterator) continue width = WIDTH height = HEIGHT x = WIDTH * (day % 7) y = (HEIGHT + WEEK_HEADER_HEIGHT) * non_empty_week + HEADER_HEIGHT if not x: # Print week header image.fill(BACKGROUND_COLOR, ((x, y), (SIZE[0], WEEK_HEADER_HEIGHT))) monday = datetime.date.fromordinal(day + 1).strftime("%d/%m") sunday = datetime.date.fromordinal(day + 7).strftime("%d/%m") text = "%s - %s" % (monday, sunday) font = common.fit_font(common.FONT1, text, (SIZE[0], int(WEEK_HEADER_HEIGHT * 0.9))) common.blit_text(image, font, (x, y), text, common.GRAY[100], BACKGROUND_COLOR, size=(SIZE[0], WEEK_HEADER_HEIGHT), anchor="") y += WEEK_HEADER_HEIGHT if day % 7 == 6: width += EXTRA_WIDTH try: value = data[day] except KeyError: continue color = get_color(value, total_average, color_step) if type(value) is int: text = "%s %d" % ("▲" if value > total_average else "▼", value) else: text = "%s %.2f" % ("▲" if value > total_average else "▼", value) size = (width - 4, height - 4) text_size = (int(width * 0.8), int(height * 0.8)) pos = (x + 2, y + 2) font = common.fit_font(common.FONT_MONO, text, text_size) image.fill(color, (pos, size)) common.blit_text(image, font, pos, text, common.BLACK, color, size=size, anchor="") y = SIZE[1] - PREFOOTER_HEIGHT - FOOTER_HEIGHT # Paint prefooter prefooter_font = common.fit_font(common.FONT1, "Average:0.00", (SIZE[0], PREFOOTER_HEIGHT)) color = get_color(total_average, total_average, color_step) common.blit_text(image, prefooter_font, (0, y), "Average: %.2f" % total_average, common.BLACK, color, size=(SIZE[0], PREFOOTER_HEIGHT), anchor="") y += PREFOOTER_HEIGHT # Paint footer footer_font = common.fit_font(common.FONT1, "▲ 00.00", (WIDTH, FOOTER_HEIGHT)) for wDay, i in enumerate(weekly_values): try: average = sum(i) / len(i) except ZeroDivisionError: continue x = wDay * WIDTH width = WIDTH if wDay == 6: width += EXTRA_WIDTH color = get_color(average, total_average, color_step) text = "%s %.2f" % ("▲" if average > total_average else "▼", average) common.blit_text(image, footer_font, (x, y), text, common.BLACK, color, size=(width, FOOTER_HEIGHT), anchor="") return common.save(image, path=path_)
driver = get_driver() driver.get('https://nanoreview.net/ru/phone-list/endurance-rating') items = driver.find_elements(By.CSS_SELECTOR, '.table-list > tbody > tr') data = [] for item in items: name = item.find_element(By.CSS_SELECTOR, 'td:nth-child(2)').text rating = item.find_element(By.CSS_SELECTOR, 'td:nth-child(3)').text wifi_surfing = item.find_element(By.CSS_SELECTOR, 'td:nth-child(4)').text video_watch = item.find_element(By.CSS_SELECTOR, 'td:nth-child(5)').text talk_3g = item.find_element(By.CSS_SELECTOR, 'td:nth-child(6)').text battery = item.find_element(By.CSS_SELECTOR, 'td:nth-child(7)').text data.append({ 'name': name, 'rating': rating, 'wifi_surfing_time': wifi_surfing, 'video_watch_time': video_watch, '3g_talk': talk_3g, 'battery': battery }) driver.close() driver.quit() save(data, 'battery.csv')
# train epoch = 0 error = 10 error_ar = [] start = time() * 1000 while epoch < 100000 and error > acceptable_error: error_ar.clear() for j in range(len(input_list)): r = numpy.random.randint(0, len(input_list)) network.train(input_list[r], target_list[r]) error_ar.append(mean_square_error(network.query, input_list, target_list)) error = numpy.sum(error_ar) / len(error_ar) if epoch % 100 == 0: print(str(epoch) + '\t\terror = ' + str(error)) epoch += 1 stop = time() * 1000 print('\n' + str(epoch) + '\t\terror = ' + str(error)) # save save(network, path + '.ser') # save(network.w_ih.tolist(), 'w_ih.ser') # save(network.w_ho.tolist(), 'w_ho.ser') # save(network.b_ih.tolist(), 'b_ih.ser') # save(network.b_ho.tolist(), 'b_ho.ser') with open(path + '.txt', 'w', encoding='utf-8') as f: f.write(str(error) + '\n') f.write(str(epoch) + '\n') f.write(str(format(stop - start, '.3f')))
def train(trace_length=10, render_eval=False, h_size=512, target_update_freq=10000, ckpt_freq=500000, summary_freq=1000, eval_freq=10000, discovery=False, batch_size=32, env_name='Pong', total_iteration=5e7, use_actions=0, pretrain_steps=50000, num_quant=0): # network = dist_Qnetwork if num_quant else Qnetwork # env_name += 'NoFrameskip-v4' absolute_start_time = time.time() model = 'drqn' if not use_actions else 'adrqn' if num_quant: model = 'dist-' + model KICKSTART_EXP_BUF_FILE = 'cache/exp_buf_random_policy_{}_{}_{}.p'.format( model, env_name, pretrain_steps) ExpBuf = FixedTraceBuf if not use_actions else FixedActionTraceBuf model_args = {} identity = 'stack={},env={},mod={},h_size={}'.format( trace_length, env_name, model, h_size) if num_quant: identity += ',quantile={}'.format(num_quant) if use_actions: identity += ',action_dim={}'.format(use_actions) model_args['action_hidden_size'] = use_actions print(identity) env = Env(env_name=env_name, skip=4) mainQN = Qnetwork(h_size, env.n_actions, 1, 'main', train_batch_size=batch_size, model=model, train_trace_length=trace_length, model_kwargs=model_args, num_quant=num_quant) saver = tf.train.Saver(max_to_keep=5) summary_writer = tf.summary.FileWriter('./log/' + identity, mainQN.sess.graph) if util.checkpoint_exists(identity): (exp_buf, env, last_iteration, is_done, prev_life_count, action, mainQN.hidden_state, S) = util.load_checkpoint(mainQN.sess, saver, identity) start_time = time.time() else: exp_buf = FixedTraceBuf(trace_length, buf_length=500000) last_iteration = 1 - pretrain_steps if os.path.isfile(KICKSTART_EXP_BUF_FILE): print('Filling buffer with random episodes on disk.') exp_buf, last_iteration = util.load(KICKSTART_EXP_BUF_FILE), 1 is_done = True prev_life_count = None mainQN.update_target_network() summaryOps = tf.summary.merge_all() eval_summary_ph = tf.placeholder(tf.float32, shape=(4, ), name='evaluation') evalOps = (tf.summary.scalar('performance', eval_summary_ph[0]), tf.summary.scalar('perform_std', eval_summary_ph[1]), tf.summary.scalar('flicker_performance', eval_summary_ph[2]), tf.summary.scalar('flicker_perform_std', eval_summary_ph[3])) online_summary_ph = tf.placeholder(tf.float32, shape=(2, ), name='online') onlineOps = (tf.summary.scalar('online_performance', online_summary_ph[0]), tf.summary.scalar('online_scenario_length', online_summary_ph[1])) for i in range(last_iteration, int(total_iteration)): if is_done: scen_R, scen_L = exp_buf.flush_scenario() if i > 0: online_perf_and_length = np.array([scen_R, scen_L]) online_perf, online_episode_count = mainQN.sess.run( onlineOps, feed_dict={online_summary_ph: online_perf_and_length}) summary_writer.add_summary(online_perf, i) summary_writer.add_summary(online_episode_count, i) S, r, prev_life_count = env.reset() S = np.reshape(S, (1, 84, 84)) mainQN.reset_hidden_state() action, _ = mainQN.get_action_stateful(S, prev_a=0) S_new, r, is_done, life_count = env.step(action, epsilon=util.epsilon_at(i)) S_new = np.reshape(S_new, (1, 84, 84)) exp_buf.append_trans(( S, action, r, S_new, # not cliping reward (huber loss) (prev_life_count and life_count < prev_life_count or is_done))) S = S_new prev_life_count = life_count if not i: util.save(exp_buf, KICKSTART_EXP_BUF_FILE) if discovery and time.time( ) - absolute_start_time > 85500: # 23 hours and 45 minutes util.Exiting = 1 if util.Exiting or not i % ckpt_freq: util.checkpoint(mainQN.sess, saver, identity, exp_buf, env, i, is_done, prev_life_count, action, mainQN.hidden_state, S) if util.Exiting: raise SystemExit if i < 0: continue # TRAIN _, summary = mainQN.update_model_stateful( *exp_buf.sample_traces(batch_size), addtional_ops=[summaryOps]) # Summary if not i % summary_freq: summary_writer.add_summary(summary, i) # Target Update if not i % target_update_freq: mainQN.update_target_network() cur_time = time.time() print(i, identity) try: print('[{}:{}:{}K] took {} seconds to {} steps'.format( model, env_name, util.unit_convert(i), int(cur_time - start_time), target_update_freq), flush=1) except: pass start_time = cur_time # Evaluate if not i % eval_freq: eval_res = np.array( evaluate(mainQN, env_name, is_render=render_eval)) eval_vals = mainQN.sess.run(evalOps, feed_dict={eval_summary_ph: eval_res}) for v in eval_vals: summary_writer.add_summary(v, i) util.checkpoint(mainQN.sess, saver, identity)
price = driver.find_elements(By.CLASS_NAME, 'product-buy__price') except Exception as e: pass price = price[0].text if price else None info['price'] = price info['url'] = page_url # print(info, end='\n\n') return info infos = [] bar = IncrementalBar('Parsing', max = len(urls)) i = 1 for url in urls: infos.append(load_info(url)) bar.next() i+=1 if i % 100 == 0: save(infos, 'smartphones.csv') driver.close() driver.quit() save(infos, 'smartphones.csv')
def main( ds_name="a_example", tparam=1.0, mode="div_points", pick_rand_topn=1, threshold_idle_time=np.inf, idle_quantile=0.99, ): if ds_name in cache: if mode in cache[ds_name]: if tparam in cache[ds_name][mode]: return cache[ds_name][mode][tparam] data = load(ds_name) num_books_added_by_last_lib = 1 books_left = set(range(len(data["S"]))) unused_libs = list(data["libs"]) output = [] days_left = data["D"] scores = data["S"] est_score = 0 if mode == "div_points": def lib_score(lib): nbr_books, nbr_points, idle_time = compute_nbr_books_processable_nbr_points_makeable_idle_time( lib, days_left=days_left, books_to_scan_left=books_left, scores=scores, quantile=idle_quantile, ) return (idle_time < threshold_idle_time, nbr_points / lib["t"] ** tparam) elif mode == "sub_points": def lib_score(lib): nbr_books, nbr_points, idle_time = compute_nbr_books_processable_nbr_points_makeable_idle_time( lib, days_left=days_left, books_to_scan_left=books_left, scores=scores, quantile=idle_quantile, ) return (idle_time < threshold_idle_time, nbr_points - lib["t"] * tparam) elif mode == "div_books": def lib_score(lib): nbr_books, nbr_points, idle_time = compute_nbr_books_processable_nbr_points_makeable_idle_time( lib, days_left=days_left, books_to_scan_left=books_left, scores=scores, quantile=idle_quantile, ) return (idle_time < threshold_idle_time, nbr_books / lib["t"] ** tparam) else: assert mode == "sub_books" def lib_score(lib): nbr_books, nbr_points, idle_time = compute_nbr_books_processable_nbr_points_makeable_idle_time( lib, days_left=days_left, books_to_scan_left=books_left, scores=scores, quantile=idle_quantile, ) return (idle_time < threshold_idle_time, nbr_books - lib["t"] * tparam) while num_books_added_by_last_lib > 0 and len(unused_libs) > 0 and days_left >= 0: print("start next days left", days_left) sel_lib = sorted(unused_libs, key=lib_score)[-pick_rand_topn:] np.random.shuffle(sel_lib) sel_lib = sel_lib[0] est_score += compute_nbr_books_processable_nbr_points_makeable_idle_time( sel_lib, days_left=days_left, books_to_scan_left=books_left, scores=scores, quantile=idle_quantile, )[1] if days_left - sel_lib["t"] <= 0: break del unused_libs[unused_libs.index(sel_lib)] output.append( get_lib_to_output( sel_lib, days_left=days_left, books_to_scan_left=books_left, scores=scores, ) ) # print("sel_lib", sel_lib) days_left -= sel_lib["t"] # print("days_left", days_left) num_books_added_by_last_lib = len(output[-1]["ids"]) # print("num_books_added_by_last_lib", num_books_added_by_last_lib) print(est_score) save( output, method_name="greedy_best_%s_tparam_%015.7f_pick_top_%d_thresh_idle_%d_quantile_%.3f" % (mode, tparam, pick_rand_topn, int(threshold_idle_time), idle_quantile), ds_name=ds_name, ) if ds_name not in cache: cache[ds_name] = {} if mode not in cache[ds_name]: cache[ds_name][mode] = {} assert tparam not in cache[ds_name][mode] cache[ds_name][mode][tparam] = est_score return est_score
def part_b(): plt = weighted_regression(0.8) save(plt, "q2_b_0.8.png") plt.show()
'td:nth-child(3) div').text rating_mark = item.find_element(By.CSS_SELECTOR, 'td:nth-child(3) span').text antutu9 = item.find_element(By.CSS_SELECTOR, 'td:nth-child(4)').text.strip() geekbench5 = item.find_element(By.CSS_SELECTOR, 'td:nth-child(5)').text.strip() core_count = item.find_element(By.CSS_SELECTOR, 'td:nth-child(6) div').text freq = item.find_element(By.CSS_SELECTOR, 'td:nth-child(7)').text graphics = item.find_element(By.CSS_SELECTOR, 'td:nth-child(8) div').text data.append({ 'name': name, 'rating_score': rating_score, 'rating_mark': rating_mark, 'Antutu 9': antutu9, 'Geekbench 5': geekbench5, 'cores': core_count, 'frequency': freq, 'graphics_name': graphics, }) driver.close() driver.quit() save(data, 'processors.csv')
parser.add_argument('--wav', action='store_true') parser.add_argument('--cp', action='store_true') parser.add_argument('--long', action='store_true') parser.add_argument('--mir', type=int) args = parser.parse_args() alg = args.alg.upper() if args.mir is not None: args.input = "../base/MIR-1K/Wavfile/" + np.sort( os.listdir("../base/MIR-1K/Wavfile"))[args.mir - 1] rate, audio = load(args.input, True) if args.cp: cp.run('apply(args.alg, audio, rate)') else: start = time.time() voice, music = apply(args.alg, audio, rate) l = time.time() - start print('{:.1f}s za {:.1f}s'.format(l, len(audio) / rate)) out = args.output if out is None: out = '../outputs' Path(out).mkdir(parents=True, exist_ok=True) name = Path(args.input).stem ext = 'wav' if args.wav else 'mp3' save(voice, rate, "{}/{}-{}-voice.{}".format(out, name, args.alg, ext), not args.wav) save(music, rate, "{}/{}-{}-music.{}".format(out, name, args.alg, ext), not args.wav)
from common import get_driver, save driver = get_driver() driver.get('https://www.dxomark.com/smartphones/#sort-camera/device-') driver.find_element(By.CLASS_NAME, 'mfp-close').click() button = driver.find_element(By.CSS_SELECTOR, '.button.wide') for _ in range(0, 5): button.click() items = driver.find_elements(By.CSS_SELECTOR, '.row.device-row') data = [] for item in items: name = item.find_element(By.CLASS_NAME, 'deviceName').text camera_score = item.find_element(By.CSS_SELECTOR, '.deviceScore.selected').text data.append({'name': name, 'camera_score': camera_score}) driver.close() driver.quit() save(data, 'camera.csv')
def generate(iDay_, weeks_, birthdays_, periods_, smoothFactor_, path_=None): def paint_day(day): def smooth_color(color, isOdd): val = (255 - smoothFactor_) // 5 if isOdd: return [smoothFactor_ + i * val // 51 for i in color] else: return [smoothFactor_ + 5 + i * (val - 1) // 51 for i in color] # Figure postion, size and color date = datetime.date.fromordinal(day + 1) dayNumber = date.day birthdayDate = dayNumber + date.month * 100 week = (day - iDay) // 7 x = day % 7 * WIDTH y = HEADER_HEIGHT + week * HEIGHT + (dayNumber < 8) width = WIDTH + (EXTRA_WIDTH if day % 7 == 6 else 0) height = HEIGHT - (dayNumber < 8) - 1 if week == weeks - 1: height += EXTRA_HEIGHT + 1 _color, periodName = periods.pop(day, (common.WHITE, None)) color = smooth_color(_color, day % 7 & 1) # Blit day number if dayNumber == 1: dayText = "%d %s" % (dayNumber, date.strftime("%b")) if day % 7: x += 1 width -= 1 dayFont = common.fit_font(common.FONT_MONO, dayText, (width, height // 3)) else: dayText = str(dayNumber) dayFont = common.fit_font(common.FONT_MONO, dayText, (width // 4, height // 3)) daySize = dayFont.size(dayText) common.blit_text(image, dayFont, (x, y), dayText, common.BLACK, color, size=(width, height), anchor="NW") topY = y if dayNumber == 1: topY += daySize[1] bottomY = y + height # Blit birthdays try: names = enumerate(birthdays[birthdayDate].split("\n")) except KeyError: names = [] for a, name in names: offset = (daySize[0] if not a and dayNumber != 1 else 0) birthdayHeigth = min(bottomY - topY, height // 3) if birthdayHeigth >= 8: birthdayImageSide = min(int(birthdayHeigth * 0.8), width // 4) birthdayImageSize = (birthdayImageSide, birthdayImageSide) birthdayImage = common.pg_scale(GIFT_PNG, birthdayImageSize) image.blit(birthdayImage, (x + offset, topY)) offset += birthdayImageSide birthdayWidth = (width - offset) birthdaySize = (birthdayWidth, birthdayImageSide) birthdayFont = common.fit_font(BIRTHDAY_FONT_NAME, name, birthdaySize) common.blit_text(image, birthdayFont, (x + offset, topY), name, common.GRAY[100], color, size=birthdaySize, anchor="SW") topY += birthdayImageSide # Blit period if day in show: periodText = "{%s}" % periodName periodSize = (width, min(bottomY - topY, height // 3)) font = common.fit_font(TEXT_FONT_NAME, periodText, periodSize) common.blit_text(image, font, (x, bottomY - periodSize[1]), periodText, common.BLACK, color, size=periodSize, anchor="SW") # Process arguments weeks = min(max(weeks_, 1), 52) iDay = str2Date(iDay_) // 7 * 7 birthdays = dict(map(str2Birthday, birthdays_)) periods = {} for period in periods_: iDay_ = str2Date(period["iDay"]) fDay = str2Date(period.pop("fDay", period["iDay"])) name = period.pop("name", None) if name == "": name = None _color = period.pop("color", "F00") color = get_color(_color) weekend = get_color(period.pop("weekend", _color)) exceptions = set(map(str2Date, period.pop("exceptions", tuple()))) for day in range(iDay_, fDay + 1): if day in exceptions: continue if day % 7 > 4: periods[day] = (weekend, name) else: periods[day] = (color, name) show = set() last = [] for day, (_, name) in periods.items(): if name is not None and name not in last[-7:]: show.add(day) last.append(name) # Create image image = common.pg_surface(SIZE) image.fill(common.BLACK) HEIGHT, EXTRA_HEIGHT = divmod(SIZE[1] - HEADER_HEIGHT, weeks) for week in range(1, weeks): y = HEADER_HEIGHT + week * HEIGHT - 1 image.fill(common.GRAY[200], rect=((0, y), (SIZE[0], 1))) # Paint header headerFont = common.fit_font(common.FONT_MONO, "0", (WIDTH // 4, HEADER_HEIGHT)) for wDay in range(7): x = wDay * WIDTH width = WIDTH if wDay == 6: width += EXTRA_WIDTH color = common.GRAY[215 if wDay & 1 else 225] common.blit_text(image, headerFont, (x, 0), common.WEEK_DAYS[wDay], common.WHITE, color, size=(width, HEADER_HEIGHT), anchor="") # Blit every week for day in range(iDay, iDay + weeks * 7): paint_day(day) return common.save(image, path=path_)
for episode in episodes: state = State() while not state.terminal: player = state.player dealer = state.dealer epsilon = float(n_zero) / (n_zero + n_s[(dealer, player)]) action = epsilon_greedy_policy(action_value_function, state, epsilon) n_s[(dealer, player)] += 1 n_s_a[(dealer, player, action)] += 1 reward = step(state, action) # update the action value function alpha = 1.0 / n_s_a[(dealer, player, action)] new_reward = action_value_function[(dealer, player, action)] action_value_function[(dealer, player, action)] += alpha * (reward - new_reward) pbar.update(episode) pbar.finish() value_function = action_value_to_value_function(action_value_function) plot_value_function(value_function, "Optimal Value Function: Question 2") return action_value_function if __name__ == '__main__': mc_action_value_function = monte_carlo_control() save(mc_action_value_function, "mc_result.dat")