def make_impex_areas(tsid): """create ImpexArea instances""" con = DB.OracleConnection() ImpexArea.clear() for new_row in con.script_cursor(ias, tsid=DB.Partition(tsid)): ImpexArea(new_row)
def last(self): keys = sorted(list( map(lambda x: float(x), filter(lambda x: x[0].isdigit(), DB.keys()))), reverse=True) if keys: self.last_score = str(DB.store_get(str(keys[0])))
def make_dpgs(tsid): """create Dpg instances""" con = DB.OracleConnection() Dpg.clear() DpgSupply.clear() DpgImpex.clear() DpgDemand.clear() DpgDemandFSK.clear() DpgDemandSystem.clear() DpgDemandLoad.clear() for new_row in con.script_cursor(cs, tsid=DB.Partition(tsid)): if new_row.is_fsk: DpgDemandFSK(new_row) elif new_row.is_system: DpgDemandSystem(new_row) else: DpgDemandLoad(new_row) for new_row in con.script_cursor(dds, tsid=DB.Partition(tsid)): Dpg.by_id[new_row.dpg_id].add_disqualified_data(new_row) for new_row in con.script_cursor(gs, tsid=DB.Partition(tsid)): DpgSupply(new_row) for new_row in con.script_cursor(imp_s, tsid=DB.Partition(tsid)): DpgImpex(new_row)
def make_stations(tsid): """create Station instances""" con = DB.OracleConnection() Station.clear() for new_row in con.script_cursor(ss, tsid=DB.Partition(tsid)): Station(new_row)
def make_settings(tsid): """create Settings instances""" con = DB.OracleConnection() Setting.clear() for new_row in con.script_cursor(ss, tsid=DB.Partition(tsid)): Setting(new_row)
def get_most_popular_movies(self): if self.most_popular_movies == None: if self.landing_page == None: self.landing_page = self.get_landing_page() # Navigate to Menu and Click element_menu = self.driver.find_element_by_id( "imdbHeader-navDrawerOpen") ActionChains(self.driver).move_to_element(element_menu).click( element_menu).perform() # Navigate to Movie Category and Click element_movie = element_menu.find_element( By.XPATH, "//label[@for='nav-link-categories-mov']") ActionChains(self.driver).move_to_element(element_movie).click( element_movie).perform() # Navigate to Most Popuplar Movies and Click element_mpm = element_movie.find_element( By.XPATH, "//a[@href='/chart/moviemeter/?ref_=nv_mv_mpm']") ActionChains(self.driver).move_to_element(element_mpm).click( element_mpm).perform() WebDriverWait(self.driver, 10) html = self.driver.page_source self.most_popular_movies = IMDB.dfparse_table_movies(html) DB.save_search(self.most_popular_movies, suffix='mpm') return self.most_popular_movies
def web_hook(): """Telegram will send new messages to this web-hook""" # Translate POST request data to JSON format. j = request.json update = bot.json_to_update(j) if not update.message.text: return "invalid_message" # If the request is `start` command. if update.message.text.lower() in ["/start", "/start@sample_bot"]: chat_id = update.message.chat_id DB.add_client(chat_id) bot.send_message(update.message.chat_id, Configs.Messages.START_TEXT) # If the request is `help` command. if update.message.text.lower() in ["/help", "/help@sample_bot"]: bot.send_message(update.message.chat_id, Configs.Messages.HELP_TEXT) # If the request is `switchreg` command. # `switchreg` command will switch client registration. # Unregistered clients won't be notified of new rss entries anymore until they re-register. if update.message.text.lower() in ["/switchreg", "/switchreg@sample_bot"]: chat_id = update.message.chat_id DB.switch_registration(chat_id) bot.send_message(update.message.chat_id, Configs.Messages.SWITCH_REGISTRATION_TEXT) # The request was processed successfully. return "ok"
def make_wsumgen(tsid): """create Wsumgen instances""" con = DB.OracleConnection() Wsumgen.clear() for new_row in con.script_cursor(ws, tsid=DB.Partition(tsid)): Wsumgen(new_row)
def create_descriptors_pca(self, dim=90): ''' 计算描述子pca :param dim: :return: ''' print("start create_descriptors_pca ...") query = DB.DescriptorModel.select( DB.DescriptorModel.id, DB.DescriptorModel.descriptor).tuples().iterator() features = numpy.array(map(lambda x: [x[0]] + list(x[1]), query)) print("create_descriptors_pca,count=%d,dim=%d" % (len(features), dim)) start = time() print("build eigenvectors start time %s" % start) mean, eigenvectors = cv2.PCACompute(features[:, 1:], None, maxComponents=dim) fitted = cv2.PCAProject(features[:, 1:], mean, eigenvectors) #pca = PCA(n_components=dim) #fitted = pca.fit_transform(features[:,1:]) print("build eigenvectors cost time %s" % (time() - start)) print("saving data ...") #scaler = preprocessing.MinMaxScaler() #pca = scaler.fit_transform(pca) DB.db.connect() with DB.db.transaction(): DB.PcaModel.drop_table(fail_silently=True) DB.PcaModel.create_table() #res = DB.TrainingResult() #res.name = "daisy_pca" #res.data = pca #res.save() for i in range(0, len(fitted)): model = DB.PcaModel() model.pca = fitted[i] model.feature = features[i][0] model.save() DB.TrainingResult.delete().where( DB.TrainingResult.name == "pca_mean").execute() DB.TrainingResult.delete().where( DB.TrainingResult.name == "pca_eigenvectors").execute() tr = DB.TrainingResult() tr.name = "pca_mean" tr.data = mean tr.save() tr = DB.TrainingResult() tr.name = "pca_eigenvectors" tr.data = eigenvectors tr.save() print("create_descriptors_pca done")
def create_classifier(self): DB.db.connect() clf = SGDClassifier(loss="modified_huber") labs_map = NameToIndex() with DB.db.transaction(): offset = 0 words_count = self.get_words_count() classes = numpy.arange(0, words_count) x_all = [] y_all = [] while True: print ' %d partial_fit %d' % (time(), offset) query = DB.Vocabulary\ .select(DB.Vocabulary.lv1, DB.Vocabulary.lv2)\ .join(DB.PcaModel, on=(DB.Vocabulary.feature == DB.PcaModel.feature)).order_by( DB.Vocabulary.feature).offset(offset).limit(1000)\ .tuples().iterator() features = numpy.array( map(lambda x: [x[0]] + list(x[1]), query)) offset += len(features) if len(features) == 0: break Y = features[:, 0] X = features[:, 1:] labs = [] for lab in Y: labs.append(labs_map.map(lab)) if (len(x_all) < 10000): x_all = x_all + X.tolist() y_all = y_all + labs labs = numpy.array(labs) #clf = LinearSVC() #clf = OneVsRestClassifier(SVC(probability=True, kernel='linear')) #clf.fit(X,labs) clf.partial_fit(X, labs, classes) print clf.score(x_all, y_all) DB.TrainingResult.delete().where( DB.TrainingResult.name == self.__class__.__name__ + "_clf").execute() DB.TrainingResult.delete().where( DB.TrainingResult.name == self.__class__.__name__ + "_labs_map").execute() tr = DB.TrainingResult() tr.name = self.__class__.__name__ + "_clf" tr.data = clf tr.save() tr = DB.TrainingResult() tr.name = self.__class__.__name__ + "_labs_map" tr.data = labs_map tr.save()
def make_price_zones(tsid): """create PriceZone instances""" con = DB.OracleConnection() PriceZone.clear() for new_row in con.script_cursor(pzs, tsid=DB.Partition(tsid)): price_zone = PriceZone(new_row) for hour in range(HOURCOUNT): price_zone.add_price_zone_hour_data(hour)
def __init__(self): self._db = DB() self._tip = self._db.get('l') if not self._tip: block = Block().pow_of_block() self._db.put(block.hash, pickle.dumps(block)) self._db.put(Blockchain.latest, block.hash) self._tip = block.hash
def make_areas(tsid): """create Area instances""" con = DB.OracleConnection() Area.clear() for new_row in con.script_cursor(ra, tsid=DB.Partition(tsid)): area = Area[new_row.area] if not area: area = Area(new_row) area.add_area_hour_data(new_row)
def make_loads(tsid): """create Load instances""" con = DB.OracleConnection() Load.clear() for new_row in con.script_cursor(rl, tsid=DB.Partition(tsid)): load = Load[new_row.consumer_code] if not load: load = Load(new_row) load.add_load_hour_data(new_row)
def make_consumers(tsid): """create Consumer instances""" con = DB.OracleConnection() Consumer.clear() for new_row in con.script_cursor(rc, tsid=DB.Partition(tsid)): consumer = Consumer[new_row.consumer_code] if not consumer: consumer = Consumer(new_row) consumer.add_consumer_hour_data(new_row)
def make_nodes(tsid): """create Node instances""" con = DB.OracleConnection() Node.clear() for new_row in con.script_cursor(ns, tsid=DB.Partition(tsid)): node = Node[new_row.node_code] if not node: node = Node(new_row) node.add_node_hour_data(new_row)
def __init__(self): self.db = DB() if self.db.get('latest'): self.height = int(self.db.get('latest')) self.prev_block = pickle.loads(self.db.get(int(self.height))) else: new_block = Block(0, "Blockchain initialized!", "") self.prev_block = new_block.pow_on_block() self.db.put(0, pickle.dumps(self.prev_block)) self.db.put('latest', 0) self.height = 0
def make_peak_so(tsid, tdate): """create Settings instances""" con = DB.OracleConnection() PeakSO.clear() done = False for new_row in con.script_cursor(pss, tsid=DB.Partition(tsid), tdate=tdate): PeakSO(new_row) done = True if not done: for new_row in con.script_cursor(pbs): PeakSO(new_row)
def make_lines(tsid): """create Line instances""" con = DB.OracleConnection() Line.clear() for new_row in con.script_cursor(ls, tsid=DB.Partition(tsid)): node_from_code = new_row.node_from node_to_code = new_row.node_to line_par_num = new_row.n_par line = Line.by_key[node_from_code, node_to_code, line_par_num] if not line: line = Line(new_row) line.add_line_hour_data(new_row)
def counter(self): if self.game_on: time_diff = datetime.now() - self.game_at self.game_since.text = str(time_diff).rsplit(".", 1)[0][3:] self.current_score.text = '{} wpm'.format(self.score) if time_diff.total_seconds() >= self.max_seconds: root = find_parent(self, TenFingers) root.switch_final_window() DB.store_put(datetime.now().timestamp(), self.score) DB.store_sync() root.current_screen.children[0].score.text = '{} wpm'.format( self.score) else: Clock.schedule_once(lambda dt: self.counter(), .1)
def make_dgu_groups(tsid): """create DguGroup instances""" con = DB.OracleConnection() DguGroup.clear() for new_row in con.script_cursor(rgs, tsid=DB.Partition(tsid)): dgu_group = DguGroup[new_row.group_code] if not dgu_group: dgu_group = DguGroup(new_row) dgu_group.add_dgu(new_row) for new_row in con.script_cursor(prs, tsid=DB.Partition(tsid)): dgu_group = DguGroup[new_row.group_code] if dgu_group: dgu_group.add_reserve_data(new_row)
def create_descriptors(self): ''' 生成特征描述子 :return: ''' print("start create_descriptors") start = time() dbs = {} count = 0 with DB.db.transaction(): DB.DescriptorModel.drop_table(fail_silently=True) DB.DescriptorModel.create_table() lv1 = [] for f in DB.Feature.select(DB.Feature.id, DB.Feature.ori).iterator(): assert isinstance(f, DB.Feature) model = DB.DescriptorModel() model.lv1 = self.get_descriptor_lv1(f.ori) model.lv2 = self.get_descriptor_lv2(f.ori) model.feature = f.id model.save() count += 1 if count % 100 == 0: print "did %d features" % count print "did %d features" % count print("create_descriptors done")
def add_lines_vertica(lines, scenario, tdate): scenario = {'scenario': scenario} print('adding lines from vertica for date %s' % tdate) start_time = time.time() con = DB.VerticaConnection() @DB.process_cursor(con, lines_s_v, scenario) def add_lines(new_row, lines_list): node_from_code = new_row[lines_s_v['node_from']] node_to_code = new_row[lines_s_v['node_to']] line_par_num = new_row[lines_s_v['n_par']] if not lines_list.get_line(node_from_code, node_to_code, line_par_num): lines_list.add_line(new_row) line = lines_list.get_line(node_from_code, node_to_code, line_par_num) for hour in range(HOURCOUNT): row = new_row[:lines_s_v['hour']] + [hour] + new_row[ (lines_s_v['hour'] + 1):] line.add_line_hour_data(row) add_lines(lines) print('%s %s seconds %s' % (15 * '-', round(time.time() - start_time, 3), 15 * '-')) return lines
def add_bids_vertica(bids, scenario, tdate): scenario = {'scenario': scenario} print('adding bids from vertica for date %s' % tdate) start_time = time.time() con = DB.VerticaConnection() @DB.process_cursor(con, bid_init_s, scenario) def add_bids(new_row, bids_list): bids_list.add_bid(new_row) @DB.process_cursor(con, bid_hours_s, scenario) def add_bid_hours_data(new_row, bids_list): dpg_id = new_row[bid_hours_s['dpg_id']] bid = bids_list[dpg_id] if bid: bid.add_hour_data(new_row) @DB.process_cursor(con, bid_pairs_s, scenario) def add_bid_pairs_data(new_row, bids_list): dpg_id = new_row[bid_pairs_s['dpg_id']] bid = bids_list[dpg_id] if bid: bid.add_intervals_data(new_row) add_bids(bids) add_bid_hours_data(bids) add_bid_pairs_data(bids) print('%s %s seconds %s' % (15 * '-', round(time.time() - start_time, 3), 15 * '-')) return bids
def add_dgus_vertica(dgus, scenario, tdate): scenario = {'scenario': scenario} print('adding dgus from vertica for date %s' % tdate) start_time = time.time() con = DB.VerticaConnection() @DB.process_cursor(con, dgs_v, scenario) def add_dgus(new_row, dgus_list): dgus_list.add_dgu(new_row, NEWDGU) @DB.process_cursor(con, rgs_v, scenario) def add_dgus_data(new_row, dgus_list): dgu_code = new_row[rgs_v['rge_code']] dgu = dgus_list.get_dgu_by_code(dgu_code) if dgu: dgu.add_dgu_hour_data(new_row) # [(int(n) if int(n) == n else float(n)) if isinstance(n, decimal.Decimal) else n for n in new_row] add_dgus(dgus) add_dgus_data(dgus) print('%s %s seconds %s' % (15 * '-', round(time.time() - start_time, 3), 15 * '-')) return dgus
def make_gus(tsid): """create Gu instances""" con = DB.OracleConnection() Gu.clear() for new_row in con.script_cursor(gs, tsid=DB.Partition(tsid)): Gu(new_row) for new_row in con.script_cursor(ns, tsid=DB.Partition(tsid)): gu_code = new_row.gu_code if not Gu.by_code[gu_code]: Gu.from_nblock(new_row) print('WARNING!! added Gu %i not from RIO!' % gu_code) unit_hd = GuHourData(new_row) for unit in Gu.by_code[gu_code]: unit.add_gu_hour_data(new_row, unit_hd)
def add_lines_vertica(scenario): """add Line instances from Vertica DB""" con = DB.VerticaConnection() new_lines = [] for new_row in con.script_cursor(ls_v, scenario=scenario): key = (new_row.node_from, new_row.node_to, new_row.n_par) line = Line.by_key[key] if line and key not in new_lines: raise Exception( 'Vertica contains already existing line %i -> %i: %i' % key) if not line: line = Line(new_row, is_new=True) new_lines.append(key) # for hour in range(HOURCOUNT): # row = ls_v.Tup(*(new_row[:ls_v['hour']] + (hour,) + new_row[(ls_v['hour'] + 1):])) line.add_line_hour_data(new_row) for key in con.script_cursor(lo_v, scenario=scenario): line = Line.by_key[key] if line: line.is_turned_off = True for _hd in line.hour_data: _hd.state = False
def add_areas_vertica(scenario): """add Area instances from Vertica DB""" con = DB.VerticaConnection() for new_row in con.script_cursor(ra_v, scenario=scenario): area = Area[new_row.area] if not area: area = Area(new_row, is_new=True) area.add_area_hour_data(new_row)
def make_dgus(tsid): """create Dgu instances""" con = DB.OracleConnection() Dgu.clear() for new_row in con.script_cursor(dgs, tsid=DB.Partition(tsid)): Dgu(new_row) for new_row in con.script_cursor(rgs, tsid=DB.Partition(tsid)): dgu = Dgu.by_code[new_row.rge_code] if dgu: dgu.add_dgu_hour_data(new_row) for new_row in con.script_cursor(glhs, tsid=DB.Partition(tsid)): dgu = Dgu.by_code[new_row.rge_code] if dgu: dgu.set_last_hour(new_row)
def make_bids(tsid): """create Bid instances""" con = DB.OracleConnection() Bid.clear() for new_row in con.script_cursor(bis, tsid=DB.Partition(tsid)): Bid(new_row) for new_row in con.script_cursor(bhs, tsid=DB.Partition(tsid)): bid = Bid[new_row.dpg_id] if bid: bid.add_hour_data(new_row) for new_row in con.script_cursor(bps, tsid=DB.Partition(tsid)): bid = Bid[new_row.dpg_id] if bid: bid.add_intervals_data(new_row)
class Blockchain(object): """ Blockchain keeps a sequence of Blocks Attributes: _tip (bytes): Point to the latest hash of block. _db (DB): DB instance """ latest = 'l' def __init__(self): self._db = DB() self._tip = self._db.get('l') if not self._tip: block = Block().pow_of_block() self._db.put(block.hash, pickle.dumps(block)) self._db.put(Blockchain.latest, block.hash) self._tip = block.hash def add_block(self, data): # AddBlock saves provided data as a block in the blockchain last_hash = self._db.get(Blockchain.latest) new_block = Block(data, last_hash) # Update DB with the new block self._db.put(new_block.hash, pickle.dumps(new_block)) # Tip point to the new block self._db.put(Blockchain.latest, new_block.hash) # Update tip self._tip = new_block.hash @property def blocks(self): current_tip = self._tip while True: encoded_block = self._db.get(current_tip) block = pickle.load(encoded_block) yield block current_tip = block.prev_block_hash