def make_impex_areas(tsid):
"""create ImpexArea instances"""
con = DB.OracleConnection()
ImpexArea.clear()
for new_row in con.script_cursor(ias, tsid=DB.Partition(tsid)):
ImpexArea(new_row)
def last(self):
keys = sorted(list(
map(lambda x: float(x), filter(lambda x: x[0].isdigit(),
DB.keys()))),
reverse=True)
if keys:
self.last_score = str(DB.store_get(str(keys[0])))
def make_dpgs(tsid):
"""create Dpg instances"""
con = DB.OracleConnection()
Dpg.clear()
DpgSupply.clear()
DpgImpex.clear()
DpgDemand.clear()
DpgDemandFSK.clear()
DpgDemandSystem.clear()
DpgDemandLoad.clear()
for new_row in con.script_cursor(cs, tsid=DB.Partition(tsid)):
if new_row.is_fsk:
DpgDemandFSK(new_row)
elif new_row.is_system:
DpgDemandSystem(new_row)
else:
DpgDemandLoad(new_row)
for new_row in con.script_cursor(dds, tsid=DB.Partition(tsid)):
Dpg.by_id[new_row.dpg_id].add_disqualified_data(new_row)
for new_row in con.script_cursor(gs, tsid=DB.Partition(tsid)):
DpgSupply(new_row)
for new_row in con.script_cursor(imp_s, tsid=DB.Partition(tsid)):
DpgImpex(new_row)
def make_stations(tsid):
"""create Station instances"""
con = DB.OracleConnection()
Station.clear()
for new_row in con.script_cursor(ss, tsid=DB.Partition(tsid)):
Station(new_row)
def make_settings(tsid):
"""create Settings instances"""
con = DB.OracleConnection()
Setting.clear()
for new_row in con.script_cursor(ss, tsid=DB.Partition(tsid)):
Setting(new_row)
def get_most_popular_movies(self):
if self.most_popular_movies == None:
if self.landing_page == None:
self.landing_page = self.get_landing_page()
# Navigate to Menu and Click
element_menu = self.driver.find_element_by_id(
"imdbHeader-navDrawerOpen")
ActionChains(self.driver).move_to_element(element_menu).click(
element_menu).perform()
# Navigate to Movie Category and Click
element_movie = element_menu.find_element(
By.XPATH, "//label[@for='nav-link-categories-mov']")
ActionChains(self.driver).move_to_element(element_movie).click(
element_movie).perform()
# Navigate to Most Popuplar Movies and Click
element_mpm = element_movie.find_element(
By.XPATH, "//a[@href='/chart/moviemeter/?ref_=nv_mv_mpm']")
ActionChains(self.driver).move_to_element(element_mpm).click(
element_mpm).perform()
WebDriverWait(self.driver, 10)
html = self.driver.page_source
self.most_popular_movies = IMDB.dfparse_table_movies(html)
DB.save_search(self.most_popular_movies, suffix='mpm')
return self.most_popular_movies
def web_hook():
"""Telegram will send new messages to this web-hook"""
# Translate POST request data to JSON format.
j = request.json
update = bot.json_to_update(j)
if not update.message.text:
return "invalid_message"
# If the request is `start` command.
if update.message.text.lower() in ["/start", "/start@sample_bot"]:
chat_id = update.message.chat_id
DB.add_client(chat_id)
bot.send_message(update.message.chat_id, Configs.Messages.START_TEXT)
# If the request is `help` command.
if update.message.text.lower() in ["/help", "/help@sample_bot"]:
bot.send_message(update.message.chat_id, Configs.Messages.HELP_TEXT)
# If the request is `switchreg` command.
# `switchreg` command will switch client registration.
# Unregistered clients won't be notified of new rss entries anymore until they re-register.
if update.message.text.lower() in ["/switchreg", "/switchreg@sample_bot"]:
chat_id = update.message.chat_id
DB.switch_registration(chat_id)
bot.send_message(update.message.chat_id,
Configs.Messages.SWITCH_REGISTRATION_TEXT)
# The request was processed successfully.
return "ok"
def make_wsumgen(tsid):
"""create Wsumgen instances"""
con = DB.OracleConnection()
Wsumgen.clear()
for new_row in con.script_cursor(ws, tsid=DB.Partition(tsid)):
Wsumgen(new_row)
def create_descriptors_pca(self, dim=90):
'''
计算描述子pca
:param dim:
:return:
'''
print("start create_descriptors_pca ...")
query = DB.DescriptorModel.select(
DB.DescriptorModel.id,
DB.DescriptorModel.descriptor).tuples().iterator()
features = numpy.array(map(lambda x: [x[0]] + list(x[1]), query))
print("create_descriptors_pca,count=%d,dim=%d" % (len(features), dim))
start = time()
print("build eigenvectors start time %s" % start)
mean, eigenvectors = cv2.PCACompute(features[:, 1:],
None,
maxComponents=dim)
fitted = cv2.PCAProject(features[:, 1:], mean, eigenvectors)
#pca = PCA(n_components=dim)
#fitted = pca.fit_transform(features[:,1:])
print("build eigenvectors cost time %s" % (time() - start))
print("saving data ...")
#scaler = preprocessing.MinMaxScaler()
#pca = scaler.fit_transform(pca)
DB.db.connect()
with DB.db.transaction():
DB.PcaModel.drop_table(fail_silently=True)
DB.PcaModel.create_table()
#res = DB.TrainingResult()
#res.name = "daisy_pca"
#res.data = pca
#res.save()
for i in range(0, len(fitted)):
model = DB.PcaModel()
model.pca = fitted[i]
model.feature = features[i][0]
model.save()
DB.TrainingResult.delete().where(
DB.TrainingResult.name == "pca_mean").execute()
DB.TrainingResult.delete().where(
DB.TrainingResult.name == "pca_eigenvectors").execute()
tr = DB.TrainingResult()
tr.name = "pca_mean"
tr.data = mean
tr.save()
tr = DB.TrainingResult()
tr.name = "pca_eigenvectors"
tr.data = eigenvectors
tr.save()
print("create_descriptors_pca done")
def create_classifier(self):
DB.db.connect()
clf = SGDClassifier(loss="modified_huber")
labs_map = NameToIndex()
with DB.db.transaction():
offset = 0
words_count = self.get_words_count()
classes = numpy.arange(0, words_count)
x_all = []
y_all = []
while True:
print ' %d partial_fit %d' % (time(), offset)
query = DB.Vocabulary\
.select(DB.Vocabulary.lv1, DB.Vocabulary.lv2)\
.join(DB.PcaModel, on=(DB.Vocabulary.feature == DB.PcaModel.feature)).order_by( DB.Vocabulary.feature).offset(offset).limit(1000)\
.tuples().iterator()
features = numpy.array(
map(lambda x: [x[0]] + list(x[1]), query))
offset += len(features)
if len(features) == 0:
break
Y = features[:, 0]
X = features[:, 1:]
labs = []
for lab in Y:
labs.append(labs_map.map(lab))
if (len(x_all) < 10000):
x_all = x_all + X.tolist()
y_all = y_all + labs
labs = numpy.array(labs)
#clf = LinearSVC()
#clf = OneVsRestClassifier(SVC(probability=True, kernel='linear'))
#clf.fit(X,labs)
clf.partial_fit(X, labs, classes)
print clf.score(x_all, y_all)
DB.TrainingResult.delete().where(
DB.TrainingResult.name == self.__class__.__name__ +
"_clf").execute()
DB.TrainingResult.delete().where(
DB.TrainingResult.name == self.__class__.__name__ +
"_labs_map").execute()
tr = DB.TrainingResult()
tr.name = self.__class__.__name__ + "_clf"
tr.data = clf
tr.save()
tr = DB.TrainingResult()
tr.name = self.__class__.__name__ + "_labs_map"
tr.data = labs_map
tr.save()
def make_price_zones(tsid):
"""create PriceZone instances"""
con = DB.OracleConnection()
PriceZone.clear()
for new_row in con.script_cursor(pzs, tsid=DB.Partition(tsid)):
price_zone = PriceZone(new_row)
for hour in range(HOURCOUNT):
price_zone.add_price_zone_hour_data(hour)
def __init__(self):
self._db = DB()
self._tip = self._db.get('l')
if not self._tip:
block = Block().pow_of_block()
self._db.put(block.hash, pickle.dumps(block))
self._db.put(Blockchain.latest, block.hash)
self._tip = block.hash
def make_areas(tsid):
"""create Area instances"""
con = DB.OracleConnection()
Area.clear()
for new_row in con.script_cursor(ra, tsid=DB.Partition(tsid)):
area = Area[new_row.area]
if not area:
area = Area(new_row)
area.add_area_hour_data(new_row)
def make_loads(tsid):
"""create Load instances"""
con = DB.OracleConnection()
Load.clear()
for new_row in con.script_cursor(rl, tsid=DB.Partition(tsid)):
load = Load[new_row.consumer_code]
if not load:
load = Load(new_row)
load.add_load_hour_data(new_row)
def make_consumers(tsid):
"""create Consumer instances"""
con = DB.OracleConnection()
Consumer.clear()
for new_row in con.script_cursor(rc, tsid=DB.Partition(tsid)):
consumer = Consumer[new_row.consumer_code]
if not consumer:
consumer = Consumer(new_row)
consumer.add_consumer_hour_data(new_row)
def make_nodes(tsid):
"""create Node instances"""
con = DB.OracleConnection()
Node.clear()
for new_row in con.script_cursor(ns, tsid=DB.Partition(tsid)):
node = Node[new_row.node_code]
if not node:
node = Node(new_row)
node.add_node_hour_data(new_row)
def __init__(self):
self.db = DB()
if self.db.get('latest'):
self.height = int(self.db.get('latest'))
self.prev_block = pickle.loads(self.db.get(int(self.height)))
else:
new_block = Block(0, "Blockchain initialized!", "")
self.prev_block = new_block.pow_on_block()
self.db.put(0, pickle.dumps(self.prev_block))
self.db.put('latest', 0)
self.height = 0
def make_peak_so(tsid, tdate):
"""create Settings instances"""
con = DB.OracleConnection()
PeakSO.clear()
done = False
for new_row in con.script_cursor(pss, tsid=DB.Partition(tsid), tdate=tdate):
PeakSO(new_row)
done = True
if not done:
for new_row in con.script_cursor(pbs):
PeakSO(new_row)
def make_lines(tsid):
"""create Line instances"""
con = DB.OracleConnection()
Line.clear()
for new_row in con.script_cursor(ls, tsid=DB.Partition(tsid)):
node_from_code = new_row.node_from
node_to_code = new_row.node_to
line_par_num = new_row.n_par
line = Line.by_key[node_from_code, node_to_code, line_par_num]
if not line:
line = Line(new_row)
line.add_line_hour_data(new_row)
def counter(self):
if self.game_on:
time_diff = datetime.now() - self.game_at
self.game_since.text = str(time_diff).rsplit(".", 1)[0][3:]
self.current_score.text = '{} wpm'.format(self.score)
if time_diff.total_seconds() >= self.max_seconds:
root = find_parent(self, TenFingers)
root.switch_final_window()
DB.store_put(datetime.now().timestamp(), self.score)
DB.store_sync()
root.current_screen.children[0].score.text = '{} wpm'.format(
self.score)
else:
Clock.schedule_once(lambda dt: self.counter(), .1)
def make_dgu_groups(tsid):
"""create DguGroup instances"""
con = DB.OracleConnection()
DguGroup.clear()
for new_row in con.script_cursor(rgs, tsid=DB.Partition(tsid)):
dgu_group = DguGroup[new_row.group_code]
if not dgu_group:
dgu_group = DguGroup(new_row)
dgu_group.add_dgu(new_row)
for new_row in con.script_cursor(prs, tsid=DB.Partition(tsid)):
dgu_group = DguGroup[new_row.group_code]
if dgu_group:
dgu_group.add_reserve_data(new_row)
def create_descriptors(self):
'''
生成特征描述子
:return:
'''
print("start create_descriptors")
start = time()
dbs = {}
count = 0
with DB.db.transaction():
DB.DescriptorModel.drop_table(fail_silently=True)
DB.DescriptorModel.create_table()
lv1 = []
for f in DB.Feature.select(DB.Feature.id, DB.Feature.ori).iterator():
assert isinstance(f, DB.Feature)
model = DB.DescriptorModel()
model.lv1 = self.get_descriptor_lv1(f.ori)
model.lv2 = self.get_descriptor_lv2(f.ori)
model.feature = f.id
model.save()
count += 1
if count % 100 == 0:
print "did %d features" % count
print "did %d features" % count
print("create_descriptors done")
def add_lines_vertica(lines, scenario, tdate):
scenario = {'scenario': scenario}
print('adding lines from vertica for date %s' % tdate)
start_time = time.time()
con = DB.VerticaConnection()
@DB.process_cursor(con, lines_s_v, scenario)
def add_lines(new_row, lines_list):
node_from_code = new_row[lines_s_v['node_from']]
node_to_code = new_row[lines_s_v['node_to']]
line_par_num = new_row[lines_s_v['n_par']]
if not lines_list.get_line(node_from_code, node_to_code, line_par_num):
lines_list.add_line(new_row)
line = lines_list.get_line(node_from_code, node_to_code,
line_par_num)
for hour in range(HOURCOUNT):
row = new_row[:lines_s_v['hour']] + [hour] + new_row[
(lines_s_v['hour'] + 1):]
line.add_line_hour_data(row)
add_lines(lines)
print('%s %s seconds %s' %
(15 * '-', round(time.time() - start_time, 3), 15 * '-'))
return lines
def add_bids_vertica(bids, scenario, tdate):
scenario = {'scenario': scenario}
print('adding bids from vertica for date %s' % tdate)
start_time = time.time()
con = DB.VerticaConnection()
@DB.process_cursor(con, bid_init_s, scenario)
def add_bids(new_row, bids_list):
bids_list.add_bid(new_row)
@DB.process_cursor(con, bid_hours_s, scenario)
def add_bid_hours_data(new_row, bids_list):
dpg_id = new_row[bid_hours_s['dpg_id']]
bid = bids_list[dpg_id]
if bid:
bid.add_hour_data(new_row)
@DB.process_cursor(con, bid_pairs_s, scenario)
def add_bid_pairs_data(new_row, bids_list):
dpg_id = new_row[bid_pairs_s['dpg_id']]
bid = bids_list[dpg_id]
if bid:
bid.add_intervals_data(new_row)
add_bids(bids)
add_bid_hours_data(bids)
add_bid_pairs_data(bids)
print('%s %s seconds %s' %
(15 * '-', round(time.time() - start_time, 3), 15 * '-'))
return bids
def add_dgus_vertica(dgus, scenario, tdate):
scenario = {'scenario': scenario}
print('adding dgus from vertica for date %s' % tdate)
start_time = time.time()
con = DB.VerticaConnection()
@DB.process_cursor(con, dgs_v, scenario)
def add_dgus(new_row, dgus_list):
dgus_list.add_dgu(new_row, NEWDGU)
@DB.process_cursor(con, rgs_v, scenario)
def add_dgus_data(new_row, dgus_list):
dgu_code = new_row[rgs_v['rge_code']]
dgu = dgus_list.get_dgu_by_code(dgu_code)
if dgu:
dgu.add_dgu_hour_data(new_row)
# [(int(n) if int(n) == n else float(n)) if isinstance(n, decimal.Decimal) else n for n in new_row]
add_dgus(dgus)
add_dgus_data(dgus)
print('%s %s seconds %s' %
(15 * '-', round(time.time() - start_time, 3), 15 * '-'))
return dgus
def make_gus(tsid):
"""create Gu instances"""
con = DB.OracleConnection()
Gu.clear()
for new_row in con.script_cursor(gs, tsid=DB.Partition(tsid)):
Gu(new_row)
for new_row in con.script_cursor(ns, tsid=DB.Partition(tsid)):
gu_code = new_row.gu_code
if not Gu.by_code[gu_code]:
Gu.from_nblock(new_row)
print('WARNING!! added Gu %i not from RIO!' % gu_code)
unit_hd = GuHourData(new_row)
for unit in Gu.by_code[gu_code]:
unit.add_gu_hour_data(new_row, unit_hd)
def add_lines_vertica(scenario):
"""add Line instances from Vertica DB"""
con = DB.VerticaConnection()
new_lines = []
for new_row in con.script_cursor(ls_v, scenario=scenario):
key = (new_row.node_from, new_row.node_to, new_row.n_par)
line = Line.by_key[key]
if line and key not in new_lines:
raise Exception(
'Vertica contains already existing line %i -> %i: %i' % key)
if not line:
line = Line(new_row, is_new=True)
new_lines.append(key)
# for hour in range(HOURCOUNT):
# row = ls_v.Tup(*(new_row[:ls_v['hour']] + (hour,) + new_row[(ls_v['hour'] + 1):]))
line.add_line_hour_data(new_row)
for key in con.script_cursor(lo_v, scenario=scenario):
line = Line.by_key[key]
if line:
line.is_turned_off = True
for _hd in line.hour_data:
_hd.state = False
def add_areas_vertica(scenario):
"""add Area instances from Vertica DB"""
con = DB.VerticaConnection()
for new_row in con.script_cursor(ra_v, scenario=scenario):
area = Area[new_row.area]
if not area:
area = Area(new_row, is_new=True)
area.add_area_hour_data(new_row)
def make_dgus(tsid):
"""create Dgu instances"""
con = DB.OracleConnection()
Dgu.clear()
for new_row in con.script_cursor(dgs, tsid=DB.Partition(tsid)):
Dgu(new_row)
for new_row in con.script_cursor(rgs, tsid=DB.Partition(tsid)):
dgu = Dgu.by_code[new_row.rge_code]
if dgu:
dgu.add_dgu_hour_data(new_row)
for new_row in con.script_cursor(glhs, tsid=DB.Partition(tsid)):
dgu = Dgu.by_code[new_row.rge_code]
if dgu:
dgu.set_last_hour(new_row)
def make_bids(tsid):
"""create Bid instances"""
con = DB.OracleConnection()
Bid.clear()
for new_row in con.script_cursor(bis, tsid=DB.Partition(tsid)):
Bid(new_row)
for new_row in con.script_cursor(bhs, tsid=DB.Partition(tsid)):
bid = Bid[new_row.dpg_id]
if bid:
bid.add_hour_data(new_row)
for new_row in con.script_cursor(bps, tsid=DB.Partition(tsid)):
bid = Bid[new_row.dpg_id]
if bid:
bid.add_intervals_data(new_row)
class Blockchain(object):
""" Blockchain keeps a sequence of Blocks
Attributes:
_tip (bytes): Point to the latest hash of block.
_db (DB): DB instance
"""
latest = 'l'
def __init__(self):
self._db = DB()
self._tip = self._db.get('l')
if not self._tip:
block = Block().pow_of_block()
self._db.put(block.hash, pickle.dumps(block))
self._db.put(Blockchain.latest, block.hash)
self._tip = block.hash
def add_block(self, data):
# AddBlock saves provided data as a block in the blockchain
last_hash = self._db.get(Blockchain.latest)
new_block = Block(data, last_hash)
# Update DB with the new block
self._db.put(new_block.hash, pickle.dumps(new_block))
# Tip point to the new block
self._db.put(Blockchain.latest, new_block.hash)
# Update tip
self._tip = new_block.hash
@property
def blocks(self):
current_tip = self._tip
while True:
encoded_block = self._db.get(current_tip)
block = pickle.load(encoded_block)
yield block
current_tip = block.prev_block_hash