def change_password():
uid = request.args.get('uid', None)
user = None
if uid and current_user.is_superuser:
user = get_user(uid)
else:
user = current_user
errs = None
info = None
if request.method == 'POST':
try:
uname = request.form['username']
pw1 = request.form['password']
pw2 = request.form['password-confirm']
assert pw1 == pw2
if not uname == current_user.username and not current_user.is_superuser:
raise NoPermissionError()
u = get_user(uname)
if len(pw1) < 4:
raise ValueError()
u.password = generate_password_hash(pw1)
write_to_db(u)
info = "Password changed for '{}'".format(uname)
except NoPermissionError:
errs = ["Permission denied."]
except Exception as ex:
print(ex)
errs = ["Password change failed; mismatching passwords?"]
return render_template('auth/change_password.html',
user=user,
errors=errs,
info=info)
def fetch_pollution_api_data(db_file, table, bid, lat, lon, key_itr):
pollution_data = db.get_pollution_data()
#db.update_realtime_db(rt_db_file, table, pollution_data)
for x, lt, ln in zip(bid, lat, lon):
if key_itr % 2 == 0:
pollution_key = "8c970d0f7c3c48c5944b94de6aeb1e96"
else:
pollution_key = "b5130bf2c1974aa9929e240212237a6c"
#pollution_key = "b5130bf2c1974aa9929e240212237a6c"
pollution_api = "https://api.breezometer.com/air-quality/v2/current-conditions?lat=" + lt + "&lon=" + ln + "&key=" + pollution_key + "&features=breezometer_aqi,pollutants_concentrations"
data = get_response(pollution_api)
#print(data)
if data['data']['data_available']:
for i in data['data']['indexes']['baqi']:
for key in pollution_data:
if i in key:
pollution_data[key] = data['data']['indexes']['baqi'][
key]
for i in data['data']['pollutants']:
for key in pollution_data:
if i in key:
pollution_data[key] = data['data']['pollutants'][key][
'concentration']['value']
pollution_data['number'] = x
pollution_data['lat'] = lt
pollution_data['lng'] = ln
db.write_to_db(db_file, table, pollution_data)
def set_consensus_chain(self):
# 通过POW机制选取nonce最大的链作为公共链
for block_index in list(self.candidate_blocks.keys()):
if block_index <= db.get_block_height(
self.get_wallet_address()) - 1:
curr_block = db.get_block_data_by_index(
self.get_wallet_address(), block_index)
max_nonce_block = curr_block
for candidate_block in self.candidate_blocks[block_index]:
if (candidate_block.previous_hash
== curr_block.previous_hash) and (
candidate_block.nonce > max_nonce_block.nonce):
max_nonce_block = candidate_block
# 校验每一笔交易
valid_flag = True
for idx in range(len(max_nonce_block.transactions)):
tx = max_nonce_block.transactions[-1 - idx]
if not self.verify_transaction(tx):
valid_flag = False
if valid_flag and max_nonce_block.current_hash != curr_block.current_hash:
print('[Info] consensusing, replace with new block',
max_nonce_block.current_hash)
db.write_to_db(self.get_wallet_address(), max_nonce_block)
def handle_sendblock(self, payload):
new_block = payload
with self.server.node_manager.lock:
blockchain = self.server.node_manager.blockchain
block_height = db.get_block_height(blockchain.wallet.address)
latest_block = db.get_block_data_by_index(
blockchain.get_wallet_address(), block_height - 1)
if (latest_block.current_hash
== new_block.previous_hash) and (latest_block.index + 1
== new_block.index):
# 校验交易是否有效
is_valid = True
for idx in range(len(new_block.transactions)):
tx = new_block.transactions[-1 - idx]
if not blockchain.verify_transaction(tx):
is_valid = False
break
if is_valid:
db.write_to_db(blockchain.wallet.address, new_block)
# 重新挖矿
blockchain.current_transactions = []
db.clear_unconfirmed_tx_from_disk(
blockchain.wallet.address)
else:
self.add_to_candidate_blocks(blockchain, new_block)
blockchain.set_consensus_chain()
def main():
r = requests.get(URL)
snaps = extract_snapshots(r_text=r.text, select_month="202001")
for snap in snaps:
r = requests.get(snap["URL"])
snap["DAILY_INTEREST"] = hits_table(r.text)
db_snaps = db_prepare_snaps(snaps)
db.write_to_db(db_snaps)
def analyze():
print("POST call for analyzing image")
print("request.files:" + str(request.files['file']))
response = get_image_details(request.files['file'])
print("response: " + json.dumps(response))
write_to_db(response)
return response
def handle_sendblock(self, payload):
'''
处理收到的某些缺失view的信息
'''
print "---handle send block---"
self.server.node_manager.receiveblock = True
if not payload == -1:
del (self.server.node_manager.failhash[0])
db.write_to_db(self.server.node_manager.blockchain.wallet.address,
payload)
def fetch_traffic_api_data(db_file, table, bid, lat1, lon1, lat2, lon2):
traffic_data = db.get_traffic_data()
#db.update_realtime_db(rt_db_file, table, traffic_data)
for x, i, j in zip(bid, lat2, lon2):
traffic_data = get_traffic_api_data(lat1, lon1, i, j)
traffic_data['number'] = x
traffic_data['lat'] = i
traffic_data['lng'] = j
db.write_to_db(db_file, table, traffic_data)
#db.write_to_db(rt_db_file, table, traffic_data)
def create_user(name, password, sudo=False):
if not name or not password or len(name) < 3 or len(
password
) < 4 or name.isdigit(
): # Disallow unames that are numbers to avoid confusing the ID catcher
raise ValueError()
if get_user(name.lower()):
raise UserExists()
u = User(username=name.lower(), password=generate_password_hash(password))
u.is_superuser = sudo
write_to_db(u)
def __init__(self, genisus_node=False):
"""
:param genisus_node: 判断是否是创世节点，如果是则读取本地（genisus_public.pem和genisus_private.pem）密钥对，
创始区块的第一笔交易就是奖励给genisus_public.pem
"""
self.difficulty = 4
self.current_transactions = []
self.wallet = wallet.Wallet(genisus_node)
genius_block = self.get_genius_block() # 创世区块
db.write_to_db(self.wallet.address, genius_block)
self.candidate_blocks = {}
def fetch_bike_api_data(db_file, table):
bike_key = "5a9fca8c93d0a38ba40af732c366af7863d6f8c5"
bike_api = "https://api.jcdecaux.com/vls/v1/stations?contract=Dublin&apiKey=" + bike_key
data = get_response(bike_api)
bike_data = db.get_bike_data()
#db.update_realtime_db(rt_db_file, table, bike_data)
for i in data:
for key in bike_data:
if key not in i:
bike_data[key] = i['position'][key]
else:
bike_data[key] = i[key]
db.write_to_db(db_file, table, bike_data)
def __init__(self, genisus_node=False):
"""
:param genisus_node: 判断是否是创世节点，如果是则读取本地（genisus_public.pem和genisus_private.pem）密钥对，
创始区块的第一笔交易就是奖励给genisus_public.pem
"""
self.difficulty = 4
self.current_transactions = [] #收集节点交换的交易
self.received_transactions = [] #用户发送的交易
self.send_transactions = [] # sendalltx时发送的交易
self.wallet = wallet.Wallet(genisus_node)
genius_block = self.get_genius_block() # 创世区块
db.write_to_db(self.wallet.address, genius_block)
self.candidate_blocks = {}
self.prepareMessages = []
self.commitMessages = []
def demote_user():
if not current_user.is_superuser:
return redirect('/')
uid = request.args.get('uid', None)
if not uid:
return redirect(url_for('auth.manage_accounts'))
user = get_user(uid)
if not user:
return redirect(url_for('auth.manage_accounts'))
if request.method == 'POST':
user.is_superuser = False
uname = user.username
write_to_db(user)
return redirect(
url_for('auth.manage_accounts', info="{} demoted.".format(uname)))
return render_template('auth/demote.html', user=user)
def main():
args = parse_arguments()
y_pred_dict = bw_to_dict(args.pred_npy_or_bw, args.chrom, args.window_size,
args.blacklist_file)
y_true_dict = bw_to_dict(args.true_npy_or_bw, args.chrom, args.window_size,
args.blacklist_file)
if args.var_npy is None:
y_var_dict = None
elif args.var_npy.endswith(('.npy', '.npz')):
log.info('Opening truth var numpy array file...')
y_var_dict = load_npy(args.var_npy)
else:
raise ValueError('Var true file should be a binned .npy or .npz.')
enh_annotations = load_bed(args.enh_annotations)
gene_annotations = load_bed(args.gene_annotations)
gc.disable()
cell, assay = parse_submission_filename(args.pred_npy_or_bw)
for k, bootstrap_chrom in args.bootstrap_chrom:
log.info('Calculating score for bootstrap {} case...'.format(k))
score_output = score(y_pred_dict, y_true_dict, bootstrap_chrom,
gene_annotations, enh_annotations,
args.window_size, args.prom_loc, y_var_dict)
s = "\t".join(['bootstrap_' + str(k)] + [str(o) for o in score_output])
print(s)
# write to DB
if args.db_file is not None:
score_db_record = ScoreDBRecord(
args.submission_id, args.team_id,
os.path.basename(args.pred_npy_or_bw), cell, assay, k,
*score_output)
write_to_db(score_db_record, args.db_file)
gc.collect()
log.info('All done')
def fetch_events_api_data(db_file, table, lat, lon):
event_key = "sig_id=239072385&sig=7c49a476cedde45274f33d3cc95137d5478346c3"
events_api = "https://api.meetup.com/2/open_events?and_text=False&offset=0&format=json&lon=" + lon + "&limited_events=False&photo-host=public&page=20&radius=25.0&lat=" + lat + "&desc=False&status=upcoming&" + event_key
data = get_response(events_api)
events_data = db.get_event_data()
#db.update_realtime_db(rt_db_file, table, events_data)
to_write_data = events_data
for i in data['results']:
if 'venue' in i and 'Dublin' in i['venue']['city']:
#print(i['venue']['city'])
for key in events_data:
if key in i:
events_data[key] = i[key]
else:
if 'venue' in i and key in i['venue']:
events_data[key] = i['venue'][key]
if key == 'time':
events_data[key] = str(events_data[key])[:10]
db.write_to_db(db_file, table, events_data)
requests = ['Автосалон', 'Салон красоты', 'Новости']
engine_parser = EngineParser()
engines = ['Google', 'Bing', 'Youtube']
num_of_links = 10
bot = User(name='TestBot')
start = time()
for engine in engines:
results = []
for request in requests:
results.append(
engine_parser.start_engine_scrapping(request,
number=num_of_links,
user=bot,
language_code='ru',
engine=engine,
use_proxy=False,
timeout_range=(3, 6),
print_output=False))
write_to_db(results, engine)
print('\n\nTIME:', time() - start)
write_user_to_db(bot)
# with open('output.json', 'w', encoding='utf-8') as outfile:
# json.dump(results, outfile,indent=4,sort_keys=True,ensure_ascii=False)
def do_mine(self):
"""
进行挖矿，验证当前节点收集的交易，并将有效交易打包成区块
:return:
"""
nonce = 0
timestamp = int(time())
# print('Minning a block...')
new_block_found = False
new_block_attempt = None
# 验证每一笔交易的有效性(备注：从最新的开始验证)
for idx in range(len(self.current_transactions)):
tx = self.current_transactions[-1 - idx]
if not self.verify_transaction(tx):
txid = tx.txid
print "[Info] Invalid transaction, remove it, tx:"
raise Error(
"[Error] do mine:Invalid transaction, remove it. Txid:" +
txid)
if len(self.current_transactions) < 5:
# 至少要有5个以上的交易才可以开始进行挖矿
raise Error("[Error] Not enough valid transactions!")
# TODO
# 给工作量证明的节点提供奖励
# 发送者为"0" 表明新挖出的币
# coinbase_tx = self.new_coinbase_tx(self.get_wallet_address())
# valid_transactions.append(coinbase_tx)
self.current_transactions = sorted(self.current_transactions,
key=lambda x: x.timestamp,
reverse=False) # 时间由小到大排
merkletrees = MerkleTrees(self.current_transactions)
merkleroot = merkletrees.get_root_leaf()
previous_block = self.get_last_block()
next_index = previous_block.index + 1
previous_hash = previous_block.current_hash
while not new_block_found:
cal_hash = calculate_hash(next_index, previous_hash, timestamp,
merkleroot, nonce, self.difficulty)
if cal_hash[0:self.difficulty] == '0' * self.difficulty:
new_block_attempt = self.generate_block(
merkleroot, timestamp, nonce)
end_timestamp = int(time())
cos_timestamp = end_timestamp - timestamp
print '[Info] New block found with nonce ' + str(
nonce) + ' in ' + str(round(cos_timestamp,
2)) + ' seconds.'
# 交易按照timestamp从旧到新（小->大）
new_block_attempt.transactions = self.current_transactions
# 将所有交易保存成Merkle树
new_block_attempt.merkleroot = merkleroot
db.write_to_db(self.wallet.address, new_block_attempt)
self.current_transactions = []
db.clear_unconfirmed_tx_from_disk(self.wallet.address)
new_block_found = True
else:
nonce += 1
return new_block_attempt
def test_read_write_db(db_file, table, data):
db.write_to_db(db_file, table, data)
rows = db.get_all_data(db_file, table)
assert rows[0][2] == 'REAL'
assert rows[0][3] == 'REAL'
assert rows[0][4] == 'TEXT'
def sendrequest(self, payload):
"""
广播一個request消息
:param payload 上一轮commit的hash:
:return:
"""
print "-------send request: the end of pre-prepare-------"
if payload != -1:
if payload != self.blockcache.current_hash:
self.failhash.append(payload)
else:
# print "pre-prepared1"
#自身确认入链
if self.view >= 1:
db.write_to_db(self.blockchain.wallet.address,
self.blockcache)
print "write to db"
if self.view >= 1:
self.commitMessages = []
self.maxTime = 0
self.blockchain.received_transactions = self.blockchain.received_transactions[
len(self.blockchain.send_transactions):]
self.blockchain.current_transactions = self.blockchain.current_transactions[
self.txinblock:]
self.receivealltx -= self.receivealltx_last
self.blockchain.send_transactions = deepcopy(
self.blockchain.received_transactions)
# print "transaction list reset"
self.view += 1
print "view:", self.view
for node in self.committee_member:
msg_obj = packet.Message(
"sendrequest", {
"hash": payload,
"address": (self.client.ip, self.client.port),
"GST": self.GST
})
msg_bytes = pickle.dumps(msg_obj)
self.client.sendrequest(self.server.socket,
(node.ip, node.port), msg_bytes)
# print "++++++++++++++sendrequest&tx++++++++++++++++"
self.sendalltx(self.blockchain.send_transactions)
self.startflag = True
# print "pre-prepared2"
else:
self.GST += 5
for node in self.committee_member:
msg_obj = packet.Message(
"sendrequest", {
"hash": payload,
"address": (self.client.ip, self.client.port),
"GST": self.GST
})
msg_bytes = pickle.dumps(msg_obj)
self.client.sendrequest(self.server.socket,
(node.ip, node.port), msg_bytes)
self.commitMessages = []
self.replyMessage = 0
self.maxTime = 0
self.blockchain.current_transactions = self.blockchain.current_transactions[
self.txinblock:]
self.blockchain.send_transactions = deepcopy(
self.blockchain.received_transactions)
self.receivealltx -= self.receivealltx_last
# print "transaction list reset"
self.view += 1
print "view:", self.view
# print "++++++++++++++sendrequest&tx++++++++++++++++"
self.sendalltx(self.blockchain.send_transactions)
self.startflag = True
# print "pre-prepared2"
self.replyflag = False
def handle_sendrequest(self, payload):
print "------handle request: the start of prepare------"
# with self.server.node_manager.lock:
if payload["hash"] != -1:
if payload[
"hash"] != self.server.node_manager.blockcache.current_hash:
self.server.node_manager.failhash.append(payload["hash"])
else:
# print payload["address"]
print "the hash of last view:"
print payload["hash"]
self.server.node_manager.primary_node_address = payload[
"address"]
# self.server.node_manager.startflag = True
# print "pre-prepared"
#确认入链
if self.server.node_manager.view >= 1:
db.write_to_db(
self.server.node_manager.blockchain.wallet.address,
self.server.node_manager.blockcache)
print "write to db"
if self.server.node_manager.view >= 1:
self.server.node_manager.commitMessages = []
self.server.node_manager.maxTime = 0
# for b1 in b.transactions:
# if b1 in self.server.node_manager.received_transactions:
# self.server.node_manager.received_transactions.remove(b1)
# self.server.node_manager.current_transactions.remove(b1)
# self.server.node_manager.current_transactions=[]
# self.server.node_manager.received_transactions=[]
self.server.node_manager.blockchain.received_transactions = self.server.node_manager.blockchain.received_transactions[
len(self.server.node_manager.blockchain.send_transactions
):]
self.server.node_manager.blockchain.send_transactions = deepcopy(
self.server.node_manager.blockchain.received_transactions)
self.server.node_manager.blockchain.current_transactions = self.server.node_manager.blockchain.current_transactions[
self.server.node_manager.txinblock:]
self.server.node_manager.receivealltx -= self.server.node_manager.receivealltx_last
# self.server.node_manager.startflag = True
# db.clear_unconfirmed_tx_from_disk(self.server.node_manager.blockchain.wallet.address)
# print "transaction list clear"
# time.sleep(15)
self.server.node_manager.view += 1
print "view:", self.server.node_manager.view
if (self.server.node_manager.is_committee):
print "the tx we want to send:"
print self.server.node_manager.blockchain.send_transactions
self.server.node_manager.sendalltx(
self.server.node_manager.blockchain.send_transactions)
self.server.node_manager.startflag = True
else:
self.server.node_manager.view += 1
print "pbft failed, start again"
print "view:", self.server.node_manager.view
self.server.node_manager.GST = payload["GST"]
self.server.node_manager.primary_node_address = payload["address"]
self.server.node_manager.commitMessages = []
self.server.node_manager.maxTime = 0
self.server.node_manager.blockchain.current_transactions = self.server.node_manager.blockchain.current_transactions[
self.server.node_manager.txinblock:]
self.server.node_manager.blockchain.send_transactions = deepcopy(
self.server.node_manager.blockchain.received_transactions)
self.server.node_manager.receivealltx -= self.server.node_manager.receivealltx_last
if (self.server.node_manager.is_committee):
self.server.node_manager.sendalltx(
self.server.node_manager.blockchain.send_transactions)
self.server.node_manager.startflag = True
self.server.node_manager.replyflag = False
def score_submission(submission, status, args, syn, gene_annotations,
enh_annotations):
status['status'] = 'INVALID'
submission_dir = os.path.join(os.path.abspath(args.submission_dir),
submission.id)
mkdir_p(submission_dir)
chosen_score = None # first bootstrap score
metadata = {'id': submission.id, 'team': 'undefined'}
try:
metadata['team'] = get_team_name(syn, None, submission.teamId)
log.info('Downloading submission... {}'.format(submission.id))
submission = syn.getSubmission(submission,
downloadLocation=submission_dir,
ifcollision='overwrite.local')
print()
submission_fname = submission.filePath
cell, assay = parse_submission_filename(submission_fname)
if not is_valid_leaderboard_cell_assay(cell, assay):
raise Exception('Invalid cell/assay combination for '
'leaderboard round')
log.info('Downloading done {}, {}, {}, {}, {}'.format(
submission_fname, submission.id, submission.teamId, cell, assay))
# read pred npy (submission)
log.info('Converting to dict...{}'.format(submission.id))
y_pred_dict = bw_to_dict(submission_fname, args.chrom,
args.window_size, args.blacklist_file,
args.validated)
#gc.collect()
# read truth npy
npy_true = os.path.join(args.true_npy_dir,
'{}{}.npy'.format(cell, assay))
y_true_dict = bw_to_dict(npy_true, args.chrom, args.window_size,
args.blacklist_file)
#gc.collect()
# read var npy
if args.var_npy_dir is not None:
var_npy = os.path.join(args.var_npy_dir,
'var_{}.npy'.format(assay))
y_var_dict = load_npy(var_npy)
else:
y_var_dict = None
#gc.collect()
score_outputs = []
for k, bootstrap_chrom in args.bootstrap_chrom:
# score it for each bootstrap chroms
log.info('Scoring... k={}, submission_id={}'.format(
k, submission.id))
r = score(y_pred_dict, y_true_dict, bootstrap_chrom,
gene_annotations, enh_annotations, args.window_size,
args.prom_loc, y_var_dict)
#gc.collect() # free memory for bootstrapped arrays
log.info('Scored: {}, {}, {}, {}'.format(submission.id,
submission.teamId, k, r))
for m in r:
if math.isnan(m) or m == float('inf') or m == float('-inf'):
raise Exception('NaN or +-Inf found in score {}'.format(r))
score_outputs.append((k, r))
# score to be shown on wiki (first bootstrap score)
chosen_score = score_outputs[0][1]
# write to db and report
for k, score_output in score_outputs:
if not args.dry_run:
score_db_record = ScoreDBRecord(int(submission.id),
int(submission.teamId),
submission_fname, cell, assay,
k, *score_output)
write_to_db(score_db_record, args.db_file)
# mark is as scored
status['status'] = 'SCORED'
# free memory
y_pred_dict = None
y_true_dict = None
y_var_dict = None
#gc.collect()
subject = 'Successfully scored submission %s %s %s:\n' % (
submission.name, submission.id, submission.teamId)
message = 'Score (bootstrap_idx: score)\n'
message += '\n'.join(['{}: {}'.format(k, s) for k, s in score_outputs])
log.info(subject + message)
except Exception as ex1:
if 'teamId' in submission:
teamId = submission.teamId
else:
teamId = 'undefined'
subject = 'Error scoring submission %s %s %s:\n' % (
submission.name, submission.id, teamId)
st = StringIO()
traceback.print_exc(file=st)
message = st.getvalue()
log.error(subject + message)
finally:
# remove submissions (both bigwig, npy) to save disk space
shutil.rmtree(submission_dir)
pass
# send message
users_to_send_msg = []
if args.send_msg_to_user:
users_to_send_msg.append(submission.userId)
if args.send_msg_to_admin:
users_to_send_msg.extend(args.admin_id)
send_message(syn, users_to_send_msg, subject, message)
if not args.dry_run:
# update metadata with score
if chosen_score is not None:
for field in Score._fields:
metadata[field] = getattr(chosen_score, field)
metadata['cell'] = cell
metadata['assay'] = assay
status[
'annotations'] = synapseclient.annotations.to_submission_status_annotations(
metadata, is_private=False)
status = syn.store(status)
return status
