def change_password(): uid = request.args.get('uid', None) user = None if uid and current_user.is_superuser: user = get_user(uid) else: user = current_user errs = None info = None if request.method == 'POST': try: uname = request.form['username'] pw1 = request.form['password'] pw2 = request.form['password-confirm'] assert pw1 == pw2 if not uname == current_user.username and not current_user.is_superuser: raise NoPermissionError() u = get_user(uname) if len(pw1) < 4: raise ValueError() u.password = generate_password_hash(pw1) write_to_db(u) info = "Password changed for '{}'".format(uname) except NoPermissionError: errs = ["Permission denied."] except Exception as ex: print(ex) errs = ["Password change failed; mismatching passwords?"] return render_template('auth/change_password.html', user=user, errors=errs, info=info)
def fetch_pollution_api_data(db_file, table, bid, lat, lon, key_itr): pollution_data = db.get_pollution_data() #db.update_realtime_db(rt_db_file, table, pollution_data) for x, lt, ln in zip(bid, lat, lon): if key_itr % 2 == 0: pollution_key = "8c970d0f7c3c48c5944b94de6aeb1e96" else: pollution_key = "b5130bf2c1974aa9929e240212237a6c" #pollution_key = "b5130bf2c1974aa9929e240212237a6c" pollution_api = "https://api.breezometer.com/air-quality/v2/current-conditions?lat=" + lt + "&lon=" + ln + "&key=" + pollution_key + "&features=breezometer_aqi,pollutants_concentrations" data = get_response(pollution_api) #print(data) if data['data']['data_available']: for i in data['data']['indexes']['baqi']: for key in pollution_data: if i in key: pollution_data[key] = data['data']['indexes']['baqi'][ key] for i in data['data']['pollutants']: for key in pollution_data: if i in key: pollution_data[key] = data['data']['pollutants'][key][ 'concentration']['value'] pollution_data['number'] = x pollution_data['lat'] = lt pollution_data['lng'] = ln db.write_to_db(db_file, table, pollution_data)
def set_consensus_chain(self): # 通过POW机制选取nonce最大的链作为公共链 for block_index in list(self.candidate_blocks.keys()): if block_index <= db.get_block_height( self.get_wallet_address()) - 1: curr_block = db.get_block_data_by_index( self.get_wallet_address(), block_index) max_nonce_block = curr_block for candidate_block in self.candidate_blocks[block_index]: if (candidate_block.previous_hash == curr_block.previous_hash) and ( candidate_block.nonce > max_nonce_block.nonce): max_nonce_block = candidate_block # 校验每一笔交易 valid_flag = True for idx in range(len(max_nonce_block.transactions)): tx = max_nonce_block.transactions[-1 - idx] if not self.verify_transaction(tx): valid_flag = False if valid_flag and max_nonce_block.current_hash != curr_block.current_hash: print('[Info] consensusing, replace with new block', max_nonce_block.current_hash) db.write_to_db(self.get_wallet_address(), max_nonce_block)
def handle_sendblock(self, payload): new_block = payload with self.server.node_manager.lock: blockchain = self.server.node_manager.blockchain block_height = db.get_block_height(blockchain.wallet.address) latest_block = db.get_block_data_by_index( blockchain.get_wallet_address(), block_height - 1) if (latest_block.current_hash == new_block.previous_hash) and (latest_block.index + 1 == new_block.index): # 校验交易是否有效 is_valid = True for idx in range(len(new_block.transactions)): tx = new_block.transactions[-1 - idx] if not blockchain.verify_transaction(tx): is_valid = False break if is_valid: db.write_to_db(blockchain.wallet.address, new_block) # 重新挖矿 blockchain.current_transactions = [] db.clear_unconfirmed_tx_from_disk( blockchain.wallet.address) else: self.add_to_candidate_blocks(blockchain, new_block) blockchain.set_consensus_chain()
def main(): r = requests.get(URL) snaps = extract_snapshots(r_text=r.text, select_month="202001") for snap in snaps: r = requests.get(snap["URL"]) snap["DAILY_INTEREST"] = hits_table(r.text) db_snaps = db_prepare_snaps(snaps) db.write_to_db(db_snaps)
def analyze(): print("POST call for analyzing image") print("request.files:" + str(request.files['file'])) response = get_image_details(request.files['file']) print("response: " + json.dumps(response)) write_to_db(response) return response
def handle_sendblock(self, payload): ''' 处理收到的某些缺失view的信息 ''' print "---handle send block---" self.server.node_manager.receiveblock = True if not payload == -1: del (self.server.node_manager.failhash[0]) db.write_to_db(self.server.node_manager.blockchain.wallet.address, payload)
def fetch_traffic_api_data(db_file, table, bid, lat1, lon1, lat2, lon2): traffic_data = db.get_traffic_data() #db.update_realtime_db(rt_db_file, table, traffic_data) for x, i, j in zip(bid, lat2, lon2): traffic_data = get_traffic_api_data(lat1, lon1, i, j) traffic_data['number'] = x traffic_data['lat'] = i traffic_data['lng'] = j db.write_to_db(db_file, table, traffic_data) #db.write_to_db(rt_db_file, table, traffic_data)
def create_user(name, password, sudo=False): if not name or not password or len(name) < 3 or len( password ) < 4 or name.isdigit( ): # Disallow unames that are numbers to avoid confusing the ID catcher raise ValueError() if get_user(name.lower()): raise UserExists() u = User(username=name.lower(), password=generate_password_hash(password)) u.is_superuser = sudo write_to_db(u)
def __init__(self, genisus_node=False): """ :param genisus_node: 判断是否是创世节点,如果是则读取本地(genisus_public.pem和genisus_private.pem)密钥对, 创始区块的第一笔交易就是奖励给genisus_public.pem """ self.difficulty = 4 self.current_transactions = [] self.wallet = wallet.Wallet(genisus_node) genius_block = self.get_genius_block() # 创世区块 db.write_to_db(self.wallet.address, genius_block) self.candidate_blocks = {}
def fetch_bike_api_data(db_file, table): bike_key = "5a9fca8c93d0a38ba40af732c366af7863d6f8c5" bike_api = "https://api.jcdecaux.com/vls/v1/stations?contract=Dublin&apiKey=" + bike_key data = get_response(bike_api) bike_data = db.get_bike_data() #db.update_realtime_db(rt_db_file, table, bike_data) for i in data: for key in bike_data: if key not in i: bike_data[key] = i['position'][key] else: bike_data[key] = i[key] db.write_to_db(db_file, table, bike_data)
def __init__(self, genisus_node=False): """ :param genisus_node: 判断是否是创世节点,如果是则读取本地(genisus_public.pem和genisus_private.pem)密钥对, 创始区块的第一笔交易就是奖励给genisus_public.pem """ self.difficulty = 4 self.current_transactions = [] #收集节点交换的交易 self.received_transactions = [] #用户发送的交易 self.send_transactions = [] # sendalltx时发送的交易 self.wallet = wallet.Wallet(genisus_node) genius_block = self.get_genius_block() # 创世区块 db.write_to_db(self.wallet.address, genius_block) self.candidate_blocks = {} self.prepareMessages = [] self.commitMessages = []
def demote_user(): if not current_user.is_superuser: return redirect('/') uid = request.args.get('uid', None) if not uid: return redirect(url_for('auth.manage_accounts')) user = get_user(uid) if not user: return redirect(url_for('auth.manage_accounts')) if request.method == 'POST': user.is_superuser = False uname = user.username write_to_db(user) return redirect( url_for('auth.manage_accounts', info="{} demoted.".format(uname))) return render_template('auth/demote.html', user=user)
def main(): args = parse_arguments() y_pred_dict = bw_to_dict(args.pred_npy_or_bw, args.chrom, args.window_size, args.blacklist_file) y_true_dict = bw_to_dict(args.true_npy_or_bw, args.chrom, args.window_size, args.blacklist_file) if args.var_npy is None: y_var_dict = None elif args.var_npy.endswith(('.npy', '.npz')): log.info('Opening truth var numpy array file...') y_var_dict = load_npy(args.var_npy) else: raise ValueError('Var true file should be a binned .npy or .npz.') enh_annotations = load_bed(args.enh_annotations) gene_annotations = load_bed(args.gene_annotations) gc.disable() cell, assay = parse_submission_filename(args.pred_npy_or_bw) for k, bootstrap_chrom in args.bootstrap_chrom: log.info('Calculating score for bootstrap {} case...'.format(k)) score_output = score(y_pred_dict, y_true_dict, bootstrap_chrom, gene_annotations, enh_annotations, args.window_size, args.prom_loc, y_var_dict) s = "\t".join(['bootstrap_' + str(k)] + [str(o) for o in score_output]) print(s) # write to DB if args.db_file is not None: score_db_record = ScoreDBRecord( args.submission_id, args.team_id, os.path.basename(args.pred_npy_or_bw), cell, assay, k, *score_output) write_to_db(score_db_record, args.db_file) gc.collect() log.info('All done')
def fetch_events_api_data(db_file, table, lat, lon): event_key = "sig_id=239072385&sig=7c49a476cedde45274f33d3cc95137d5478346c3" events_api = "https://api.meetup.com/2/open_events?and_text=False&offset=0&format=json&lon=" + lon + "&limited_events=False&photo-host=public&page=20&radius=25.0&lat=" + lat + "&desc=False&status=upcoming&" + event_key data = get_response(events_api) events_data = db.get_event_data() #db.update_realtime_db(rt_db_file, table, events_data) to_write_data = events_data for i in data['results']: if 'venue' in i and 'Dublin' in i['venue']['city']: #print(i['venue']['city']) for key in events_data: if key in i: events_data[key] = i[key] else: if 'venue' in i and key in i['venue']: events_data[key] = i['venue'][key] if key == 'time': events_data[key] = str(events_data[key])[:10] db.write_to_db(db_file, table, events_data)
requests = ['Автосалон', 'Салон красоты', 'Новости'] engine_parser = EngineParser() engines = ['Google', 'Bing', 'Youtube'] num_of_links = 10 bot = User(name='TestBot') start = time() for engine in engines: results = [] for request in requests: results.append( engine_parser.start_engine_scrapping(request, number=num_of_links, user=bot, language_code='ru', engine=engine, use_proxy=False, timeout_range=(3, 6), print_output=False)) write_to_db(results, engine) print('\n\nTIME:', time() - start) write_user_to_db(bot) # with open('output.json', 'w', encoding='utf-8') as outfile: # json.dump(results, outfile,indent=4,sort_keys=True,ensure_ascii=False)
def do_mine(self): """ 进行挖矿,验证当前节点收集的交易,并将有效交易打包成区块 :return: """ nonce = 0 timestamp = int(time()) # print('Minning a block...') new_block_found = False new_block_attempt = None # 验证每一笔交易的有效性(备注:从最新的开始验证) for idx in range(len(self.current_transactions)): tx = self.current_transactions[-1 - idx] if not self.verify_transaction(tx): txid = tx.txid print "[Info] Invalid transaction, remove it, tx:" raise Error( "[Error] do mine:Invalid transaction, remove it. Txid:" + txid) if len(self.current_transactions) < 5: # 至少要有5个以上的交易才可以开始进行挖矿 raise Error("[Error] Not enough valid transactions!") # TODO # 给工作量证明的节点提供奖励 # 发送者为"0" 表明新挖出的币 # coinbase_tx = self.new_coinbase_tx(self.get_wallet_address()) # valid_transactions.append(coinbase_tx) self.current_transactions = sorted(self.current_transactions, key=lambda x: x.timestamp, reverse=False) # 时间由小到大排 merkletrees = MerkleTrees(self.current_transactions) merkleroot = merkletrees.get_root_leaf() previous_block = self.get_last_block() next_index = previous_block.index + 1 previous_hash = previous_block.current_hash while not new_block_found: cal_hash = calculate_hash(next_index, previous_hash, timestamp, merkleroot, nonce, self.difficulty) if cal_hash[0:self.difficulty] == '0' * self.difficulty: new_block_attempt = self.generate_block( merkleroot, timestamp, nonce) end_timestamp = int(time()) cos_timestamp = end_timestamp - timestamp print '[Info] New block found with nonce ' + str( nonce) + ' in ' + str(round(cos_timestamp, 2)) + ' seconds.' # 交易按照timestamp从旧到新(小->大) new_block_attempt.transactions = self.current_transactions # 将所有交易保存成Merkle树 new_block_attempt.merkleroot = merkleroot db.write_to_db(self.wallet.address, new_block_attempt) self.current_transactions = [] db.clear_unconfirmed_tx_from_disk(self.wallet.address) new_block_found = True else: nonce += 1 return new_block_attempt
def test_read_write_db(db_file, table, data): db.write_to_db(db_file, table, data) rows = db.get_all_data(db_file, table) assert rows[0][2] == 'REAL' assert rows[0][3] == 'REAL' assert rows[0][4] == 'TEXT'
def sendrequest(self, payload): """ 广播一個request消息 :param payload 上一轮commit的hash: :return: """ print "-------send request: the end of pre-prepare-------" if payload != -1: if payload != self.blockcache.current_hash: self.failhash.append(payload) else: # print "pre-prepared1" #自身确认入链 if self.view >= 1: db.write_to_db(self.blockchain.wallet.address, self.blockcache) print "write to db" if self.view >= 1: self.commitMessages = [] self.maxTime = 0 self.blockchain.received_transactions = self.blockchain.received_transactions[ len(self.blockchain.send_transactions):] self.blockchain.current_transactions = self.blockchain.current_transactions[ self.txinblock:] self.receivealltx -= self.receivealltx_last self.blockchain.send_transactions = deepcopy( self.blockchain.received_transactions) # print "transaction list reset" self.view += 1 print "view:", self.view for node in self.committee_member: msg_obj = packet.Message( "sendrequest", { "hash": payload, "address": (self.client.ip, self.client.port), "GST": self.GST }) msg_bytes = pickle.dumps(msg_obj) self.client.sendrequest(self.server.socket, (node.ip, node.port), msg_bytes) # print "++++++++++++++sendrequest&tx++++++++++++++++" self.sendalltx(self.blockchain.send_transactions) self.startflag = True # print "pre-prepared2" else: self.GST += 5 for node in self.committee_member: msg_obj = packet.Message( "sendrequest", { "hash": payload, "address": (self.client.ip, self.client.port), "GST": self.GST }) msg_bytes = pickle.dumps(msg_obj) self.client.sendrequest(self.server.socket, (node.ip, node.port), msg_bytes) self.commitMessages = [] self.replyMessage = 0 self.maxTime = 0 self.blockchain.current_transactions = self.blockchain.current_transactions[ self.txinblock:] self.blockchain.send_transactions = deepcopy( self.blockchain.received_transactions) self.receivealltx -= self.receivealltx_last # print "transaction list reset" self.view += 1 print "view:", self.view # print "++++++++++++++sendrequest&tx++++++++++++++++" self.sendalltx(self.blockchain.send_transactions) self.startflag = True # print "pre-prepared2" self.replyflag = False
def handle_sendrequest(self, payload): print "------handle request: the start of prepare------" # with self.server.node_manager.lock: if payload["hash"] != -1: if payload[ "hash"] != self.server.node_manager.blockcache.current_hash: self.server.node_manager.failhash.append(payload["hash"]) else: # print payload["address"] print "the hash of last view:" print payload["hash"] self.server.node_manager.primary_node_address = payload[ "address"] # self.server.node_manager.startflag = True # print "pre-prepared" #确认入链 if self.server.node_manager.view >= 1: db.write_to_db( self.server.node_manager.blockchain.wallet.address, self.server.node_manager.blockcache) print "write to db" if self.server.node_manager.view >= 1: self.server.node_manager.commitMessages = [] self.server.node_manager.maxTime = 0 # for b1 in b.transactions: # if b1 in self.server.node_manager.received_transactions: # self.server.node_manager.received_transactions.remove(b1) # self.server.node_manager.current_transactions.remove(b1) # self.server.node_manager.current_transactions=[] # self.server.node_manager.received_transactions=[] self.server.node_manager.blockchain.received_transactions = self.server.node_manager.blockchain.received_transactions[ len(self.server.node_manager.blockchain.send_transactions ):] self.server.node_manager.blockchain.send_transactions = deepcopy( self.server.node_manager.blockchain.received_transactions) self.server.node_manager.blockchain.current_transactions = self.server.node_manager.blockchain.current_transactions[ self.server.node_manager.txinblock:] self.server.node_manager.receivealltx -= self.server.node_manager.receivealltx_last # self.server.node_manager.startflag = True # db.clear_unconfirmed_tx_from_disk(self.server.node_manager.blockchain.wallet.address) # print "transaction list clear" # time.sleep(15) self.server.node_manager.view += 1 print "view:", self.server.node_manager.view if (self.server.node_manager.is_committee): print "the tx we want to send:" print self.server.node_manager.blockchain.send_transactions self.server.node_manager.sendalltx( self.server.node_manager.blockchain.send_transactions) self.server.node_manager.startflag = True else: self.server.node_manager.view += 1 print "pbft failed, start again" print "view:", self.server.node_manager.view self.server.node_manager.GST = payload["GST"] self.server.node_manager.primary_node_address = payload["address"] self.server.node_manager.commitMessages = [] self.server.node_manager.maxTime = 0 self.server.node_manager.blockchain.current_transactions = self.server.node_manager.blockchain.current_transactions[ self.server.node_manager.txinblock:] self.server.node_manager.blockchain.send_transactions = deepcopy( self.server.node_manager.blockchain.received_transactions) self.server.node_manager.receivealltx -= self.server.node_manager.receivealltx_last if (self.server.node_manager.is_committee): self.server.node_manager.sendalltx( self.server.node_manager.blockchain.send_transactions) self.server.node_manager.startflag = True self.server.node_manager.replyflag = False
def score_submission(submission, status, args, syn, gene_annotations, enh_annotations): status['status'] = 'INVALID' submission_dir = os.path.join(os.path.abspath(args.submission_dir), submission.id) mkdir_p(submission_dir) chosen_score = None # first bootstrap score metadata = {'id': submission.id, 'team': 'undefined'} try: metadata['team'] = get_team_name(syn, None, submission.teamId) log.info('Downloading submission... {}'.format(submission.id)) submission = syn.getSubmission(submission, downloadLocation=submission_dir, ifcollision='overwrite.local') print() submission_fname = submission.filePath cell, assay = parse_submission_filename(submission_fname) if not is_valid_leaderboard_cell_assay(cell, assay): raise Exception('Invalid cell/assay combination for ' 'leaderboard round') log.info('Downloading done {}, {}, {}, {}, {}'.format( submission_fname, submission.id, submission.teamId, cell, assay)) # read pred npy (submission) log.info('Converting to dict...{}'.format(submission.id)) y_pred_dict = bw_to_dict(submission_fname, args.chrom, args.window_size, args.blacklist_file, args.validated) #gc.collect() # read truth npy npy_true = os.path.join(args.true_npy_dir, '{}{}.npy'.format(cell, assay)) y_true_dict = bw_to_dict(npy_true, args.chrom, args.window_size, args.blacklist_file) #gc.collect() # read var npy if args.var_npy_dir is not None: var_npy = os.path.join(args.var_npy_dir, 'var_{}.npy'.format(assay)) y_var_dict = load_npy(var_npy) else: y_var_dict = None #gc.collect() score_outputs = [] for k, bootstrap_chrom in args.bootstrap_chrom: # score it for each bootstrap chroms log.info('Scoring... k={}, submission_id={}'.format( k, submission.id)) r = score(y_pred_dict, y_true_dict, bootstrap_chrom, gene_annotations, enh_annotations, args.window_size, args.prom_loc, y_var_dict) #gc.collect() # free memory for bootstrapped arrays log.info('Scored: {}, {}, {}, {}'.format(submission.id, submission.teamId, k, r)) for m in r: if math.isnan(m) or m == float('inf') or m == float('-inf'): raise Exception('NaN or +-Inf found in score {}'.format(r)) score_outputs.append((k, r)) # score to be shown on wiki (first bootstrap score) chosen_score = score_outputs[0][1] # write to db and report for k, score_output in score_outputs: if not args.dry_run: score_db_record = ScoreDBRecord(int(submission.id), int(submission.teamId), submission_fname, cell, assay, k, *score_output) write_to_db(score_db_record, args.db_file) # mark is as scored status['status'] = 'SCORED' # free memory y_pred_dict = None y_true_dict = None y_var_dict = None #gc.collect() subject = 'Successfully scored submission %s %s %s:\n' % ( submission.name, submission.id, submission.teamId) message = 'Score (bootstrap_idx: score)\n' message += '\n'.join(['{}: {}'.format(k, s) for k, s in score_outputs]) log.info(subject + message) except Exception as ex1: if 'teamId' in submission: teamId = submission.teamId else: teamId = 'undefined' subject = 'Error scoring submission %s %s %s:\n' % ( submission.name, submission.id, teamId) st = StringIO() traceback.print_exc(file=st) message = st.getvalue() log.error(subject + message) finally: # remove submissions (both bigwig, npy) to save disk space shutil.rmtree(submission_dir) pass # send message users_to_send_msg = [] if args.send_msg_to_user: users_to_send_msg.append(submission.userId) if args.send_msg_to_admin: users_to_send_msg.extend(args.admin_id) send_message(syn, users_to_send_msg, subject, message) if not args.dry_run: # update metadata with score if chosen_score is not None: for field in Score._fields: metadata[field] = getattr(chosen_score, field) metadata['cell'] = cell metadata['assay'] = assay status[ 'annotations'] = synapseclient.annotations.to_submission_status_annotations( metadata, is_private=False) status = syn.store(status) return status