def finish(self, path): print "compress" compress_file = self.hash + "_out.tar.gz" compress(self.name, self.dirOut, self.dirFinish + compress_file) os.chdir(self.path) os.remove(self.dirOut + self.name) print "finish"
def merge_big_and_small_logs(logs, log_level_ids_to_add, log_level_messages, log_level_ids_merged): """Merge big message logs with small ones""" new_logs = [] for log in logs: if not log["_source"]["message"].strip(): continue log_level = log["_source"]["log_level"] if log["_id"] in log_level_ids_to_add[log_level]: merged_small_logs = utils.compress( log_level_messages["message"][log_level]) new_logs.append( LogMerger.prepare_new_log(log, log["_id"], False, merged_small_logs)) for log_level in log_level_messages["message"]: if not log_level_ids_to_add[log_level] and\ log_level_messages["message"][log_level].strip(): log = log_level_ids_merged[log_level] new_log = LogMerger.prepare_new_log( log, str(log["_id"]) + "_m", True, utils.compress(log_level_messages["message"][log_level]), fields_to_clean=[ "message", "detected_message", "only_numbers", "detected_message_with_numbers", "stacktrace", "found_exceptions_extended", "detected_message_extended", "detected_message_without_params_extended", "stacktrace_extended", "message_extended", "message_without_params_extended", "urls", "paths", "message_params", "message_without_params_and_brackets", "detected_message_without_params_and_brackets" ]) for field in log_level_messages: if field in ["message"]: continue new_log["_source"][field] = utils.compress( log_level_messages[field][log_level]) new_log["_source"][ "found_exceptions_extended"] = utils.compress( utils.enrich_found_exceptions( log_level_messages["found_exceptions"][log_level])) new_logs.append(new_log) return new_logs
def process(self, p=None): petition = self.dequeue() if p is None else p names = self.splitPath(petition) try: list = decompress(names[0], petition) except OSError as osE: print "error with a file" os.chdir(self.path) shutil.rmtree("../out/" + names[0]) self.process(petition) listOut = [] os.mkdir(list['dir'] + "/result/") try: for l in list['listDir']: if names[0].split("_")[1] == "wiki": params = self.wiki.search(l, names[0], list['dir'] + "/result/") if params and len(params) > 0: for d in params: listOut.append(d) compress_file = names[0] + "_out.tar.gz" compress(listOut, list['dir'] + "/result/", list['dir'] + "/../" + compress_file) elif names[0].split("_")[1] == "youtube": self.youtube.search(l, names[0], list['dir'] + "/result/", names[0]) except OSError as osE: print osE print "error with a file" os.chdir(self.path) shutil.rmtree(list['dir']) except Exception as eEx: print eEx os.chdir(self.path) shutil.rmtree(list['dir']) time.sleep(15) self.process(petition) except e: print e print "cannot get resources, check internet connection!" os.chdir(self.path) shutil.rmtree(list['dir']) time.sleep(15) self.process(petition) print "remove" print petition print os.getcwd() os.remove(petition) os.chdir(self.path)
def send_block_hashes(): log_ip(request, inspect.stack()[0][3]) peer_height = int(request.forms.get("myheight")) hash_list = [] for i in range(peer_height, BLOCKCHAIN.active_chain.length): hash_list.append(dhash(BLOCKCHAIN.active_chain.header_list[i])) return compress(json.dumps(hash_list)).decode()
def send_transaction(): log_ip(request, inspect.stack()[0][3]) data = request.json transaction = Transaction.from_json(data["transaction"]).object() sig = data["signature"] transaction.add_sign(sig) logger.debug(transaction) logger.info("Wallet: Attempting to Send Transaction") try: r = requests.post( "http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) + "/newtransaction", data=compress(transaction.to_json()), timeout=(5, 1), ) if r.status_code == 400: response.status = 400 logger.error( "Wallet: Could not Send Transaction. Invalid transaction: " + r.text) return "Invalid Transaction: " + r.text except Exception as e: response.status = 400 logger.error("Wallet: Could not Send Transaction. Try Again." + str(e)) return "Internal error, try again" else: logger.info("Wallet: Transaction Sent, Wait for it to be Mined") return "Done"
def __mine(self, mempool: Set[Transaction], chain: Chain, wallet: Wallet) -> Block: c_pool = list(copy.deepcopy(mempool)) mlist = self.__calculate_transactions(c_pool) logger.debug(len(mlist)) block_header = BlockHeader( version=consts.MINER_VERSION, height=chain.length, prev_block_hash=dhash(chain.header_list[-1]), merkle_root=merkle_hash(mlist), timestamp=int(time.time()), signature="", ) sign = wallet.sign(dhash(block_header)) block_header.signature = sign block = Block(header=block_header, transactions=mlist) r = requests.post("http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) + "/newblock", data=compress(block.to_json())) if r.text == "Block Received": vjti_chain_relayer = VJTIChainRelayer(wallet) vjti_chain_relayer.new_block(block) logger.info(f"Miner: Mined Block with {len(mlist)} transaction(s)") return block else: logger.info( f"Miner: Could not mine block with {len(mlist)} transaction(s)" ) return None
def send_block_hashes(): peer_height = int(request.forms.get("myheight")) hash_list = [] for i in range(peer_height, BLOCKCHAIN.active_chain.length): hash_list.append(dhash(BLOCKCHAIN.active_chain.header_list[i])) # logger.debug("Server: Sending Peer this Block Hash List: " + str(hash_list)) return compress(json.dumps(hash_list)).decode()
def send_bounty(receiver_public_keys: List[str], amounts: List[int]): current_balance = check_balance(MY_WALLET.public_key) for key in receiver_public_keys: if len(key) < consts.PUBLIC_KEY_LENGTH: logger.debug("Invalid Public Key Length") return False total_amount = sum(amounts) if current_balance < total_amount: logger.debug("Insuficient balance") elif MY_WALLET.public_key in receiver_public_keys: logger.debug("Cannot send to myself") else: transaction = create_transaction(receiver_public_keys, amounts, MY_WALLET.public_key, message="Authority: Faucet Money") transaction.sign(MY_WALLET) logger.info("Wallet: Attempting to Send Transaction") try: r = requests.post( "http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) + "/newtransaction", data=compress(transaction.to_json()), timeout=(5, 1), ) if r.status_code == 400: logger.info("Wallet: Could not Send Transaction. Invalid Transaction") else: logger.info("Wallet: Transaction Sent, Wait for it to be Mined") return True except Exception as e: logger.error("Wallet: Could not Send Transaction. Try Again." + str(e)) return False
def process(self): petition = self.dequeue() names = self.splitPath(petition) list = decompress(names[0], petition) listOut = [] os.mkdir(list['dir']+"/result/") for l in list['listDir']: params = self.wiki.search(l, names[0], list['dir']+"/result/") for d in params: listOut.append(d) print listOut compress_file = names[0]+"_out.tar.gz" compress(listOut,list['dir']+"/result/", list['dir']+"/../"+compress_file) os.chdir(self.path) shutil.rmtree(list['dir'])
def cached_get_block(headerhash: str) -> str: if headerhash: db_block = get_block_from_db(headerhash) if db_block: return compress(db_block) else: logger.error("ERROR CALLED GETBLOCK FOR NON EXISTENT BLOCK") return "Invalid Hash"
def __mine(self, mempool: Set[Transaction], chain: Chain, payout_addr: str) -> Block: c_pool = list(copy.deepcopy(mempool)) mlist, fees = self.__calculate_best_transactions(c_pool) # logger.debug(f"Miner: Will mine {len(mlist)} transactions and get {fees} scoins in fees") coinbase_tx_in = { 0: TxIn(payout=None, sig="Receiving some Money", pub_key="Does it matter?") } coinbase_tx_out = { 0: TxOut(amount=chain.current_block_reward(), address=payout_addr), 1: TxOut(amount=fees, address=payout_addr), } coinbase_tx = Transaction( is_coinbase=True, version=consts.MINER_VERSION, fees=0, timestamp=int(time.time()), locktime=-1, vin=coinbase_tx_in, vout=coinbase_tx_out, ) mlist.insert(0, coinbase_tx) block_header = BlockHeader( version=consts.MINER_VERSION, height=chain.length, prev_block_hash=dhash(chain.header_list[-1]), merkle_root=merkle_hash(mlist), timestamp=int(time.time()), target_difficulty=chain.target_difficulty, nonce=0, ) DONE = False for n in range(2**64): block_header.nonce = n bhash = dhash(block_header) if chain.is_proper_difficulty(bhash): block = Block(header=block_header, transactions=mlist) requests.post("http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) + "/newblock", data=compress(block.to_json())) logger.info( f"Miner: Mined Block with {len(mlist)} transactions, Got {fees} in fees and {chain.current_block_reward()} as reward" ) DONE = True break if not DONE: logger.error( "Miner: Exhausted all 2 ** 64 values without finding proper hash" )
def process(self, path, metadata, data, source): self._logger.info("Exiting media processor") media_files = fs.get_media_list_on_path(path) media_files = fs.sort_files_by_part(media_files) for mf in media_files: full_media_file_path = os.path.join(path, mf) # get raw content of image media_file_content = fs.read_file_as_binary(full_media_file_path) # computet md5 crc crc = utils.md5_raw_content(media_file_content) # gzip content media_file_gzip = utils.compress(media_file_content) # prepare avro chema compatible data avro_schema_data = \ transform.avro_msg_serializer(media_file_gzip, mf, metadata, data, source, crc) # prepare avro binary avro_raw_data = transform.encode_avro_message(avro_schema_data) # import self._importer.imp(avro_raw_data, avro_schema_data["filename"]) self._logger.info("Exiting media processor")
def send_bounty(bounty: int, receiver_public_key: str, fees: int): current_balance = check_balance() if current_balance < bounty + fees: print("Insuficient balance ") print("Current balance : " + str(current_balance)) print("You need " + str(current_balance - bounty) + "more money") else: transaction = Transaction( version=1, locktime=0, timestamp=2, is_coinbase=False, fees=0, vin={}, vout={ 0: TxOut(amount=bounty, address=receiver_public_key), 1: TxOut(amount=0, address=MY_WALLET.public_key) }, ) calculate_transaction_fees(transaction, MY_WALLET, bounty, fees) logger.debug(transaction) logger.info("Wallet: Attempting to Send Transaction") try: requests.post( "http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) + "/newtransaction", data=compress(transaction.to_json()), timeout=(5, 1), ) except Exception as e: logger.error("Wallet: Could not Send Transaction. Try Again." + str(e)) else: logger.info("Wallet: Transaction Sent, Wait for it to be Mined")
def onExit(): compress_file = "medicamentos_vademecum_out.tar.gz" listOut = os.listdir("./vademecum/resultsMedicamentos/") compress(listOut, "./vademecum/resultsMedicamentos/", "../out/" + compress_file) shutil.rmtree("./vademecum/resultsMedicamentos/")
def get_cc_co_cp_by_contract_address(contract_address: str) -> Tuple[str, str, str]: r = requests.post("http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) + "/get_cc_co_cp", data=compress(contract_address)) data = r.json() return data.get('cc', ''), data.get('co', ''), data.get('cp', '')
def compress_config(config): config['task_args'] = compress(config['task_args']) config['dataset_args'] = compress(config['dataset_args']) config['ptracker_args'] = compress(config['ptracker_args']) config['strategy_args'] = compress(config['strategy_args']) return config