def finish(self, path):
print "compress"
compress_file = self.hash + "_out.tar.gz"
compress(self.name, self.dirOut, self.dirFinish + compress_file)
os.chdir(self.path)
os.remove(self.dirOut + self.name)
print "finish"
def merge_big_and_small_logs(logs, log_level_ids_to_add,
log_level_messages, log_level_ids_merged):
"""Merge big message logs with small ones"""
new_logs = []
for log in logs:
if not log["_source"]["message"].strip():
continue
log_level = log["_source"]["log_level"]
if log["_id"] in log_level_ids_to_add[log_level]:
merged_small_logs = utils.compress(
log_level_messages["message"][log_level])
new_logs.append(
LogMerger.prepare_new_log(log, log["_id"], False,
merged_small_logs))
for log_level in log_level_messages["message"]:
if not log_level_ids_to_add[log_level] and\
log_level_messages["message"][log_level].strip():
log = log_level_ids_merged[log_level]
new_log = LogMerger.prepare_new_log(
log,
str(log["_id"]) + "_m",
True,
utils.compress(log_level_messages["message"][log_level]),
fields_to_clean=[
"message", "detected_message", "only_numbers",
"detected_message_with_numbers", "stacktrace",
"found_exceptions_extended",
"detected_message_extended",
"detected_message_without_params_extended",
"stacktrace_extended", "message_extended",
"message_without_params_extended", "urls", "paths",
"message_params",
"message_without_params_and_brackets",
"detected_message_without_params_and_brackets"
])
for field in log_level_messages:
if field in ["message"]:
continue
new_log["_source"][field] = utils.compress(
log_level_messages[field][log_level])
new_log["_source"][
"found_exceptions_extended"] = utils.compress(
utils.enrich_found_exceptions(
log_level_messages["found_exceptions"][log_level]))
new_logs.append(new_log)
return new_logs
def process(self, p=None):
petition = self.dequeue() if p is None else p
names = self.splitPath(petition)
try:
list = decompress(names[0], petition)
except OSError as osE:
print "error with a file"
os.chdir(self.path)
shutil.rmtree("../out/" + names[0])
self.process(petition)
listOut = []
os.mkdir(list['dir'] + "/result/")
try:
for l in list['listDir']:
if names[0].split("_")[1] == "wiki":
params = self.wiki.search(l, names[0],
list['dir'] + "/result/")
if params and len(params) > 0:
for d in params:
listOut.append(d)
compress_file = names[0] + "_out.tar.gz"
compress(listOut, list['dir'] + "/result/",
list['dir'] + "/../" + compress_file)
elif names[0].split("_")[1] == "youtube":
self.youtube.search(l, names[0], list['dir'] + "/result/",
names[0])
except OSError as osE:
print osE
print "error with a file"
os.chdir(self.path)
shutil.rmtree(list['dir'])
except Exception as eEx:
print eEx
os.chdir(self.path)
shutil.rmtree(list['dir'])
time.sleep(15)
self.process(petition)
except e:
print e
print "cannot get resources, check internet connection!"
os.chdir(self.path)
shutil.rmtree(list['dir'])
time.sleep(15)
self.process(petition)
print "remove"
print petition
print os.getcwd()
os.remove(petition)
os.chdir(self.path)
def send_block_hashes():
log_ip(request, inspect.stack()[0][3])
peer_height = int(request.forms.get("myheight"))
hash_list = []
for i in range(peer_height, BLOCKCHAIN.active_chain.length):
hash_list.append(dhash(BLOCKCHAIN.active_chain.header_list[i]))
return compress(json.dumps(hash_list)).decode()
def send_transaction():
log_ip(request, inspect.stack()[0][3])
data = request.json
transaction = Transaction.from_json(data["transaction"]).object()
sig = data["signature"]
transaction.add_sign(sig)
logger.debug(transaction)
logger.info("Wallet: Attempting to Send Transaction")
try:
r = requests.post(
"http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) +
"/newtransaction",
data=compress(transaction.to_json()),
timeout=(5, 1),
)
if r.status_code == 400:
response.status = 400
logger.error(
"Wallet: Could not Send Transaction. Invalid transaction: " +
r.text)
return "Invalid Transaction: " + r.text
except Exception as e:
response.status = 400
logger.error("Wallet: Could not Send Transaction. Try Again." + str(e))
return "Internal error, try again"
else:
logger.info("Wallet: Transaction Sent, Wait for it to be Mined")
return "Done"
def __mine(self, mempool: Set[Transaction], chain: Chain,
wallet: Wallet) -> Block:
c_pool = list(copy.deepcopy(mempool))
mlist = self.__calculate_transactions(c_pool)
logger.debug(len(mlist))
block_header = BlockHeader(
version=consts.MINER_VERSION,
height=chain.length,
prev_block_hash=dhash(chain.header_list[-1]),
merkle_root=merkle_hash(mlist),
timestamp=int(time.time()),
signature="",
)
sign = wallet.sign(dhash(block_header))
block_header.signature = sign
block = Block(header=block_header, transactions=mlist)
r = requests.post("http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) +
"/newblock",
data=compress(block.to_json()))
if r.text == "Block Received":
vjti_chain_relayer = VJTIChainRelayer(wallet)
vjti_chain_relayer.new_block(block)
logger.info(f"Miner: Mined Block with {len(mlist)} transaction(s)")
return block
else:
logger.info(
f"Miner: Could not mine block with {len(mlist)} transaction(s)"
)
return None
def send_block_hashes():
peer_height = int(request.forms.get("myheight"))
hash_list = []
for i in range(peer_height, BLOCKCHAIN.active_chain.length):
hash_list.append(dhash(BLOCKCHAIN.active_chain.header_list[i]))
# logger.debug("Server: Sending Peer this Block Hash List: " + str(hash_list))
return compress(json.dumps(hash_list)).decode()
def send_bounty(receiver_public_keys: List[str], amounts: List[int]):
current_balance = check_balance(MY_WALLET.public_key)
for key in receiver_public_keys:
if len(key) < consts.PUBLIC_KEY_LENGTH:
logger.debug("Invalid Public Key Length")
return False
total_amount = sum(amounts)
if current_balance < total_amount:
logger.debug("Insuficient balance")
elif MY_WALLET.public_key in receiver_public_keys:
logger.debug("Cannot send to myself")
else:
transaction = create_transaction(receiver_public_keys, amounts, MY_WALLET.public_key, message="Authority: Faucet Money")
transaction.sign(MY_WALLET)
logger.info("Wallet: Attempting to Send Transaction")
try:
r = requests.post(
"http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) + "/newtransaction",
data=compress(transaction.to_json()),
timeout=(5, 1),
)
if r.status_code == 400:
logger.info("Wallet: Could not Send Transaction. Invalid Transaction")
else:
logger.info("Wallet: Transaction Sent, Wait for it to be Mined")
return True
except Exception as e:
logger.error("Wallet: Could not Send Transaction. Try Again." + str(e))
return False
def process(self):
petition = self.dequeue()
names = self.splitPath(petition)
list = decompress(names[0], petition)
listOut = []
os.mkdir(list['dir']+"/result/")
for l in list['listDir']:
params = self.wiki.search(l, names[0], list['dir']+"/result/")
for d in params:
listOut.append(d)
print listOut
compress_file = names[0]+"_out.tar.gz"
compress(listOut,list['dir']+"/result/", list['dir']+"/../"+compress_file)
os.chdir(self.path)
shutil.rmtree(list['dir'])
def cached_get_block(headerhash: str) -> str:
if headerhash:
db_block = get_block_from_db(headerhash)
if db_block:
return compress(db_block)
else:
logger.error("ERROR CALLED GETBLOCK FOR NON EXISTENT BLOCK")
return "Invalid Hash"
def __mine(self, mempool: Set[Transaction], chain: Chain,
payout_addr: str) -> Block:
c_pool = list(copy.deepcopy(mempool))
mlist, fees = self.__calculate_best_transactions(c_pool)
# logger.debug(f"Miner: Will mine {len(mlist)} transactions and get {fees} scoins in fees")
coinbase_tx_in = {
0:
TxIn(payout=None,
sig="Receiving some Money",
pub_key="Does it matter?")
}
coinbase_tx_out = {
0: TxOut(amount=chain.current_block_reward(), address=payout_addr),
1: TxOut(amount=fees, address=payout_addr),
}
coinbase_tx = Transaction(
is_coinbase=True,
version=consts.MINER_VERSION,
fees=0,
timestamp=int(time.time()),
locktime=-1,
vin=coinbase_tx_in,
vout=coinbase_tx_out,
)
mlist.insert(0, coinbase_tx)
block_header = BlockHeader(
version=consts.MINER_VERSION,
height=chain.length,
prev_block_hash=dhash(chain.header_list[-1]),
merkle_root=merkle_hash(mlist),
timestamp=int(time.time()),
target_difficulty=chain.target_difficulty,
nonce=0,
)
DONE = False
for n in range(2**64):
block_header.nonce = n
bhash = dhash(block_header)
if chain.is_proper_difficulty(bhash):
block = Block(header=block_header, transactions=mlist)
requests.post("http://0.0.0.0:" +
str(consts.MINER_SERVER_PORT) + "/newblock",
data=compress(block.to_json()))
logger.info(
f"Miner: Mined Block with {len(mlist)} transactions, Got {fees} in fees and {chain.current_block_reward()} as reward"
)
DONE = True
break
if not DONE:
logger.error(
"Miner: Exhausted all 2 ** 64 values without finding proper hash"
)
def process(self, path, metadata, data, source):
self._logger.info("Exiting media processor")
media_files = fs.get_media_list_on_path(path)
media_files = fs.sort_files_by_part(media_files)
for mf in media_files:
full_media_file_path = os.path.join(path, mf)
# get raw content of image
media_file_content = fs.read_file_as_binary(full_media_file_path)
# computet md5 crc
crc = utils.md5_raw_content(media_file_content)
# gzip content
media_file_gzip = utils.compress(media_file_content)
# prepare avro chema compatible data
avro_schema_data = \
transform.avro_msg_serializer(media_file_gzip, mf, metadata, data, source, crc)
# prepare avro binary
avro_raw_data = transform.encode_avro_message(avro_schema_data)
# import
self._importer.imp(avro_raw_data, avro_schema_data["filename"])
self._logger.info("Exiting media processor")
def send_bounty(bounty: int, receiver_public_key: str, fees: int):
current_balance = check_balance()
if current_balance < bounty + fees:
print("Insuficient balance ")
print("Current balance : " + str(current_balance))
print("You need " + str(current_balance - bounty) + "more money")
else:
transaction = Transaction(
version=1,
locktime=0,
timestamp=2,
is_coinbase=False,
fees=0,
vin={},
vout={
0: TxOut(amount=bounty, address=receiver_public_key),
1: TxOut(amount=0, address=MY_WALLET.public_key)
},
)
calculate_transaction_fees(transaction, MY_WALLET, bounty, fees)
logger.debug(transaction)
logger.info("Wallet: Attempting to Send Transaction")
try:
requests.post(
"http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) +
"/newtransaction",
data=compress(transaction.to_json()),
timeout=(5, 1),
)
except Exception as e:
logger.error("Wallet: Could not Send Transaction. Try Again." +
str(e))
else:
logger.info("Wallet: Transaction Sent, Wait for it to be Mined")
def onExit():
compress_file = "medicamentos_vademecum_out.tar.gz"
listOut = os.listdir("./vademecum/resultsMedicamentos/")
compress(listOut, "./vademecum/resultsMedicamentos/",
"../out/" + compress_file)
shutil.rmtree("./vademecum/resultsMedicamentos/")
def get_cc_co_cp_by_contract_address(contract_address: str) -> Tuple[str, str, str]:
r = requests.post("http://0.0.0.0:" + str(consts.MINER_SERVER_PORT) + "/get_cc_co_cp", data=compress(contract_address))
data = r.json()
return data.get('cc', ''), data.get('co', ''), data.get('cp', '')
def compress_config(config):
config['task_args'] = compress(config['task_args'])
config['dataset_args'] = compress(config['dataset_args'])
config['ptracker_args'] = compress(config['ptracker_args'])
config['strategy_args'] = compress(config['strategy_args'])
return config
