def main():
argument_parser = argparse.ArgumentParser(
description="Crawl Etherscan contract byte code. "
+ "No more than one process or thread should be used for the same contract.")
argument_parser.add_argument("contracts", type=argparse.FileType("r"),
help="File path containing contracts to crawl, one address per line.")
argument_parser.add_argument("--update", action="store_true", default=False,
help="Update contract if exists. If not set, throw an error when the contract exists.")
arguments = argument_parser.parse_args()
addresses = address_list_from_file(arguments.contracts)
etherscan_client = config.create_etherscan_client()
sqlalchemy_engine = config.create_sqlalchemy_engine()
sqlalchemy_session = sessionmaker(bind=sqlalchemy_engine)()
crawler = ByteCodeCrawler(sqlalchemy_session, etherscan_client)
logger.info("Crawling byte code for {:d} addresses...".format(len(addresses)))
for address in addresses:
try:
crawler.crawl(address, arguments.update)
# if something goes wrong, skip the address and continue
except Exception:
logger.exception("Error requesting transactions for address {}:".format(address))
logger.info("Done crawling transactions.")
def main():
argument_parser = argparse.ArgumentParser(
description="Create features per contract.")
argument_parser.add_argument(
"contracts",
type=argparse.FileType("r"),
help="File path containing contracts to crawl, one address per line.")
argument_parser.add_argument("output",
type=str,
help="Output file in csv format.")
argument_parser.add_argument(
"--processes",
type=int,
help="Number of processes. Default is cpu_count() - 1.")
argument_parser.add_argument("--log_every",
type=int,
default=5,
help="How many seconds between count logs.")
arguments = argument_parser.parse_args()
addresses = address_list_from_file(arguments.contracts)
multiprocess_by_address(addresses,
TransactionFeatureWorker,
arguments.output,
COLUMNS,
num_processes=arguments.processes,
log_every=arguments.log_every)
def main():
argument_parser = argparse.ArgumentParser(
description="Dump propagated honey badger labels into a csv file.")
argument_parser.add_argument(
"contracts",
type=argparse.FileType("r"),
help="File path containing contracts to label, one address per line.")
argument_parser.add_argument("output",
type=argparse.FileType("w"),
help="Output csv file.")
arguments = argument_parser.parse_args()
addresses = address_list_from_file(arguments.contracts)
sqlalchemy_engine = config.create_sqlalchemy_engine()
sqlalchemy_session = sessionmaker(bind=sqlalchemy_engine)()
csv_writer = csv.DictWriter(arguments.output, [
"contract_address", "contract_evaluation_positive",
"contract_label_id", "contract_label_name"
])
csv_writer.writeheader()
label_id_to_value = {}
labels = sqlalchemy_session.query(HoneyBadgerLabel).all()
for label in labels:
label_id_to_value[label.id] = label.value
for address in addresses:
entry = sqlalchemy_session.query(HoneyBadgerContractLabel).\
filter(HoneyBadgerContractLabel.address == address).one_or_none()
if entry is None:
csv_writer.writerow({
"contract_address": address,
"contract_evaluation_positive": 0,
"contract_label_id": 0,
"contract_label_name": "Not Honeypot"
})
else:
csv_writer.writerow({
"contract_address":
address,
"contract_evaluation_positive":
entry.evaluation_positive,
"contract_label_id":
entry.honey_badger_label_id,
"contract_label_name":
label_id_to_value[entry.honey_badger_label_id]
})
def main():
argument_parser = argparse.ArgumentParser(
description="Crawl Etherscan contract transactions. " +
"No more than one process or thread should be used for the same contract."
)
argument_parser.add_argument(
"contracts",
type=argparse.FileType("r"),
help="File path containing contracts to crawl, one address per line.")
argument_parser.add_argument(
"--max_requests",
type=int,
help=
"Maximum number of requests per address on each iteration (horizontal crawl)."
)
argument_parser.add_argument(
"--max_iterations",
type=int,
default=0,
help="Maximum number of iterations (horizontal crawl).")
argument_parser.add_argument("--size",
type=int,
help="Number of transactions per response.")
arguments = argument_parser.parse_args()
addresses = address_list_from_file(arguments.contracts)
etherscan_client = config.create_etherscan_client()
sqlalchemy_engine = config.create_sqlalchemy_engine()
sqlalchemy_session = sessionmaker(bind=sqlalchemy_engine)()
crawler = TransactionCrawler(sqlalchemy_session, etherscan_client)
logger.info("Crawling transactions for {:d} addresses...".format(
len(addresses)))
# while there are addresses to crawl
i = 0
while len(addresses) > 0 and (arguments.max_iterations == 0
or i < arguments.max_iterations):
remaining_addresses = []
for address in addresses:
try:
transaction_crawls = crawler.crawl(
address,
max_requests=arguments.max_requests,
size=arguments.size)
# check if there are remaining transactions for this address
normal_transaction_crawl, internal_transaction_crawl = transaction_crawls
if not normal_transaction_crawl.finished or not internal_transaction_crawl.finished:
remaining_addresses.append(address)
# if something goes wrong, skip the address and continue
except Exception:
logger.exception(
"Error requesting transactions for address {}:".format(
address))
# if the crawl is not horizontal, do not continue
if arguments.max_requests is None:
addresses = []
# if the crawl is horizontal, continue with the remaining addresses
else:
addresses = remaining_addresses
# next iteration
i += 1
logger.info("Done crawling transactions.")