def tx_based_analysis_with_filters(tx_fin_name): """ Performs a transaction data analysis using different filters, to obtain for example the amount of coinbase transactions. :param tx_fin_name: Input file path which contains the chainstate transaction dump. :type: str :return: None :rtype: None """ x_attributes = 'height' xlabels = ['Height'] out_names = ['tx_height_coinbase'] filters = [lambda x: x["coinbase"]] samples = get_filtered_samples(x_attributes, fin_name=tx_fin_name, filtr=filters) xs, ys = get_cdf(samples, normalize=True) for label, out in zip(xlabels, out_names): plots_from_samples(xs=xs, ys=ys, xlabel=label, save_fig=out, ylabel="Number of txs")
def tx_based_analysis_with_filters(tx_fin_name, version=0.15): """ Performs a transaction data analysis using different filters, to obtain for example the amount of coinbase transactions. :param tx_fin_name: Input file path which contains the chainstate transaction dump. :type: str :param version: Bitcoin core version, used to decide the folder in which to store the data. :type version: float :return: None :rtype: None """ x_attributes = 'height' xlabels = ['Height'] out_names = ['tx_height_coinbase'] filters = [lambda x: x["coinbase"]] samples = get_filtered_samples(x_attributes, fin_name=tx_fin_name, filtr=filters) for attribute, label, out in zip(x_attributes, xlabels, out_names): plots_from_samples(x_attribute=attribute, samples=samples, xlabel=label, save_fig=out, version=str(version), ylabel="Number of txs")
def comparative_data_analysis(tx_fin_name, utxo_fin_name, version): """ Performs a comparative data analysis between a transaction dump data file and an utxo dump one. :param tx_fin_name: Input file path which contains the chainstate transaction dump. :type: str :param utxo_fin_name: Input file path which contains the chainstate utxo dump. :type: str :param version: Bitcoin core version, used to decide the folder in which to store the data. :type version: float :return: None :rtype: None """ # Generate plots with both transaction and utxo data (f_parsed_txs and f_parsed_utxos) tx_attributes = ['total_value', 'height'] utxo_attributes = ['amount', 'tx_height'] xlabels = ['Amount (Satoshi)', 'Height'] out_names = ['tx_utxo_amount', 'tx_utxo_height'] legends = [['Tx.', 'UTXO'], ['Tx.', 'UTXO']] legend_locations = [1, 2] tx_samples = get_samples(tx_attributes, tx_fin_name) utxo_samples = get_samples(utxo_attributes, utxo_fin_name) for tx_attr, utxo_attr, label, out, legend, leg_loc in zip(tx_attributes, utxo_attributes, xlabels, out_names, legends, legend_locations): plots_from_samples(x_attribute=[tx_attr, utxo_attr], xlabel=label, save_fig=out, version=str(version), samples=[tx_samples[tx_attr], utxo_samples[utxo_attr]], legend=legend, legend_loc=leg_loc, ylabel="Number of registers", comparative=True)
def compare_dust(dust_files, legend, suffix=''): """ Compares dust of two given dust files. :param dust_files: List of dust file names :type dust_files: list of str :param legend: Legend for the charts :type legend: list of str :param suffix: Suffix for the output file names. :type suffix: str :return: None :rtype: None """ outs = [ "cmp_dust_utxos" + suffix, "cmp_dust_value" + suffix, "cmp_dust_data_len" + suffix ] totals = ['total_utxos', 'total_value', 'total_data_len'] utxos = [] value = [] length = [] for f in dust_files: data = load(open(CFG.data_path + f)) utxos.append(data['npest_utxos']) value.append(data['npest_value']) length.append(data['npest_data_len']) xs_utxos, ys_utxos = [sorted(u.keys(), key=int) for u in utxos ], [sorted(u.values(), key=int) for u in utxos] xs_value, ys_value = [sorted(v.keys(), key=int) for v in value ], [sorted(v.values(), key=int) for v in value] xs_length, ys_length = [sorted(l.keys(), key=int) for l in length ], [sorted(l.values(), key=int) for l in length] x_samples = [xs_utxos, xs_value, xs_length] y_samples = [ys_utxos, ys_value, ys_length] for xs, ys, out, total in zip(x_samples, y_samples, outs, totals): plots_from_samples(xs=xs, ys=ys, save_fig=out, legend=legend, xlabel='Fee rate(sat/byte)', ylabel="Number of UTXOs") # Get values in percentage ys_perc = [] for y_samples in ys: y_perc = [y / float(data[total]) for y in y_samples] ys_perc.append(y_perc) plots_from_samples(xs=xs, ys=ys_perc, save_fig='perc_' + out, legend=legend, xlabel='Fee rate(sat/byte)', ylabel="Number of UTXOs")
def dust_analysis(utxo_fin_name, f_dust, fltr=None): """ Performs a dust analysis by aggregating al the dust of a utxo dump file. :param utxo_fin_name: Input file path which contains the chainstate utxo dump. :type: str :param f_dust: Output file name where the aggregated dust will be stored. :type f_dust: str :param fltr: Filter to be applied to the samples. None by default. :type fltr: function :return: None :rtype: None """ # Generate plots for dust analysis (including percentage scale). # First, the dust accumulation file is generated data = aggregate_dust_np(utxo_fin_name, fout_name=f_dust, fltr=fltr) # # Or we can load it from a dust file if we have already created it # data = load(open(CFG.data_path + f_dust)) dict_labels = [["dust_utxos", "np_utxos", "npest_utxos"], ["dust_value", "np_value", "npest_value"], ["dust_data_len", "np_data_len", "npest_data_len"]] outs = ["dust_utxos", "dust_value", "dust_data_len"] totals = ['total_utxos', 'total_value', 'total_data_len'] ylabels = [ "Number of UTXOs", "UTXOs Amount (satoshis)", "UTXOs Sizes (bytes)" ] legend = ["Dust", "Non-profitable min.", "Non-profitable est."] for labels, out, total, ylabel in zip(dict_labels, outs, totals, ylabels): xs = [sorted(data[l].keys(), key=int) for l in labels] ys = [sorted(data[l].values(), key=int) for l in labels] plots_from_samples(xs=xs, ys=ys, save_fig=out, legend=legend, legend_loc=4, xlabel='Fee rate (sat./byte)', ylabel=ylabel) # Get values in percentage ys_perc = [] for y_samples in ys: y_perc = [y / float(data[total]) for y in y_samples] ys_perc.append(y_perc) plots_from_samples(xs=xs, ys=ys_perc, save_fig='perc_' + out, legend=legend, legend_loc=4, xlabel='Fee rate (sat./byte)', ylabel=ylabel)
def tx_based_analysis(tx_fin_name): """ Performs a transaction based analysis from a given input file (resulting from a transaction dump of the chainstate) :param tx_fin_name: Input file path which contains the chainstate transaction dump. :type: str :return: None :rtype: None """ x_attributes = ['height', 'total_len', 'total_value', 'num_utxos'] xlabels = [ 'Height', 'Total length (bytes)', 'Total value', 'Number of UTXOs per tx' ] out_names = [ "tx_height", ["tx_total_len", "tx_total_len_logx"], "tx_total_value_logx", ["tx_num_utxos", "tx_num_utxos_logx"] ] log_axis = [False, [False, 'x'], 'x', [False, 'x']] x_attr_pie = 'coinbase' xlabels_pie = [['Coinbase', 'No-coinbase']] out_names_pie = ['tx_coinbase'] pie_groups = [[[1], [0]]] pie_colors = [["#165873", "#428C5C"]] samples = get_samples(x_attributes + [x_attr_pie], fin_name=tx_fin_name) samples_pie = samples.pop(x_attr_pie) for attribute, label, log, out in zip(x_attributes, xlabels, log_axis, out_names): xs, ys = get_cdf(samples[attribute], normalize=True) plots_from_samples(xs=xs, ys=ys, xlabel=label, log_axis=log, save_fig=out, ylabel="Number of txs") for label, out, groups, colors in (zip(xlabels_pie, out_names_pie, pie_groups, pie_colors)): plot_pie_chart_from_samples(samples=samples_pie, save_fig=out, labels=label, title="", groups=groups, colors=colors, labels_out=True)
def utxo_based_analysis(utxo_fin_name, version=0.15): """ Performs a utxo based analysis from a given input file (resulting from a utxo dump of the chainstate) :param utxo_fin_name: Input file path which contains the chainstate utxo dump. :type: str :param version: Bitcoin core version, used to decide the folder in which to store the data. :type version: float :return: None :rtype: None """ x_attributes = ['tx_height', 'amount', 'index', 'out_type', 'utxo_data_len'] xlabels = ['Tx. height', 'Amount', 'UTXO Index', 'Out type', 'UTXO data len.'] out_names = ["utxo_tx_height", "utxo_amount_logx", ["utxo_index", "utxo_index_logx"], ["utxo_out_type", "utxo_out_type_logx"], ["utxo_data_len", "utxo_data_len_logx"]] # Getting register length in case of 0.15 onwards. For previous versions this chart is printed in the tx based # analysis, since data is already aggregated. if version >= 0.15: x_attributes += ['register_len'] xlabels += ['Register len.'] out_names += ['utxo_register_len', 'utxo_register_len_logx'] log_axis = [False, 'x', [False, 'x'], [False, 'x'], [False, 'x'], [False, 'x']] x_attributes_pie = ['out_type', 'out_type'] xlabels_pie = [['C-even', 'C-odd', 'U-even', 'U-odd'], ['P2PKH', 'P2PK', 'P2SH', 'Other']] out_names_pie = ["utxo_pk_types", "utxo_types"] pie_groups = [[[2], [3], [4], [5]], [[0], [2, 3, 4, 5], [1]]] x_attribute_special = 'non_std_type' # Since the attributes for the pie chart are already included in the normal chart, we won't pass them to the # sampling function. samples = get_samples(x_attributes + [x_attribute_special], fin_name=utxo_fin_name) for attribute, label, log, out in zip(x_attributes, xlabels, log_axis, out_names): plots_from_samples(x_attribute=attribute, samples=samples[attribute], xlabel=label, log_axis=log, save_fig=out, version=str(version), ylabel="Number of UTXOs") for attribute, label, out, groups in (zip(x_attributes_pie, xlabels_pie, out_names_pie, pie_groups)): plot_pie_chart_from_samples(samples=samples[attribute], save_fig=out, labels=label, title="", version=version, groups=groups, colors=["#165873", "#428C5C", "#4EA64B", "#ADD96C"]) # Special case: non-standard non_std_outs_analysis(samples[x_attribute_special], version)
def utxo_based_analysis_with_filters(utxo_fin_name, version=0.15): """ Performs an utxo data analysis using different filters, to obtain for examples the amount of SegWit outputs. :param utxo_fin_name: Input file path which contains the chainstate utxo dump. :type: str :param version: Bitcoin core version, used to decide the folder in which to store the data. :type version: float :return: None :rtype: None """ x_attribute = 'tx_height' xlabel = 'Block height' out_names = [ 'utxo_height_out_type', 'utxo_height_amount', 'segwit_upper_bound' ] filters = [ lambda x: x["out_type"] == 0, lambda x: x["out_type"] == 1, lambda x: x["out_type"] in [2, 3, 4, 5], lambda x: x["out_type"] not in range(0, 6), lambda x: x["amount"] < 10**2, lambda x: x["amount"] < 10**4, lambda x: x["amount"] < 10**6, lambda x: x["amount"] < 10**8, lambda x: x["out_type"] == 1 ] legends = [['P2PKH', 'P2SH', 'P2PK', 'Other'], ['$<10^2$', '$<10^4$', '$<10^6$', '$<10^8$'], ['P2SH']] comparative = [True, True, False] legend_loc = 2 offset = 0 samples = get_filtered_samples(x_attribute, fin_name=utxo_fin_name, filtr=filters) for out, flt, legend, comp in zip(out_names, filters, legends, comparative): plots_from_samples(x_attribute=[x_attribute] * len(legend), samples=samples[offset:offset + len(legend)], xlabel=xlabel, save_fig=out, legend=legend, legend_loc=legend_loc, version=str(version), comparative=comp, ylabel="Number of UTXOs") offset += len(legend)
def tx_based_analysis(tx_fin_name, version=0.15): """ Performs a transaction based analysis from a given input file (resulting from a transaction dump of the chainstate) :param tx_fin_name: Input file path which contains the chainstate transaction dump. :type: str :param version: Bitcoin core version, used to decide the folder in which to store the data. :type version: float :return: None :rtype: None """ x_attributes = ['height', 'total_len', 'version', 'total_value', 'num_utxos'] xlabels = ['Height', 'Total length (bytes)', 'Version', 'Total value', 'Number of UTXOs per tx'] out_names = ["tx_height", ["tx_total_len", "tx_total_len_logx"], 'tx_version', "tx_total_value_logx", ["tx_num_utxos", "tx_num_utxos_logx"]] log_axis = [False, [False, 'x'], False, 'x', [False, 'x']] x_attr_pie = ['coinbase'] xlabels_pie = [['Coinbase', 'No-coinbase']] out_names_pie = ['tx_coinbase'] pie_groups = [[[1], [0]]] pie_colors = [["#165873", "#428C5C"]] # Version has been dropped of in version 0.15, so there is no need of parsing Null data for 0.15 onwards. if version >= 0.15: i = x_attributes.index('version') x_attributes.pop(i) xlabels.pop(i) out_names.pop(i) log_axis.pop(i) samples = get_samples(x_attributes + x_attr_pie, fin_name=tx_fin_name) for attribute, label, log, out in zip(x_attributes, xlabels, log_axis, out_names): plots_from_samples(x_attribute=attribute, samples=samples[attribute], xlabel=label, log_axis=log, save_fig=out, version=str(version), ylabel="Number of txs") for attribute, label, out, groups, colors in (zip(x_attr_pie, xlabels_pie, out_names_pie, pie_groups, pie_colors)): plot_pie_chart_from_samples(samples=samples[attribute], save_fig=out, labels=label, title="", version=version, groups=groups, colors=colors)
def comparative_data_analysis(tx_fin_name, utxo_fin_name): """ Performs a comparative data analysis between a transaction dump data file and an utxo dump one. :param tx_fin_name: Input file path which contains the chainstate transaction dump. :type: str :param utxo_fin_name: Input file path which contains the chainstate utxo dump. :type: str :return: None :rtype: None """ # Generate plots with both transaction and utxo data (f_parsed_txs and f_parsed_utxos) tx_attributes = ['total_value', 'height'] utxo_attributes = ['amount', 'tx_height'] xlabels = ['Amount (Satoshi)', 'Height'] out_names = ['tx_utxo_amount', 'tx_utxo_height'] legends = [['Tx.', 'UTXO'], ['Tx.', 'UTXO']] legend_locations = [1, 2] tx_samples = get_samples(tx_attributes, tx_fin_name) utxo_samples = get_samples(utxo_attributes, utxo_fin_name) for tx_attr, utxo_attr, label, out, legend, leg_loc in zip( tx_attributes, utxo_attributes, xlabels, out_names, legends, legend_locations): xs_txs, ys_txs = get_cdf(tx_samples[tx_attr], normalize=True) xs_utxos, ys_utxos = get_cdf(utxo_samples[utxo_attr], normalize=True) plots_from_samples(xs=[xs_txs, xs_utxos], ys=[ys_txs, ys_utxos], xlabel=label, save_fig=out, legend=legend, legend_loc=leg_loc, ylabel="Number of registers")
def dust_analysis_all_fees(utxo_fin_name): """ Performs a dust analysis for all fee rates, that is, up until all samples are considered dust (plot shows cdf up until 1). :param utxo_fin_name: Input file path which contains the chainstate utxo dump. :type: str :return: None :rtype: None """ x_attributes = [["dust", "non_profitable", "non_profitable_est"]] xlabels = ['Dust/non_prof_min/non_prof_est value'] out_names = ["dust_utxos_all"] legends = [["Dust", "Non-profitable min.", "Non-profitable est."]] log_axis = ['x'] for attribute, label, log, out, legend in zip(x_attributes, xlabels, log_axis, out_names, legends): samples = get_samples(attribute, fin_name=utxo_fin_name) xs = [] ys = [] for a in attribute: x, y = get_cdf(samples[a], normalize=True) xs.append(x) ys.append(y) plots_from_samples(xs=xs, ys=ys, xlabel=label, log_axis=log, save_fig=out, ylabel="Number of UTXOs", legend=legend, legend_loc=4)
def compare_attribute(fin_names, x_attribute, out_name, xlabel='', legend=''): """ Performs a comparative analysis between different files and a fixed attribute. Useful to compare the evolution of a parameter throughout different snapshots. :param fin_names: List of file names to load data from. :type fin_names: list str :param x_attribute: Attribute to be compared. :type x_attribute: str :param out_name: Name of the generated chart. :type out_name: str :param xlabel: Label of the x axis of the resulting chart. :type xlabel: str :param legend: Legend to be included in the chart. :type legend: str or list :return: None :rtyp: None """ samples = [get_samples(x_attribute, fin) for fin in fin_names] xs = [] ys = [] for _ in range(len(samples)): x, y = get_cdf(samples.pop(0).values(), normalize=True) xs.append(x) ys.append(y) plots_from_samples(xs=xs, ys=ys, xlabel=xlabel, save_fig=out_name, legend=legend, log_axis='x', ylabel="Number of UTXOs", legend_loc=2)
def utxo_based_analysis_with_filters(utxo_fin_name): """ Performs an utxo data analysis using different filters, to obtain for examples the amount of SegWit outputs. :param utxo_fin_name: Input file path which contains the chainstate utxo dump. :type: str :return: None :rtype: None """ x_attribute = 'tx_height' xlabel = 'Block height' out_names = [ 'utxo_height_out_type', 'utxo_height_amount', 'segwit_upper_bound', 'utxo_height_1_satoshi' ] filters = [ lambda x: x["out_type"] == 0, lambda x: x["out_type"] == 1, lambda x: x["out_type"] in [2, 3, 4, 5], lambda x: x["non_std_type"] == "P2WPKH", lambda x: x["non_std_type"] == "P2WSH", lambda x: x[ "non_std_type"] is not False and "multisig" in x["non_std_type"], lambda x: x["non_std_type"] is False, lambda x: x["amount"] == 1, lambda x: 1 < x["amount"] <= 10**1, lambda x: 10 < x["amount"] <= 10**2, lambda x: 10**2 < x["amount"] <= 10**4, lambda x: 10**4 < x["amount"] <= 10**6, lambda x: 10**6 < x["amount"] <= 10**8, lambda x: x["amount"] > 10**8, lambda x: x["out_type"] == 1, lambda x: x["amount"] == 1 ] legends = [[ 'P2PKH', 'P2SH', 'P2PK', 'P2WPKH', 'P2WSH', 'Multisig', 'Other' ], [ '$=1$', '$1 < x \leq 10$', '$10 < x \leq 10^2$', '$10^2 < x \leq 10^4$', '$10^4 < x \leq 10^6$', '$10^6 < x \leq 10^8$', '$10^8 < x$' ], ['P2SH'], ['Amount = 1']] comparative = [True, True, False, False] legend_loc = 2 samples = get_filtered_samples(x_attribute, fin_name=utxo_fin_name, filtr=filters) for out, legend, comp in zip(out_names, legends, comparative): xs = [] ys = [] for _ in range(len(legend)): x, y = get_cdf(samples.pop(0), normalize=True) xs.append(x) ys.append(y) plots_from_samples(xs=xs, ys=ys, xlabel=xlabel, save_fig=out, legend=legend, legend_loc=legend_loc, ylabel="Number of UTXOs")
def utxo_based_analysis(utxo_fin_name): """ Performs a utxo based analysis from a given input file (resulting from a utxo dump of the chainstate) :param utxo_fin_name: Input file path which contains the chainstate utxo dump. :type: str :return: None :rtype: None """ x_attributes = [ 'tx_height', 'amount', 'index', 'out_type', 'utxo_data_len', 'register_len' ] xlabels = [ 'Tx. height', 'Amount', 'UTXO index', 'Out type', 'UTXO data length', 'Register length' ] out_names = [ "utxo_tx_height", "utxo_amount_logx", ["utxo_index", "utxo_index_logx"], ["utxo_out_type", "utxo_out_type_logx"], ["utxo_data_len", "utxo_data_len_logx"], ['utxo_register_len', 'utxo_register_len_logx'] ] log_axis = [ False, 'x', [False, 'x'], [False, 'x'], [False, 'x'], [False, 'x'] ] x_attributes_pie = ['out_type', 'out_type'] xlabels_pie = [['C-even', 'C-odd', 'U-even', 'U-odd'], ['P2PKH', 'P2PK', 'P2SH', 'Other']] out_names_pie = ["utxo_pk_types", "utxo_types"] pie_groups = [[[2], [3], [4], [5]], [[0], [2, 3, 4, 5], [1]]] x_attribute_special = 'non_std_type' # Since the attributes for the pie chart are already included in the normal chart, we won't pass them to the # sampling function. samples = get_samples(x_attributes + [x_attribute_special], fin_name=utxo_fin_name) samples_special = samples.pop(x_attribute_special) for attribute, label, log, out in zip(x_attributes, xlabels, log_axis, out_names): xs, ys = get_cdf(samples[attribute], normalize=True) plots_from_samples(xs=xs, ys=ys, xlabel=label, log_axis=log, save_fig=out, ylabel="Number of UTXOs") for attribute, label, out, groups in (zip(x_attributes_pie, xlabels_pie, out_names_pie, pie_groups)): plot_pie_chart_from_samples( samples=samples[attribute], save_fig=out, labels=label, title="", groups=groups, colors=["#165873", "#428C5C", "#4EA64B", "#ADD96C"], labels_out=True) # Special case: non-standard non_std_outs_analysis(samples_special)
def run_comparative_analysis_14_15(): # Generate plots from tx and utxo data version_str = "0.14-0.15" # 0.14 v = 0.14 tx_fin_name_14 = str(v) + "/parsed_txs.json" utxo_fin_name_14 = str(v) + "/parsed_utxos_wp2sh.json" tx_attributes_14 = ['total_len'] utxo_attributes_14 = ['utxo_data_len'] tx_samples_14 = get_samples(tx_attributes_14, tx_fin_name_14) utxo_samples_14 = get_samples(utxo_attributes_14, utxo_fin_name_14) # 0.15 v = 0.15 tx_fin_name_15 = str(v) + "/parsed_txs.json" utxo_fin_name_15 = str(v) + "/parsed_utxos_wp2sh.json" tx_attributes_15 = ['total_len'] utxo_attributes_15 = ['register_len', 'utxo_data_len'] tx_samples_15 = get_samples(tx_attributes_15, tx_fin_name_15) utxo_samples_15 = get_samples(utxo_attributes_15, utxo_fin_name_15) # TODO: why is x_attribute used? # Compare transaction storage space: # - register length of v0.14 (tx -> total_len) # - sum of register lengths of utxos from that tx in v0.15 (tx -> total_len) plots_from_samples( samples=[tx_samples_14['total_len'], tx_samples_15['total_len']], x_attribute=['total_len'], xlabel="Transaction len. (bytes)", log_axis="x", version=version_str, comparative=True, save_fig="tx_total_len_logx_0.14-0.15", legend=["v0.14", "v0.15"]) # Compare register lenghts of v0.14 (tx) vs v0.15 (utxo) # - transation total_len of v0.14 # - utxo register_len of v0.15 # x_attribute=['total_len', 'register_len'] plots_from_samples( samples=[tx_samples_14['total_len'], utxo_samples_15['register_len']], x_attribute=['aa'], xlabel="Register size (in bytes)", log_axis="x", version=version_str, comparative=True, save_fig="register_len_logx_0.14-0.15", legend=["v0.14", "v0.15"]) # Compare utxo data storage space: # - utxo_data_len of v0.15 # - utxo_data_len of v0.14 # TODO: check what have we stored exactly here! Does this make sense? plots_from_samples(samples=[ utxo_samples_14['utxo_data_len'], utxo_samples_15['utxo_data_len'] ], x_attribute=['aa'], xlabel="UTXO data len (in bytes)", log_axis="x", version=version_str, comparative=True, save_fig="utxo_data_len_logx_0.14-0.15", legend=["v0.14", "v0.15"])