def load_input_objects(self):
"""
Loads input objects to an array
:return:
"""
for file in self.args.files:
io = common.FileInputObject(fname=file, fmode='rb')
io.check()
self.input_objects.append(io)
if len(self.input_objects) == 0 or self.args.stdin:
self.input_objects.append(common.StdinInputObject(desc=self.args.stdin_desc))
def work(self, bin_data=None):
"""
Main entry point - data processing
:return:
"""
config = self.config_data
if self.args.config_file:
with open(self.args.config_file) as fh:
config = json.load(fh)
NRES_TO_DUMP = 128
self.timer_data_read.reset()
self.timer_data_bins.reset()
self.timer_process.reset()
cpu_pcnt_load_before = misc.try_get_cpu_percent()
cpu_load_before = misc.try_get_cpu_load()
hw_cfg = config['hwanalysis']
test_run = config['config']
data_file = common.defvalkeys(config, 'spec.data_file')
skip_finished = common.defvalkey(config, 'skip_finished', False)
self.do_halving = common.defvalkey(config, 'halving', False)
self.halving_top = common.defvalkey(config, 'halving_top',
NRES_TO_DUMP)
res_file = common.defvalkey(config, 'res_file')
backup_dir = common.defvalkey(config, 'backup_dir')
all_zscores = common.defvalkey(config, 'all_zscores')
if res_file and self.check_res_file(res_file):
if skip_finished:
logger.info('Already computed in %s' % res_file)
return
elif backup_dir:
misc.file_backup(res_file, backup_dir=backup_dir)
self.hw_cfg = hw_cfg
self.hwanalysis = HWAnalysis()
self.hwanalysis.from_json(hw_cfg)
self.check_ref_db(self.hwanalysis)
self.blocklen = self.hwanalysis.blocklen
self.deg = self.hwanalysis.deg
self.top_comb = self.hwanalysis.top_comb
self.top_k = self.hwanalysis.top_k
self.all_deg = self.hwanalysis.all_deg_compute
self.zscore_thresh = self.hwanalysis.zscore_thresh
self.json_nice = True
self.json_top = min(NRES_TO_DUMP, self.halving_top)
if all_zscores:
self.hwanalysis.all_zscore_comp = True
self.rounds = common.defvalkey(test_run['spec'], 'data_rounds')
tvsize = common.defvalkey(test_run['spec'], 'data_size')
# Load input polynomials
# self.load_input_poly()
logger.info('Basic settings, deg: %s, blocklen: %s, TV size: %s' %
(self.hwanalysis.deg, self.hwanalysis.blocklen, tvsize))
total_terms = int(
common.comb(self.hwanalysis.blocklen, self.hwanalysis.deg, True))
logger.info('Initializing test')
time_test_start = time.time()
self.hwanalysis.init()
# Process input object
iobj = None
if data_file:
iobj = common.FileInputObject(data_file)
elif bin_data:
iobj = common.BinaryInputObject(bin_data)
else:
iobj = common.StdinInputObject('stdin')
size = iobj.size()
logger.info('Testing input object: %s, size: %d kB' %
(iobj, size / 1024.0))
# size smaller than TV? Adapt tv then
tvsize = self.adjust_tvsize(tvsize, size)
self.hwanalysis.reset()
logger.info(
'BlockLength: %d, deg: %d, terms: %d' %
(self.hwanalysis.blocklen, self.hwanalysis.deg, total_terms))
jscres = []
with iobj:
self.analyze_iobj(iobj, 0, tvsize, jscres)
data_hash = iobj.sha1.hexdigest()
logger.info('Finished processing %s ' % iobj)
logger.info('Data read %s ' % iobj.data_read)
logger.info('Read data hash %s ' % data_hash)
# All zscore list for statistical processing / theory check
if all_zscores and res_file:
self.all_zscore_process(res_file, self.hwanalysis)
return
# RESULT process...
total_results = len(
self.hwanalysis.last_res) if self.hwanalysis.last_res else 0
best_dists = self.hwanalysis.last_res[0:min(
NRES_TO_DUMP, total_results)] if self.hwanalysis.last_res else None
halving_pvals_ok = False
best_dist_hlv_zscore = None
if self.do_halving and len(jscres) > 1 and 'halvings' in jscres[
1] and jscres[1]['halvings']:
halving_pvals_ok = True
# Re-sort best distinguishers by the halving ordering
sorder = self.build_poly_sort_index(
[common.jsunwrap(x['poly']) for x in jscres[1]['halvings']])
best_dists.sort(key=lambda x: sorder[common.immutable_poly(
common.jsunwrap(x.poly))])
# Dists from halving
try:
dists1s = sorted(jscres[1]['dists'],
key=lambda x: sorder[common.immutable_poly(
common.jsunwrap(x['poly']))])
best_dist_hlv_zscore = dists1s[0]['zscore']
except Exception as e:
logger.warning('Exception best_dist_hlv_zscore: %s' % (e, ),
exc_info=e)
# Add pvalue from the halving to the best distingushers
mrange = min(len(jscres[1]['halvings']), len(best_dists))
best_dists = [
(list(best_dists[ix]) + [jscres[1]['halvings'][ix]['pval']])
for ix in range(mrange)
]
best_dists_json = [NoIndent(x) for x in best_dists
] if best_dists is not None else None
jsres = collections.OrderedDict()
if best_dists:
jsres['best_zscore'] = best_dists[0][4] # .zscore
jsres['best_poly'] = NoIndent(best_dists[0][0]) # .poly
if halving_pvals_ok:
jsres['best_pval'] = jscres[1]['halvings'][0]['pval']
jsres['best_zscore_hlv'] = best_dist_hlv_zscore
for ix, rr in enumerate(jscres):
if 'dists' in rr:
rr['dists'] = [
NoIndent(common.jsunwrap(x)) for x in rr['dists']
]
if 'halvings' in rr:
rr['halvings'] = [
NoIndent(common.jsunwrap(x)) for x in rr['halvings']
]
jsres['blocklen'] = self.hwanalysis.blocklen
jsres['degree'] = self.hwanalysis.deg
jsres['comb_degree'] = self.hwanalysis.top_comb
jsres['top_k'] = self.top_k
jsres['all_deg'] = self.all_deg
jsres['time_elapsed'] = time.time() - time_test_start
jsres['time_data_read'] = self.timer_data_read.total()
jsres['time_data_bins'] = self.timer_data_bins.total()
jsres['time_process'] = self.timer_process.total()
jsres['data_hash'] = data_hash
jsres['data_read'] = iobj.data_read
jsres['generator'] = self.config_js
jsres['best_dists'] = best_dists_json
jsres['config'] = config
jsres['booltest_res'] = jscres
if self.dump_cpu_info:
jsres['hostname'] = misc.try_get_hostname()
jsres['cpu_pcnt_load_before'] = cpu_pcnt_load_before
jsres['cpu_load_before'] = cpu_load_before
jsres['cpu_pcnt_load_after'] = misc.try_get_cpu_percent()
jsres['cpu_load_after'] = misc.try_get_cpu_load()
jsres['cpu'] = misc.try_get_cpu_info()
if res_file:
with open(res_file, 'w+') as fh:
fh.write(common.json_dumps(jsres, indent=2))
misc.try_chmod_gr(res_file)
return common.jsunwrap(jsres)
def work(self):
"""
Main entry point - data processing
:return:
"""
config = None
with open(self.args.config_file) as fh:
config = json.load(fh)
hw_cfg = config['hwanalysis']
test_run = config['config']
data_file = test_run['spec']['data_file']
skip_finished = common.defvalkey(config, 'skip_finished', False)
res_file = common.defvalkey(config, 'res_file')
if skip_finished and res_file and self.check_res_file(res_file):
logger.info('Already computed in %s' % res_file)
return
hwanalysis = HWAnalysis()
hwanalysis.from_json(hw_cfg)
rounds = None
size_mb = test_run['spec']['data_size'] / 1024 / 1024
tvsize = test_run['spec']['data_size']
# Load input polynomials
# self.load_input_poly()
logger.info('Basic settings, deg: %s, blocklen: %s, TV size: %s' %
(hwanalysis.deg, hwanalysis.blocklen, tvsize))
total_terms = int(
scipy.misc.comb(hwanalysis.blocklen, hwanalysis.deg, True))
logger.info('Initializing test')
time_test_start = time.time()
hwanalysis.init()
# Process input object
iobj = common.FileInputObject(
data_file) if data_file else common.StdinInputObject('stdin')
size = iobj.size()
logger.info('Testing input object: %s, size: %d kB' %
(iobj, size / 1024.0))
# size smaller than TV? Adapt tv then
if size >= 0 and size < tvsize:
logger.info('File size is smaller than TV, updating TV to %d' %
size)
tvsize = size
if tvsize * 8 % hwanalysis.blocklen != 0:
rem = tvsize * 8 % hwanalysis.blocklen
logger.warning('Input data size not aligned to the block size. '
'Input bytes: %d, block bits: %d, rem: %d' %
(tvsize, hwanalysis.blocklen, rem))
tvsize -= rem // 8
logger.info('Updating TV to %d' % tvsize)
hwanalysis.reset()
logger.info('BlockLength: %d, deg: %d, terms: %d' %
(hwanalysis.blocklen, hwanalysis.deg, total_terms))
with iobj:
data_read = 0
cur_round = 0
while size < 0 or data_read < size:
if rounds is not None and cur_round > rounds:
break
data = iobj.read(tvsize)
if (len(data) * 8 % hwanalysis.blocklen) != 0:
logger.warning(
'Not aligned block read, terminating. Data: %s bits remainder: %s'
% (len(data) * 8, hwanalysis.blocklen))
break
bits = common.to_bitarray(data)
if len(bits) == 0:
logger.info('File read completely')
break
logger.info(
'Pre-computing with TV, deg: %d, blocklen: %04d, tvsize: %08d = %8.2f kB = %8.2f MB, '
'round: %d, avail: %d' %
(hwanalysis.deg, hwanalysis.blocklen, tvsize, tvsize /
1024.0, tvsize / 1024.0 / 1024.0, cur_round, len(bits)))
hwanalysis.proces_chunk(bits, None)
cur_round += 1
pass
# RESULT process...
total_results = len(hwanalysis.last_res) if hwanalysis.last_res else 0
best_dists = hwanalysis.last_res[
0:min(128, total_results)] if hwanalysis.last_res else None
data_hash = iobj.sha1.hexdigest()
jsres = collections.OrderedDict()
if best_dists:
jsres['best_zscore'] = best_dists[0].zscore
jsres['best_poly'] = best_dists[0].poly
jsres['blocklen'] = hwanalysis.blocklen
jsres['degree'] = hwanalysis.deg
jsres['comb_degree'] = hwanalysis.top_comb
jsres['top_k'] = self.top_k
jsres['all_deg'] = self.all_deg
jsres['time_elapsed'] = time.time() - time_test_start
jsres['data_hash'] = data_hash
jsres['data_read'] = iobj.data_read
jsres['generator'] = self.config_js
jsres['best_dists'] = best_dists
jsres['config'] = config
res_file_path = config['res_file']
with open(res_file_path, 'w+') as fh:
fh.write(common.json_dumps(jsres, indent=2))
logger.info('Finished processing %s ' % iobj)
logger.info('Data read %s ' % iobj.data_read)
logger.info('Read data hash %s ' % data_hash)
return jsres