def wait_on(self, outputs, todo, sleep=1):
"""Wait for jobs in ``todo`` to return and store results in ``outputs``.
:param outputs: store results here
:param todo: these are running jobs
:param sleep: seconds to sleep before checking if new results are availabl.
"""
fmtstr = self._majorminor_format_str()
while todo:
collect = [(tag, res) for (tag, res) in todo if res.ready()]
for tag, res in collect:
major, minor = tag
try:
res = res.get()
if major not in outputs:
outputs[major] = []
outputs[major].append((minor, res))
self.logger.debug(fmtstr %
(major, minor, pretty_dict(res.data)))
if self.pickle_jar is not None:
Conductor.dump(self.outputs, self.pickle_jar)
except ReductionError:
self.logger.debug("ReductionError for %s(%s)." %
(major, minor))
todo = todo.difference(collect)
time.sleep(sleep)
return outputs
def log_averages(self, tags, outputs):
"""
Log average values for all entries tagged as ``tags`` in ``outputs``.
"""
fmtstr = self._majorminor_format_str()
avg = OrderedDict()
for major, minor in tags:
if major in avg:
continue
avg[major] = OrderedDict()
n = len(outputs[major])
for minor, output in outputs[major]:
for k, v in output.data.items():
avg[major][k] = avg[major].get(k, 0.0) + float(v) / n
self.logger.info(fmtstr % (major, "avg", pretty_dict(avg[major])))
def exit(self, **kwds):
"""
When the label is a tour then the status is printed if verbosity > 0.
"""
node = self.current
label = node.label
if label[0] == "tour":
data = basis_quality([2 ** (2 * r_) for r_ in self.instance.r])
for k, v in data.items():
if k == "/":
node.data[k] = Accumulator(v, repr="max")
else:
node.data[k] = Accumulator(v, repr="min")
if self.verbosity and label[0] == "tour":
report = OrderedDict()
report["i"] = label[1]
report["#enum"] = node.sum("#enum")
report["r_0"] = node["r_0"]
report["/"] = node["/"]
print(pretty_dict(report))
self.current = self.current.parent
def simulate(r, param):
"""
BKZ simulation algorithm as proposed by Chen and Nguyen in "BKZ 2.0: Better Lattice Security
Estimates". Returns the reduced squared norms of the GSO vectors of the basis and the number of
BKZ tours simulated. This version terminates when no substantial progress is made anymore or at
most ``max_loops`` tours were simulated. If no ``max_loops`` is given, at most ``d`` tours are
performed, where ``d`` is the dimension of the lattice.
:param r: squared norms of the GSO vectors of the basis.
:param param: BKZ parameters
EXAMPLE:
>>> from fpylll import IntegerMatrix, GSO, LLL, FPLLL, BKZ
>>> FPLLL.set_random_seed(1337)
>>> A = LLL.reduction(IntegerMatrix.random(100, "qary", bits=30, k=50))
>>> M = GSO.Mat(A)
>>> from fpylll.tools.bkz_simulator import simulate
>>> _ = simulate(M, BKZ.Param(block_size=40, max_loops=4, flags=BKZ.VERBOSE))
{"i": 0, "r_0": 2^33.3, "r_0/gh": 6.110565, "rhf": 1.018340, "/": -0.07013, "hv/hv": 2.424131}
{"i": 1, "r_0": 2^32.7, "r_0/gh": 4.018330, "rhf": 1.016208, "/": -0.06161, "hv/hv": 2.156298}
{"i": 2, "r_0": 2^32.3, "r_0/gh": 2.973172, "rhf": 1.014679, "/": -0.05745, "hv/hv": 2.047014}
{"i": 3, "r_0": 2^32.1, "r_0/gh": 2.583479, "rhf": 1.013966, "/": -0.05560, "hv/hv": 2.000296}
"""
if isinstance(r, IntegerMatrix):
r = GSO.Mat(r)
if isinstance(r, MatGSO):
r.update_gso()
r = r.r()
d = len(r)
# code uses log2 of norms, FPLLL uses squared norms
r = list(map(lambda x: log(x, 2) / 2., r))
r1 = copy(r)
r2 = copy(r)
c = [rk[-i] - sum(rk[-i:]) / i for i in range(1, 46)]
c += [(lgamma(beta / 2. + 1) * (1. / beta) - log(sqrt(pi))) / log(2.)
for beta in range(46, param.block_size + 1)]
if param.max_loops:
max_loops = param.max_loops
else:
max_loops = d
for i in range(max_loops):
phi = True
for k in range(d - min(45, param.block_size)):
beta = min(param.block_size, d - k)
f = k + beta
logV = sum(r1[:f]) - sum(r2[:k])
lma = logV / beta + c[beta - 1]
if phi:
if lma < r1[k]:
r2[k] = lma
phi = False
else:
r2[k] = lma
# early termination
if phi or r1 == r2:
break
else:
beta = min(45, param.block_size)
logV = sum(r1) - sum(r2[:-beta])
if param.block_size < 45:
tmp = sum(rk[-param.block_size:]) / param.block_size
rk1 = [r_ - tmp for r_ in rk[-param.block_size:]]
else:
rk1 = rk
for k, r in zip(range(d - beta, d), rk1):
r2[k] = logV / beta + r
r1 = copy(r2)
if param.flags & BKZ.VERBOSE:
r = OrderedDict()
r["i"] = i
for k, v in basis_quality(list(map(lambda x: 2.**(2 * x),
r1))).items():
r[k] = v
print(pretty_dict(r))
r1 = list(map(lambda x: 2.**(2 * x), r1))
return r1, i + 1
def exit(self, **kwds):
"""
By default CPU and wall time are recorded. More information is recorded for sieve labels.
"""
node = self.current
node.data["cputime"] += process_time()
node.data["walltime"] += time.time()
self.instance.M.update_gso()
if self.is_sieve_node(node.label):
if isinstance(self.instance, Siever):
instance = self.instance
else:
instance = self.instance.sieve
node.data["|db|"] = Accumulator(
len(instance), repr="max") + node.data.get("|db|", None)
# determine the type of sieve:
# idstring should be among SieveTreeTraces.recognized_sieves or "all".
# This is used to look up what statistics to include in Siever.all_statistics
if isinstance(node.label, str):
idstring = node.label
elif isinstance(node.label, tuple):
idstring = node.label[0]
else:
idstring = "all"
logging.warning("Unrecognized algorithm in Tracer")
for key in Siever.all_statistics:
# Siever.all_statistics[key][3] is a list of algorithms for which the statistic
# indexed by key is meaningful instance.get_stat(key) will return None if support for
# the statistics was not compiled in Siever.all_statistics[key][1] is a short string
# that identifies the statistic
if ((idstring == "all") or
(idstring in Siever.all_statistics[key][3])) and (
instance.get_stat(key) is not None):
if (len(Siever.all_statistics[key]) <= 4):
node.data[Siever.all_statistics[key][1]] = Accumulator(
0, repr="sum")
else:
node.data[Siever.all_statistics[key][1]] = Accumulator(
0, repr=Siever.all_statistics[key][4])
node.data[Siever.all_statistics[key][1]] += node.data.get(
Siever.all_statistics[key][1], None)
try:
i, length, v = (instance.best_lifts())[0]
if i == 0:
node.data["|v|"] = length
else:
self.instance.update_gso(0, self.instance.full_n)
node.data["|v|"] = self.instance.M.get_r(0, 0)
except (IndexError, AttributeError):
node.data["|v|"] = None
data = basis_quality(self.instance.M)
for k, v in data.items():
if k == "/":
node.data[k] = Accumulator(v, repr="max")
else:
node.data[k] = Accumulator(v, repr="min")
if kwds.get("dump_gso", node.level <= 1):
node.data["r"] = self.instance.M.r()
verbose_labels = ["tour", "prog_tour"]
if self.verbosity and node.label[0] in verbose_labels:
report = OrderedDict()
report["i"] = node.label[1]
report["cputime"] = node["cputime"]
report["walltime"] = node["walltime"]
try:
report["preproc"] = node.find("preprocessing", True)["cputime"]
except KeyError:
pass
try:
report["svp"] = node.find("sieve", True)["cputime"]
# TODO: re-implement
# report["sieve sat"] = node.find("sieve", True)["saturation"]
except KeyError:
pass
report["r_0"] = node["r_0"]
report["/"] = node["/"]
print(pretty_dict(report))
self.current = self.current.parent
return self.trace
def simulate_prob(r, param, prng_seed=0xdeadbeef):
"""
BKZ simulation algorithm as proposed by Bai and Stehlé and Wen in "Measuring, simulating and
exploiting the head concavity phenomenon in BKZ". Returns the reduced squared norms of the
GSO vectors of the basis and the number of BKZ tours simulated. This version terminates when
no substantial progress is made anymore or at most ``max_loops`` tours were simulated.
If no ``max_loops`` is given, at most ``d`` tours are performed, where ``d`` is the dimension
of the lattice.
:param r: squared norms of the GSO vectors of the basis.
:param param: BKZ parameters
EXAMPLE:
>>> from fpylll import IntegerMatrix, GSO, LLL, FPLLL, BKZ
>>> FPLLL.set_random_seed(1337)
>>> A = LLL.reduction(IntegerMatrix.random(100, "qary", bits=30, k=50))
>>> M = GSO.Mat(A)
>>> from fpylll.tools.bkz_simulator import simulate_prob
>>> _ = simulate_prob(M, BKZ.Param(block_size=40, max_loops=4, flags=BKZ.VERBOSE))
{"i": 0, "r_0": 2^33.1, "r_0/gh": 5.193166, "rhf": 1.017512, "/": -0.07022, "hv/hv": 2.428125}
{"i": 1, "r_0": 2^32.7, "r_0/gh": 3.997766, "rhf": 1.016182, "/": -0.06214, "hv/hv": 2.168460}
{"i": 2, "r_0": 2^32.3, "r_0/gh": 3.020156, "rhf": 1.014759, "/": -0.05808, "hv/hv": 2.059562}
{"i": 3, "r_0": 2^32.2, "r_0/gh": 2.783102, "rhf": 1.014344, "/": -0.05603, "hv/hv": 2.013191}
"""
if param.block_size <= 2:
raise ValueError("The BSW18 simulator requires block size >= 3.")
# fix PRNG seed
random.seed(prng_seed if prng_seed else FPLLL.randint(0, 2**32 - 1))
r = _extract_log_norms(r)
d = len(r)
r1 = copy(r)
r2 = copy(r)
c = [rk[-j] - sum(rk[-j:]) / j for j in range(1, 46)]
c += [(lgamma(beta / 2.0 + 1) * (1.0 / beta) - log(sqrt(pi))) / log(2.0)
for beta in range(46, param.block_size + 1)]
if param.max_loops:
N = param.max_loops
else:
N = d
t0 = [True for _ in range(d)]
for i in range(N):
t1 = [False for _ in range(d)]
for k in range(d - min(45, param.block_size)):
beta = min(param.block_size, d - k)
f = k + beta
phi = False
for kp in range(k, f):
phi |= t0[kp]
logV = sum(r1[:f]) - sum(r2[:k])
if phi:
X = random.expovariate(.5)
lma = (log(X, 2) + logV) / beta + c[beta - 1]
if lma < r1[k]:
r2[k] = lma
r2[k + 1] = r1[k] + log(sqrt(1 - 1. / beta), 2)
dec = (r1[k] - lma) + (r1[k + 1] - r2[k + 1])
for j in range(k + 2, f):
r2[j] = r1[j] + dec / (beta - 2.)
t1[j] = True
phi = False
for j in range(k, f):
r1[j] = r2[j]
# early termination
if True not in t1:
break
# last block
beta = min(45, param.block_size)
logV = sum(r1) - sum(r2[:-beta])
if param.block_size < 45:
rk1 = normalize_GSO_unitary(rk[-beta:])
else:
rk1 = rk
K = range(d - beta, d)
for k, r in zip(K, rk1):
r2[k] = logV / beta + r
t1[k] = True
# early termination
if (r1 == r2):
break
r1 = copy(r2)
t0 = copy(t1)
if param.flags & BKZ.VERBOSE:
r = OrderedDict()
r["i"] = i
for k, v in basis_quality(list(map(lambda x: 2.0**(2 * x),
r1))).items():
r[k] = v
print(pretty_dict(r))
r1 = list(map(lambda x: 2.0**(2 * x), r1))
return r1, i + 1
def __call__(self, jobs, current=None):
"""
Call ``jobs`` in parallel.
The parameter jobs is a list with the following format. Each entry is one of the following:
- a tuple ``((major, minor), (BKZ, A, block_size, tours, progressive_step_size))``,
where ``major`` and ``minor`` are arbitrary hashable tags and the rest are valid
inputs to ``play``.
- A list with elements of the same format as above.
Entries at the same level are considered to be a group. All jobs in the same group go into
the same execution pool. At the end of the execution of a group the average across all
``minor`` tags of a ``major`` tag are shown.
.. note :: Recursive jobs, i.e. those in a sub-list are run first, this is an
implementation artefact. Typically, we don't expect jobs and lists of jobs to be mixed at
the same level, though, this is supported.
"""
inputs = OrderedDict()
if current is None:
current = self.outputs
# filter out sub-jobs that should be grouped and call recursively
for tag, job in jobs:
if isinstance(job[0], (list, tuple)):
self.logger.info("")
self.logger.info("# %s (size: %d) #" % (tag, len(job)))
current[tag] = OrderedDict()
self(job, current=current[tag])
else:
major, minor = tag
self._update_strlens(major, minor)
if major not in current:
current[major] = list()
inputs[tag] = job
self.logger.debug("")
# base case
if self.threads > 1:
todo = set()
for tag in inputs:
todo.add((tag, self.pool.apply_async(play, inputs[tag])))
current = self.wait_on(current, todo)
else:
fmtstr = self._majorminor_format_str()
for tag in inputs:
major, minor = tag
try:
res = play(*inputs[tag])
current[major].append((minor, res))
self.logger.debug(fmtstr %
(major, minor, pretty_dict(res.data)))
if self.pickle_jar is not None:
Conductor.dump(self.outputs, self.pickle_jar)
except ReductionError:
self.logger.debug("ReductionError for %s(%s)." %
(major, minor))
self.logger.debug("")
# print averages per major tag
self.log_averages(inputs.keys(), current)
if self.pickle_jar is not None:
Conductor.dump(self.outputs, self.pickle_jar)
return self.outputs
def __call__(self, seed, threads=2, samples=2, tours=1):
"""
:param seed: A random seed, each matrix will be created with seed increased by one
:param threads: number of threads to use
:param samples: number of reductions to perform
:param tours: number of BKZ tours to run
"""
logger = logging.getLogger("compare")
results = OrderedDict()
fmtstring = " %%%ds" % max(
[len(BKZ_.__name__) for BKZ_ in self.classes])
for dimension in self.dimensions:
results[dimension] = OrderedDict()
for block_size in self.block_sizes:
seed_ = seed
if dimension < block_size:
continue
L = OrderedDict([(BKZ_.__name__, OrderedDict())
for BKZ_ in self.classes])
logger.info("dimension: %3d, block_size: %2d" %
(dimension, block_size))
tasks = []
matrixf = self.matrixf(dimension=dimension,
block_size=block_size)
for i in range(samples):
set_random_seed(seed_)
A = IntegerMatrix.random(dimension, **matrixf)
for BKZ_ in self.classes:
args = (BKZ_, A, block_size, tours,
self.progressive_step_size)
tasks.append(((seed_, BKZ_), args))
seed_ += 1
if threads > 1:
pool = Pool(processes=threads)
tasks = dict([(key, pool.apply_async(bkz_call, args_))
for key, args_ in tasks])
pool.close()
while tasks:
ready = [key for key in tasks if tasks[key].ready()]
for key in ready:
seed_, BKZ_ = key
try:
trace_ = tasks[key].get()
L[BKZ_.__name__][seed_] = trace_
logger.debug(fmtstring % (BKZ_.__name__) +
" 0x%08x %s" %
(seed_, pretty_dict(trace_.data)))
except ReductionError:
logger.debug(
"ReductionError in %s with seed 0x%08x" %
(BKZ_.__name__, seed_))
del tasks[key]
time.sleep(1)
else:
for key, args_ in tasks:
seed_, BKZ_ = key
try:
trace_ = apply(bkz_call, args_)
L[BKZ_.__name__][seed_] = trace_
logger.debug(fmtstring % (BKZ_.__name__) +
" 0x%08x %s" %
(seed_, pretty_dict(trace_.data)))
except ReductionError:
logger.debug(
"ReductionError in %s with seed 0x%08x" %
(BKZ_.__name__, seed_))
logger.debug("")
for name, vals in L.items():
if vals:
vals = OrderedDict(
zip(
vals.items()[0][1].data,
zip(*
[d[1].data.values()
for d in vals.items()])))
vals = OrderedDict((k, float(sum(v)) / len(v))
for k, v in vals.items())
logger.info(fmtstring % (name) + " average %s" %
(pretty_dict(vals)))
logger.info("")
results[dimension][block_size] = L
self.write_log(results)
return results