def compute_selected_method(button):
args = []
for i in self.inputs.children:
try:
arg = i.value or i.placeholder
evaled_arg = eval_in_main(arg)
if not arg:
self.output.value = to_html("Argument '%s' is empty!" % i.description)
return
args.append(evaled_arg)
except:
self.output.value = to_html("Could not evaluate argument '%s'" % i.description)
return
try:
if AlarmInterrupt:
alarm(TIMEOUT)
out = self.selected_menu_value.member(obj, *args)
if AlarmInterrupt:
cancel_alarm()
except AlarmInterrupt:
self.output.value = to_html("Timeout!")
except Exception as e:
self.output.value = to_html(e)
return
self.output.value = to_html(out)
def report(F, title, systems=['sage', 'magma'], **kwds):
"""
Run benchmarks with default arguments for each function in the list F.
INPUT:
- ``F`` - a list of callables used for benchmarking
- ``title`` - a string describing this report
- ``systems`` - a list of systems (supported entries are 'sage' and 'magma')
- ``**kwds`` - keyword arguments passed to all functions in ``F``
EXAMPLES::
sage: import sage.matrix.benchmark as b
sage: print("starting"); import sys; sys.stdout.flush(); b.report([b.det_ZZ], 'Test', systems=['sage'])
starting...
======================================================================
Test
======================================================================
...
======================================================================
"""
import os
if len(systems) > 2:
raise NotImplementedError("at most two systems ('sage' or 'magma')")
print('=' * 70)
print(' ' * 10 + title)
print('=' * 70)
os.system('uname -a')
print('\n')
for f in F:
print("-" * 70)
print(f.__doc__.strip())
print(('%15s' * len(systems)) % tuple(systems))
w = []
for s in systems:
alarm(timeout)
try:
t = f(system=s, **kwds)
except AlarmInterrupt:
t = -timeout
cancel_alarm()
w.append(float(t))
if len(w) > 1:
if w[1] == 0:
w.append(0.0)
else:
w.append(w[0] / w[1])
w = tuple(w)
print(('%15.3f' * len(w)) % w)
print('=' * 70)
def report(F, title, systems = ['sage', 'magma'], **kwds):
"""
Run benchmarks with default arguments for each function in the list F.
INPUT:
- ``F`` - a list of callables used for benchmarking
- ``title`` - a string describing this report
- ``systems`` - a list of systems (supported entries are 'sage' and 'magma')
- ``**kwds`` - keyword arguments passed to all functions in ``F``
EXAMPLES::
sage: import sage.matrix.benchmark as b
sage: print "starting"; import sys; sys.stdout.flush(); b.report([b.det_ZZ], 'Test', systems=['sage'])
starting...
======================================================================
Test
======================================================================
...
======================================================================
"""
import os
if len(systems) > 2:
raise NotImplementedError("at most two systems ('sage' or 'magma')")
print '='*70
print ' '*10 + title
print '='*70
os.system('uname -a')
print '\n'
for f in F:
print "-"*70
print f.__doc__.strip()
print ('%15s'*len(systems))%tuple(systems)
w = []
for s in systems:
alarm(timeout)
try:
t = f(system=s, **kwds)
except AlarmInterrupt:
t = -timeout
cancel_alarm()
w.append(float(t))
if len(w) > 1:
if w[1] == 0:
w.append(0.0)
else:
w.append(w[0]/w[1])
w = tuple(w)
print ('%15.3f'*len(w))%w
print '='*70
def benchmark(pbound=[3, 2**10],
nbound=[3, 2**8],
cbound=[1, Infinity],
obound=[1, Infinity],
loops=10,
tloop=Infinity,
tmax=Infinity,
prime=False,
even=False,
check=False,
fname=None,
write=False,
overwrite=False,
verbose=True,
skip_pari=False,
skip_magma=False,
skip_rains=False,
skip_kummer=False):
if write:
mode = 'w' if overwrite else 'a'
f = open(fname, mode, 0)
else:
f = sys.stdout
pmin, pmax = pbound
nmin, nmax = nbound
omin, omax = obound
cmin, cmax = cbound
M = Magma()
for p in xrange(pmin, pmax):
p = ZZ(p)
if not p.is_prime():
continue
for n in xrange(nmin, nmax):
n = ZZ(n)
if (prime == 1 and not is_prime(n)) or (prime == 2
and not is_prime_power(n)):
continue
if n < 2:
continue
if n % p == 0:
continue
if (not even) and (n % 2 == 0):
continue
o, G = find_root_order(p, [n, n], n, verbose=False)
m = G[0][0].parent().order()
c = Mod(p, n).multiplicative_order()
if verbose:
sys.stdout.write("\r" + " " * 79)
print("\rp = {}, n = {}, (o = {}, c = {})".format(p, n, o, c))
if verbose:
t = mytime()
sys.stdout.write("Constructing fields ({})".format(
time.strftime("%c")))
sys.stdout.flush()
q = p**n
k = GF(q, name='z')
k_rand = GF(q, modulus='random', name='z')
k_flint = GF_flint(p, k.modulus(), name='z')
if verbose > 1:
sys.stdout.write("\ntotal: {}s\n".format(mytime(t)))
sys.stdout.flush()
# Magma
if verbose:
sys.stdout.write("\r" + " " * 79)
sys.stdout.write("\rMagma ({})".format(time.strftime("%c")))
sys.stdout.flush()
tloops = 0
for l in xrange(loops):
if skip_magma:
break
if (o > omax) or (o == p):
break
# let's assume that launching a new Magma instance is cheaper
# than computing random irreducible polynomials
try:
M._start()
except OSError as err:
# but it can also cause fork issues...
# let's accept this
# and fail as the situation will only worsen
# unless it is "just" a memory issue
# which should be mitigated by COW but is not
#print(err)
if err.errno == errno.ENOMEM:
break
else:
raise
try:
k_magma = M(k)
k_rand_magma = M(k_rand)
if tloop is not Infinity:
alarm(tloop)
t = mytime()
k_magma.Embed(k_rand_magma, nvals=0)
#M._eval_line("Embed(k_magma, k_rand_magma);", wait_for_prompt=False)
tloops += mytime(t)
except TypeError:
# sage/magma interface sometimes gets confused
pass
except (KeyboardInterrupt, AlarmInterrupt):
# sage interface eats KeyboardInterrupt
# and AlarmInterrupt derives from it
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
M.quit()
# sage pexpect interface leaves zombies around
try:
while os.waitpid(-1, os.WNOHANG)[0]:
pass
# but sometimes every child is already buried
# and we get an ECHILD error...
except OSError:
pass
if tloops > tmax:
break
tmagma = tloops / (l + 1)
if verbose > 1:
sys.stdout.write("\ntotal: {}s, per loop: {}s\n".format(
tloops, tloops / (l + 1)))
sys.stdout.flush()
# Rains algorithms
if verbose:
sys.stdout.write("\r" + " " * 79)
sys.stdout.write("\rCyclotomic Rains ({})".format(
time.strftime("%c")))
sys.stdout.flush()
trains = []
tloops = 0
for l in xrange(loops):
if skip_rains:
break
if (o > omax) or (o == p):
break
try:
if tloop is not Infinity:
alarm(tloop)
t = mytime()
a, b = find_gens_cyclorains(k_flint,
k_flint,
use_lucas=False)
tloops += mytime(t)
except (KeyboardInterrupt, AlarmInterrupt):
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
if check and (l == 0 or check > 1):
g = a.minpoly()
if g.degree() != n:
raise RuntimeError("wrong degree")
if g != b.minpoly():
raise RuntimeError("different minpolys")
if tloops > tmax:
break
trains.append(tloops / (l + 1))
if verbose > 1:
sys.stdout.write("\ntotal: {}s, per loop: {}s\n".format(
tloops, tloops / (l + 1)))
sys.stdout.flush()
# Conic Rains
if verbose:
sys.stdout.write("\r" + " " * 79)
sys.stdout.write("\rConic Rains ({})".format(
time.strftime("%c")))
sys.stdout.flush()
tloops = 0
for l in xrange(loops):
if skip_rains:
break
if (o != 2) or (o > omax) or (o == p):
break
try:
if tloop is not Infinity:
alarm(tloop)
t = mytime()
a, b = find_gens_cyclorains(k_flint,
k_flint,
use_lucas=True)
tloops += mytime(t)
except (KeyboardInterrupt, AlarmInterrupt):
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
if check and (l == 0 or check > 1):
g = a.minpoly()
if g.degree() != n:
raise RuntimeError("wrong degree")
if g != b.minpoly():
raise RuntimeError("different minpolys")
if tloops > tmax:
break
trains.append(tloops / (l + 1))
if verbose > 1:
sys.stdout.write("\ntotal: {}s, per loop: {}s\n".format(
tloops, tloops / (l + 1)))
sys.stdout.flush()
# Elliptic Rains
if verbose:
sys.stdout.write("\r" + " " * 79)
sys.stdout.write("\rElliptic Rains ({})".format(
time.strftime("%c")))
sys.stdout.flush()
tloops = 0
for l in xrange(loops):
if skip_rains:
break
try:
if tloop is not Infinity:
alarm(tloop)
t = mytime()
a, b = find_gens_ellrains(k_flint, k_flint)
tloops += mytime(t)
except RuntimeError:
# sometimes no suitable elliptic curve exists
pass
except (KeyboardInterrupt, AlarmInterrupt):
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
if check and (l == 0 or check > 1):
g = a.minpoly()
if g.degree() != n:
raise RuntimeError("wrong degree")
if g != b.minpoly():
raise RuntimeError("different minpolys")
if tloops > tmax:
break
trains.append(tloops / (l + 1))
if verbose > 1:
sys.stdout.write("\ntotal: {}s, per loop: {}s\n".format(
tloops, tloops / (l + 1)))
sys.stdout.flush()
# PARI/GP
if verbose:
sys.stdout.write("\r" + " " * 79)
sys.stdout.write("\rPARI/GP ({})".format(time.strftime("%c")))
sys.stdout.flush()
tloops = 0
for l in xrange(loops):
if skip_pari:
break
if c < cmin or c > cmax:
break
try:
if tloop is not Infinity:
alarm(tloop)
t = mytime()
a, b = find_gens_pari(k, k)
tloops += mytime(t)
except (KeyboardInterrupt, AlarmInterrupt):
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
if check and (l == 0 or check > 1):
g = a.minpoly()
if g.degree() != n:
raise RuntimeError("wrong degree")
if g != b.minpoly():
raise RuntimeError("different minpolys")
if tloops > tmax:
break
tpari = tloops / (l + 1)
# Kummer algorithms
tkummer = []
# only linalg and modcomp implemented for c==1
for i, algo in enumerate(kummer_algolist[2 * (c == 1):-2 *
(c == 1) - 1]):
if verbose:
sys.stdout.write("\r" + " " * 79)
sys.stdout.write("\rKummer {} ({})".format(
kummer_namelist[2 * (c == 1) + i],
time.strftime("%c")))
sys.stdout.flush()
tloops = 0
for l in xrange(loops):
if skip_kummer:
break
if c < cmin or c > cmax:
break
try:
if tloop is not Infinity:
alarm(tloop)
t = mytime()
a, b = find_gens_kummer(k_flint, k_flint, n, algo)
tloops += mytime(t)
except (KeyboardInterrupt, AlarmInterrupt):
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
if check and (l == 0 or check > 1):
g = a.minpoly()
if g.degree() != n:
raise RuntimeError("wrong degree")
if g != b.minpoly():
raise RuntimeError("different minpolys")
if tloops > tmax:
break
tkummer.append(tloops / (l + 1))
if verbose > 1:
sys.stdout.write("\ntotal: {}s, per loop: {}s\n".format(
tloops, tloops / (l + 1)))
sys.stdout.flush()
tkummer = 2 * (c == 1) * [0] + tkummer + 2 * (c == 1) * [0]
if verbose:
sys.stdout.write("\r")
sys.stdout.flush()
f.write(("{} {} ({}, {})" + " {}" + " {}" * len(trains) + " {}" +
" {}" * len(tkummer) + "\n").format(
p, n, o, c, *([tmagma] + trains + [tpari] + tkummer)))
if write:
f.close()
def __call__(self, f, inputs):
"""
Parallel iterator using ``fork()``.
INPUT:
- ``f`` -- a function (or more general, any callable)
- ``inputs`` -- a list of pairs ``(args, kwds)`` to be used as
arguments to ``f``, where ``args`` is a tuple and ``kwds`` is
a dictionary.
OUTPUT:
EXAMPLES::
sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
[(([10], {}), 100), (([20], {}), 400)]
sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
[(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]
TESTS:
The output of functions decorated with :func:`parallel` is read
as a pickle by the parent process. We intentionally break the
unpickling and demonstrate that this failure is handled
gracefully (the exception is put in the list instead of the
answer)::
sage: Polygen = parallel(polygen)
sage: list(Polygen([QQ]))
[(((Rational Field,), {}), x)]
sage: from sage.misc.persist import unpickle_override, register_unpickle_override
sage: register_unpickle_override('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint', Integer)
sage: L = list(Polygen([QQ]))
sage: L
[(((Rational Field,), {}),
'INVALID DATA __init__() takes at most 2 positional arguments (4 given)')]
Fix the unpickling::
sage: del unpickle_override[('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint')]
sage: list(Polygen([QQ,QQ]))
[(((Rational Field,), {}), x), (((Rational Field,), {}), x)]
"""
n = self.ncpus
v = list(inputs)
import os
import sys
import signal
from sage.misc.persist import loads
from sage.misc.temporary_file import tmp_dir
dir = tmp_dir()
timeout = self.timeout
workers = {}
try:
while v or workers:
# Spawn up to n subprocesses
while v and len(workers) < n:
v0 = v.pop(0) # Input value for the next subprocess
with ContainChildren():
pid = os.fork()
# The way fork works is that pid returns the
# nonzero pid of the subprocess for the master
# process and returns 0 for the subprocess.
if not pid:
# This is the subprocess.
self._subprocess(f, dir, *v0)
workers[pid] = WorkerData(v0)
if len(workers) > 0:
# Now wait for one subprocess to finish and report the result.
# However, wait at most the time since the oldest process started.
T = walltime()
if timeout:
oldest = min(W.starttime for W in workers.values())
alarm(max(timeout - (T - oldest), 0.1))
try:
pid = os.wait()[0]
cancel_alarm()
W = workers.pop(pid)
except AlarmInterrupt:
# Kill workers that are too old
for pid, W in workers.items():
if T - W.starttime > timeout:
if self.verbose:
print(
"Killing subprocess %s with input %s which took too long"
% (pid, W.input))
os.kill(pid, signal.SIGKILL)
W.failure = " (timed out)"
except KeyError:
# Some other process exited, not our problem...
pass
else:
# collect data from process that successfully terminated
sobj = os.path.join(dir, '%s.sobj' % pid)
try:
with open(sobj, "rb") as file:
data = file.read()
except IOError:
answer = "NO DATA" + W.failure
else:
os.unlink(sobj)
try:
answer = loads(data, compress=False)
except Exception as E:
answer = "INVALID DATA {}".format(E)
out = os.path.join(dir, '%s.out' % pid)
try:
with open(out) as file:
sys.stdout.write(file.read())
os.unlink(out)
except IOError:
pass
yield (W.input, answer)
finally:
# Send SIGKILL signal to workers that are left.
if workers:
if self.verbose:
print("Killing any remaining workers...")
sys.stdout.flush()
for pid in workers:
try:
os.kill(pid, signal.SIGKILL)
except OSError:
# If kill() failed, it is most likely because
# the process already exited.
pass
else:
try:
os.waitpid(pid, 0)
except OSError as msg:
if self.verbose:
print(msg)
# Clean up all temporary files.
rmtree(dir)
def __call__(self, f, inputs):
"""
Parallel iterator using ``fork()``.
INPUT:
- ``f`` -- a Python function that need not be pickleable or anything else!
- ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
``args`` is a tuple and ``kwds`` is a dictionary.
OUTPUT:
EXAMPLES::
sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
[(([10], {}), 100), (([20], {}), 400)]
sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
[(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]
TESTS:
The output of functions decorated with :func:parallel is read
as a pickle by the parent process. We intentionally break the
unpickling and demonstrate that this failure is handled
gracefully (an exception is displayed and an empty list is
returned)::
sage: Polygen = parallel(polygen)
sage: list(Polygen([QQ]))
[(((Rational Field,), {}), x)]
sage: from sage.structure.sage_object import unpickle_override, register_unpickle_override
sage: register_unpickle_override('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint', Integer)
sage: L = list(Polygen([QQ]))
('__init__() takes at most 2 positional arguments (4 given)', <type 'sage.rings.integer.Integer'>, (Univariate Polynomial Ring in x over Rational Field, [0, 1], False, True))
sage: L
[]
Fix the unpickling::
sage: del unpickle_override[('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint')]
sage: list(Polygen([QQ,QQ]))
[(((Rational Field,), {}), x), (((Rational Field,), {}), x)]
"""
n = self.ncpus
v = list(inputs)
import os, sys, signal
from sage.structure.sage_object import load
from sage.misc.all import tmp_dir, walltime
dir = tmp_dir()
timeout = self.timeout
workers = {}
try:
while len(v) > 0 or len(workers) > 0:
# Spawn up to n subprocesses
while len(v) > 0 and len(workers) < n:
# Subprocesses shouldn't inherit unflushed buffers (cf. #11778):
sys.stdout.flush()
sys.stderr.flush()
pid = os.fork()
# The way fork works is that pid returns the
# nonzero pid of the subprocess for the master
# process and returns 0 for the subprocess.
if pid:
# This is the parent master process.
workers[pid] = [v[0], walltime(), '']
del v[0]
else:
# This is the subprocess.
self._subprocess(f, dir, v[0])
if len(workers) > 0:
# Now wait for one subprocess to finish and report the result.
# However, wait at most the time since the oldest process started.
if timeout:
oldest = min([X[1] for X in workers.values()])
alarm(max(timeout - (walltime()-oldest), 0.1))
try:
pid = os.wait()[0]
cancel_alarm()
w = workers.pop(pid)
except AlarmInterrupt:
cancel_alarm()
# Kill workers that are too old
for pid, X in workers.iteritems():
if walltime() - X[1] > timeout:
if self.verbose:
print(
"Killing subprocess %s with input %s which took too long"
% (pid, X[0]) )
os.kill(pid, signal.SIGKILL)
X[-1] = ' (timed out)'
except KeyError:
# Some other process exited, not our problem...
pass
else:
# collect data from process that successfully terminated
sobj = os.path.join(dir, '%s.sobj'%pid)
if not os.path.exists(sobj):
X = "NO DATA" + w[-1] # the message field
else:
X = load(sobj, compress=False)
os.unlink(sobj)
out = os.path.join(dir, '%s.out'%pid)
if not os.path.exists(out):
output = "NO OUTPUT"
else:
output = open(out).read()
os.unlink(out)
if output.strip():
print output,
yield (w[0], X)
except Exception as msg:
print msg
finally:
# Clean up all temporary files.
try:
for X in os.listdir(dir):
os.unlink(os.path.join(dir, X))
os.rmdir(dir)
except OSError as msg:
if self.verbose:
print msg
# Send "kill -9" signal to workers that are left.
if len(workers) > 0:
if self.verbose:
print "Killing any remaining workers..."
sys.stdout.flush()
for pid in workers.keys():
try:
os.kill(pid, signal.SIGKILL)
os.waitpid(pid, 0)
except OSError as msg:
if self.verbose:
print msg
def __call__(self, f, inputs):
"""
Parallel iterator using ``fork()``.
INPUT:
- ``f`` -- a Python function that need not be pickleable or anything else!
- ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
``args`` is a tuple and ``kwds`` is a dictionary.
OUTPUT:
EXAMPLES::
sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
[(([10], {}), 100), (([20], {}), 400)]
sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
[(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]
TESTS:
The output of functions decorated with :func:parallel is read
as a pickle by the parent process. We intentionally break the
unpickling and demonstrate that this failure is handled
gracefully (an exception is displayed and an empty list is
returned)::
sage: Polygen = parallel(polygen)
sage: list(Polygen([QQ]))
[(((Rational Field,), {}), x)]
sage: from sage.structure.sage_object import unpickle_override, register_unpickle_override
sage: register_unpickle_override('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint', Integer)
sage: L = list(Polygen([QQ]))
('__init__() takes at most 2 positional arguments (4 given)', <type 'sage.rings.integer.Integer'>, (Univariate Polynomial Ring in x over Rational Field, [0, 1], False, True))
sage: L
[]
Fix the unpickling::
sage: del unpickle_override[('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint')]
sage: list(Polygen([QQ,QQ]))
[(((Rational Field,), {}), x), (((Rational Field,), {}), x)]
"""
n = self.ncpus
v = list(inputs)
import os, sys, signal
from sage.structure.sage_object import load
from sage.misc.all import tmp_dir, walltime
dir = tmp_dir()
timeout = self.timeout
workers = {}
try:
while len(v) > 0 or len(workers) > 0:
# Spawn up to n subprocesses
while len(v) > 0 and len(workers) < n:
# Subprocesses shouldn't inherit unflushed buffers (cf. #11778):
sys.stdout.flush()
sys.stderr.flush()
pid = os.fork()
# The way fork works is that pid returns the
# nonzero pid of the subprocess for the master
# process and returns 0 for the subprocess.
if pid:
# This is the parent master process.
workers[pid] = [v[0], walltime(), '']
del v[0]
else:
# This is the subprocess.
self._subprocess(f, dir, v[0])
if len(workers) > 0:
# Now wait for one subprocess to finish and report the result.
# However, wait at most the time since the oldest process started.
if timeout:
oldest = min([X[1] for X in workers.values()])
alarm(max(timeout - (walltime() - oldest), 0.1))
try:
pid = os.wait()[0]
cancel_alarm()
w = workers.pop(pid)
except AlarmInterrupt:
cancel_alarm()
# Kill workers that are too old
for pid, X in workers.iteritems():
if walltime() - X[1] > timeout:
if self.verbose:
print(
"Killing subprocess %s with input %s which took too long"
% (pid, X[0]))
os.kill(pid, signal.SIGKILL)
X[-1] = ' (timed out)'
except KeyError:
# Some other process exited, not our problem...
pass
else:
# collect data from process that successfully terminated
sobj = os.path.join(dir, '%s.sobj' % pid)
if not os.path.exists(sobj):
X = "NO DATA" + w[-1] # the message field
else:
X = load(sobj, compress=False)
os.unlink(sobj)
out = os.path.join(dir, '%s.out' % pid)
if not os.path.exists(out):
output = "NO OUTPUT"
else:
output = open(out).read()
os.unlink(out)
if output.strip():
print output,
yield (w[0], X)
except Exception as msg:
print msg
finally:
# Clean up all temporary files.
try:
for X in os.listdir(dir):
os.unlink(os.path.join(dir, X))
os.rmdir(dir)
except OSError as msg:
if self.verbose:
print msg
# Send "kill -9" signal to workers that are left.
if len(workers) > 0:
if self.verbose:
print "Killing any remaining workers..."
sys.stdout.flush()
for pid in workers.keys():
try:
os.kill(pid, signal.SIGKILL)
os.waitpid(pid, 0)
except OSError as msg:
if self.verbose:
print msg
def __call__(self, f, inputs):
"""
Parallel iterator using ``fork()``.
INPUT:
- ``f`` -- a function (or more general, any callable)
- ``inputs`` -- a list of pairs ``(args, kwds)`` to be used as
arguments to ``f``, where ``args`` is a tuple and ``kwds`` is
a dictionary.
OUTPUT:
EXAMPLES::
sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
[(([10], {}), 100), (([20], {}), 400)]
sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
[(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]
TESTS:
The output of functions decorated with :func:`parallel` is read
as a pickle by the parent process. We intentionally break the
unpickling and demonstrate that this failure is handled
gracefully (the exception is put in the list instead of the
answer)::
sage: Polygen = parallel(polygen)
sage: list(Polygen([QQ]))
[(((Rational Field,), {}), x)]
sage: from sage.structure.sage_object import unpickle_override, register_unpickle_override
sage: register_unpickle_override('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint', Integer)
sage: L = list(Polygen([QQ]))
sage: L
[(((Rational Field,), {}),
'INVALID DATA __init__() takes at most 2 positional arguments (4 given)')]
Fix the unpickling::
sage: del unpickle_override[('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint')]
sage: list(Polygen([QQ,QQ]))
[(((Rational Field,), {}), x), (((Rational Field,), {}), x)]
"""
n = self.ncpus
v = list(inputs)
import os, sys, signal
from sage.structure.sage_object import loads
from sage.misc.temporary_file import tmp_dir
dir = tmp_dir()
timeout = self.timeout
workers = {}
try:
while len(v) > 0 or len(workers) > 0:
# Spawn up to n subprocesses
while len(v) > 0 and len(workers) < n:
v0 = v.pop(0) # Input value for the next subprocess
with ContainChildren():
pid = os.fork()
# The way fork works is that pid returns the
# nonzero pid of the subprocess for the master
# process and returns 0 for the subprocess.
if not pid:
# This is the subprocess.
self._subprocess(f, dir, *v0)
workers[pid] = WorkerData(v0)
if len(workers) > 0:
# Now wait for one subprocess to finish and report the result.
# However, wait at most the time since the oldest process started.
T = walltime()
if timeout:
oldest = min(W.starttime for W in workers.values())
alarm(max(timeout - (T - oldest), 0.1))
try:
pid = os.wait()[0]
cancel_alarm()
W = workers.pop(pid)
except AlarmInterrupt:
# Kill workers that are too old
for pid, W in workers.items():
if T - W.starttime > timeout:
if self.verbose:
print(
"Killing subprocess %s with input %s which took too long"
% (pid, W.input) )
os.kill(pid, signal.SIGKILL)
W.failure = " (timed out)"
except KeyError:
# Some other process exited, not our problem...
pass
else:
# collect data from process that successfully terminated
sobj = os.path.join(dir, '%s.sobj'%pid)
try:
with open(sobj) as file:
data = file.read()
except IOError:
answer = "NO DATA" + W.failure
else:
os.unlink(sobj)
try:
answer = loads(data, compress=False)
except Exception as E:
answer = "INVALID DATA {}".format(E)
out = os.path.join(dir, '%s.out'%pid)
try:
with open(out) as file:
sys.stdout.write(file.read())
os.unlink(out)
except IOError:
pass
yield (W.input, answer)
finally:
# Send SIGKILL signal to workers that are left.
if workers:
if self.verbose:
print("Killing any remaining workers...")
sys.stdout.flush()
for pid in workers:
try:
os.kill(pid, signal.SIGKILL)
except OSError:
# If kill() failed, it is most likely because
# the process already exited.
pass
else:
try:
os.waitpid(pid, 0)
except OSError as msg:
if self.verbose:
print(msg)
# Clean up all temporary files.
rmtree(dir)
def benchmark(pbound = [3, 2**10], nbound = [3, 2**8], cbound = [1, Infinity], obound = [1, Infinity], loops = 10, tloop = Infinity, tmax = Infinity, prime = False, even = False, check = False, fname = None, write = False, overwrite = False, verbose = True, skip_pari = False, skip_magma = False, skip_rains = False, skip_kummer = False):
if write:
mode = 'w' if overwrite else 'a'
f = open(fname, mode, 0)
else:
f = sys.stdout
pmin, pmax = pbound
nmin, nmax = nbound
omin, omax = obound
cmin, cmax = cbound
M = Magma()
for p in xrange(pmin, pmax):
p = ZZ(p)
if not p.is_prime():
continue
for n in xrange(nmin, nmax):
n = ZZ(n)
if (prime == 1 and not is_prime(n)) or (prime == 2 and not is_prime_power(n)):
continue
if n < 2:
continue
if n % p == 0:
continue
if (not even) and (n % 2 == 0):
continue
o, G = find_root_order(p, [n, n], n, verbose=False)
m = G[0][0].parent().order()
c = Mod(p,n).multiplicative_order()
if verbose:
sys.stdout.write("\r"+" "*79)
print("\rp = {}, n = {}, (o = {}, c = {})".format(p, n, o, c))
if verbose:
t = mytime()
sys.stdout.write("Constructing fields ({})".format(time.strftime("%c")))
sys.stdout.flush()
q = p**n
k = GF(q, name='z')
k_rand = GF(q, modulus='random', name='z')
k_flint = GF_flint(p, k.modulus(), name='z')
if verbose > 1:
sys.stdout.write("\ntotal: {}s\n".format(mytime(t)))
sys.stdout.flush()
# Magma
if verbose:
sys.stdout.write("\r"+" "*79)
sys.stdout.write("\rMagma ({})".format(time.strftime("%c")))
sys.stdout.flush()
tloops = 0
for l in xrange(loops):
if skip_magma:
break
if (o > omax) or (o == p):
break
# let's assume that launching a new Magma instance is cheaper
# than computing random irreducible polynomials
try:
M._start()
except OSError as err:
# but it can also cause fork issues...
# let's accept this
# and fail as the situation will only worsen
# unless it is "just" a memory issue
# which should be mitigated by COW but is not
#print(err)
if err.errno == errno.ENOMEM:
break
else:
raise
try:
k_magma = M(k)
k_rand_magma = M(k_rand)
if tloop is not Infinity:
alarm(tloop)
t = mytime()
k_magma.Embed(k_rand_magma, nvals=0)
#M._eval_line("Embed(k_magma, k_rand_magma);", wait_for_prompt=False)
tloops += mytime(t)
except TypeError:
# sage/magma interface sometimes gets confused
pass
except (KeyboardInterrupt, AlarmInterrupt):
# sage interface eats KeyboardInterrupt
# and AlarmInterrupt derives from it
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
M.quit()
# sage pexpect interface leaves zombies around
try:
while os.waitpid(-1, os.WNOHANG)[0]:
pass
# but sometimes every child is already buried
# and we get an ECHILD error...
except OSError:
pass
if tloops > tmax:
break
tmagma = tloops / (l+1)
if verbose > 1:
sys.stdout.write("\ntotal: {}s, per loop: {}s\n".format(tloops, tloops/(l+1)))
sys.stdout.flush()
# Rains algorithms
if verbose:
sys.stdout.write("\r"+" "*79)
sys.stdout.write("\rCyclotomic Rains ({})".format(time.strftime("%c")))
sys.stdout.flush()
trains = []
tloops = 0
for l in xrange(loops):
if skip_rains:
break
if (o > omax) or (o == p):
break
try:
if tloop is not Infinity:
alarm(tloop)
t = mytime()
a, b = find_gens_cyclorains(k_flint, k_flint, use_lucas = False)
tloops += mytime(t)
except (KeyboardInterrupt, AlarmInterrupt):
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
if check and (l == 0 or check > 1):
g = a.minpoly()
if g.degree() != n:
raise RuntimeError("wrong degree")
if g != b.minpoly():
raise RuntimeError("different minpolys")
if tloops > tmax:
break
trains.append(tloops / (l+1))
if verbose > 1:
sys.stdout.write("\ntotal: {}s, per loop: {}s\n".format(tloops, tloops/(l+1)))
sys.stdout.flush()
# Conic Rains
if verbose:
sys.stdout.write("\r"+" "*79)
sys.stdout.write("\rConic Rains ({})".format(time.strftime("%c")))
sys.stdout.flush()
tloops = 0
for l in xrange(loops):
if skip_rains:
break
if (o != 2) or (o > omax) or (o == p):
break
try:
if tloop is not Infinity:
alarm(tloop)
t = mytime()
a, b = find_gens_cyclorains(k_flint, k_flint, use_lucas = True)
tloops += mytime(t)
except (KeyboardInterrupt, AlarmInterrupt):
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
if check and (l == 0 or check > 1):
g = a.minpoly()
if g.degree() != n:
raise RuntimeError("wrong degree")
if g != b.minpoly():
raise RuntimeError("different minpolys")
if tloops > tmax:
break
trains.append(tloops / (l+1))
if verbose > 1:
sys.stdout.write("\ntotal: {}s, per loop: {}s\n".format(tloops, tloops/(l+1)))
sys.stdout.flush()
# Elliptic Rains
if verbose:
sys.stdout.write("\r"+" "*79)
sys.stdout.write("\rElliptic Rains ({})".format(time.strftime("%c")))
sys.stdout.flush()
tloops = 0
for l in xrange(loops):
if skip_rains:
break
try:
if tloop is not Infinity:
alarm(tloop)
t = mytime()
a, b = find_gens_ellrains(k_flint, k_flint)
tloops += mytime(t)
except RuntimeError:
# sometimes no suitable elliptic curve exists
pass
except (KeyboardInterrupt, AlarmInterrupt):
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
if check and (l == 0 or check > 1):
g = a.minpoly()
if g.degree() != n:
raise RuntimeError("wrong degree")
if g != b.minpoly():
raise RuntimeError("different minpolys")
if tloops > tmax:
break
trains.append(tloops / (l+1))
if verbose > 1:
sys.stdout.write("\ntotal: {}s, per loop: {}s\n".format(tloops, tloops/(l+1)))
sys.stdout.flush()
# PARI/GP
if verbose:
sys.stdout.write("\r"+" "*79)
sys.stdout.write("\rPARI/GP ({})".format(time.strftime("%c")))
sys.stdout.flush()
tloops = 0
for l in xrange(loops):
if skip_pari:
break
if c < cmin or c > cmax:
break
try:
if tloop is not Infinity:
alarm(tloop)
t = mytime()
a, b = find_gens_pari(k, k)
tloops += mytime(t)
except (KeyboardInterrupt, AlarmInterrupt):
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
if check and (l == 0 or check > 1):
g = a.minpoly()
if g.degree() != n:
raise RuntimeError("wrong degree")
if g != b.minpoly():
raise RuntimeError("different minpolys")
if tloops > tmax:
break
tpari = tloops / (l+1)
# Kummer algorithms
tkummer = []
# only linalg and modcomp implemented for c==1
for i, algo in enumerate(kummer_algolist[2*(c==1):-2*(c==1)-1]):
if verbose:
sys.stdout.write("\r"+" "*79)
sys.stdout.write("\rKummer {} ({})".format(kummer_namelist[2*(c==1)+i], time.strftime("%c")))
sys.stdout.flush()
tloops = 0
for l in xrange(loops):
if skip_kummer:
break
if c < cmin or c > cmax:
break
try:
if tloop is not Infinity:
alarm(tloop)
t = mytime()
a, b = find_gens_kummer(k_flint, k_flint, n, algo)
tloops += mytime(t)
except (KeyboardInterrupt, AlarmInterrupt):
tloops = 0
break
finally:
if tloop is not Infinity:
cancel_alarm()
if check and (l == 0 or check > 1):
g = a.minpoly()
if g.degree() != n:
raise RuntimeError("wrong degree")
if g != b.minpoly():
raise RuntimeError("different minpolys")
if tloops > tmax:
break
tkummer.append(tloops / (l+1))
if verbose > 1:
sys.stdout.write("\ntotal: {}s, per loop: {}s\n".format(tloops, tloops/(l+1)))
sys.stdout.flush()
tkummer = 2*(c == 1)*[0] + tkummer + 2*(c == 1)*[0]
if verbose:
sys.stdout.write("\r")
sys.stdout.flush()
f.write(("{} {} ({}, {})" + " {}" + " {}"*len(trains) + " {}" + " {}"*len(tkummer)+"\n").format(p, n, o, c, *([tmagma] + trains + [tpari] + tkummer)))
if write:
f.close()