def g(N):
from sage.misc.misc import alarm, cancel_alarm
alarm(maxtime)
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
engine = create_engine('postgresql://[email protected]:6432/mrc2', echo=False)
s = sessionmaker(bind=engine)()
t = cputime()
x,y,z = ideal_to_tuple(N)
q = s.query(Space).filter(Space.x==x).filter(Space.y==y).filter(Space.z==z)
if q.count() > 0:
M = q.one()
if M.number_of_rational_newforms == len(M.rational_newforms):
return "Already done with level %s (norm = %s)"%(N, N.norm())
H = M.hmf()
else:
H = QuaternionicModule(N)
M = store_space(s, H)
V, rational_oldform_dimension = H.dual_rational_newforms(verb=verb)
for vdual, aplist in V:
f = RationalNewform()
for p, ap in aplist:
f.store_eigenvalue(p, ap)
M.rational_newforms.append(f)
f.dual_vector = ns_str(canonically_scale(vdual))
M.number_of_rational_newforms = len(V)
M.rational_oldform_dimension = int(rational_oldform_dimension)
M.time_to_compute_newforms = cputime(t)
s.commit()
cancel_alarm()
return N.norm(), cputime(t), len(V)
def find_next_available_port(start, max_tries=100, verbose=False):
"""
Find the next port that is available to be used, where available means that
currently trying to connect to it gives a 'Connection refused'
error message.
INPUT:
start -- integer; a port number to start scanning for a new port at
max_tries -- integer (default: 100); how many ports to try
verbose -- bool (default: True); whether or not to print out info about scanning results.
OUTPUT:
an integer, or if no port is found, raises a RuntimError exception
EXAMPLES:
sage: sage.server.misc.find_next_available_port(9000, verbose=False) # random output -- depends on network
9002
"""
from sage.misc.misc import alarm, cancel_alarm
alarm_count = 0
for port in range(start, start + max_tries + 1):
try:
alarm(1)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(('', port))
except socket.error, msg:
if msg[1] == 'Connection refused':
if verbose: print "Using port = %s" % port
return port
except KeyboardInterrupt:
if verbose: print "alarm"
alarm_count += 1
if alarm_count >= 10:
break
pass
def report(F, title, systems=['sage', 'magma'], **kwds):
"""
Run benchmarks with default arguments for each function in the list F.
INPUT:
- ``F`` - a list of callables used for benchmarking
- ``title`` - a string describing this report
- ``systems`` - a list of systems (supported entries are 'sage' and 'magma')
- ``**kwds`` - keyword arguments passed to all functions in ``F``
EXAMPLES::
sage: import sage.matrix.benchmark as b
sage: print "starting"; import sys; sys.stdout.flush(); b.report([b.det_ZZ], 'Test', systems=['sage'])
starting...
======================================================================
Test
======================================================================
...
======================================================================
"""
import os
if len(systems) > 2:
raise NotImplementedError("at most two systems ('sage' or 'magma')")
print '=' * 70
print ' ' * 10 + title
print '=' * 70
os.system('uname -a')
print '\n'
for f in F:
print "-" * 70
print f.__doc__.strip()
print('%15s' * len(systems)) % tuple(systems)
w = []
for s in systems:
alarm(timeout)
try:
t = f(system=s, **kwds)
except AlarmInterrupt:
t = -timeout
cancel_alarm()
w.append(float(t))
if len(w) > 1:
if w[1] == 0:
w.append(0.0)
else:
w.append(w[0] / w[1])
w = tuple(w)
print('%15.3f' * len(w)) % w
print '=' * 70
def report(F, title, systems = ['sage', 'magma'], **kwds):
"""
Run benchmarks with default arguments for each function in the list F.
INPUT:
- ``F`` - a list of callables used for benchmarking
- ``title`` - a string describing this report
- ``systems`` - a list of systems (supported entries are 'sage' and 'magma')
- ``**kwds`` - keyword arguments passed to all functions in ``F``
EXAMPLES::
sage: import sage.matrix.benchmark as b
sage: print "starting"; import sys; sys.stdout.flush(); b.report([b.det_ZZ], 'Test', systems=['sage'])
starting...
======================================================================
Test
======================================================================
...
======================================================================
"""
import os
if len(systems) > 2:
raise NotImplementedError, "at most two systems ('sage' or 'magma')"
print '='*70
print ' '*10 + title
print '='*70
os.system('uname -a')
print '\n'
for f in F:
print "-"*70
print f.__doc__.strip()
print ('%15s'*len(systems))%tuple(systems)
w = []
for s in systems:
alarm(timeout)
try:
t = f(system=s, **kwds)
except KeyboardInterrupt:
t = -timeout
alarm(0)
w.append(float(t))
if len(w) > 1:
if w[1] == 0:
w.append(0.0)
else:
w.append(w[0]/w[1])
w = tuple(w)
print ('%15.3f'*len(w))%w
print '='*70
def find_next_available_port(host, start, max_tries=100, verbose=False):
"""
Find the next available port on a given host, that is, a port for
which a current connection attempt returns a 'Connection refused'
error message. If no port is found, raise a RuntimeError exception.
INPUT:
- ``host`` - address to check
- ``start`` - an int; the starting port number for the scan
- ``max_tries`` - an int (default: 100); how many ports to scan
- ``verbose`` - a bool (default: True); whether to print information
about the scan
OUTPUT:
- an int - the port number
EXAMPLES::
sage: from sage.server.misc import find_next_available_port
sage: find_next_available_port('127.0.0.1', 9000, verbose=False) # random output -- depends on network
9002
"""
from sage.misc.misc import alarm, cancel_alarm
alarm_count = 0
for port in range(start, start+max_tries+1):
try:
alarm(1)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
except socket.error as msg:
if msg[1] == 'Connection refused':
if verbose:
print "Using port = %s" % port
return port
except KeyboardInterrupt:
if verbose:
print "alarm"
alarm_count += 1
if alarm_count >= 10:
break
pass
finally:
cancel_alarm()
if verbose:
print "Port %s is already in use." % port
print "Trying next port..."
raise RuntimeError("no available port.")
def _geturl(self, url, use_alarm=True):
"""
Download the given url. If use_alarm is True (the default)
timeout and return the TIMEOUT object if the default download
timeout is exceeded.
"""
if not use_alarm:
return urllib2.urlopen(url).read()
try:
alarm(self._url_timeout)
return urllib2.urlopen(url).read()
except KeyboardInterrupt:
return TIMEOUT
finally:
cancel_alarm()
def _timeit(self, code):
"""
Time evaluation of the given code, timing out after
self._url_timeout seconds, and using the default number and
repeat values as options to timeit.
"""
try:
alarm(self._url_timeout)
T = sage_timeit(code, globals(), number=self._timeit_number,
repeat=self._timeit_repeat)
except KeyboardInterrupt:
return TIMEOUT
finally:
cancel_alarm()
return T
def find_next_available_port(start, max_tries=100, verbose=False):
"""
Find the next available port, that is, a port for which a
current connection attempt returns a 'Connection refused' error
message. If no port is found, raise a RuntimError exception.
INPUT:
- ``start`` - an int; the starting port number for the scan
- ``max_tries`` - an int (default: 100); how many ports to scan
- ``verbose`` - a bool (default: True); whether to print information
about the scan
OUTPUT:
- an int - the port number
EXAMPLES::
sage: sage.server.misc.find_next_available_port(9000, verbose=False) # random output -- depends on network
9002
"""
from sage.misc.misc import alarm, cancel_alarm
alarm_count = 0
for port in range(start, start + max_tries + 1):
try:
alarm(1)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("", port))
except socket.error, msg:
if msg[1] == "Connection refused":
if verbose:
print "Using port = %s" % port
return port
except KeyboardInterrupt:
if verbose:
print "alarm"
alarm_count += 1
if alarm_count >= 10:
break
pass
def test_allpub(self):
"""
View every single one of the published worksheets on the
FEMhub online lab server.
"""
if self._verbose: print "testing download of all published worksheets..."
tm = walltime()
pub = self.get_urls_of_published_worksheets()
try:
alarm(self._url_timeout)
for i, X in enumerate(pub):
t0 = walltime()
self._geturl(X, use_alarm=False)
if self._verbose:
print "Got %s [%s/%s] %.2f seconds"%(X,i+1,len(pub), walltime(t0))
return walltime(tm)
except KeyboardInterrupt:
return TIMEOUT
finally:
cancel_alarm()
def find_next_available_port(interface, start, max_tries=100, verbose=False):
"""
Find the next available port, that is, a port for which a
current connection attempt returns a 'Connection refused' error
message. If no port is found, raise a RuntimError exception.
INPUT:
- ``start`` - an int; the starting port number for the scan
- ``max_tries`` - an int (default: 100); how many ports to scan
- ``verbose`` - a bool (default: True); whether to print information
about the scan
OUTPUT:
- an int - the port number
EXAMPLES::
sage: import sagenb
sage: sagenb.misc.misc.find_next_available_port('127.0.0.1', 9000, verbose=False) # random output -- depends on network
9002
"""
alarm_count = 0
for port in range(start, start + max_tries + 1):
try:
alarm(1)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((interface, port))
except socket.error, msg:
if msg[1] == 'Connection refused':
if verbose: print "Using port = %s" % port
return port
except KeyboardInterrupt:
if verbose: print "alarm"
alarm_count += 1
if alarm_count >= 10:
break
pass
def find_next_available_port(start, max_tries=100, verbose=False):
"""
Find the next port that is available to be used, where available means that
currently trying to connect to it gives a 'Connection refused'
error message.
INPUT:
start -- integer; a port number to start scanning for a new port at
max_tries -- integer (default: 100); how many ports to try
verbose -- bool (default: True); whether or not to print out info about scanning results.
OUTPUT:
an integer, or if no port is found, raises a RuntimError exception
EXAMPLES:
sage: sage.server.misc.find_next_available_port(9000, verbose=False) # random output -- depends on network
9002
"""
from sage.misc.misc import alarm, cancel_alarm
alarm_count = 0
for port in range(start, start + max_tries + 1):
try:
alarm(1)
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect(("", port))
except socket.error, msg:
if msg[1] == "Connection refused":
if verbose:
print "Using port = %s" % port
return port
except KeyboardInterrupt:
if verbose:
print "alarm"
alarm_count += 1
if alarm_count >= 10:
break
pass
def __call__(self, f, inputs):
"""
Parallel iterator using ``fork()``.
INPUT:
- ``f`` -- a Python function that need not be pickleable or anything else!
- ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
``args`` is a tuple and ``kwds`` is a dictionary.
OUTPUT:
EXAMPLES::
sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
[(([10], {}), 100), (([20], {}), 400)]
sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
[(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]
TESTS:
The output of functions decorated with :func:parallel is read
as a pickle by the parent process. We intentionally break the
unpickling and demonstrate that this failure is handled
gracefully (an exception is displayed and an empty list is
returned)::
sage: Polygen = parallel(polygen)
sage: list(Polygen([QQ]))
[(((Rational Field,), {}), x)]
sage: from sage.structure.sage_object import unpickle_override, register_unpickle_override
sage: register_unpickle_override('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint', Integer)
sage: L = list(Polygen([QQ]))
('__init__() takes at most 2 positional arguments (4 given)', <type 'sage.rings.integer.Integer'>, (Univariate Polynomial Ring in x over Rational Field, [0, 1], False, True))
sage: L
[]
Fix the unpickling::
sage: del unpickle_override[('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint')]
sage: list(Polygen([QQ,QQ]))
[(((Rational Field,), {}), x), (((Rational Field,), {}), x)]
"""
n = self.ncpus
v = list(inputs)
import os, sys, signal
from sage.structure.sage_object import load
from sage.misc.all import tmp_dir, walltime
dir = tmp_dir()
timeout = self.timeout
workers = {}
try:
while len(v) > 0 or len(workers) > 0:
# Spawn up to n subprocesses
while len(v) > 0 and len(workers) < n:
# Subprocesses shouldn't inherit unflushed buffers (cf. #11778):
sys.stdout.flush()
sys.stderr.flush()
pid = os.fork()
# The way fork works is that pid returns the
# nonzero pid of the subprocess for the master
# process and returns 0 for the subprocess.
if pid:
# This is the parent master process.
workers[pid] = [v[0], walltime(), '']
del v[0]
else:
# This is the subprocess.
self._subprocess(f, dir, v[0])
if len(workers) > 0:
# Now wait for one subprocess to finish and report the result.
# However, wait at most the time since the oldest process started.
if timeout:
oldest = min([X[1] for X in workers.values()])
alarm(max(timeout - (walltime() - oldest), 0.1))
try:
pid = os.wait()[0]
cancel_alarm()
w = workers.pop(pid)
except AlarmInterrupt:
cancel_alarm()
# Kill workers that are too old
for pid, X in workers.iteritems():
if walltime() - X[1] > timeout:
if self.verbose:
print(
"Killing subprocess %s with input %s which took too long"
% (pid, X[0]))
os.kill(pid, signal.SIGKILL)
X[-1] = ' (timed out)'
except KeyError:
# Some other process exited, not our problem...
pass
else:
# collect data from process that successfully terminated
sobj = os.path.join(dir, '%s.sobj' % pid)
if not os.path.exists(sobj):
X = "NO DATA" + w[-1] # the message field
else:
X = load(sobj, compress=False)
os.unlink(sobj)
out = os.path.join(dir, '%s.out' % pid)
if not os.path.exists(out):
output = "NO OUTPUT"
else:
output = open(out).read()
os.unlink(out)
if output.strip():
print output,
yield (w[0], X)
except Exception as msg:
print msg
finally:
# Clean up all temporary files.
try:
for X in os.listdir(dir):
os.unlink(os.path.join(dir, X))
os.rmdir(dir)
except OSError as msg:
if self.verbose:
print msg
# Send "kill -9" signal to workers that are left.
if len(workers) > 0:
if self.verbose:
print "Killing any remaining workers..."
sys.stdout.flush()
for pid in workers.keys():
try:
os.kill(pid, signal.SIGKILL)
os.waitpid(pid, 0)
except OSError as msg:
if self.verbose:
print msg
def __call__(self, f, inputs):
"""
Parallel iterator using ``fork()``.
INPUT:
- ``f`` -- a Python function that need not be pickleable or anything else!
- ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
``args`` is a tuple and ``kwds`` is a dictionary.
OUTPUT:
EXAMPLES::
sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
[(([10], {}), 100), (([20], {}), 400)]
sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
[(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]
TESTS:
The output of functions decorated with :func:parallel is read
as a pickle by the parent process. We intentionally break the
unpickling and demonstrate that this failure is handled
gracefully (an exception is displayed and an empty list is
returned)::
sage: Polygen = parallel(polygen)
sage: list(Polygen([QQ]))
[(((Rational Field,), {}), x)]
sage: from sage.structure.sage_object import unpickle_override, register_unpickle_override
sage: register_unpickle_override('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint', Integer)
sage: L = list(Polygen([QQ]))
('__init__() takes at most 2 positional arguments (4 given)', <type 'sage.rings.integer.Integer'>, (Univariate Polynomial Ring in x over Rational Field, [0, 1], False, True))
sage: L
[]
Fix the unpickling::
sage: del unpickle_override[('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint')]
sage: list(Polygen([QQ,QQ]))
[(((Rational Field,), {}), x), (((Rational Field,), {}), x)]
"""
n = self.ncpus
v = list(inputs)
import os, sys, signal
from sage.structure.sage_object import load
from sage.misc.all import tmp_dir, walltime
dir = tmp_dir()
timeout = self.timeout
workers = {}
try:
while len(v) > 0 or len(workers) > 0:
# Spawn up to n subprocesses
while len(v) > 0 and len(workers) < n:
# Subprocesses shouldn't inherit unflushed buffers (cf. #11778):
sys.stdout.flush()
sys.stderr.flush()
pid = os.fork()
# The way fork works is that pid returns the
# nonzero pid of the subprocess for the master
# process and returns 0 for the subprocess.
if pid:
# This is the parent master process.
workers[pid] = [v[0], walltime(), '']
del v[0]
else:
# This is the subprocess.
self._subprocess(f, dir, v[0])
if len(workers) > 0:
# Now wait for one subprocess to finish and report the result.
# However, wait at most the time since the oldest process started.
if timeout:
oldest = min([X[1] for X in workers.values()])
alarm(max(timeout - (walltime()-oldest), 0.1))
try:
pid = os.wait()[0]
cancel_alarm()
w = workers.pop(pid)
except AlarmInterrupt:
cancel_alarm()
# Kill workers that are too old
for pid, X in workers.iteritems():
if walltime() - X[1] > timeout:
if self.verbose:
print(
"Killing subprocess %s with input %s which took too long"
% (pid, X[0]) )
os.kill(pid, signal.SIGKILL)
X[-1] = ' (timed out)'
except KeyError:
# Some other process exited, not our problem...
pass
else:
# collect data from process that successfully terminated
sobj = os.path.join(dir, '%s.sobj'%pid)
if not os.path.exists(sobj):
X = "NO DATA" + w[-1] # the message field
else:
X = load(sobj, compress=False)
os.unlink(sobj)
out = os.path.join(dir, '%s.out'%pid)
if not os.path.exists(out):
output = "NO OUTPUT"
else:
output = open(out).read()
os.unlink(out)
if output.strip():
print output,
yield (w[0], X)
except Exception as msg:
print msg
finally:
# Clean up all temporary files.
try:
for X in os.listdir(dir):
os.unlink(os.path.join(dir, X))
os.rmdir(dir)
except OSError as msg:
if self.verbose:
print msg
# Send "kill -9" signal to workers that are left.
if len(workers) > 0:
if self.verbose:
print "Killing any remaining workers..."
sys.stdout.flush()
for pid in workers.keys():
try:
os.kill(pid, signal.SIGKILL)
os.waitpid(pid, 0)
except OSError as msg:
if self.verbose:
print msg
