def tmpdir():
global _tmp_dir
if _tmp_dir:
return
X = tmp_dir('qsieve')
os.environ['TMPDIR'] = X
_tmp_dir = True
def tmpdir():
global _tmp_dir
if _tmp_dir:
return
X = tmp_dir('qsieve')
os.environ['TMPDIR'] = X
_tmp_dir = True
def qsieve_block(n, time, verbose=False):
"""
Compute the factorization of n using Hart's quadratic Sieve
blocking until complete.
"""
cmd = ['QuadraticSieve']
if time:
cmd = ['time'] + cmd
if six.PY2:
enc_kwds = {}
else:
enc_kwds = {'encoding': 'latin1'}
env = os.environ.copy()
env['TMPDIR'] = tmp_dir('qsieve')
p = sp.Popen(cmd,
env=env,
stdout=sp.PIPE,
stderr=sp.STDOUT,
stdin=sp.PIPE,
**enc_kwds)
out, err = p.communicate(str(n))
z = data_to_list(out, n, time=time)
if verbose:
print(z[-1])
return z[:2]
def unpickle_all(dir, debug=False):
"""
Unpickle all sobj's in the given directory, reporting failures as
they occur. Also printed the number of successes and failure.
INPUT:
- ``dir`` - string; a directory or name of a .tar.bz2 file that
decompresses to give a directo pickirectory.
EXAMPLES::
sage: dir = tmp_dir()
sage: sage.structure.sage_object.picklejar('hello', dir)
sage: sage.structure.sage_object.unpickle_all(dir)
Successfully unpickled 1 objects.
Failed to unpickle 0 objects.
We unpickle the standard pickle jar. This doctest tests that
all "standard pickles" unpickle. Every so often the standard pickle jar
should be updated by running the doctest suite with the environment variable
SAGE_PICKLE_JAR set, then copying the files from SAGE_ROOT/tmp/pickle_jar*
into the standard pickle jar.
::
sage: std = os.environ['SAGE_DATA'] + '/extcode/pickle_jar/pickle_jar.tar.bz2'
sage: sage.structure.sage_object.unpickle_all(std)
doctest:...: DeprecationWarning: RQDF is deprecated; use RealField(212) instead.
Successfully unpickled 487 objects.
Failed to unpickle 0 objects.
"""
i = 0
j = 0
failed = []
tracebacks = []
if dir.endswith('.tar.bz2'):
# create a temporary directory
from sage.misc.all import tmp_dir
T = tmp_dir()
# extract tarball to it
os.system('cd "%s"; bunzip2 -c "%s" | tar fx - ' %
(T, os.path.abspath(dir)))
# Now use the directory in the tarball instead of dir
dir = T + "/" + os.listdir(T)[0]
for A in sorted(os.listdir(dir)):
if A.endswith('.sobj'):
try:
load(dir + '/' + A)
i += 1
except Exception, msg:
j += 1
print "** failed: ", A
failed.append(A)
if debug:
tracebacks.append(sys.exc_info())
def __init__(self, n, time):
self._n = n
if time:
cmd = 'time QuadraticSieve'
else:
cmd = 'QuadraticSieve'
env = os.environ.copy()
env['TMPDIR'] = tmp_dir('qsieve')
self._p = SageSpawn(cmd, env=env)
cleaner.cleaner(self._p.pid, 'QuadraticSieve')
self._p.sendline(str(self._n) + '\n\n\n')
self._done = False
self._out = ''
self._time = ''
self._do_time = time
def __init__(self, n, time):
self._n = n
if time:
cmd = 'time QuadraticSieve'
else:
cmd = 'QuadraticSieve'
env = os.environ.copy()
env['TMPDIR'] = tmp_dir('qsieve')
self._p = SageSpawn(cmd, env=env)
cleaner.cleaner(self._p.pid, 'QuadraticSieve')
self._p.sendline(str(self._n)+'\n\n\n')
self._done = False
self._out = ''
self._time = ''
self._do_time = time
def tmp_dir():
"""
Return a temporary directory.
OUTPUT:
String. The absolute filename of the directory.
EXAMPLES::
sage: from sage.dev.misc import tmp_dir
sage: tmp_dir().startswith(str(SAGE_TMP))
True
"""
try:
from sage.misc.all import tmp_dir
return tmp_dir()
except ImportError:
from tempfile import mkdtemp
return mkdtemp(dir=get_sage_tmp())
def tmp_dir():
"""
Return a temporary directory.
OUTPUT:
String. The absolute filename of the directory.
EXAMPLES::
sage: from sage.dev.misc import tmp_dir
sage: tmp_dir().startswith(str(SAGE_TMP))
True
"""
try:
from sage.misc.all import tmp_dir
return tmp_dir()
except ImportError:
from tempfile import mkdtemp
return mkdtemp(dir=get_sage_tmp())
def qsieve_block(n, time, verbose=False):
"""
Compute the factorization of n using Hart's quadratic Sieve
blocking until complete.
"""
cmd = ['QuadraticSieve']
if time:
cmd = ['time'] + cmd
if six.PY2:
enc_kwds = {}
else:
enc_kwds = {'encoding': 'latin1'}
env = os.environ.copy()
env['TMPDIR'] = tmp_dir('qsieve')
p = sp.Popen(cmd, env=env, stdout=sp.PIPE, stderr=sp.STDOUT,
stdin=sp.PIPE, **enc_kwds)
out, err = p.communicate(str(n))
z = data_to_list(out, n, time=time)
if verbose:
print(z[-1])
return z[:2]
def upload_worksheet():
from sage.misc.all import tmp_filename, tmp_dir
from werkzeug.utils import secure_filename
import zipfile
if g.notebook.readonly_user(g.username):
return current_app.message(_("Account is in read-only mode"), cont=url_for('home', username=g.username))
backlinks = _("""Return to <a href="/upload" title="Upload a worksheet"><strong>Upload File</strong></a>.""")
url = request.values['url'].strip()
dir = ''
if url != '':
#Downloading a file from the internet
# The file will be downloaded from the internet and saved
# to a temporary file with the same extension
path = urlparse.urlparse(url).path
extension = os.path.splitext(path)[1].lower()
if extension not in ["", ".txt", ".sws", ".zip", ".html", ".rst"]:
# Or shall we try to import the document as an sws when in doubt?
return current_app.message(_("Unknown worksheet extension: %(ext)s. %(links)s", ext=extension, links=backlinks))
filename = tmp_filename()+extension
try:
import re
matches = re.match("file://(?:localhost)?(/.+)", url)
if matches:
if g.notebook.interface != 'localhost':
return current_app.message(_("Unable to load file URL's when not running on localhost.\n%(backlinks)s",backlinks=backlinks))
import shutil
shutil.copy(matches.group(1),filename)
else:
my_urlretrieve(url, filename, backlinks=backlinks)
except RetrieveError as err:
return current_app.message(str(err))
else:
#Uploading a file from the user's computer
dir = tmp_dir()
file = request.files['file']
if file.filename is None:
return current_app.message(_("Please specify a worksheet to load.\n%(backlinks)s",backlinks=backlinks))
filename = secure_filename(file.filename)
if len(filename)==0:
return current_app.message(_("Invalid filename.\n%(backlinks)s",backlinks=backlinks))
filename = os.path.join(dir, filename)
file.save(filename)
new_name = request.values.get('name', None)
try:
try:
if filename.endswith('.zip'):
# Extract all the .sws files from a zip file.
zip_file = zipfile.ZipFile(filename)
for subfilename in zip_file.namelist():
prefix, extension = os.path.splitext(subfilename)
# Mac zip files contain files like __MACOSX/._worksheet.sws
# which are metadata files, so we skip those as
# well as any other files we won't understand
if extension in ['.sws', '.html', '.txt', '.rst'] and not prefix.startswith('__MACOSX/'):
tmpfilename = os.path.join(dir, "tmp" + extension)
try:
tmpfilename = zip_file.extract(subfilename, tmpfilename)
except AttributeError:
open(tmpfilename, 'w').write(zip_file.read(subfilename))
W = g.notebook.import_worksheet(tmpfilename, g.username)
if new_name:
W.set_name("%s - %s" % (new_name, W.name()))
else:
print "Unknown extension, file %s is ignored" % subfilename
return redirect(url_for('home', username=g.username))
else:
if url and extension in ['', '.html']:
linked_sws = parse_link_rel(url, filename)
if linked_sws:
# just grab 1st URL; perhaps later add interface for
# downloading multiple linked .sws
try:
filename = my_urlretrieve(linked_sws[0]['url'], backlinks=backlinks)[0]
print 'Importing {0}, linked to from {1}'.format(linked_sws[0]['url'], url)
except RetrieveError as err:
return current_app.message(str(err))
W = g.notebook.import_worksheet(filename, g.username)
except Exception, msg:
print 'error uploading worksheet', msg
s = _('There was an error uploading the worksheet. It could be an old unsupported format or worse. If you desperately need its contents contact the <a href="http://groups.google.com/group/sage-support">sage-support group</a> and post a link to your worksheet. Alternatively, an sws file is just a bzip2 tarball; take a look inside!\n%(backlinks)s', backlinks=backlinks)
return current_app.message(s, url_for('home', username=g.username))
finally:
# Clean up the temporarily uploaded filename.
os.unlink(filename)
# if a temp directory was created, we delete it now.
if dir:
import shutil
shutil.rmtree(dir)
def __call__(self, f, inputs):
"""
Parallel iterator using ``fork()``.
INPUT:
- ``f`` -- a Python function that need not be pickleable or anything else!
- ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
``args`` is a tuple and ``kwds`` is a dictionary.
OUTPUT:
EXAMPLES::
sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
[(([10], {}), 100), (([20], {}), 400)]
sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
[(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]
TESTS:
The output of functions decorated with :func:parallel is read
as a pickle by the parent process. We intentionally break the
unpickling and demonstrate that this failure is handled
gracefully (an exception is displayed and an empty list is
returned)::
sage: Polygen = parallel(polygen)
sage: list(Polygen([QQ]))
[(((Rational Field,), {}), x)]
sage: from sage.structure.sage_object import unpickle_override, register_unpickle_override
sage: register_unpickle_override('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint', Integer)
sage: L = list(Polygen([QQ]))
('__init__() takes at most 2 positional arguments (4 given)', <type 'sage.rings.integer.Integer'>, (Univariate Polynomial Ring in x over Rational Field, [0, 1], False, True))
sage: L
[]
Fix the unpickling::
sage: del unpickle_override[('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint')]
sage: list(Polygen([QQ,QQ]))
[(((Rational Field,), {}), x), (((Rational Field,), {}), x)]
"""
n = self.ncpus
v = list(inputs)
import os, sys, signal
from sage.structure.sage_object import load
from sage.misc.all import tmp_dir, walltime
dir = tmp_dir()
timeout = self.timeout
workers = {}
try:
while len(v) > 0 or len(workers) > 0:
# Spawn up to n subprocesses
while len(v) > 0 and len(workers) < n:
# Subprocesses shouldn't inherit unflushed buffers (cf. #11778):
sys.stdout.flush()
sys.stderr.flush()
pid = os.fork()
# The way fork works is that pid returns the
# nonzero pid of the subprocess for the master
# process and returns 0 for the subprocess.
if pid:
# This is the parent master process.
workers[pid] = [v[0], walltime(), '']
del v[0]
else:
# This is the subprocess.
self._subprocess(f, dir, v[0])
if len(workers) > 0:
# Now wait for one subprocess to finish and report the result.
# However, wait at most the time since the oldest process started.
if timeout:
oldest = min([X[1] for X in workers.values()])
alarm(max(timeout - (walltime() - oldest), 0.1))
try:
pid = os.wait()[0]
cancel_alarm()
w = workers.pop(pid)
except AlarmInterrupt:
cancel_alarm()
# Kill workers that are too old
for pid, X in workers.iteritems():
if walltime() - X[1] > timeout:
if self.verbose:
print(
"Killing subprocess %s with input %s which took too long"
% (pid, X[0]))
os.kill(pid, signal.SIGKILL)
X[-1] = ' (timed out)'
except KeyError:
# Some other process exited, not our problem...
pass
else:
# collect data from process that successfully terminated
sobj = os.path.join(dir, '%s.sobj' % pid)
if not os.path.exists(sobj):
X = "NO DATA" + w[-1] # the message field
else:
X = load(sobj, compress=False)
os.unlink(sobj)
out = os.path.join(dir, '%s.out' % pid)
if not os.path.exists(out):
output = "NO OUTPUT"
else:
output = open(out).read()
os.unlink(out)
if output.strip():
print output,
yield (w[0], X)
except Exception as msg:
print msg
finally:
# Clean up all temporary files.
try:
for X in os.listdir(dir):
os.unlink(os.path.join(dir, X))
os.rmdir(dir)
except OSError as msg:
if self.verbose:
print msg
# Send "kill -9" signal to workers that are left.
if len(workers) > 0:
if self.verbose:
print "Killing any remaining workers..."
sys.stdout.flush()
for pid in workers.keys():
try:
os.kill(pid, signal.SIGKILL)
os.waitpid(pid, 0)
except OSError as msg:
if self.verbose:
print msg
def upload_worksheet():
from sage.misc.all import tmp_filename, tmp_dir
from werkzeug.utils import secure_filename
import zipfile
if g.notebook.readonly_user(g.username):
return current_app.message(_("Account is in read-only mode"),
cont=url_for('home', username=g.username))
backlinks = _(
"""Return to <a href="/upload" title="Upload a worksheet"><strong>Upload File</strong></a>."""
)
url = request.values['url'].strip()
dir = ''
if url != '':
#Downloading a file from the internet
# The file will be downloaded from the internet and saved
# to a temporary file with the same extension
path = urlparse.urlparse(url).path
extension = os.path.splitext(path)[1].lower()
if extension not in ["", ".txt", ".sws", ".zip", ".html", ".rst"]:
# Or shall we try to import the document as an sws when in doubt?
return current_app.message(
_("Unknown worksheet extension: %(ext)s. %(links)s",
ext=extension,
links=backlinks))
filename = tmp_filename() + extension
try:
import re
matches = re.match("file://(?:localhost)?(/.+)", url)
if matches:
if g.notebook.interface != 'localhost':
return current_app.message(
_("Unable to load file URL's when not running on localhost.\n%(backlinks)s",
backlinks=backlinks))
import shutil
shutil.copy(matches.group(1), filename)
else:
my_urlretrieve(url, filename, backlinks=backlinks)
except RetrieveError as err:
return current_app.message(str(err))
else:
#Uploading a file from the user's computer
dir = tmp_dir()
file = request.files['file']
if file.filename is None:
return current_app.message(
_("Please specify a worksheet to load.\n%(backlinks)s",
backlinks=backlinks))
filename = secure_filename(file.filename)
if len(filename) == 0:
return current_app.message(
_("Invalid filename.\n%(backlinks)s", backlinks=backlinks))
filename = os.path.join(dir, filename)
file.save(filename)
new_name = request.values.get('name', None)
try:
try:
if filename.endswith('.zip'):
# Extract all the .sws files from a zip file.
zip_file = zipfile.ZipFile(filename)
for subfilename in zip_file.namelist():
prefix, extension = os.path.splitext(subfilename)
# Mac zip files contain files like __MACOSX/._worksheet.sws
# which are metadata files, so we skip those as
# well as any other files we won't understand
if extension in ['.sws', '.html', '.txt', '.rst'
] and not prefix.startswith('__MACOSX/'):
tmpfilename = os.path.join(dir, "tmp" + extension)
try:
tmpfilename = zip_file.extract(
subfilename, tmpfilename)
except AttributeError:
open(tmpfilename,
'w').write(zip_file.read(subfilename))
W = g.notebook.import_worksheet(
tmpfilename, g.username)
if new_name:
W.set_name("%s - %s" % (new_name, W.name()))
else:
print "Unknown extension, file %s is ignored" % subfilename
return redirect(url_for('home', username=g.username))
else:
if url and extension in ['', '.html']:
linked_sws = parse_link_rel(url, filename)
if linked_sws:
# just grab 1st URL; perhaps later add interface for
# downloading multiple linked .sws
try:
filename = my_urlretrieve(linked_sws[0]['url'],
backlinks=backlinks)[0]
print 'Importing {0}, linked to from {1}'.format(
linked_sws[0]['url'], url)
except RetrieveError as err:
return current_app.message(str(err))
W = g.notebook.import_worksheet(filename, g.username)
except Exception, msg:
print 'error uploading worksheet', msg
s = _(
'There was an error uploading the worksheet. It could be an old unsupported format or worse. If you desperately need its contents contact the <a href="http://groups.google.com/group/sage-support">sage-support group</a> and post a link to your worksheet. Alternatively, an sws file is just a bzip2 tarball; take a look inside!\n%(backlinks)s',
backlinks=backlinks)
return current_app.message(s, url_for('home', username=g.username))
finally:
# Clean up the temporarily uploaded filename.
os.unlink(filename)
# if a temp directory was created, we delete it now.
if dir:
import shutil
shutil.rmtree(dir)
def __call__(self, f, inputs):
"""
Parallel iterator using ``fork()``.
INPUT:
- ``f`` -- a Python function that need not be pickleable or anything else!
- ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
``args`` is a tuple and ``kwds`` is a dictionary.
OUTPUT:
EXAMPLES::
sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
[(([10], {}), 100), (([20], {}), 400)]
sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
[(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]
TESTS:
The output of functions decorated with :func:parallel is read
as a pickle by the parent process. We intentionally break the
unpickling and demonstrate that this failure is handled
gracefully (an exception is displayed and an empty list is
returned)::
sage: Polygen = parallel(polygen)
sage: list(Polygen([QQ]))
[(((Rational Field,), {}), x)]
sage: from sage.structure.sage_object import unpickle_override, register_unpickle_override
sage: register_unpickle_override('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint', Integer)
sage: L = list(Polygen([QQ]))
('__init__() takes at most 2 positional arguments (4 given)', <type 'sage.rings.integer.Integer'>, (Univariate Polynomial Ring in x over Rational Field, [0, 1], False, True))
sage: L
[]
Fix the unpickling::
sage: del unpickle_override[('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint')]
sage: list(Polygen([QQ,QQ]))
[(((Rational Field,), {}), x), (((Rational Field,), {}), x)]
"""
n = self.ncpus
v = list(inputs)
import os, sys, signal
from sage.structure.sage_object import load
from sage.misc.all import tmp_dir, walltime
dir = tmp_dir()
timeout = self.timeout
workers = {}
try:
while len(v) > 0 or len(workers) > 0:
# Spawn up to n subprocesses
while len(v) > 0 and len(workers) < n:
# Subprocesses shouldn't inherit unflushed buffers (cf. #11778):
sys.stdout.flush()
sys.stderr.flush()
pid = os.fork()
# The way fork works is that pid returns the
# nonzero pid of the subprocess for the master
# process and returns 0 for the subprocess.
if pid:
# This is the parent master process.
workers[pid] = [v[0], walltime(), '']
del v[0]
else:
# This is the subprocess.
self._subprocess(f, dir, v[0])
if len(workers) > 0:
# Now wait for one subprocess to finish and report the result.
# However, wait at most the time since the oldest process started.
if timeout:
oldest = min([X[1] for X in workers.values()])
alarm(max(timeout - (walltime()-oldest), 0.1))
try:
pid = os.wait()[0]
cancel_alarm()
w = workers.pop(pid)
except AlarmInterrupt:
cancel_alarm()
# Kill workers that are too old
for pid, X in workers.iteritems():
if walltime() - X[1] > timeout:
if self.verbose:
print(
"Killing subprocess %s with input %s which took too long"
% (pid, X[0]) )
os.kill(pid, signal.SIGKILL)
X[-1] = ' (timed out)'
except KeyError:
# Some other process exited, not our problem...
pass
else:
# collect data from process that successfully terminated
sobj = os.path.join(dir, '%s.sobj'%pid)
if not os.path.exists(sobj):
X = "NO DATA" + w[-1] # the message field
else:
X = load(sobj, compress=False)
os.unlink(sobj)
out = os.path.join(dir, '%s.out'%pid)
if not os.path.exists(out):
output = "NO OUTPUT"
else:
output = open(out).read()
os.unlink(out)
if output.strip():
print output,
yield (w[0], X)
except Exception as msg:
print msg
finally:
# Clean up all temporary files.
try:
for X in os.listdir(dir):
os.unlink(os.path.join(dir, X))
os.rmdir(dir)
except OSError as msg:
if self.verbose:
print msg
# Send "kill -9" signal to workers that are left.
if len(workers) > 0:
if self.verbose:
print "Killing any remaining workers..."
sys.stdout.flush()
for pid in workers.keys():
try:
os.kill(pid, signal.SIGKILL)
os.waitpid(pid, 0)
except OSError as msg:
if self.verbose:
print msg
