def upload_worksheet():
from sage.misc.misc import tmp_filename, tmp_dir
from werkzeug import secure_filename
import zipfile
#XXX: i18n
backlinks = """ Return to <a href="/upload" title="Upload a worksheet"><strong>Upload File</strong></a>."""
url = request.values['url'].strip()
dir = ''
if url:
#Downloading a file from the internet
import urllib
filename = tmp_filename() + ('.zip'
if url.endswith('.zip') else '.sws')
urllib.urlretrieve(url, filename)
else:
#Uploading a file from the user's computer
dir = tmp_dir()
file = request.files['file']
if file.filename == '':
return current_app.message(
"Please specify a worksheet to load.%s" % backlinks)
filename = secure_filename(file.filename)
filename = os.path.join(dir, filename)
file.save(filename)
new_name = request.values.get('name', None)
try:
try:
if filename.endswith('.zip'):
# Extract all the .sws files from a zip file.
zip_file = zipfile.ZipFile(filename)
sws_file = os.path.join(dir, "tmp.sws")
for sws in zip_file.namelist():
if sws.endswith('.sws'):
open(sws_file, 'w').write(zip_file.read(
sws)) # 2.6 zip_file.extract(sws, sws_file)
W = g.notebook.import_worksheet(sws_file, g.username)
if new_name:
W.set_name("%s - %s" % (new_name, W.name()))
return redirect(url_for('home', username=g.username))
else:
W = g.notebook.import_worksheet(filename, g.username)
except Exception, msg:
s = 'There was an error uploading the worksheet. It could be an old unsupported format or worse. If you desperately need its contents contact the <a href="http://groups.google.com/group/sage-support">sage-support group</a> and post a link to your worksheet. Alternatively, an sws file is just a bzip2 tarball; take a look inside!%s' % backlinks
return current_app.message(s, url_for('home', username=g.username))
finally:
# Clean up the temporarily uploaded filename.
os.unlink(filename)
# if a temp directory was created, we delete it now.
if dir:
import shutil
shutil.rmtree(dir)
def upload_worksheet():
from sage.misc.misc import tmp_filename, tmp_dir
from werkzeug import secure_filename
import zipfile
#XXX: i18n
backlinks = """ Return to <a href="/upload" title="Upload a worksheet"><strong>Upload File</strong></a>."""
url = request.values['url'].strip()
dir = ''
if url:
#Downloading a file from the internet
import urllib
filename = tmp_filename() + ('.zip' if url.endswith('.zip') else '.sws')
urllib.urlretrieve(url, filename)
else:
#Uploading a file from the user's computer
dir = tmp_dir()
file = request.files['file']
if file.filename == '':
return current_app.message("Please specify a worksheet to load.%s" % backlinks)
filename = secure_filename(file.filename)
filename = os.path.join(dir, filename)
file.save(filename)
new_name = request.values.get('name', None)
try:
try:
if filename.endswith('.zip'):
# Extract all the .sws files from a zip file.
zip_file = zipfile.ZipFile(filename)
sws_file = os.path.join(dir, "tmp.sws")
for sws in zip_file.namelist():
if sws.endswith('.sws'):
open(sws_file, 'w').write(zip_file.read(sws)) # 2.6 zip_file.extract(sws, sws_file)
W = g.notebook.import_worksheet(sws_file, g.username)
if new_name:
W.set_name("%s - %s" % (new_name, W.name()))
return redirect(url_for('home', username=g.username))
else:
W = g.notebook.import_worksheet(filename, g.username)
except Exception, msg:
s = 'There was an error uploading the worksheet. It could be an old unsupported format or worse. If you desperately need its contents contact the <a href="http://groups.google.com/group/sage-support">sage-support group</a> and post a link to your worksheet. Alternatively, an sws file is just a bzip2 tarball; take a look inside!%s' % backlinks
return current_app.message(s, url_for('home', username=g.username))
finally:
# Clean up the temporarily uploaded filename.
os.unlink(filename)
# if a temp directory was created, we delete it now.
if dir:
import shutil
shutil.rmtree(dir)
def tmp_dir():
"""
Return a temporary directory.
OUTPUT:
String. The absolute filename of the directory.
EXAMPLES::
sage: from sage.dev.misc import tmp_dir
sage: tmp_dir().startswith(str(SAGE_TMP))
True
"""
try:
from sage.misc.misc import tmp_dir
return tmp_dir()
except ImportError:
from tempfile import mkdtemp
return mkdtemp(dir=get_sage_tmp())
def upload_worksheet():
from sage.misc.misc import tmp_filename, tmp_dir
from werkzeug.utils import secure_filename
import zipfile
if g.notebook.readonly_user(g.username):
return current_app.message(_("Account is in read-only mode"), cont=url_for('home', username=g.username))
backlinks = _("""Return to <a href="/upload" title="Upload a worksheet"><strong>Upload File</strong></a>.""")
url = request.values['url'].strip()
dir = ''
if url != '':
#Downloading a file from the internet
# The file will be downloaded from the internet and saved
# to a temporary file with the same extension
path = urlparse.urlparse(url).path
extension = os.path.splitext(path)[1].lower()
if extension not in ["", ".txt", ".sws", ".zip", ".html", ".rst"]:
# Or shall we try to import the document as an sws when in doubt?
return current_app.message("Unknown worksheet extension: %s. %s" % (extension, backlinks))
filename = tmp_filename()+extension
try:
import re
matches = re.match("file://(?:localhost)?(/.+)", url)
if matches:
if g.notebook.interface != 'localhost':
return current_app.message(_("Unable to load file URL's when not running on localhost.\n%(backlinks)s",backlinks=backlinks))
import shutil
shutil.copy(matches.group(1),filename)
else:
my_urlretrieve(url, filename, backlinks=backlinks)
except RetrieveError as err:
return current_app.message(str(err))
else:
#Uploading a file from the user's computer
dir = tmp_dir()
file = request.files['file']
if file.filename is None:
return current_app.message(_("Please specify a worksheet to load.\n%(backlinks)s",backlinks=backlinks))
filename = secure_filename(file.filename)
if len(filename)==0:
return current_app.message(_("Invalid filename.\n%(backlinks)s",backlinks=backlinks))
filename = os.path.join(dir, filename)
file.save(filename)
new_name = request.values.get('name', None)
try:
try:
if filename.endswith('.zip'):
# Extract all the .sws files from a zip file.
zip_file = zipfile.ZipFile(filename)
for subfilename in zip_file.namelist():
prefix, extension = os.path.splitext(subfilename)
# Mac zip files contain files like __MACOSX/._worksheet.sws
# which are metadata files, so we skip those as
# well as any other files we won't understand
if extension in ['.sws', '.html', '.txt', '.rst'] and not prefix.startswith('__MACOSX/'):
tmpfilename = os.path.join(dir, "tmp" + extension)
try:
tmpfilename = zip_file.extract(subfilename, tmpfilename)
except AttributeError:
open(tmpfilename, 'w').write(zip_file.read(subfilename))
W = g.notebook.import_worksheet(tmpfilename, g.username)
if new_name:
W.set_name("%s - %s" % (new_name, W.name()))
else:
print "Unknown extension, file %s is ignored" % subfilename
return redirect(url_for('home', username=g.username))
else:
if url and extension in ['', '.html']:
linked_sws = parse_link_rel(url, filename)
if linked_sws:
# just grab 1st URL; perhaps later add interface for
# downloading multiple linked .sws
try:
filename = my_urlretrieve(linked_sws[0]['url'], backlinks=backlinks)[0]
print 'Importing {0}, linked to from {1}'.format(linked_sws[0]['url'], url)
except RetrieveError as err:
return current_app.message(str(err))
W = g.notebook.import_worksheet(filename, g.username)
except Exception, msg:
print 'error uploading worksheet', msg
s = _('There was an error uploading the worksheet. It could be an old unsupported format or worse. If you desperately need its contents contact the <a href="http://groups.google.com/group/sage-support">sage-support group</a> and post a link to your worksheet. Alternatively, an sws file is just a bzip2 tarball; take a look inside!\n%(backlinks)s', backlinks=backlinks)
return current_app.message(s, url_for('home', username=g.username))
finally:
# Clean up the temporarily uploaded filename.
os.unlink(filename)
# if a temp directory was created, we delete it now.
if dir:
import shutil
shutil.rmtree(dir)
def upload_worksheet():
from sage.misc.misc import tmp_filename, tmp_dir
from werkzeug.utils import secure_filename
import zipfile
backlinks = _("""Return to <a href="/upload" title="Upload a worksheet"><strong>Upload File</strong></a>.""")
url = request.values['url'].strip()
dir = ''
if url:
#Downloading a file from the internet
import urllib, urlparse
filename = tmp_filename() + ('.zip' if url.endswith('.zip') else '.sws')
# The file will be downloaded from the internet and saved
# to a temporary file with the same extension
path = urlparse.urlparse(url).path
extension = os.path.splitext(path)[1].lower()
if extension not in [".txt", ".sws", ".zip", ".html", ".rst"]:
# Or shall we try to import the document as an sws in doubt?
return current_app.message("Unknown worksheet extension: %s. %s" % (extension, backlinks))
filename = tmp_filename()+extension
try:
urllib.urlretrieve(url, filename)
except IOError as err:
if err.strerror == 'unknown url type' and err.filename == 'https':
return current_app.message(_("This Sage notebook is not configured to load worksheets from 'https' URLs. Try a different URL or download the worksheet and upload it directly from your computer.\n%(backlinks)s",backlinks=backlinks))
else:
raise
else:
#Uploading a file from the user's computer
dir = tmp_dir()
file = request.files['file']
if file.filename is None:
return current_app.message(_("Please specify a worksheet to load.\n%(backlinks)s",backlinks=backlinks))
filename = secure_filename(file.filename)
if len(filename)==0:
return current_app.message(_("Invalid filename.\n%(backlinks)s",backlinks=backlinks))
filename = os.path.join(dir, filename)
file.save(filename)
new_name = request.values.get('name', None)
try:
try:
if filename.endswith('.zip'):
# Extract all the .sws files from a zip file.
zip_file = zipfile.ZipFile(filename)
for subfilename in zip_file.namelist():
prefix, extension = os.path.splitext(subfilename)
if extension in ['.sws', '.html', '.txt', '.rst'] :
tmpfilename = os.path.join(dir, "tmp" + extension)
try:
tmpfilename = zip_file.extract(subfilename, tmpfilename)
except AttributeError:
open(tmpfilename, 'w').write(zip_file.read(subfilename))
W = g.notebook.import_worksheet(tmpfilename, g.username)
if new_name:
W.set_name("%s - %s" % (new_name, W.name()))
else:
print "Unknown extension, file %s is ignored" % subfilename
return redirect(url_for('home', username=g.username))
else:
W = g.notebook.import_worksheet(filename, g.username)
except Exception, msg:
print 'error uploading worksheet', msg
s = _('There was an error uploading the worksheet. It could be an old unsupported format or worse. If you desperately need its contents contact the <a href="http://groups.google.com/group/sage-support">sage-support group</a> and post a link to your worksheet. Alternatively, an sws file is just a bzip2 tarball; take a look inside!\n%(backlinks)s', backlinks=backlinks)
return current_app.message(s, url_for('home', username=g.username))
finally:
# Clean up the temporarily uploaded filename.
os.unlink(filename)
# if a temp directory was created, we delete it now.
if dir:
import shutil
shutil.rmtree(dir)
def __call__(self, f, inputs):
"""
Parallel iterator using ``fork()``.
INPUT:
- ``f`` -- a Python function that need not be pickleable or anything else!
- ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
``args`` is a tuple and ``kwds`` is a dictionary.
OUTPUT:
EXAMPLES::
sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
[(([10], {}), 100), (([20], {}), 400)]
sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
[(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]
"""
n = self.ncpus
v = list(inputs)
import os, sys, signal
from sage.structure.sage_object import load
from sage.misc.misc import tmp_dir, walltime
dir = tmp_dir()
timeout = self.timeout
# Subprocesses shouldn't inherit unflushed buffers (cf. #11778):
sys.stdout.flush()
sys.stderr.flush()
workers = {}
try:
while len(v) > 0 or len(workers) > 0:
# Spawn up to n subprocesses
while len(v) > 0 and len(workers) < n:
pid = os.fork()
# The way fork works is that pid returns the
# nonzero pid of the subprocess for the master
# process and returns 0 for the subprocess.
if pid:
# This is the parent master process.
workers[pid] = [v[0], walltime(), '']
del v[0]
else:
# This is the subprocess.
self._subprocess(f, dir, v[0])
if len(workers) > 0:
# Now wait for one subprocess to finish and report the result.
# However, wait at most the time since the oldest process started.
if timeout:
def mysig(a,b):
raise RuntimeError, "SIGALRM"
oldest = min([X[1] for X in workers.values()])
signal.signal(signal.SIGALRM, mysig)
signal.alarm(max(int(timeout - (walltime()-oldest)), 1))
try:
pid = os.wait()[0]
signal.signal(signal.SIGALRM, signal.SIG_IGN)
except RuntimeError:
signal.signal(signal.SIGALRM, signal.SIG_IGN)
# Kill workers that are too old
for pid, X in workers.iteritems():
if walltime() - X[1] > timeout:
if self.verbose:
print(
"Killing subprocess %s with input %s which took too long"
% (pid, X[0]) )
sys.stdout.flush()
os.kill(pid,9)
X[-1] = ' (timed out)'
else:
# If the computation was interrupted the pid
# might not be in the workers list, in which
# case we skip this.
if pid in workers:
# collect data from process that successfully terminated
sobj = os.path.join(dir, '%s.sobj'%pid)
if not os.path.exists(sobj):
X = "NO DATA" + workers[pid][-1] # the message field
else:
X = load(sobj, compress=False)
os.unlink(sobj)
out = os.path.join(dir, '%s.out'%pid)
if not os.path.exists(out):
output = "NO OUTPUT"
else:
output = open(out).read()
os.unlink(out)
if output.strip():
print output,
sys.stdout.flush()
yield (workers[pid][0], X)
del workers[pid]
except Exception, msg:
print msg
sys.stdout.flush()
def upload_worksheet():
from sage.misc.misc import tmp_filename, tmp_dir
from werkzeug.utils import secure_filename
import zipfile
backlinks = _(
"""Return to <a href="/upload" title="Upload a worksheet"><strong>Upload File</strong></a>."""
)
url = request.values['url'].strip()
dir = ''
if url:
#Downloading a file from the internet
import urllib, urlparse
filename = tmp_filename() + ('.zip'
if url.endswith('.zip') else '.sws')
# The file will be downloaded from the internet and saved
# to a temporary file with the same extension
path = urlparse.urlparse(url).path
extension = os.path.splitext(path)[1].lower()
if extension not in [".txt", ".sws", ".zip", ".html", ".rst"]:
# Or shall we try to import the document as an sws in doubt?
return current_app.message("Unknown worksheet extension: %s. %s" %
(extension, backlinks))
filename = tmp_filename() + extension
urllib.urlretrieve(url, filename)
else:
#Uploading a file from the user's computer
dir = tmp_dir()
file = request.files['file']
if file.filename is None:
return current_app.message(
_("Please specify a worksheet to load.\n%(backlinks)s",
backlinks=backlinks))
filename = secure_filename(file.filename)
if len(filename) == 0:
return current_app.message(
_("Invalid filename.\n%(backlinks)s", backlinks=backlinks))
filename = os.path.join(dir, filename)
file.save(filename)
new_name = request.values.get('name', None)
try:
try:
if filename.endswith('.zip'):
# Extract all the .sws files from a zip file.
zip_file = zipfile.ZipFile(filename)
for subfilename in zip_file.namelist():
prefix, extension = os.path.splitext(subfilename)
if extension in ['.sws', '.html', '.txt', '.rst']:
tmpfilename = os.path.join(dir, "tmp" + extension)
try:
tmpfilename = zip_file.extract(
subfilename, tmpfilename)
except AttributeError:
open(tmpfilename,
'w').write(zip_file.read(subfilename))
W = g.notebook.import_worksheet(
tmpfilename, g.username)
if new_name:
W.set_name("%s - %s" % (new_name, W.name()))
else:
print "Unknown extension, file %s is ignored" % subfilename
return redirect(url_for('home', username=g.username))
else:
W = g.notebook.import_worksheet(filename, g.username)
except Exception, msg:
print 'error uploading worksheet', msg
s = _(
'There was an error uploading the worksheet. It could be an old unsupported format or worse. If you desperately need its contents contact the <a href="http://groups.google.com/group/sage-support">sage-support group</a> and post a link to your worksheet. Alternatively, an sws file is just a bzip2 tarball; take a look inside!\n%(backlinks)s',
backlinks=backlinks)
return current_app.message(s, url_for('home', username=g.username))
finally:
# Clean up the temporarily uploaded filename.
os.unlink(filename)
# if a temp directory was created, we delete it now.
if dir:
import shutil
shutil.rmtree(dir)
def upload_worksheet():
from sage.misc.misc import tmp_filename, tmp_dir
from werkzeug.utils import secure_filename
import zipfile
if g.notebook.readonly_user(g.username):
return current_app.message(_("Account is in read-only mode"), cont=url_for('home', username=g.username))
backlinks = _("""Return to <a href="/upload" title="Upload a worksheet"><strong>Upload File</strong></a>.""")
url = request.values['url'].strip()
dir = ''
if url != '':
#Downloading a file from the internet
# The file will be downloaded from the internet and saved
# to a temporary file with the same extension
path = urlparse.urlparse(url).path
extension = os.path.splitext(path)[1].lower()
if extension not in ["", ".txt", ".sws", ".zip", ".html", ".rst"]:
# Or shall we try to import the document as an sws when in doubt?
return current_app.message("Unknown worksheet extension: %s. %s" % (extension, backlinks))
filename = tmp_filename()+extension
try:
import re
matches = re.match("file://(?:localhost)?(/.+)", url)
if matches:
if g.notebook.interface != 'localhost':
return current_app.message(_("Unable to load file URL's when not running on localhost.\n%(backlinks)s",backlinks=backlinks))
import shutil
shutil.copy(matches.group(1),filename)
else:
my_urlretrieve(url, filename, backlinks=backlinks)
except RetrieveError as err:
return current_app.message(str(err))
else:
#Uploading a file from the user's computer
dir = tmp_dir()
file = request.files['file']
if file.filename is None:
return current_app.message(_("Please specify a worksheet to load.\n%(backlinks)s",backlinks=backlinks))
filename = secure_filename(file.filename)
if len(filename)==0:
return current_app.message(_("Invalid filename.\n%(backlinks)s",backlinks=backlinks))
filename = os.path.join(dir, filename)
file.save(filename)
new_name = request.values.get('name', None)
try:
try:
if filename.endswith('.zip'):
# Extract all the .sws files from a zip file.
zip_file = zipfile.ZipFile(filename)
for subfilename in zip_file.namelist():
prefix, extension = os.path.splitext(subfilename)
# Mac zip files contain files like __MACOSX/._worksheet.sws
# which are metadata files, so we skip those as
# well as any other files we won't understand
if extension in ['.sws', '.html', '.txt', '.rst'] and not prefix.startswith('__MACOSX/'):
tmpfilename = os.path.join(dir, "tmp" + extension)
try:
tmpfilename = zip_file.extract(subfilename, tmpfilename)
except AttributeError:
open(tmpfilename, 'w').write(zip_file.read(subfilename))
W = g.notebook.import_worksheet(tmpfilename, g.username)
if new_name:
W.set_name("%s - %s" % (new_name, W.name()))
else:
print "Unknown extension, file %s is ignored" % subfilename
return redirect(url_for('home', username=g.username))
else:
if url and extension in ['', '.html']:
linked_sws = parse_link_rel(url, filename)
if linked_sws:
# just grab 1st URL; perhaps later add interface for
# downloading multiple linked .sws
try:
filename = my_urlretrieve(linked_sws[0]['url'], backlinks=backlinks)[0]
print 'Importing {0}, linked to from {1}'.format(linked_sws[0]['url'], url)
except RetrieveError as err:
return current_app.message(str(err))
W = g.notebook.import_worksheet(filename, g.username)
except Exception, msg:
print 'error uploading worksheet', msg
s = _('There was an error uploading the worksheet. It could be an old unsupported format or worse. If you desperately need its contents contact the <a href="http://groups.google.com/group/sage-support">sage-support group</a> and post a link to your worksheet. Alternatively, an sws file is just a bzip2 tarball; take a look inside!\n%(backlinks)s', backlinks=backlinks)
return current_app.message(s, url_for('home', username=g.username))
finally:
# Clean up the temporarily uploaded filename.
os.unlink(filename)
# if a temp directory was created, we delete it now.
if dir:
import shutil
shutil.rmtree(dir)
def __call__(self, f, inputs):
"""
Parallel iterator using ``fork()``.
INPUT:
- ``f`` -- a Python function that need not be pickleable or anything else!
- ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
``args`` is a tuple and ``kwds`` is a dictionary.
OUTPUT:
EXAMPLES::
sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
[(([10], {}), 100), (([20], {}), 400)]
sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
[(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]
"""
n = self.ncpus
v = list(inputs)
import os, sys, signal
from sage.structure.sage_object import load
from sage.misc.misc import tmp_dir, walltime
dir = tmp_dir()
timeout = self.timeout
# Subprocesses shouldn't inherit unflushed buffers (cf. #11778):
sys.stdout.flush()
sys.stderr.flush()
workers = {}
try:
while len(v) > 0 or len(workers) > 0:
# Spawn up to n subprocesses
while len(v) > 0 and len(workers) < n:
pid = os.fork()
# The way fork works is that pid returns the
# nonzero pid of the subprocess for the master
# process and returns 0 for the subprocess.
if pid:
# This is the parent master process.
workers[pid] = [v[0], walltime(), '']
del v[0]
else:
# This is the subprocess.
self._subprocess(f, dir, v[0])
if len(workers) > 0:
# Now wait for one subprocess to finish and report the result.
# However, wait at most the time since the oldest process started.
if timeout:
def mysig(a, b):
raise RuntimeError, "SIGALRM"
oldest = min([X[1] for X in workers.values()])
signal.signal(signal.SIGALRM, mysig)
signal.alarm(
max(int(timeout - (walltime() - oldest)), 1))
try:
pid = os.wait()[0]
signal.signal(signal.SIGALRM, signal.SIG_IGN)
except RuntimeError:
signal.signal(signal.SIGALRM, signal.SIG_IGN)
# Kill workers that are too old
for pid, X in workers.iteritems():
if walltime() - X[1] > timeout:
if self.verbose:
print(
"Killing subprocess %s with input %s which took too long"
% (pid, X[0]))
sys.stdout.flush()
os.kill(pid, 9)
X[-1] = ' (timed out)'
else:
# If the computation was interrupted the pid
# might not be in the workers list, in which
# case we skip this.
if pid in workers:
# collect data from process that successfully terminated
sobj = os.path.join(dir, '%s.sobj' % pid)
if not os.path.exists(sobj):
X = "NO DATA" + workers[pid][
-1] # the message field
else:
X = load(sobj, compress=False)
os.unlink(sobj)
out = os.path.join(dir, '%s.out' % pid)
if not os.path.exists(out):
output = "NO OUTPUT"
else:
output = open(out).read()
os.unlink(out)
if output.strip():
print output,
sys.stdout.flush()
yield (workers[pid][0], X)
del workers[pid]
except Exception, msg:
print msg
sys.stdout.flush()