예제 #1
0
파일: qsieve.py 프로젝트: wdv4758h/sage
def tmpdir():
    global _tmp_dir
    if _tmp_dir:
        return
    X = tmp_dir('qsieve')
    os.environ['TMPDIR'] = X
    _tmp_dir = True
예제 #2
0
def tmpdir():
    global _tmp_dir
    if _tmp_dir:
        return
    X = tmp_dir('qsieve')
    os.environ['TMPDIR'] = X
    _tmp_dir = True
예제 #3
0
파일: qsieve.py 프로젝트: yjjcc/sage
def qsieve_block(n, time, verbose=False):
    """
    Compute the factorization of n using Hart's quadratic Sieve
    blocking until complete.
    """

    cmd = ['QuadraticSieve']
    if time:
        cmd = ['time'] + cmd

    if six.PY2:
        enc_kwds = {}
    else:
        enc_kwds = {'encoding': 'latin1'}

    env = os.environ.copy()
    env['TMPDIR'] = tmp_dir('qsieve')
    p = sp.Popen(cmd,
                 env=env,
                 stdout=sp.PIPE,
                 stderr=sp.STDOUT,
                 stdin=sp.PIPE,
                 **enc_kwds)
    out, err = p.communicate(str(n))
    z = data_to_list(out, n, time=time)
    if verbose:
        print(z[-1])
    return z[:2]
예제 #4
0
def unpickle_all(dir, debug=False):
    """
    Unpickle all sobj's in the given directory, reporting failures as
    they occur.  Also printed the number of successes and failure.
    
    INPUT:

    - ``dir`` - string; a directory or name of a .tar.bz2 file that
      decompresses to give a directo pickirectory.

    EXAMPLES::
    
        sage: dir = tmp_dir()
        sage: sage.structure.sage_object.picklejar('hello', dir)
        sage: sage.structure.sage_object.unpickle_all(dir)
        Successfully unpickled 1 objects.
        Failed to unpickle 0 objects.

    We unpickle the standard pickle jar. This doctest tests that
    all "standard pickles" unpickle.  Every so often the standard pickle jar
    should be updated by running the doctest suite with the environment variable
    SAGE_PICKLE_JAR set, then copying the files from SAGE_ROOT/tmp/pickle_jar*
    into the standard pickle jar.

    ::
    
        sage: std = os.environ['SAGE_DATA'] + '/extcode/pickle_jar/pickle_jar.tar.bz2'
        sage: sage.structure.sage_object.unpickle_all(std)
        doctest:...: DeprecationWarning: RQDF is deprecated; use RealField(212) instead.
        Successfully unpickled 487 objects.
        Failed to unpickle 0 objects.
    """
    i = 0
    j = 0
    failed = []
    tracebacks = []
    if dir.endswith('.tar.bz2'):
        # create a temporary directory
        from sage.misc.all import tmp_dir
        T = tmp_dir()
        # extract tarball to it
        os.system('cd "%s"; bunzip2 -c "%s" | tar fx - ' %
                  (T, os.path.abspath(dir)))
        # Now use the directory in the tarball instead of dir
        dir = T + "/" + os.listdir(T)[0]

    for A in sorted(os.listdir(dir)):
        if A.endswith('.sobj'):
            try:
                load(dir + '/' + A)
                i += 1
            except Exception, msg:
                j += 1
                print "** failed: ", A
                failed.append(A)
                if debug:
                    tracebacks.append(sys.exc_info())
예제 #5
0
파일: qsieve.py 프로젝트: yjjcc/sage
 def __init__(self, n, time):
     self._n = n
     if time:
         cmd = 'time QuadraticSieve'
     else:
         cmd = 'QuadraticSieve'
     env = os.environ.copy()
     env['TMPDIR'] = tmp_dir('qsieve')
     self._p = SageSpawn(cmd, env=env)
     cleaner.cleaner(self._p.pid, 'QuadraticSieve')
     self._p.sendline(str(self._n) + '\n\n\n')
     self._done = False
     self._out = ''
     self._time = ''
     self._do_time = time
예제 #6
0
 def __init__(self, n, time):
     self._n = n
     if time:
         cmd = 'time QuadraticSieve'
     else:
         cmd = 'QuadraticSieve'
     env = os.environ.copy()
     env['TMPDIR'] = tmp_dir('qsieve')
     self._p = SageSpawn(cmd, env=env)
     cleaner.cleaner(self._p.pid, 'QuadraticSieve')
     self._p.sendline(str(self._n)+'\n\n\n')
     self._done = False
     self._out = ''
     self._time = ''
     self._do_time = time
예제 #7
0
def tmp_dir():
    """
    Return a temporary directory.

    OUTPUT:

    String. The absolute filename of the directory.

    EXAMPLES::

        sage: from sage.dev.misc import tmp_dir
        sage: tmp_dir().startswith(str(SAGE_TMP))
        True
    """
    try:
        from sage.misc.all import tmp_dir
        return tmp_dir()
    except ImportError:
        from tempfile import mkdtemp
        return mkdtemp(dir=get_sage_tmp())
예제 #8
0
파일: misc.py 프로젝트: Babyll/sage
def tmp_dir():
    """
    Return a temporary directory.

    OUTPUT:

    String. The absolute filename of the directory.

    EXAMPLES::

        sage: from sage.dev.misc import tmp_dir
        sage: tmp_dir().startswith(str(SAGE_TMP))
        True
    """
    try:
        from sage.misc.all import tmp_dir
        return tmp_dir()
    except ImportError:
        from tempfile import mkdtemp
        return mkdtemp(dir=get_sage_tmp())
예제 #9
0
def qsieve_block(n, time, verbose=False):
    """
    Compute the factorization of n using Hart's quadratic Sieve
    blocking until complete.
    """

    cmd = ['QuadraticSieve']
    if time:
        cmd = ['time'] + cmd

    if six.PY2:
        enc_kwds = {}
    else:
        enc_kwds = {'encoding': 'latin1'}

    env = os.environ.copy()
    env['TMPDIR'] = tmp_dir('qsieve')
    p = sp.Popen(cmd, env=env, stdout=sp.PIPE, stderr=sp.STDOUT,
                 stdin=sp.PIPE, **enc_kwds)
    out, err = p.communicate(str(n))
    z = data_to_list(out, n, time=time)
    if verbose:
        print(z[-1])
    return z[:2]
예제 #10
0
def upload_worksheet():
    from sage.misc.all import tmp_filename, tmp_dir
    from werkzeug.utils import secure_filename
    import zipfile

    if g.notebook.readonly_user(g.username):
        return current_app.message(_("Account is in read-only mode"), cont=url_for('home', username=g.username))

    backlinks = _("""Return to <a href="/upload" title="Upload a worksheet"><strong>Upload File</strong></a>.""")

    url = request.values['url'].strip()
    dir = ''
    if url != '':
        #Downloading a file from the internet
        # The file will be downloaded from the internet and saved
        # to a temporary file with the same extension
        path = urlparse.urlparse(url).path
        extension = os.path.splitext(path)[1].lower()
        if extension not in ["", ".txt", ".sws", ".zip", ".html", ".rst"]:
            # Or shall we try to import the document as an sws when in doubt?
            return current_app.message(_("Unknown worksheet extension: %(ext)s. %(links)s", ext=extension, links=backlinks))
        filename = tmp_filename()+extension
        try:
            import re
            matches = re.match("file://(?:localhost)?(/.+)", url)
            if matches:
                if g.notebook.interface != 'localhost':
                    return current_app.message(_("Unable to load file URL's when not running on localhost.\n%(backlinks)s",backlinks=backlinks))

                import shutil
                shutil.copy(matches.group(1),filename)
            else:
                my_urlretrieve(url, filename, backlinks=backlinks)

        except RetrieveError as err:
            return current_app.message(str(err))

    else:
        #Uploading a file from the user's computer
        dir = tmp_dir()
        file = request.files['file']
        if file.filename is None:
            return current_app.message(_("Please specify a worksheet to load.\n%(backlinks)s",backlinks=backlinks))

        filename = secure_filename(file.filename)
        if len(filename)==0:
            return current_app.message(_("Invalid filename.\n%(backlinks)s",backlinks=backlinks))

        filename = os.path.join(dir, filename)
        file.save(filename)

    new_name = request.values.get('name', None)

    try:
        try:
            if filename.endswith('.zip'):
                # Extract all the .sws files from a zip file.
                zip_file = zipfile.ZipFile(filename)
                for subfilename in zip_file.namelist():
                    prefix, extension = os.path.splitext(subfilename)
                    # Mac zip files contain files like __MACOSX/._worksheet.sws
                    # which are metadata files, so we skip those as
                    # well as any other files we won't understand
                    if extension in ['.sws', '.html', '.txt', '.rst'] and not prefix.startswith('__MACOSX/'):
                        tmpfilename = os.path.join(dir, "tmp" + extension)
                        try:
                            tmpfilename = zip_file.extract(subfilename, tmpfilename)
                        except AttributeError:
                            open(tmpfilename, 'w').write(zip_file.read(subfilename))
                        W = g.notebook.import_worksheet(tmpfilename, g.username)
                        if new_name:
                            W.set_name("%s - %s" % (new_name, W.name()))
                    else:
                        print "Unknown extension, file %s is ignored" % subfilename
                return redirect(url_for('home', username=g.username))

            else:
                if url and extension in ['', '.html']:
                    linked_sws = parse_link_rel(url, filename)
                    if linked_sws:
                        # just grab 1st URL; perhaps later add interface for
                        # downloading multiple linked .sws
                        try:
                            filename = my_urlretrieve(linked_sws[0]['url'], backlinks=backlinks)[0]
                            print 'Importing {0}, linked to from {1}'.format(linked_sws[0]['url'], url)
                        except RetrieveError as err:
                            return current_app.message(str(err))
                W = g.notebook.import_worksheet(filename, g.username)
        except Exception, msg:
            print 'error uploading worksheet', msg
            s = _('There was an error uploading the worksheet.  It could be an old unsupported format or worse.  If you desperately need its contents contact the <a href="http://groups.google.com/group/sage-support">sage-support group</a> and post a link to your worksheet.  Alternatively, an sws file is just a bzip2 tarball; take a look inside!\n%(backlinks)s', backlinks=backlinks)
            return current_app.message(s, url_for('home', username=g.username))
        finally:
            # Clean up the temporarily uploaded filename.
            os.unlink(filename)
            # if a temp directory was created, we delete it now.
            if dir:
                import shutil
                shutil.rmtree(dir)
예제 #11
0
    def __call__(self, f, inputs):
        """
        Parallel iterator using ``fork()``.

        INPUT:

        - ``f`` -- a Python function that need not be pickleable or anything else!

        - ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
          ``args`` is a tuple and ``kwds`` is a dictionary.

        OUTPUT:

        EXAMPLES::

            sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
            sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
            [(([10], {}), 100), (([20], {}), 400)]
            sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
            [(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]

        TESTS:

        The output of functions decorated with :func:parallel is read
        as a pickle by the parent process. We intentionally break the
        unpickling and demonstrate that this failure is handled
        gracefully (an exception is displayed and an empty list is
        returned)::

            sage: Polygen = parallel(polygen)
            sage: list(Polygen([QQ]))
            [(((Rational Field,), {}), x)]
            sage: from sage.structure.sage_object import unpickle_override, register_unpickle_override
            sage: register_unpickle_override('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint', Integer)
            sage: L = list(Polygen([QQ]))
            ('__init__() takes at most 2 positional arguments (4 given)', <type 'sage.rings.integer.Integer'>, (Univariate Polynomial Ring in x over Rational Field, [0, 1], False, True))
            sage: L
            []

        Fix the unpickling::

            sage: del unpickle_override[('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint')]
            sage: list(Polygen([QQ,QQ]))
            [(((Rational Field,), {}), x), (((Rational Field,), {}), x)]
        """
        n = self.ncpus
        v = list(inputs)
        import os, sys, signal
        from sage.structure.sage_object import load
        from sage.misc.all import tmp_dir, walltime
        dir = tmp_dir()
        timeout = self.timeout

        workers = {}
        try:
            while len(v) > 0 or len(workers) > 0:
                # Spawn up to n subprocesses
                while len(v) > 0 and len(workers) < n:
                    # Subprocesses shouldn't inherit unflushed buffers (cf. #11778):
                    sys.stdout.flush()
                    sys.stderr.flush()

                    pid = os.fork()
                    # The way fork works is that pid returns the
                    # nonzero pid of the subprocess for the master
                    # process and returns 0 for the subprocess.
                    if pid:
                        # This is the parent master process.
                        workers[pid] = [v[0], walltime(), '']
                        del v[0]
                    else:
                        # This is the subprocess.
                        self._subprocess(f, dir, v[0])

                if len(workers) > 0:
                    # Now wait for one subprocess to finish and report the result.
                    # However, wait at most the time since the oldest process started.
                    if timeout:
                        oldest = min([X[1] for X in workers.values()])
                        alarm(max(timeout - (walltime() - oldest), 0.1))

                    try:
                        pid = os.wait()[0]
                        cancel_alarm()
                        w = workers.pop(pid)
                    except AlarmInterrupt:
                        cancel_alarm()
                        # Kill workers that are too old
                        for pid, X in workers.iteritems():
                            if walltime() - X[1] > timeout:
                                if self.verbose:
                                    print(
                                        "Killing subprocess %s with input %s which took too long"
                                        % (pid, X[0]))
                                os.kill(pid, signal.SIGKILL)
                                X[-1] = ' (timed out)'
                    except KeyError:
                        # Some other process exited, not our problem...
                        pass
                    else:
                        # collect data from process that successfully terminated
                        sobj = os.path.join(dir, '%s.sobj' % pid)
                        if not os.path.exists(sobj):
                            X = "NO DATA" + w[-1]  # the message field
                        else:
                            X = load(sobj, compress=False)
                            os.unlink(sobj)
                        out = os.path.join(dir, '%s.out' % pid)
                        if not os.path.exists(out):
                            output = "NO OUTPUT"
                        else:
                            output = open(out).read()
                            os.unlink(out)

                        if output.strip():
                            print output,

                        yield (w[0], X)

        except Exception as msg:
            print msg

        finally:
            # Clean up all temporary files.
            try:
                for X in os.listdir(dir):
                    os.unlink(os.path.join(dir, X))
                os.rmdir(dir)
            except OSError as msg:
                if self.verbose:
                    print msg

            # Send "kill -9" signal to workers that are left.
            if len(workers) > 0:
                if self.verbose:
                    print "Killing any remaining workers..."
                sys.stdout.flush()
                for pid in workers.keys():
                    try:
                        os.kill(pid, signal.SIGKILL)
                        os.waitpid(pid, 0)
                    except OSError as msg:
                        if self.verbose:
                            print msg
예제 #12
0
def upload_worksheet():
    from sage.misc.all import tmp_filename, tmp_dir
    from werkzeug.utils import secure_filename
    import zipfile

    if g.notebook.readonly_user(g.username):
        return current_app.message(_("Account is in read-only mode"),
                                   cont=url_for('home', username=g.username))

    backlinks = _(
        """Return to <a href="/upload" title="Upload a worksheet"><strong>Upload File</strong></a>."""
    )

    url = request.values['url'].strip()
    dir = ''
    if url != '':
        #Downloading a file from the internet
        # The file will be downloaded from the internet and saved
        # to a temporary file with the same extension
        path = urlparse.urlparse(url).path
        extension = os.path.splitext(path)[1].lower()
        if extension not in ["", ".txt", ".sws", ".zip", ".html", ".rst"]:
            # Or shall we try to import the document as an sws when in doubt?
            return current_app.message(
                _("Unknown worksheet extension: %(ext)s. %(links)s",
                  ext=extension,
                  links=backlinks))
        filename = tmp_filename() + extension
        try:
            import re
            matches = re.match("file://(?:localhost)?(/.+)", url)
            if matches:
                if g.notebook.interface != 'localhost':
                    return current_app.message(
                        _("Unable to load file URL's when not running on localhost.\n%(backlinks)s",
                          backlinks=backlinks))

                import shutil
                shutil.copy(matches.group(1), filename)
            else:
                my_urlretrieve(url, filename, backlinks=backlinks)

        except RetrieveError as err:
            return current_app.message(str(err))

    else:
        #Uploading a file from the user's computer
        dir = tmp_dir()
        file = request.files['file']
        if file.filename is None:
            return current_app.message(
                _("Please specify a worksheet to load.\n%(backlinks)s",
                  backlinks=backlinks))

        filename = secure_filename(file.filename)
        if len(filename) == 0:
            return current_app.message(
                _("Invalid filename.\n%(backlinks)s", backlinks=backlinks))

        filename = os.path.join(dir, filename)
        file.save(filename)

    new_name = request.values.get('name', None)

    try:
        try:
            if filename.endswith('.zip'):
                # Extract all the .sws files from a zip file.
                zip_file = zipfile.ZipFile(filename)
                for subfilename in zip_file.namelist():
                    prefix, extension = os.path.splitext(subfilename)
                    # Mac zip files contain files like __MACOSX/._worksheet.sws
                    # which are metadata files, so we skip those as
                    # well as any other files we won't understand
                    if extension in ['.sws', '.html', '.txt', '.rst'
                                     ] and not prefix.startswith('__MACOSX/'):
                        tmpfilename = os.path.join(dir, "tmp" + extension)
                        try:
                            tmpfilename = zip_file.extract(
                                subfilename, tmpfilename)
                        except AttributeError:
                            open(tmpfilename,
                                 'w').write(zip_file.read(subfilename))
                        W = g.notebook.import_worksheet(
                            tmpfilename, g.username)
                        if new_name:
                            W.set_name("%s - %s" % (new_name, W.name()))
                    else:
                        print "Unknown extension, file %s is ignored" % subfilename
                return redirect(url_for('home', username=g.username))

            else:
                if url and extension in ['', '.html']:
                    linked_sws = parse_link_rel(url, filename)
                    if linked_sws:
                        # just grab 1st URL; perhaps later add interface for
                        # downloading multiple linked .sws
                        try:
                            filename = my_urlretrieve(linked_sws[0]['url'],
                                                      backlinks=backlinks)[0]
                            print 'Importing {0}, linked to from {1}'.format(
                                linked_sws[0]['url'], url)
                        except RetrieveError as err:
                            return current_app.message(str(err))
                W = g.notebook.import_worksheet(filename, g.username)
        except Exception, msg:
            print 'error uploading worksheet', msg
            s = _(
                'There was an error uploading the worksheet.  It could be an old unsupported format or worse.  If you desperately need its contents contact the <a href="http://groups.google.com/group/sage-support">sage-support group</a> and post a link to your worksheet.  Alternatively, an sws file is just a bzip2 tarball; take a look inside!\n%(backlinks)s',
                backlinks=backlinks)
            return current_app.message(s, url_for('home', username=g.username))
        finally:
            # Clean up the temporarily uploaded filename.
            os.unlink(filename)
            # if a temp directory was created, we delete it now.
            if dir:
                import shutil
                shutil.rmtree(dir)
예제 #13
0
파일: use_fork.py 프로젝트: ProgVal/sage
    def __call__(self, f, inputs):
        """
        Parallel iterator using ``fork()``.

        INPUT:

        - ``f`` -- a Python function that need not be pickleable or anything else!

        - ``inputs`` -- a list of pickleable pairs ``(args, kwds)``, where
          ``args`` is a tuple and ``kwds`` is a dictionary.

        OUTPUT:

        EXAMPLES::

            sage: F = sage.parallel.use_fork.p_iter_fork(2,3)
            sage: sorted(list( F( (lambda x: x^2), [([10],{}), ([20],{})])))
            [(([10], {}), 100), (([20], {}), 400)]
            sage: sorted(list( F( (lambda x, y: x^2+y), [([10],{'y':1}), ([20],{'y':2})])))
            [(([10], {'y': 1}), 101), (([20], {'y': 2}), 402)]

        TESTS:

        The output of functions decorated with :func:parallel is read
        as a pickle by the parent process. We intentionally break the
        unpickling and demonstrate that this failure is handled
        gracefully (an exception is displayed and an empty list is
        returned)::

            sage: Polygen = parallel(polygen)
            sage: list(Polygen([QQ]))
            [(((Rational Field,), {}), x)]
            sage: from sage.structure.sage_object import unpickle_override, register_unpickle_override
            sage: register_unpickle_override('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint', Integer)
            sage: L = list(Polygen([QQ]))
            ('__init__() takes at most 2 positional arguments (4 given)', <type 'sage.rings.integer.Integer'>, (Univariate Polynomial Ring in x over Rational Field, [0, 1], False, True))
            sage: L
            []

        Fix the unpickling::

            sage: del unpickle_override[('sage.rings.polynomial.polynomial_rational_flint', 'Polynomial_rational_flint')]
            sage: list(Polygen([QQ,QQ]))
            [(((Rational Field,), {}), x), (((Rational Field,), {}), x)]
        """
        n = self.ncpus
        v = list(inputs)
        import os, sys, signal
        from sage.structure.sage_object import load
        from sage.misc.all import tmp_dir, walltime
        dir = tmp_dir()
        timeout = self.timeout

        workers = {}
        try:
            while len(v) > 0 or len(workers) > 0:
                # Spawn up to n subprocesses
                while len(v) > 0 and len(workers) < n:
                    # Subprocesses shouldn't inherit unflushed buffers (cf. #11778):
                    sys.stdout.flush()
                    sys.stderr.flush()

                    pid = os.fork()
                    # The way fork works is that pid returns the
                    # nonzero pid of the subprocess for the master
                    # process and returns 0 for the subprocess.
                    if pid:
                        # This is the parent master process.
                        workers[pid] = [v[0], walltime(), '']
                        del v[0]
                    else:
                        # This is the subprocess.
                        self._subprocess(f, dir, v[0])

                if len(workers) > 0:
                    # Now wait for one subprocess to finish and report the result.
                    # However, wait at most the time since the oldest process started.
                    if timeout:
                        oldest = min([X[1] for X in workers.values()])
                        alarm(max(timeout - (walltime()-oldest), 0.1))

                    try:
                        pid = os.wait()[0]
                        cancel_alarm()
                        w = workers.pop(pid)
                    except AlarmInterrupt:
                        cancel_alarm()
                        # Kill workers that are too old
                        for pid, X in workers.iteritems():
                            if walltime() - X[1] > timeout:
                                if self.verbose:
                                    print(
                                        "Killing subprocess %s with input %s which took too long"
                                         % (pid, X[0]) )
                                os.kill(pid, signal.SIGKILL)
                                X[-1] = ' (timed out)'
                    except KeyError:
                        # Some other process exited, not our problem...
                        pass
                    else:
                        # collect data from process that successfully terminated
                        sobj = os.path.join(dir, '%s.sobj'%pid)
                        if not os.path.exists(sobj):
                            X = "NO DATA" + w[-1]  # the message field
                        else:
                            X = load(sobj, compress=False)
                            os.unlink(sobj)
                        out = os.path.join(dir, '%s.out'%pid)
                        if not os.path.exists(out):
                            output = "NO OUTPUT"
                        else:
                            output = open(out).read()
                            os.unlink(out)

                        if output.strip():
                            print output,

                        yield (w[0], X)

        except Exception as msg:
            print msg

        finally:
            # Clean up all temporary files.
            try:
                for X in os.listdir(dir):
                    os.unlink(os.path.join(dir, X))
                os.rmdir(dir)
            except OSError as msg:
                if self.verbose:
                    print msg

            # Send "kill -9" signal to workers that are left.
            if len(workers) > 0:
                if self.verbose:
                    print "Killing any remaining workers..."
                sys.stdout.flush()
                for pid in workers.keys():
                    try:
                        os.kill(pid, signal.SIGKILL)
                        os.waitpid(pid, 0)
                    except OSError as msg:
                        if self.verbose:
                            print msg