def mksfile(filename):
"""
Create a file safely and return (fd, abspath)
If filename already exists, add '(n)' as suffix before extension (will
try up to os.TMP_MAX or 10000 for n)
Basically, this works in a similar way as _mkstemp_inner from the
standard library 'tempfile' module.
"""
flags = tempfile._bin_openflags
fname, ext = os.path.splitext(filename)
for seq in range(tempfile.TMP_MAX):
if not seq:
pth = filename
else:
pth = "%s(%i)%s" % (fname, seq, ext)
try:
fd = os.open(pth, flags, 0o600)
tempfile._set_cloexec(fd)
return (fd, os.path.abspath(pth))
except OSError as e:
if e.errno == errno.EEXIST:
continue # Try again
raise
raise IOError(errno.EEXIST, "No usable temporary file name found")
def createUniqueFile(req):
"""Create a file with unique file id in upload directory.
Returns file descriptor, path, like tempfile.mkstemp, but in
addition returns unique file id.
"""
create_paths(req)
# XXX we're relying on implementation of tempfile
while True:
file_id = str(random.randrange(sys.maxint))
# do not accept files already known in the repository
# this is normally not changing so this should be relatively
# safe
if os.path.exists(os.path.join(tramline_repository_path(req),
file_id)):
continue # try again
path = os.path.join(tramline_upload_path(req), file_id)
try:
fd = os.open(path, tempfile._bin_openflags)
tempfile._set_cloexec(fd)
return fd, path, file_id
except OSError, e:
if e.errno == errno.EEXIST:
continue # try again
raise
def _try_create_desired_file(mode, bufsize, dir_path, desired_name, delete):
# try to match desired_name directly
# most of this code is adapted from the tempfile source
# pylint: disable=W0212
if 'b' in mode:
flags = tempfile._bin_openflags
else:
flags = tempfile._text_openflags
fd = os.open(os.path.join(dir_path, desired_name), flags, 0600)
tempfile._set_cloexec(fd)
file_name = os.path.abspath(os.path.join(dir_path, desired_name))
file_handle = os.fdopen(fd, mode, bufsize)
return tempfile._TemporaryFileWrapper(file_handle, file_name, delete)
def _try_create_desired_file(mode, bufsize, dir_path, desired_name, delete):
# try to match desired_name directly
# most of this code is adapted from the tempfile source
# pylint: disable=W0212
if 'b' in mode:
flags = tempfile._bin_openflags
else:
flags = tempfile._text_openflags
fd = os.open(os.path.join(dir_path, desired_name), flags, 0600)
tempfile._set_cloexec(fd)
file_name = os.path.abspath(os.path.join(dir_path, desired_name))
file_handle = os.fdopen(fd, mode, bufsize)
return tempfile._TemporaryFileWrapper(file_handle, file_name, delete)
def __init__(self, mode='w+b', bufsize=-1, suffix=["shp", "dbf", "shx"],
prefix=tmp.template, dir=None, delete=True):
from collections import Counter
super(NamedTemporaryFiles, self).__init__()
# first check whether no invalid suffix is part
suffix = list(suffix)
for s in (s for s in suffix if not s):
raise ValueError("Empty suffixes are not allowed!")
# now check whether one suffix occurs multiple times
for item in (item for item in Counter(suffix).items() if item[1] > 1):
raise ValueError("Found suffix %s multiple (%i) times!" % item)
suffix = ['.'+s for s in suffix]
files = [0] * len(suffix)
success = False
for _ in xrange(tmp.TMP_MAX):
f = tmp.NamedTemporaryFile(
mode=mode, bufsize=bufsize, suffix=suffix[0], prefix=prefix,
dir=dir, delete=delete)
basename = os.path.splitext(f.name)[0]
success = all(not os.path.exists(basename+s) for s in suffix[1:])
if success:
break
if not success:
f.close()
raise IOError("No usable temporary file name found")
if 'b' in mode:
flags = tmp._bin_openflags
else:
flags = tmp._text_openflags
# Setting O_TEMPORARY in the flags causes the OS to delete
# the file when it is closed. This is only supported by Windows.
if os.name == 'nt' and delete:
flags |= os.O_TEMPORARY
files[0] = f
for i, s in enumerate(suffix[1:], start=1):
fd = os.open(basename+s, flags, 0600)
tmp._set_cloexec(fd)
f = os.fdopen(fd, mode, bufsize)
files[i] = tmp._TemporaryFileWrapper(f, basename+s, delete=delete)
self.name = basename
self.update(zip([s[1:] for s in suffix], files))
def mkstemp(dirpath, prefix='', suffix='', unlink=1, mode=0600):
"""Creates a file in directory *dir* using *prefix* and *suffix* to
name it:
(dir)/<prefix><random_string><postfix>
Returns a couple of files (pysec.io.fd.File):
(Read_File, Write_File)
If *unlink* is true registers a unlink function at exit.
"""
dirpath = os.path.abspath(dirpath)
mode = int(mode)
if not fcheck.mode_check(mode):
raise ValueError("wrong mode: %r" % oct(mode))
names = _get_candidate_names()
for _ in xrange(TMP_MAX):
name = names.next()
fpath = os.path.join(dirpath, '%s%s%s' % (prefix, name, suffix))
if unlink:
atexit.register(os.unlink, fpath)
fdr = fdw = -1
try:
fdr = os.open(fpath, os.O_RDONLY | os.O_CREAT | os.O_EXCL, mode)
fdw = os.open(fpath, os.O_WRONLY, mode)
_set_cloexec(fdr)
_set_cloexec(fdw)
return fd.File(fdr), fd.File(fdw)
except OSError, ex:
if fdr != -1:
os.close(fdr)
if fdw != -1:
os.close(fdw)
if ex.errno == errno.EEXIST:
continue
else:
try:
os.unlink(fpath)
except IOError:
pass
raise
except:
def mkstemp(dirpath, prefix='', suffix='', unlink=1, mode=0600):
"""Creates a file in directory *dir* using *prefix* and *suffix* to
name it:
(dir)/<prefix><random_string><postfix>
Returns a couple of files (pysec.io.fd.File):
(Read_File, Write_File)
If *unlink* is true registers a unlink function at exit.
"""
dirpath = os.path.abspath(dirpath)
mode = int(mode)
if not fcheck.mode_check(mode):
raise ValueError("wrong mode: %r" % oct(mode))
names = _get_candidate_names()
for _ in xrange(TMP_MAX):
name = names.next()
fpath = os.path.join(dirpath, '%s%s%s' % (prefix, name, suffix))
if unlink:
atexit.register(os.unlink, fpath)
fdr = fdw = -1
try:
fdr = os.open(fpath, os.O_RDONLY | os.O_CREAT | os.O_EXCL, mode)
fdw = os.open(fpath, os.O_WRONLY, mode)
_set_cloexec(fdr)
_set_cloexec(fdw)
return fd.File(fdr), fd.File(fdw)
except OSError, ex:
if fdr != -1:
os.close(fdr)
if fdw != -1:
os.close(fdw)
if ex.errno == errno.EEXIST:
continue
else:
try:
os.unlink(fpath)
except IOError:
pass
raise
except:
def dump_item(inbox, type='file', prefix=None, suffix=None, dir=None, \
timeout=320, buffer=None):
"""
Writes the first element of the inbox as a file of a specified type. The
type can be 'file', 'fifo' or 'socket' corresponding to typical files,
named pipes (FIFOs). FIFOs and TCP sockets and are volatile i.e. exists only
as long as the Python process, which created them. FIFOs are local i.e.
allow to communicate processes only on the same computer.
This function returns a semi-random name of the file written. By default
creates files and fifos in the default temporary directory. To use named
pipes the operating system has to support both forks and fifos (not
Windows). Sockets should work on all operating systems.
This function is useful to efficently communicate parallel ``Pipers``
without the overhead of using queues.
Arguments:
- type('file', 'fifo', 'socket') [default: 'file'] Type of the created
file-like object.
- prefix(``str``) [default: ``"tmp_papy_%type%"``] Prefix of the file to
be created. Should probably identify the ``Worker`` and ``Piper`` type.
- suffix(``str``) [default: ``''``] Suffix of the file to be created.
Should probably identify the format of the serialization protocol e.g.
``"pickle"`` or deserialized data e.g. ``"nubox"``.
- dir(``str``) [default: ``tempfile.gettempdir()``] Directory to safe the
file to. (can be changed only for types ``"file"`` and ``"fifo"``)
- timeout(``int``) [default: ``320``] Number of seconds to keep the
process at the write-end of the ``"socket"`` or ``"pipe"`` alive.
"""
# get a random filename generator
names = tempfile._get_candidate_names()
names.rng.seed() # re-seed rng after the fork
# try to own the file
if type in ('file', 'fifo'):
prefix = prefix or 'tmp_papy_%s' % type
suffix = suffix or ''
dir = dir or tempfile.gettempdir()
while True:
# create a random file name
file = prefix + names.next() + suffix#IGNORE:W0622
if type in ('file', 'fifo'):
file = os.path.join(dir, file)
try:
if type == 'file':
fd = os.open(file, tempfile._bin_openflags, 0600)
tempfile._set_cloexec(fd) # ?, but still open
elif type == 'fifo':
os.mkfifo(file)
file = os.path.abspath(file)
break
except OSError, excp:
# first try to close the fd
try:
os.close(fd)
except OSError, excp_:
if excp_.errno == errno.EBADF:
pass
# strange error better raise it
raise excp_
if excp.errno == errno.EEXIST:
# file exists try another one
continue
# all other errors should be raise
raise excp
