Exemplo n.º 1
0
def main(infile, outfile, level, jobs=1):
    from tempfile import TemporaryDirectory
    from shutil import copy

    class OptimizeOptions:
        """Emulate ocrmypdf's options"""

        def __init__(self, jobs, optimize, jpeg_quality, png_quality, jb2lossy):
            self.jobs = jobs
            self.optimize = optimize
            self.jpeg_quality = jpeg_quality
            self.png_quality = png_quality
            self.jbig2_page_group_size = 0
            self.jbig2_lossy = jb2lossy

    logging.basicConfig(level=logging.DEBUG)
    log = logging.getLogger()

    ctx = JobContext()
    options = OptimizeOptions(
        jobs=jobs,
        optimize=int(level),
        jpeg_quality=0,  # Use default
        png_quality=0,
        jb2lossy=False,
    )
    ctx.set_options(options)

    with TemporaryDirectory() as td:
        tmpout = Path(td) / 'out.pdf'
        optimize(infile, tmpout, log, ctx)
        copy(fspath(tmpout), fspath(outfile))
Exemplo n.º 2
0
def relpath(path, start=None):
    """Return a relative version of a path"""

    if not path:
        raise ValueError("no path specified")

    path = os.fspath(path)
    if isinstance(path, bytes):
        curdir = b'.'
        sep = b'/'
        pardir = b'..'
    else:
        curdir = '.'
        sep = '/'
        pardir = '..'

    if start is None:
        start = curdir
    else:
        start = os.fspath(start)

    try:
        start_list = [x for x in abspath(start).split(sep) if x]
        path_list = [x for x in abspath(path).split(sep) if x]
        # Work out how much of the filepath is shared by start and path.
        i = len(commonprefix([start_list, path_list]))

        rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
        if not rel_list:
            return curdir
        return join(*rel_list)
    except (TypeError, AttributeError, BytesWarning, DeprecationWarning):
        genericpath._check_arg_types('relpath', path, start)
        raise
Exemplo n.º 3
0
def test_leptonica_compile(tmpdir):
    from ocrmypdf.lib.compile_leptonica import ffibuilder

    # Compile the library but build it somewhere that won't interfere with
    # existing compiled library. Also compile in API mode so that we test
    # the interfaces, even though we use it ABI mode.
    ffibuilder.compile(tmpdir=fspath(tmpdir), target=fspath(tmpdir / 'lepttest.*'))
Exemplo n.º 4
0
def test_no_unpaper(resources, no_outpdf):
    input_ = fspath(resources / "c02-22.pdf")
    output = fspath(no_outpdf)
    options = main.parser.parse_args(args=["--clean", input_, output])

    with patch("ocrmypdf.exec.unpaper.version") as mock_unpaper_version:
        mock_unpaper_version.side_effect = FileNotFoundError("unpaper")
        with pytest.raises(SystemExit):
            main.check_options(options, log=logging.getLogger())
Exemplo n.º 5
0
def test_srgb_in_unicode_path(tmpdir):
    """Test that we can produce pdfmark when install path is not ASCII"""

    dstdir = Path(fspath(tmpdir)) / b'\xe4\x80\x80'.decode('utf-8')
    dstdir.mkdir()
    dst = dstdir / 'sRGB.icc'

    copyfile(SRGB_ICC_PROFILE, fspath(dst))

    with patch('ocrmypdf.pdfa.SRGB_ICC_PROFILE', new=str(dst)):
        generate_pdfa_ps(dstdir / 'out.ps')
Exemplo n.º 6
0
def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0):
    if quiet < 2 and isinstance(dir, os.PathLike):
        dir = os.fspath(dir)
    if not quiet:
        print('Listing {!r}...'.format(dir))
    try:
        names = os.listdir(dir)
    except OSError:
        if quiet < 2:
            print("Can't list {!r}".format(dir))
        names = []
    names.sort()
    for name in names:
        if name == '__pycache__':
            continue
        fullname = os.path.join(dir, name)
        if ddir is not None:
            dfile = os.path.join(ddir, name)
        else:
            dfile = None
        if not os.path.isdir(fullname):
            yield fullname
        elif (maxlevels > 0 and name != os.curdir and name != os.pardir and
              os.path.isdir(fullname) and not os.path.islink(fullname)):
            yield from _walk_dir(fullname, ddir=dfile,
                                 maxlevels=maxlevels - 1, quiet=quiet)
Exemplo n.º 7
0
def _getpath(p):
    if (3, 6) <= sys.version_info:
        import os
        return os.fspath(p)
    if _detect_pathlib_path(p):
        return str(p)
    return p
Exemplo n.º 8
0
def main(*, args):
	params = docopt.docopt(__doc__, argv=args, help=True, version=True, options_first=False)
	command = params.pop("<command>")
	assert not params, params

	print(command, end="\n\n")
	command_path = shutil.which(command)

	command_path_last = None
	while command_path_last != command_path:
		print(command_path)
		command_path_last = command_path

		parts = pathlib.Path(command_path).parts
		assert parts[0] == os.path.sep, parts[0]
		first = list(parts[:1])
		second = list(parts[1:])

		while second:
			first.append(second.pop(0))
			p = pathlib.Path(*first)
			if not p.is_symlink():
				continue
			t = os.readlink(p)
			print("\t", p, " -> ", t, sep="", end="\n\n")
			first = list(pathlib.Path(os.path.normpath((p.parent / t))).parts)
			command_path = os.fspath(pathlib.Path(*first) / pathlib.Path(*second))
			break
		else:
			continue
		command_path_last = None
Exemplo n.º 9
0
def is_file_writable(test_file):
    """Intentionally racy test if target is writable.

    We intend to write to the output file if and only if we succeed and
    can replace it atomically. Before doing the OCR work, make sure
    the location is writable.
    """
    p = Path(test_file)

    if p.is_symlink():
        p = p.resolve(strict=False)

    # p.is_file() throws an exception in some cases
    if p.exists() and p.is_file():
        return os.access(
            os.fspath(p),
            os.W_OK,
            effective_ids=(os.access in os.supports_effective_ids),
        )
    else:
        try:
            fp = p.open('wb')
        except OSError:
            return False
        else:
            fp.close()
            with suppress(OSError):
                p.unlink()
        return True
Exemplo n.º 10
0
    def __init__(self, file=None, parent=None, name=None, **kwargs):
        from .collection import Collection

        if file is None:
            return
        # parent
        if parent is None:
            parent = ""
        if parent == "":
            parent = pathlib.PurePosixPath("/")
            path = pathlib.PurePosixPath("/")
        else:
            path = pathlib.PurePosixPath(parent) / name
        name = path.name if path.name else name
        parent = path.parent
        self.filepath = file.filename
        if name != "" and parent.name != "":
            # Ensure that the parent Collection object is made first
            Collection(file, parent=parent.parent, name=parent.name, edit_local=True)
        file.require_group(str(path))
        h5py.Group.__init__(self, bind=file[str(path)].id)
        self.__n = 0
        self.fid = self.file.id
        self.natural_name = name
        # attrs
        self.attrs["class"] = self.class_name
        if "created" not in self.attrs.keys():
            self.attrs["created"] = wt_kit.TimeStamp().RFC3339
        for key, value in kwargs.items():
            try:
                if isinstance(value, pathlib.Path):
                    value = str(value)
                elif (
                    isinstance(value, list)
                    and len(value) > 0
                    and isinstance(value[0], (str, os.PathLike))
                ):
                    value = np.array(value, dtype="S")
                elif sys.version_info > (3, 6):
                    try:
                        value = os.fspath(value)
                    except TypeError:
                        pass  # Not all things that can be stored have fspath
                self.attrs[key] = value
            except TypeError:
                # some values have no native HDF5 equivalent
                message = "'{}' not included in attrs because its Type ({}) cannot be represented"
                message = message.format(key, type(value))
                warnings.warn(message)
        # the following are populated if not already recorded
        self.__version__
        self.item_names

        parent = file[str(parent)]
        if parent.name == self.name:
            pass  # at root, dont add to item_names
        elif self.natural_name.encode() not in parent.attrs["item_names"]:
            parent.attrs["item_names"] = np.append(
                parent.attrs["item_names"], self.natural_name.encode()
            )
Exemplo n.º 11
0
 def test_excelfile_fspath(self):
     with tm.ensure_clean('foo.xlsx') as path:
         df = DataFrame({"A": [1, 2]})
         df.to_excel(path)
         xl = ExcelFile(path)
         result = os.fspath(xl)
         assert result == path
Exemplo n.º 12
0
def expanduser(path):
    """Expand ~ and ~user constructs.

    If user or $HOME is unknown, do nothing."""
    path = os.fspath(path)
    if isinstance(path, bytes):
        tilde = b'~'
    else:
        tilde = '~'
    if not path.startswith(tilde):
        return path
    i, n = 1, len(path)
    while i < n and path[i] not in _get_bothseps(path):
        i += 1

    if 'HOME' in os.environ:
        userhome = os.environ['HOME']
    elif 'USERPROFILE' in os.environ:
        userhome = os.environ['USERPROFILE']
    elif not 'HOMEPATH' in os.environ:
        return path
    else:
        try:
            drive = os.environ['HOMEDRIVE']
        except KeyError:
            drive = ''
        userhome = join(drive, os.environ['HOMEPATH'])

    if isinstance(path, bytes):
        userhome = os.fsencode(userhome)

    if i != 1: #~user
        userhome = join(dirname(userhome), path[1:i])

    return userhome + path[i:]
Exemplo n.º 13
0
def normcase(s):
    """Normalize case of pathname.  Has no effect under Posix"""
    s = os.fspath(s)
    if not isinstance(s, (bytes, str)):
        raise TypeError("normcase() argument must be str or bytes, "
                        "not '{}'".format(s.__class__.__name__))
    return s
Exemplo n.º 14
0
    def test_magic_methods_fspath(self):
        mock = MagicMock()
        expected_path = mock.__fspath__()
        mock.reset_mock()

        self.assertEqual(os.fspath(mock), expected_path)
        mock.__fspath__.assert_called_once()
Exemplo n.º 15
0
def as_handle(handleish, mode='r', **kwargs):
    r"""Context manager to ensure we are using a handle.

    Context manager for arguments that can be passed to SeqIO and AlignIO read, write,
    and parse methods: either file objects or path-like objects (strings, pathlib.Path
    instances, or more generally, anything that can be handled by the builtin 'open'
    function).

    When given a path-like object, returns an open file handle to that path, with provided
    mode, which will be closed when the manager exits.

    All other inputs are returned, and are *not* closed.

    Arguments:
     - handleish  - Either a file handle or path-like object (anything which can be
                    passed to the builtin 'open' function: str, bytes, and under
                    Python >= 3.6, pathlib.Path, os.DirEntry)
     - mode       - Mode to open handleish (used only if handleish is a string)
     - kwargs     - Further arguments to pass to open(...)

    Examples
    --------
    >>> with as_handle('seqs.fasta', 'w') as fp:
    ...     fp.write('>test\nACGT')
    >>> fp.closed
    True

    >>> handle = open('seqs.fasta', 'w')
    >>> with as_handle(handle) as fp:
    ...     fp.write('>test\nACGT')
    >>> fp.closed
    False
    >>> fp.close()

    Note that if the mode argument includes U (for universal new lines)
    this will be removed under Python 3 where is is redundant and has
    been deprecated (this happens automatically in text mode).

    """
    # If we're running under a version of Python that supports PEP 519, try
    # to convert `handleish` to a string with `os.fspath`.
    if hasattr(os, 'fspath'):
        try:
            handleish = os.fspath(handleish)
        except TypeError:
            # handleish isn't path-like, and it remains unchanged -- we'll yield it below
            pass

    if isinstance(handleish, basestring):
        if sys.version_info[0] >= 3 and "U" in mode:
            mode = mode.replace("U", "")
        if 'encoding' in kwargs:
            with codecs.open(handleish, mode, **kwargs) as fp:
                yield fp
        else:
            with open(handleish, mode, **kwargs) as fp:
                yield fp
    else:
        yield handleish
Exemplo n.º 16
0
 def test_pathlib_file(self):
     t1 = Path(self.writeTmp("Pathlib file."))
     with FileInput(t1) as fi:
         line = fi.readline()
         self.assertEqual(line, 'Pathlib file.')
         self.assertEqual(fi.lineno(), 1)
         self.assertEqual(fi.filelineno(), 1)
         self.assertEqual(fi.filename(), os.fspath(t1))
Exemplo n.º 17
0
    def _readline(self):
        if not self._files:
            if 'b' in self._mode:
                return b''
            else:
                return ''
        self._filename = self._files[0]
        self._files = self._files[1:]
        self._startlineno = self.lineno()
        self._filelineno = 0
        self._file = None
        self._isstdin = False
        self._backupfilename = 0
        if self._filename == '-':
            self._filename = '<stdin>'
            if 'b' in self._mode:
                self._file = getattr(sys.stdin, 'buffer', sys.stdin)
            else:
                self._file = sys.stdin
            self._isstdin = True
        else:
            if self._inplace:
                self._backupfilename = (
                    os.fspath(self._filename) + (self._backup or ".bak"))
                try:
                    os.unlink(self._backupfilename)
                except OSError:
                    pass
                # The next few lines may raise OSError
                os.rename(self._filename, self._backupfilename)
                self._file = open(self._backupfilename, self._mode)
                try:
                    perm = os.fstat(self._file.fileno()).st_mode
                except OSError:
                    self._output = open(self._filename, "w")
                else:
                    mode = os.O_CREAT | os.O_WRONLY | os.O_TRUNC
                    if hasattr(os, 'O_BINARY'):
                        mode |= os.O_BINARY

                    fd = os.open(self._filename, mode, perm)
                    self._output = os.fdopen(fd, "w")
                    try:
                        if hasattr(os, 'chmod'):
                            os.chmod(self._filename, perm)
                    except OSError:
                        pass
                self._savestdout = sys.stdout
                sys.stdout = self._output
            else:
                # This may raise OSError
                if self._openhook:
                    self._file = self._openhook(self._filename, self._mode)
                else:
                    self._file = open(self._filename, self._mode)
        self._readline = self._file.readline  # hide FileInput._readline
        return self._readline()
Exemplo n.º 18
0
def splitext(p):
    p = os.fspath(p)
    if isinstance(p, bytes):
        sep = b'/'
        extsep = b'.'
    else:
        sep = '/'
        extsep = '.'
    return genericpath._splitext(p, sep, None, extsep)
Exemplo n.º 19
0
def dirname(p):
    """Returns the directory component of a pathname"""
    p = os.fspath(p)
    sep = _get_sep(p)
    i = p.rfind(sep) + 1
    head = p[:i]
    if head and head != sep*len(head):
        head = head.rstrip(sep)
    return head
Exemplo n.º 20
0
 def test_magic_methods_fspath(self):
     mock = MagicMock()
     if sys.version_info < (3, 6):
         self.assertRaises(AttributeError, lambda: mock.__fspath__)
     else:
         expected_path = mock.__fspath__()
         mock.reset_mock()
         self.assertEqual(os.fspath(mock), expected_path)
         mock.__fspath__.assert_called_once()
Exemplo n.º 21
0
def test_no_languages(ensure_tess4, tmpdir):
    env = ensure_tess4
    (tmpdir / 'tessdata').mkdir()
    env['TESSDATA_PREFIX'] = fspath(tmpdir)

    with modified_os_environ(env):
        with pytest.raises(MissingDependencyError):
            tesseract.languages.cache_clear()
            tesseract.languages()
Exemplo n.º 22
0
def pkg_config_setup_extension(
        ext, package,
        atleast_version=None, alt_exec=None, default_libraries=()):
    """Add parameters to the given *ext* for the given *package*."""

    # First, try to get the flags from pkg-config.

    pkg_config = get_pkg_config()
    cmd = [pkg_config, package] if pkg_config else alt_exec
    if cmd is not None:
        try:
            if pkg_config and atleast_version:
                subprocess.check_call(
                    [*cmd, f"--atleast-version={atleast_version}"])
            # Use sys.getfilesystemencoding() to allow round-tripping
            # when passed back to later subprocess calls; do not use
            # locale.getpreferredencoding() which universal_newlines=True
            # would do.
            cflags = shlex.split(
                os.fsdecode(subprocess.check_output([*cmd, "--cflags"])))
            libs = shlex.split(
                os.fsdecode(subprocess.check_output([*cmd, "--libs"])))
        except (OSError, subprocess.CalledProcessError):
            pass
        else:
            ext.extra_compile_args.extend(cflags)
            ext.extra_link_args.extend(libs)
            return

    # If that fails, fall back on the defaults.

    # conda Windows header and library paths.
    # https://github.com/conda/conda/issues/2312 re: getting the env dir.
    if sys.platform == 'win32':
        conda_env_path = (os.getenv('CONDA_PREFIX')  # conda >= 4.1
                          or os.getenv('CONDA_DEFAULT_ENV'))  # conda < 4.1
        if conda_env_path and os.path.isdir(conda_env_path):
            ext.include_dirs.append(os.fspath(
                pathlib.Path(conda_env_path, "Library/include")))
            ext.library_dirs.append(os.fspath(
                pathlib.Path(conda_env_path, "Library/lib")))

    # Default linked libs.
    ext.libraries.extend(default_libraries)
Exemplo n.º 23
0
def relpath(path, start=None):
    """Return a relative version of a path"""
    path = os.fspath(path)
    if isinstance(path, bytes):
        sep = b'\\'
        curdir = b'.'
        pardir = b'..'
    else:
        sep = '\\'
        curdir = '.'
        pardir = '..'

    if start is None:
        start = curdir

    if not path:
        raise ValueError("no path specified")

    start = os.fspath(start)
    try:
        start_abs = abspath(normpath(start))
        path_abs = abspath(normpath(path))
        start_drive, start_rest = splitdrive(start_abs)
        path_drive, path_rest = splitdrive(path_abs)
        if normcase(start_drive) != normcase(path_drive):
            raise ValueError("path is on mount %r, start on mount %r" % (
                path_drive, start_drive))

        start_list = [x for x in start_rest.split(sep) if x]
        path_list = [x for x in path_rest.split(sep) if x]
        # Work out how much of the filepath is shared by start and path.
        i = 0
        for e1, e2 in zip(start_list, path_list):
            if normcase(e1) != normcase(e2):
                break
            i += 1

        rel_list = [pardir] * (len(start_list)-i) + path_list[i:]
        if not rel_list:
            return curdir
        return join(*rel_list)
    except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning):
        genericpath._check_arg_types('relpath', path, start)
        raise
Exemplo n.º 24
0
def transcode_jpegs(pike, jpegs, root, log, options):
    for xref in jpegs:
        in_jpg = Path(jpg_name(root, xref))
        opt_jpg = in_jpg.with_suffix('.opt.jpg')

        # This produces a debug warning from PIL
        # DEBUG:PIL.Image:Error closing: 'NoneType' object has no attribute
        # 'close'.  Seems to be mostly harmless
        # https://github.com/python-pillow/Pillow/issues/1144
        with Image.open(fspath(in_jpg)) as im:
            im.save(fspath(opt_jpg), optimize=True, quality=options.jpeg_quality)

        if opt_jpg.stat().st_size > in_jpg.stat().st_size:
            log.debug("xref %s, jpeg, made larger - skip", xref)
            continue

        compdata = leptonica.CompressedData.open(opt_jpg)
        im_obj = pike.get_object(xref, 0)
        im_obj.write(compdata.read(), filter=Name.DCTDecode)
Exemplo n.º 25
0
 def jbig2_group_futures(executor, root, groups):
     for group, xref_exts in groups.items():
         prefix = f'group{group:08d}'
         future = executor.submit(
             jbig2enc.convert_group,
             cwd=fspath(root),
             infiles=(img_name(root, xref, ext) for xref, ext in xref_exts),
             out_prefix=prefix,
         )
         yield future
Exemplo n.º 26
0
 def abspath(path):
     """Return the absolute version of a path."""
     path = os.fspath(path)
     if not isabs(path):
         if isinstance(path, bytes):
             cwd = os.getcwdb()
         else:
             cwd = os.getcwd()
         path = join(cwd, path)
     return normpath(path)
Exemplo n.º 27
0
 def __init__(self, data, options=0, fn=None):
     self.face = None
     if fn:
         self.face = gr2.gr_make_face(bytes(data), fn, options)
     else:
         if not os.path.isfile(data):
             raise FileNotFoundError(errno.ENOENT,
                                     os.strerror(errno.ENOENT), data)
         if hasattr(data, '__fspath__'): data = os.fspath(data)
         self.face = gr2.gr_make_file_face(data.encode('utf_8'), options)
Exemplo n.º 28
0
def split(p):
    """Split a pathname.  Returns tuple "(head, tail)" where "tail" is
    everything after the final slash.  Either part may be empty."""
    p = os.fspath(p)
    sep = _get_sep(p)
    i = p.rfind(sep) + 1
    head, tail = p[:i], p[i:]
    if head and head != sep*len(head):
        head = head.rstrip(sep)
    return head, tail
Exemplo n.º 29
0
    def guessit(self, string, options=None):  # pylint: disable=too-many-branches
        """
        Retrieves all matches from string as a dict
        :param string: the filename or release name
        :type string: str|Path
        :param options:
        :type options: str|dict
        :return:
        :rtype:
        """
        try:
            from pathlib import Path
            if isinstance(string, Path):
                try:
                    # Handle path-like object
                    string = os.fspath(string)
                except AttributeError:
                    string = str(string)
        except ImportError:
            pass

        try:
            options = parse_options(options, True)
            options = self._fix_encoding(options)
            config = self.configure(options, sanitize_options=False)
            options = merge_options(config, options)
            result_decode = False
            result_encode = False

            if six.PY2:
                if isinstance(string, six.text_type):
                    string = string.encode("utf-8")
                    result_decode = True
                elif isinstance(string, six.binary_type):
                    string = six.binary_type(string)
            if six.PY3:
                if isinstance(string, six.binary_type):
                    string = string.decode('ascii')
                    result_encode = True
                elif isinstance(string, six.text_type):
                    string = six.text_type(string)

            matches = self.rebulk.matches(string, options)
            if result_decode:
                for match in matches:
                    if isinstance(match.value, six.binary_type):
                        match.value = match.value.decode("utf-8")
            if result_encode:
                for match in matches:
                    if isinstance(match.value, six.text_type):
                        match.value = match.value.encode("ascii")
            return matches.to_dict(options.get('advanced', False), options.get('single_value', False),
                                   options.get('enforce_list', False))
        except:
            raise GuessitException(string, options)
Exemplo n.º 30
0
    def create_unix_server(self, protocol_factory, path=None, *,
                           sock=None, backlog=100, ssl=None):
        if isinstance(ssl, bool):
            raise TypeError('ssl argument must be an SSLContext or None')

        if path is not None:
            if sock is not None:
                raise ValueError(
                    'path and sock can not be specified at the same time')

            path = os.fspath(path)
            sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)

            # Check for abstract socket. `str` and `bytes` paths are supported.
            if path[0] not in (0, '\x00'):
                try:
                    if stat.S_ISSOCK(os.stat(path).st_mode):
                        os.remove(path)
                except FileNotFoundError:
                    pass
                except OSError as err:
                    # Directory may have permissions only to create socket.
                    logger.error('Unable to check or remove stale UNIX socket '
                                 '%r: %r', path, err)

            try:
                sock.bind(path)
            except OSError as exc:
                sock.close()
                if exc.errno == errno.EADDRINUSE:
                    # Let's improve the error message by adding
                    # with what exact address it occurs.
                    msg = 'Address {!r} is already in use'.format(path)
                    raise OSError(errno.EADDRINUSE, msg) from None
                else:
                    raise
            except:
                sock.close()
                raise
        else:
            if sock is None:
                raise ValueError(
                    'path was not specified, and no sock specified')

            if (sock.family != socket.AF_UNIX or
                    not base_events._is_stream_socket(sock)):
                raise ValueError(
                    'A UNIX Domain Stream Socket was expected, got {!r}'
                    .format(sock))

        server = base_events.Server(self, [sock])
        sock.listen(backlog)
        sock.setblocking(False)
        self._start_serving(protocol_factory, sock, ssl, server)
        return server
Exemplo n.º 31
0
def splitdrive(p):
    """Split a pathname into drive and path. On Posix, drive is always
    empty."""
    p = os.fspath(p)
    return p[:0], p
Exemplo n.º 32
0
def realpath(filename):
    """Return the canonical path of the specified filename, eliminating any
symbolic links encountered in the path."""
    filename = os.fspath(filename)
    path, ok = _joinrealpath(filename[:0], filename, {})
    return abspath(path)
Exemplo n.º 33
0
def isabs(s):
    """Test whether a path is absolute"""
    s = os.fspath(s)
    sep = _get_sep(s)
    return s.startswith(sep)
import os
from pathlib import Path
import logging


PARENT_PATH = os.fspath(Path(__file__).parents[0])
LOGGING_FILE_PATH = os.path.join(PARENT_PATH,
                                 "__logger",
                                 "{}.log")


def set_logger(file_path_extension):
    '''A logging helper.
    Keeps the logged experiments in the __logger path.
    Both prints out on the Terminal and writes on the
    .log file.'''
    logging.basicConfig(
        level=logging.INFO,
        format="%(asctime)-7s: %(levelname)-1s %(message)s",
        datefmt="%Y-%m-%d %H:%M:%S",
        handlers=[
            logging.FileHandler(
                LOGGING_FILE_PATH.format(file_path_extension)
                ),
            logging.StreamHandler()
        ])
    return logging
Exemplo n.º 35
0
Arquivo: db.py Projeto: SNH48Live/wxj
import os
import peewee

from .common import DATADIR

DBPATH = DATADIR / 'data.db'
db = peewee.SqliteDatabase(os.fspath(DBPATH))


class Status(peewee.Model):
    status_id = peewee.IntegerField(unique=True)
    created_at = peewee.IntegerField()
    body = peewee.TextField()
    images = peewee.TextField()

    repost = peewee.BooleanField(default=False)
    orig_body = peewee.TextField(default='')
    orig_images = peewee.TextField(default='')

    class Meta:
        database = db


db.connect()
db.create_table(Status, safe=True)
Exemplo n.º 36
0
def isabs(s):
    """Test whether a path is absolute"""
    s = os.fspath(s)
    s = splitdrive(s)[1]
    return len(s) > 0 and s[0] in _get_bothseps(s)
Exemplo n.º 37
0
def create_new_project(project,
                       experimenter,
                       videos,
                       working_directory=None,
                       copy_videos=False,
                       videotype='.avi'):
    """Creates a new project directory, sub-directories and a basic configuration file. The configuration file is loaded with the default values. Change its parameters to your projects need.

    Parameters
    ----------
    project : string
        String containing the name of the project.

    experimenter : string
        String containing the name of the experimenter.

    videos : list
        A list of string containing the full paths of the videos to include in the project.
        Attention: Can also be a directory, then all videos of videotype will be imported. Do not pass it as a list!

    working_directory : string, optional
        The directory where the project will be created. The default is the ``current working directory``; if provided, it must be a string.

    copy_videos : bool, optional
        If this is set to True, the videos are copied to the ``videos`` directory. If it is False,symlink of the videos are copied to the project/videos directory. The default is ``False``; if provided it must be either
        ``True`` or ``False``.

    Example
    --------
    Linux/MacOs
    >>> deeplabcut.create_new_project('reaching-task','Linus',['/data/videos/mouse1.avi','/data/videos/mouse2.avi','/data/videos/mouse3.avi'],'/analysis/project/')
    >>> deeplabcut.create_new_project('reaching-task','Linus','/data/videos',videotype='.mp4')

    Windows:
    >>> deeplabcut.create_new_project('reaching-task','Bill',[r'C:\yourusername\rig-95\Videos\reachingvideo1.avi'], copy_videos=True)
    Users must format paths with either:  r'C:\ OR 'C:\\ <- i.e. a double backslash \ \ )

    """
    from datetime import datetime as dt
    from deeplabcut.utils import auxiliaryfunctions
    date = dt.today()
    month = date.strftime("%B")
    day = date.day
    d = str(month[0:3] + str(day))
    date = dt.today().strftime('%Y-%m-%d')
    if working_directory == None:
        working_directory = '.'
    wd = Path(working_directory).resolve()
    project_name = '{pn}-{exp}-{date}'.format(pn=project,
                                              exp=experimenter,
                                              date=date)
    project_path = wd / project_name

    # Create project and sub-directories
    if not DEBUG and project_path.exists():
        print('Project "{}" already exists!'.format(project_path))
        return
    video_path = project_path / 'videos'
    data_path = project_path / 'labeled-data'
    shuffles_path = project_path / 'training-datasets'
    results_path = project_path / 'dlc-models'
    for p in [video_path, data_path, shuffles_path, results_path]:
        p.mkdir(parents=True, exist_ok=DEBUG)
        print('Created "{}"'.format(p))

    # Import all videos in a folder or if just one video withouth [] passed, then make it a list.
    if isinstance(videos, str):
        #there are two cases:
        if os.path.isdir(videos):  # it is a path!
            path = videos
            videos = [
                os.path.join(path, vp) for vp in os.listdir(path)
                if videotype in vp
            ]
            if len(videos) == 0:
                print("No videos found in", path, os.listdir(path))
                print(
                    "Perhaps change the videotype, which is currently set to:",
                    videotype)
            else:
                print("Directory entered, ", len(videos),
                      " videos were found.")
        else:
            if os.path.isfile(videos):
                videos = [videos]

    videos = [Path(vp) for vp in videos]
    dirs = [data_path / Path(i.stem) for i in videos]
    for p in dirs:
        """
        Creates directory under data
        """
        p.mkdir(parents=True, exist_ok=True)

    destinations = [video_path.joinpath(vp.name) for vp in videos]
    if copy_videos == True:
        print("Copying the videos")
        for src, dst in zip(videos, destinations):
            shutil.copy(
                os.fspath(src),
                os.fspath(dst))  #https://www.python.org/dev/peps/pep-0519/
            #https://github.com/AlexEMG/DeepLabCut/issues/105 (for windows)
            #try:
            #    #shutil.copy(src,dst)
            #except OSError or TypeError: #https://github.com/AlexEMG/DeepLabCut/issues/105 (for windows)
            #    shutil.copy(os.fspath(src),os.fspath(dst))
    else:
        # creates the symlinks of the video and puts it in the videos directory.
        print("Creating the symbolic link of the video")
        for src, dst in zip(videos, destinations):
            if dst.exists() and not DEBUG:
                raise FileExistsError('Video {} exists already!'.format(dst))
            try:
                src = str(src)
                dst = str(dst)
                os.symlink(src, dst)
            except OSError:
                import subprocess
                subprocess.check_call('mklink %s %s' % (dst, src), shell=True)
            print('Created the symlink of {} to {}'.format(src, dst))
            videos = destinations

    if copy_videos == True:
        videos = destinations  # in this case the *new* location should be added to the config file

    # adds the video list to the config.yaml file
    video_sets = {}
    for video in videos:
        print(video)
        try:
            rel_video_path = os.path.realpath(video)
        except:
            rel_video_path = os.readlink(str(video))

        vcap = cv2.VideoCapture(rel_video_path)
        if vcap.isOpened():
            width = int(vcap.get(cv2.CAP_PROP_FRAME_WIDTH))
            height = int(vcap.get(cv2.CAP_PROP_FRAME_HEIGHT))
            video_sets[rel_video_path] = {
                'crop': ', '.join(map(str, [0, width, 0, height]))
            }
        else:
            print("Cannot open the video file!")
            video_sets = None

    #        Set values to config file:
    cfg_file, ruamelFile = auxiliaryfunctions.create_config_template()
    cfg_file
    cfg_file['Task'] = project
    cfg_file['scorer'] = experimenter
    cfg_file['video_sets'] = video_sets
    cfg_file['project_path'] = str(project_path)
    cfg_file['date'] = d
    cfg_file['bodyparts'] = ['Hand', 'Finger1', 'Finger2', 'Joystick']
    cfg_file['cropping'] = False
    cfg_file['start'] = 0
    cfg_file['stop'] = 1
    cfg_file['numframes2pick'] = 20
    cfg_file['TrainingFraction'] = [0.95]
    cfg_file['iteration'] = 0
    cfg_file['resnet'] = 50
    cfg_file['snapshotindex'] = -1
    cfg_file['x1'] = 0
    cfg_file['x2'] = 640
    cfg_file['y1'] = 277
    cfg_file['y2'] = 624
    cfg_file[
        'batch_size'] = 4  #batch size during inference (video - analysis); see https://www.biorxiv.org/content/early/2018/10/30/457242
    cfg_file['corner2move2'] = (50, 50)
    cfg_file['move2corner'] = True
    cfg_file['pcutoff'] = 0.1
    cfg_file['dotsize'] = 12  #for plots size of dots
    cfg_file['alphavalue'] = 0.7  #for plots transparency of markers
    cfg_file['colormap'] = 'jet'  #for plots type of colormap

    projconfigfile = os.path.join(str(project_path), 'config.yaml')
    # Write dictionary to yaml  config file
    auxiliaryfunctions.write_config(projconfigfile, cfg_file)

    print('Generated "{}"'.format(project_path / 'config.yaml'))
    print(
        "\nA new project with name %s is created at %s and a configurable file (config.yaml) is stored there. Change the parameters in this file to adapt to your project's needs.\n Once you have changed the configuration file, use the function 'extract_frames' to select frames for labeling.\n. [OPTIONAL] Use the function 'add_new_videos' to add new videos to your project (at any stage)."
        % (project_name, str(wd)))
    return projconfigfile
Exemplo n.º 38
0
 def test_get(self):
     cache = ODBManager(self.dvc).local.hash_to_path_info(self.cache1_md5)
     self.assertEqual(os.fspath(cache), self.cache1)
Exemplo n.º 39
0
    def sign(self, command, build_dir, bcfg, formats):
        if not formats:
            return

        args = command.args
        b = pathlib.Path(build_dir)

        tool_path = self.find_imgtool(command, args)
        # The vector table offset is set in Kconfig:
        vtoff = self.get_cfg(command, bcfg, 'CONFIG_ROM_START_OFFSET')
        # Flash device write alignment and the partition's slot size
        # come from devicetree:
        flash = self.edt_flash_node(b, args.quiet)
        align, addr, size = self.edt_flash_params(flash)

        dot_config_file = b / 'zephyr' / '.config'
        if not dot_config_file.is_file():
            log.die(f"no .config found at {dot_config_file}")

        dot_config = load_dot_config(dot_config_file)

        if dot_config.get('CONFIG_BOOTLOADER_MCUBOOT', 'n') != 'y':
            log.wrn("CONFIG_BOOTLOADER_MCUBOOT is not set to y in "
                    f"{dot_config_file}; this probably won't work")

        kernel = dot_config.get('CONFIG_KERNEL_BIN_NAME', 'zephyr')

        if 'bin' in formats:
            in_bin = b / 'zephyr' / f'{kernel}.bin'
            if not in_bin.is_file():
                log.die(f"no unsigned .bin found at {in_bin}")
            in_bin = os.fspath(in_bin)
        else:
            in_bin = None
        if 'hex' in formats:
            in_hex = b / 'zephyr' / f'{kernel}.hex'
            if not in_hex.is_file():
                log.die(f"no unsigned .hex found at {in_hex}")
            in_hex = os.fspath(in_hex)
        else:
            in_hex = None

        if not args.quiet:
            log.banner('image configuration:')
            log.inf('partition offset: {0} (0x{0:x})'.format(addr))
            log.inf('partition size: {0} (0x{0:x})'.format(size))
            log.inf('rom start offset: {0} (0x{0:x})'.format(vtoff))

        # Base sign command.
        #
        # We provide a default --version in case the user is just
        # messing around and doesn't want to set one. It will be
        # overridden if there is a --version in args.tool_args.
        sign_base = [
            tool_path, 'sign', '--version', '0.0.0+0', '--align',
            str(align), '--header-size',
            str(vtoff), '--slot-size',
            str(size)
        ]
        sign_base.extend(args.tool_args)

        if not args.quiet:
            log.banner('signing binaries')
        if in_bin:
            out_bin = args.sbin or str(b / 'zephyr' / 'zephyr.signed.bin')
            sign_bin = sign_base + [in_bin, out_bin]
            if not args.quiet:
                log.inf(f'unsigned bin: {in_bin}')
                log.inf(f'signed bin:   {out_bin}')
                log.dbg(quote_sh_list(sign_bin))
            subprocess.check_call(sign_bin)
        if in_hex:
            out_hex = args.shex or str(b / 'zephyr' / 'zephyr.signed.hex')
            sign_hex = sign_base + [in_hex, out_hex]
            if not args.quiet:
                log.inf(f'unsigned hex: {in_hex}')
                log.inf(f'signed hex:   {out_hex}')
                log.dbg(quote_sh_list(sign_hex))
            subprocess.check_call(sign_hex)
Exemplo n.º 40
0
def compile_file(fullname,
                 ddir=None,
                 force=False,
                 rx=None,
                 quiet=0,
                 legacy=False,
                 optimize=-1):
    """Byte-compile one file.

    Arguments (only fullname is required):

    fullname:  the file to byte-compile
    ddir:      if given, the directory name compiled in to the
               byte-code file.
    force:     if True, force compilation, even if timestamps are up-to-date
    quiet:     full output with False or 0, errors only with 1,
               no output with 2
    legacy:    if True, produce legacy pyc paths instead of PEP 3147 paths
    optimize:  optimization level or -1 for level of the interpreter
    """
    success = True
    if quiet < 2 and isinstance(fullname, os.PathLike):
        fullname = os.fspath(fullname)
    name = os.path.basename(fullname)
    if ddir is not None:
        dfile = os.path.join(ddir, name)
    else:
        dfile = None
    if rx is not None:
        mo = rx.search(fullname)
        if mo:
            return success
    if os.path.isfile(fullname):
        if legacy:
            cfile = fullname + 'c'
        else:
            if optimize >= 0:
                opt = optimize if optimize >= 1 else ''
                cfile = importlib.util.cache_from_source(fullname,
                                                         optimization=opt)
            else:
                cfile = importlib.util.cache_from_source(fullname)
            cache_dir = os.path.dirname(cfile)
        head, tail = name[:-3], name[-3:]
        if tail == '.py':
            if not force:
                try:
                    mtime = int(os.stat(fullname).st_mtime)
                    expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER,
                                         mtime)
                    with open(cfile, 'rb') as chandle:
                        actual = chandle.read(8)
                    if expect == actual:
                        return success
                except OSError:
                    pass
            if not quiet:
                print('Compiling {!r}...'.format(fullname))
            try:
                ok = py_compile.compile(fullname,
                                        cfile,
                                        dfile,
                                        True,
                                        optimize=optimize)
            except py_compile.PyCompileError as err:
                success = False
                if quiet >= 2:
                    return success
                elif quiet:
                    print('*** Error compiling {!r}...'.format(fullname))
                else:
                    print('*** ', end='')
                # escape non-printable characters in msg
                msg = err.msg.encode(sys.stdout.encoding,
                                     errors='backslashreplace')
                msg = msg.decode(sys.stdout.encoding)
                print(msg)
            except (SyntaxError, UnicodeError, OSError) as e:
                success = False
                if quiet >= 2:
                    return success
                elif quiet:
                    print('*** Error compiling {!r}...'.format(fullname))
                else:
                    print('*** ', end='')
                print(e.__class__.__name__ + ':', e)
            else:
                if ok == 0:
                    success = False
    return success
Exemplo n.º 41
0
 def fromfile(cls, filepath):
     filepath = os.fspath(filepath)
     c = cls(**yaml.safe_load(open(filepath)))
     c.origin = filepath
     return c
Exemplo n.º 42
0
    def guess_type(self, url, strict=True):
        """Guess the type of a file which is either a URL or a path-like object.

        Return value is a tuple (type, encoding) where type is None if
        the type can't be guessed (no or unknown suffix) or a string
        of the form type/subtype, usable for a MIME Content-type
        header; and encoding is None for no encoding or the name of
        the program used to encode (e.g. compress or gzip).  The
        mappings are table driven.  Encoding suffixes are case
        sensitive; type suffixes are first tried case sensitive, then
        case insensitive.

        The suffixes .tgz, .taz and .tz (case sensitive!) are all
        mapped to '.tar.gz'.  (This is table-driven too, using the
        dictionary suffix_map.)

        Optional `strict' argument when False adds a bunch of commonly found,
        but non-standard types.
        """
        url = os.fspath(url)
        scheme, url = urllib.parse._splittype(url)
        if scheme == 'data':
            # syntax of data URLs:
            # dataurl   := "data:" [ mediatype ] [ ";base64" ] "," data
            # mediatype := [ type "/" subtype ] *( ";" parameter )
            # data      := *urlchar
            # parameter := attribute "=" value
            # type/subtype defaults to "text/plain"
            comma = url.find(',')
            if comma < 0:
                # bad data URL
                return None, None
            semi = url.find(';', 0, comma)
            if semi >= 0:
                type = url[:semi]
            else:
                type = url[:comma]
            if '=' in type or '/' not in type:
                type = 'text/plain'
            return type, None           # never compressed, so encoding is None
        base, ext = posixpath.splitext(url)
        while ext in self.suffix_map:
            base, ext = posixpath.splitext(base + self.suffix_map[ext])
        if ext in self.encodings_map:
            encoding = self.encodings_map[ext]
            base, ext = posixpath.splitext(base)
        else:
            encoding = None
        types_map = self.types_map[True]
        if ext in types_map:
            return types_map[ext], encoding
        elif ext.lower() in types_map:
            return types_map[ext.lower()], encoding
        elif strict:
            return None, encoding
        types_map = self.types_map[False]
        if ext in types_map:
            return types_map[ext], encoding
        elif ext.lower() in types_map:
            return types_map[ext.lower()], encoding
        else:
            return None, encoding
Exemplo n.º 43
0
def expandvars(path):
    """Expand shell variables of the forms $var, ${var} and %var%.

    Unknown variables are left unchanged."""
    path = os.fspath(path)
    if isinstance(path, bytes):
        if b'$' not in path and b'%' not in path:
            return path
        import string
        varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii')
        quote = b'\''
        percent = b'%'
        brace = b'{'
        rbrace = b'}'
        dollar = b'$'
        environ = getattr(os, 'environb', None)
    else:
        if '$' not in path and '%' not in path:
            return path
        import string
        varchars = string.ascii_letters + string.digits + '_-'
        quote = '\''
        percent = '%'
        brace = '{'
        rbrace = '}'
        dollar = '$'
        environ = os.environ
    res = path[:0]
    index = 0
    pathlen = len(path)
    while index < pathlen:
        c = path[index:index + 1]
        if c == quote:  # no expansion within single quotes
            path = path[index + 1:]
            pathlen = len(path)
            try:
                index = path.index(c)
                res += c + path[:index + 1]
            except ValueError:
                res += c + path
                index = pathlen - 1
        elif c == percent:  # variable or '%'
            if path[index + 1:index + 2] == percent:
                res += c
                index += 1
            else:
                path = path[index + 1:]
                pathlen = len(path)
                try:
                    index = path.index(percent)
                except ValueError:
                    res += percent + path
                    index = pathlen - 1
                else:
                    var = path[:index]
                    try:
                        if environ is None:
                            value = os.fsencode(os.environ[os.fsdecode(var)])
                        else:
                            value = environ[var]
                    except KeyError:
                        value = percent + var + percent
                    res += value
        elif c == dollar:  # variable or '$$'
            if path[index + 1:index + 2] == dollar:
                res += c
                index += 1
            elif path[index + 1:index + 2] == brace:
                path = path[index + 2:]
                pathlen = len(path)
                try:
                    index = path.index(rbrace)
                except ValueError:
                    res += dollar + brace + path
                    index = pathlen - 1
                else:
                    var = path[:index]
                    try:
                        if environ is None:
                            value = os.fsencode(os.environ[os.fsdecode(var)])
                        else:
                            value = environ[var]
                    except KeyError:
                        value = dollar + brace + var + rbrace
                    res += value
            else:
                var = path[:0]
                index += 1
                c = path[index:index + 1]
                while c and c in varchars:
                    var += c
                    index += 1
                    c = path[index:index + 1]
                try:
                    if environ is None:
                        value = os.fsencode(os.environ[os.fsdecode(var)])
                    else:
                        value = environ[var]
                except KeyError:
                    value = dollar + var
                res += value
                if c:
                    index -= 1
        else:
            res += c
        index += 1
    return res
Exemplo n.º 44
0
def Calc(calc: MaybeSequence[str],
         outfile: Optional[PathLikeOrStr] = None,
         NoDataValue: Optional[Number] = None,
         type: Optional[Union[GDALDataType, str]] = None,
         format: Optional[str] = None,
         creation_options: Optional[Sequence[str]] = None,
         allBands: str = '',
         overwrite: bool = False,
         hideNoData: bool = False,
         projectionCheck: bool = False,
         color_table: Optional[ColorTableLike] = None,
         extent: Optional[Extent] = None,
         projwin: Optional[Union[Tuple, GeoRectangle]] = None,
         user_namespace: Optional[Dict] = None,
         debug: bool = False,
         quiet: bool = False,
         **input_files):

    if debug:
        print(f"gdal_calc.py starting calculation {calc}")

    # Single calc value compatibility
    if isinstance(calc, (list, tuple)):
        calc = calc
    else:
        calc = [calc]
    calc = [c.strip('"') for c in calc]

    creation_options = creation_options or []

    # set up global namespace for eval with all functions of gdal_array, numpy
    global_namespace = {
        key: getattr(module, key)
        for module in [gdal_array, numpy] for key in dir(module)
        if not key.startswith('__')
    }

    if user_namespace:
        global_namespace.update(user_namespace)

    if not calc:
        raise Exception("No calculation provided.")
    elif not outfile and format.upper() != 'MEM':
        raise Exception("No output file provided.")

    if format is None:
        format = GetOutputDriverFor(outfile)

    if isinstance(extent, GeoRectangle):
        pass
    elif projwin:
        if isinstance(projwin, GeoRectangle):
            extent = projwin
        else:
            extent = GeoRectangle.from_lurd(*projwin)
    elif not extent:
        extent = Extent.IGNORE
    else:
        extent = extent_util.parse_extent(extent)

    compatible_gt_eps = 0.000001
    gt_diff_support = {
        GT.INCOMPATIBLE_OFFSET: extent != Extent.FAIL,
        GT.INCOMPATIBLE_PIXEL_SIZE: False,
        GT.INCOMPATIBLE_ROTATION: False,
        GT.NON_ZERO_ROTATION: False,
    }
    gt_diff_error = {
        GT.INCOMPATIBLE_OFFSET: 'different offset',
        GT.INCOMPATIBLE_PIXEL_SIZE: 'different pixel size',
        GT.INCOMPATIBLE_ROTATION: 'different rotation',
        GT.NON_ZERO_ROTATION: 'non zero rotation',
    }

    ################################################################
    # fetch details of input layers
    ################################################################

    # set up some lists to store data for each band
    myFileNames = []  # input filenames
    myFiles = []  # input DataSets
    myBands = []  # input bands
    myAlphaList = []  # input alpha letter that represents each input file
    myDataType = []  # string representation of the datatype of each input file
    myDataTypeNum = []  # datatype of each input file
    myNDV = []  # nodatavalue for each input file
    DimensionsCheck = None  # dimensions of the output
    Dimensions = []  # Dimensions of input files
    ProjectionCheck = None  # projection of the output
    GeoTransformCheck = None  # GeoTransform of the output
    GeoTransforms = []  # GeoTransform of each input file
    GeoTransformDiffer = False  # True if we have inputs with different GeoTransforms
    myTempFileNames = []  # vrt filename from each input file
    myAlphaFileLists = []  # list of the Alphas which holds a list of inputs

    # loop through input files - checking dimensions
    for alphas, filenames in input_files.items():
        if isinstance(filenames, (list, tuple)):
            # alpha is a list of files
            myAlphaFileLists.append(alphas)
        elif is_path_like(filenames) or isinstance(filenames, gdal.Dataset):
            # alpha is a single filename or a Dataset
            filenames = [filenames]
            alphas = [alphas]
        else:
            # I guess this alphas should be in the global_namespace,
            # It would have been better to pass it as user_namepsace, but I'll accept it anyway
            global_namespace[alphas] = filenames
            continue
        for alpha, filename in zip(alphas * len(filenames), filenames):
            if not alpha.endswith("_band"):
                # check if we have asked for a specific band...
                alpha_band = f"{alpha}_band"
                if alpha_band in input_files:
                    myBand = input_files[alpha_band]
                else:
                    myBand = 1

                myF_is_ds = not is_path_like(filename)
                if myF_is_ds:
                    myFile = filename
                    filename = None
                else:
                    myFile = open_ds(filename, gdal.GA_ReadOnly)
                if not myFile:
                    raise IOError(f"No such file or directory: '{filename}'")

                myFileNames.append(filename)
                myFiles.append(myFile)
                myBands.append(myBand)
                myAlphaList.append(alpha)
                dt = myFile.GetRasterBand(myBand).DataType
                myDataType.append(gdal.GetDataTypeName(dt))
                myDataTypeNum.append(dt)
                myNDV.append(None if hideNoData else myFile.
                             GetRasterBand(myBand).GetNoDataValue())

                # check that the dimensions of each layer are the same
                myFileDimensions = [myFile.RasterXSize, myFile.RasterYSize]
                if DimensionsCheck:
                    if DimensionsCheck != myFileDimensions:
                        GeoTransformDiffer = True
                        if extent in [Extent.IGNORE, Extent.FAIL]:
                            raise Exception(
                                f"Error! Dimensions of file {filename} ({myFileDimensions[0]:d}, "
                                f"{myFileDimensions[1]:d}) are different from other files "
                                f"({DimensionsCheck[0]:d}, {DimensionsCheck[1]:d}).  Cannot proceed"
                            )
                else:
                    DimensionsCheck = myFileDimensions

                # check that the Projection of each layer are the same
                myProjection = myFile.GetProjection()
                if ProjectionCheck:
                    if projectionCheck and ProjectionCheck != myProjection:
                        raise Exception(
                            f"Error! Projection of file {filename} {myProjection} "
                            f"are different from other files {ProjectionCheck}.  Cannot proceed"
                        )
                else:
                    ProjectionCheck = myProjection

                # check that the GeoTransforms of each layer are the same
                myFileGeoTransform = myFile.GetGeoTransform(
                    can_return_null=True)
                if extent == Extent.IGNORE:
                    GeoTransformCheck = myFileGeoTransform
                else:
                    Dimensions.append(myFileDimensions)
                    GeoTransforms.append(myFileGeoTransform)
                    if not GeoTransformCheck:
                        GeoTransformCheck = myFileGeoTransform
                    else:
                        my_gt_diff = extent_util.gt_diff(
                            GeoTransformCheck,
                            myFileGeoTransform,
                            eps=compatible_gt_eps,
                            diff_support=gt_diff_support)
                        if my_gt_diff not in [GT.SAME, GT.ALMOST_SAME]:
                            GeoTransformDiffer = True
                            if my_gt_diff != GT.COMPATIBLE_DIFF:
                                raise Exception(
                                    f"Error! GeoTransform of file {filename} {myFileGeoTransform} is incompatible "
                                    f"({gt_diff_error[my_gt_diff]}), first file GeoTransform is {GeoTransformCheck}. "
                                    f"Cannot proceed")
                if debug:
                    print(
                        f"file {alpha}: {filename}, dimensions: "
                        f"{DimensionsCheck[0]}, {DimensionsCheck[1]}, type: {myDataType[-1]}"
                    )

    # process allBands option
    allBandsIndex = None
    allBandsCount = 1
    if allBands:
        if len(calc) > 1:
            raise Exception("Error! --allBands implies a single --calc")
        try:
            allBandsIndex = myAlphaList.index(allBands)
        except ValueError:
            raise Exception(
                f"Error! allBands option was given but Band {allBands} not found.  Cannot proceed"
            )
        allBandsCount = myFiles[allBandsIndex].RasterCount
        if allBandsCount <= 1:
            allBandsIndex = None
    else:
        allBandsCount = len(calc)

    if extent not in [
            Extent.IGNORE, Extent.FAIL
    ] and (GeoTransformDiffer or isinstance(extent, GeoRectangle)):
        # mixing different GeoTransforms/Extents
        GeoTransformCheck, DimensionsCheck, ExtentCheck = extent_util.calc_geotransform_and_dimensions(
            GeoTransforms, Dimensions, extent)
        if GeoTransformCheck is None:
            raise Exception(
                "Error! The requested extent is empty. Cannot proceed")
        for i in range(len(myFileNames)):
            temp_vrt_filename, temp_vrt_ds = extent_util.make_temp_vrt(
                myFiles[i], ExtentCheck)
            myTempFileNames.append(temp_vrt_filename)
            myFiles[i] = None  # close original ds
            myFiles[i] = temp_vrt_ds  # replace original ds with vrt_ds

            # update the new precise dimensions and gt from the new ds
            GeoTransformCheck = temp_vrt_ds.GetGeoTransform()
            DimensionsCheck = [
                temp_vrt_ds.RasterXSize, temp_vrt_ds.RasterYSize
            ]
        temp_vrt_ds = None

    ################################################################
    # set up output file
    ################################################################

    # open output file exists
    if outfile and os.path.isfile(outfile) and not overwrite:
        if allBandsIndex is not None:
            raise Exception(
                "Error! allBands option was given but Output file exists, must use --overwrite option!"
            )
        if len(calc) > 1:
            raise Exception(
                "Error! multiple calc options were given but Output file exists, must use --overwrite option!"
            )
        if debug:
            print(
                f"Output file {outfile} exists - filling in results into file")

        myOut = open_ds(outfile, gdal.GA_Update)
        if myOut is None:
            error = 'but cannot be opened for update'
        elif [myOut.RasterXSize, myOut.RasterYSize] != DimensionsCheck:
            error = 'but is the wrong size'
        elif ProjectionCheck and ProjectionCheck != myOut.GetProjection():
            error = 'but is the wrong projection'
        elif GeoTransformCheck and GeoTransformCheck != myOut.GetGeoTransform(
                can_return_null=True):
            error = 'but is the wrong geotransform'
        else:
            error = None
        if error:
            raise Exception(
                f"Error! Output exists, {error}.  Use the --overwrite option "
                f"to automatically overwrite the existing file")

        myOutB = myOut.GetRasterBand(1)
        myOutNDV = myOutB.GetNoDataValue()
        myOutType = myOutB.DataType

    else:
        if outfile:
            # remove existing file and regenerate
            if os.path.isfile(outfile):
                os.remove(outfile)
            # create a new file
            if debug:
                print(f"Generating output file {outfile}")
        else:
            outfile = ''

        # find data type to use
        if not type:
            # use the largest type of the input files
            myOutType = max(myDataTypeNum)
        else:
            myOutType = type
            if isinstance(myOutType, str):
                myOutType = gdal.GetDataTypeByName(myOutType)

        # create file
        myOutDrv = gdal.GetDriverByName(format)
        myOut = myOutDrv.Create(os.fspath(outfile), DimensionsCheck[0],
                                DimensionsCheck[1], allBandsCount, myOutType,
                                creation_options)

        # set output geo info based on first input layer
        if not GeoTransformCheck:
            GeoTransformCheck = myFiles[0].GetGeoTransform(
                can_return_null=True)
        if GeoTransformCheck:
            myOut.SetGeoTransform(GeoTransformCheck)

        if not ProjectionCheck:
            ProjectionCheck = myFiles[0].GetProjection()
        if ProjectionCheck:
            myOut.SetProjection(ProjectionCheck)

        if NoDataValue is None:
            myOutNDV = DefaultNDVLookup[
                myOutType]  # use the default noDataValue for this datatype
        elif isinstance(NoDataValue, str) and NoDataValue.lower() == 'none':
            myOutNDV = None  # not to set any noDataValue
        else:
            myOutNDV = NoDataValue  # use the given noDataValue

        for i in range(1, allBandsCount + 1):
            myOutB = myOut.GetRasterBand(i)
            if myOutNDV is not None:
                myOutB.SetNoDataValue(myOutNDV)
            if color_table:
                # set color table and color interpretation
                if is_path_like(color_table):
                    color_table = get_color_table(color_table)
                myOutB.SetRasterColorTable(color_table)
                myOutB.SetRasterColorInterpretation(gdal.GCI_PaletteIndex)

            myOutB = None  # write to band

        if hideNoData:
            myOutNDV = None

    myOutTypeName = gdal.GetDataTypeName(myOutType)
    if debug:
        print(
            f"output file: {outfile}, dimensions: {myOut.RasterXSize}, {myOut.RasterYSize}, type: {myOutTypeName}"
        )

    ################################################################
    # find block size to chop grids into bite-sized chunks
    ################################################################

    # use the block size of the first layer to read efficiently
    myBlockSize = myFiles[0].GetRasterBand(myBands[0]).GetBlockSize()
    # find total x and y blocks to be read
    nXBlocks = (int)(
        (DimensionsCheck[0] + myBlockSize[0] - 1) / myBlockSize[0])
    nYBlocks = (int)(
        (DimensionsCheck[1] + myBlockSize[1] - 1) / myBlockSize[1])
    myBufSize = myBlockSize[0] * myBlockSize[1]

    if debug:
        print(f"using blocksize {myBlockSize[0]} x {myBlockSize[1]}")

    # variables for displaying progress
    ProgressCt = -1
    ProgressMk = -1
    ProgressEnd = nXBlocks * nYBlocks * allBandsCount

    ################################################################
    # start looping through each band in allBandsCount
    ################################################################

    for bandNo in range(1, allBandsCount + 1):

        ################################################################
        # start looping through blocks of data
        ################################################################

        # store these numbers in variables that may change later
        nXValid = myBlockSize[0]
        nYValid = myBlockSize[1]

        # loop through X-lines
        for X in range(0, nXBlocks):

            # in case the blocks don't fit perfectly
            # change the block size of the final piece
            if X == nXBlocks - 1:
                nXValid = DimensionsCheck[0] - X * myBlockSize[0]

            # find X offset
            myX = X * myBlockSize[0]

            # reset buffer size for start of Y loop
            nYValid = myBlockSize[1]
            myBufSize = nXValid * nYValid

            # loop through Y lines
            for Y in range(0, nYBlocks):
                ProgressCt += 1
                if 10 * ProgressCt / ProgressEnd % 10 != ProgressMk and not quiet:
                    ProgressMk = 10 * ProgressCt / ProgressEnd % 10
                    from sys import version_info
                    if version_info >= (3, 0, 0):
                        exec('print("%d.." % (10*ProgressMk), end=" ")')
                    else:
                        exec('print 10*ProgressMk, "..",')

                # change the block size of the final piece
                if Y == nYBlocks - 1:
                    nYValid = DimensionsCheck[1] - Y * myBlockSize[1]
                    myBufSize = nXValid * nYValid

                # find Y offset
                myY = Y * myBlockSize[1]

                # create empty buffer to mark where nodata occurs
                myNDVs = None

                # make local namespace for calculation
                local_namespace = {}

                val_lists = defaultdict(list)

                # fetch data for each input layer
                for i, Alpha in enumerate(myAlphaList):

                    # populate lettered arrays with values
                    if allBandsIndex is not None and allBandsIndex == i:
                        myBandNo = bandNo
                    else:
                        myBandNo = myBands[i]
                    myval = gdal_array.BandReadAsArray(
                        myFiles[i].GetRasterBand(myBandNo),
                        xoff=myX,
                        yoff=myY,
                        win_xsize=nXValid,
                        win_ysize=nYValid)
                    if myval is None:
                        raise Exception(
                            f'Input block reading failed from filename {filename[i]}'
                        )

                    # fill in nodata values
                    if myNDV[i] is not None:
                        # myNDVs is a boolean buffer.
                        # a cell equals to 1 if there is NDV in any of the corresponding cells in input raster bands.
                        if myNDVs is None:
                            # this is the first band that has NDV set. we initializes myNDVs to a zero buffer
                            # as we didn't see any NDV value yet.
                            myNDVs = numpy.zeros(myBufSize)
                            myNDVs.shape = (nYValid, nXValid)
                        myNDVs = 1 * numpy.logical_or(myNDVs == 1, myval
                                                      == myNDV[i])

                    # add an array of values for this block to the eval namespace
                    if Alpha in myAlphaFileLists:
                        val_lists[Alpha].append(myval)
                    else:
                        local_namespace[Alpha] = myval
                    myval = None

                for lst in myAlphaFileLists:
                    local_namespace[lst] = val_lists[lst]

                # try the calculation on the array blocks
                this_calc = calc[bandNo - 1 if len(calc) > 1 else 0]
                try:
                    myResult = eval(this_calc, global_namespace,
                                    local_namespace)
                except:
                    print(f"evaluation of calculation {this_calc} failed")
                    raise

                # Propagate nodata values (set nodata cells to zero
                # then add nodata value to these cells).
                if myNDVs is not None and myOutNDV is not None:
                    myResult = (
                        (1 * (myNDVs == 0)) * myResult) + (myOutNDV * myNDVs)
                elif not isinstance(myResult, numpy.ndarray):
                    myResult = numpy.ones((nYValid, nXValid)) * myResult

                # write data block to the output file
                myOutB = myOut.GetRasterBand(bandNo)
                if gdal_array.BandWriteArray(
                        myOutB, myResult, xoff=myX, yoff=myY) != 0:
                    raise Exception('Block writing failed')
                myOutB = None  # write to band

    # remove temp files
    for idx, tempFile in enumerate(myTempFileNames):
        myFiles[idx] = None
        os.remove(tempFile)

    gdal.ErrorReset()
    myOut.FlushCache()
    if gdal.GetLastErrorMsg() != '':
        raise Exception('Dataset writing failed')

    if not quiet:
        print("100 - Done")

    return myOut
Exemplo n.º 45
0
 def static_folder(self, value):
     if value is not None:
         value = os.fspath(value).rstrip(r"\/")
     self._static_folder = value
Exemplo n.º 46
0
async def test_open_file_can_open_path(path):
    async with await trio.open_file(path, "w") as f:
        assert f.name == os.fspath(path)
Exemplo n.º 47
0
 def collect_stages():
     return {stage.relpath for stage in Repo(os.fspath(tmp_dir)).stages}
Exemplo n.º 48
0
 def serialize(self):
     return repr(os.fspath(self.value)), {}
Exemplo n.º 49
0
import os, shutil
import time
import subprocess
import json

from pathlib import Path
from pylib import mongo
import base64
from pylib import textual
import datetime

db = mongo.get_makalu()

parent_dir = "/home/report"
directory = "generated_reports_coll/"
Original_report_location = os.fspath(
    "/opt/makalu/app_store/reports/generatedreports/")
new_path = os.path.join(parent_dir, directory)
mode = 0o766
error_file = os.path.join(parent_dir, "error.txt")
counter_file = os.path.join(parent_dir, "counter.txt")

if not os.path.exists(new_path):
    os.mkdir(new_path, mode)


def write_error(location, error):
    with open(error_file, "a") as record:
        record.write(f"{datetime.datetime.now()}: {location} {error}\n")


def lookup_mongo(report_name):
# Static files (CSS, JavaScript, Images)
# https://docs.djangoproject.com/en/3.1/howto/static-files/

STATIC_URL = "/static/"

# Keep this as the last section of this file!
try:
    from .local_settings import *
except ImportError:
    pass
# Intended to be a temporary hack for overriding settings in production.
# TODO: figure out a better way to do this, probably with a different path
override_settings_file = Path("/opt/secret/override_settings.py")
if override_settings_file.is_file():
    print("Reading production override settings from", override_settings_file)
    sys.path.append(fspath(override_settings_file.parent))
    try:
        from override_settings import *
    except ImportError as e:
        print("Couldn't read override settings found at",
              override_settings_file)
        raise
# Sometimes we do need to define settings in terms of other settings, so
# this is a good place to do so, after override settings are loaded.
# Shouldn't define any constants at this point though

# !!! overrides that depend on other (including local) settings

MONGO_HOST_AND_PORT = f"mongodb://{MONGO_USERNAME}:{MONGO_PASSWORD}@{MONGO_HOSTNAME}:{MONGO_PORT}/"

# /!!! overrides that depend on other (including local) settings
Exemplo n.º 51
0
    def __init__(
        self,
        root: Union[str, Path],
        url: str = URL,
        folder_in_archive: str = FOLDER_IN_ARCHIVE,
        download: bool = False,
        subset: Optional[str] = None,
    ) -> None:

        assert subset is None or subset in [
            "training", "validation", "testing"
        ], ("When `subset` not None, it must take a value from " +
            "{'training', 'validation', 'testing'}.")

        if url in [
                "speech_commands_v0.01",
                "speech_commands_v0.02",
        ]:
            base_url = "https://storage.googleapis.com/download.tensorflow.org/data/"
            ext_archive = ".tar.gz"

            url = os.path.join(base_url, url + ext_archive)

        # Get string representation of 'root' in case Path object is passed
        root = os.fspath(root)

        basename = os.path.basename(url)
        archive = os.path.join(root, basename)

        basename = basename.rsplit(".", 2)[0]
        folder_in_archive = os.path.join(folder_in_archive, basename)

        self._path = os.path.join(root, folder_in_archive)

        if download:
            if not os.path.isdir(self._path):
                if not os.path.isfile(archive):
                    checksum = _CHECKSUMS.get(url, None)
                    download_url(url,
                                 root,
                                 hash_value=checksum,
                                 hash_type="md5")
                extract_archive(archive, self._path)

        if subset == "validation":
            self._walker = _load_list(self._path, "validation_list.txt")
        elif subset == "testing":
            self._walker = _load_list(self._path, "testing_list.txt")
        elif subset == "training":
            excludes = set(
                _load_list(self._path, "validation_list.txt",
                           "testing_list.txt"))
            walker = sorted(str(p) for p in Path(self._path).glob('*/*.wav'))
            self._walker = [
                w for w in walker
                if HASH_DIVIDER in w and EXCEPT_FOLDER not in w
                and os.path.normpath(w) not in excludes
            ]
        else:
            walker = sorted(str(p) for p in Path(self._path).glob('*/*.wav'))
            self._walker = [
                w for w in walker
                if HASH_DIVIDER in w and EXCEPT_FOLDER not in w
            ]
Exemplo n.º 52
0
 def _get_local_path(self, path: str, **kwargs: Any) -> str:
     self._check_kwargs(kwargs)
     return os.fspath(path)
Exemplo n.º 53
0
def basename(p):
    """Returns the final component of a pathname"""
    p = os.fspath(p)
    sep = _get_sep(p)
    i = p.rfind(sep) + 1
    return p[i:]
Exemplo n.º 54
0
 def __init__(self, *args, directory=None, **kwargs):
     if directory is None:
         directory = os.getcwd()
     self.directory = os.fspath(directory)
     super().__init__(*args, **kwargs)
Exemplo n.º 55
0
 def set_file(self, file):
     """
     Set the filename of the fontfile to use.  In this case, all
     other properties will be ignored.
     """
     self._file = os.fspath(file) if file is not None else None
Exemplo n.º 56
0
def splitext(p):
    p = os.fspath(p)
    if isinstance(p, bytes):
        return genericpath._splitext(p, b'\\', b'/', b'.')
    else:
        return genericpath._splitext(p, '\\', '/', '.')
Exemplo n.º 57
0
def run_PETSurfaceCrossSectional(
    input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path
) -> None:
    import shutil

    import nibabel as nib
    import numpy as np

    from clinica.pipelines.pet_surface.pet_surface_pipeline import PetSurface

    shutil.copytree(input_dir / "caps", output_dir / "caps", copy_function=shutil.copy)

    parameters = {
        "acq_label": "FDG",
        "suvr_reference_region": "pons",
        "pvc_psf_tsv": fspath(input_dir / "subjects.tsv"),
        "longitudinal": False,
        "skip_question": False,
    }
    pipeline = PetSurface(
        bids_directory=fspath(input_dir / "bids"),
        caps_directory=fspath(output_dir / "caps"),
        tsv_file=fspath(input_dir / "subjects.tsv"),
        base_dir=fspath(working_dir),
        parameters=parameters,
    )
    pipeline.build()
    pipeline.run(bypass_check=True)

    # Check files
    out_files = [
        (
            output_dir
            / "caps/subjects/sub-ADNI011S4105/ses-M00/pet/surface"
            / (
                "sub-ADNI011S4105_ses-M00_task-rest_acq-fdg_pet_space-fsaverage_suvr-pons_pvc-iy_hemi-"
                + h
                + "_fwhm-"
                + str(f)
                + "_projection.mgh"
            )
        )
        for h in ["lh", "rh"]
        for f in [0, 5, 10, 15, 20, 25]
    ]
    ref_files = [
        (
            ref_dir
            / (
                "sub-ADNI011S4105_ses-M00_task-rest_acq-fdg_pet_space-fsaverage_suvr-pons_pvc-iy_hemi-"
                + h
                + "_fwhm-"
                + str(f)
                + "_projection.mgh"
            )
        )
        for h in ["lh", "rh"]
        for f in [0, 5, 10, 15, 20, 25]
    ]

    for i in range(len(out_files)):
        assert np.allclose(
            np.squeeze(nib.load(fspath(out_files[i])).get_fdata(dtype="float32")),
            np.squeeze(nib.load(fspath(ref_files[i])).get_fdata(dtype="float32")),
            rtol=3e-2,
            equal_nan=True,
        )
Exemplo n.º 58
0
 def fspath(self):
     return os.fspath(self)
Exemplo n.º 59
0
 def __init__(self, symlink_file_to_convert: os.PathLike, **kwargs) -> None:
     super().__init__(**kwargs)
     self.symlink_file_to_convert = os.fspath(symlink_file_to_convert)
Exemplo n.º 60
0
def run_PETVolume(
    input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path
) -> None:
    import shutil

    from clinica.pipelines.pet_volume.pet_volume_pipeline import PETVolume

    # Arrange
    shutil.copytree(input_dir / "caps", output_dir / "caps", copy_function=shutil.copy)

    parameters = {
        "group_label": "UnitTest",
        "acq_label": "fdg",
        "suvr_reference_region": "pons",
        "skip_question": False,
    }
    pipeline = PETVolume(
        bids_directory=fspath(input_dir / "bids"),
        caps_directory=fspath(output_dir / "caps"),
        tsv_file=fspath(input_dir / "subjects.tsv"),
        base_dir=fspath(working_dir),
        parameters=parameters,
    )
    # Acte
    pipeline.build()
    pipeline.run(plugin="MultiProc", plugin_args={"n_procs": 4}, bypass_check=True)

    # Assert
    subjects = [
        "sub-ADNI011S4105",
        "sub-ADNI023S4020",
        "sub-ADNI035S4082",
        "sub-ADNI128S4832",
    ]
    out_files = [
        fspath(
            output_dir
            / "caps"
            / "subjects"
            / sub
            / "ses-M00/pet/preprocessing/group-UnitTest"
            / (
                sub
                + "_ses-M00_task-rest_acq-fdg_pet_space-Ixi549Space_suvr-pons_mask-brain_fwhm-8mm_pet.nii.gz"
            )
        )
        for sub in subjects
    ]
    ref_files = [
        fspath(
            ref_dir
            / (
                sub
                + "_ses-M00_task-rest_acq-fdg_pet_space-Ixi549Space_suvr-pons_mask-brain_fwhm-8mm_pet.nii.gz"
            )
        )
        for sub in subjects
    ]

    for i in range(len(out_files)):
        assert likeliness_measure(
            out_files[i], ref_files[i], (1e-2, 0.25), (1e-1, 0.001)
        )