def main(infile, outfile, level, jobs=1): from tempfile import TemporaryDirectory from shutil import copy class OptimizeOptions: """Emulate ocrmypdf's options""" def __init__(self, jobs, optimize, jpeg_quality, png_quality, jb2lossy): self.jobs = jobs self.optimize = optimize self.jpeg_quality = jpeg_quality self.png_quality = png_quality self.jbig2_page_group_size = 0 self.jbig2_lossy = jb2lossy logging.basicConfig(level=logging.DEBUG) log = logging.getLogger() ctx = JobContext() options = OptimizeOptions( jobs=jobs, optimize=int(level), jpeg_quality=0, # Use default png_quality=0, jb2lossy=False, ) ctx.set_options(options) with TemporaryDirectory() as td: tmpout = Path(td) / 'out.pdf' optimize(infile, tmpout, log, ctx) copy(fspath(tmpout), fspath(outfile))
def relpath(path, start=None): """Return a relative version of a path""" if not path: raise ValueError("no path specified") path = os.fspath(path) if isinstance(path, bytes): curdir = b'.' sep = b'/' pardir = b'..' else: curdir = '.' sep = '/' pardir = '..' if start is None: start = curdir else: start = os.fspath(start) try: start_list = [x for x in abspath(start).split(sep) if x] path_list = [x for x in abspath(path).split(sep) if x] # Work out how much of the filepath is shared by start and path. i = len(commonprefix([start_list, path_list])) rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir return join(*rel_list) except (TypeError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise
def test_leptonica_compile(tmpdir): from ocrmypdf.lib.compile_leptonica import ffibuilder # Compile the library but build it somewhere that won't interfere with # existing compiled library. Also compile in API mode so that we test # the interfaces, even though we use it ABI mode. ffibuilder.compile(tmpdir=fspath(tmpdir), target=fspath(tmpdir / 'lepttest.*'))
def test_no_unpaper(resources, no_outpdf): input_ = fspath(resources / "c02-22.pdf") output = fspath(no_outpdf) options = main.parser.parse_args(args=["--clean", input_, output]) with patch("ocrmypdf.exec.unpaper.version") as mock_unpaper_version: mock_unpaper_version.side_effect = FileNotFoundError("unpaper") with pytest.raises(SystemExit): main.check_options(options, log=logging.getLogger())
def test_srgb_in_unicode_path(tmpdir): """Test that we can produce pdfmark when install path is not ASCII""" dstdir = Path(fspath(tmpdir)) / b'\xe4\x80\x80'.decode('utf-8') dstdir.mkdir() dst = dstdir / 'sRGB.icc' copyfile(SRGB_ICC_PROFILE, fspath(dst)) with patch('ocrmypdf.pdfa.SRGB_ICC_PROFILE', new=str(dst)): generate_pdfa_ps(dstdir / 'out.ps')
def _walk_dir(dir, ddir=None, maxlevels=10, quiet=0): if quiet < 2 and isinstance(dir, os.PathLike): dir = os.fspath(dir) if not quiet: print('Listing {!r}...'.format(dir)) try: names = os.listdir(dir) except OSError: if quiet < 2: print("Can't list {!r}".format(dir)) names = [] names.sort() for name in names: if name == '__pycache__': continue fullname = os.path.join(dir, name) if ddir is not None: dfile = os.path.join(ddir, name) else: dfile = None if not os.path.isdir(fullname): yield fullname elif (maxlevels > 0 and name != os.curdir and name != os.pardir and os.path.isdir(fullname) and not os.path.islink(fullname)): yield from _walk_dir(fullname, ddir=dfile, maxlevels=maxlevels - 1, quiet=quiet)
def _getpath(p): if (3, 6) <= sys.version_info: import os return os.fspath(p) if _detect_pathlib_path(p): return str(p) return p
def main(*, args): params = docopt.docopt(__doc__, argv=args, help=True, version=True, options_first=False) command = params.pop("<command>") assert not params, params print(command, end="\n\n") command_path = shutil.which(command) command_path_last = None while command_path_last != command_path: print(command_path) command_path_last = command_path parts = pathlib.Path(command_path).parts assert parts[0] == os.path.sep, parts[0] first = list(parts[:1]) second = list(parts[1:]) while second: first.append(second.pop(0)) p = pathlib.Path(*first) if not p.is_symlink(): continue t = os.readlink(p) print("\t", p, " -> ", t, sep="", end="\n\n") first = list(pathlib.Path(os.path.normpath((p.parent / t))).parts) command_path = os.fspath(pathlib.Path(*first) / pathlib.Path(*second)) break else: continue command_path_last = None
def is_file_writable(test_file): """Intentionally racy test if target is writable. We intend to write to the output file if and only if we succeed and can replace it atomically. Before doing the OCR work, make sure the location is writable. """ p = Path(test_file) if p.is_symlink(): p = p.resolve(strict=False) # p.is_file() throws an exception in some cases if p.exists() and p.is_file(): return os.access( os.fspath(p), os.W_OK, effective_ids=(os.access in os.supports_effective_ids), ) else: try: fp = p.open('wb') except OSError: return False else: fp.close() with suppress(OSError): p.unlink() return True
def __init__(self, file=None, parent=None, name=None, **kwargs): from .collection import Collection if file is None: return # parent if parent is None: parent = "" if parent == "": parent = pathlib.PurePosixPath("/") path = pathlib.PurePosixPath("/") else: path = pathlib.PurePosixPath(parent) / name name = path.name if path.name else name parent = path.parent self.filepath = file.filename if name != "" and parent.name != "": # Ensure that the parent Collection object is made first Collection(file, parent=parent.parent, name=parent.name, edit_local=True) file.require_group(str(path)) h5py.Group.__init__(self, bind=file[str(path)].id) self.__n = 0 self.fid = self.file.id self.natural_name = name # attrs self.attrs["class"] = self.class_name if "created" not in self.attrs.keys(): self.attrs["created"] = wt_kit.TimeStamp().RFC3339 for key, value in kwargs.items(): try: if isinstance(value, pathlib.Path): value = str(value) elif ( isinstance(value, list) and len(value) > 0 and isinstance(value[0], (str, os.PathLike)) ): value = np.array(value, dtype="S") elif sys.version_info > (3, 6): try: value = os.fspath(value) except TypeError: pass # Not all things that can be stored have fspath self.attrs[key] = value except TypeError: # some values have no native HDF5 equivalent message = "'{}' not included in attrs because its Type ({}) cannot be represented" message = message.format(key, type(value)) warnings.warn(message) # the following are populated if not already recorded self.__version__ self.item_names parent = file[str(parent)] if parent.name == self.name: pass # at root, dont add to item_names elif self.natural_name.encode() not in parent.attrs["item_names"]: parent.attrs["item_names"] = np.append( parent.attrs["item_names"], self.natural_name.encode() )
def test_excelfile_fspath(self): with tm.ensure_clean('foo.xlsx') as path: df = DataFrame({"A": [1, 2]}) df.to_excel(path) xl = ExcelFile(path) result = os.fspath(xl) assert result == path
def expanduser(path): """Expand ~ and ~user constructs. If user or $HOME is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): tilde = b'~' else: tilde = '~' if not path.startswith(tilde): return path i, n = 1, len(path) while i < n and path[i] not in _get_bothseps(path): i += 1 if 'HOME' in os.environ: userhome = os.environ['HOME'] elif 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] elif not 'HOMEPATH' in os.environ: return path else: try: drive = os.environ['HOMEDRIVE'] except KeyError: drive = '' userhome = join(drive, os.environ['HOMEPATH']) if isinstance(path, bytes): userhome = os.fsencode(userhome) if i != 1: #~user userhome = join(dirname(userhome), path[1:i]) return userhome + path[i:]
def normcase(s): """Normalize case of pathname. Has no effect under Posix""" s = os.fspath(s) if not isinstance(s, (bytes, str)): raise TypeError("normcase() argument must be str or bytes, " "not '{}'".format(s.__class__.__name__)) return s
def test_magic_methods_fspath(self): mock = MagicMock() expected_path = mock.__fspath__() mock.reset_mock() self.assertEqual(os.fspath(mock), expected_path) mock.__fspath__.assert_called_once()
def as_handle(handleish, mode='r', **kwargs): r"""Context manager to ensure we are using a handle. Context manager for arguments that can be passed to SeqIO and AlignIO read, write, and parse methods: either file objects or path-like objects (strings, pathlib.Path instances, or more generally, anything that can be handled by the builtin 'open' function). When given a path-like object, returns an open file handle to that path, with provided mode, which will be closed when the manager exits. All other inputs are returned, and are *not* closed. Arguments: - handleish - Either a file handle or path-like object (anything which can be passed to the builtin 'open' function: str, bytes, and under Python >= 3.6, pathlib.Path, os.DirEntry) - mode - Mode to open handleish (used only if handleish is a string) - kwargs - Further arguments to pass to open(...) Examples -------- >>> with as_handle('seqs.fasta', 'w') as fp: ... fp.write('>test\nACGT') >>> fp.closed True >>> handle = open('seqs.fasta', 'w') >>> with as_handle(handle) as fp: ... fp.write('>test\nACGT') >>> fp.closed False >>> fp.close() Note that if the mode argument includes U (for universal new lines) this will be removed under Python 3 where is is redundant and has been deprecated (this happens automatically in text mode). """ # If we're running under a version of Python that supports PEP 519, try # to convert `handleish` to a string with `os.fspath`. if hasattr(os, 'fspath'): try: handleish = os.fspath(handleish) except TypeError: # handleish isn't path-like, and it remains unchanged -- we'll yield it below pass if isinstance(handleish, basestring): if sys.version_info[0] >= 3 and "U" in mode: mode = mode.replace("U", "") if 'encoding' in kwargs: with codecs.open(handleish, mode, **kwargs) as fp: yield fp else: with open(handleish, mode, **kwargs) as fp: yield fp else: yield handleish
def test_pathlib_file(self): t1 = Path(self.writeTmp("Pathlib file.")) with FileInput(t1) as fi: line = fi.readline() self.assertEqual(line, 'Pathlib file.') self.assertEqual(fi.lineno(), 1) self.assertEqual(fi.filelineno(), 1) self.assertEqual(fi.filename(), os.fspath(t1))
def _readline(self): if not self._files: if 'b' in self._mode: return b'' else: return '' self._filename = self._files[0] self._files = self._files[1:] self._startlineno = self.lineno() self._filelineno = 0 self._file = None self._isstdin = False self._backupfilename = 0 if self._filename == '-': self._filename = '<stdin>' if 'b' in self._mode: self._file = getattr(sys.stdin, 'buffer', sys.stdin) else: self._file = sys.stdin self._isstdin = True else: if self._inplace: self._backupfilename = ( os.fspath(self._filename) + (self._backup or ".bak")) try: os.unlink(self._backupfilename) except OSError: pass # The next few lines may raise OSError os.rename(self._filename, self._backupfilename) self._file = open(self._backupfilename, self._mode) try: perm = os.fstat(self._file.fileno()).st_mode except OSError: self._output = open(self._filename, "w") else: mode = os.O_CREAT | os.O_WRONLY | os.O_TRUNC if hasattr(os, 'O_BINARY'): mode |= os.O_BINARY fd = os.open(self._filename, mode, perm) self._output = os.fdopen(fd, "w") try: if hasattr(os, 'chmod'): os.chmod(self._filename, perm) except OSError: pass self._savestdout = sys.stdout sys.stdout = self._output else: # This may raise OSError if self._openhook: self._file = self._openhook(self._filename, self._mode) else: self._file = open(self._filename, self._mode) self._readline = self._file.readline # hide FileInput._readline return self._readline()
def splitext(p): p = os.fspath(p) if isinstance(p, bytes): sep = b'/' extsep = b'.' else: sep = '/' extsep = '.' return genericpath._splitext(p, sep, None, extsep)
def dirname(p): """Returns the directory component of a pathname""" p = os.fspath(p) sep = _get_sep(p) i = p.rfind(sep) + 1 head = p[:i] if head and head != sep*len(head): head = head.rstrip(sep) return head
def test_magic_methods_fspath(self): mock = MagicMock() if sys.version_info < (3, 6): self.assertRaises(AttributeError, lambda: mock.__fspath__) else: expected_path = mock.__fspath__() mock.reset_mock() self.assertEqual(os.fspath(mock), expected_path) mock.__fspath__.assert_called_once()
def test_no_languages(ensure_tess4, tmpdir): env = ensure_tess4 (tmpdir / 'tessdata').mkdir() env['TESSDATA_PREFIX'] = fspath(tmpdir) with modified_os_environ(env): with pytest.raises(MissingDependencyError): tesseract.languages.cache_clear() tesseract.languages()
def pkg_config_setup_extension( ext, package, atleast_version=None, alt_exec=None, default_libraries=()): """Add parameters to the given *ext* for the given *package*.""" # First, try to get the flags from pkg-config. pkg_config = get_pkg_config() cmd = [pkg_config, package] if pkg_config else alt_exec if cmd is not None: try: if pkg_config and atleast_version: subprocess.check_call( [*cmd, f"--atleast-version={atleast_version}"]) # Use sys.getfilesystemencoding() to allow round-tripping # when passed back to later subprocess calls; do not use # locale.getpreferredencoding() which universal_newlines=True # would do. cflags = shlex.split( os.fsdecode(subprocess.check_output([*cmd, "--cflags"]))) libs = shlex.split( os.fsdecode(subprocess.check_output([*cmd, "--libs"]))) except (OSError, subprocess.CalledProcessError): pass else: ext.extra_compile_args.extend(cflags) ext.extra_link_args.extend(libs) return # If that fails, fall back on the defaults. # conda Windows header and library paths. # https://github.com/conda/conda/issues/2312 re: getting the env dir. if sys.platform == 'win32': conda_env_path = (os.getenv('CONDA_PREFIX') # conda >= 4.1 or os.getenv('CONDA_DEFAULT_ENV')) # conda < 4.1 if conda_env_path and os.path.isdir(conda_env_path): ext.include_dirs.append(os.fspath( pathlib.Path(conda_env_path, "Library/include"))) ext.library_dirs.append(os.fspath( pathlib.Path(conda_env_path, "Library/lib"))) # Default linked libs. ext.libraries.extend(default_libraries)
def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) if isinstance(path, bytes): sep = b'\\' curdir = b'.' pardir = b'..' else: sep = '\\' curdir = '.' pardir = '..' if start is None: start = curdir if not path: raise ValueError("no path specified") start = os.fspath(start) try: start_abs = abspath(normpath(start)) path_abs = abspath(normpath(path)) start_drive, start_rest = splitdrive(start_abs) path_drive, path_rest = splitdrive(path_abs) if normcase(start_drive) != normcase(path_drive): raise ValueError("path is on mount %r, start on mount %r" % ( path_drive, start_drive)) start_list = [x for x in start_rest.split(sep) if x] path_list = [x for x in path_rest.split(sep) if x] # Work out how much of the filepath is shared by start and path. i = 0 for e1, e2 in zip(start_list, path_list): if normcase(e1) != normcase(e2): break i += 1 rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir return join(*rel_list) except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise
def transcode_jpegs(pike, jpegs, root, log, options): for xref in jpegs: in_jpg = Path(jpg_name(root, xref)) opt_jpg = in_jpg.with_suffix('.opt.jpg') # This produces a debug warning from PIL # DEBUG:PIL.Image:Error closing: 'NoneType' object has no attribute # 'close'. Seems to be mostly harmless # https://github.com/python-pillow/Pillow/issues/1144 with Image.open(fspath(in_jpg)) as im: im.save(fspath(opt_jpg), optimize=True, quality=options.jpeg_quality) if opt_jpg.stat().st_size > in_jpg.stat().st_size: log.debug("xref %s, jpeg, made larger - skip", xref) continue compdata = leptonica.CompressedData.open(opt_jpg) im_obj = pike.get_object(xref, 0) im_obj.write(compdata.read(), filter=Name.DCTDecode)
def jbig2_group_futures(executor, root, groups): for group, xref_exts in groups.items(): prefix = f'group{group:08d}' future = executor.submit( jbig2enc.convert_group, cwd=fspath(root), infiles=(img_name(root, xref, ext) for xref, ext in xref_exts), out_prefix=prefix, ) yield future
def abspath(path): """Return the absolute version of a path.""" path = os.fspath(path) if not isabs(path): if isinstance(path, bytes): cwd = os.getcwdb() else: cwd = os.getcwd() path = join(cwd, path) return normpath(path)
def __init__(self, data, options=0, fn=None): self.face = None if fn: self.face = gr2.gr_make_face(bytes(data), fn, options) else: if not os.path.isfile(data): raise FileNotFoundError(errno.ENOENT, os.strerror(errno.ENOENT), data) if hasattr(data, '__fspath__'): data = os.fspath(data) self.face = gr2.gr_make_file_face(data.encode('utf_8'), options)
def split(p): """Split a pathname. Returns tuple "(head, tail)" where "tail" is everything after the final slash. Either part may be empty.""" p = os.fspath(p) sep = _get_sep(p) i = p.rfind(sep) + 1 head, tail = p[:i], p[i:] if head and head != sep*len(head): head = head.rstrip(sep) return head, tail
def guessit(self, string, options=None): # pylint: disable=too-many-branches """ Retrieves all matches from string as a dict :param string: the filename or release name :type string: str|Path :param options: :type options: str|dict :return: :rtype: """ try: from pathlib import Path if isinstance(string, Path): try: # Handle path-like object string = os.fspath(string) except AttributeError: string = str(string) except ImportError: pass try: options = parse_options(options, True) options = self._fix_encoding(options) config = self.configure(options, sanitize_options=False) options = merge_options(config, options) result_decode = False result_encode = False if six.PY2: if isinstance(string, six.text_type): string = string.encode("utf-8") result_decode = True elif isinstance(string, six.binary_type): string = six.binary_type(string) if six.PY3: if isinstance(string, six.binary_type): string = string.decode('ascii') result_encode = True elif isinstance(string, six.text_type): string = six.text_type(string) matches = self.rebulk.matches(string, options) if result_decode: for match in matches: if isinstance(match.value, six.binary_type): match.value = match.value.decode("utf-8") if result_encode: for match in matches: if isinstance(match.value, six.text_type): match.value = match.value.encode("ascii") return matches.to_dict(options.get('advanced', False), options.get('single_value', False), options.get('enforce_list', False)) except: raise GuessitException(string, options)
def create_unix_server(self, protocol_factory, path=None, *, sock=None, backlog=100, ssl=None): if isinstance(ssl, bool): raise TypeError('ssl argument must be an SSLContext or None') if path is not None: if sock is not None: raise ValueError( 'path and sock can not be specified at the same time') path = os.fspath(path) sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) # Check for abstract socket. `str` and `bytes` paths are supported. if path[0] not in (0, '\x00'): try: if stat.S_ISSOCK(os.stat(path).st_mode): os.remove(path) except FileNotFoundError: pass except OSError as err: # Directory may have permissions only to create socket. logger.error('Unable to check or remove stale UNIX socket ' '%r: %r', path, err) try: sock.bind(path) except OSError as exc: sock.close() if exc.errno == errno.EADDRINUSE: # Let's improve the error message by adding # with what exact address it occurs. msg = 'Address {!r} is already in use'.format(path) raise OSError(errno.EADDRINUSE, msg) from None else: raise except: sock.close() raise else: if sock is None: raise ValueError( 'path was not specified, and no sock specified') if (sock.family != socket.AF_UNIX or not base_events._is_stream_socket(sock)): raise ValueError( 'A UNIX Domain Stream Socket was expected, got {!r}' .format(sock)) server = base_events.Server(self, [sock]) sock.listen(backlog) sock.setblocking(False) self._start_serving(protocol_factory, sock, ssl, server) return server
def splitdrive(p): """Split a pathname into drive and path. On Posix, drive is always empty.""" p = os.fspath(p) return p[:0], p
def realpath(filename): """Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.""" filename = os.fspath(filename) path, ok = _joinrealpath(filename[:0], filename, {}) return abspath(path)
def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) sep = _get_sep(s) return s.startswith(sep)
import os from pathlib import Path import logging PARENT_PATH = os.fspath(Path(__file__).parents[0]) LOGGING_FILE_PATH = os.path.join(PARENT_PATH, "__logger", "{}.log") def set_logger(file_path_extension): '''A logging helper. Keeps the logged experiments in the __logger path. Both prints out on the Terminal and writes on the .log file.''' logging.basicConfig( level=logging.INFO, format="%(asctime)-7s: %(levelname)-1s %(message)s", datefmt="%Y-%m-%d %H:%M:%S", handlers=[ logging.FileHandler( LOGGING_FILE_PATH.format(file_path_extension) ), logging.StreamHandler() ]) return logging
import os import peewee from .common import DATADIR DBPATH = DATADIR / 'data.db' db = peewee.SqliteDatabase(os.fspath(DBPATH)) class Status(peewee.Model): status_id = peewee.IntegerField(unique=True) created_at = peewee.IntegerField() body = peewee.TextField() images = peewee.TextField() repost = peewee.BooleanField(default=False) orig_body = peewee.TextField(default='') orig_images = peewee.TextField(default='') class Meta: database = db db.connect() db.create_table(Status, safe=True)
def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) s = splitdrive(s)[1] return len(s) > 0 and s[0] in _get_bothseps(s)
def create_new_project(project, experimenter, videos, working_directory=None, copy_videos=False, videotype='.avi'): """Creates a new project directory, sub-directories and a basic configuration file. The configuration file is loaded with the default values. Change its parameters to your projects need. Parameters ---------- project : string String containing the name of the project. experimenter : string String containing the name of the experimenter. videos : list A list of string containing the full paths of the videos to include in the project. Attention: Can also be a directory, then all videos of videotype will be imported. Do not pass it as a list! working_directory : string, optional The directory where the project will be created. The default is the ``current working directory``; if provided, it must be a string. copy_videos : bool, optional If this is set to True, the videos are copied to the ``videos`` directory. If it is False,symlink of the videos are copied to the project/videos directory. The default is ``False``; if provided it must be either ``True`` or ``False``. Example -------- Linux/MacOs >>> deeplabcut.create_new_project('reaching-task','Linus',['/data/videos/mouse1.avi','/data/videos/mouse2.avi','/data/videos/mouse3.avi'],'/analysis/project/') >>> deeplabcut.create_new_project('reaching-task','Linus','/data/videos',videotype='.mp4') Windows: >>> deeplabcut.create_new_project('reaching-task','Bill',[r'C:\yourusername\rig-95\Videos\reachingvideo1.avi'], copy_videos=True) Users must format paths with either: r'C:\ OR 'C:\\ <- i.e. a double backslash \ \ ) """ from datetime import datetime as dt from deeplabcut.utils import auxiliaryfunctions date = dt.today() month = date.strftime("%B") day = date.day d = str(month[0:3] + str(day)) date = dt.today().strftime('%Y-%m-%d') if working_directory == None: working_directory = '.' wd = Path(working_directory).resolve() project_name = '{pn}-{exp}-{date}'.format(pn=project, exp=experimenter, date=date) project_path = wd / project_name # Create project and sub-directories if not DEBUG and project_path.exists(): print('Project "{}" already exists!'.format(project_path)) return video_path = project_path / 'videos' data_path = project_path / 'labeled-data' shuffles_path = project_path / 'training-datasets' results_path = project_path / 'dlc-models' for p in [video_path, data_path, shuffles_path, results_path]: p.mkdir(parents=True, exist_ok=DEBUG) print('Created "{}"'.format(p)) # Import all videos in a folder or if just one video withouth [] passed, then make it a list. if isinstance(videos, str): #there are two cases: if os.path.isdir(videos): # it is a path! path = videos videos = [ os.path.join(path, vp) for vp in os.listdir(path) if videotype in vp ] if len(videos) == 0: print("No videos found in", path, os.listdir(path)) print( "Perhaps change the videotype, which is currently set to:", videotype) else: print("Directory entered, ", len(videos), " videos were found.") else: if os.path.isfile(videos): videos = [videos] videos = [Path(vp) for vp in videos] dirs = [data_path / Path(i.stem) for i in videos] for p in dirs: """ Creates directory under data """ p.mkdir(parents=True, exist_ok=True) destinations = [video_path.joinpath(vp.name) for vp in videos] if copy_videos == True: print("Copying the videos") for src, dst in zip(videos, destinations): shutil.copy( os.fspath(src), os.fspath(dst)) #https://www.python.org/dev/peps/pep-0519/ #https://github.com/AlexEMG/DeepLabCut/issues/105 (for windows) #try: # #shutil.copy(src,dst) #except OSError or TypeError: #https://github.com/AlexEMG/DeepLabCut/issues/105 (for windows) # shutil.copy(os.fspath(src),os.fspath(dst)) else: # creates the symlinks of the video and puts it in the videos directory. print("Creating the symbolic link of the video") for src, dst in zip(videos, destinations): if dst.exists() and not DEBUG: raise FileExistsError('Video {} exists already!'.format(dst)) try: src = str(src) dst = str(dst) os.symlink(src, dst) except OSError: import subprocess subprocess.check_call('mklink %s %s' % (dst, src), shell=True) print('Created the symlink of {} to {}'.format(src, dst)) videos = destinations if copy_videos == True: videos = destinations # in this case the *new* location should be added to the config file # adds the video list to the config.yaml file video_sets = {} for video in videos: print(video) try: rel_video_path = os.path.realpath(video) except: rel_video_path = os.readlink(str(video)) vcap = cv2.VideoCapture(rel_video_path) if vcap.isOpened(): width = int(vcap.get(cv2.CAP_PROP_FRAME_WIDTH)) height = int(vcap.get(cv2.CAP_PROP_FRAME_HEIGHT)) video_sets[rel_video_path] = { 'crop': ', '.join(map(str, [0, width, 0, height])) } else: print("Cannot open the video file!") video_sets = None # Set values to config file: cfg_file, ruamelFile = auxiliaryfunctions.create_config_template() cfg_file cfg_file['Task'] = project cfg_file['scorer'] = experimenter cfg_file['video_sets'] = video_sets cfg_file['project_path'] = str(project_path) cfg_file['date'] = d cfg_file['bodyparts'] = ['Hand', 'Finger1', 'Finger2', 'Joystick'] cfg_file['cropping'] = False cfg_file['start'] = 0 cfg_file['stop'] = 1 cfg_file['numframes2pick'] = 20 cfg_file['TrainingFraction'] = [0.95] cfg_file['iteration'] = 0 cfg_file['resnet'] = 50 cfg_file['snapshotindex'] = -1 cfg_file['x1'] = 0 cfg_file['x2'] = 640 cfg_file['y1'] = 277 cfg_file['y2'] = 624 cfg_file[ 'batch_size'] = 4 #batch size during inference (video - analysis); see https://www.biorxiv.org/content/early/2018/10/30/457242 cfg_file['corner2move2'] = (50, 50) cfg_file['move2corner'] = True cfg_file['pcutoff'] = 0.1 cfg_file['dotsize'] = 12 #for plots size of dots cfg_file['alphavalue'] = 0.7 #for plots transparency of markers cfg_file['colormap'] = 'jet' #for plots type of colormap projconfigfile = os.path.join(str(project_path), 'config.yaml') # Write dictionary to yaml config file auxiliaryfunctions.write_config(projconfigfile, cfg_file) print('Generated "{}"'.format(project_path / 'config.yaml')) print( "\nA new project with name %s is created at %s and a configurable file (config.yaml) is stored there. Change the parameters in this file to adapt to your project's needs.\n Once you have changed the configuration file, use the function 'extract_frames' to select frames for labeling.\n. [OPTIONAL] Use the function 'add_new_videos' to add new videos to your project (at any stage)." % (project_name, str(wd))) return projconfigfile
def test_get(self): cache = ODBManager(self.dvc).local.hash_to_path_info(self.cache1_md5) self.assertEqual(os.fspath(cache), self.cache1)
def sign(self, command, build_dir, bcfg, formats): if not formats: return args = command.args b = pathlib.Path(build_dir) tool_path = self.find_imgtool(command, args) # The vector table offset is set in Kconfig: vtoff = self.get_cfg(command, bcfg, 'CONFIG_ROM_START_OFFSET') # Flash device write alignment and the partition's slot size # come from devicetree: flash = self.edt_flash_node(b, args.quiet) align, addr, size = self.edt_flash_params(flash) dot_config_file = b / 'zephyr' / '.config' if not dot_config_file.is_file(): log.die(f"no .config found at {dot_config_file}") dot_config = load_dot_config(dot_config_file) if dot_config.get('CONFIG_BOOTLOADER_MCUBOOT', 'n') != 'y': log.wrn("CONFIG_BOOTLOADER_MCUBOOT is not set to y in " f"{dot_config_file}; this probably won't work") kernel = dot_config.get('CONFIG_KERNEL_BIN_NAME', 'zephyr') if 'bin' in formats: in_bin = b / 'zephyr' / f'{kernel}.bin' if not in_bin.is_file(): log.die(f"no unsigned .bin found at {in_bin}") in_bin = os.fspath(in_bin) else: in_bin = None if 'hex' in formats: in_hex = b / 'zephyr' / f'{kernel}.hex' if not in_hex.is_file(): log.die(f"no unsigned .hex found at {in_hex}") in_hex = os.fspath(in_hex) else: in_hex = None if not args.quiet: log.banner('image configuration:') log.inf('partition offset: {0} (0x{0:x})'.format(addr)) log.inf('partition size: {0} (0x{0:x})'.format(size)) log.inf('rom start offset: {0} (0x{0:x})'.format(vtoff)) # Base sign command. # # We provide a default --version in case the user is just # messing around and doesn't want to set one. It will be # overridden if there is a --version in args.tool_args. sign_base = [ tool_path, 'sign', '--version', '0.0.0+0', '--align', str(align), '--header-size', str(vtoff), '--slot-size', str(size) ] sign_base.extend(args.tool_args) if not args.quiet: log.banner('signing binaries') if in_bin: out_bin = args.sbin or str(b / 'zephyr' / 'zephyr.signed.bin') sign_bin = sign_base + [in_bin, out_bin] if not args.quiet: log.inf(f'unsigned bin: {in_bin}') log.inf(f'signed bin: {out_bin}') log.dbg(quote_sh_list(sign_bin)) subprocess.check_call(sign_bin) if in_hex: out_hex = args.shex or str(b / 'zephyr' / 'zephyr.signed.hex') sign_hex = sign_base + [in_hex, out_hex] if not args.quiet: log.inf(f'unsigned hex: {in_hex}') log.inf(f'signed hex: {out_hex}') log.dbg(quote_sh_list(sign_hex)) subprocess.check_call(sign_hex)
def compile_file(fullname, ddir=None, force=False, rx=None, quiet=0, legacy=False, optimize=-1): """Byte-compile one file. Arguments (only fullname is required): fullname: the file to byte-compile ddir: if given, the directory name compiled in to the byte-code file. force: if True, force compilation, even if timestamps are up-to-date quiet: full output with False or 0, errors only with 1, no output with 2 legacy: if True, produce legacy pyc paths instead of PEP 3147 paths optimize: optimization level or -1 for level of the interpreter """ success = True if quiet < 2 and isinstance(fullname, os.PathLike): fullname = os.fspath(fullname) name = os.path.basename(fullname) if ddir is not None: dfile = os.path.join(ddir, name) else: dfile = None if rx is not None: mo = rx.search(fullname) if mo: return success if os.path.isfile(fullname): if legacy: cfile = fullname + 'c' else: if optimize >= 0: opt = optimize if optimize >= 1 else '' cfile = importlib.util.cache_from_source(fullname, optimization=opt) else: cfile = importlib.util.cache_from_source(fullname) cache_dir = os.path.dirname(cfile) head, tail = name[:-3], name[-3:] if tail == '.py': if not force: try: mtime = int(os.stat(fullname).st_mtime) expect = struct.pack('<4sl', importlib.util.MAGIC_NUMBER, mtime) with open(cfile, 'rb') as chandle: actual = chandle.read(8) if expect == actual: return success except OSError: pass if not quiet: print('Compiling {!r}...'.format(fullname)) try: ok = py_compile.compile(fullname, cfile, dfile, True, optimize=optimize) except py_compile.PyCompileError as err: success = False if quiet >= 2: return success elif quiet: print('*** Error compiling {!r}...'.format(fullname)) else: print('*** ', end='') # escape non-printable characters in msg msg = err.msg.encode(sys.stdout.encoding, errors='backslashreplace') msg = msg.decode(sys.stdout.encoding) print(msg) except (SyntaxError, UnicodeError, OSError) as e: success = False if quiet >= 2: return success elif quiet: print('*** Error compiling {!r}...'.format(fullname)) else: print('*** ', end='') print(e.__class__.__name__ + ':', e) else: if ok == 0: success = False return success
def fromfile(cls, filepath): filepath = os.fspath(filepath) c = cls(**yaml.safe_load(open(filepath))) c.origin = filepath return c
def guess_type(self, url, strict=True): """Guess the type of a file which is either a URL or a path-like object. Return value is a tuple (type, encoding) where type is None if the type can't be guessed (no or unknown suffix) or a string of the form type/subtype, usable for a MIME Content-type header; and encoding is None for no encoding or the name of the program used to encode (e.g. compress or gzip). The mappings are table driven. Encoding suffixes are case sensitive; type suffixes are first tried case sensitive, then case insensitive. The suffixes .tgz, .taz and .tz (case sensitive!) are all mapped to '.tar.gz'. (This is table-driven too, using the dictionary suffix_map.) Optional `strict' argument when False adds a bunch of commonly found, but non-standard types. """ url = os.fspath(url) scheme, url = urllib.parse._splittype(url) if scheme == 'data': # syntax of data URLs: # dataurl := "data:" [ mediatype ] [ ";base64" ] "," data # mediatype := [ type "/" subtype ] *( ";" parameter ) # data := *urlchar # parameter := attribute "=" value # type/subtype defaults to "text/plain" comma = url.find(',') if comma < 0: # bad data URL return None, None semi = url.find(';', 0, comma) if semi >= 0: type = url[:semi] else: type = url[:comma] if '=' in type or '/' not in type: type = 'text/plain' return type, None # never compressed, so encoding is None base, ext = posixpath.splitext(url) while ext in self.suffix_map: base, ext = posixpath.splitext(base + self.suffix_map[ext]) if ext in self.encodings_map: encoding = self.encodings_map[ext] base, ext = posixpath.splitext(base) else: encoding = None types_map = self.types_map[True] if ext in types_map: return types_map[ext], encoding elif ext.lower() in types_map: return types_map[ext.lower()], encoding elif strict: return None, encoding types_map = self.types_map[False] if ext in types_map: return types_map[ext], encoding elif ext.lower() in types_map: return types_map[ext.lower()], encoding else: return None, encoding
def expandvars(path): """Expand shell variables of the forms $var, ${var} and %var%. Unknown variables are left unchanged.""" path = os.fspath(path) if isinstance(path, bytes): if b'$' not in path and b'%' not in path: return path import string varchars = bytes(string.ascii_letters + string.digits + '_-', 'ascii') quote = b'\'' percent = b'%' brace = b'{' rbrace = b'}' dollar = b'$' environ = getattr(os, 'environb', None) else: if '$' not in path and '%' not in path: return path import string varchars = string.ascii_letters + string.digits + '_-' quote = '\'' percent = '%' brace = '{' rbrace = '}' dollar = '$' environ = os.environ res = path[:0] index = 0 pathlen = len(path) while index < pathlen: c = path[index:index + 1] if c == quote: # no expansion within single quotes path = path[index + 1:] pathlen = len(path) try: index = path.index(c) res += c + path[:index + 1] except ValueError: res += c + path index = pathlen - 1 elif c == percent: # variable or '%' if path[index + 1:index + 2] == percent: res += c index += 1 else: path = path[index + 1:] pathlen = len(path) try: index = path.index(percent) except ValueError: res += percent + path index = pathlen - 1 else: var = path[:index] try: if environ is None: value = os.fsencode(os.environ[os.fsdecode(var)]) else: value = environ[var] except KeyError: value = percent + var + percent res += value elif c == dollar: # variable or '$$' if path[index + 1:index + 2] == dollar: res += c index += 1 elif path[index + 1:index + 2] == brace: path = path[index + 2:] pathlen = len(path) try: index = path.index(rbrace) except ValueError: res += dollar + brace + path index = pathlen - 1 else: var = path[:index] try: if environ is None: value = os.fsencode(os.environ[os.fsdecode(var)]) else: value = environ[var] except KeyError: value = dollar + brace + var + rbrace res += value else: var = path[:0] index += 1 c = path[index:index + 1] while c and c in varchars: var += c index += 1 c = path[index:index + 1] try: if environ is None: value = os.fsencode(os.environ[os.fsdecode(var)]) else: value = environ[var] except KeyError: value = dollar + var res += value if c: index -= 1 else: res += c index += 1 return res
def Calc(calc: MaybeSequence[str], outfile: Optional[PathLikeOrStr] = None, NoDataValue: Optional[Number] = None, type: Optional[Union[GDALDataType, str]] = None, format: Optional[str] = None, creation_options: Optional[Sequence[str]] = None, allBands: str = '', overwrite: bool = False, hideNoData: bool = False, projectionCheck: bool = False, color_table: Optional[ColorTableLike] = None, extent: Optional[Extent] = None, projwin: Optional[Union[Tuple, GeoRectangle]] = None, user_namespace: Optional[Dict] = None, debug: bool = False, quiet: bool = False, **input_files): if debug: print(f"gdal_calc.py starting calculation {calc}") # Single calc value compatibility if isinstance(calc, (list, tuple)): calc = calc else: calc = [calc] calc = [c.strip('"') for c in calc] creation_options = creation_options or [] # set up global namespace for eval with all functions of gdal_array, numpy global_namespace = { key: getattr(module, key) for module in [gdal_array, numpy] for key in dir(module) if not key.startswith('__') } if user_namespace: global_namespace.update(user_namespace) if not calc: raise Exception("No calculation provided.") elif not outfile and format.upper() != 'MEM': raise Exception("No output file provided.") if format is None: format = GetOutputDriverFor(outfile) if isinstance(extent, GeoRectangle): pass elif projwin: if isinstance(projwin, GeoRectangle): extent = projwin else: extent = GeoRectangle.from_lurd(*projwin) elif not extent: extent = Extent.IGNORE else: extent = extent_util.parse_extent(extent) compatible_gt_eps = 0.000001 gt_diff_support = { GT.INCOMPATIBLE_OFFSET: extent != Extent.FAIL, GT.INCOMPATIBLE_PIXEL_SIZE: False, GT.INCOMPATIBLE_ROTATION: False, GT.NON_ZERO_ROTATION: False, } gt_diff_error = { GT.INCOMPATIBLE_OFFSET: 'different offset', GT.INCOMPATIBLE_PIXEL_SIZE: 'different pixel size', GT.INCOMPATIBLE_ROTATION: 'different rotation', GT.NON_ZERO_ROTATION: 'non zero rotation', } ################################################################ # fetch details of input layers ################################################################ # set up some lists to store data for each band myFileNames = [] # input filenames myFiles = [] # input DataSets myBands = [] # input bands myAlphaList = [] # input alpha letter that represents each input file myDataType = [] # string representation of the datatype of each input file myDataTypeNum = [] # datatype of each input file myNDV = [] # nodatavalue for each input file DimensionsCheck = None # dimensions of the output Dimensions = [] # Dimensions of input files ProjectionCheck = None # projection of the output GeoTransformCheck = None # GeoTransform of the output GeoTransforms = [] # GeoTransform of each input file GeoTransformDiffer = False # True if we have inputs with different GeoTransforms myTempFileNames = [] # vrt filename from each input file myAlphaFileLists = [] # list of the Alphas which holds a list of inputs # loop through input files - checking dimensions for alphas, filenames in input_files.items(): if isinstance(filenames, (list, tuple)): # alpha is a list of files myAlphaFileLists.append(alphas) elif is_path_like(filenames) or isinstance(filenames, gdal.Dataset): # alpha is a single filename or a Dataset filenames = [filenames] alphas = [alphas] else: # I guess this alphas should be in the global_namespace, # It would have been better to pass it as user_namepsace, but I'll accept it anyway global_namespace[alphas] = filenames continue for alpha, filename in zip(alphas * len(filenames), filenames): if not alpha.endswith("_band"): # check if we have asked for a specific band... alpha_band = f"{alpha}_band" if alpha_band in input_files: myBand = input_files[alpha_band] else: myBand = 1 myF_is_ds = not is_path_like(filename) if myF_is_ds: myFile = filename filename = None else: myFile = open_ds(filename, gdal.GA_ReadOnly) if not myFile: raise IOError(f"No such file or directory: '{filename}'") myFileNames.append(filename) myFiles.append(myFile) myBands.append(myBand) myAlphaList.append(alpha) dt = myFile.GetRasterBand(myBand).DataType myDataType.append(gdal.GetDataTypeName(dt)) myDataTypeNum.append(dt) myNDV.append(None if hideNoData else myFile. GetRasterBand(myBand).GetNoDataValue()) # check that the dimensions of each layer are the same myFileDimensions = [myFile.RasterXSize, myFile.RasterYSize] if DimensionsCheck: if DimensionsCheck != myFileDimensions: GeoTransformDiffer = True if extent in [Extent.IGNORE, Extent.FAIL]: raise Exception( f"Error! Dimensions of file {filename} ({myFileDimensions[0]:d}, " f"{myFileDimensions[1]:d}) are different from other files " f"({DimensionsCheck[0]:d}, {DimensionsCheck[1]:d}). Cannot proceed" ) else: DimensionsCheck = myFileDimensions # check that the Projection of each layer are the same myProjection = myFile.GetProjection() if ProjectionCheck: if projectionCheck and ProjectionCheck != myProjection: raise Exception( f"Error! Projection of file {filename} {myProjection} " f"are different from other files {ProjectionCheck}. Cannot proceed" ) else: ProjectionCheck = myProjection # check that the GeoTransforms of each layer are the same myFileGeoTransform = myFile.GetGeoTransform( can_return_null=True) if extent == Extent.IGNORE: GeoTransformCheck = myFileGeoTransform else: Dimensions.append(myFileDimensions) GeoTransforms.append(myFileGeoTransform) if not GeoTransformCheck: GeoTransformCheck = myFileGeoTransform else: my_gt_diff = extent_util.gt_diff( GeoTransformCheck, myFileGeoTransform, eps=compatible_gt_eps, diff_support=gt_diff_support) if my_gt_diff not in [GT.SAME, GT.ALMOST_SAME]: GeoTransformDiffer = True if my_gt_diff != GT.COMPATIBLE_DIFF: raise Exception( f"Error! GeoTransform of file {filename} {myFileGeoTransform} is incompatible " f"({gt_diff_error[my_gt_diff]}), first file GeoTransform is {GeoTransformCheck}. " f"Cannot proceed") if debug: print( f"file {alpha}: {filename}, dimensions: " f"{DimensionsCheck[0]}, {DimensionsCheck[1]}, type: {myDataType[-1]}" ) # process allBands option allBandsIndex = None allBandsCount = 1 if allBands: if len(calc) > 1: raise Exception("Error! --allBands implies a single --calc") try: allBandsIndex = myAlphaList.index(allBands) except ValueError: raise Exception( f"Error! allBands option was given but Band {allBands} not found. Cannot proceed" ) allBandsCount = myFiles[allBandsIndex].RasterCount if allBandsCount <= 1: allBandsIndex = None else: allBandsCount = len(calc) if extent not in [ Extent.IGNORE, Extent.FAIL ] and (GeoTransformDiffer or isinstance(extent, GeoRectangle)): # mixing different GeoTransforms/Extents GeoTransformCheck, DimensionsCheck, ExtentCheck = extent_util.calc_geotransform_and_dimensions( GeoTransforms, Dimensions, extent) if GeoTransformCheck is None: raise Exception( "Error! The requested extent is empty. Cannot proceed") for i in range(len(myFileNames)): temp_vrt_filename, temp_vrt_ds = extent_util.make_temp_vrt( myFiles[i], ExtentCheck) myTempFileNames.append(temp_vrt_filename) myFiles[i] = None # close original ds myFiles[i] = temp_vrt_ds # replace original ds with vrt_ds # update the new precise dimensions and gt from the new ds GeoTransformCheck = temp_vrt_ds.GetGeoTransform() DimensionsCheck = [ temp_vrt_ds.RasterXSize, temp_vrt_ds.RasterYSize ] temp_vrt_ds = None ################################################################ # set up output file ################################################################ # open output file exists if outfile and os.path.isfile(outfile) and not overwrite: if allBandsIndex is not None: raise Exception( "Error! allBands option was given but Output file exists, must use --overwrite option!" ) if len(calc) > 1: raise Exception( "Error! multiple calc options were given but Output file exists, must use --overwrite option!" ) if debug: print( f"Output file {outfile} exists - filling in results into file") myOut = open_ds(outfile, gdal.GA_Update) if myOut is None: error = 'but cannot be opened for update' elif [myOut.RasterXSize, myOut.RasterYSize] != DimensionsCheck: error = 'but is the wrong size' elif ProjectionCheck and ProjectionCheck != myOut.GetProjection(): error = 'but is the wrong projection' elif GeoTransformCheck and GeoTransformCheck != myOut.GetGeoTransform( can_return_null=True): error = 'but is the wrong geotransform' else: error = None if error: raise Exception( f"Error! Output exists, {error}. Use the --overwrite option " f"to automatically overwrite the existing file") myOutB = myOut.GetRasterBand(1) myOutNDV = myOutB.GetNoDataValue() myOutType = myOutB.DataType else: if outfile: # remove existing file and regenerate if os.path.isfile(outfile): os.remove(outfile) # create a new file if debug: print(f"Generating output file {outfile}") else: outfile = '' # find data type to use if not type: # use the largest type of the input files myOutType = max(myDataTypeNum) else: myOutType = type if isinstance(myOutType, str): myOutType = gdal.GetDataTypeByName(myOutType) # create file myOutDrv = gdal.GetDriverByName(format) myOut = myOutDrv.Create(os.fspath(outfile), DimensionsCheck[0], DimensionsCheck[1], allBandsCount, myOutType, creation_options) # set output geo info based on first input layer if not GeoTransformCheck: GeoTransformCheck = myFiles[0].GetGeoTransform( can_return_null=True) if GeoTransformCheck: myOut.SetGeoTransform(GeoTransformCheck) if not ProjectionCheck: ProjectionCheck = myFiles[0].GetProjection() if ProjectionCheck: myOut.SetProjection(ProjectionCheck) if NoDataValue is None: myOutNDV = DefaultNDVLookup[ myOutType] # use the default noDataValue for this datatype elif isinstance(NoDataValue, str) and NoDataValue.lower() == 'none': myOutNDV = None # not to set any noDataValue else: myOutNDV = NoDataValue # use the given noDataValue for i in range(1, allBandsCount + 1): myOutB = myOut.GetRasterBand(i) if myOutNDV is not None: myOutB.SetNoDataValue(myOutNDV) if color_table: # set color table and color interpretation if is_path_like(color_table): color_table = get_color_table(color_table) myOutB.SetRasterColorTable(color_table) myOutB.SetRasterColorInterpretation(gdal.GCI_PaletteIndex) myOutB = None # write to band if hideNoData: myOutNDV = None myOutTypeName = gdal.GetDataTypeName(myOutType) if debug: print( f"output file: {outfile}, dimensions: {myOut.RasterXSize}, {myOut.RasterYSize}, type: {myOutTypeName}" ) ################################################################ # find block size to chop grids into bite-sized chunks ################################################################ # use the block size of the first layer to read efficiently myBlockSize = myFiles[0].GetRasterBand(myBands[0]).GetBlockSize() # find total x and y blocks to be read nXBlocks = (int)( (DimensionsCheck[0] + myBlockSize[0] - 1) / myBlockSize[0]) nYBlocks = (int)( (DimensionsCheck[1] + myBlockSize[1] - 1) / myBlockSize[1]) myBufSize = myBlockSize[0] * myBlockSize[1] if debug: print(f"using blocksize {myBlockSize[0]} x {myBlockSize[1]}") # variables for displaying progress ProgressCt = -1 ProgressMk = -1 ProgressEnd = nXBlocks * nYBlocks * allBandsCount ################################################################ # start looping through each band in allBandsCount ################################################################ for bandNo in range(1, allBandsCount + 1): ################################################################ # start looping through blocks of data ################################################################ # store these numbers in variables that may change later nXValid = myBlockSize[0] nYValid = myBlockSize[1] # loop through X-lines for X in range(0, nXBlocks): # in case the blocks don't fit perfectly # change the block size of the final piece if X == nXBlocks - 1: nXValid = DimensionsCheck[0] - X * myBlockSize[0] # find X offset myX = X * myBlockSize[0] # reset buffer size for start of Y loop nYValid = myBlockSize[1] myBufSize = nXValid * nYValid # loop through Y lines for Y in range(0, nYBlocks): ProgressCt += 1 if 10 * ProgressCt / ProgressEnd % 10 != ProgressMk and not quiet: ProgressMk = 10 * ProgressCt / ProgressEnd % 10 from sys import version_info if version_info >= (3, 0, 0): exec('print("%d.." % (10*ProgressMk), end=" ")') else: exec('print 10*ProgressMk, "..",') # change the block size of the final piece if Y == nYBlocks - 1: nYValid = DimensionsCheck[1] - Y * myBlockSize[1] myBufSize = nXValid * nYValid # find Y offset myY = Y * myBlockSize[1] # create empty buffer to mark where nodata occurs myNDVs = None # make local namespace for calculation local_namespace = {} val_lists = defaultdict(list) # fetch data for each input layer for i, Alpha in enumerate(myAlphaList): # populate lettered arrays with values if allBandsIndex is not None and allBandsIndex == i: myBandNo = bandNo else: myBandNo = myBands[i] myval = gdal_array.BandReadAsArray( myFiles[i].GetRasterBand(myBandNo), xoff=myX, yoff=myY, win_xsize=nXValid, win_ysize=nYValid) if myval is None: raise Exception( f'Input block reading failed from filename {filename[i]}' ) # fill in nodata values if myNDV[i] is not None: # myNDVs is a boolean buffer. # a cell equals to 1 if there is NDV in any of the corresponding cells in input raster bands. if myNDVs is None: # this is the first band that has NDV set. we initializes myNDVs to a zero buffer # as we didn't see any NDV value yet. myNDVs = numpy.zeros(myBufSize) myNDVs.shape = (nYValid, nXValid) myNDVs = 1 * numpy.logical_or(myNDVs == 1, myval == myNDV[i]) # add an array of values for this block to the eval namespace if Alpha in myAlphaFileLists: val_lists[Alpha].append(myval) else: local_namespace[Alpha] = myval myval = None for lst in myAlphaFileLists: local_namespace[lst] = val_lists[lst] # try the calculation on the array blocks this_calc = calc[bandNo - 1 if len(calc) > 1 else 0] try: myResult = eval(this_calc, global_namespace, local_namespace) except: print(f"evaluation of calculation {this_calc} failed") raise # Propagate nodata values (set nodata cells to zero # then add nodata value to these cells). if myNDVs is not None and myOutNDV is not None: myResult = ( (1 * (myNDVs == 0)) * myResult) + (myOutNDV * myNDVs) elif not isinstance(myResult, numpy.ndarray): myResult = numpy.ones((nYValid, nXValid)) * myResult # write data block to the output file myOutB = myOut.GetRasterBand(bandNo) if gdal_array.BandWriteArray( myOutB, myResult, xoff=myX, yoff=myY) != 0: raise Exception('Block writing failed') myOutB = None # write to band # remove temp files for idx, tempFile in enumerate(myTempFileNames): myFiles[idx] = None os.remove(tempFile) gdal.ErrorReset() myOut.FlushCache() if gdal.GetLastErrorMsg() != '': raise Exception('Dataset writing failed') if not quiet: print("100 - Done") return myOut
def static_folder(self, value): if value is not None: value = os.fspath(value).rstrip(r"\/") self._static_folder = value
async def test_open_file_can_open_path(path): async with await trio.open_file(path, "w") as f: assert f.name == os.fspath(path)
def collect_stages(): return {stage.relpath for stage in Repo(os.fspath(tmp_dir)).stages}
def serialize(self): return repr(os.fspath(self.value)), {}
import os, shutil import time import subprocess import json from pathlib import Path from pylib import mongo import base64 from pylib import textual import datetime db = mongo.get_makalu() parent_dir = "/home/report" directory = "generated_reports_coll/" Original_report_location = os.fspath( "/opt/makalu/app_store/reports/generatedreports/") new_path = os.path.join(parent_dir, directory) mode = 0o766 error_file = os.path.join(parent_dir, "error.txt") counter_file = os.path.join(parent_dir, "counter.txt") if not os.path.exists(new_path): os.mkdir(new_path, mode) def write_error(location, error): with open(error_file, "a") as record: record.write(f"{datetime.datetime.now()}: {location} {error}\n") def lookup_mongo(report_name):
# Static files (CSS, JavaScript, Images) # https://docs.djangoproject.com/en/3.1/howto/static-files/ STATIC_URL = "/static/" # Keep this as the last section of this file! try: from .local_settings import * except ImportError: pass # Intended to be a temporary hack for overriding settings in production. # TODO: figure out a better way to do this, probably with a different path override_settings_file = Path("/opt/secret/override_settings.py") if override_settings_file.is_file(): print("Reading production override settings from", override_settings_file) sys.path.append(fspath(override_settings_file.parent)) try: from override_settings import * except ImportError as e: print("Couldn't read override settings found at", override_settings_file) raise # Sometimes we do need to define settings in terms of other settings, so # this is a good place to do so, after override settings are loaded. # Shouldn't define any constants at this point though # !!! overrides that depend on other (including local) settings MONGO_HOST_AND_PORT = f"mongodb://{MONGO_USERNAME}:{MONGO_PASSWORD}@{MONGO_HOSTNAME}:{MONGO_PORT}/" # /!!! overrides that depend on other (including local) settings
def __init__( self, root: Union[str, Path], url: str = URL, folder_in_archive: str = FOLDER_IN_ARCHIVE, download: bool = False, subset: Optional[str] = None, ) -> None: assert subset is None or subset in [ "training", "validation", "testing" ], ("When `subset` not None, it must take a value from " + "{'training', 'validation', 'testing'}.") if url in [ "speech_commands_v0.01", "speech_commands_v0.02", ]: base_url = "https://storage.googleapis.com/download.tensorflow.org/data/" ext_archive = ".tar.gz" url = os.path.join(base_url, url + ext_archive) # Get string representation of 'root' in case Path object is passed root = os.fspath(root) basename = os.path.basename(url) archive = os.path.join(root, basename) basename = basename.rsplit(".", 2)[0] folder_in_archive = os.path.join(folder_in_archive, basename) self._path = os.path.join(root, folder_in_archive) if download: if not os.path.isdir(self._path): if not os.path.isfile(archive): checksum = _CHECKSUMS.get(url, None) download_url(url, root, hash_value=checksum, hash_type="md5") extract_archive(archive, self._path) if subset == "validation": self._walker = _load_list(self._path, "validation_list.txt") elif subset == "testing": self._walker = _load_list(self._path, "testing_list.txt") elif subset == "training": excludes = set( _load_list(self._path, "validation_list.txt", "testing_list.txt")) walker = sorted(str(p) for p in Path(self._path).glob('*/*.wav')) self._walker = [ w for w in walker if HASH_DIVIDER in w and EXCEPT_FOLDER not in w and os.path.normpath(w) not in excludes ] else: walker = sorted(str(p) for p in Path(self._path).glob('*/*.wav')) self._walker = [ w for w in walker if HASH_DIVIDER in w and EXCEPT_FOLDER not in w ]
def _get_local_path(self, path: str, **kwargs: Any) -> str: self._check_kwargs(kwargs) return os.fspath(path)
def basename(p): """Returns the final component of a pathname""" p = os.fspath(p) sep = _get_sep(p) i = p.rfind(sep) + 1 return p[i:]
def __init__(self, *args, directory=None, **kwargs): if directory is None: directory = os.getcwd() self.directory = os.fspath(directory) super().__init__(*args, **kwargs)
def set_file(self, file): """ Set the filename of the fontfile to use. In this case, all other properties will be ignored. """ self._file = os.fspath(file) if file is not None else None
def splitext(p): p = os.fspath(p) if isinstance(p, bytes): return genericpath._splitext(p, b'\\', b'/', b'.') else: return genericpath._splitext(p, '\\', '/', '.')
def run_PETSurfaceCrossSectional( input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path ) -> None: import shutil import nibabel as nib import numpy as np from clinica.pipelines.pet_surface.pet_surface_pipeline import PetSurface shutil.copytree(input_dir / "caps", output_dir / "caps", copy_function=shutil.copy) parameters = { "acq_label": "FDG", "suvr_reference_region": "pons", "pvc_psf_tsv": fspath(input_dir / "subjects.tsv"), "longitudinal": False, "skip_question": False, } pipeline = PetSurface( bids_directory=fspath(input_dir / "bids"), caps_directory=fspath(output_dir / "caps"), tsv_file=fspath(input_dir / "subjects.tsv"), base_dir=fspath(working_dir), parameters=parameters, ) pipeline.build() pipeline.run(bypass_check=True) # Check files out_files = [ ( output_dir / "caps/subjects/sub-ADNI011S4105/ses-M00/pet/surface" / ( "sub-ADNI011S4105_ses-M00_task-rest_acq-fdg_pet_space-fsaverage_suvr-pons_pvc-iy_hemi-" + h + "_fwhm-" + str(f) + "_projection.mgh" ) ) for h in ["lh", "rh"] for f in [0, 5, 10, 15, 20, 25] ] ref_files = [ ( ref_dir / ( "sub-ADNI011S4105_ses-M00_task-rest_acq-fdg_pet_space-fsaverage_suvr-pons_pvc-iy_hemi-" + h + "_fwhm-" + str(f) + "_projection.mgh" ) ) for h in ["lh", "rh"] for f in [0, 5, 10, 15, 20, 25] ] for i in range(len(out_files)): assert np.allclose( np.squeeze(nib.load(fspath(out_files[i])).get_fdata(dtype="float32")), np.squeeze(nib.load(fspath(ref_files[i])).get_fdata(dtype="float32")), rtol=3e-2, equal_nan=True, )
def fspath(self): return os.fspath(self)
def __init__(self, symlink_file_to_convert: os.PathLike, **kwargs) -> None: super().__init__(**kwargs) self.symlink_file_to_convert = os.fspath(symlink_file_to_convert)
def run_PETVolume( input_dir: Path, output_dir: Path, ref_dir: Path, working_dir: Path ) -> None: import shutil from clinica.pipelines.pet_volume.pet_volume_pipeline import PETVolume # Arrange shutil.copytree(input_dir / "caps", output_dir / "caps", copy_function=shutil.copy) parameters = { "group_label": "UnitTest", "acq_label": "fdg", "suvr_reference_region": "pons", "skip_question": False, } pipeline = PETVolume( bids_directory=fspath(input_dir / "bids"), caps_directory=fspath(output_dir / "caps"), tsv_file=fspath(input_dir / "subjects.tsv"), base_dir=fspath(working_dir), parameters=parameters, ) # Acte pipeline.build() pipeline.run(plugin="MultiProc", plugin_args={"n_procs": 4}, bypass_check=True) # Assert subjects = [ "sub-ADNI011S4105", "sub-ADNI023S4020", "sub-ADNI035S4082", "sub-ADNI128S4832", ] out_files = [ fspath( output_dir / "caps" / "subjects" / sub / "ses-M00/pet/preprocessing/group-UnitTest" / ( sub + "_ses-M00_task-rest_acq-fdg_pet_space-Ixi549Space_suvr-pons_mask-brain_fwhm-8mm_pet.nii.gz" ) ) for sub in subjects ] ref_files = [ fspath( ref_dir / ( sub + "_ses-M00_task-rest_acq-fdg_pet_space-Ixi549Space_suvr-pons_mask-brain_fwhm-8mm_pet.nii.gz" ) ) for sub in subjects ] for i in range(len(out_files)): assert likeliness_measure( out_files[i], ref_files[i], (1e-2, 0.25), (1e-1, 0.001) )