Beispiel #1
0
def test_download_bad_url():
    """
    Check that we error when the url is bad

    CommandLine:
        python -m ubelt.tests.test_download test_download_bad_url --verbose
    """
    url = 'http://a-very-incorrect-url'

    if not ub.argflag('--network'):
        pytest.skip('not running network tests')

    dpath = ub.ensure_app_cache_dir('ubelt', 'tests')
    fname = basename(url)
    fpath = join(dpath, fname)

    ub.delete(fpath)
    assert not exists(fpath)

    # from ubelt.util_download import URLError
    import six
    if six.PY2:  # nocover
        from urllib2 import URLError  # NOQA
    else:
        from urllib.error import URLError  # NOQA
    with pytest.raises(URLError):
        ub.download(url, fpath=fpath, verbose=1)
Beispiel #2
0
    def make_lowres_scene(task, scene):
        scene_gtfull_dpath = task.datasubdir('gt' + 'full', scene)
        scene_imfull_dpath = task.datasubdir('im' + 'full', scene)
        gt_fpaths = sorted(glob.glob(join(scene_gtfull_dpath, '*.png')))
        im_fpaths = sorted(glob.glob(join(scene_imfull_dpath, '*.png')))

        # Define the output path for this preprocessing mode
        mode = 'lowres'
        scene_gtout_dpath = task.datasubdir('gt' + mode, scene)
        scene_imout_dpath = task.datasubdir('im' + mode, scene)

        # Start fresh. Remove existing files
        ub.delete(scene_gtout_dpath, verbose=False)
        ub.delete(scene_imout_dpath, verbose=False)
        ub.ensuredir(scene_gtout_dpath)
        ub.ensuredir(scene_imout_dpath)

        target_dsize = tuple(task.input_shape[::-1])
        for impath, gtpath in zip(im_fpaths, gt_fpaths):
            im = cv2.imread(impath, flags=cv2.IMREAD_UNCHANGED)
            gt = cv2.imread(gtpath, flags=cv2.IMREAD_UNCHANGED)
            im_lowres = cv2.resize(im,
                                   target_dsize,
                                   interpolation=cv2.INTER_LANCZOS4)
            gt_lowres = cv2.resize(gt,
                                   target_dsize,
                                   interpolation=cv2.INTER_NEAREST)

            fname = basename(impath)
            cv2.imwrite(join(scene_imout_dpath, fname), im_lowres)
            cv2.imwrite(join(scene_gtout_dpath, fname), gt_lowres)
        return scene_imout_dpath, scene_gtout_dpath
Beispiel #3
0
def _convert_to_cog_worker(src_fpath, dst_fpath, cog_config):
    """ worker function """
    if not exists(dst_fpath):
        _cli_convert_cloud_optimized_geotiff(src_fpath, dst_fpath,
                                             **cog_config)

    success = False
    max_tries = 3
    for try_num in range(max_tries):
        info = _validate_cog_worker(dst_fpath=dst_fpath, orig_fpath=src_fpath)
        if info['status'] == 'pass':
            success = True
            break
        else:
            print('ATTEMPT TO RECOVER FROM ERROR info = {!r}'.format(info))
            print('src_fpath = {!r}'.format(src_fpath))
            print('dst_fpath = {!r}'.format(dst_fpath))
            ub.delete(dst_fpath)
            _cli_convert_cloud_optimized_geotiff(src_fpath, dst_fpath,
                                                 **cog_config)

    if not success:
        raise Exception(
            'ERROR CONVERTING TO COG: src_fpath={}, dst_fpath={}'.format(
                src_fpath, dst_fpath))
    return dst_fpath
Beispiel #4
0
def stage_self(ROOT, staging_dpath):
    import shutil
    # stage the important files in this repo
    dist_paths = [
        'pyhesaff', 'src', 'CMakeLists.txt', 'setup.py', 'run_doctests.sh',
        'CMake', 'run_tests.py'
    ]
    from os.path import isfile, exists

    def copy3(src, dst):
        if exists(dst) and isfile(dst):
            os.unlink(dst)
        shutil.copy2(src, dst)

    mirror_dpath = ub.ensuredir((staging_dpath, 'hesaff'))

    copy_function = shutil.copy2
    copy_function = copy3
    print('======')
    for pname in dist_paths:
        src = join(ROOT, pname)
        dst = join(mirror_dpath, pname)
        print('src={!r}, dst={!r}'.format(src, dst))

        if os.path.isdir(pname):
            ub.delete(dst)
            shutil.copytree(src, dst, copy_function=copy_function)
        else:
            copy_function(src, dst)
    print('======')
Beispiel #5
0
def test_broken_link():
    """
    CommandLine:
        python -m ubelt.tests.test_links test_broken_link
    """
    dpath = ub.ensure_app_cache_dir('ubelt', 'test_broken_link')

    ub.delete(dpath, verbose=2)
    ub.ensuredir(dpath, verbose=2)
    util_links._dirstats(dpath)

    broken_fpath = join(dpath, 'broken_fpath.txt')
    broken_flink = join(dpath, 'broken_flink.txt')

    ub.touch(broken_fpath, verbose=2)
    util_links._dirstats(dpath)
    ub.symlink(broken_fpath, broken_flink, verbose=2)

    util_links._dirstats(dpath)
    ub.delete(broken_fpath, verbose=2)

    util_links._dirstats(dpath)

    # make sure I am sane that this is the correct check.
    can_symlink = util_links._can_symlink()
    print('can_symlink = {!r}'.format(can_symlink))
    if can_symlink:
        # normal behavior
        assert islink(broken_flink)
        assert not exists(broken_flink)
    else:
        # on windows hard links are essentially the same file.
        # there is no trace that it was actually a link.
        assert exists(broken_flink)
Beispiel #6
0
def test_download_bad_url():
    """
    Check that we error when the url is bad

    Notes:
        For some reason this can take a long time to realize there is no URL,
        even if the timeout is specified and fairly low.

    CommandLine:
        python tests/test_download.py test_download_bad_url --verbose
    """
    url = 'http://www.a-very-incorrect-url.gov/does_not_exist.txt'
    # if not ub.argflag('--network'):
    #     pytest.skip('not running network tests')

    # Ensure the opener exist
    import sys
    if sys.version_info[0] == 2:  # nocover
        # import urllib2 as urllib_x
        from urllib2 import URLError  # NOQA
    else:
        # import urllib.request as urllib_x
        from urllib.error import URLError  # NOQA
    # if urllib_x._opener is None:
    #     urllib_x.install_opener(urllib_x.build_opener())

    dpath = ub.ensure_app_cache_dir('ubelt', 'tests')
    fname = basename(url)
    fpath = join(dpath, fname)

    ub.delete(fpath)
    assert not exists(fpath)

    with pytest.raises(URLError):
        ub.download(url, fpath=fpath, verbose=1, timeout=1.0)
Beispiel #7
0
def test_cant_overwrite_file_with_symlink():
    if ub.WIN32:
        # Can't distinguish this case on windows
        pytest.skip()

    dpath = ub.ensure_app_cache_dir('ubelt',
                                    'test_cant_overwrite_file_with_symlink')
    ub.delete(dpath, verbose=2)
    ub.ensuredir(dpath, verbose=2)

    happy_fpath = join(dpath, 'happy_fpath.txt')
    happy_flink = join(dpath, 'happy_flink.txt')

    for verbose in [2, 1, 0]:
        print('=======')
        print('verbose = {!r}'.format(verbose))
        ub.delete(dpath, verbose=verbose)
        ub.ensuredir(dpath, verbose=verbose)
        ub.touch(happy_fpath, verbose=verbose)
        ub.touch(happy_flink)  # create a file where a link should be

        util_links._dirstats(dpath)
        with pytest.raises(FileExistsError):  # file exists error
            ub.symlink(happy_fpath,
                       happy_flink,
                       overwrite=False,
                       verbose=verbose)

        with pytest.raises(FileExistsError):  # file exists error
            ub.symlink(happy_fpath,
                       happy_flink,
                       overwrite=True,
                       verbose=verbose)
Beispiel #8
0
def clean():
    """
    __file__ = ub.truepath('~/code/clab/setup.py')
    """
    import ubelt as ub
    import os

    modname = 'clab'
    repodir = dirname(__file__)
    # pkgdir = join(repodir, modname)

    toremove = []
    for root, dnames, fnames in os.walk(repodir):

        if os.path.basename(root) == modname + '.egg-info':
            toremove.append(root)
            del dnames[:]

        if os.path.basename(root) == '__pycache__':
            toremove.append(root)
            del dnames[:]

        if os.path.basename(root) == '_ext':
            # Remove torch extensions
            toremove.append(root)
            del dnames[:]

        if os.path.basename(root) == 'build':
            # Remove python c extensions
            if len(dnames) == 1 and dnames[0].startswith('temp.'):
                toremove.append(root)
                del dnames[:]

        # Remove simple pyx inplace extensions
        for fname in fnames:
            if fname.endswith('.so') or fname.endswith('.c'):
                if fname.split('.')[0] + '.pyx' in fnames:
                    toremove.append(join(root, fname))

    def enqueue(d):
        if exists(d) and d not in toremove:
            toremove.append(d)

    enqueue(join(repodir, 'clab/models/yolo2/utils/cython_yolo.c') )
    enqueue(join(repodir, 'clab/models/yolo2/utils/cython_bbox.c') )
    enqueue(join(repodir, 'clab/models/yolo2/utils/nms/cpu_nms.c') )
    enqueue(join(repodir, 'clab/models/yolo2/utils/nms/cpu_nms.c') )
    enqueue(join(repodir, 'clab/models/yolo2/utils/nms/cpu_nms.cpp') )

    enqueue(join(repodir, 'clab/models/yolo2/layers/roi_pooling/_ext') )
    enqueue(join(repodir, 'clab/models/yolo2/layers/reorg/_ext') )
    import glob

    for d in glob.glob(join(repodir, 'clab/models/yolo2/utils/nms/*_nms.*so')):
        enqueue(d)

    for dpath in toremove:
        # print('Removing dpath = {!r}'.format(dpath))
        ub.delete(dpath, verbose=1)
Beispiel #9
0
def main():
    try:
        script = __file__
    except NameError:
        raise
        # for Ipython hacking
        script = ub.expandpath(
            '~/code/ndsampler/dev/devcheck_python23_isect_index_cache.py')

    # py2 = ub.find_exe('python2')
    # py3 = ub.find_exe('python3')
    # ub.cmd([py2, script, 'load_regions'], shell=True)
    # ub.cmd([py3, script, 'save_regions'], shell=True)

    # Register scripts for activating python 2/3 virtual envs that have
    # ndsampler installed

    import getpass
    username = getpass.getuser()

    if username in ['joncrall', 'jon.crall']:
        # Hack for Jon's computer
        activate_cmds = {
            'python2': 'we py2.7',
            'python3': 'we venv3.6',
        }
    else:
        assert False, 'need to customize activation scripts for your machine'
        activate_cmds = {
            'python2': 'source ~/venv27/bin/activate',
            'python3': 'conda activate py36',
        }

    def run(py):
        bash_cmd = ' && '.join([
            'source $HOME/.bashrc',
            activate_cmds[py],
            'python {} access_cache'.format(script),
        ])
        sh_cmd = 'bash -c "{}"'.format(bash_cmd)
        info = ub.cmd(sh_cmd, shell=True, verbose=3)
        return info

    workdir = ub.ensure_app_cache_dir('ndsampler', 'tests', '23_regions')

    # Save in python3, load in python2
    print('\n\n--- SAVE Python3, LOAD Python2 ---')
    ub.delete(workdir, verbose=1)
    info = run('python3')  # NOQA
    assert info['ret'] == 0
    info = run('python2')  # NOQA
    assert info['ret'] == 0

    print('\n\n--- SAVE Python2, LOAD Python3 ---')
    ub.delete(workdir, verbose=1)  # Clear the cache
    info = run('python2')  # NOQA
    assert info['ret'] == 0
    info = run('python3')  # NOQA
    assert info['ret'] == 0
Beispiel #10
0
def test_npy_backend():
    dset = ndsampler.CocoSampler.demo('shapes11', backend='npy')
    assert dset.frames._backend['type'] == 'npy'
    assert dset.frames.cache_dpath is not None
    ub.delete(dset.frames.cache_dpath)
    ub.ensuredir(dset.frames.cache_dpath)
    raw_img = dset.load_image(1)
    assert isinstance(raw_img, np.memmap)
Beispiel #11
0
 def _startfresh(task):
     scene_im_dpaths, scene_gt_dpaths = task._all_scene_dpaths()
     keys = task._preprocessing_keys()
     for scene, key in it.product(task.scene_ids, keys):
         im_dpath = task.datasubdir('im' + key, scene)
         gt_dpath = task.datasubdir('gt' + key, scene)
         ub.delete(gt_dpath)
         ub.delete(im_dpath)
Beispiel #12
0
def test_grabdata_delete_hash_stamp():
    import ubelt as ub
    fname = 'foo3.bar'
    url = _demo_url(128 * 12)
    prefix1 = '43f92597d7eb08b57c88b636'
    fpath = ub.grabdata(url, fname=fname, hash_prefix=prefix1)
    stamp_fpath = fpath + '.sha512.hash'
    ub.delete(stamp_fpath)
    fpath = ub.grabdata(url, fname=fname, hash_prefix=prefix1)
Beispiel #13
0
def _devcheck_manage_monitor(workdir, dry=True):

    all_sessions = collect_sessions(workdir)

    # Get all the images in the monitor directories
    # (this is a convention and not something netharn does by default)

    all_files = []
    # factor = 100
    max_keep = 300

    def _choose_action(file_infos):
        import kwarray
        file_infos = kwarray.shuffle(file_infos, rng=0)
        n_keep = max_keep
        # n_keep = (len(file_infos) // factor) + 1
        # n_keep = min(max_keep, n_keep)

        for info in file_infos[:n_keep]:
            info['action'] = 'keep'
        for info in file_infos[n_keep:]:
            info['action'] = 'delete'

    for session in ub.ProgIter(all_sessions, desc='checking monitor files'):
        dpaths = [
            join(session.dpath, 'monitor', 'train', 'batch'),
            join(session.dpath, 'monitor', 'vali', 'batch'),
            join(session.dpath, 'monitor', 'train'),
            join(session.dpath, 'monitor', 'vali'),
        ]
        exts = ['*.jpg', '*.png']
        for dpath in dpaths:
            for ext in exts:
                fpaths = list(glob.glob(join(dpath, ext)))
                file_infos = [{
                    'size': os.stat(p).st_size,
                    'fpath': p
                } for p in fpaths]
                _choose_action(file_infos)
                all_files.extend(file_infos)

    grouped_actions = ub.group_items(all_files, lambda x: x['action'])

    for key, group in grouped_actions.items():
        size = byte_str(sum([s['size'] for s in group]))
        print('{:>4} images:  {:>4}, size={}'.format(key.capitalize(),
                                                     len(group), size))

    if dry:
        print('Dry run')
    else:
        delete = grouped_actions.get('delete', [])
        delete_fpaths = [item['fpath'] for item in delete]
        for p in ub.ProgIter(delete_fpaths, desc='deleting'):
            ub.delete(p)
Beispiel #14
0
def clean_repo(repodir, modname, rel_paths=[]):
    """
    repodir = ub.truepath('~/code/netharn/')
    modname = 'netharn'
    rel_paths = [
        'netharn/util/nms/cpu_nms.c',
        'netharn/util/nms/cpu_nms.c',
        'netharn/util/nms/cpu_nms.cpp',
        'netharn/util/nms/cython_boxes.c',
        'netharn/util/nms/cython_boxes.html',
    ]
    """
    toremove = []
    for root, dnames, fnames in os.walk(repodir):

        if os.path.basename(root) == modname + '.egg-info':
            toremove.append(root)
            del dnames[:]

        if os.path.basename(root) == '__pycache__':
            toremove.append(root)
            del dnames[:]

        if os.path.basename(root) == '_ext':
            # Remove torch extensions
            toremove.append(root)
            del dnames[:]

        if os.path.basename(root) == 'build':
            # Remove python c extensions
            if len(dnames) == 1 and dnames[0].startswith('temp.'):
                toremove.append(root)
                del dnames[:]

        # Remove simple pyx inplace extensions
        for fname in fnames:
            if fname.endswith('.so') or fname.endswith('.c'):
                if fname.split('.')[0] + '.pyx' in fnames:
                    toremove.append(join(root, fname))

    def enqueue(d):
        if exists(d) and d not in toremove:
            toremove.append(d)

    abs_paths = [
        join(repodir, p) for pat in rel_paths
        for p in glob.glob(pat, recursive=True)
    ]
    for abs_path in abs_paths:
        enqueue(abs_path)

    for dpath in toremove:
        # print('Removing dpath = {!r}'.format(dpath))
        ub.delete(dpath, verbose=1)
Beispiel #15
0
    def make_part_scene(task, scene, scale=1):
        """
        Slices the full scene into smaller parts that fit into the network but
        are at the original resolution (or higher).

        >>> scene = '0001'
        >>> scale = 1
        """
        if task.part_overlap < 0 or task.part_overlap >= 1:
            raise ValueError(('part overlap was {}, but it must be '
                              'in the range [0, 1)').format(task.part_overlap))

        input_shape = task.input_shape
        overlap = task.part_overlap
        keepbound = task.part_keepbound

        scene_gtfull_dpath = task.datasubdir('gt' + 'full', scene)
        scene_imfull_dpath = task.datasubdir('im' + 'full', scene)
        gt_fpaths = sorted(glob.glob(join(scene_gtfull_dpath, '*.png')))
        im_fpaths = sorted(glob.glob(join(scene_imfull_dpath, '*.png')))

        # Define the output path for this preprocessing mode
        mode = 'part-scale{}'.format(scale)
        scene_gtout_dpath = task.datasubdir('gt' + mode, scene)
        scene_imout_dpath = task.datasubdir('im' + mode, scene)
        # Start fresh. Remove existing files
        ub.delete(scene_gtout_dpath, verbose=False)
        ub.delete(scene_imout_dpath, verbose=False)
        ub.ensuredir(scene_gtout_dpath)
        ub.ensuredir(scene_imout_dpath)

        for impath, gtpath in zip(im_fpaths, gt_fpaths):
            im = cv2.imread(impath, flags=cv2.IMREAD_UNCHANGED)
            gt = cv2.imread(gtpath, flags=cv2.IMREAD_UNCHANGED)

            if scale != 1.0:
                im = imutil.imscale(im, scale, cv2.INTER_LANCZOS4)[0]
                gt = imutil.imscale(gt, scale, cv2.INTER_NEAREST)[0]

            assert gt.max() <= task.labels.max(), (
                'be careful not to change gt labels')

            fname_we = splitext(basename(impath))[0]
            sl_gen = imutil.image_slices(im.shape[0:2], input_shape, overlap,
                                         keepbound)
            for idx, rc_slice in enumerate(sl_gen):
                # encode the slice in the image name?
                fname = '{}_part{:0=4d}.png'.format(fname_we, idx)
                im_part = im[rc_slice]
                gt_part = gt[rc_slice]

                cv2.imwrite(join(scene_imout_dpath, fname), im_part)
                cv2.imwrite(join(scene_gtout_dpath, fname), gt_part)
        return scene_imout_dpath, scene_gtout_dpath
Beispiel #16
0
def test_static_find_locals():
    """
    python ~/code/mkinit/tests/test_with_dummy.py test_static_find_locals
    """
    import mkinit
    cache_dpath = ub.ensure_app_cache_dir('mkinit/tests')
    paths = make_dummy_package(cache_dpath)
    ub.delete(paths['root_init'])
    modpath = paths['root']
    imports = list(mkinit.static_mkinit._find_local_submodules(modpath))
    print('imports = {!r}'.format(imports))
Beispiel #17
0
def ensuredir(dpath, mode=0o1777, verbose=0, recreate=False):
    r"""
    Ensures that directory will exist. Creates new dir with sticky bits by
    default

    Args:
        dpath (str | PathLike | Tuple[str | PathLike]): dir to ensure. Can also
            be a tuple to send to join
        mode (int, default=0o1777): octal mode of directory
        verbose (int, default=0): verbosity
        recreate (bool, default=False): if True removes the directory and
            all of its contents and creates a fresh new directory.
            USE CAREFULLY.

    Returns:
        str: path - the ensured directory

    SeeAlso:
        :func:`ubelt.Path.ensuredir`

    Note:
        This function is not thread-safe in Python2

    Example:
        >>> from ubelt.util_path import *  # NOQA
        >>> import ubelt as ub
        >>> cache_dpath = ub.ensure_app_cache_dir('ubelt')
        >>> dpath = join(cache_dpath, 'ensuredir')
        >>> if exists(dpath):
        ...     os.rmdir(dpath)
        >>> assert not exists(dpath)
        >>> ub.ensuredir(dpath)
        >>> assert exists(dpath)
        >>> os.rmdir(dpath)
    """
    if isinstance(dpath, (list, tuple)):
        dpath = join(*dpath)

    if recreate:
        import ubelt as ub
        ub.delete(dpath, verbose=verbose)

    if not exists(dpath):
        if verbose:
            print('Ensuring directory (creating {!r})'.format(dpath))
        if PY2:  # nocover
            os.makedirs(normpath(dpath), mode=mode)
        else:
            os.makedirs(normpath(dpath), mode=mode, exist_ok=True)
    else:
        if verbose:
            print('Ensuring directory (existing {!r})'.format(dpath))
    return dpath
Beispiel #18
0
def test_static_import_without_init():
    """
    python ~/code/mkinit/tests/test_with_dummy.py test_static_import_without_init
    """
    import mkinit
    cache_dpath = ub.ensure_app_cache_dir('mkinit/tests')
    paths = make_dummy_package(cache_dpath)
    ub.delete(paths['root_init'])

    modpath = paths['root']
    text = mkinit.static_init(modpath)
    check_dummy_root_init(text)
Beispiel #19
0
    def make_augment_scene(task, mode, scene, rng=None):
        """
        Augments data in a scene of a specific "mode"

        mode = 'part-scale1'
        scene = '0000'
        rng = 'determ'

        gtdir = task.datasubdir('gtpart', scene))
        imdir = task.datasubdir('impart', scene))
        """
        assert task.enable_augment

        if rng == 'determ':
            # Make a determenistic seed based on the scene and mode
            seed = int(hashutil.hash_data([scene, mode], alphabet='hex'), 16)
            seed = seed % (2**32 - 1)
            rng = np.random.RandomState(seed)

        auger = augment.SSegAugmentor(rng=rng, ignore_label=task.ignore_label)
        auger.params = task.aug_params

        # rng = np.random.RandomState(0)
        imdir = task.datasubdir('im' + mode, scene)
        gtdir = task.datasubdir('gt' + mode, scene)
        im_fpaths = sorted(glob.glob(join(imdir, '*.png')))
        gt_fpaths = sorted(glob.glob(join(gtdir, '*.png')))

        # Define the output path for the augmentation of this mode
        key = mode + '-aug'
        scene_imout_dpath = task.datasubdir('im' + key, scene)
        scene_gtout_dpath = task.datasubdir('gt' + key, scene)

        # Start fresh. Remove existing files
        ub.delete(scene_gtout_dpath, verbose=False)
        ub.delete(scene_imout_dpath, verbose=False)
        ub.ensuredir(scene_gtout_dpath)
        ub.ensuredir(scene_imout_dpath)

        for impath, gtpath in ub.ProgIter(
                list(zip(im_fpaths, gt_fpaths)),
                label='   * augment mode={}'.format(mode)):
            fname_we = splitext(basename(impath))[0]
            im = cv2.imread(impath, flags=cv2.IMREAD_UNCHANGED)
            gt = cv2.imread(gtpath, flags=cv2.IMREAD_UNCHANGED)
            aug_gen = auger.augment(im, gt)
            for augx, aug_data in enumerate(aug_gen):
                (im_aug, gt_aug) = aug_data[0:2]
                fname = '{}_aug{:0=4d}.png'.format(fname_we, augx)
                cv2.imwrite(join(scene_imout_dpath, fname), im_aug)
                cv2.imwrite(join(scene_gtout_dpath, fname), gt_aug)
        return scene_imout_dpath, scene_gtout_dpath
Beispiel #20
0
def make_simple_dummy_package():
    """
    Creates a dummy package structure with or without __init__ files

    ANY EXISTING FILES ARE DELETED
    """
    # Fresh start
    dpath = ub.ensure_app_cache_dir("mkinit/test/simple_demo/")
    ub.delete(dpath)
    ub.ensuredir(dpath)
    rel_paths = {
        "root": "mkinit_demo_pkg",
        "root_init": "mkinit_demo_pkg/__init__.py",
        "submod": "mkinit_demo_pkg/submod.py",
        "subpkg": "mkinit_demo_pkg/subpkg",
        "subpkg_init": "mkinit_demo_pkg/subpkg/__init__.py",
        "nested": "mkinit_demo_pkg/subpkg/nested.py",
    }
    paths = {key: join(dpath, path) for key, path in rel_paths.items()}

    for key, path in paths.items():
        if not path.endswith(".py"):
            ub.ensuredir(path)

    for key, path in paths.items():
        if path.endswith(".py"):
            ub.touch(path)

    with open(paths["submod"], "w") as file:
        file.write(
            ub.codeblock(
                """
            print('SUBMOD SIDE EFFECT')

            def submod_func():
                print('This is a submod func in {}'.format(__file__))
            """
            )
        )

    with open(paths["nested"], "w") as file:
        file.write(
            ub.codeblock(
                """
            print('NESTED SIDE EFFECT')

            def nested_func():
                print('This is a nested func in {}'.format(__file__))
            """
            )
        )
    return paths
Beispiel #21
0
 def _cleanup(self):
     # print('self._cleanup = {!r}'.format(self._cleanup))
     if self._handle is not None:
         if not getattr(self, 'closed', True):
             closemethod = getattr(self, 'close', None)
             if closemethod is not None:  # nocover
                 closemethod()
             closemethod = None
     self._handle = None
     if self._temp_dpath and exists(self._temp_dpath):
         # os.unlink(self._temp_dpath)
         import ubelt as ub
         ub.delete(self._temp_dpath)
Beispiel #22
0
def test_cog_backend():
    try:
        import gdal  # NOQA
    except ImportError:
        import pytest
        pytest.skip('cog requires gdal')

    dset = ndsampler.CocoSampler.demo('shapes13', backend='cog')
    assert dset.frames._backend['type'] == 'cog'
    assert dset.frames.cache_dpath is not None
    ub.delete(dset.frames.cache_dpath)
    ub.ensuredir(dset.frames.cache_dpath)
    raw_img = dset.load_image(5)
    assert not isinstance(raw_img, np.ndarray)
Beispiel #23
0
def test_rel_file_link():
    dpath = ub.ensure_app_cache_dir('ubelt', 'test_rel_file_link')
    ub.delete(dpath, verbose=2)
    ub.ensuredir(dpath, verbose=2)

    real_fpath = join(ub.ensuredir((dpath, 'dir1')), 'real')
    link_fpath = join(ub.ensuredir((dpath, 'dir2')), 'link')
    ub.touch(real_fpath)

    orig = os.getcwd()
    try:
        os.chdir(dpath)
        real_path = relpath(real_fpath, dpath)
        link_path = relpath(link_fpath, dpath)
        link = ub.symlink(real_path, link_path)
        import sys
        if sys.platform.startswith('win32') and isfile(link):
            # Note: if windows hard links the file there is no way we can
            # tell that it was a symlink. Just verify it exists.
            from ubelt import _win32_links
            assert _win32_links._win32_is_hardlinked(real_fpath, link_fpath)
        else:
            pointed = ub.util_links._readlink(link)
            resolved = os.path.realpath(
                ub.expandpath(join(dirname(link), pointed)))
            assert os.path.realpath(ub.expandpath(real_fpath)) == resolved
    except Exception:
        util_links._dirstats(dpath)
        util_links._dirstats(join(dpath, 'dir1'))
        util_links._dirstats(join(dpath, 'dir2'))
        print('TEST FAILED: test_rel_link')
        print('real_fpath = {!r}'.format(real_fpath))
        print('link_fpath = {!r}'.format(link_fpath))
        print('real_path = {!r}'.format(real_path))
        print('link_path = {!r}'.format(link_path))
        try:
            if 'link' in vars():
                print('link = {!r}'.format(link))
            if 'pointed' in vars():
                print('pointed = {!r}'.format(pointed))
            if 'resolved' in vars():
                print('resolved = {!r}'.format(resolved))
        except Exception:
            print('...rest of the names are not available')
        raise
    finally:
        util_links._dirstats(dpath)
        util_links._dirstats(join(dpath, 'dir1'))
        util_links._dirstats(join(dpath, 'dir2'))
        os.chdir(orig)
Beispiel #24
0
def test_download_no_fpath():
    url = 'http://i.imgur.com/rqwaDag.png'

    dpath = ub.ensure_app_cache_dir('ubelt')
    fname = basename(url)
    fpath = join(dpath, fname)

    ub.delete(fpath)
    assert not exists(fpath)

    got_fpath = ub.download(url)

    assert got_fpath == fpath
    assert exists(fpath)
Beispiel #25
0
def test_download_bad_url():
    """
    Check where the url is downloaded to when fpath is not specified.
    """
    url = 'http://averyincorrecturl'

    dpath = ub.ensure_app_cache_dir('ubelt', 'tests')
    fname = basename(url)
    fpath = join(dpath, fname)

    ub.delete(fpath)
    assert not exists(fpath)

    with pytest.raises(Exception):
        ub.download(url, fpath=fpath)
Beispiel #26
0
def test_download_chunksize():
    # url = 'https://www.dropbox.com/s/jl506apezj42zjz/ibeis-win32-setup-ymd_hm-2015-08-01_16-28.exe?dl=1'
    url = 'http://i.imgur.com/rqwaDag.png'

    dpath = ub.ensure_app_cache_dir('ubelt')
    fname = basename(url)
    fpath = join(dpath, fname)

    ub.delete(fpath)
    assert not exists(fpath)

    got_fpath = ub.download(url, chunksize=2)

    assert got_fpath == fpath
    assert exists(fpath)
Beispiel #27
0
def clean_tensorboard_protobufs(dpath):
    """
    Removes event logs that only contain conflicting information

    dpath = '/home/local/KHQ/jon.crall/data/work_phase2/train/unet2/'

    CommandLine:
        python -m clab.util.misc clean_tensorboard_protobufs

    Example:
        >>> dpath = '.'
        >>> clean_tensorboard_protobufs(dpath)
    """

    # from tensorflow.python.summary import event_accumulator
    from tensorboard.backend.event_processing import event_accumulator
    import glob
    from os.path import join
    from collections import defaultdict
    import ubelt as ub
    import tqdm

    # Clean out iterno overrides
    event_paths = sorted(glob.glob(join(dpath, 'events.out.tfevents*')))

    bad_paths = set()
    good_paths = set()
    low_steps = defaultdict(lambda: float('inf'))
    for p in tqdm.tqdm(list(reversed(event_paths)), desc='cleaning'):
        ea = event_accumulator.EventAccumulator(p)
        ea.Reload()
        for key in ea.scalars.Keys():
            events = ea.scalars.Items(key)
            for e in reversed(sorted(events, key=lambda e: e.wall_time)):
                if e.step < low_steps[key]:
                    low_steps[key] = e.step
                    good_paths.add(p)
                else:
                    # Can we individually remove scalars?
                    bad_paths.add(p)
                    # print('skip e = {}, {!r}'.format(key, e))

    # these paths only have conflicting old information. Just kill them
    onlybad_paths = bad_paths - good_paths
    print('onlybad_paths = {!r}'.format(onlybad_paths))
    for p in onlybad_paths:
        ub.delete(p)
Beispiel #28
0
def test_download_no_fpath():
    # url = 'http://i.imgur.com/rqwaDag.png'
    # if not ub.argflag('--network'):
    #     pytest.skip('not running network tests')
    url = _demo_url()

    dpath = ub.ensure_app_cache_dir('ubelt')
    fname = basename(url)
    fpath = join(dpath, fname)

    ub.delete(fpath)
    assert not exists(fpath)

    got_fpath = ub.download(url)

    assert got_fpath == fpath
    assert exists(fpath)
Beispiel #29
0
def test_download_with_fpath():
    url = 'http://i.imgur.com/rqwaDag.png'

    dpath = ub.ensure_app_cache_dir('ubelt', 'tests')
    fname = basename(url)
    fpath = join(dpath, fname)

    ub.delete(fpath)
    assert not exists(fpath)

    got_fpath = ub.download(url, fpath=fpath)
    assert got_fpath == fpath
    assert exists(fpath)

    with open(got_fpath, 'rb') as file:
        data = file.read()
    assert len(data) > 1200, 'should have downloaded some bytes'
Beispiel #30
0
    def cleanup_snapshots(harn):
        """
        remove old snapshots

        TODO:
            [ ] - keep the top epochs for every metric
        """
        snapshots = harn.prev_snapshots()
        existing_epochs = sorted([
            int(parse.parse('{}_epoch_{num:d}.pt', path).named['num'])
            for path in snapshots
        ])

        num_keep = harn.config['num_keep']

        def _epochs_to_remove(existing_epochs, num_keep):
            """
            doctest:
                >>> import netharn as nh
                >>> harn = FitHarn({})
                >>> rng = np.random.RandomState(0)
                >>> harn.monitor = nh.Monitor(minimize=['loss'], maximize=['miou'])
                >>> for epoch in range(200):
                >>>     harn.monitor.update(epoch, {'loss': rng.rand(),
                >>>                                 'miou': rng.rand()})
                >>> existing_epochs = list(range(0, 200, 4))
                >>> num_keep = 10
            """
            keep = set()

            recent = existing_epochs[-num_keep:]
            keep.update(recent)

            if harn.monitor:
                for best_epochs in harn.monitor.best_epochs().values():
                    best = ub.oset(best_epochs).intersection(existing_epochs)
                keep.update(best[:num_keep])

            to_remove = set(existing_epochs) - keep
            return to_remove

        epoch_to_fpath = dict(zip(existing_epochs, snapshots))
        to_remove = _epochs_to_remove(existing_epochs, num_keep)
        for fpath in ub.take(epoch_to_fpath, to_remove):
            ub.delete(fpath)
Beispiel #31
0
def demo(config=None):
    """
    Runs the algorithm end-to-end.
    """
    # dataset = 'test'
    # dataset = 'haul83'

    if config is None:
        import argparse
        parser = argparse.ArgumentParser(description='Standalone camtrawl demo')

        parser.add_argument('--cal', help='path to matlab or numpy stereo calibration file', default='cal.npz')
        parser.add_argument('--left', help='path to directory containing left images', default='left')
        parser.add_argument('--right', help='path to directory containing right images', default='right')
        parser.add_argument('--out', help='output directory', default='./out')
        parser.add_argument('-f', '--overwrite', action='store_true', help='will delete any existing output')
        parser.add_argument('--draw', action='store_true', help='draw visualization of algorithm steps')

        parser.add_argument('--dataset', default=None,
                            help='Developer convenience assumes you have demo '
                                 ' data downloaded and available. If you dont '
                                 ' specify the other args.')

        args = parser.parse_args()
        config = args.__dict__.copy()
        config = FrozenKeyDict(config)

    if config['dataset'] is not None:
        img_path1, img_path2, cal_fpath = demodata_input(dataset=config['dataset'])
        config['left'] = img_path1
        config['right'] = img_path2
        config['cal'] = cal_fpath

    img_path1, img_path2, cal_fpath = ub.take(config, [
        'left', 'right', 'cal'])
    out_dpath = config['out']
    logging.info('Demo Config = {!r}'.format(config))

    ub.ensuredir(out_dpath)

    # ----
    # Choose parameter configurations
    # ----

    # Use GMM based model
    gmm_params = {
    }
    triangulate_params = {
    }

    DRAWING = config['draw']

    # ----
    # Initialize algorithms
    # ----

    detector1 = ctalgo.GMMForegroundObjectDetector(**gmm_params)
    detector2 = ctalgo.GMMForegroundObjectDetector(**gmm_params)
    triangulator = ctalgo.FishStereoMeasurments(**triangulate_params)

    try:
        import pyfiglet
        print(pyfiglet.figlet_format('CAMTRAWL', font='cybermedium'))
    except ImportError:
        logging.debug('pyfiglet is not installed')
        print('========')
        print('CAMTRAWL')
        print('========')
    logging.info('Detector1 Config: ' + ub.repr2(detector1.config, nl=1))
    logging.info('Detector2 Config: ' + ub.repr2(detector2.config, nl=1))
    logging.info('Triangulate Config: ' + ub.repr2(triangulator.config, nl=1))
    logging.info('DRAWING = {!r}'.format(DRAWING))

    cal = ctalgo.StereoCalibration.from_file(cal_fpath)

    stream = StereoFrameStream(img_path1, img_path2)
    stream.preload()

    # HACK IN A BEGIN FRAME
    if len(stream) > 2200:
        stream.seek(2200)

    # ----
    # Run the algorithm
    # ----

    # n_frames = 2000
    # stream.aligned_frameids = stream.aligned_frameids[:stream.index]

    measure_fpath = join(out_dpath, 'measurements.csv')
    if exists(measure_fpath):
        if config['overwrite']:
            ub.delete(measure_fpath)
        else:
            raise IOError('Measurement path already exists')
    output_file = open(measure_fpath, 'a')

    if DRAWING:
        drawing_dpath = join(out_dpath, 'visual')
        if exists(drawing_dpath):
            if config['overwrite']:
                ub.delete(drawing_dpath)
            else:
                raise IOError('Output path already exists')
        ub.ensuredir(drawing_dpath)

    headers = ['current_frame', 'fishlen', 'range', 'error', 'dz', 'box_pts1',
               'box_pts2']
    output_file.write(','.join(headers) + '\n')
    output_file.flush()

    measurements = []

    logger.info('begin camtrawl iteration')

    import tqdm
    # prog = ub.ProgIter(iter(stream), total=len(stream), desc='camtrawl demo',
    #                    clearline=False, freq=1, adjust=False)
    prog = tqdm.tqdm(iter(stream), total=len(stream), desc='camtrawl demo',
                     leave=True)

    def csv_repr(d):
        if isinstance(d, np.ndarray):
            d = d.tolist()
        s = repr(d)
        return s.replace('\n', '').replace(',', ';').replace(' ', '')

    for frame_num, (frame_id, img1, img2) in enumerate(prog):
        logger.debug('frame_num = {!r}'.format(frame_num))

        detections1 = list(detector1.detect(img1))
        detections2 = list(detector2.detect(img2))
        masks1 = detector1._masks
        masks2 = detector2._masks

        any_detected = len(detections1) > 0 or len(detections2) > 0

        if any_detected:
            assignment, assign_data, cand_errors = triangulator.find_matches(
                cal, detections1, detections2)
            # Append assignments to the measurements
            for data in assign_data:
                data['current_frame'] = int(frame_id)
                measurements.append(data)
                line = ','.join([csv_repr(d) for d in ub.take(data, headers)])
                output_file.write(line + '\n')
                output_file.flush()
        else:
            cand_errors = None
            assignment, assign_data = None, None

        if DRAWING >= 2 or (DRAWING and any_detected):
            DRAWING = 3
            stacked = DrawHelper.draw_stereo_detections(img1, detections1, masks1,
                                                        img2, detections2, masks2,
                                                        assignment, assign_data,
                                                        cand_errors)
            if cv2.__version__.startswith('2'):
                cv2.putText(stacked,
                            text='frame #{}, id={}'.format(frame_num,
                                                           frame_id),
                            org=(10, 50),
                            fontFace=cv2.FONT_HERSHEY_SIMPLEX,
                            fontScale=1, color=(255, 0, 0),
                            thickness=2, lineType=cv2.cv.CV_AA)
            else:
                stacked = cv2.putText(stacked,
                                      text='frame #{}, id={}'.format(frame_num,
                                                                     frame_id),
                                      org=(10, 50),
                                      fontFace=cv2.FONT_HERSHEY_SIMPLEX,
                                      fontScale=1, color=(255, 0, 0),
                                      thickness=2, lineType=cv2.LINE_AA)
            cv2.imwrite(drawing_dpath + '/mask{}_draw.png'.format(frame_id), stacked)
    output_file.close()

    n_total = len(measurements)
    logger.info('n_total = {!r}'.format(n_total))
    if n_total:
        all_errors = np.array([d['error'] for d in measurements])
        all_lengths = np.array([d['fishlen'] for d in measurements])
        logger.info('ave_error = {:.2f} +- {:.2f}'.format(all_errors.mean(), all_errors.std()))
        logger.info('ave_lengths = {:.2f} +- {:.2f} '.format(all_lengths.mean(), all_lengths.std()))
    return measurements