Beispiel #1
0
def _describe_system():
    import platform as pl
    from datalad import get_encoding_info
    from datalad.utils import get_linux_distribution
    try:
        dist = get_linux_distribution()
    except Exception as exc:
        lgr.warning("Failed to get distribution information: %s", exc_str(exc))
        dist = tuple()

    return {
        'type':
        os.name,
        'name':
        pl.system(),
        'release':
        pl.release(),
        'version':
        pl.version(),
        'distribution':
        ' '.join([_t2s(dist),
                  _t2s(pl.mac_ver()),
                  _t2s(pl.win32_ver())]).rstrip(),
        'max_path_length':
        get_max_path_length(getpwd()),
        'encoding':
        get_encoding_info(),
    }
Beispiel #2
0
def _describe_system():
    import platform as pl
    from datalad import get_encoding_info
    from datalad.utils import get_linux_distribution
    try:
        dist = get_linux_distribution()
    except Exception as exc:
        ce = CapturedException(exc)
        lgr.warning("Failed to get distribution information: %s", ce)
        dist = tuple()

    return {
        'type':
        os.name,
        'name':
        pl.system(),
        'release':
        pl.release(),
        'version':
        pl.version(),
        'distribution':
        ' '.join([_t2s(dist),
                  _t2s(pl.mac_ver()),
                  _t2s(pl.win32_ver())]).rstrip(),
        'max_path_length':
        get_max_path_length(getpwd()),
        'encoding':
        get_encoding_info(),
        'filesystem': {
            l: _get_fs_type(l, p)
            for l, p in [('CWD',
                          Path.cwd()), ('TMP', Path(tempfile.gettempdir())
                                        ), ('HOME', Path.home())]
        }
    }
Beispiel #3
0
def _describe_system():
    import platform as pl
    from datalad import get_encoding_info

    if hasattr(pl, 'dist'):
        dist = pl.dist()
    else:
        # Python 3.8 removed .dist but recommended "distro" is slow, so we
        # try it only if needed
        try:
            import distro
            dist = distro.linux_distribution(full_distribution_name=False)
        except ImportError:
            lgr.info(
                "Please install 'distro' package to obtain distribution information"
            )
            dist = tuple()
        except Exception as exc:
            lgr.warning(
                "No distribution information will be provided since 'distro' "
                "fails to import/run: %s", exc_str(exc)
            )
            dist = tuple()

    return {
        'type': os.name,
        'name': pl.system(),
        'release': pl.release(),
        'version': pl.version(),
        'distribution': ' '.join([_t2s(dist),
                                  _t2s(pl.mac_ver()),
                                  _t2s(pl.win32_ver())]).rstrip(),
        'max_path_length': get_max_path_length(getpwd()),
        'encoding': get_encoding_info(),
    }
Beispiel #4
0
 def time_addurls(self, exclude_autometa):
     lgr.warning("CSV: " + self.listfile.read_text())
     ret = dl.addurls(self.ds,
                      str(self.listfile),
                      '{url}',
                      '{filename}',
                      exclude_autometa=exclude_autometa)
     assert not any(r['status'] == 'error' for r in ret)
Beispiel #5
0
def _describe_system():
    import platform as pl
    from datalad import get_encoding_info

    if hasattr(pl, 'dist'):
        dist = pl.dist()
    else:
        # Python 3.8 removed .dist but recommended "distro" is slow, so we
        # try it only if needed
        try:
            import distro
            dist = distro.linux_distribution(full_distribution_name=False)
        except ImportError:
            lgr.info(
                "Please install 'distro' package to obtain distribution information"
            )
            dist = tuple()
        except Exception as exc:
            lgr.warning(
                "No distribution information will be provided since 'distro' "
                "fails to import/run: %s", exc_str(exc))
            dist = tuple()

    return {
        'type':
        os.name,
        'name':
        pl.system(),
        'release':
        pl.release(),
        'version':
        pl.version(),
        'distribution':
        ' '.join([_t2s(dist),
                  _t2s(pl.mac_ver()),
                  _t2s(pl.win32_ver())]).rstrip(),
        'max_path_length':
        get_max_path_length(getpwd()),
        'encoding':
        get_encoding_info(),
    }
    annex = AnnexRepo('.', create=False, init=False)
    # enable datalad special remote
    urls_to_register = defaultdict(list)  # key: urls
    try:
        annex.call_annex(["enableremote", "datalad"])
        # go through each and see where urls aren't yet under web
        # seems might have also --in=datalad to restrict
        w = annex.whereis([], options=['--all'], output='full')
        lgr.info("Got %d entries", len(w))
        for k, rec in tqdm(w.items()):
            datalad_urls = get_remote_urls(rec, 'datalad')
            web_urls = set(get_remote_urls(rec, 'web'))
            for url in datalad_urls:
                if url not in web_urls:
                    if 'openneuro.s3' in url or 'openfmri.s3' in url:
                        urls_to_register[k].append(url)
                    else:
                        lgr.warning("Found unexpected url %s" % url)

    finally:
        # disable datalad special remote
        annex.remove_remote("datalad")  # need to disable it first
    lgr.info("Got %d entries which could get new urls", len(urls_to_register))
    for k, urls in tqdm(urls_to_register.items()):
        for url in urls:
            annex.call_annex(
                ["registerurl", '-c', 'annex.alwayscommit=false', k, url])
    # to cause annex to commit all the changes
    annex.call_annex(["merge"])
    annex.gc(allow_background=False)
    urls_to_register = defaultdict(list)  # key: urls
    try:
        annex._annex_custom_command([], ["git", "annex", "enableremote", "datalad"])
        # go through each and see where urls aren't yet under web
        # seems might have also --in=datalad to restrict
        w = annex.whereis([], options=['--all'], output='full')
        lgr.info("Got %d entries", len(w))
        for k, rec in tqdm(w.items()):
            datalad_urls = get_remote_urls(rec, 'datalad')
            web_urls = set(get_remote_urls(rec, 'web'))
            for url in datalad_urls:
                if url not in web_urls:
                    if 'openneuro.s3' in url or 'openfmri.s3' in url:
                        urls_to_register[k].append(url)
                    else:
                        lgr.warning("Found unexpected url %s" % url)

    finally:
        # disable datalad special remote
        annex.remove_remote("datalad") # need to disable it first
    lgr.info(
        "Got %d entries which could get new urls",
        len(urls_to_register)
    )
    for k, urls in tqdm(urls_to_register.items()):
        for url in urls:
            annex._annex_custom_command(
                [],
                ["git", "annex", "registerurl", '-c', 'annex.alwayscommit=false', k, url]
            )
    # to cause annex to commit all the changes