def _get_untracked_content(dspath, report_untracked, paths=None): cmd = [ 'git', '--work-tree=.', 'status', '--porcelain', # file names NULL terminated '-z', # we never want to touch submodules, they cannot be untracked '--ignore-submodules=all', # fully untracked dirs as such, the rest as files '--untracked={}'.format(report_untracked) ] try: stdout, stderr = GitRunner(cwd=dspath).run(cmd, log_stderr=True, log_stdout=True, log_online=False, expect_stderr=False, shell=False, expect_fail=True) except CommandError as e: # TODO should we catch any and handle them in here? raise e if paths: paths = [r['path'] for r in paths] if len(paths) == 1 and paths[0] == dspath: # nothing to filter paths = None from datalad.utils import assure_unicode for line in stdout.split('\0'): if not line: continue line = assure_unicode(line) if not line.startswith('?? '): # nothing untracked, ignore, task of `diff` continue apath = opj( dspath, # strip state marker line[3:]) norm_apath = normpath(apath) if paths and not any(norm_apath == p or path_startswith(apath, p) for p in paths): # we got a whitelist for paths, don't report any other continue ap = dict(path=norm_apath, parentds=dspath, state='untracked', type='directory' if isdir(apath) else 'file') yield ap
def _get_untracked_content(dspath, report_untracked, paths=None): cmd = ['git', '--work-tree=.', 'status', '--porcelain', # file names NULL terminated '-z', # we never want to touch submodules, they cannot be untracked '--ignore-submodules=all', # fully untracked dirs as such, the rest as files '--untracked={}'.format(report_untracked)] try: stdout, stderr = GitRunner(cwd=dspath).run( cmd, log_stderr=True, log_stdout=True, log_online=False, expect_stderr=False, shell=False, expect_fail=True) except CommandError as e: # TODO should we catch any and handle them in here? raise e if paths: paths = [r['path'] for r in paths] if len(paths) == 1 and paths[0] == dspath: # nothing to filter paths = None from datalad.utils import assure_unicode for line in stdout.split('\0'): if not line: continue line = assure_unicode(line) if not line.startswith('?? '): # nothing untracked, ignore, task of `diff` continue apath = opj( dspath, # strip state marker line[3:]) norm_apath = normpath(apath) if paths and not any(norm_apath == p or path_startswith(apath, p) for p in paths): # we got a whitelist for paths, don't report any other continue ap = dict( path=norm_apath, parentds=dspath, state='untracked', type='directory' if isdir(apath) else 'file') yield ap
def _parse_git_submodules(dspath, recursive): """All known ones with some properties""" if not exists(opj(dspath, ".gitmodules")): # easy way out. if there is no .gitmodules file # we cannot have (functional) subdatasets return # this will not work in direct mode, need better way #1422 cmd = ['git', '--work-tree=.', 'submodule', 'status'] if recursive: cmd.append('--recursive') # need to go rogue and cannot use proper helper in GitRepo # as they also pull in all of GitPython's magic try: stdout, stderr = GitRunner(cwd=dspath).run( cmd, log_stderr=True, log_stdout=True, # not sure why exactly, but log_online has to be false! log_online=False, expect_stderr=False, shell=False, # we don't want it to scream on stdout expect_fail=True) except CommandError as e: raise InvalidGitRepositoryError(exc_str(e)) for line in stdout.split('\n'): if not line: continue sm = {} sm['state'] = status_map[line[0]] props = submodule_full_props.match(line[1:]) if props: sm['revision'] = props.group(1) sm['path'] = opj(dspath, props.group(2)) sm['revision_descr'] = props.group(3) else: props = submodule_nodescribe_props.match(line[1:]) sm['revision'] = props.group(1) sm['path'] = opj(dspath, props.group(2)) yield sm
def _parse_git_submodules(dspath): """All known ones with some properties""" if not exists(opj(dspath, ".gitmodules")): # easy way out. if there is no .gitmodules file # we cannot have (functional) subdatasets return # this will not work in direct mode, need better way #1422 cmd = ['git', 'ls-files', '--stage', '-z'] # need to go rogue and cannot use proper helper in GitRepo # as they also pull in all of GitPython's magic try: stdout, stderr = GitRunner(cwd=dspath).run( cmd, log_stderr=True, log_stdout=True, # not sure why exactly, but log_online has to be false! log_online=False, expect_stderr=False, shell=False, # we don't want it to scream on stdout expect_fail=True) except CommandError as e: raise InvalidGitRepositoryError(exc_str(e)) for line in stdout.split('\0'): if not line or not line.startswith('160000'): continue sm = {} props = submodule_full_props.match(line) sm['revision'] = props.group(2) subpath = _path_(dspath, props.group(4)) sm['path'] = subpath if not exists(subpath) or not GitRepo.is_valid_repo(subpath): sm['state'] = 'absent' yield sm
def _parse_git_diff(dspath, diff_thingie=None, paths=None, ignore_submodules='none', staged=False): # use '--work-tree=.' to get direct omde to cooperate cmd = ['git', '--work-tree=.', 'diff', '--raw', # file names NULL terminated '-z', # how to treat submodules (see git diff docs) '--ignore-submodules={}'.format(ignore_submodules), # never abbreviate sha sums '--abbrev=40'] if staged: cmd.append('--staged') if diff_thingie: cmd.append(diff_thingie) if paths: cmd.append('--') cmd.extend(ap['path'] for ap in paths if ap.get('raw_input', False)) try: stdout, stderr = GitRunner(cwd=dspath).run( cmd, log_stderr=True, log_stdout=True, log_online=False, expect_stderr=False, shell=False, expect_fail=True) except CommandError as e: if 'bad revision' in e.stderr: yield dict( path=dspath, type='dataset', status='impossible', message=e.stderr.strip()) return raise ap = None for line in stdout.split('\0'): if not line: continue if line.startswith(':'): # a new path # yield any existing one if ap: yield ap ap = None # start new record m_src, m_dst, sha_src, sha_dst, status = \ line[1:].split() ap = dict( mode_src=int(m_src, base=8), mode=int(m_dst, base=8), revision_src=sha_src if sha_src != '0' * 40 else None, revision=sha_dst if sha_dst != '0' * 40 else None, parentds=dspath) _translate_status(status, ap) _translate_type(ap['mode'], ap, 'type') _translate_type(ap['mode_src'], ap, 'type_src') else: # a filename if 'path' in ap: ap['path_src'] = ap['path'] ap['path'] = opj(dspath, line) if ap: yield ap
def _parse_git_diff(dspath, diff_thingie=None, paths=None, ignore_submodules='none', staged=False): # use '--work-tree=.' to get direct omde to cooperate cmd = [ 'git', '--work-tree=.', 'diff', '--raw', # file names NULL terminated '-z', # how to treat submodules (see git diff docs) '--ignore-submodules={}'.format(ignore_submodules), # never abbreviate sha sums '--abbrev=40' ] if staged: cmd.append('--staged') if diff_thingie: cmd.append(diff_thingie) if paths: cmd.append('--') cmd.extend(ap['path'] for ap in paths if ap.get('raw_input', False)) try: stdout, stderr = GitRunner(cwd=dspath).run(cmd, log_stderr=True, log_stdout=True, log_online=False, expect_stderr=False, shell=False, expect_fail=True) except CommandError as e: if 'bad revision' in e.stderr: yield dict(path=dspath, type='dataset', status='impossible', message=e.stderr.strip()) return raise e ap = None for line in stdout.split('\0'): if not line: continue if line.startswith(':'): # a new path # yield any existing one if ap: yield ap ap = None # start new record m_src, m_dst, sha_src, sha_dst, status = \ line[1:].split() ap = dict(mode_src=int(m_src, base=8), mode=int(m_dst, base=8), revision_src=sha_src if sha_src != '0' * 40 else None, revision=sha_dst if sha_dst != '0' * 40 else None, parentds=dspath) _translate_status(status, ap) _translate_type(ap['mode'], ap, 'type') _translate_type(ap['mode_src'], ap, 'type_src') else: # a filename if 'path' in ap: ap['path_src'] = ap['path'] ap['path'] = opj(dspath, line) if ap: yield ap