def _get_untracked_content(dspath, report_untracked, paths=None):
cmd = [
'git',
'--work-tree=.',
'status',
'--porcelain',
# file names NULL terminated
'-z',
# we never want to touch submodules, they cannot be untracked
'--ignore-submodules=all',
# fully untracked dirs as such, the rest as files
'--untracked={}'.format(report_untracked)
]
try:
stdout, stderr = GitRunner(cwd=dspath).run(cmd,
log_stderr=True,
log_stdout=True,
log_online=False,
expect_stderr=False,
shell=False,
expect_fail=True)
except CommandError as e:
# TODO should we catch any and handle them in here?
raise e
if paths:
paths = [r['path'] for r in paths]
if len(paths) == 1 and paths[0] == dspath:
# nothing to filter
paths = None
from datalad.utils import assure_unicode
for line in stdout.split('\0'):
if not line:
continue
line = assure_unicode(line)
if not line.startswith('?? '):
# nothing untracked, ignore, task of `diff`
continue
apath = opj(
dspath,
# strip state marker
line[3:])
norm_apath = normpath(apath)
if paths and not any(norm_apath == p or path_startswith(apath, p)
for p in paths):
# we got a whitelist for paths, don't report any other
continue
ap = dict(path=norm_apath,
parentds=dspath,
state='untracked',
type='directory' if isdir(apath) else 'file')
yield ap
def _get_untracked_content(dspath, report_untracked, paths=None):
cmd = ['git', '--work-tree=.', 'status', '--porcelain',
# file names NULL terminated
'-z',
# we never want to touch submodules, they cannot be untracked
'--ignore-submodules=all',
# fully untracked dirs as such, the rest as files
'--untracked={}'.format(report_untracked)]
try:
stdout, stderr = GitRunner(cwd=dspath).run(
cmd,
log_stderr=True,
log_stdout=True,
log_online=False,
expect_stderr=False,
shell=False,
expect_fail=True)
except CommandError as e:
# TODO should we catch any and handle them in here?
raise e
if paths:
paths = [r['path'] for r in paths]
if len(paths) == 1 and paths[0] == dspath:
# nothing to filter
paths = None
from datalad.utils import assure_unicode
for line in stdout.split('\0'):
if not line:
continue
line = assure_unicode(line)
if not line.startswith('?? '):
# nothing untracked, ignore, task of `diff`
continue
apath = opj(
dspath,
# strip state marker
line[3:])
norm_apath = normpath(apath)
if paths and not any(norm_apath == p or path_startswith(apath, p)
for p in paths):
# we got a whitelist for paths, don't report any other
continue
ap = dict(
path=norm_apath,
parentds=dspath,
state='untracked',
type='directory' if isdir(apath) else 'file')
yield ap
def _parse_git_submodules(dspath, recursive):
"""All known ones with some properties"""
if not exists(opj(dspath, ".gitmodules")):
# easy way out. if there is no .gitmodules file
# we cannot have (functional) subdatasets
return
# this will not work in direct mode, need better way #1422
cmd = ['git', '--work-tree=.', 'submodule', 'status']
if recursive:
cmd.append('--recursive')
# need to go rogue and cannot use proper helper in GitRepo
# as they also pull in all of GitPython's magic
try:
stdout, stderr = GitRunner(cwd=dspath).run(
cmd,
log_stderr=True,
log_stdout=True,
# not sure why exactly, but log_online has to be false!
log_online=False,
expect_stderr=False,
shell=False,
# we don't want it to scream on stdout
expect_fail=True)
except CommandError as e:
raise InvalidGitRepositoryError(exc_str(e))
for line in stdout.split('\n'):
if not line:
continue
sm = {}
sm['state'] = status_map[line[0]]
props = submodule_full_props.match(line[1:])
if props:
sm['revision'] = props.group(1)
sm['path'] = opj(dspath, props.group(2))
sm['revision_descr'] = props.group(3)
else:
props = submodule_nodescribe_props.match(line[1:])
sm['revision'] = props.group(1)
sm['path'] = opj(dspath, props.group(2))
yield sm
def _parse_git_submodules(dspath):
"""All known ones with some properties"""
if not exists(opj(dspath, ".gitmodules")):
# easy way out. if there is no .gitmodules file
# we cannot have (functional) subdatasets
return
# this will not work in direct mode, need better way #1422
cmd = ['git', 'ls-files', '--stage', '-z']
# need to go rogue and cannot use proper helper in GitRepo
# as they also pull in all of GitPython's magic
try:
stdout, stderr = GitRunner(cwd=dspath).run(
cmd,
log_stderr=True,
log_stdout=True,
# not sure why exactly, but log_online has to be false!
log_online=False,
expect_stderr=False,
shell=False,
# we don't want it to scream on stdout
expect_fail=True)
except CommandError as e:
raise InvalidGitRepositoryError(exc_str(e))
for line in stdout.split('\0'):
if not line or not line.startswith('160000'):
continue
sm = {}
props = submodule_full_props.match(line)
sm['revision'] = props.group(2)
subpath = _path_(dspath, props.group(4))
sm['path'] = subpath
if not exists(subpath) or not GitRepo.is_valid_repo(subpath):
sm['state'] = 'absent'
yield sm
def _parse_git_submodules(dspath):
"""All known ones with some properties"""
if not exists(opj(dspath, ".gitmodules")):
# easy way out. if there is no .gitmodules file
# we cannot have (functional) subdatasets
return
# this will not work in direct mode, need better way #1422
cmd = ['git', 'ls-files', '--stage', '-z']
# need to go rogue and cannot use proper helper in GitRepo
# as they also pull in all of GitPython's magic
try:
stdout, stderr = GitRunner(cwd=dspath).run(
cmd,
log_stderr=True,
log_stdout=True,
# not sure why exactly, but log_online has to be false!
log_online=False,
expect_stderr=False,
shell=False,
# we don't want it to scream on stdout
expect_fail=True)
except CommandError as e:
raise InvalidGitRepositoryError(exc_str(e))
for line in stdout.split('\0'):
if not line or not line.startswith('160000'):
continue
sm = {}
props = submodule_full_props.match(line)
sm['revision'] = props.group(2)
subpath = _path_(dspath, props.group(4))
sm['path'] = subpath
if not exists(subpath) or not GitRepo.is_valid_repo(subpath):
sm['state'] = 'absent'
yield sm
def _parse_git_diff(dspath, diff_thingie=None, paths=None,
ignore_submodules='none', staged=False):
# use '--work-tree=.' to get direct omde to cooperate
cmd = ['git', '--work-tree=.', 'diff', '--raw',
# file names NULL terminated
'-z',
# how to treat submodules (see git diff docs)
'--ignore-submodules={}'.format(ignore_submodules),
# never abbreviate sha sums
'--abbrev=40']
if staged:
cmd.append('--staged')
if diff_thingie:
cmd.append(diff_thingie)
if paths:
cmd.append('--')
cmd.extend(ap['path'] for ap in paths if ap.get('raw_input', False))
try:
stdout, stderr = GitRunner(cwd=dspath).run(
cmd,
log_stderr=True,
log_stdout=True,
log_online=False,
expect_stderr=False,
shell=False,
expect_fail=True)
except CommandError as e:
if 'bad revision' in e.stderr:
yield dict(
path=dspath,
type='dataset',
status='impossible',
message=e.stderr.strip())
return
raise
ap = None
for line in stdout.split('\0'):
if not line:
continue
if line.startswith(':'):
# a new path
# yield any existing one
if ap:
yield ap
ap = None
# start new record
m_src, m_dst, sha_src, sha_dst, status = \
line[1:].split()
ap = dict(
mode_src=int(m_src, base=8),
mode=int(m_dst, base=8),
revision_src=sha_src if sha_src != '0' * 40 else None,
revision=sha_dst if sha_dst != '0' * 40 else None,
parentds=dspath)
_translate_status(status, ap)
_translate_type(ap['mode'], ap, 'type')
_translate_type(ap['mode_src'], ap, 'type_src')
else:
# a filename
if 'path' in ap:
ap['path_src'] = ap['path']
ap['path'] = opj(dspath, line)
if ap:
yield ap
def _parse_git_diff(dspath,
diff_thingie=None,
paths=None,
ignore_submodules='none',
staged=False):
# use '--work-tree=.' to get direct omde to cooperate
cmd = [
'git',
'--work-tree=.',
'diff',
'--raw',
# file names NULL terminated
'-z',
# how to treat submodules (see git diff docs)
'--ignore-submodules={}'.format(ignore_submodules),
# never abbreviate sha sums
'--abbrev=40'
]
if staged:
cmd.append('--staged')
if diff_thingie:
cmd.append(diff_thingie)
if paths:
cmd.append('--')
cmd.extend(ap['path'] for ap in paths if ap.get('raw_input', False))
try:
stdout, stderr = GitRunner(cwd=dspath).run(cmd,
log_stderr=True,
log_stdout=True,
log_online=False,
expect_stderr=False,
shell=False,
expect_fail=True)
except CommandError as e:
if 'bad revision' in e.stderr:
yield dict(path=dspath,
type='dataset',
status='impossible',
message=e.stderr.strip())
return
raise e
ap = None
for line in stdout.split('\0'):
if not line:
continue
if line.startswith(':'):
# a new path
# yield any existing one
if ap:
yield ap
ap = None
# start new record
m_src, m_dst, sha_src, sha_dst, status = \
line[1:].split()
ap = dict(mode_src=int(m_src, base=8),
mode=int(m_dst, base=8),
revision_src=sha_src if sha_src != '0' * 40 else None,
revision=sha_dst if sha_dst != '0' * 40 else None,
parentds=dspath)
_translate_status(status, ap)
_translate_type(ap['mode'], ap, 'type')
_translate_type(ap['mode_src'], ap, 'type_src')
else:
# a filename
if 'path' in ap:
ap['path_src'] = ap['path']
ap['path'] = opj(dspath, line)
if ap:
yield ap
