def curr_commit(self, branches=False):
Logger.debug('[dvc-git] Getting current git commit. Command: git rev-parse --short HEAD')
if branches:
cmd = 'git rev-parse --abbrev-ref HEAD'
else:
cmd = 'git rev-parse HEAD'
code, out, err = Executor.exec_cmd(cmd.split())
if code != 0:
raise ExecutorError('[dvc-git] Commit command error - {}'.format(err))
Logger.debug('[dvc-git] Getting current git commit. Success.')
return out
def init_git_repo(self):
Executor.exec_cmd_only_success(['git', 'init'])
msg = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME)
msg += '\n'
msg += os.path.join(ConfigI.CONFIG_DIR, '.' + ConfigI.CONFIG + '.lock')
self.create_file('.gitignore', msg)
Executor.exec_cmd_only_success(['git', 'add', '.gitignore'])
Executor.exec_cmd_only_success(['git', 'commit', '-m', '"Init test repo"'])
def get_changed_files(target_commit):
Logger.debug('[dvc-git] Identify changes. Command: '
'git diff --name-only HEAD {}'.format(target_commit))
changed_files_str = Executor.exec_cmd_only_success(['git',
'diff',
'--name-only',
'HEAD',
target_commit])
changed_files = changed_files_str.strip('"').split('\n')
Logger.debug('[dvc-git] Identify changes. Success. Changed files: {}'.format(
', '.join(changed_files)))
return changed_files
def branches(regexp):
cmd = 'git branch --format "%(refname)"'
lines = Executor.exec_cmd_only_success(cmd.split()).split('\n')
lines = map(lambda l: l.strip('"'), lines)
if not regexp or '/' not in regexp:
# remove remotes like 'refs/heads'
lines = map(lambda s: s.split('/')[-1], lines)
if not regexp:
return lines
reg = re.compile(regexp)
return list(filter(reg.search, lines))
def __init__(self, cmd_args, settings, stdout, stderr, shell=False):
self._settings = settings
stemps_before = self.data_file_timesteps()
Logger.debug(
u'[Repository change] Exec command: {}. stdout={}, stderr={}, shell={}'
.format(u' '.join(cmd_args), stdout, stderr, shell))
Executor.exec_cmd_only_success(cmd_args, stdout, stderr, shell=shell)
stemps_after = self.data_file_timesteps()
sym_diff = stemps_after ^ stemps_before
self._modified_content_filenames = set(
[filename for filename, timestemp in sym_diff])
Logger.debug(
u'[Repository change] Identified modifications: {}'.format(
u', '.join(self._modified_content_filenames)))
self._stated_data_items = []
self._externally_created_files = []
self._created_status_files = []
self._init_file_states()
def is_target(self, hash, target, settings):
git_cmd = ['git', 'show', '--pretty=', '--name-only', hash]
files = set(Executor.exec_cmd_only_success(git_cmd).split('\n'))
if target in files:
symlink_content = self._get_symlink_content(hash, target, settings)
if symlink_content is not None:
metric = self.target_metric_from_git_history(
hash, symlink_content, target, settings)
else:
metric = None
return True, metric
return False, None
def _read_metric_from_state_file(self, hash, target, settings):
try:
data_item = settings.path_factory.data_item(target)
except DataItemError as ex:
Logger.warn('Target file {} is not data item: {}'.format(target, ex))
return None
try:
cmd_corresponded_state_file = ['git', 'show', '{}:{}'.format(hash, data_item.state.relative)]
state_file_content = Executor.exec_cmd_only_success(cmd_corresponded_state_file)
except ExecutorError as ex:
msg = '[dvc-git] Cannot obtain content of target symbolic file {} with hash {}: {}'
Logger.warn(msg.format(target, hash, ex))
return None
state_file = StateFile.loads(state_file_content, settings)
return state_file.single_target_metric
def test_custom_file_outputs(self):
stdout_file = os.path.join(self.test_dir, 'stdout.txt')
stderr_file = os.path.join(self.test_dir, 'stderr.txt')
output = Executor.exec_cmd_only_success(['ls', self.test_dir],
stdout_file, stderr_file)
self.assertEqual(output, '')
output_set = set(open(stdout_file).read().split())
expected = {
self.file1, self.file2,
os.path.basename(stdout_file),
os.path.basename(stderr_file)
}
self.assertEqual(output_set, expected)
os.remove(stdout_file)
os.remove(stderr_file)
def git_dir(self):
if self._git_dir:
return self._git_dir
try:
Logger.debug('[dvc-git] Getting git directory. Command: git rev-parse --show-toplevel')
code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--show-toplevel'])
if code != 0:
raise ExecutorError('[dvc-git] Git directory command error - {}'.format(err))
Logger.debug('[dvc-git] Getting git directory. Success.')
self._git_dir = out
return self._git_dir
except ExecutorError:
raise
except Exception as e:
raise ExecutorError('Unable to run git command: {}'.format(e))
pass
def get_all_commits(self, target, settings):
# git log --all --abbrev=7 --pretty=format:"%h|%p|%an|%ai|%s"
try:
merges_map = GitWrapper.get_merges_map()
format_str = GitWrapper.LOG_SEPARATOR.join(GitWrapper.LOG_FORMAT)
git_cmd = [
'git', 'log', '--all',
'--abbrev={}'.format(GitWrapper.COMMIT_LEN),
'--pretty=format:{}'.format(format_str)
]
lines = Executor.exec_cmd_only_success(git_cmd).split('\n')
branches_multimap = GitWrapper.branches_multimap()
wf = Workflow(target, merges_map, branches_multimap)
for line in lines:
items = line.split(GitWrapper.LOG_SEPARATOR,
len(GitWrapper.LOG_FORMAT))
assert len(
items
) == 5, 'Git wrapper: git log format has {} items, 5 expected'.format(
len(items))
hash, parent_hash, name, date, comment = items
commit = Commit(hash,
parent_hash,
name,
date,
comment,
*self.was_target_changed(
hash, target, settings),
branch_tips=branches_multimap.get(hash))
wf.add_commit(commit)
return wf
except ExecutorError:
raise
def _get_symlink_content(self, hash, target, settings):
try:
settings.path_factory.data_item(target)
except DataItemError as ex:
Logger.warn('Target file {} is not data item: {}'.format(
target, ex))
return None
try:
cmd_symlink_data = ['git', 'show', '{}:{}'.format(hash, target)]
symlink_content = Executor.exec_cmd_only_success(
cmd_symlink_data).split('\n')
except ExecutorError as ex:
msg = '[dvc-git] Cannot obtain content of target symbolic file {} with hash {}: {}'
Logger.warn(msg.format(target, hash, ex))
return None
if not symlink_content or len(symlink_content) != 1:
msg = '[dvc-git] Target symbolic file {} with hash {} has wrong format'
Logger.warn(msg.format(target, hash))
return None
return symlink_content[0]
def test_suppress_output(self):
output = Executor.exec_cmd_only_success(['ls', self.test_dir])
output_set = set(output.split())
self.assertEqual(output_set, {self.file1, self.file2})
def is_inside_work_tree(self):
code, out, err = Executor.exec_cmd(['git', 'rev-parse', '--is-inside-work-tree'])
if code != 0:
Logger.error('[dvc-git] Not inside git tree');
return code == 0
def init_git_repo(self):
Executor.exec_cmd_only_success(['git', 'init'])
self.create_file('.gitignore', 'cache\n.dvc.conf.lock')
Executor.exec_cmd_only_success(['git', 'add', '.gitignore'])
Executor.exec_cmd_only_success(
['git', 'commit', '-m', '"Init test repo"'])
def run(self):
if self.cmd:
Executor.exec_cmd_only_success(self.cmd, cwd=str(self.cwd), shell=True)
self.save()
def commit_file(file_name, message='Just a commit'):
Executor.exec_cmd_only_success(['git', 'add', file_name])
Executor.exec_cmd_only_success(['git', 'commit', '-m', message])
def get_target_commit(file):
try:
commit = Executor.exec_cmd_only_success(['git', 'log', '-1', '--pretty=format:"%h"', file])
return commit.strip('"')
except ExecutorError:
return None
def checkout_file_before_last_merge(fname):
cmd = 'git checkout @{1} -- ' + fname
Logger.debug(
'[dvc-git] Checkout file {} before last merge. Command: {}'.format(
fname, cmd))
Executor.exec_cmd_only_success(cmd.split())
def test_wrong_command(self):
with self.assertRaises(ExecutorError):
Executor.exec_cmd_only_success(
['--wrong-command--', self.test_dir])
def checkout(commit_or_branch, create_new=False):
prefix = '-b ' if create_new else ''
Logger.debug('[dvc-git] Checkout {}{}'.format(prefix,
commit_or_branch))
cmd = 'git checkout {}{}'.format(prefix, commit_or_branch)
return Executor.exec_cmd(cmd.split())
def checkout_previous():
Logger.debug('[dvc-git] Checkout previous')
cmd = 'git checkout -'
return Executor.exec_cmd(cmd.split())
def all_untracked_files():
cmd = 'git ls-files --others --exclude-standard'
res = Executor.exec_cmd_only_success(cmd.split())
if not res.strip():
return []
return res.split('\n')
def get_file_content(fname, branch):
cmd = 'git show {}:{}'.format(branch, fname)
return Executor.exec_cmd_only_success(cmd.split())
def git_prev_commit():
cmd = 'git check-ref-format --branch @{-1}'
code, out, err = Executor.exec_cmd(cmd.split())
if code != 0:
return ''
return out
def test_output_to_std_console(self):
output = Executor.exec_cmd_only_success(['ls', self.test_dir], '-',
'-')
self.assertEqual(output, '')
def run_command(settings, state):
Executor.exec_cmd_only_success(state.cmd, cwd=state.cwd, shell=True)
CmdRun.move_output_to_cache(settings, state)
CmdRun.update_state_file(settings, state)