def were_files_changed(self, file, code_dependencies, path_factory):
Logger.debug('[Git] Identify changes. Commands: {} {} && {}'.format(
'git log -1 --pretty=format:"%h"', file,
'git diff --name-only HEAD *'))
commit = Executor.exec_cmd_only_success(
['git', 'log', '-1', '--pretty=format:"%h"', file])
commit = commit.strip('"')
Logger.debug('[Git] Identify changes. Success.')
changed_files = Executor.exec_cmd_only_success(
['git', 'diff', '--name-only', 'HEAD', commit])
changed_files = changed_files.strip('"')
code_files, code_dirs = self.separate_dependency_files_and_dirs(
code_dependencies)
code_files_set = set([path_factory.path(x).dvc for x in code_files])
for changed_file in changed_files.split('\n'):
if changed_file in code_files_set:
return True
for dir in code_dirs:
if changed_file.startswith(dir):
return True
return False
def __init__(self, command, settings, stdout, stderr, shell=False):
self._settings = settings
stemps_before = self.data_file_timesteps()
Logger.debug(
u'[Repository change] Exec command: {}. stdout={}, stderr={}, shell={}'
.format(command, stdout, stderr, shell))
Executor.exec_cmd_only_success(command.split(' '),
stdout,
stderr,
shell=shell)
stemps_after = self.data_file_timesteps()
sym_diff = stemps_after ^ stemps_before
self._modified_content_filenames = set(
[filename for filename, timestemp in sym_diff])
Logger.debug(
u'[Repository change] Identified modifications: {}'.format(
u', '.join(self._modified_content_filenames)))
self._stated_data_items = []
self._externally_created_files = []
self._created_status_files = []
self._init_file_states()
def commit_all_changes(message):
Logger.debug('[dvc-git] Commit all changes. Commands: {} && {} && {}'.format(
'git add --all', 'git status --porcelain', 'git commit -m'))
Executor.exec_cmd_only_success(['git', 'add', '--all'])
out_status = Executor.exec_cmd_only_success(['git', 'status', '--porcelain'])
Executor.exec_cmd_only_success(['git', 'commit', '-m', message])
Logger.debug('[dvc-git] Commit all changes. Success.')
return GitWrapper.parse_porcelain_files(out_status)
def __init__(self, cmd_args, settings, stdout, stderr):
self._settings = settings
Logger.debug(
u'[Repository change] Exec command: {}. stdout={}, stderr={}'.
format(u' '.join(cmd_args), stdout, stderr))
Executor.exec_cmd_only_success(cmd_args, stdout, stderr)
self._stated_data_items = []
self._externally_created_files = []
self._init_file_states()
def get_all_commits(self, target, settings):
# git log --all --abbrev=7 --pretty=format:"%h|%p|%an|%ai|%s"
try:
merges_map = GitWrapper.get_merges_map()
format_str = GitWrapper.LOG_SEPARATOR.join(GitWrapper.LOG_FORMAT)
git_cmd = ['git', 'log', '--all', '--abbrev={}'.format(GitWrapper.COMMIT_LEN),
'--pretty=format:{}'.format(format_str)]
lines = Executor.exec_cmd_only_success(git_cmd).split('\n')
branches_multimap = GitWrapper.branches_multimap()
wf = Workflow(target, merges_map, branches_multimap)
for line in lines:
items = line.split(GitWrapper.LOG_SEPARATOR, len(GitWrapper.LOG_FORMAT))
assert len(items) == 5, 'Git wrapper: git log format has {} items, 5 expected'.format(len(items))
hash, parent_hash, name, date, comment = items
commit = Commit(hash, parent_hash, name, date, comment,
*self.was_target_changed(hash, target, settings),
branch_tips=branches_multimap.get(hash))
wf.add_commit(commit)
return wf
except ExecutorError:
raise
def get_target_commit(file):
try:
commit = Executor.exec_cmd_only_success(
['git', 'log', '-1', '--pretty=format:"%h"', file])
return commit.strip('"')
except ExecutorError:
return None
def get_last_merge_changed_files():
cmd = 'git diff --name-only @{1}'
Logger.debug('[dvc-git] Get last merge changed files. Command: {}'.format(cmd))
try:
fstr = Executor.exec_cmd_only_success(cmd.split())
return fstr.split('\n')
except ExecutorError:
return None
def get_target_commit(fname):
Logger.debug('[dvc-git] Get target commit. Command: '
'git log -1 --pretty=format:"%h" {}'.format(fname))
try:
commit = Executor.exec_cmd_only_success(
['git', 'log', '-1', '--pretty=format:"%h"', fname])
return commit.strip('"')
except ExecutorError:
return None
def branches_multimap():
git_cmd = ['git', 'show-ref', '--abbrev={}'.format(GitWrapper.COMMIT_LEN)]
lines = Executor.exec_cmd_only_success(git_cmd).split('\n')
items_full = map(unicode.split, lines)
items = map(lambda it: (it[0], re.sub(r'^refs/heads/', '', it[1])), items_full)
result = defaultdict(list)
for (hash, branch) in items:
result[hash].append(branch)
return result
def get_changed_files(target_commit):
Logger.debug('[dvc-git] Identify changes. Command: git diff --name-only HEAD {}'.format(
target_commit))
changed_files_str = Executor.exec_cmd_only_success(['git', 'diff', '--name-only', 'HEAD', target_commit])
changed_files = changed_files_str.strip('"').split('\n')
Logger.debug('[dvc-git] Identify changes. Success. Changed files: {}'.format(
', '.join(changed_files)))
return changed_files
def was_target_changed(self, hash, target, settings):
git_cmd = ['git', 'show', '--pretty=', '--name-only', hash]
changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n'))
if target not in changed_files:
return False, None
metric = self._read_metric_from_state_file(hash, target, settings)
if metric is None:
return False, None
return True, metric
def was_target_changed(self, hash, target):
git_cmd = ['git', 'show', '--pretty=', '--name-only', hash]
changed_files = set(Executor.exec_cmd_only_success(git_cmd).split('\n'))
if target not in changed_files:
return False, None
metric = utils.parse_target_metric_file(target)
if metric is None:
return False, None
return True, metric
def init_git_repo(self):
Executor.exec_cmd_only_success(['git', 'init'])
msg = os.path.join(ConfigI.CONFIG_DIR, ConfigI.CACHE_DIR_NAME)
msg += '\n'
msg += os.path.join(ConfigI.CONFIG_DIR, '.' + ConfigI.CONFIG + '.lock')
self.create_file('.gitignore', msg)
Executor.exec_cmd_only_success(['git', 'add', '.gitignore'])
Executor.exec_cmd_only_success(['git', 'commit', '-m', '"Init test repo"'])
def branches(regexp):
cmd = 'git branch --format "%(refname)"'
lines = Executor.exec_cmd_only_success(cmd.split()).split('\n')
lines = map(lambda l: l.strip('"'), lines)
if not regexp or '/' not in regexp:
# remove remotes like 'refs/heads'
lines = map(lambda s: s.split('/')[-1], lines)
if not regexp:
return lines
reg = re.compile(regexp)
return list(filter(reg.search, lines))
def is_target(self, hash, target, settings):
git_cmd = ['git', 'show', '--pretty=', '--name-only', hash]
files = set(Executor.exec_cmd_only_success(git_cmd).split('\n'))
if target in files:
symlink_content = self._get_symlink_content(hash, target, settings)
if symlink_content is not None:
metric = self.target_metric_from_git_history(
hash, symlink_content, target, settings)
else:
metric = None
return True, metric
return False, None
def _read_metric_from_state_file(self, hash, target, settings):
try:
data_item = settings.path_factory.data_item(target)
except DataItemError as ex:
Logger.warn('Target file {} is not data item: {}'.format(target, ex))
return None
try:
cmd_corresponded_state_file = ['git', 'show', '{}:{}'.format(hash, data_item.state.relative)]
state_file_content = Executor.exec_cmd_only_success(cmd_corresponded_state_file)
except ExecutorError as ex:
msg = '[dvc-git] Cannot obtain content of target symbolic file {} with hash {}: {}'
Logger.warn(msg.format(target, hash, ex))
return None
state_file = StateFile.loads(state_file_content, settings)
return state_file.single_target_metric
def test_custom_file_outputs(self):
stdout_file = os.path.join(self.test_dir, 'stdout.txt')
stderr_file = os.path.join(self.test_dir, 'stderr.txt')
output = Executor.exec_cmd_only_success(['ls', self.test_dir],
stdout_file, stderr_file)
self.assertEqual(output, '')
output_set = set(open(stdout_file).read().split())
expected = {
self.file1, self.file2,
os.path.basename(stdout_file),
os.path.basename(stderr_file)
}
self.assertEqual(output_set, expected)
os.remove(stdout_file)
os.remove(stderr_file)
def _get_symlink_content(self, hash, target, settings):
try:
settings.path_factory.data_item(target)
except DataItemError as ex:
Logger.warn('Target file {} is not data item: {}'.format(
target, ex))
return None
try:
cmd_symlink_data = ['git', 'show', '{}:{}'.format(hash, target)]
symlink_content = Executor.exec_cmd_only_success(
cmd_symlink_data).split('\n')
except ExecutorError as ex:
msg = '[dvc-git] Cannot obtain content of target symbolic file {} with hash {}: {}'
Logger.warn(msg.format(target, hash, ex))
return None
if not symlink_content or len(symlink_content) != 1:
msg = '[dvc-git] Target symbolic file {} with hash {} has wrong format'
Logger.warn(msg.format(target, hash))
return None
return symlink_content[0]
def init_git_repo(self):
Executor.exec_cmd_only_success(['git', 'init'])
self.create_file('.gitignore', 'cache\n.dvc.conf.lock')
Executor.exec_cmd_only_success(['git', 'add', '.gitignore'])
Executor.exec_cmd_only_success(
['git', 'commit', '-m', '"Init test repo"'])
def run(self):
if self.cmd:
Executor.exec_cmd_only_success(self.cmd, cwd=str(self.cwd), shell=True)
self.save()
def commit_file(file_name, message='Just a commit'):
Executor.exec_cmd_only_success(['git', 'add', file_name])
Executor.exec_cmd_only_success(['git', 'commit', '-m', message])
def test_wrong_command(self):
with self.assertRaises(ExecutorError):
Executor.exec_cmd_only_success(
['--wrong-command--', self.test_dir])
def run_command(settings, state):
Executor.exec_cmd_only_success(state.cmd, cwd=state.cwd, shell=True)
CmdRun.move_output_to_cache(settings, state)
CmdRun.update_state_file(settings, state)
def test_suppress_output(self):
output = Executor.exec_cmd_only_success(['ls', self.test_dir])
output_set = set(output.split())
self.assertEqual(output_set, {self.file1, self.file2})
def all_untracked_files():
cmd = 'git ls-files --others --exclude-standard'
res = Executor.exec_cmd_only_success(cmd.split())
if not res.strip():
return []
return res.split('\n')
def get_file_content(fname, branch):
cmd = 'git show {}:{}'.format(branch, fname)
return Executor.exec_cmd_only_success(cmd.split())
def test_output_to_std_console(self):
output = Executor.exec_cmd_only_success(['ls', self.test_dir], '-',
'-')
self.assertEqual(output, '')
def checkout_file_before_last_merge(fname):
cmd = 'git checkout @{1} -- ' + fname
Logger.debug(
'[dvc-git] Checkout file {} before last merge. Command: {}'.format(
fname, cmd))
Executor.exec_cmd_only_success(cmd.split())
