def run_command(self,
cmd_args,
data_items_from_args,
not_data_items_from_args,
stdout=None,
stderr=None,
shell=False):
Logger.debug(
'Run command with args: {}. Data items from args: {}. stdout={}, stderr={}, shell={}'
.format(' '.join(cmd_args),
', '.join([x.data.dvc for x in data_items_from_args]),
stdout, stderr, shell))
repo_change = RepositoryChange(cmd_args,
self.settings,
stdout,
stderr,
shell=shell)
if not self.no_git_actions and not self._validate_file_states(
repo_change):
self.remove_new_files(repo_change)
raise RunError('Errors occurred.')
output_set = set(self.declaration_output_data_items +
repo_change.changed_data_items)
output_files_dvc = [x.data.dvc for x in output_set]
input_set = set(data_items_from_args +
self.declaration_input_data_items) - output_set
input_files_dvc = [x.data.dvc for x in input_set]
code_dependencies_dvc = self.git.abs_paths_to_dvc(
self.code_dependencies + not_data_items_from_args)
result = []
for data_item in repo_change.changed_data_items:
Logger.debug(
'Move output file "{}" to cache dir "{}" and create a symlink'.
format(data_item.data.relative, data_item.cache.relative))
data_item.move_data_to_cache()
Logger.debug('Create state file "{}"'.format(
data_item.state.relative))
state_file = StateFile(StateFile.COMMAND_RUN,
data_item.state.relative,
self.settings,
input_files_dvc,
output_files_dvc,
code_dependencies_dvc,
argv=cmd_args,
lock=self.lock,
stdout=self._stdout_to_dvc(stdout),
stderr=self._stdout_to_dvc(stderr),
shell=shell)
state_file.save()
result.append(state_file)
return result
def test(self):
not_in_data_dir_file = 'file1.txt'
self.create_file(not_in_data_dir_file)
change = RepositoryChange(['ls', '-la'],
self.settings,
not_in_data_dir_file,
None)
self.assertEqual([os.path.realpath(not_in_data_dir_file)],
change.externally_created_files)
def test(self):
os.mkdir(os.path.join('data', 'dir32'))
deep_file = os.path.join('data', 'dir32', 'file1.txt')
self.create_file(deep_file)
change = RepositoryChange(['ls', '-la'],
self.settings,
deep_file,
None)
self.assertEqual([x.data.dvc for x in change.new_data_items], [deep_file])
def test(self):
not_in_data_dir_file = 'file1.txt'
self.create_file(not_in_data_dir_file)
change = RepositoryChange(['echo', 'textsd'],
self.settings,
not_in_data_dir_file,
None,
shell=True)
self.assertEqual([System.realpath(not_in_data_dir_file)],
change.externally_created_files)
def test_all(self):
file_to_modify = os.path.join('data', 'file1.txt')
file_created_before_run = os.path.join('data', 'file2.txt')
file_created_by_run = os.path.join('data', 'out.txt')
file_to_remove = os.path.join('data', 'file3.txt')
self.create_file(file_to_modify)
self.create_file(file_created_before_run)
self.create_file(file_to_remove)
subprocess.Popen(['git', 'add', file_to_modify, file_to_remove],
stdout=self._devnull,
stderr=None).wait()
subprocess.Popen(['git', 'commit', '-m', '"Adding one file3"'],
stdout=self._devnull,
stderr=None).wait()
os.remove(file_to_remove)
change = RepositoryChange(['echo', 'test'],
self.settings,
file_created_by_run,
file_to_modify,
shell=True)
expected_new = [file_created_by_run, file_created_before_run]
new_file_abs = [x.data.dvc for x in change.new_data_items]
self.assertEqual(set(new_file_abs), set(expected_new))
self.assertEqual(len(change.removed_data_items), 1)
self.assertEqual(change.removed_data_items[0].data.dvc, file_to_remove)
self.assertEqual(len(change.modified_data_items), 1)
self.assertEqual(change.modified_data_items[0].data.dvc,
file_to_modify)
expected_to_change_set = set(expected_new + [file_to_modify])
changed_file_names = [x.data.dvc for x in change.changed_data_items]
self.assertEqual(set(changed_file_names), expected_to_change_set)
self.assertEqual([], change.externally_created_files)
pass