def setUp(self):
super(ReproBasicEnv, self).setUp()
self.file_name1 = 'data/file1'
self.file1_code_file = 'file1.py'
self.create_file_and_commit(self.file1_code_file, 'An awesome code...')
self.settings._args = ['printf', 'Hello\nMary', '--not-repro',
'--stdout', self.file_name1, '--code', self.file1_code_file]
cmd_file1 = CmdRun(self.settings)
self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file])
cmd_file1.run()
self.file_name11 = 'data/file11'
self.settings._args = ['head', '-n', '1', self.file_name1, '--stdout', self.file_name11]
CmdRun(self.settings).run()
self.file_name2 = 'data/file2'
self.settings._args = ['printf', 'Bobby', '--not-repro', '--stdout', self.file_name2]
CmdRun(self.settings).run()
self.file_res_code_file = 'code_res.py'
self.create_file_and_commit(self.file_res_code_file, 'Another piece of code')
self.file_name_res = 'data/file_res'
self.settings._args = ['cat', self.file_name11, self.file_name2, '--stdout', self.file_name_res,
'--code', self.file_res_code_file]
cmd_res = CmdRun(self.settings)
self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file])
cmd_res.run()
self.assertEqual(open(self.file_name_res).read(), 'Hello\nBobby')
def setUp(self):
super(RunTwoFilesBase, self).setUp()
self.settings._args = []
cmd_run = CmdRun(self.settings)
self.input_param_file = os.path.join('data', 'extra_input')
cmd_run.parsed_args.input = [self.input_param_file]
self.extra_output_file = os.path.join('data', 'extra_output')
cmd_run.parsed_args.output = [self.extra_output_file]
self.file_name1 = os.path.join('data', 'file1')
self.file_name2 = os.path.join('data', 'file2')
self.state_objs = cmd_run.run_command(['echo', 'test'], [], [],
shell=True,
stdout=self.file_name1,
stderr=self.file_name2)
self.state_file_name1 = os.path.join(
'state', 'file1' + DataItem.STATE_FILE_SUFFIX)
self.state_file_name2 = os.path.join(
'state', 'file2' + DataItem.STATE_FILE_SUFFIX)
self.state_file1 = None
self.state_file2 = None
for s in self.state_objs:
if s.file == self.state_file_name1:
self.state_file1 = s
elif s.file == self.state_file_name2:
self.state_file2 = s
pass
def setUp(self):
super(RunTwoFilesBase, self).setUp()
self.input_param_file = os.path.join('data', 'extra_input')
self.extra_output_file = os.path.join('data', 'extra_output')
self.settings.parse_args('run --input {} --output {} cmd'.format(self.input_param_file, self.extra_output_file))
self.settings._args = []
cmd_run = CmdRun(self.settings)
self.file_name1 = os.path.join('data', 'file1')
self.file_name2 = os.path.join('data', 'file2')
self.state_objs = cmd_run.run_command(['echo', 'test'], [], [], shell=True,
stdout=self.file_name1,
stderr=self.file_name2)
self.state_file_name1 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name1 + DataItem.STATE_FILE_SUFFIX)
self.state_file_name2 = os.path.join(ConfigI.CONFIG_DIR, ConfigI.STATE_DIR_NAME, self.file_name2 + DataItem.STATE_FILE_SUFFIX)
self.state_file1 = None
self.state_file2 = None
for s in self.state_objs:
if s.file == self.state_file_name1:
self.state_file1 = s
elif s.file == self.state_file_name2:
self.state_file2 = s
pass
def test(self):
self.settings._args = []
cmd_run = CmdRun(self.settings)
with self.assertRaises(RunError):
cmd_run.run_command(['echo', 'test'], [], [], shell=True,
stdout='file1', stderr='file2')
pass
def setUp(self):
super(ReproBasicEnv, self).setUp()
self.file_name1 = os.path.join('data', 'file1')
self.file1_code_file = 'file1.py'
self.create_file_and_commit(
self.file1_code_file,
'print("Hello")' + os.linesep + 'print("Mary")')
self.settings.parse_args([
'run', '--stdout', self.file_name1, '--code', self.file1_code_file,
'python', self.file1_code_file, '--not-repro'
])
cmd_file1 = CmdRun(self.settings)
self.assertEqual(cmd_file1.code_dependencies, [self.file1_code_file])
cmd_file1.run()
self.file_name11 = os.path.join('data', 'file11')
self.file11_code_file = 'file11.py'
self.create_file_and_commit(
self.file11_code_file, 'import sys' + os.linesep +
'print(open(sys.argv[1]).readline().strip())')
self.settings.parse_args([
'run', '--stdout', self.file_name11, '--code',
self.file11_code_file, 'python', self.file11_code_file,
self.file_name1
])
CmdRun(self.settings).run()
self.file_name2 = os.path.join('data', 'file2')
self.file2_code_file = 'file2.py'
self.create_file_and_commit(self.file2_code_file, 'print("Bobby")')
self.settings.parse_args([
'run', '--stdout', self.file_name2, 'python', self.file2_code_file,
'--not-repro'
])
CmdRun(self.settings).run()
self.file_res_code_file = 'code_res.py'
self.create_file_and_commit(
self.file_res_code_file,
'import sys' + os.linesep + 'text1 = open(sys.argv[1]).read()' +
os.linesep + 'text2 = open(sys.argv[2]).read()' + os.linesep +
'print(text1 + text2)')
self.file_name_res = os.path.join('data', 'file_res')
self.settings.parse_args([
'run', '--stdout', self.file_name_res, '--code',
self.file_res_code_file, 'python', self.file_res_code_file,
self.file_name11, self.file_name2
])
cmd_res = CmdRun(self.settings)
self.assertEqual(cmd_res.code_dependencies, [self.file_res_code_file])
cmd_res.run()
lines = list(
filter(None, map(str.strip,
open(self.file_name_res).readlines())))
self.assertEqual(lines, ['Hello', 'Bobby'])
def test_stage_file_name(self):
fname = 'path/to/file'
outs = [fname, 'dummy']
dvcfile = os.path.basename(fname) + Stage.STAGE_FILE_SUFFIX
ret = CmdRun.stage_file_name(None, [], [])
self.assertEqual(ret, Stage.STAGE_FILE)
ret = CmdRun.stage_file_name(None, [], outs)
self.assertEqual(ret, dvcfile)
ret = CmdRun.stage_file_name(None, outs, [])
self.assertEqual(ret, dvcfile)
ret = CmdRun.stage_file_name(fname, [], [])
self.assertEqual(ret, fname)
def reproduce_data_item(self, changed_files):
Logger.debug('Reproducing data item {}.'.format(self._data_item.data.dvc))
for output_dvc in self._state.output_files:
Logger.debug('Removing output file {} before reproduction.'.format(output_dvc))
try:
data_item = self.cmd_obj.settings.path_factory.existing_data_item_from_dvc_path(output_dvc)
os.remove(data_item.data.relative)
except Exception as ex:
msg = 'Data item {} cannot be removed before reproduction: {}'
Logger.error(msg.format(output_dvc, ex))
changed_files.add(output_dvc)
if self.state.is_import_file:
Logger.debug('Reproducing data item {}. Re-import cmd: {}'.format(
self._data_item.data.relative, ' '.join(self.state.argv)))
if len(self.state.argv) != 2:
msg = 'Data item "{}" cannot be re-imported because of arguments number {} is incorrect. Argv: {}'
raise ReproError(msg.format(self._data_item.data.relative, len(self.state.argv), self.state.argv))
input = self.state.argv[0]
output = self.state.argv[1]
cmd = CmdImportFile(self._settings)
cmd.set_git_action(True)
cmd.set_locker(False)
Logger.info(u'Reproducing import command: {}'.format(output))
if cmd.import_and_commit_if_needed(input, output, lock=True, check_if_ready=False) != 0:
raise ReproError('Import command reproduction failed')
return True
elif self.state.is_run:
cmd = CmdRun(self._settings)
cmd.set_git_action(True)
cmd.set_locker(False)
Logger.info('Reproducing run command for data item {}. Args: {}'.format(
self._data_item.data.relative, ' '.join(self.state.argv)))
data_items_from_args, not_data_items_from_args = self.cmd_obj.argv_files_by_type(self.state.argv)
if cmd.run_and_commit_if_needed(self.state.argv,
data_items_from_args,
not_data_items_from_args,
self.state.stdout,
self.state.stderr,
self.state.shell,
check_if_ready=False) != 0:
raise ReproError('Run command reproduction failed')
return True
else:
# Ignore EMPTY_FILE command
pass
pass
def recreate_file1(self):
self.settings._args = [self.file_name1, '--keep-in-cloud']
CmdDataRemove(self.settings).run()
file1_code_file = 'file1_2.py'
self.create_file_and_commit(
file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")')
self.settings._args = [
'python', file1_code_file, '--not-repro', '--stdout',
self.file_name1, '--code', file1_code_file
]
CmdRun(self.settings).run()
def recreate_file1(self):
self.settings.parse_args('remove {} --keep-in-cloud'.format(
self.file_name1))
CmdRemove(self.settings).run()
file1_code_file = 'file1_2.py'
self.create_file_and_commit(
file1_code_file, 'print("Goodbye")' + os.linesep + 'print("Jack")')
self.settings.parse_args([
'run', '--stdout', self.file_name1, '--code', file1_code_file,
'python', file1_code_file, '--not-repro'
])
CmdRun(self.settings).run()
def recreate_file1(self):
self.settings._args = [self.file_name1, '--keep-in-cloud']
CmdDataRemove(self.settings).run()
self.settings._args = ['printf', 'Goodbye\nJack', '--stdout', self.file_name1]
CmdRun(self.settings).run()
def reproduce_run(self):
Logger.info('Reproducing run command for stage {}. Args: {}'.format(
self.stage.path, self.stage.cmd))
CmdRun.run_command(self.settings, self.stage)