def save(self):
try:
self.configobj.write()
except Exception as exc:
Logger.error(
"Failed to write config '{}'".format(self.configobj.filename),
exc)
return 1
return 0
def run_cmd(self):
from dvc.project import Project, InitError
try:
Project.init('.', no_scm=self.args.no_scm, force=self.args.force)
except InitError as e:
Logger.error('Failed to initiate dvc', e)
return 1
return 0
def is_ready_to_go(self):
statuses = self.git_file_statuses()
if len(statuses) > 0:
Logger.error('[dvc-git] Commit all changed files before running reproducible command. Changed files:')
for status, file in statuses:
Logger.error("{} {}".format(status, file))
return False
return True
def unset(self):
try:
del self.configobj[self.section][self.opt]
self.configobj.write()
except Exception as exc:
Logger.error('Failed to unset \'{}\': {}'.format(self.parsed_args.name, exc))
return 1
return 0
def run(self):
for target in self.args.targets:
try:
outs_only = self._is_outs_only(target)
self.project.remove(target, outs_only=outs_only)
except DvcException as ex:
Logger.error('Failed to remove {}'.format(target), ex)
return 1
return 0
def add(self, paths):
# NOTE: GitPython is not currently able to handle index version >= 3.
# See https://github.com/iterative/dvc/issues/610 for more details.
try:
self.repo.index.add(paths)
except AssertionError as exc:
msg = 'Failed to add \'{}\' to git. You can add those files manually using \'git add\'. '
msg += 'See \'https://github.com/iterative/dvc/issues/610\' for more details.'
Logger.error(msg.format(str(paths)), exc)
def reproduce_data_item(self, changed_files):
Logger.debug('Reproducing data item {}.'.format(self._data_item.data.dvc))
for output_dvc in self._state.output_files:
Logger.debug('Removing output file {} before reproduction.'.format(output_dvc))
try:
data_item = self.cmd_obj.settings.path_factory.existing_data_item_from_dvc_path(output_dvc)
os.remove(data_item.data.relative)
except Exception as ex:
msg = 'Data item {} cannot be removed before reproduction: {}'
Logger.error(msg.format(output_dvc, ex))
changed_files.add(output_dvc)
if self.state.is_import_file:
Logger.debug('Reproducing data item {}. Re-import cmd: {}'.format(
self._data_item.data.relative, ' '.join(self.state.argv)))
if len(self.state.argv) != 2:
msg = 'Data item "{}" cannot be re-imported because of arguments number {} is incorrect. Argv: {}'
raise ReproError(msg.format(self._data_item.data.relative, len(self.state.argv), self.state.argv))
input = self.state.argv[0]
output = self.state.argv[1]
cmd = CmdImportFile(self._settings)
cmd.set_git_action(True)
cmd.set_locker(False)
Logger.info(u'Reproducing import command: {}'.format(output))
if cmd.import_and_commit_if_needed(input, output, lock=True, check_if_ready=False) != 0:
raise ReproError('Import command reproduction failed')
return True
elif self.state.is_run:
cmd = CmdRun(self._settings)
cmd.set_git_action(True)
cmd.set_locker(False)
Logger.info('Reproducing run command for data item {}. Args: {}'.format(
self._data_item.data.relative, ' '.join(self.state.argv)))
data_items_from_args, not_data_items_from_args = self.cmd_obj.argv_files_by_type(self.state.argv)
if cmd.run_and_commit_if_needed(self.state.argv,
data_items_from_args,
not_data_items_from_args,
self.state.stdout,
self.state.stderr,
self.state.shell,
check_if_ready=False) != 0:
raise ReproError('Run command reproduction failed')
return True
else:
# Ignore EMPTY_FILE command
pass
pass
def _import(self, bucket, i, out, item):
tmp_file = self.tmp_file(out)
try:
copyfile(i, tmp_file)
os.rename(tmp_file, out)
except Exception as exc:
Logger.error('Failed to copy "{}": {}'.format(i, exc))
return None
return item
def load(self):
if not self.state_file or not os.path.isfile(self.state_file):
return {}
with open(self.state_file, 'r') as fd:
try:
return json.load(fd)
except ValueError as exc:
Logger.error('Failed to load \'{}\''.format(self.state_file), exc)
return {}
def test_stderr(self, mock_stdout, mock_stderr):
error_message = 'error msg'
Logger.init()
Logger.error(error_message)
self.assertEqual('', mock_stdout.getvalue())
self.assertEqual(
'Error: {}\n\nHaving any troubles? '
'Hit us up at dvc.org/support, we '
'are always happy to help!\n'.format(error_message),
mock_stderr.getvalue())
def _import(self, bucket, i, item):
tmp_file = self.tmp_file(item.data.dvc)
try:
copyfile(i, tmp_file)
item.import_cache(tmp_file)
except Exception as exc:
Logger.error('Failed to copy "{}": {}'.format(i, exc))
return None
return item
def set(self):
try:
self.configobj[self.section][self.opt] = self.parsed_args.value
self.configobj.write()
except Exception as exc:
Logger.error('Failed to set \'{}\' to \'{}\': {}'.format(
self.parsed_args.name, self.parsed_args.value, exc))
return 1
return 0
def remove_target(self, target):
try:
if os.path.isdir(target):
self.remove_dir(target)
else:
self.remove_file(target)
return True
except DvcException as ex:
Logger.error('[Cmd-Remove] Unable to remove data item "{}": {}'.format(target, ex))
return False
def _validate_file_states(repo_change):
error = False
for data_item in repo_change.removed_data_items:
Logger.error('Error: file "{}" was removed'.format(data_item.data.relative))
error = True
for file in GitWrapper.abs_paths_to_relative(repo_change.externally_created_files):
Logger.error('Error: file "{}" was created outside of the data directory'.format(file))
error = True
return not error
def _push_key(self, key, path):
""" push, aws version """
try:
self._push_multipart(key, path)
except Exception as exc:
Logger.error('Failed to upload "{}": {}'.format(path, exc))
return None
progress.finish_target(os.path.basename(path))
return path
def _unset(self, section, opt=None, configobj=None):
if configobj is None:
configobj = self.configobj
try:
self.config.unset(configobj, section, opt)
self.config.save(configobj)
except DvcException as exc:
Logger.error("Failed to unset '{}'".format(self.args.name), exc)
return 1
return 0
def _set(self, section, opt, value):
try:
self.config.set(self.configobj, section, opt, value)
self.config.save(self.configobj)
except DvcException as exc:
Logger.error("Failed to set '{}.{}' to '{}'".format(section,
opt,
value),
exc)
return 1
return 0
def sync_dir(self, dir):
for file in os.listdir(dir):
try:
fname = None
fname = os.path.join(dir, file)
self.sync_object(fname)
except DataSyncError as ex:
Logger.debug(ex)
except Exception as ex:
Logger.error('Cannot sync file {}: {}'.format(fname, ex))
pass
def draw(self, g, target, fname):
try:
A = nx.nx_agraph.to_agraph(g)
A.write(fname + '.dot')
A.draw(fname + '.svg', prog='dot')
except Exception as exc:
Logger.error('Failed to draw dependency graph for {}: {}'.format(
target, exc))
return 1
return 0
def _push_key(self, key, path):
""" push, aws version """
try:
self._push_multipart(key, path)
except Exception as exc:
Logger.error('Failed to upload "{}": {}'.format(path, exc))
return None
progress.finish_target(
os.path.relpath(path, self._cloud_settings.cache.cache_dir))
return path
def unset(self, section, opt=None):
if section not in self.configobj.keys():
Logger.error("Section '{}' doesn't exist".format(section))
return 1
if opt in self.configobj[section].keys():
del self.configobj[section][opt]
if len(self.configobj[section]) == 0 or opt == None:
del self.configobj[section]
return self.save()
def download(self,
from_infos,
to_infos,
no_progress_bar=False,
names=None):
names = self._verify_path_args(from_infos, to_infos, names)
s3 = self.s3
for to_info, from_info, name in zip(to_infos, from_infos, names):
if from_info['scheme'] != 's3':
raise NotImplementedError
if to_info['scheme'] == 's3':
self._copy(from_info, to_info, s3=s3)
continue
if to_info['scheme'] != 'local':
raise NotImplementedError
msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'],
from_info['key'],
to_info['path'])
Logger.debug(msg)
tmp_file = self.tmp_file(to_info['path'])
if not name:
name = os.path.basename(to_info['path'])
if no_progress_bar:
cb = None
else:
total = s3.head_object(Bucket=from_info['bucket'],
Key=from_info['key'])['ContentLength']
cb = Callback(name, total)
self._makedirs(to_info['path'])
try:
s3.download_file(from_info['bucket'],
from_info['key'],
tmp_file,
Callback=cb)
except Exception as exc:
msg = "Failed to download '{}/{}'".format(
from_info['bucket'], from_info['key'])
Logger.error(msg, exc)
return
os.rename(tmp_file, to_info['path'])
if not no_progress_bar:
progress.finish_target(name)
def _import(self, bucket, i, path):
inp = os.path.join(self.storage_path, i)
tmp_file = self.tmp_file(path)
try:
copyfile(inp, tmp_file)
except Exception as exc:
Logger.error('Failed to copy "{}": {}'.format(i, exc))
return None
os.rename(tmp_file, path)
return path
def draw(self, g, target, fname_suffix):
fname = 'pipeline_' + fname_suffix
try:
A = nx.nx_agraph.to_agraph(g)
A.write(fname + '.dot')
A.draw(fname + '.jpeg', format='jpeg', prog='dot')
except Exception as exc:
Logger.error('Failed to draw dependency graph for {}: {}'.format(
target, exc))
return 1
return 0
def _pull_key(self, key, path):
self._makedirs(path)
tmp_file = self.tmp_file(path)
try:
copyfile(key.path, tmp_file)
except Exception as exc:
Logger.error('Failed to copy "{}": {}'.format(key.path, exc))
return None
os.rename(tmp_file, path)
return path
def check_config(self):
# Sanity check to make sure we will be able to commit
name = self.git_config_get('user.name')
if name == None:
Logger.error('[dvc-git] Please setup user.name in git config')
return False
email = self.git_config_get('user.email')
if email == None:
Logger.error('[dvc-git] Please setup user.email in git config')
return False
return True
def is_ready_to_go(self):
if not self.is_inside_work_tree():
return False
statuses = self.git_file_statuses()
if len(statuses) > 0:
Logger.error('[dvc-git] Commit all changed files before running '
'reproducible command. Changed files:')
for status, file in statuses:
Logger.error("{} {}".format(status, file))
return False
return self.check_config()
def repro_target(self, target, recursive, force):
if not self.no_git_actions and not self.git.is_ready_to_go():
return 1
data_item_list, external_files_names = self.settings.path_factory.to_data_items(target)
if external_files_names:
Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'.
format(self.config.data_dir, ' '.join(external_files_names)))
return 1
if self.repro_data_items(data_item_list, recursive, force):
return 0
return 1
def _push_key(self, key, path):
""" push, aws version """
name = os.path.relpath(path, self._cloud_settings.cache.cache_dir)
cb = self.create_cb_push(name, path)
try:
self.s3.Object(key.bucket, key.name).upload_file(path, Callback=cb)
except Exception as exc:
Logger.error('Failed to upload "{}": {}'.format(path, exc))
return None
progress.finish_target(name)
return path
def test_empty_exc_msg(self, mock_stdout, mock_stderr, mock_logger):
from dvc.exceptions import DvcException
Logger.init()
err_msg = 'msg'
Logger.error(err_msg, exc=DvcException(""))
self.assertEqual('', mock_stdout.getvalue())
self.assertEqual(
'Error: {}\n\nHaving any troubles? '
'Hit us up at dvc.org/support, we '
'are always happy to help!\n'.format(err_msg),
mock_stderr.getvalue())
