def save(self): try: self.configobj.write() except Exception as exc: Logger.error( "Failed to write config '{}'".format(self.configobj.filename), exc) return 1 return 0
def run_cmd(self): from dvc.project import Project, InitError try: Project.init('.', no_scm=self.args.no_scm, force=self.args.force) except InitError as e: Logger.error('Failed to initiate dvc', e) return 1 return 0
def is_ready_to_go(self): statuses = self.git_file_statuses() if len(statuses) > 0: Logger.error('[dvc-git] Commit all changed files before running reproducible command. Changed files:') for status, file in statuses: Logger.error("{} {}".format(status, file)) return False return True
def unset(self): try: del self.configobj[self.section][self.opt] self.configobj.write() except Exception as exc: Logger.error('Failed to unset \'{}\': {}'.format(self.parsed_args.name, exc)) return 1 return 0
def run(self): for target in self.args.targets: try: outs_only = self._is_outs_only(target) self.project.remove(target, outs_only=outs_only) except DvcException as ex: Logger.error('Failed to remove {}'.format(target), ex) return 1 return 0
def add(self, paths): # NOTE: GitPython is not currently able to handle index version >= 3. # See https://github.com/iterative/dvc/issues/610 for more details. try: self.repo.index.add(paths) except AssertionError as exc: msg = 'Failed to add \'{}\' to git. You can add those files manually using \'git add\'. ' msg += 'See \'https://github.com/iterative/dvc/issues/610\' for more details.' Logger.error(msg.format(str(paths)), exc)
def reproduce_data_item(self, changed_files): Logger.debug('Reproducing data item {}.'.format(self._data_item.data.dvc)) for output_dvc in self._state.output_files: Logger.debug('Removing output file {} before reproduction.'.format(output_dvc)) try: data_item = self.cmd_obj.settings.path_factory.existing_data_item_from_dvc_path(output_dvc) os.remove(data_item.data.relative) except Exception as ex: msg = 'Data item {} cannot be removed before reproduction: {}' Logger.error(msg.format(output_dvc, ex)) changed_files.add(output_dvc) if self.state.is_import_file: Logger.debug('Reproducing data item {}. Re-import cmd: {}'.format( self._data_item.data.relative, ' '.join(self.state.argv))) if len(self.state.argv) != 2: msg = 'Data item "{}" cannot be re-imported because of arguments number {} is incorrect. Argv: {}' raise ReproError(msg.format(self._data_item.data.relative, len(self.state.argv), self.state.argv)) input = self.state.argv[0] output = self.state.argv[1] cmd = CmdImportFile(self._settings) cmd.set_git_action(True) cmd.set_locker(False) Logger.info(u'Reproducing import command: {}'.format(output)) if cmd.import_and_commit_if_needed(input, output, lock=True, check_if_ready=False) != 0: raise ReproError('Import command reproduction failed') return True elif self.state.is_run: cmd = CmdRun(self._settings) cmd.set_git_action(True) cmd.set_locker(False) Logger.info('Reproducing run command for data item {}. Args: {}'.format( self._data_item.data.relative, ' '.join(self.state.argv))) data_items_from_args, not_data_items_from_args = self.cmd_obj.argv_files_by_type(self.state.argv) if cmd.run_and_commit_if_needed(self.state.argv, data_items_from_args, not_data_items_from_args, self.state.stdout, self.state.stderr, self.state.shell, check_if_ready=False) != 0: raise ReproError('Run command reproduction failed') return True else: # Ignore EMPTY_FILE command pass pass
def _import(self, bucket, i, out, item): tmp_file = self.tmp_file(out) try: copyfile(i, tmp_file) os.rename(tmp_file, out) except Exception as exc: Logger.error('Failed to copy "{}": {}'.format(i, exc)) return None return item
def load(self): if not self.state_file or not os.path.isfile(self.state_file): return {} with open(self.state_file, 'r') as fd: try: return json.load(fd) except ValueError as exc: Logger.error('Failed to load \'{}\''.format(self.state_file), exc) return {}
def test_stderr(self, mock_stdout, mock_stderr): error_message = 'error msg' Logger.init() Logger.error(error_message) self.assertEqual('', mock_stdout.getvalue()) self.assertEqual( 'Error: {}\n\nHaving any troubles? ' 'Hit us up at dvc.org/support, we ' 'are always happy to help!\n'.format(error_message), mock_stderr.getvalue())
def _import(self, bucket, i, item): tmp_file = self.tmp_file(item.data.dvc) try: copyfile(i, tmp_file) item.import_cache(tmp_file) except Exception as exc: Logger.error('Failed to copy "{}": {}'.format(i, exc)) return None return item
def set(self): try: self.configobj[self.section][self.opt] = self.parsed_args.value self.configobj.write() except Exception as exc: Logger.error('Failed to set \'{}\' to \'{}\': {}'.format( self.parsed_args.name, self.parsed_args.value, exc)) return 1 return 0
def remove_target(self, target): try: if os.path.isdir(target): self.remove_dir(target) else: self.remove_file(target) return True except DvcException as ex: Logger.error('[Cmd-Remove] Unable to remove data item "{}": {}'.format(target, ex)) return False
def _validate_file_states(repo_change): error = False for data_item in repo_change.removed_data_items: Logger.error('Error: file "{}" was removed'.format(data_item.data.relative)) error = True for file in GitWrapper.abs_paths_to_relative(repo_change.externally_created_files): Logger.error('Error: file "{}" was created outside of the data directory'.format(file)) error = True return not error
def _push_key(self, key, path): """ push, aws version """ try: self._push_multipart(key, path) except Exception as exc: Logger.error('Failed to upload "{}": {}'.format(path, exc)) return None progress.finish_target(os.path.basename(path)) return path
def _unset(self, section, opt=None, configobj=None): if configobj is None: configobj = self.configobj try: self.config.unset(configobj, section, opt) self.config.save(configobj) except DvcException as exc: Logger.error("Failed to unset '{}'".format(self.args.name), exc) return 1 return 0
def _set(self, section, opt, value): try: self.config.set(self.configobj, section, opt, value) self.config.save(self.configobj) except DvcException as exc: Logger.error("Failed to set '{}.{}' to '{}'".format(section, opt, value), exc) return 1 return 0
def sync_dir(self, dir): for file in os.listdir(dir): try: fname = None fname = os.path.join(dir, file) self.sync_object(fname) except DataSyncError as ex: Logger.debug(ex) except Exception as ex: Logger.error('Cannot sync file {}: {}'.format(fname, ex)) pass
def draw(self, g, target, fname): try: A = nx.nx_agraph.to_agraph(g) A.write(fname + '.dot') A.draw(fname + '.svg', prog='dot') except Exception as exc: Logger.error('Failed to draw dependency graph for {}: {}'.format( target, exc)) return 1 return 0
def _push_key(self, key, path): """ push, aws version """ try: self._push_multipart(key, path) except Exception as exc: Logger.error('Failed to upload "{}": {}'.format(path, exc)) return None progress.finish_target( os.path.relpath(path, self._cloud_settings.cache.cache_dir)) return path
def unset(self, section, opt=None): if section not in self.configobj.keys(): Logger.error("Section '{}' doesn't exist".format(section)) return 1 if opt in self.configobj[section].keys(): del self.configobj[section][opt] if len(self.configobj[section]) == 0 or opt == None: del self.configobj[section] return self.save()
def download(self, from_infos, to_infos, no_progress_bar=False, names=None): names = self._verify_path_args(from_infos, to_infos, names) s3 = self.s3 for to_info, from_info, name in zip(to_infos, from_infos, names): if from_info['scheme'] != 's3': raise NotImplementedError if to_info['scheme'] == 's3': self._copy(from_info, to_info, s3=s3) continue if to_info['scheme'] != 'local': raise NotImplementedError msg = "Downloading '{}/{}' to '{}'".format(from_info['bucket'], from_info['key'], to_info['path']) Logger.debug(msg) tmp_file = self.tmp_file(to_info['path']) if not name: name = os.path.basename(to_info['path']) if no_progress_bar: cb = None else: total = s3.head_object(Bucket=from_info['bucket'], Key=from_info['key'])['ContentLength'] cb = Callback(name, total) self._makedirs(to_info['path']) try: s3.download_file(from_info['bucket'], from_info['key'], tmp_file, Callback=cb) except Exception as exc: msg = "Failed to download '{}/{}'".format( from_info['bucket'], from_info['key']) Logger.error(msg, exc) return os.rename(tmp_file, to_info['path']) if not no_progress_bar: progress.finish_target(name)
def _import(self, bucket, i, path): inp = os.path.join(self.storage_path, i) tmp_file = self.tmp_file(path) try: copyfile(inp, tmp_file) except Exception as exc: Logger.error('Failed to copy "{}": {}'.format(i, exc)) return None os.rename(tmp_file, path) return path
def draw(self, g, target, fname_suffix): fname = 'pipeline_' + fname_suffix try: A = nx.nx_agraph.to_agraph(g) A.write(fname + '.dot') A.draw(fname + '.jpeg', format='jpeg', prog='dot') except Exception as exc: Logger.error('Failed to draw dependency graph for {}: {}'.format( target, exc)) return 1 return 0
def _pull_key(self, key, path): self._makedirs(path) tmp_file = self.tmp_file(path) try: copyfile(key.path, tmp_file) except Exception as exc: Logger.error('Failed to copy "{}": {}'.format(key.path, exc)) return None os.rename(tmp_file, path) return path
def check_config(self): # Sanity check to make sure we will be able to commit name = self.git_config_get('user.name') if name == None: Logger.error('[dvc-git] Please setup user.name in git config') return False email = self.git_config_get('user.email') if email == None: Logger.error('[dvc-git] Please setup user.email in git config') return False return True
def is_ready_to_go(self): if not self.is_inside_work_tree(): return False statuses = self.git_file_statuses() if len(statuses) > 0: Logger.error('[dvc-git] Commit all changed files before running ' 'reproducible command. Changed files:') for status, file in statuses: Logger.error("{} {}".format(status, file)) return False return self.check_config()
def repro_target(self, target, recursive, force): if not self.no_git_actions and not self.git.is_ready_to_go(): return 1 data_item_list, external_files_names = self.settings.path_factory.to_data_items(target) if external_files_names: Logger.error('Files from outside of the data directory "{}" could not be reproduced: {}'. format(self.config.data_dir, ' '.join(external_files_names))) return 1 if self.repro_data_items(data_item_list, recursive, force): return 0 return 1
def _push_key(self, key, path): """ push, aws version """ name = os.path.relpath(path, self._cloud_settings.cache.cache_dir) cb = self.create_cb_push(name, path) try: self.s3.Object(key.bucket, key.name).upload_file(path, Callback=cb) except Exception as exc: Logger.error('Failed to upload "{}": {}'.format(path, exc)) return None progress.finish_target(name) return path
def test_empty_exc_msg(self, mock_stdout, mock_stderr, mock_logger): from dvc.exceptions import DvcException Logger.init() err_msg = 'msg' Logger.error(err_msg, exc=DvcException("")) self.assertEqual('', mock_stdout.getvalue()) self.assertEqual( 'Error: {}\n\nHaving any troubles? ' 'Hit us up at dvc.org/support, we ' 'are always happy to help!\n'.format(err_msg), mock_stderr.getvalue())