def run(self): if self.is_locker: lock = fasteners.InterProcessLock(self.git.lock_file) gotten = lock.acquire(timeout=5) if not gotten: Logger.info('Cannot perform the cmd since DVC is busy and locked. Please retry the cmd later.') return 1 good = self.config.sanity_check() if not good[0]: Logger.error('config \'%s\' is not correctly setup. Please fix:' % Runtime.CONFIG) for e in good[1]: Logger.error(' ' + e) return 1 try: for target in self.parsed_args.targets: data_item = self.settings.path_factory.data_item(target) if System.islink(target): self.sync_symlink(data_item) elif os.path.isdir(target): self.sync_dir(target) else: raise DataSyncError('File "{}" does not exit'.format(target)) finally: if self.is_locker: lock.release()
def __init__(self, data_item, cmd_obj, globally_changed_files, recursive, force): self._data_item = data_item self.git = cmd_obj.git self._cmd_obj = cmd_obj self._globally_changed_files = globally_changed_files self._recursive = recursive self._force = force if not System.islink(data_item.data.relative): raise ReproError('data item {} is not symlink'.format( data_item.data.relative)) try: self._state = StateFile.load(data_item.state.relative, self.git) except Exception as ex: raise ReproError( 'Error: state file "{}" cannot be loaded: {}'.format( data_item.state.relative, ex)) if not self.state.argv: raise ReproError( 'Error: parameter {} is not defined in state file "{}"'.format( StateFile.PARAM_ARGV, data_item.state.relative)) if len(self.state.argv) < 1: raise ReproError( 'Error: reproducible cmd in state file "{}" is too short'. format(self.state.file)) self._settings = copy.copy(self._cmd_obj.settings) self._settings.set_args(self.state.argv) pass
def sync_object(self, fname): if os.path.isdir(fname): self.sync_dir(fname) elif System.islink(fname): self.sync_symlink(self.settings.path_factory.data_item(fname)) else: raise DvcException('Unsupported file type "{}"'.format(fname))
def _collect_target(self, target): if System.islink(target): item = self._settings.path_factory.data_item(target) return [item] elif os.path.isdir(target): return self._collect_dir(target) Logger.warn('Target "{}" does not exist'.format(target)) return []
def run(self): with DvcLock(self.is_locker, self.git): self.sync_sanity_check() for target in self.parsed_args.targets: data_item = self.settings.path_factory.data_item(target) if System.islink(target): self.sync_symlink(data_item) elif os.path.isdir(target): self.sync_dir(target) else: raise DataSyncError( 'File "{}" does not exit'.format(target)) pass
def sync_dir(self, dir): for file in os.listdir(dir): try: fname = None fname = os.path.join(dir, file) if os.path.isdir(fname): self.sync_dir(fname) elif System.islink(fname): self.sync_symlink(self.settings.path_factory.data_item(fname)) else: raise DvcException('Unsupported file type "{}"'.format(fname)) except DataSyncError as ex: Logger.debug(ex) except Exception as ex: Logger.error('Cannot sync file {}: {}'.format(fname, ex)) pass
def run(self): with DvcLock(self.is_locker, self.git): cloud = DataCloud(self.settings) targets = [] if len(self.parsed_args.targets) == 0: raise DataSyncError('Sync target is not specified') for target in self.parsed_args.targets: if System.islink(target): targets.append(target) elif os.path.isdir(target): for root, dirs, files in os.walk(target): for f in files: targets.append(os.path.join(root, f)) else: raise DataSyncError( 'File "{}" does not exit'.format(target)) map_progress(cloud.sync, targets, self.parsed_args.jobs) pass
def existing_data_item(self, file): if not System.islink(file): raise DataItemError(u'Data file "%s" must be a symbolic link' % file) resolved_symlink = System.realpath(file) return DataItem(file, self._git, self._config, resolved_symlink)