def _resync_partitions(self, path, block_list_old, block_list_new): partition_resync_handler = self._splitter.get_resync_handler() progress = ProgressActivity(progress_max=self.get_parameter_len(), msg='Writing resyncronized dataset partitions (progress is estimated)') path_tmp = path + '.tmp' try: resync_result = partition_resync_handler.resync(self._splitter, self._reader, block_list_old, block_list_new) DataSplitter.save_partitions(path_tmp, resync_result.partition_iter, progress) except Exception: raise DatasetError('Unable to resync %r' % self.get_datasource_name()) os.rename(path_tmp, path) return (resync_result.pnum_list_redo, resync_result.pnum_list_disable)
def _resync_partitions(self, path, block_list_old, block_list_new): partition_resync_handler = self._splitter.get_resync_handler() progress = ProgressActivity( progress_max=self.get_parameter_len(), msg= 'Writing resyncronized dataset partitions (progress is estimated)') path_tmp = path + '.tmp' try: resync_result = partition_resync_handler.resync( self._splitter, self._reader, block_list_old, block_list_new) DataSplitter.save_partitions(path_tmp, resync_result.partition_iter, progress) except Exception: raise DatasetError('Unable to resync %r' % self.get_datasource_name()) os.rename(path_tmp, path) return (resync_result.pnum_list_redo, resync_result.pnum_list_disable)
def _init_reader(self): # look for aborted inits / resyncs - and try to restore old state if possible if self._exists_data_path('map.tar.resync') and self._exists_data_path('cache.dat.resync'): rename_file(self._get_data_path('cache.dat.resync'), self._get_data_path('cache.dat')) rename_file(self._get_data_path('map.tar.resync'), self._get_data_path('map.tar')) elif self._exists_data_path('map.tar.resync') or self._exists_data_path('cache.dat.resync'): raise DatasetError('Found broken dataset partition resync state in work directory') if self._exists_data_path('map.tar') and not self._exists_data_path('cache.dat'): raise DatasetError('Found broken dataset partition in work directory') elif not self._exists_data_path('map.tar'): # create initial partition map file if not self._exists_data_path('cache.dat'): provider = self._provider else: provider = DataProvider.load_from_file(self._get_data_path('cache.dat')) block_iter = DataProvider.save_to_file_iter(self._get_data_path('cache.dat.init'), provider.get_block_list_cached(show_stats=True)) partition_iter = self._splitter.split_partitions(block_iter) DataSplitter.save_partitions(self._get_data_path('map.tar.init'), partition_iter) rename_file(self._get_data_path('cache.dat.init'), self._get_data_path('cache.dat')) rename_file(self._get_data_path('map.tar.init'), self._get_data_path('map.tar')) return DataSplitter.load_partitions(self._get_data_path('map.tar'))
def _init_reader(self): # look for aborted inits / resyncs - and try to restore old state if possible if self._exists_data_path('map.tar.resync') and self._exists_data_path( 'cache.dat.resync'): rename_file(self._get_data_path('cache.dat.resync'), self._get_data_path('cache.dat')) rename_file(self._get_data_path('map.tar.resync'), self._get_data_path('map.tar')) elif self._exists_data_path( 'map.tar.resync') or self._exists_data_path( 'cache.dat.resync'): raise DatasetError( 'Found broken dataset partition resync state in work directory' ) if self._exists_data_path( 'map.tar') and not self._exists_data_path('cache.dat'): raise DatasetError( 'Found broken dataset partition in work directory') elif not self._exists_data_path('map.tar'): # create initial partition map file if not self._exists_data_path('cache.dat'): provider = self._provider else: provider = DataProvider.load_from_file( self._get_data_path('cache.dat')) block_iter = DataProvider.save_to_file_iter( self._get_data_path('cache.dat.init'), provider.get_block_list_cached(show_stats=True)) partition_iter = self._splitter.split_partitions(block_iter) DataSplitter.save_partitions(self._get_data_path('map.tar.init'), partition_iter) rename_file(self._get_data_path('cache.dat.init'), self._get_data_path('cache.dat')) rename_file(self._get_data_path('map.tar.init'), self._get_data_path('map.tar')) return DataSplitter.load_partitions(self._get_data_path('map.tar'))