def _resync_partitions(self, path, block_list_old, block_list_new):
partition_resync_handler = self._splitter.get_resync_handler()
progress = ProgressActivity(progress_max=self.get_parameter_len(),
msg='Writing resyncronized dataset partitions (progress is estimated)')
path_tmp = path + '.tmp'
try:
resync_result = partition_resync_handler.resync(self._splitter,
self._reader, block_list_old, block_list_new)
DataSplitter.save_partitions(path_tmp, resync_result.partition_iter, progress)
except Exception:
raise DatasetError('Unable to resync %r' % self.get_datasource_name())
os.rename(path_tmp, path)
return (resync_result.pnum_list_redo, resync_result.pnum_list_disable)
def _resync_partitions(self, path, block_list_old, block_list_new):
partition_resync_handler = self._splitter.get_resync_handler()
progress = ProgressActivity(
progress_max=self.get_parameter_len(),
msg=
'Writing resyncronized dataset partitions (progress is estimated)')
path_tmp = path + '.tmp'
try:
resync_result = partition_resync_handler.resync(
self._splitter, self._reader, block_list_old, block_list_new)
DataSplitter.save_partitions(path_tmp,
resync_result.partition_iter,
progress)
except Exception:
raise DatasetError('Unable to resync %r' %
self.get_datasource_name())
os.rename(path_tmp, path)
return (resync_result.pnum_list_redo, resync_result.pnum_list_disable)
def _init_reader(self):
# look for aborted inits / resyncs - and try to restore old state if possible
if self._exists_data_path('map.tar.resync') and self._exists_data_path('cache.dat.resync'):
rename_file(self._get_data_path('cache.dat.resync'), self._get_data_path('cache.dat'))
rename_file(self._get_data_path('map.tar.resync'), self._get_data_path('map.tar'))
elif self._exists_data_path('map.tar.resync') or self._exists_data_path('cache.dat.resync'):
raise DatasetError('Found broken dataset partition resync state in work directory')
if self._exists_data_path('map.tar') and not self._exists_data_path('cache.dat'):
raise DatasetError('Found broken dataset partition in work directory')
elif not self._exists_data_path('map.tar'):
# create initial partition map file
if not self._exists_data_path('cache.dat'):
provider = self._provider
else:
provider = DataProvider.load_from_file(self._get_data_path('cache.dat'))
block_iter = DataProvider.save_to_file_iter(self._get_data_path('cache.dat.init'),
provider.get_block_list_cached(show_stats=True))
partition_iter = self._splitter.split_partitions(block_iter)
DataSplitter.save_partitions(self._get_data_path('map.tar.init'), partition_iter)
rename_file(self._get_data_path('cache.dat.init'), self._get_data_path('cache.dat'))
rename_file(self._get_data_path('map.tar.init'), self._get_data_path('map.tar'))
return DataSplitter.load_partitions(self._get_data_path('map.tar'))
def _init_reader(self):
# look for aborted inits / resyncs - and try to restore old state if possible
if self._exists_data_path('map.tar.resync') and self._exists_data_path(
'cache.dat.resync'):
rename_file(self._get_data_path('cache.dat.resync'),
self._get_data_path('cache.dat'))
rename_file(self._get_data_path('map.tar.resync'),
self._get_data_path('map.tar'))
elif self._exists_data_path(
'map.tar.resync') or self._exists_data_path(
'cache.dat.resync'):
raise DatasetError(
'Found broken dataset partition resync state in work directory'
)
if self._exists_data_path(
'map.tar') and not self._exists_data_path('cache.dat'):
raise DatasetError(
'Found broken dataset partition in work directory')
elif not self._exists_data_path('map.tar'):
# create initial partition map file
if not self._exists_data_path('cache.dat'):
provider = self._provider
else:
provider = DataProvider.load_from_file(
self._get_data_path('cache.dat'))
block_iter = DataProvider.save_to_file_iter(
self._get_data_path('cache.dat.init'),
provider.get_block_list_cached(show_stats=True))
partition_iter = self._splitter.split_partitions(block_iter)
DataSplitter.save_partitions(self._get_data_path('map.tar.init'),
partition_iter)
rename_file(self._get_data_path('cache.dat.init'),
self._get_data_path('cache.dat'))
rename_file(self._get_data_path('map.tar.init'),
self._get_data_path('map.tar'))
return DataSplitter.load_partitions(self._get_data_path('map.tar'))
