def sync_from_block_store(self, root_block_id, src_block_store, **kwargs) :
if self.__handle__ is not None :
raise Exception("key value store must be closed to sync")
default_minimum_duration = pconfig.shared_configuration(['Replication', 'MinimumDuration'], 5)
minimum_duration = kwargs.get('minimum_duration', default_minimum_duration)
default_duration = pconfig.shared_configuration(['Replication', 'Duration'], 60)
duration = kwargs.get('duration', default_duration)
try :
root_block = src_block_store.get_block(root_block_id)
except Exception :
logger.exception('failed to get root block from remote kv store')
raise
try :
root_block = root_block.decode('utf8')
except AttributeError :
pass
root_block = root_block.rstrip('\0')
root_block_json = json.loads(root_block)
block_ids = [root_block_id] + root_block_json['BlockIds']
# check to see which blocks need to be pushed
blocks_to_extend = []
blocks_to_pull = []
for block_id in block_ids :
block_status = self.__check_block__(block_id)
# if the size is 0 then the block is unknown to the storage service
if block_status['size'] == 0 :
blocks_to_pull.append(block_id)
# if the expiration is nearing, then add to the list to extend, the
# policy here is to extend if the block is within 5 seconds of expiring
elif block_status['duration'] < minimum_duration :
blocks_to_extend.append(block_id)
# there is currently no operation to simply extend the expiration of
# an existing block, so for now just add the blocks to extend onto
# the end of the blocks to push
blocks_to_pull += blocks_to_extend
if len(blocks_to_pull) == 0 :
return 0
block_data_list = src_block_store.get_blocks(blocks_to_pull)
for block_data in block_data_list :
self.__store_block__(block_data, input_encoding='raw')
# if block_store_list is None :
# raise Exception('failed to push blocks to block_store')
self.hash_identity = root_block_id
return len(blocks_to_pull)
def build_file_name(basename, data_dir=None, data_sub=None, extension=''):
"""build a file name from the basename and directory; this is a
common operation for scripts that process a configuration file
:param str basename: base name of a file, may be a full path, may have an extension
:param str data_dir: directory where the file will be placed
:param str data_sub: subdirectory where the files of this type are stored
:param str extension: the extension to add to the file if it doesnt have one
"""
global __default_data_directory__
if data_dir is None:
if __default_data_directory__ is None:
import pdo.common.config as pconfig
__default_data_directory__ = pconfig.shared_configuration(
['Contract', 'DataDirectory'], "./data")
data_dir = __default_data_directory__
if data_sub is not None:
data_dir = os.path.join(data_dir, data_sub)
# os.path.abspath only works for full paths, not relative paths
# this check should catch './abc'
if os.path.split(basename)[0]:
return os.path.realpath(basename)
if basename[-len(extension):] == extension:
return os.path.join(data_dir, basename)
else:
return os.path.join(data_dir, basename + extension)
def local_block_manager():
global __local_block_manager__
if __local_block_manager__ is None:
block_store_file = pconfig.shared_configuration(
['StorageService', 'BlockStore'], "./blockstore.mdb")
__local_block_manager__ = BlockStoreManager(block_store_file, True)
return __local_block_manager__
def KeyValueInitialize(block_store_file = None) :
global __block_store_initialized__
if __block_store_initialized__ :
raise Exception("duplicate block store initialization")
if block_store_file is None :
block_store_file = pconfig.shared_configuration(['StorageService', 'KeyValueStore'], "./keyvalue.mdb")
kvs.SetLogger(logger)
kvs.block_store_open(block_store_file)
__block_store_initialized__ = True
def set_replication_parameters(self,
num_provable_replicas=None,
availability_duration=None,
**kwargs):
# pull the defaults from the configuration if they are not
# otherwise set by the caller
if num_provable_replicas is None:
num_provable_replicas = pconfig.shared_configuration(
['Replication', 'NumProvableReplicas'], 1)
if availability_duration is None:
availability_duration = pconfig.shared_configuration(
['Replication', 'Duration'], 120)
self.replication_params = dict()
self.replication_params[
'num_provable_replicas'] = num_provable_replicas
self.replication_params[
'availability_duration'] = availability_duration #seconds
# we replicate to storage services associated with all provisioned encalves
self.replication_params['service_ids'] = self.enclave_map.keys()
def sync_block_store(src_block_store,
dst_block_store,
root_block_id,
root_block=None,
**kwargs):
"""
ensure that required blocks are stored in the storage service
assumes that all of the blocks referenced by root_block_id are in the source
block manager
:param src_block_store object implementing the block_store_manager interface
:param dst_block_store object implementing the block_store_manager interface
:param root_block_id string: block identifier for the root block
:param root_block string: block data for the root block
"""
if root_block is None:
root_block = src_block_store.get_block(root_block_id)
block_ids = [root_block_id]
try:
root_block = root_block.decode('utf8')
except AttributeError:
pass
root_block = root_block.rstrip('\0')
root_block_json = json.loads(root_block)
block_ids.extend(root_block_json['BlockIds'])
default_minimum_duration = pconfig.shared_configuration(
['Replication', 'MinimumDuration'], 5)
minimum_duration = kwargs.get('minimum_duration', default_minimum_duration)
default_duration = pconfig.shared_configuration(
['Replication', 'Duration'], 60)
duration = kwargs.get('duration', default_duration)
# check to see which blocks need to be pushed
blocks_to_push = []
blocks_to_extend = []
block_status_list = dst_block_store.check_blocks(block_ids)
for block_status in block_status_list:
# if the size is 0 then the block is unknown to the storage service
if block_status['size'] == 0:
blocks_to_push.append(block_status['block_id'])
# if the expiration is nearing, then add to the list to extend, the
# policy here is to extend if the block is within 5 seconds of expiring
elif block_status['duration'] < minimum_duration:
blocks_to_extend.append(block_status['block_id'])
# there is currently no operation to simply extend the expiration of
# an existing block, so for now just add the blocks to extend onto
# the end of the blocks to push
blocks_to_push += blocks_to_extend
if len(blocks_to_push) == 0:
return 0
block_data_list = src_block_store.get_blocks(blocks_to_push)
block_store_list = dst_block_store.store_blocks(block_data_list,
duration=duration)
if block_store_list is None:
raise Exception('failed to push blocks to block_store')
return len(blocks_to_push)