class TestNullDriver(unittest.TestCase):
def setUp(self):
self.null_driver = ECDriver(library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
def tearDown(self):
pass
def test_null_driver(self):
self.null_driver.encode("")
self.null_driver.decode([])
class TestNullDriver(unittest.TestCase):
def setUp(self):
self.null_driver = ECDriver(
library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
def tearDown(self):
pass
def test_null_driver(self):
self.null_driver.encode('')
self.null_driver.decode([])
class Eraser(object):
"""A wrapper for pyeclib erasure coding driver (ECDriver)"""
def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
self.k = k
self.m = m
self.ec_type = ec_type
if EC_TYPE == "pylonghair":
self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
elif EC_TYPE == "striping" or EC_TYPE == "bypass":
self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
else:
self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
def encode(self, data):
"""Encode a string of bytes in flattened string of byte strips"""
raw_strips = self.driver.encode(data)
strips = []
for raw_strip in raw_strips:
strip = Strip()
strip.data = raw_strip
strips.append(strip)
return strips
def decode(self, playcloud_strips):
"""Decode byte strips in a string of bytes"""
strips = convert_strips_to_bytes_list(playcloud_strips)
return self.driver.decode(strips)
class Eraser(object):
"""A wrapper for pyeclib erasure coding driver (ECDriver)"""
def __init__(self, k=8, m=2, ec_type="liberasurecode_rs_vand"):
self.k = k
self.m = m
self.ec_type = ec_type
if EC_TYPE == "pylonghair":
self.driver = PylonghairDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
elif EC_TYPE == "striping" or EC_TYPE == "bypass":
self.driver = ECStripingDriver(k=EC_K, m=0, hd=None)
else:
self.driver = ECDriver(k=EC_K, m=EC_M, ec_type=EC_TYPE)
def encode(self, data):
"""Encode a string of bytes in flattened string of byte strips"""
raw_strips = self.driver.encode(data)
strips = []
for raw_strip in raw_strips:
strip = Strip()
strip.data = raw_strip
strips.append(strip)
return strips
def decode(self, playcloud_strips):
"""Decode byte strips in a string of bytes"""
strips = convert_strips_to_bytes_list(playcloud_strips)
return self.driver.decode(strips)
def decode(k, m, base_filename_to_decode, ec_type):
#print("k = %d, m = %d" % (k, m))
#print("ec_type = %s" % ec_type)
home = expanduser("~")
Disk_base_dir = home + '/Disk'
key_list = base_filename_to_decode.split('.')
jdecoder_types = [
"reed_sol_van", "reed_sol_r6_op", "cauchy_orig", "cauchy_good",
"liberation", "blaum_roth", "liber8tion"
]
if ec_type in jdecoder_types:
print "Decoding started "
jdecoder = ctypes.CDLL(
'/home/raven/Downloads/Jerasure-master/Examples/jdecoder.so')
key = key_list[0]
decoded_data = str(jdecoder.decode(key, Disk_base_dir))
print "decoded data = ", decoded_data
else:
ec_driver = ECDriver(k=k, m=m, ec_type=ec_type)
fragment_list = []
i = 0
while i <= k:
if os.path.exists(Disk_base_dir + "/" + "Disk_" + str(i)):
curr_dir = Disk_base_dir + "/" + "Disk_" + str(i)
if i == 0:
if os.path.exists(curr_dir + "/" +
base_filename_to_decode):
with open(curr_dir + "/" + base_filename_to_decode,
'rb') as fp:
decoded_data = fp.read()
fp.close()
return decoded_data
else:
if os.path.exists(curr_dir + "/" +
base_filename_to_decode + "_fragment_" +
str(i)):
with open(
curr_dir + "/" + base_filename_to_decode +
"_fragment_" + str(i), 'rb') as fp:
fragment = fp.read()
fp.close()
fragment_list.append(fragment)
i = i + 1
if len(fragment_list) < k:
return -1 # Not enough fragments to decode
else:
decoded_data = ec_driver.decode(fragment_list)
return decoded_data
def decode(k,m,base_filename_to_decode,ec_type) :
#print("k = %d, m = %d" % (k, m))
#print("ec_type = %s" % ec_type)
home = expanduser("~")
Disk_base_dir = home + '/Disk'
key_list = base_filename_to_decode.split('.')
jdecoder_types = ["reed_sol_van", "reed_sol_r6_op","cauchy_orig","cauchy_good","liberation","blaum_roth","liber8tion"]
if ec_type in jdecoder_types :
print "Decoding started "
jdecoder = ctypes.CDLL('/home/raven/Downloads/Jerasure-master/Examples/jdecoder.so')
key = key_list[0]
decoded_data = str(jdecoder.decode(key,Disk_base_dir))
print "decoded data = ", decoded_data
else :
ec_driver = ECDriver(k = k, m = m, ec_type = ec_type)
fragment_list = []
i = 0
while i <=k :
if os.path.exists(Disk_base_dir + "/" + "Disk_" + str(i)) :
curr_dir = Disk_base_dir + "/" + "Disk_" + str(i)
if i == 0 :
if os.path.exists(curr_dir + "/" + base_filename_to_decode) :
with open(curr_dir + "/" + base_filename_to_decode,'rb') as fp :
decoded_data = fp.read()
fp.close()
return decoded_data
else :
if os.path.exists(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i)) :
with open(curr_dir + "/" + base_filename_to_decode + "_fragment_" + str(i),'rb') as fp :
fragment = fp.read()
fp.close()
fragment_list.append(fragment)
i = i + 1
if len(fragment_list) < k :
return -1 # Not enough fragments to decode
else :
decoded_data = ec_driver.decode(fragment_list)
return decoded_data
class Eraser(object):
"""A wrapper for pyeclib erasure coding driver (ECDriver)"""
def __init__(self,
ec_k,
ec_m,
ec_type="liberasurecode_rs_vand",
aes_enabled=True):
self.ec_type = ec_type
if aes_enabled:
self.aes = AESDriver()
logger.info("Eraser will use AES encryption")
else:
logger.info("Eraser will not use AES encryption")
expected_module_name = "drivers." + ec_type.lower() + "_driver"
expected_class_name = ec_type[0].upper() + ec_type[1:].lower(
) + "Driver"
try:
mod = __import__(expected_module_name,
fromlist=[expected_class_name])
driver_class = None
driver_class = getattr(mod, expected_class_name)
self.driver = driver_class(k=ec_k,
m=ec_m,
ec_type=ec_type,
hd=None)
except (ImportError, AttributeError):
logger.exception("Driver " + ec_type +
" could not be loaded as a custom driver")
try:
self.driver = ECDriver(k=ec_k, m=ec_m, ec_type=ec_type)
except Exception as error:
logger.exception("Driver " + ec_type +
" could not be loaded by pyeclib")
raise error
def encode(self, data):
"""Encode a string of bytes in flattened string of byte strips"""
payload = data
if hasattr(self, 'aes'):
payload = self.aes.encode(data)[0]
strips = self.driver.encode(payload)
return strips
def decode(self, strips):
"""Decode byte strips in a string of bytes"""
payload = self.driver.decode(strips)
if hasattr(self, 'aes'):
return self.aes.decode([payload])
return payload
def reconstruct(self, available_payload_fragments, missing_indices):
"""
Reconstruct missing fragments of data
Args:
available_payload_fragments(list(bytes)): Available fragments of data
missing_indices(list(int)): List of the indices of the missing blocks
Returns:
list(bytes): A list of the reconstructed fragments
"""
return self.driver.reconstruct(available_payload_fragments,
missing_indices)
def fragments_needed(self, missing_indices):
"""
Return a list of the fragments needed to recover the missing ones
Args:
missing_indices(list(int)): The list of indices of the fragments to recover
Returns:
list(int): A list of the indices of the fragments required to recover the missing ones
"""
return self.driver.fragments_needed(missing_indices)
k = 0
#for i in range(0, k + m, 1):
start_time = 0
end_time = 0
downed_nodes = []
for i in range(0, num_stripes, 1):
frags = []
for j in range(args.m+args.k):
try:
f = open("./nodes/%d/%s.%d.%d" % (j, args.filename, j, i), 'rb')
#print j
frags.append(f.read())
except IOError:
if j not in downed_nodes:
print("Logging data loss at node %d, notify name_node of downed node" % j)
downed_nodes.append(j)
start_time = timeit.default_timer()
dec_chunk = ec_driver.decode(frags)
end_time += timeit.default_timer()-start_time
new_file.write(dec_chunk)
print("Decoding time: %f" % end_time)
from pyeclib.ec_iface import ECDriver
import pickle
from random import shuffle
if __name__ == '__main__':
with open('results.pkl', 'rb') as f:
results_dict = pickle.load(f)
for key, frags in results_dict.items():
k, m = key
driver = ECDriver(ec_type='jerasure_rs_vand', k=k, m=m)
for num in range(10):
shuffle(frags)
assert 'a'*100 == driver.decode(
frags[:k], force_metadata_checks=True)
print '%s passed' % str(key)
class Client:
def __init__(self, cfgfile):
self.hw_addr = self.get_hwaddr()
self.uid = (-1, self.hw_addr)
self.loop = asyncio.get_event_loop()
config = configparser.ConfigParser()
config.read(cfgfile)
nbr_of_servers = int(config['General']['n'])
chunks_size = int(config['General']['chunks_size'])
f = int(config['General']['f'])
e = int(config['General']['e'])
k = nbr_of_servers - 2 * (f + e)
if (k < 1):
raise Exception("Coded elements less than 1")
if (nbr_of_servers + k > 32):
raise Exception(
"[liberasurecode] Total number of fragments (k + m) must be at most 32"
)
quorum_size = math.ceil((nbr_of_servers + k + 2 * e) / 2)
self.majority = math.ceil((nbr_of_servers + 1) / 2)
self.ec_driver = ECDriver(k=k,
m=nbr_of_servers,
ec_type='liberasurecode_rs_vand')
self.p = QuorumSend(quorum_size, PingPongMessage)
t = Thread(target=self.start_event_loop,
args=(self.loop, config['Nodes'], chunks_size))
t.daemon = True
t.start()
def get_hwaddr(self):
with open('/sys/class/net/lo/address') as f:
hw_addr = f.read().splitlines()[0]
if (hw_addr == '00:00:00:00:00:00'):
hw_addr = ':'.join(['%02x'] * 6) % (
r.randint(0, 255), r.randint(0, 255), r.randint(0, 255),
r.randint(0, 255), r.randint(0, 255), r.randint(0, 255))
return hw_addr
def start_event_loop(self, loop, nodes, chunks_size):
asyncio.set_event_loop(loop)
i = 0
for node in nodes:
ip, port = nodes[node].split(':')
c = SenderChannel(i,
self.hw_addr,
0,
self.p,
ip,
port,
chunks_size=chunks_size)
asyncio.ensure_future(c.start())
print("Create channel to {}:{}".format(ip, port))
i = i + 1
loop.run_forever()
def qrmAccess(self, msg, opt_size=None):
fut = asyncio.run_coroutine_threadsafe(self.p.phaseInit(msg, opt_size),
self.loop)
return fut.result()
def write(self, msg):
elements = self.ec_driver.encode(msg)
res = self.qrmAccess((None, None, 'qry', 'write'))
max_tag = max([x.get_tag() for x in res])
new_int = int(max_tag[0]) + 1
new_tag = (new_int, self.uid)
self.qrmAccess((new_tag, elements, 'pre', 'write'))
self.qrmAccess((new_tag, None, 'fin', 'write'))
self.qrmAccess((new_tag, None, 'FIN', 'write'))
def read(self):
res = self.qrmAccess((None, None, 'qry', 'read'))
max_tag = max([x.get_tag() for x in res])
res = self.qrmAccess((max_tag, None, 'fin', 'read'))
elements = [x.get_data() for x in res if x.get_data()]
try:
decoded_msg = self.ec_driver.decode(elements)
except:
decoded_msg = None
return decoded_msg
def check_uid(self):
res = self.qrmAccess((None, None, 'cntrQry', None),
opt_size=self.majority)
max_cntr = max([struct.unpack("i", x.get_data()) for x in res])[0]
if (max_cntr != self.uid[0]):
new_inc_nbr = max(max_cntr, self.uid[0]) + 1
new_cntr = struct.pack("i", new_inc_nbr)
self.qrmAccess((None, new_cntr, 'incCntr', None),
opt_size=self.majority)
self.uid = (new_inc_nbr, self.hw_addr)
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('fragments',
metavar='fragment',
nargs='+',
help='fragments to decode')
parser.add_argument('filename', help='output file')
args = parser.parse_args()
print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("fragments = %s" % args.fragments)
print("filename = %s" % args.filename)
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
fragment_list = []
# read fragments
for fragment in args.fragments:
with open(("%s" % fragment), "rb") as fp:
fragment_list.append(fp.read())
# decode
decoded_file = ec_driver.decode(fragment_list)
# write
with open("%s.decoded" % args.filename, "wb") as fp:
fp.write(decoded_file)
class StepEntangler(object):
"""
Basic implementation of STeP based entanglement
"""
# Use the Group Separator from the ASCII table as the delimiter between the
# entanglement header and the data itself
# https://www.lammertbies.nl/comm/info/ascii-characters.html
HEADER_DELIMITER = chr(29)
def __init__(self, source, s, t, p, ec_type="isa_l_rs_vand"):
"""
StepEntangler constructor
Args:
source(Source): Block source implementing the get_random_blocks and
get_block primitives
s(int): Number of source blocks or the number of chunks to make from
the original data
t(int): Number of old blocks to entangle with
p(int): Number of parity blocks to produce using Reed-Solomon
"""
if not source or \
not callable(getattr(source, "get_block", None)) or \
not callable(getattr(source, "get_random_blocks", None)):
raise ValueError(
"source argument must implement a get_random_blocks and a get_block method"
)
if s <= 0:
raise ValueError("s({:d}) must be greater or equal to 1".format(s))
if t < 0:
raise ValueError("t({:d}) must be greater or equal to 0".format(t))
if p < s:
raise ValueError(
"p({:d}) must be greater or equal to s({:d})".format(p, s))
self.s = s
self.t = t
self.p = p
self.e = self.p - self.s
self.k = s + t
self.source = source
self.driver = ECDriver(k=self.k, m=self.p, ec_type=ec_type)
@staticmethod
def __get_original_size_from_strip(strip):
"""
Returns the size of the original data located
Args:
strip(bytes): The bytes containing the header, size and data
Returns:
int: The size of the original data
"""
start = strip.find(EntanglementDriver.HEADER_DELIMITER) +\
len(EntanglementDriver.HEADER_DELIMITER)
end = strip.find(EntanglementDriver.HEADER_DELIMITER, start)
return int(strip[start:end])
@staticmethod
def __get_data_from_strip(strip):
"""
Returns the data part of the bytes strip
Args:
strip(bytes): The bytes containing the header and the data
Returns:
bytes: The data part of the strip
"""
first_pos = strip.find(EntanglementDriver.HEADER_DELIMITER) +\
len(EntanglementDriver.HEADER_DELIMITER)
pos = strip.find(EntanglementDriver.HEADER_DELIMITER, first_pos) +\
len(EntanglementDriver.HEADER_DELIMITER)
return strip[pos:]
@staticmethod
def compute_fragment_size(document_size, fragments):
"""
Computes the fragment size that will be used to process a document of size
`document_size`.
Args:
document_size(int): Document size in bytes
fragments(int): Number of fragments
Returns:
int: The required fragment size in bytes
Raises:
ValueError: if the document size argument is not an integer or lower than 0
"""
if not isinstance(document_size, int) or document_size < 0:
raise ValueError(
"document_size argument must be an integer greater or equal to 0"
)
if not isinstance(fragments, int) or fragments <= 0:
raise ValueError(
"fragments argument must be an integer greater than 0")
fragment_size = int(math.ceil(document_size / float(fragments)))
if (fragment_size % 2) == 1:
fragment_size += 1
return fragment_size
def encode(self, data):
"""
Encodes data using combining entanglemend and Reed-Solomon(n, k)
where k = s + t and n = k + p.
Args:
data(bytes): The original data to encode
Returns:
list(bytes): The encoded bytes to store
"""
pointer_blocks = []
if self.t > 0:
pointer_blocks = self.source.get_random_blocks(self.t)
block_header = serialize_entanglement_header(pointer_blocks)
size = len(data)
fragment_size = StepEntangler.compute_fragment_size(size, self.s)
padded_size = fragment_size * self.s
padded_data = pad(data, padded_size)
pointer_blocks = [
pad(self.__get_data_from_strip(block.data)[80:], fragment_size)
for block in pointer_blocks
]
encoded = self.entangle(padded_data, pointer_blocks)
parity_blocks = [
block_header + self.HEADER_DELIMITER + str(size) +
self.HEADER_DELIMITER + parity_block
for parity_block in encoded[self.k:]
]
return parity_blocks
def entangle(self, data, blocks):
"""
Performs entanglement combining the data and the extra blocks using
Reed-Solomon.
Args:
data(bytes): The original piece of data
blocks(list(bytes)): The pointer blocks
Returns:
list(bytes): The parity blocks produced by combining the data and
pointer blocks and running them through Reed-Solomon
encoding
"""
return self.driver.encode(data + "".join(blocks))
def fetch_and_prep_pointer_blocks(self, pointers, fragment_size,
original_data_size):
"""
Fetches the pointer blocks and rewrites their liberasurecode header so
that they can be reused for reconstruction or decoding
Args:
pointer_blocks(list(list)): A list of 2 elements lists namely the
filename and the index of each pointer
block
fragment_size(int): Size of each fragment
original_data_size(int): Size of the original piece of data
Returns:
list(bytes): A list of cleaned up liberasurecode fragments formatted
and padded to fit the code
Raises:
ValueError: If the pointers argument is not of type list,
The fragment_size argument is not an int or is lower or
equal to 0,
The original_data_size argument is not an int or is lower
or equal to 0
"""
if not isinstance(pointers, list):
raise ValueError("pointers argument must be of type list")
if not isinstance(fragment_size, int) or fragment_size <= 0:
raise ValueError(
"fragment_size argument must be an integer greater than 0")
if not isinstance(original_data_size, int) or original_data_size <= 0:
raise ValueError(
"original_data_size argument must be an integer greater than 0"
)
if original_data_size < fragment_size:
raise ValueError(
"original_data_size must be greater or equal to fragment_size")
pointer_collection = {}
fetchers = []
for pointer_index, coordinates in enumerate(pointers):
path = coordinates[0]
index = coordinates[1]
fragment_index = self.s + pointer_index
fetcher = PointerHandler(self.source, path, index, fragment_index,
fragment_size, original_data_size,
pointer_collection)
fetcher.start()
fetchers.append(fetcher)
for fetcher in fetchers:
fetcher.join()
return [
pointer_collection[key]
for key in sorted(pointer_collection.keys())
]
def decode(self, strips, path=None):
"""
Decodes data using the entangled blocks and Reed-Solomon.
Args:
strips(list(bytes)): The encoded strips of data
Returns:
bytes: The decoded data
Raises:
ECDriverError: if the number of fragments is too low for Reed-Solomon
decoding
"""
logger = logging.getLogger("entangled_driver")
model_fragment_header = FragmentHeader(
self.__get_data_from_strip(strips[0])[:80])
fragment_size = model_fragment_header.metadata.size
orig_data_size = model_fragment_header.metadata.orig_data_size
modified_pointer_blocks = []
original_data_size = self.__get_original_size_from_strip(strips[0])
block_header_text = get_entanglement_header_from_strip(strips[0])
strips = [self.__get_data_from_strip(strip) for strip in strips]
if self.t > 0:
block_header = parse_entanglement_header(block_header_text)
modified_pointer_blocks = self.fetch_and_prep_pointer_blocks(
block_header, fragment_size, orig_data_size)
# Filter to see what pointers we were able to fetch from the proxy
initial_length = len(modified_pointer_blocks)
modified_pointer_blocks = [
mpb for mpb in modified_pointer_blocks if mpb
]
filtered_length = len(modified_pointer_blocks)
if filtered_length != initial_length:
logger.warn("Only found {:d} pointers out of {:d}".format(
filtered_length, initial_length))
biggest_index = max(
[FragmentHeader(s[:80]).metadata.index for s in strips])
missing = initial_length - filtered_length
if missing > self.e:
message = "Configuration of Step (s={:d}, t={:d}, e={:d}, p={:d}) " + \
"does not allow for reconstruction with {:d} missing fragments"
raise ECDriverError(
message.format(self.s, self.t, self.e, self.p,
missing))
extra_parities_needed = [
index - self.k
for index in xrange(biggest_index + 1, biggest_index + 1 +
missing, 1)
]
logger.info("We need blocks {} from {:s}".format(
sorted(extra_parities_needed), path))
for index in extra_parities_needed:
strips.append(
self.__get_data_from_strip(
self.source.get_block(path, index).data))
decoded = self.disentangle(strips, modified_pointer_blocks)
return decoded[:original_data_size]
def disentangle(self, parity_blocks, pointer_blocks):
"""
Performs disentanglement in order to reconstruct and decode the original
data.
Args:
parity_blocks(bytes): The parity blocks produced from the original encoding
pointer_blocks(bytes): The blocks used in the original entanglement adjusted to the right size
Returns:
bytes: The data is it was originally mixed with the pointer blocks before encoding
"""
available_blocks = pointer_blocks + parity_blocks
return self.driver.decode(available_blocks)
def fragments_needed(self, missing_fragment_indexes):
"""
Returns the list of fragments necessary for decoding/reconstruction of data
Args:
missing_fragment_indexes(list(int)): The list of missing fragments
Returns:
list(int): The list of fragments required for decoding/reconstruction
Raises:
ECDriverError: If the number of missing indexes to work around is
greater than self.p - self.s
ValueError: if one of the missing indexes is out of scope
(index < 0 || (self.s + self.s + self.p) <= index)
"""
if self.e == 0:
message = (
"Configuration of Step (s={:d}, t={:d}, e={:d}, p={:d}) does not allow for reconstruction"
.format(self.s, self.t, self.e, self.p))
raise ECDriverError(message)
if self.e < len(missing_fragment_indexes):
message = (
"Configuration of Step (s={:d}, t={:d}, e={:d}, p={:d}) does not allow for reconstruction of {:d} missing blocks"
.format(self.s, self.t, self.e, self.p,
len(missing_fragment_indexes)))
raise ECDriverError(message)
for index in missing_fragment_indexes:
if index < 0 or (self.s + self.t + self.p) <= index:
raise ValueError(
"Index {:d} is out of range(0 <= index < {:d})".format(
index, self.s + self.t + self.p))
required_indices = []
for index in xrange(self.k, self.k + self.p):
if not index in missing_fragment_indexes:
required_indices.append(index)
if len(required_indices) == self.s:
break
return required_indices
def reconstruct(self, available_fragment_payloads,
missing_fragment_indexes):
"""
Reconstruct the missing fragements
Args:
list(bytes): Avalaible fragments
list(int): Indices of the missing fragments
Returns:
list(bytes): The list of reconstructed blocks
"""
header_text = get_entanglement_header_from_strip(
available_fragment_payloads[0])
list_of_pointer_blocks = parse_entanglement_header(header_text)
parity_header = FragmentHeader(
self.__get_data_from_strip(available_fragment_payloads[0])[:80])
data_size = self.__get_original_size_from_strip(
available_fragment_payloads[0])
parity_blocks = [
self.__get_data_from_strip(block)
for block in available_fragment_payloads
]
missing_fragment_indexes = [
index + self.s + self.t for index in missing_fragment_indexes
]
# Get pointer blocks
modified_pointer_blocks = self.fetch_and_prep_pointer_blocks(
list_of_pointer_blocks, parity_header.metadata.size,
parity_header.metadata.orig_data_size)
# Filter to remove responses for pointers that are missing
modified_pointer_blocks = [
mpb for mpb in modified_pointer_blocks if mpb
]
assembled = modified_pointer_blocks + parity_blocks
reconstructed = self.driver.reconstruct(assembled,
missing_fragment_indexes)
requested_blocks = []
for index, block in enumerate(reconstructed):
requested_block = header_text + self.HEADER_DELIMITER + str(
data_size) + self.HEADER_DELIMITER + block
requested_blocks.append(requested_block)
return requested_blocks
def __repr__(self):
return "StepEntangler(s=" + str(self.s) + ", t=" + str(
self.t) + ", p=" + str(self.p) + ")"
class cas:
def __init__(self, cfgfile, verbose=False):
uid = self.get_hwaddr()
config = configparser.ConfigParser()
config.read(cfgfile)
N = int(config['General']['n'])
F = int(config['General']['f'])
addrs = list(config['Nodes'].values())
self.verbose = verbose
context = zmq.Context()
self.k = N-2*F
self.ec_driver = ECDriver(k=self.k, m=N, ec_type='liberasurecode_rs_vand')
self.socketid = {}
self.sockets = []
self.uid = uid
self.id = "Client-{}".format(uid)
self.poller = zmq.Poller()
self.quorum = int(math.ceil((N+self.k)/2))
print("Starting client {}".format(uid))
for addr in addrs:
socket = context.socket(zmq.DEALER)
self.socketid[socket] = "Server-%s" % addr
self.poller.register(socket, zmq.POLLIN)
socket.connect("tcp://%s" % addr);
self.vprint("Connecting to server with address {}".format(addr))
self.sockets.append(socket)
def get_hwaddr(self):
with open('/sys/class/net/lo/address') as f:
hw_addr = f.read().splitlines()[0]
if (hw_addr == '00:00:00:00:00:00'):
hw_addr = ':'.join(['%02x']*6) % (
r.randint(0, 255),
r.randint(0, 255),
r.randint(0, 255),
r.randint(0, 255),
r.randint(0, 255),
r.randint(0, 255)
)
return hw_addr
def vprint(self, *args, **kwargs):
if self.verbose:
print(*args, **kwargs)
def createMessage(self, msg):
elements = self.ec_driver.encode(msg)
return elements
def assembleMessage(self, elements):
decoded_msg = self.ec_driver.decode(elements)
return decoded_msg
#Send different coded element to everyone
def sendCodedElements(self, msg, tag, msg_id):
elements = self.createMessage(msg)
for i in range(len(self.sockets)):
payload = (tag, 0, 'pre')
msg = ["PREWRITE", str(payload), msg_id, str(self.uid)]
bmsg = [e.encode() for e in [""]+msg] + [elements[i]]
self.sockets[i].send_multipart(bmsg)
# mon_msg = str(["PREWRITE", str(payload), msg_id])
self.log(bmsg[1:], "s", self.socketid[self.sockets[i]])
def send(self, msg_type, payload, msg_id):
msg = [msg_type, str(payload), msg_id, str(self.uid)]
bmsg = [e.encode() for e in [""]+msg]
for s in self.sockets:
s.send_multipart(bmsg)
self.log(bmsg[1:], "s", self.socketid[s])
def receive(self, socket, expected=None):
msg = socket.recv_multipart()
self.log(msg, "r", self.socketid[socket])
if expected:
if msg[2] == expected:
return msg
else:
self.vprint("[{me}] Did not match message: {msg}".format(me=self.id, msg=msg))
return None
return msg
def preWrite(self, msg, tag):
msg_id = str(self.hash(tag, datetime.now()))
self.sendCodedElements(msg, tag, msg_id)
acks = 0
while acks < self.quorum:
reply = dict(self.poller.poll())
while reply:
socket, event = reply.popitem()
message = self.receive(socket, expected=msg_id.encode())
if message:
self.vprint("[{me}] Received (pre-write) message: {msg}".format(
me=self.id, msg=message[1].decode()))
if message[0] == b"ACK":
acks += 1
def readerFinalize(self, tag):
acks = 0
listOfElements = []
payload = (tag, None, 'fin')
msg_id = str(self.hash(datetime.now(), self.uid))
self.send("RFINALIZE", payload, msg_id)
while acks < self.quorum:
reply = dict(self.poller.poll())
while reply:
socket, event = reply.popitem()
message = self.receive(socket, expected=msg_id.encode())
if message:
self.vprint("[{me}] Received (read-fin) message: {msg}".format(
me=self.id, msg=message[1].decode()))
msg_type = message[0]
if(msg_type == b"RFINALIZE"):
if(len(message) == 4):
listOfElements.append(message[-1])
acks += 1
if (len(listOfElements) >= self.k):
self.vprint("[{me}] Got enough elements to assemble message!".format(me=self.id))
return self.assembleMessage(listOfElements)
else:
self.vprint("[{me}] Did not get enough elements to assemble message!".format(me=self.id))
return None
def query(self):
maxtag = (0,0)
acks = 0
payload = (None, None, self.uid)
msg_id = str(self.hash(datetime.now(), self.uid))
self.send("QUERY", payload, msg_id)
while acks < self.quorum:
reply = dict(self.poller.poll())
while reply:
socket, event = reply.popitem()
message = self.receive(socket, expected=msg_id.encode())
self.vprint("[{me}] Received (query) message: {msg}".format(
me=self.id, msg=message))
if message:
msg_type = message[0]
if(msg_type == b"QUERY"):
msg_maxtag = literal_eval(message[1].decode())
self.vprint("[{me}] Received maxts: {m}".format(me=self.id, m=msg_maxtag))
if(self.compareTags(msg_maxtag, maxtag)):
maxtag = msg_maxtag
acks += 1
return maxtag
def read(self):
#QUERY PHASE
ts = self.query()
self.vprint("[{me}] maxts = {m}".format(me=self.id, m=ts))
res = self.readerFinalize(ts)
return res
def write(self, message):
#QUERY PHASE
maxtag = self.query()
newTag = (maxtag[0]+1, self.uid)
#PRE-WRITE PHASE
self.preWrite(message, newTag)
#WRITER FINALIZE PHASE
acks = 0
payload = (newTag, None, 'fin')
msg_id = str(self.hash(payload, datetime.now()))
self.send("WFINALIZE", payload, msg_id)
self.vprint("[{me}] sent: msg_id={mid}".format(
me=self.id, mid=msg_id))
while acks < self.quorum:
reply = dict(self.poller.poll())
while reply:
socket, event = reply.popitem()
message = self.receive(socket, expected=msg_id.encode())
if message:
msg_type = message[0]
if(msg_type == b"WFINALIZE"):
acks += 1
#Return True if the first one is larger
def compareTags(self, firstTuple, secondTuple):
return firstTuple > secondTuple
#Create a list of message payload
def unpackPayload(self, payload):
payloadList = literal_eval(payload)
return payloadList
# returns the string which is the hexdigest of args using hash funciton hf
def hash(self, *args):
h = hashlib.new('sha256')
for arg in args:
h.update(str(arg).encode('ascii'))
return h.hexdigest()
def log(self, msg, mode, dst):
if self.verbose:
uid = str(self.uid)
mode = str(mode)
time = str(datetime.now())
msg = str([e.decode() for e in msg[:4]])
log_str = "%26s %100s %2s %15s %15s" % (time, msg, mode, uid, dst)
log_dir = 'log'
log_file = log_dir + '/client_%s.log' % (uid)
try:
with open(log_file, 'a') as f:
f.write(log_str + '\n')
except OSError as e:
print("Could not open logfile: {}".format(e), file=sys.stderr)
parser = argparse.ArgumentParser(description='Decoder for PyECLib.')
parser.add_argument('k', type=int, help='number of data elements')
parser.add_argument('m', type=int, help='number of parity elements')
parser.add_argument('ec_type', help='EC algorithm used')
parser.add_argument('fragments', metavar='fragment', nargs='+',
help='fragments to decode')
parser.add_argument('filename', help='output file')
args = parser.parse_args()
print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("fragments = %s" % args.fragments)
print("filename = %s" % args.filename)
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
fragment_list = []
# read fragments
for fragment in args.fragments:
with open(("%s" % fragment), "rb") as fp:
fragment_list.append(fp.read())
# decode
decoded_file = ec_driver.decode(fragment_list)
# write
with open("%s.decoded" % args.filename, "wb") as fp:
fp.write(decoded_file)
class Bracha(object):
"""
Bracha broadcast '87
Implemented using state machine (BrachaStep)
"""
def __init__(self, factory, msg_wrapper_f=lambda _x: _x):
self._factory = factory
self._step = _BRACHA_STEP.one
self._init_count = 0
self._echo_count = 0
self._ready_count = 0
self._root = None
self._fragments = {}
self._v = None
self._done = False
self._msg_wrapper_f = msg_wrapper_f
self._n = self._factory.config.n
self._t = self._factory.config.t
self._sent_ready = False
# NOTE: #define EC_MAX_FRAGMENTS 32
# https://github.com/openstack/liberasurecode/blob/master/include/erasurecode/erasurecode.h
k = self._n - 2 * self._t
m = 2 * self._t
logging.debug("Bracha: erasure code params k={}, m={}".format(k, m))
self._ec_driver = ECDriver(k=k, m=m, ec_type='liberasurecode_rs_vand')
random.seed()
def handle(self, msg, sender_vk):
# type: (pb.Bracha) -> Handled
"""
This function is called on a new incoming message, we expect the type is correct
msg should be in the following format
struct Msg {
ty: u32,
body: String,
}
:param msg: the input message to send
:return: the delivered message when completed, otherwise None
"""
if self._done:
logging.debug("Bracha: done, doing nothing")
return Handled()
ty = msg.ty
logging.debug("Bracha: received {}".format(msg))
assert isinstance(ty, int)
assert msg.digest is not None
# initialisation
if self._root is None:
# if body is None, we must be in the initial state
assert self._init_count == 0 and self._echo_count == 0 and self._ready_count == 0
self._root = msg.digest
if self._root != msg.digest:
logging.debug("Bracha: self.root {} != root: {}, discarding"
.format(b64encode(self._root), b64encode(msg.digest)))
return Handled()
# here we update the state
if ty == _INIT:
self._init_count += 1
elif ty == _ECHO:
self._fragments[sender_vk] = msg.fragment
self._echo_count += 1
assert self._echo_count == len(self._fragments), \
"echo_count {} != fragment count {}".format(self._echo_count, len(self._fragments))
elif ty == _READY:
self._ready_count += 1
assert self._root == msg.digest, \
"self.root {} != root: {}".format(self._root, msg.digest)
else:
raise AssertionError("Bracha: unexpected msg type")
# we should only see one init message
assert self._init_count == 0 or self._init_count == 1
# everything below is the algorithm, acting on the current state
if ty == _INIT:
logging.debug("Bracha: got init value, root = {}".format(b64encode(msg.digest)))
self._upon_init(msg)
if ty == _ECHO:
logging.debug("Bracha: got echo value, root = {}".format(b64encode(msg.digest)))
# TODO check Merkle branch
pass
if ty == _ECHO and self._echo_count >= self._n - self._t:
logging.debug("Bracha: got n - t echo values, root = {}".format(b64encode(msg.digest)))
self._upon_n_minus_t_echo()
if ty == _READY and self._ready_count >= self._t + 1:
self._upon_t_plus_1_ready()
if self._ready_count >= 2 * self._t + 1 and self._echo_count >= self._n - 2*self._t:
res = self._upon_2t_plus_1_ready()
# NOTE: we use a random value to trip up tests, since it shouldn't be viewed by tests
logging.info("Bracha: DELIVER {}"
.format(random.random() if self._factory.config.from_instruction else b64encode(res)))
self._done = True
return Handled(res)
return Handled()
def _upon_init(self, msg):
assert isinstance(msg, pb.Bracha)
msg.ty = _ECHO
self.bcast(msg)
def _decode_fragments(self):
fragments = random.sample(self._fragments.values(), self._n - 2 * self._t)
v = self._ec_driver.decode(fragments)
return v
def _upon_n_minus_t_echo(self):
v = self._decode_fragments()
assert libnacl.crypto_hash_sha256(v) == self._root
self._v = v
logging.debug("Bracha: erasure decoded msg v {}".format(b64encode(v)))
if not self._sent_ready:
logging.debug("Bracha: broadcast ready 1, root = {}".format(b64encode(self._root)))
self.bcast(pb.Bracha(ty=_READY, digest=self._root))
self._sent_ready = True
def _upon_t_plus_1_ready(self):
if not self._sent_ready:
logging.debug("Bracha: broadcast ready 2, root = {}".format(b64encode(self._root)))
self.bcast(pb.Bracha(ty=_READY, digest=self._root))
self._sent_ready = True
def _upon_2t_plus_1_ready(self):
if self._v is None:
self._v = self._decode_fragments()
return self._v
def bcast_init(self, msg="some test msg!!"):
assert isinstance(msg, str)
self._bcast_init_fragments(msg)
def _bcast_init_fragments(self, msg):
"""
:param msg: some bytes
:return:
"""
fragments = self._ec_driver.encode(msg)
digest = libnacl.crypto_hash_sha256(msg)
logging.info("Bracha: initiate erasure code with {} fragments, digest {}"
.format(len(fragments), b64encode(digest)))
assert len(fragments) == len(self._factory.promoters)
for fragment, promoter in zip(fragments, self._factory.promoters):
m = pb.Bracha(ty=_INIT, digest=digest, fragment=fragment)
self._factory.send(promoter, self._msg_wrapper_f(m))
def bcast(self, msg):
self._factory.promoter_cast(self._msg_wrapper_f(msg))
class DepSkyClient:
def __init__(self, clientId, isLocal=True):
self.clientId = clientId
self.isLocal = isLocal
self.N = 4
self.F = 1
self.metadataQ = queue.Queue()
self.ec_driver = ECDriver(k=2, m=2, ec_type='liberasurecode_rs_vand')
if self.isLocal:
self.createLocalClients()
def read(self, container):
maxVersion, metadata_list = self.getMetadataFromClouds(
filename=container.duId + 'metadata')
count = 0
# Flush the metadata queue - cancel the remaining requests
try:
while True:
self.metadataQ.get_nowait()
except Empty:
pass
# Flushing complete
if maxVersion == 0:
print('No data to read! Sorry!!!')
return
dataFileName = container.duId + 'value' + str(maxVersion)
secret_unit_list = self.readDataFromClouds(metadata_list, dataFileName)
shares = []
fragments = []
for unit in secret_unit_list:
shares.append((unit.secret_id, unit.share))
fragments.append(unit.value)
key = Shamir.combine(shares)
encrypted_data = self.ec_driver.decode(fragments)
cipher = getAESCiphers(key)
data = cipher.decrypt(encrypted_data)
return data
def readDataFromClouds(self, metadata_list, filename):
results = self.readInParallel(metadata_list, filename)
return results
def readInParallel(self, metadata_list, filename):
q = queue.Queue()
cloudIds = [cloudMetadata.cloudId for cloudMetadata in metadata_list]
threads = [
threading.Thread(target=self.getFileFromCloud,
args=(i, filename, q)) for i in cloudIds
]
metadata_list_to_dict = {
metadata.cloudId: metadata
for metadata in metadata_list
}
results = []
verified_count = 0
for thread in threads:
thread.daemon = True
thread.start()
for _ in range(len(cloudIds)):
result = q.get()
result = json.loads(result)
secretUnit = SecretUnitDecoder().decode(result)
cloudId = secretUnit.cloudId
metadata_of_secret_unit = metadata_list_to_dict[cloudId]
latest_metadata = metadata_of_secret_unit.metadata_list[0]
if getHash(SecretUnitEncoder().encode(
secretUnit)).hexdigest() == latest_metadata.hashedUnit:
results.append(secretUnit)
verified_count += 1
print('Data integrity of the value in cloud', cloudId,
'succeeded')
else:
print('Data integrity of the value stored in cloud', cloudId,
'failed')
if verified_count > self.F:
# remove the remaining items from the queue
try:
while True:
q.get_nowait()
except Empty:
pass
break
if verified_count < self.F + 1:
print('Data integrity of F+1 clouds failed. Cannot recover!!')
exit(0)
return results
def createLocalClients(self):
for i in range(4):
requests.post('http://localhost:5555',
data=json.dumps({'cloudId': i}))
def generateKeyShares(self, key):
return Shamir.split(self.F + 1, self.N, key)
def erasureCode(self, value):
return self.ec_driver.encode(value)
def write(self, container, value):
maxVersion, metadata_list = self.getMetadataFromClouds(
filename=container.duId + 'metadata')
nextVersion = maxVersion + 1
value_hash = getHash(value, return_hexdigest=True)
key = get_random_bytes(16)
cipher = getAESCiphers(key)
encrypted_value = cipher.encrypt(value.encode('utf8'))
shares = self.generateKeyShares(key)
fragments = self.erasureCode(encrypted_value)
units = []
for cloudId, (share, fragment) in enumerate(zip(shares, fragments)):
idx, secret = share
unit = SecretUnit(idx, secret, fragment, cloudId)
units.append(SecretUnitEncoder().encode(unit))
self.writeToClouds(container.duId + 'value' + str(nextVersion), units)
self.writeMetadata(container.duId + 'metadata', value_hash, units,
nextVersion, metadata_list)
def writeToClouds(self, filename, units):
results = self.writeInParallel(filename, units)
def writeInParallel(self, filename, dataList):
q = queue.Queue()
threads = []
for i, data in enumerate(dataList):
threads.append(
threading.Thread(target=self.writeToCloud,
args=(i, data, filename, q)))
for thread in threads:
thread.daemon = True
thread.start()
results = []
for _ in range(self.N - self.F):
results.append(q.get())
return results
def writeToCloud(self, cloudId, data, filename, result_queue):
r = requests.post('http://localhost:5555/write',
data=json.dumps({
'cloudId': cloudId,
'data2write': data,
'filename': filename
}))
result_queue.put(r.text)
def writeMetadata(self, filename, dataHash, units, version, metadata_list):
metadata_of_clouds = []
for _ in range(self.F):
value = self.metadataQ.get()
if value == 'None':
metadata_list.append(None)
else:
result = json.loads(value)
cloudMetadata = CloudMetadataDecoder().decode(result)
metadata_list.append(cloudMetadata)
# sort the metadata - or better store as dict :)
cloudsPresent = any(metadata_list)
if cloudsPresent:
metadata_list_to_dict = {
metadata.cloudId: metadata
for metadata in metadata_list
}
for i in range(self.N):
hashedUnit = getHash(units[i]).hexdigest()
# got the cloud in ascending order
metadata = Metadata(dataHash, hashedUnit, version,
getSignature(i, hashedUnit))
if cloudsPresent:
cloud = metadata_list_to_dict[i]
cloudMetadata = cloud.add_new_metadata(metadata)
else:
cloudMetadata = CloudMetadata([metadata], i)
cloudMetadata = cloudMetadata.return_writable_content()
metadata_of_clouds.append(cloudMetadata)
results = self.writeInParallel(filename, metadata_of_clouds)
def getMetadataInParallel(self, function, filename):
q = self.metadataQ
threads = [
threading.Thread(target=function, args=(i, filename, q))
for i in range(4)
]
versions_from_metadata = []
results = []
verified_count = 0
for thread in threads:
thread.daemon = True
thread.start()
for _ in range(self.N):
result = q.get()
if result == 'None':
results.append(None)
else:
result = json.loads(result)
cloudMetadata = CloudMetadataDecoder().decode(result)
latest_metadata = cloudMetadata.metadata_list[0]
list_of_dicts = cloudMetadata.return_metadata_list_in_dicts()
if verifySignature(cloudMetadata.cloudId,
getHash(list_of_dicts).hexdigest(),
cloudMetadata.metadata_list_sig):
if verifySignature(cloudMetadata.cloudId,
latest_metadata.hashedUnit,
latest_metadata.signature):
verified_count += 1
versions_from_metadata.append(latest_metadata.version)
results.append(cloudMetadata)
print('Verified metadata of cloud',
cloudMetadata.cloudId)
else:
print("The verification of metadata of cloud",
cloudMetadata.cloudId, 'failed')
else:
print(
'The verification of signature of metadata list of cloud',
cloudMetadata.cloudId, 'failed')
# breaking condition
if not any(results) and len(results) == self.N - self.F:
break
if verified_count == self.N - self.F:
break
if verified_count < self.N - self.F and any(results):
print(
'Metadata of N-F clouds have been corrupted. Cannot proceed further!!!'
)
exit(0)
return results, max(
versions_from_metadata) if len(versions_from_metadata) > 0 else 0
def getMetadataFromClouds(self, filename):
results, maxVersion = self.getMetadataInParallel(
self.getFileFromCloud, filename)
return maxVersion, results
def getFileFromCloud(self, cloudId, filename, result_queue):
r = requests.post('http://localhost:5555/read',
data=json.dumps({
'cloudId': cloudId,
'filename': filename
}))
result_queue.put(r.text)