def setUp(self):
self.null_driver = ECDriver(
library_import_str="pyeclib.core.ECNullDriver", k=8, m=2)
def setUp(self):
self.stripe_driver = ECDriver(
library_import_str="pyeclib.core.ECStripingDriver", k=8, m=0)
parser = argparse.ArgumentParser()
parser.add_argument('k', type=int)
parser.add_argument('m', type=int)
parser.add_argument('ec_type')
parser.add_argument('file_dir')
parser.add_argument('filename')
parser.add_argument('fragment_dir')
args = parser.parse_args()
print("k = %d, m = %d" % (args.k, args.m))
print("ec_type = %s" % args.ec_type)
print("filename = %s" % args.filename)
ec_driver = ECDriver(k=args.k, m=args.m, ec_type=args.ec_type)
# read
with open(("%s/%s" % (args.file_dir, args.filename)), "rb") as fp:
whole_file_str = fp.read()
# encode
fragments = ec_driver.encode(whole_file_str)
# store
i = 0
for fragment in fragments:
with open("%s/%s.%d" % (args.fragment_dir, args.filename, i), "wb") as fp:
fp.write(fragment)
i += 1
def get_pyeclib_testspec(self, csum="none"):
pyeclib_drivers = []
_type1 = 'jerasure_rs_vand'
if _type1 in VALID_EC_TYPES:
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type=_type1, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=11, m=2, ec_type=_type1, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type=_type1, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=8, m=4, ec_type=_type1, chksum_type=csum))
_type2 = 'liberasurecode_rs_vand'
if _type2 in VALID_EC_TYPES:
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type=_type2, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=11, m=2, ec_type=_type2, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type=_type2, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=8, m=4, ec_type=_type2, chksum_type=csum))
_type3_1 = 'flat_xor_hd'
if _type3_1 in VALID_EC_TYPES:
pyeclib_drivers.append(
ECDriver(k=12, m=6, ec_type=_type3_1, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=10, m=5, ec_type=_type3_1, chksum_type=csum))
_type3_2 = 'flat_xor_hd_4'
if _type3_2 in VALID_EC_TYPES:
pyeclib_drivers.append(
ECDriver(k=12, m=6, ec_type=_type3_2, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=10, m=5, ec_type=_type3_2, chksum_type=csum))
_type4 = 'shss'
if _type4 in VALID_EC_TYPES:
pyeclib_drivers.append(
ECDriver(k=10, m=4, ec_type=_type4, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=20, m=4, ec_type=_type4, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=11, m=7, ec_type=_type4, chksum_type=csum))
_type5 = 'isa_l_rs_vand'
if _type5 in VALID_EC_TYPES:
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type=_type5, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=11, m=2, ec_type=_type5, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type=_type5, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=8, m=4, ec_type=_type5, chksum_type=csum))
_type6 = 'isa_l_rs_cauchy'
if _type6 in VALID_EC_TYPES:
pyeclib_drivers.append(
ECDriver(k=12, m=2, ec_type=_type6, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=11, m=2, ec_type=_type6, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=10, m=2, ec_type=_type6, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=8, m=4, ec_type=_type6, chksum_type=csum))
pyeclib_drivers.append(
ECDriver(k=11, m=7, ec_type=_type6, chksum_type=csum))
_type7 = 'libphazr'
if _type7 in VALID_EC_TYPES:
pyeclib_drivers.append(
ECDriver(k=4, m=4, ec_type=_type7, chksum_type=csum))
return pyeclib_drivers
def __init__(self,
idx,
name='',
aliases='',
is_default=False,
is_deprecated=False,
object_ring=None,
diskfile_module='egg:swift#erasure_coding.fs',
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
ec_type=None,
ec_ndata=None,
ec_nparity=None,
ec_duplication_factor=1):
super(ECStoragePolicy, self).__init__(idx=idx,
name=name,
aliases=aliases,
is_default=is_default,
is_deprecated=is_deprecated,
object_ring=object_ring,
diskfile_module=diskfile_module)
# Validate erasure_coding policy specific members
# ec_type is one of the EC implementations supported by PyEClib
if ec_type is None:
raise PolicyError('Missing ec_type')
if ec_type not in VALID_EC_TYPES:
raise PolicyError('Wrong ec_type %s for policy %s, should be one'
' of "%s"' %
(ec_type, self.name, ', '.join(VALID_EC_TYPES)))
self._ec_type = ec_type
# Define _ec_ndata as the number of EC data fragments
# Accessible as the property "ec_ndata"
try:
value = int(ec_ndata)
if value <= 0:
raise ValueError
self._ec_ndata = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_data_fragments %r' % ec_ndata,
index=self.idx)
# Define _ec_nparity as the number of EC parity fragments
# Accessible as the property "ec_nparity"
try:
value = int(ec_nparity)
if value <= 0:
raise ValueError
self._ec_nparity = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_parity_fragments %r' %
ec_nparity,
index=self.idx)
# Define _ec_segment_size as the encode segment unit size
# Accessible as the property "ec_segment_size"
try:
value = int(ec_segment_size)
if value <= 0:
raise ValueError
self._ec_segment_size = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_object_segment_size %r' %
ec_segment_size,
index=self.idx)
if self._ec_type == 'isa_l_rs_vand' and self._ec_nparity >= 5:
logger = logging.getLogger("swift.common.storage_policy")
if not logger.handlers:
# If nothing else, log to stderr
logger.addHandler(logging.StreamHandler(sys.__stderr__))
logger.warning(
'Storage policy %s uses an EC configuration known to harm '
'data durability. Any data in this policy should be migrated. '
'See https://bugs.launchpad.net/swift/+bug/1639691 for '
'more information.' % self.name)
if not is_deprecated:
raise PolicyError(
'Storage policy %s uses an EC configuration known to harm '
'data durability. This policy MUST be deprecated.' %
self.name)
# Initialize PyECLib EC backend
try:
self.pyeclib_driver = \
ECDriver(k=self._ec_ndata, m=self._ec_nparity,
ec_type=self._ec_type)
except ECDriverError as e:
raise PolicyError("Error creating EC policy (%s)" % e,
index=self.idx)
# quorum size in the EC case depends on the choice of EC scheme.
self._ec_quorum_size = \
self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
self._fragment_size = None
self._ec_duplication_factor = \
config_positive_int_value(ec_duplication_factor)
# Contains consants that are used by all programs. from pyeclib.ec_iface import ECDriver K = 3 M = 13 EC_TYPE = "liberasurecode_rs_vand" PORT1 = 5005 # sender.py sends to this port, intermediate.py receives from this port. PORT2 = 6006 # intermediate.py sends to this port, receiver.py receives from this port. ec_driver = ECDriver(k=K, m=M, ec_type=EC_TYPE)
oldname = FILE_NAME
os.rename(FILE_NAME, new_name)
# Delete the fragment
for i in range(0, 6):
fil = "/home/rawal/Desktop/mini_pro/File/" + new_name + '.' + str(i)
os.remove(fil)
FILE_NAME = new_name
k1 = 4
m1 = 2
ec_type1 = "liberasurecode_rs_vand"
file_dir = "/home/rawal/Desktop/mini_pro/code"
filename = FILE_NAME
fragment_dir = "/home/rawal/Desktop/mini_pro/File"
ec_driver = ECDriver(k=k1, m=m1, ec_type=ec_type1)
with open(("%s/%s" % (file_dir, filename)), "rb") as fp:
whole_file_str = fp.read()
fragments = ec_driver.encode(whole_file_str)
i = 0
for fragment in fragments:
with open("%s/%s.%d" % (fragment_dir, filename, i), "wb") as fp:
fp.write(fragment)
i += 1
for i in range(1, 6):
BUCKET_NAME = "minibuck" + str(i)
newfn = "/home/rawal/Desktop/mini_pro/File/" + new_name + '.' + str(i -
1)
s3 = boto3.client('s3')
# Results will be List[(ec_type, throughput)]
results = []
# Num iterations
num_iterations = 10
for scheme in schemes:
print(scheme)
# Generate a new string for each test
file_str = ''.join(
random.choice(string.ascii_uppercase + string.digits)
for x in range(args.s))
try:
ec_driver = ECDriver(k=scheme.k, m=scheme.m, ec_type=scheme.ec_type)
except Exception as e:
print("Scheme %s is not defined (%s)." % (scheme, e))
continue
timer.start()
for i in range(num_iterations):
ec_driver.encode(file_str)
duration = timer.stop_and_return()
results.append((scheme, duration))
timer.reset()
def __init__(self,
idx,
name='',
aliases='',
is_default=False,
is_deprecated=False,
object_ring=None,
ec_segment_size=DEFAULT_EC_OBJECT_SEGMENT_SIZE,
ec_type=None,
ec_ndata=None,
ec_nparity=None):
super(ECStoragePolicy, self).__init__(idx=idx,
name=name,
aliases=aliases,
is_default=is_default,
is_deprecated=is_deprecated,
object_ring=object_ring)
# Validate erasure_coding policy specific members
# ec_type is one of the EC implementations supported by PyEClib
if ec_type is None:
raise PolicyError('Missing ec_type')
if ec_type not in VALID_EC_TYPES:
raise PolicyError('Wrong ec_type %s for policy %s, should be one'
' of "%s"' %
(ec_type, self.name, ', '.join(VALID_EC_TYPES)))
self._ec_type = ec_type
# Define _ec_ndata as the number of EC data fragments
# Accessible as the property "ec_ndata"
try:
value = int(ec_ndata)
if value <= 0:
raise ValueError
self._ec_ndata = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_data_fragments %r' % ec_ndata,
index=self.idx)
# Define _ec_nparity as the number of EC parity fragments
# Accessible as the property "ec_nparity"
try:
value = int(ec_nparity)
if value <= 0:
raise ValueError
self._ec_nparity = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_num_parity_fragments %r' %
ec_nparity,
index=self.idx)
# Define _ec_segment_size as the encode segment unit size
# Accessible as the property "ec_segment_size"
try:
value = int(ec_segment_size)
if value <= 0:
raise ValueError
self._ec_segment_size = value
except (TypeError, ValueError):
raise PolicyError('Invalid ec_object_segment_size %r' %
ec_segment_size,
index=self.idx)
# Initialize PyECLib EC backend
try:
self.pyeclib_driver = \
ECDriver(k=self._ec_ndata, m=self._ec_nparity,
ec_type=self._ec_type)
except ECDriverError as e:
raise PolicyError("Error creating EC policy (%s)" % e,
index=self.idx)
# quorum size in the EC case depends on the choice of EC scheme.
self._ec_quorum_size = \
self._ec_ndata + self.pyeclib_driver.min_parity_fragments_needed()
