def fill_name_tag_in_tn_iface(node, dom):
"""
Obtain a random VLAN from the given list of ranges of available VLANs obtained from TNRM.
In case TNRM provides the full list only, the local domain will be iteratively
examined in order to minimise possible collisions of VLANs
"""
new_node = {}
num_iter = 0
vlan = ""
for k in node.keys():
if k == "vlan":
vlans = CommonUtils.process_range_and_set_values(
node[k][0]["description"])
is_contained = True
max_iter = int(len(vlans) - 1)
# Search for suitable (available VLANs) until found or "all" the
# range (having in mind the randomness) has been examined
while is_contained and num_iter <= max_iter:
idx_vlan = CommonUtils.get_random_list_position(max_iter)
vlan = vlans[idx_vlan]
is_contained, intersect = TNUtils.check_vlan_is_in_use(
vlan)
num_iter += 1
new_node[k] = [{"tag": str(vlan), "name": "%s+vlan" % dom}]
else:
new_node[k] = node[k]
return new_node
def fill_name_tag_in_tn_iface(node, dom):
"""
Obtain a random VLAN from the given list of ranges of available VLANs obtained from TNRM.
In case TNRM provides the full list only, the local domain will be iteratively
examined in order to minimise possible collisions of VLANs
"""
new_node = {}
num_iter = 0
vlan = ""
for k in node.keys():
if k == "vlan":
vlans = CommonUtils.process_range_and_set_values(node[k][0]["description"])
is_contained = True
max_iter = int(len(vlans)-1)
# Search for suitable (available VLANs) until found or "all" the
# range (having in mind the randomness) has been examined
while is_contained and num_iter <= max_iter:
idx_vlan = CommonUtils.get_random_list_position(max_iter)
vlan = vlans[idx_vlan]
is_contained, intersect = TNUtils.check_vlan_is_in_use(vlan)
num_iter += 1
new_node[k] = [{"tag": str(vlan), "name": "%s+vlan" % dom}]
else:
new_node[k] = node[k]
return new_node
def fill_name_tag_in_tn_iface(node, dom, transport_vlans={},
force_vlan=None):
"""
Obtain a random VLAN from the given list of ranges of available VLANs
obtained from TNRM.
In case TNRM provides the full list only, the local domain will
be iteratively examined to minimise possible collisions of VLANs.
In case restricted sets of VLANs is used, those will be
taken into account.
"""
new_node = {}
idx_vlan = 0
# num_iter = 0
vlan = ""
for k in node.keys():
if k == "vlan":
vlans = CommonUtils.process_range_and_set_values(
node[k][0]["description"])
# A) Search for suitable (available) VLANs until found or
# "all" the range (having randomness in mind) is examined
# - For this, check slice monitoring info
# contained = True
max_iter = int(len(vlans)-1)
# while contained and num_iter <= max_iter:
# idx_vlan = CommonUtils.get_random_list_position(max_iter)
# vlan = vlans[idx_vlan]
# contained, intersect = TNUtils.check_vlan_is_in_use(vlan)
# num_iter += 1
# B.1) Choose from list of restricted VLANs
if "enabled" in transport_vlans and\
"disabled" in transport_vlans:
vlans_enabled = transport_vlans.get("enabled") or set()
vlans_disabled = transport_vlans.get("disabled") or set()
# VLANs are taken from the intersection of the free VLANs
# reported by TN (to only get free VLANs) and those
# restricted by configuration (to avoid them, even if free)
vlans = set(vlans).intersection(set(vlans_enabled))
vlans = set(vlans).difference(set(vlans_disabled))
vlans = list(vlans)
max_iter = int(len(vlans)-1)
idx_vlan = CommonUtils.get_random_list_position(max_iter)
# B.2) Choose from list of available VLANs from TNRM
else:
idx_vlan = CommonUtils.get_random_list_position(max_iter)
# C) If explicit VLAN is forced, use that one
if force_vlan is not None:
vlan = force_vlan
else:
vlan = vlans[idx_vlan]
new_node[k] = [{"tag": str(vlan), "name": "%s+vlan" % dom}]
else:
new_node[k] = node[k]
return new_node
def generate_manifest(self):
manifest = {
CommonConsts.MF_KEY_BUILD: [],
CommonConsts.MF_KEY_DEPENDS: [],
CommonConsts.MF_KEY_DIRS: [],
CommonConsts.MF_KEY_FILES: []
}
if not os.path.exists(self._staging_dir):
raise ResourceNotFoundError(
"Staging directory path not available. Path: {0}".format(
self._staging_dir))
for dirpath, dirnames, files in os.walk(self._staging_dir):
dir_rel_path = os.path.relpath(dirpath, self._staging_dir)
if dir_rel_path != ".":
manifest[CommonConsts.MF_KEY_DIRS].append(
{CommonConsts.MF_KEY_FILES_ATTR_PATH: dir_rel_path})
for file_ in files:
fullpath = os.path.join(dirpath, file_)
sha1 = HashGenerator(fullpath).generate_hash()
filename = os.path.relpath(fullpath, self._staging_dir)
mode = CommonUtils.get_filepermission(fullpath)
manifest[CommonConsts.MF_KEY_FILES].append({
CommonConsts.MF_KEY_FILES_ATTR_PATH:
filename,
CommonConsts.MF_KEY_FILES_ATTR_SHA1:
sha1,
CommonConsts.MF_KEY_FILES_ATTR_MODE:
mode
})
if self._target_file_path is None:
return json.dumps(manifest)
self._write_to_file(manifest)
def test_install_update(self):
PackageInstaller(SNAPPY_PKG_NAME, SNAPPY_VERSION, SNAPPY_PLATFORM,
DIR_INSTALL, DIR_DEPOT).install()
assert os.path.exists(SNAPPY_INSTALDIR_ETC_MF)
PackageInstaller(SNAPPY_PKG_NAME, SNAPPY_VERSION, SNAPPY_PLATFORM,
DIR_INSTALL, DIR_DEPOT).install()
assert os.path.exists(SNAPPY_INSTALDIR_ETC_MF)
assert CommonUtils.get_filecount_for_dir_tree(SNAPPY_INSTALLDIR) == 17
def update(self, package_name, version, platform, staging_dir, manifest_filepath):
try:
manifest_filename = CommonUtils.generate_manifest_filename(package_name, version, platform, "json")
input_manifest_file = os.path.join(manifest_filepath, manifest_filename)
self._deploy_package(package_name, version, platform, input_manifest_file, staging_dir)
except Exception as e:
self._cleanup(package_name, version, platform, manifest_filename)
raise e
def _verify_file(self, f):
fullpath = self._get_destination_file(f)
digest = hashlib.new("sha1")
digest.update(open(fullpath, "rb").read())
if digest.hexdigest() != f[CommonConsts.MF_KEY_FILES_ATTR_SHA1]:
raise ChecksumError("FATAL: SHA1 doesn't match for installed file: {0}".format(fullpath))
mode = CommonUtils.get_filepermission(fullpath)
if mode != f[CommonConsts.MF_KEY_FILES_ATTR_MODE]:
raise PermissionError("FATAL: Permission mode doesn't match for installed file: {0}".format(fullpath))
return True
def add_tn_to_ro_request_rspec(req_rspec, sdn_utils, vl_utils):
"""
Should only be run from MRO; where TN can be processed.
"""
# Run mapper path-finder to extend SDN resources
# by adding an inherent link to the SE device
# Note: check if we had an explicit/direct TN allocation
# (in this case just skip the mapper)
request_stps = []
# Retrieve virtual links
if CommonUtils.is_virtual_links(req_rspec):
vlinks = req_rspec.vl_links()
for vlink in vlinks:
(src_dom, dst_dom) = vl_utils.get_domains_from_link(vlink)
request_stps_type = vl_utils.get_type_from_link(vlink)
request_stps.append({"src_name": src_dom, "dst_name": dst_dom, \
"link_type": request_stps_type})
logger.debug("STPs=%s" % (request_stps,))
dpid_port_ids = sdn_utils.find_dpid_port_identifiers(
req_rspec.of_groups(), req_rspec.of_matches())
logger.debug("DPIDs=%s" % (dpid_port_ids,))
tn_nodes = []
tn_links = []
try:
tn_nodes, tn_links = TNUtils.identify_tn_from_sdn_and_vl(dpid_port_ids, request_stps, sdn_utils)
except Exception as e:
m = "Could not obtain TN resources from SDN and VL resources. Details: %s" % str(e)
logger.warning(m)
tnrm_formatter = TNRMv3RequestFormatter()
for n in tn_nodes:
tnrm_formatter.node(n)
for l in tn_links:
tnrm_formatter.link(l)
# Generate RSpec from request RSpec object
rspec = req_rspec.get_rspec()
tnrm_rspec = tnrm_formatter.rspec.getchildren()
for tnrm_elem in tnrm_rspec:
rspec.append(tnrm_elem)
# Clean virtual links
vlinks = rspec.xpath("xs:link//xs:link_type[contains(@name, '%s')]" % ("virtual_link"), namespaces={"xs": DEFAULT_XMLNS})
#vlinks = rspec.xpath("xs:link//xs:link_type[@name='%s']" % ("urn:felix+virtual_link"), namespaces={"xs": DEFAULT_XMLNS})
#vlinks = rspec.findall("{%s}link//{%s}link_type[@name='%s']" % (DEFAULT_XMLNS, DEFAULT_XMLNS, "urn:felix+virtual_link"))
for v in vlinks:
v.getparent().getparent().remove(v.getparent())
rspec = etree.tostring(rspec, pretty_print=True)
logger.debug("Implicit request => request RSpec has been extended with proper resources")
logger.debug("Request RSpec passed to Allocate: %s" % str(rspec))
return rspec
def _cleanup(self, package_name, version, platform, manifest_filename):
try:
depot_mf = os.path.join(self._depot_mf_path, manifest_filename)
if os.path.exists(depot_mf):
os.remove(depot_mf)
depot_pkg = CommonUtils.generate_package_name(package_name, version, platform)
depot_df = os.path.join(self._location, CommonConsts.SW_DEPOT_DATAFILES_DIR , depot_pkg)
if os.path.exists(depot_df):
shutil.rmtree(depot_df)
except Exception as e:
print "Exception while cleanup: ", e
def _deploy_package(self, package_name, version, platform, manifest_file, staging_dir):
depot_package_name = CommonUtils.generate_package_name(package_name, version, platform)
depot_pkg_dest_path = os.path.join(self._depot_df_path, depot_package_name)
if not os.path.exists(depot_pkg_dest_path):
os.makedirs(depot_pkg_dest_path)
with open(manifest_file, "rb") as f:
manifest = json.load(f)
dirs_m = manifest[CommonConsts.MF_KEY_DIRS]
files_m = manifest[CommonConsts.MF_KEY_FILES]
for file_ in files_m:
fullpath = os.path.join(staging_dir, file_[CommonConsts.MF_KEY_FILES_ATTR_PATH])
contents = open(fullpath, 'rb').read()
sha1 = hashlib.sha1(contents).hexdigest()
if sha1 != file_[CommonConsts.MF_KEY_FILES_ATTR_SHA1]:
raise ChecksumError("FATAL: File modified in staging area before installation: {0}".format(file_[CommonConsts.MF_KEY_FILES_ATTR_PATH]))
mode = CommonUtils.get_filepermission(fullpath)
if mode != file_["mode"]:
raise PermissionError("FATAL: SHA1 doesn't match for installed file: {0}".format(fullpath))
dest_file = os.path.join(depot_pkg_dest_path, sha1)
shutil.copy(fullpath, dest_file)
shutil.copy(manifest_file, self._depot_mf_path)
def _setup(self):
try:
self._manifest_filename = CommonUtils.generate_manifest_filename(self._name, self._version, self._platform, "json")
self._depot_manifestfile_location = os.path.join(self._depot_location, CommonConsts.SW_DEPOT_MANIFEST_FILE_DIR)
self._depot_datafile_location = os.path.join(self._depot_location, CommonConsts.SW_DEPOT_DATAFILES_DIR,
CommonUtils.generate_package_name(self._name, self._version, self._platform))
self._etc_dir = os.path.join(self._install_dir, "etc", "packages")
if not os.path.exists(self._etc_dir):
os.makedirs(self._etc_dir)
self._pkg_install_dir = self._install_dir
self._is_already_installed = self._is_already_installed()
self._temp_dir = os.path.join(self._pkg_install_dir, os.path.splitext(self._manifest_filename)[0] + "-INSTALL-TEMP")
os.mkdir(self._temp_dir)
self._manifest = self._get_manifest_object()
except Exception as e:
self._cleanup()
self._log.error(e)
raise e
def add(self, package_name, version, platform, staging_dir, manifest_filepath):
try:
manifest_filename = CommonUtils.generate_manifest_filename(package_name, version, platform, "json")
if os.path.exists(os.path.join(self._depot_mf_path, manifest_filename)):
raise PackageExistsError("Package manifest file already exists in depot. Package name : {0}, manifest file: {1}.\n\
Either uninstall the package first or use update command to install the package.".format(CommonUtils.generate_package_name(package_name,
version,
platform),
manifest_filename))
input_manifest_file = os.path.join(manifest_filepath, manifest_filename)
self._deploy_package(package_name, version, platform, input_manifest_file, staging_dir)
except Exception as e:
self._cleanup(package_name, version, platform, manifest_filename)
raise e
def _install_dependency(self, dep):
mfn = dep[CommonConsts.MF_KEY_DEPENDS_ATTR_MANIFEST]
if not mfn:
mfn = CommonUtils.generate_manifest_filename(dep[CommonConsts.MF_KEY_DEPENDS_ATTR_PACKAGE],
dep[CommonConsts.MF_KEY_DEPENDS_ATTR_VERSION],
dep[CommonConsts.MF_KEY_DEPENDS_ATTR_PLATFORM],
"json")
if mfn in self._dep_status[DEP_STATUS_KEY_INSTALLED]:
return
elif mfn in self._dep_status[DEP_STATUS_KEY_INQUEUE]:
raise CyclicDependencyError("FATAL: Cyclic dependency found in manifest : {0} for {1} while installing package: {2}".format(self._manifest_filename,
mfn, self._name))
pi = PackageInstaller(dep[CommonConsts.MF_KEY_DEPENDS_ATTR_PACKAGE],
dep[CommonConsts.MF_KEY_DEPENDS_ATTR_VERSION],
dep[CommonConsts.MF_KEY_DEPENDS_ATTR_PLATFORM],
self._install_dir, self._depot_location, self._dep_status)
pi.install()
def test_genfile(self):
'''
python voltron20.py manifest genfile --package_name="snappy" -ver="1.0.5" -p="ubuntu-12.04"
-sd="/tmp/snappy/opt/couchbase" -tfp="/home/suhail/workspace/temp/manifest-files"
'''
m = ManifestGenerator(SNAPPY_PKG_NAME, SNAPPY_VERSION, SNAPPY_PLATFORM,
os.path.join(DIR_SNAPPY_STAGING), DIR_DEPOT_TEMP)
m.generate_manifest()
mfn = CommonUtils.generate_manifest_filename("snappy", "1.0.5",
"ubuntu-12.04", "json")
with open(os.path.join(DIR_DEPOT_TEMP, mfn)) as f:
manifest = json.load(f)
assert set([
CommonConsts.MF_KEY_BUILD, CommonConsts.MF_KEY_DEPENDS,
CommonConsts.MF_KEY_DIRS, CommonConsts.MF_KEY_FILES
]) == set(manifest)
assert len(manifest[CommonConsts.MF_KEY_DIRS]) == 5
assert len(manifest[CommonConsts.MF_KEY_FILES]) == 16
def generate_manifest(self):
manifest = {CommonConsts.MF_KEY_BUILD:[], CommonConsts.MF_KEY_DEPENDS:[], CommonConsts.MF_KEY_DIRS:[], CommonConsts.MF_KEY_FILES: []}
if not os.path.exists(self._staging_dir):
raise ResourceNotFoundError("Staging directory path not available. Path: {0}".format(self._staging_dir))
for dirpath, dirnames, files in os.walk(self._staging_dir):
dir_rel_path = os.path.relpath(dirpath, self._staging_dir)
if dir_rel_path != ".":
manifest[CommonConsts.MF_KEY_DIRS].append({CommonConsts.MF_KEY_FILES_ATTR_PATH:dir_rel_path})
for file_ in files:
fullpath = os.path.join(dirpath, file_)
sha1 = HashGenerator(fullpath).generate_hash()
filename = os.path.relpath(fullpath, self._staging_dir)
mode = CommonUtils.get_filepermission(fullpath)
manifest[CommonConsts.MF_KEY_FILES].append({CommonConsts.MF_KEY_FILES_ATTR_PATH: filename, CommonConsts.MF_KEY_FILES_ATTR_SHA1: sha1,
CommonConsts.MF_KEY_FILES_ATTR_MODE: mode})
if self._target_file_path is None:
return json.dumps(manifest)
self._write_to_file(manifest)
def identify_tn_from_sdn_and_vl(dpid_port_ids, request_stps,
sdn_utils, transport_vlans={}):
# TN resources
nodes = []
links = []
logger.debug("Identifying TN STPs from Virtual Links + SDN resources")
logger.debug("Request STPs=%s" % str(request_stps))
for stp in request_stps:
paths = TNUtils.find_interdomain_paths_from_stps_and_dpids(
stp, dpid_port_ids)
# A path is chosen from the mapping taking into account the
# restrictions defined implicitly by the DPIDs within the flowspace
# Note: an empty list will be returned if none fits
# path = sdn_utils.find_path_containing_all(dpid_port_ids, paths)
# Thus, path is either the previously returned (or all, if empty)
# path = path or paths
# Getting the only element of list (path) or random element (paths)
rnd_path_idx = CommonUtils.get_random_list_position(len(paths))
path = paths[0] if len(paths) == 1 else paths[rnd_path_idx]
# Whatever the search space (i.e. the path) is, this is fed to the
# methods that identify how to extend the SDN flowspace to be able
# to bind the SDN domain with the stitching (virtual) domain
items, links_constraints = \
sdn_utils.analyze_mapped_path(dpid_port_ids, [path])
src_dom, dst_dom = path["src"]["tn"], path["dst"]["tn"]
node = TNUtils.generate_tn_node(
src_dom, dst_dom, transport_vlans)
src_vlan = node["interfaces"][0]["vlan"][0]["tag"]
dst_vlan = node["interfaces"][1]["vlan"][0]["tag"]
if node is None:
break
nodes.append(node)
link = TNUtils.generate_tn_link(
src_dom, src_vlan, dst_dom, dst_vlan)
links.append(link)
logger.debug("Implicit retrieval of TN STPs has concluded: \
%s" % str(links))
return (nodes, links)
def identify_tn_from_sdn_and_vl(dpid_port_ids, request_stps, sdn_utils):
# TN resources
nodes = []
links = []
logger.debug(
"Identifying TN STPs from Virtual Links and SDN resources")
logger.debug("Request STPs=%s" % str(request_stps))
for stp in request_stps:
paths = TNUtils.find_interdomain_paths_from_stps_and_dpids(
stp, dpid_port_ids)
# A path is chosen from the mapping taking into account the
# restrictions defined implicitly by the DPIDs within the flowspace
# Note: an empty list will be returned if none fits
#path = sdn_utils.find_path_containing_all(dpid_port_ids, paths)
# Thus, path is either the previously returned (or all, if empty)
#path = path or paths
# Getting the only element of list (path) or random element (paths)
rnd_path_idx = CommonUtils.get_random_list_position(len(paths))
path = paths[0] if len(paths) == 1 else paths[rnd_path_idx]
# Whatever the search space (i.e. the path) is, this is fed to the
# methods that identify how to extend the SDN flowspace to be able
# to bind the SDN domain with the stitching (virtual) domain
items, links_constraints = sdn_utils.analyze_mapped_path(
dpid_port_ids, [path])
src_dom, dst_dom = path["src"]["tn"], path["dst"]["tn"]
node = TNUtils.generate_tn_node(src_dom, dst_dom)
src_vlan = node["interfaces"][0]["vlan"][0]["tag"]
dst_vlan = node["interfaces"][1]["vlan"][0]["tag"]
if node is None:
break
nodes.append(node)
link = TNUtils.generate_tn_link(src_dom, src_vlan, dst_dom,
dst_vlan)
links.append(link)
logger.debug("Implicit retrieval of TN STPs has concluded: %s" %
str(links))
return (nodes, links)
def delete(self, package_name, version, platform,):
manifest_filename = CommonUtils.generate_manifest_filename(package_name, version, platform, "json")
self._cleanup(package_name, version, platform, manifest_filename)
def _generate_file_name(self):
return CommonUtils.generate_manifest_filename(self._package_name, self._version, self._platform, "json")
def test_depot_update(self):
self._add_snappy_to_depot()
self._update_snappy_in_depot()
assert CommonUtils.get_filecount_for_dir_tree(
DIR_DEPOT_DATAFILES) == 14
def _generate_file_name(self):
return CommonUtils.generate_manifest_filename(self._package_name,
self._version,
self._platform, "json")
def add_tn_to_ro_request_rspec(req_rspec, sdn_utils, vl_utils):
"""
Should only be run from MRO; where TN can be processed.
"""
# Run mapper path-finder to extend SDN resources
# by adding an inherent link to the SE device
# Note: check if we had an explicit/direct TN allocation
# (in this case just skip the mapper)
request_stps = []
# Retrieve virtual links
if CommonUtils.is_virtual_links(req_rspec):
vlinks = req_rspec.vl_links()
for vlink in vlinks:
(src_dom, dst_dom) = vl_utils.get_domains_from_link(vlink)
request_stps_type = vl_utils.get_type_from_link(vlink)
request_stps.append({"src_name": src_dom, "dst_name": dst_dom, \
"link_type": request_stps_type})
logger.debug("STPs=%s" % (request_stps, ))
dpid_port_ids = sdn_utils.find_dpid_port_identifiers(
req_rspec.of_groups(), req_rspec.of_matches())
logger.debug("DPIDs=%s" % (dpid_port_ids, ))
tn_nodes = []
tn_links = []
try:
tn_nodes, tn_links = TNUtils.identify_tn_from_sdn_and_vl(
dpid_port_ids, request_stps, sdn_utils)
except Exception as e:
m = "Could not obtain TN resources from SDN and VL resources. Details: %s" % str(
e)
logger.warning(m)
tnrm_formatter = TNRMv3RequestFormatter()
for n in tn_nodes:
tnrm_formatter.node(n)
for l in tn_links:
tnrm_formatter.link(l)
# Generate RSpec from request RSpec object
rspec = req_rspec.get_rspec()
tnrm_rspec = tnrm_formatter.rspec.getchildren()
for tnrm_elem in tnrm_rspec:
rspec.append(tnrm_elem)
# Clean virtual links
vlinks = rspec.xpath("xs:link//xs:link_type[contains(@name, '%s')]" %
("virtual_link"),
namespaces={"xs": DEFAULT_XMLNS})
#vlinks = rspec.xpath("xs:link//xs:link_type[@name='%s']" % ("urn:felix+virtual_link"), namespaces={"xs": DEFAULT_XMLNS})
#vlinks = rspec.findall("{%s}link//{%s}link_type[@name='%s']" % (DEFAULT_XMLNS, DEFAULT_XMLNS, "urn:felix+virtual_link"))
for v in vlinks:
v.getparent().getparent().remove(v.getparent())
rspec = etree.tostring(rspec, pretty_print=True)
logger.debug(
"Implicit request => request RSpec has been extended with proper resources"
)
logger.debug("Request RSpec passed to Allocate: %s" % str(rspec))
return rspec