def test_merge_dict(self):
_logger.info("test_merge_dict")
for js1, js2, expect in test_cases_mergedict:
result = common.merge_object(js1, js2)
print(json.dumps(result))
self.assertEqual(json.dumps(result), json.dumps(expect))
_logger.info(
"====>\t ok : queryString=[{0}] stringToReplace=[{1}] result=[{2}]"
.format(js1, js2, result))
def prepare_isolate(self,
uid,
name,
kind,
level,
sublevel,
bundles=None,
composition=None):
"""
Prepares and returns a configuration dictionary to be stored in the
configuration broker, to start an isolate of the given kind.
:param uid: The isolate UID
:param name: The isolate name
:param kind: The kind of isolate to boot (pelix, osgi, ...)
:param level: The level of configuration (boot, java, python, ...)
:param sublevel: Category of configuration (monitor, isolate, ...)
:param bundles: Extra bundles to install
:param composition: Extra components to instantiate
:return: A configuration dictionary
:raise IOError: Unknown/unaccessible kind of isolate
:raise KeyError: A parameter is missing in the configuration files
:raise ValueError: Error reading the configuration
"""
# Load the isolate model file
configuration = self.load_conf_raw(level, sublevel)
_logger.info("load isolate conf file {}.js".format(name))
try:
# Try to load the isolate-specific configuration
# without logging "file not found" errors
isolate_conf = self.read(name + ".js", False)
if isolate_conf == None:
isolate_conf = self.read("isolate_" + name + ".js", False)
except Exception:
try:
isolate_conf = self.read("isolate_" + name + ".js", False)
except Exception as e:
_logger.error("can't load isolate_config=[{}]".format(name))
_logger.exception(e)
raise e
if isolate_conf is not None:
# Merge the configurations: this method considers that the first
# parameter has priority on the second
configuration = common.merge_object(isolate_conf, configuration)
_logger.debug("isolate configuration = {}".format(configuration))
# Extend with the boot configuration
return self._prepare_configuration(uid, name, kind, bundles,
composition, configuration)
def get_content(self, filename, want_json=False):
"""
@param filename: path we want the content. the filename can be a absolute path, a path with key value params or tag to specify what content we want.
e.g :
- file://mydir/myfile.js => this just get the content of the file myfile
- file://mydir.myfile.js?k1=v1&k2=v2 => this will get the content of the file myfile but it will replace all variable ${k1} and ${k2} by v1 and v2
- file://mydir.myfile.js?k1=v1&k2=v2#myprop => this will get the only the property myprop of the content file myfile wth replace variable
@param wantJson : boolean to defined if we want a json object as a result or a string
return a resolved content json string without commentary
"""
# multi path asked if the filename contains ; separator
if filename.find(";") != -1:
content = ",".join(
[self._get_content(name) for name in filename.split(";")])
else:
content = self._get_content(filename)
merge_content = None
if content == None:
raise IOError("file {0} doesn't exists".format(filename))
json_contents = json.loads("[" + content + "]")
for json_content in json_contents:
if isinstance(json_content, dict):
# must be always a dict to append all json dict
if merge_content == None:
merge_content = json.loads("{}")
merge_content = common.merge_object(merge_content,
json_content)
elif isinstance(json_content, list):
# must be always a list to append all json arrays
if merge_content == None:
merge_content = json.loads("[]")
for arr in json_content:
merge_content.append(arr)
if merge_content == None:
raise IOError("{0} doesn't exists ".format(filename))
if not want_json:
return json.dumps(merge_content)
_logger.debug("content=[{}]".format(json.dumps(merge_content)))
return merge_content
def prepare_isolate(self, uid, name, kind, level, sublevel,
bundles=None, composition=None):
"""
Prepares and returns a configuration dictionary to be stored in the
configuration broker, to start an isolate of the given kind.
:param uid: The isolate UID
:param name: The isolate name
:param kind: The kind of isolate to boot (pelix, osgi, ...)
:param level: The level of configuration (boot, java, python, ...)
:param sublevel: Category of configuration (monitor, isolate, ...)
:param bundles: Extra bundles to install
:param composition: Extra components to instantiate
:return: A configuration dictionary
:raise IOError: Unknown/unaccessible kind of isolate
:raise KeyError: A parameter is missing in the configuration files
:raise ValueError: Error reading the configuration
"""
# Load the isolate model file
configuration = self.load_conf_raw(level, sublevel)
_logger.info("load isolate conf file {}.js".format(name))
try:
# Try to load the isolate-specific configuration
# without logging "file not found" errors
isolate_conf = self.read(name + ".js", False)
if isolate_conf == None:
isolate_conf = self.read("isolate_" + name + ".js", False)
except Exception:
try:
isolate_conf = self.read("isolate_" + name + ".js", False)
except Exception as e:
_logger.error("can't load isolate_config=[{}]".format(name))
_logger.exception(e)
raise e
if isolate_conf is not None:
# Merge the configurations: this method considers that the first
# parameter has priority on the second
configuration = common.merge_object(isolate_conf,
configuration)
_logger.debug("isolate configuration = {}".format(configuration))
# Extend with the boot configuration
return self._prepare_configuration(uid, name, kind,
bundles, composition, configuration)
def get_content(self, filename, want_json=False):
"""
@param filename: path we want the content. the filename can be a absolute path, a path with key value params or tag to specify what content we want.
e.g :
- file://mydir/myfile.js => this just get the content of the file myfile
- file://mydir.myfile.js?k1=v1&k2=v2 => this will get the content of the file myfile but it will replace all variable ${k1} and ${k2} by v1 and v2
- file://mydir.myfile.js?k1=v1&k2=v2#myprop => this will get the only the property myprop of the content file myfile wth replace variable
@param wantJson : boolean to defined if we want a json object as a result or a string
return a resolved content json string without commentary
"""
# multi path asked if the filename contains ; separator
if filename.find(";") != -1:
content = ",".join([self._get_content(name) for name in filename.split(";")])
else:
content = self._get_content(filename)
merge_content = None
if content == None:
raise IOError("file {0} doesn't exists".format(filename))
json_contents = json.loads("[" + content + "]")
for json_content in json_contents:
if isinstance(json_content, dict):
# must be always a dict to append all json dict
if merge_content == None:
merge_content = json.loads("{}")
merge_content = common.merge_object(merge_content, json_content)
elif isinstance(json_content, list):
# must be always a list to append all json arrays
if merge_content == None:
merge_content = json.loads("[]")
for arr in json_content:
merge_content.append(arr)
if merge_content == None:
raise IOError("{0} doesn't exists ".format(filename))
if not want_json:
return json.dumps(merge_content)
_logger.debug("content=[{}]".format(json.dumps(merge_content)))
return merge_content
def _resolve_content(self, resource):
"""
return a resolve content with all include file content
"""
_logger.debug("_revolveContent")
contents = resource.get_contents()
if contents != None:
resolved_contents = []
for content in contents:
resolved_content = content
# apply regexp to remove content
# load str as json and find all $include match
resolved_json = json.loads(resolved_content)
for found_match in self._find_match_include(resolved_json):
# found_match = self._get_include_match(matches)
_logger.debug(
"_revolveContent: match found {0}".format(found_match))
sub_contents = []
# match_json = json.loads(found_match)
match_json = found_match
found_match = json.dumps(found_match)
if match_json != None:
if "$include" in match_json.keys():
# match is { $include: "file:///path of a file"}
# check condition property if it exists
include_matched = match_json["$include"]
if (self._is_condition_include(include_matched)):
for path in self._get_include_path(
include_matched):
_logger.debug(
"_revolveContent: $include - subContentPath {0}"
.format(path))
sub_content = self._get_content(
path, resource)
if sub_content != None and sub_content != "":
sub_contents.append(sub_content)
if len(sub_contents) > 0:
resolved_content = resolved_content.replace(
found_match,
str.join(",", sub_contents))
else:
resolved_content = resolved_content.replace(
found_match, "{}")
else:
resolved_content = resolved_content.replace(
found_match, "{}")
# apply regexp to remove content
for matches in self._merge.findall(resolved_content):
found_match = self._get_include_match(matches)
_logger.debug(
"_revolveContent: match found {0}".format(found_match))
sub_contents = []
# load json to get the path and resolve it
idx_sep = found_match.find(":")
end_coma = found_match.endswith(",")
start_coma = found_match.startswith(",")
if end_coma:
match_json = json.loads(found_match[idx_sep + 1:-1])
else:
match_json = json.loads(found_match[idx_sep + 1:])
if match_json != None:
resolved_content = json.loads(
resolved_content.replace(
found_match,
"," if end_coma and start_coma else ""))
for path in match_json:
_logger.debug(
"_revolveContent: $merge - subContentPath {0}".
format(path))
# merge this json with the current one
w_content = self._get_content(path, resource)
if w_content != None:
_logger.debug(
"_revolveContent: $merge - subContentPath not null {0}, content={1}"
.format(path, w_content))
to_merges = json.loads("[" + w_content + "]")
for to_merge in to_merges:
resolved_content = common.merge_object(
resolved_content, to_merge)
else:
_logger.debug(
"_revolveContent: $merge - subContentPath not null {0}"
.format(path))
resolved_content = json.dumps(resolved_content)
# replace match by list of subcontent
resolved_contents.append(resolved_content)
# check if the json is ok and reformat it for the correct application of the regexp
resource.set_contents(resolved_contents)
def _do_recursive_imports(self, filename, json_data, overridden_props,
include_stack):
"""
Recursively does the file merging according to the indications in the
given JSON data (object or array).
**TODO:**
* Refactor to avoid duplicated lines
:param filename: The name of the file used to parse this data
(can be None)
:param json_data: A parsed JSON data (object or array)
:param overridden_props: Properties to override in imported files
:return: The combined JSON result
:raise ValueError: Error parsing an imported JSON file
:raise IOError: Error reading an imported JSON file
"""
if type(json_data) is list:
# We have an array, update all of its children
i = 0
while i < len(json_data):
entry = json_data[i]
new_data = self._do_recursive_imports(
filename, entry, overridden_props, include_stack)
if new_data is not entry:
# The entry has been updated
json_data[i] = new_data
i += 1
return json_data
elif type(json_data) is dict:
# We have an object
from_filename = json_data.get(KEY_FILE_FROM)
import_filenames = json_data.get(KEY_FILES_IMPORT)
if from_filename and not filename.endswith(from_filename):
# Load & return the content of the imported file
# (this method will be called by _load_file)
new_props = self._compute_overridden_props(json_data,
overridden_props)
imported_data = self._load_file(from_filename, filename,
new_props, include_stack)
# Update properties
self._update_properties(imported_data, new_props)
# Return the imported object
return imported_data
elif import_filenames:
# Load the content of the imported file and merge with local
# values, i.e. add entries existing only in the imported file
# and merge arrays
if type(import_filenames) is not list:
import_filenames = [import_filenames]
# Remove import keys
del json_data[KEY_FILES_IMPORT]
if KEY_OVERRIDDEN_PROPERTIES in json_data:
del json_data[KEY_OVERRIDDEN_PROPERTIES]
for import_filename in import_filenames:
if not filename.endswith(import_filename):
# Import files
imported_data = self._load_file(
import_filename, filename, overridden_props,
include_stack)
# Update properties in imported data
self._update_properties(imported_data, overridden_props)
# Merge arrays with imported data
json_data = common.merge_object(json_data, imported_data)
# Do the recursive import
for key, value in json_data.items():
new_value = self._do_recursive_imports(
filename, value, overridden_props, include_stack)
if new_value is not value:
# The value has been changed
json_data[key] = value
return json_data
else:
# Standard object, look into its entries
for key, value in json_data.items():
new_value = self._do_recursive_imports(
filename, value, overridden_props, include_stack)
if new_value is not value:
# The value has been changed
json_data[key] = new_value
return json_data
# Nothing to do
return json_data
def _resolve_content(self, resource):
"""
return a resolve content with all include file content
"""
_logger.debug("_revolveContent")
contents = resource.get_contents();
if contents != None:
resolved_contents = []
for content in contents:
resolved_content = content
# apply regexp to remove content
# load str as json and find all $include match
resolved_json = json.loads(resolved_content)
for found_match in self._find_match_include(resolved_json):
# found_match = self._get_include_match(matches)
_logger.debug("_revolveContent: match found {0}".format(found_match))
sub_contents = []
# match_json = json.loads(found_match)
match_json = found_match
found_match = json.dumps(found_match)
if match_json != None:
if "$include" in match_json.keys():
# match is { $include: "file:///path of a file"}
# check condition property if it exists
include_matched = match_json["$include"]
if(self._is_condition_include(include_matched)):
for path in self._get_include_path(include_matched):
_logger.debug("_revolveContent: $include - subContentPath {0}".format(path))
sub_content = self._get_content(path, resource)
if sub_content != None and sub_content != "":
sub_contents.append(sub_content)
if len(sub_contents) > 0 :
resolved_content = resolved_content.replace(found_match, str.join(",", sub_contents))
else:
resolved_content = resolved_content.replace(found_match, "{}")
else:
resolved_content = resolved_content.replace(found_match, "{}")
# apply regexp to remove content
for matches in self._merge.findall(resolved_content):
found_match = self._get_include_match(matches)
_logger.debug("_revolveContent: match found {0}".format(found_match))
sub_contents = []
# load json to get the path and resolve it
idx_sep = found_match.find(":")
end_coma = found_match.endswith(",")
start_coma = found_match.startswith(",")
if end_coma:
match_json = json.loads(found_match[idx_sep + 1:-1])
else:
match_json = json.loads(found_match[idx_sep + 1:])
if match_json != None:
resolved_content = json.loads(
resolved_content.replace(found_match, "," if end_coma and start_coma else ""))
for path in match_json:
_logger.debug("_revolveContent: $merge - subContentPath {0}".format(path))
# merge this json with the current one
w_content = self._get_content(path, resource)
if w_content != None:
_logger.debug("_revolveContent: $merge - subContentPath not null {0}, content={1}".format(path, w_content))
to_merges = json.loads("[" + w_content + "]")
for to_merge in to_merges:
resolved_content = common.merge_object(resolved_content, to_merge)
else:
_logger.debug("_revolveContent: $merge - subContentPath not null {0}".format(path))
resolved_content = json.dumps(resolved_content)
# replace match by list of subcontent
resolved_contents.append(resolved_content);
# check if the json is ok and reformat it for the correct application of the regexp
resource.set_contents(resolved_contents)
def _do_recursive_imports(self, filename, json_data, overridden_props,
include_stack):
"""
Recursively does the file merging according to the indications in the
given JSON data (object or array).
**TODO:**
* Refactor to avoid duplicated lines
:param filename: The name of the file used to parse this data
(can be None)
:param json_data: A parsed JSON data (object or array)
:param overridden_props: Properties to override in imported files
:return: The combined JSON result
:raise ValueError: Error parsing an imported JSON file
:raise IOError: Error reading an imported JSON file
"""
if type(json_data) is list:
# We have an array, update all of its children
i = 0
while i < len(json_data):
entry = json_data[i]
new_data = self._do_recursive_imports(filename, entry,
overridden_props,
include_stack)
if new_data is not entry:
# The entry has been updated
json_data[i] = new_data
i += 1
return json_data
elif type(json_data) is dict:
# We have an object
from_filename = json_data.get(KEY_FILE_FROM)
import_filenames = json_data.get(KEY_FILES_IMPORT)
if from_filename and not filename.endswith(from_filename):
# Load & return the content of the imported file
# (this method will be called by _load_file)
new_props = self._compute_overridden_props(
json_data, overridden_props)
imported_data = self._load_file(from_filename, filename,
new_props, include_stack)
# Update properties
self._update_properties(imported_data, new_props)
# Return the imported object
return imported_data
elif import_filenames:
# Load the content of the imported file and merge with local
# values, i.e. add entries existing only in the imported file
# and merge arrays
if type(import_filenames) is not list:
import_filenames = [import_filenames]
# Remove import keys
del json_data[KEY_FILES_IMPORT]
if KEY_OVERRIDDEN_PROPERTIES in json_data:
del json_data[KEY_OVERRIDDEN_PROPERTIES]
for import_filename in import_filenames:
if not filename.endswith(import_filename):
# Import files
imported_data = self._load_file(
import_filename, filename, overridden_props,
include_stack)
# Update properties in imported data
self._update_properties(imported_data,
overridden_props)
# Merge arrays with imported data
json_data = common.merge_object(
json_data, imported_data)
# Do the recursive import
for key, value in json_data.items():
new_value = self._do_recursive_imports(
filename, value, overridden_props,
include_stack)
if new_value is not value:
# The value has been changed
json_data[key] = value
return json_data
else:
# Standard object, look into its entries
for key, value in json_data.items():
new_value = self._do_recursive_imports(
filename, value, overridden_props, include_stack)
if new_value is not value:
# The value has been changed
json_data[key] = new_value
return json_data
# Nothing to do
return json_data