def generate_random_path(**kwargs):
"""Generates a random directive path.
Given either the apache file as a string or the apache file path.
If both are given, the apache file path takes precedence.
"""
if 'apache_file_as_string' in kwargs:
apache_conf_obj = parse_config.ParseApacheConfig(
apache_file_as_string = kwargs['apache_file_as_string']
)
elif 'apache_file_path' in kwargs:
apache_conf_obj = parse_config.ParseApacheConfig(
apache_config_path = kwargs['apache_file_path']
)
else:
raise Exception("Must give apache file path or apache file as string")
conf_list = apache_conf_obj.parse_config()
stack = list(reversed(conf_list))
tag_path = []
while len(stack) > 0:
current = stack.pop()
if isinstance(current, parse_config.NestedTags):
if random.randrange(2) == 1:
tag_path.append(current.open_tag)
stack = list(reversed(current))
return tag_path
def parse_apache_config(filename,
env,
sigevent_url,
verbose,
eval_final_location=True,
eval_staging_location=True):
errors = 0
msg = []
try:
apache_config = open(filename, 'r')
if verbose:
print('Opening config file: ' + filename + '\n')
except IOError:
errors += 1
error_msg = "Cannot read Apache configuration file: " + filename
msg.append(error_msg)
log_sig_err(error_msg, sigevent_url)
return errors, error_msg
apache_config_str = apache_config.read()
# parse config doesn't like underscores in directives
for directive in allowed_apache_directives:
if "_" in directive:
apache_config_str = apache_config_str.replace(
directive, directive.replace("_", ""))
apache_config.close()
# Check for either TWMS or WMTS (default) service type
service_type = "wmts"
if "twms" in filename.lower():
service_type = "twms"
try:
apache_parse_obj = parse_config.ParseApacheConfig(
apache_file_as_string=apache_config_str)
apache_config = apache_parse_obj.parse_config()
for top_directive in apache_config:
result = eval_top_directive(
top_directive,
env,
service_type,
filename,
sigevent_url,
verbose,
eval_final_location=eval_final_location,
eval_staging_location=eval_staging_location)
errors += result[0]
msg += result[1]
except ParseException:
errors += 1
error_msg = "Invalid Apache configuration - unable to parse " + filename
msg.append(error_msg)
log_sig_err(error_msg, sigevent_url)
error_msg = "\n" + str(errors) + " errors found:\n" + "\n".join(msg)
return errors, error_msg
def test_parse_config(self):
test_files = [
f for f in listdir("./test_conf_files")
if isfile(join("./test_conf_files", f))
]
for file_name in test_files:
pac = parse_config.ParseApacheConfig("./test_conf_files/" +
file_name)
conf_list = pac.parse_config()
def test_add_directive_to_root(self):
conf_files = [
f for f in listdir("./test_conf_files")
if isfile(join("./test_conf_files", f))
]
for conf_file in conf_files:
pac = parse_config.ParseApacheConfig("./test_conf_files/" +
conf_file)
conf_list = pac.parse_config()
conf_list = pac.add_directive(conf_list, "AddingTo",
"Root Does it Work")
def test_get_apache_config(self):
test_files = [
f for f in listdir("./test_conf_files")
if isfile(join("./test_conf_files", f))
]
for file_name in test_files:
pac = parse_config.ParseApacheConfig("./test_conf_files/" +
file_name)
#pp = pprint.pprint
conf_list = pac.parse_config()
conf_string = pac.get_apache_config(conf_list)
def test_parse_config_string_file(self):
test_files = [
f for f in listdir("./test_conf_files")
if isfile(join("./test_conf_files", f))
]
for file_name in test_files:
full_file_path = "./test_conf_files/" + file_name
with open(full_file_path, 'r') as fp:
file_as_string = fp.read()
pac = parse_config.ParseApacheConfig(
apache_file_as_string=file_as_string)
conf_list = pac.parse_config()
def test_override(self):
conf_files = [
f for f in listdir("./test_conf_files")
if isfile(join("./test_conf_files", f))
]
for conf_file in conf_files:
pac = parse_config.ParseApacheConfig("./test_conf_files/" +
conf_file)
conf_list = pac.parse_config()
conf_list = pac.add_directive(conf_list, "SomeDirective",
"HasBeenOverwritten")
duplicate_directive = self._check_for_duplicate(conf_list)
self.assertFalse(duplicate_directive)
def test_add_directive(self):
conf_files = [
f for f in listdir("./test_conf_files")
if isfile(join("./test_conf_files", f))
]
for conf_file in conf_files:
conf_file = "./test_conf_files/" + conf_file
pac = parse_config.ParseApacheConfig(conf_file)
tag_path = test_utils.generate_random_path(
apache_file_path=conf_file)
conf_list = pac.parse_config()
conf_list = pac.add_directive(conf_list, "SomeRandomDirective",
"Some +ARGS", *tag_path)
def test_file_diff(self):
"""
This method takes the output of get_apache_config, and diffs
it against the orignal file, ignoring whitespace. This will test
to see how close to the original file the output of get_apache_config
is.
"""
test_files = [
f for f in listdir("./test_conf_files")
if isfile(join("./test_conf_files", f))
]
for file_name in test_files:
file_path = "./test_conf_files/" + file_name
with open(file_path, "r") as apache_config:
file_string = apache_config.read()
pac = parse_config.ParseApacheConfig(file_path)
conf_list = pac.parse_config()
conf_string = pac.get_apache_config(conf_list)
conf_string = conf_string.replace(" ", "")
conf_string = conf_string.replace("\t", "")
file_string = file_string.replace(" ", "")
file_string = file_string.replace("\t", "")
s = difflib.SequenceMatcher(None, conf_string, file_string)
self.assertTrue(s.real_quick_ratio() == 1.0)
def evaluate_ReprojectConfigurationFiles(source_config, reproject_config, env,
service_type, sigevent_url, verbose):
errors = 0
msg = []
# Figure out the layer_name from file path
layer_name = os.path.basename(source_config).replace("_source.config", "")
# Update SourcePath with layer_name
SourcePath = re.compile(r"^/(?:\w+/)+%s/default/(\$\{date\}\/)?\S+$" %
layer_name)
# Validate source reproject config
if verbose:
print "Validating " + source_config
if not os.path.isfile(source_config):
errors += 1
error_msg = source_config + " is not a valid location"
msg.append(error_msg)
log_sig_err(error_msg, sigevent_url)
return errors, msg
apache_parse_obj = parse_config.ParseApacheConfig(
apache_config_path=source_config)
source_apache_config = apache_parse_obj.parse_config()
# List of allowed source reproject config directives
allowed_source_reproject_directives = [
"Size", "PageSize", "SkippedLevels", "BoundingBox", "Projection"
]
# Add regular expressions to lookup dictionary
source_reproject_directive_rules = {}
source_reproject_directive_rules["Size"] = Size
source_reproject_directive_rules["PageSize"] = PageSize
source_reproject_directive_rules["SkippedLevels"] = SkippedLevels
source_reproject_directive_rules["BoundingBox"] = BoundingBox
source_reproject_directive_rules["Projection"] = Projection
# Evaluate directives
for directive in source_apache_config:
result = eval_directive(
directive,
env,
service_type,
source_config,
sigevent_url,
verbose,
allowed_apache_directives=allowed_source_reproject_directives,
directive_rules=source_reproject_directive_rules)
errors += result[0]
msg += result[1]
# Validate source reproject config
if verbose:
print "Validating " + reproject_config
if not os.path.isfile(reproject_config):
errors += 1
error_msg = reproject_config + " is not a valid location"
msg.append(error_msg)
log_sig_err(error_msg, sigevent_url)
return errors, msg
apache_parse_obj = parse_config.ParseApacheConfig(
apache_config_path=reproject_config)
reproject_apache_config = apache_parse_obj.parse_config()
# List of allowed reproject config directives
allowed_reproject_directives = [
"Size", "PageSize", "BoundingBox", "Projection", "Nearest",
"SourcePath", "SourcePostfix", "MimeType", "Oversample", "ExtraLevels",
"Quality", "ETagSeed"
]
# Add regular expressions to lookup dictionary
reproject_directive_rules = {}
reproject_directive_rules["Size"] = Size
reproject_directive_rules["PageSize"] = PageSize
reproject_directive_rules["BoundingBox"] = BoundingBox
reproject_directive_rules["Projection"] = Projection
reproject_directive_rules["Nearest"] = Nearest
reproject_directive_rules["SourcePath"] = SourcePath
reproject_directive_rules["SourcePostfix"] = SourcePostfix
reproject_directive_rules["MimeType"] = MimeType
reproject_directive_rules["Oversample"] = Oversample
reproject_directive_rules["ExtraLevels"] = ExtraLevels
reproject_directive_rules["Quality"] = Quality
reproject_directive_rules["ETagSeed"] = ETagSeed
# Evaluate directives
for directive in reproject_apache_config:
result = eval_directive(
directive,
env,
service_type,
reproject_config,
sigevent_url,
verbose,
allowed_apache_directives=allowed_reproject_directives,
directive_rules=reproject_directive_rules)
errors += result[0]
msg += result[1]
return errors, msg
def evaluate_tWMSConfigurationFile(config, env, service_type, sigevent_url,
verbose):
errors = 0
msg = []
# Evaluate tWMS_ConfigurationFile
tWMS_ConfigurationFile = re.compile(
r"^(%s|%s)(\$\{layer\}\/)twms.config$" %
(add_trailing_slash(str(env.reprojectLayerConfigLocation_twms)),
add_trailing_slash(env.twms_dir)))
research = tWMS_ConfigurationFile.search(config)
if research == None:
errors += 1
error_msg = "Incorrect pattern found for tWMS_ConfigurationFile " + config
msg.append(error_msg)
log_sig_err(error_msg, sigevent_url)
return errors, msg
# List of allowed directives
allowed_twms_directives = [
"Size", "PageSize", "SkippedLevels", "BoundingBox", "SourcePath",
"SourcePostfix"
]
# Search for all matching TWMS configs
config_base_path = config.split("${layer}")[0]
if verbose:
print "Search for TWMS configuration files in " + config_base_path
if not os.path.isdir(config_base_path):
errors += 1
error_msg = config_base_path + " is not a valid location"
msg.append(error_msg)
log_sig_err(error_msg, sigevent_url)
return errors, msg
else:
configsearch = config_base_path + "*/*twms.config"
if len(glob.glob(configsearch)) == 0:
errors += 1
error_msg = config + " contains no TWMS config files"
msg.append(error_msg)
log_sig_err(error_msg, configsearch)
return errors, msg
else:
for twms_config_file in glob.glob(configsearch):
# Figure out the correct source path
source_path = env.reprojectEndpoint_wmts + "/" + twms_config_file.replace(
config_base_path, "").replace("twms.config", "")
SourcePath = re.compile(
r"^%sdefault/(\$\{date\}\/)?GoogleMapsCompatible_Level[0-9]{1,2}$"
% source_path)
# Add regular expressions to lookup dictionary
twms_directive_rules = {}
twms_directive_rules["Size"] = Size
twms_directive_rules["PageSize"] = PageSize
twms_directive_rules["SkippedLevels"] = SkippedLevels
twms_directive_rules["BoundingBox"] = BoundingBox
twms_directive_rules["SourcePath"] = SourcePath
twms_directive_rules["SourcePostfix"] = SourcePostfix
if verbose:
print "Validating " + twms_config_file
apache_parse_obj = parse_config.ParseApacheConfig(
apache_config_path=twms_config_file)
twms_config = apache_parse_obj.parse_config()
# Evaluate directives
for directive in twms_config:
result = eval_directive(
directive,
env,
service_type,
config,
sigevent_url,
verbose,
allowed_apache_directives=allowed_twms_directives,
directive_rules=twms_directive_rules)
errors += result[0]
msg += result[1]
return errors, msg
help="Use cached parsed config.",
action="store_true")
arguments = parser.parse_args()
APACHE_CONF_FILE = arguments.input
OUTPUT_FILE = arguments.output
DESIRED_VHOST = arguments.servername
FROM_CACHE = arguments.cache
if FROM_CACHE:
with open('data.pkl', 'rb') as f:
apache_config = pickle.load(f)
else:
with open('data.pkl', 'wb') as f:
apache_parse_obj = parse_config.ParseApacheConfig(
apache_config_path=APACHE_CONF_FILE)
apache_config = apache_parse_obj.parse_config()
pickle.dump(apache_config, f, pickle.HIGHEST_PROTOCOL)
def remove_quotes(_str):
if _str.startswith('"'):
_str = _str[1:]
if _str.endswith('"'):
_str = _str[:-1]
return _str
def sepatare_server_names(directive_name, dir_inst, parse_dict):
if directive_name in dir_inst.name:
parse_dict[directive_name] = dir_inst.args