def generate_random_path(**kwargs): """Generates a random directive path. Given either the apache file as a string or the apache file path. If both are given, the apache file path takes precedence. """ if 'apache_file_as_string' in kwargs: apache_conf_obj = parse_config.ParseApacheConfig( apache_file_as_string = kwargs['apache_file_as_string'] ) elif 'apache_file_path' in kwargs: apache_conf_obj = parse_config.ParseApacheConfig( apache_config_path = kwargs['apache_file_path'] ) else: raise Exception("Must give apache file path or apache file as string") conf_list = apache_conf_obj.parse_config() stack = list(reversed(conf_list)) tag_path = [] while len(stack) > 0: current = stack.pop() if isinstance(current, parse_config.NestedTags): if random.randrange(2) == 1: tag_path.append(current.open_tag) stack = list(reversed(current)) return tag_path
def parse_apache_config(filename, env, sigevent_url, verbose, eval_final_location=True, eval_staging_location=True): errors = 0 msg = [] try: apache_config = open(filename, 'r') if verbose: print('Opening config file: ' + filename + '\n') except IOError: errors += 1 error_msg = "Cannot read Apache configuration file: " + filename msg.append(error_msg) log_sig_err(error_msg, sigevent_url) return errors, error_msg apache_config_str = apache_config.read() # parse config doesn't like underscores in directives for directive in allowed_apache_directives: if "_" in directive: apache_config_str = apache_config_str.replace( directive, directive.replace("_", "")) apache_config.close() # Check for either TWMS or WMTS (default) service type service_type = "wmts" if "twms" in filename.lower(): service_type = "twms" try: apache_parse_obj = parse_config.ParseApacheConfig( apache_file_as_string=apache_config_str) apache_config = apache_parse_obj.parse_config() for top_directive in apache_config: result = eval_top_directive( top_directive, env, service_type, filename, sigevent_url, verbose, eval_final_location=eval_final_location, eval_staging_location=eval_staging_location) errors += result[0] msg += result[1] except ParseException: errors += 1 error_msg = "Invalid Apache configuration - unable to parse " + filename msg.append(error_msg) log_sig_err(error_msg, sigevent_url) error_msg = "\n" + str(errors) + " errors found:\n" + "\n".join(msg) return errors, error_msg
def test_parse_config(self): test_files = [ f for f in listdir("./test_conf_files") if isfile(join("./test_conf_files", f)) ] for file_name in test_files: pac = parse_config.ParseApacheConfig("./test_conf_files/" + file_name) conf_list = pac.parse_config()
def test_add_directive_to_root(self): conf_files = [ f for f in listdir("./test_conf_files") if isfile(join("./test_conf_files", f)) ] for conf_file in conf_files: pac = parse_config.ParseApacheConfig("./test_conf_files/" + conf_file) conf_list = pac.parse_config() conf_list = pac.add_directive(conf_list, "AddingTo", "Root Does it Work")
def test_get_apache_config(self): test_files = [ f for f in listdir("./test_conf_files") if isfile(join("./test_conf_files", f)) ] for file_name in test_files: pac = parse_config.ParseApacheConfig("./test_conf_files/" + file_name) #pp = pprint.pprint conf_list = pac.parse_config() conf_string = pac.get_apache_config(conf_list)
def test_parse_config_string_file(self): test_files = [ f for f in listdir("./test_conf_files") if isfile(join("./test_conf_files", f)) ] for file_name in test_files: full_file_path = "./test_conf_files/" + file_name with open(full_file_path, 'r') as fp: file_as_string = fp.read() pac = parse_config.ParseApacheConfig( apache_file_as_string=file_as_string) conf_list = pac.parse_config()
def test_override(self): conf_files = [ f for f in listdir("./test_conf_files") if isfile(join("./test_conf_files", f)) ] for conf_file in conf_files: pac = parse_config.ParseApacheConfig("./test_conf_files/" + conf_file) conf_list = pac.parse_config() conf_list = pac.add_directive(conf_list, "SomeDirective", "HasBeenOverwritten") duplicate_directive = self._check_for_duplicate(conf_list) self.assertFalse(duplicate_directive)
def test_add_directive(self): conf_files = [ f for f in listdir("./test_conf_files") if isfile(join("./test_conf_files", f)) ] for conf_file in conf_files: conf_file = "./test_conf_files/" + conf_file pac = parse_config.ParseApacheConfig(conf_file) tag_path = test_utils.generate_random_path( apache_file_path=conf_file) conf_list = pac.parse_config() conf_list = pac.add_directive(conf_list, "SomeRandomDirective", "Some +ARGS", *tag_path)
def test_file_diff(self): """ This method takes the output of get_apache_config, and diffs it against the orignal file, ignoring whitespace. This will test to see how close to the original file the output of get_apache_config is. """ test_files = [ f for f in listdir("./test_conf_files") if isfile(join("./test_conf_files", f)) ] for file_name in test_files: file_path = "./test_conf_files/" + file_name with open(file_path, "r") as apache_config: file_string = apache_config.read() pac = parse_config.ParseApacheConfig(file_path) conf_list = pac.parse_config() conf_string = pac.get_apache_config(conf_list) conf_string = conf_string.replace(" ", "") conf_string = conf_string.replace("\t", "") file_string = file_string.replace(" ", "") file_string = file_string.replace("\t", "") s = difflib.SequenceMatcher(None, conf_string, file_string) self.assertTrue(s.real_quick_ratio() == 1.0)
def evaluate_ReprojectConfigurationFiles(source_config, reproject_config, env, service_type, sigevent_url, verbose): errors = 0 msg = [] # Figure out the layer_name from file path layer_name = os.path.basename(source_config).replace("_source.config", "") # Update SourcePath with layer_name SourcePath = re.compile(r"^/(?:\w+/)+%s/default/(\$\{date\}\/)?\S+$" % layer_name) # Validate source reproject config if verbose: print "Validating " + source_config if not os.path.isfile(source_config): errors += 1 error_msg = source_config + " is not a valid location" msg.append(error_msg) log_sig_err(error_msg, sigevent_url) return errors, msg apache_parse_obj = parse_config.ParseApacheConfig( apache_config_path=source_config) source_apache_config = apache_parse_obj.parse_config() # List of allowed source reproject config directives allowed_source_reproject_directives = [ "Size", "PageSize", "SkippedLevels", "BoundingBox", "Projection" ] # Add regular expressions to lookup dictionary source_reproject_directive_rules = {} source_reproject_directive_rules["Size"] = Size source_reproject_directive_rules["PageSize"] = PageSize source_reproject_directive_rules["SkippedLevels"] = SkippedLevels source_reproject_directive_rules["BoundingBox"] = BoundingBox source_reproject_directive_rules["Projection"] = Projection # Evaluate directives for directive in source_apache_config: result = eval_directive( directive, env, service_type, source_config, sigevent_url, verbose, allowed_apache_directives=allowed_source_reproject_directives, directive_rules=source_reproject_directive_rules) errors += result[0] msg += result[1] # Validate source reproject config if verbose: print "Validating " + reproject_config if not os.path.isfile(reproject_config): errors += 1 error_msg = reproject_config + " is not a valid location" msg.append(error_msg) log_sig_err(error_msg, sigevent_url) return errors, msg apache_parse_obj = parse_config.ParseApacheConfig( apache_config_path=reproject_config) reproject_apache_config = apache_parse_obj.parse_config() # List of allowed reproject config directives allowed_reproject_directives = [ "Size", "PageSize", "BoundingBox", "Projection", "Nearest", "SourcePath", "SourcePostfix", "MimeType", "Oversample", "ExtraLevels", "Quality", "ETagSeed" ] # Add regular expressions to lookup dictionary reproject_directive_rules = {} reproject_directive_rules["Size"] = Size reproject_directive_rules["PageSize"] = PageSize reproject_directive_rules["BoundingBox"] = BoundingBox reproject_directive_rules["Projection"] = Projection reproject_directive_rules["Nearest"] = Nearest reproject_directive_rules["SourcePath"] = SourcePath reproject_directive_rules["SourcePostfix"] = SourcePostfix reproject_directive_rules["MimeType"] = MimeType reproject_directive_rules["Oversample"] = Oversample reproject_directive_rules["ExtraLevels"] = ExtraLevels reproject_directive_rules["Quality"] = Quality reproject_directive_rules["ETagSeed"] = ETagSeed # Evaluate directives for directive in reproject_apache_config: result = eval_directive( directive, env, service_type, reproject_config, sigevent_url, verbose, allowed_apache_directives=allowed_reproject_directives, directive_rules=reproject_directive_rules) errors += result[0] msg += result[1] return errors, msg
def evaluate_tWMSConfigurationFile(config, env, service_type, sigevent_url, verbose): errors = 0 msg = [] # Evaluate tWMS_ConfigurationFile tWMS_ConfigurationFile = re.compile( r"^(%s|%s)(\$\{layer\}\/)twms.config$" % (add_trailing_slash(str(env.reprojectLayerConfigLocation_twms)), add_trailing_slash(env.twms_dir))) research = tWMS_ConfigurationFile.search(config) if research == None: errors += 1 error_msg = "Incorrect pattern found for tWMS_ConfigurationFile " + config msg.append(error_msg) log_sig_err(error_msg, sigevent_url) return errors, msg # List of allowed directives allowed_twms_directives = [ "Size", "PageSize", "SkippedLevels", "BoundingBox", "SourcePath", "SourcePostfix" ] # Search for all matching TWMS configs config_base_path = config.split("${layer}")[0] if verbose: print "Search for TWMS configuration files in " + config_base_path if not os.path.isdir(config_base_path): errors += 1 error_msg = config_base_path + " is not a valid location" msg.append(error_msg) log_sig_err(error_msg, sigevent_url) return errors, msg else: configsearch = config_base_path + "*/*twms.config" if len(glob.glob(configsearch)) == 0: errors += 1 error_msg = config + " contains no TWMS config files" msg.append(error_msg) log_sig_err(error_msg, configsearch) return errors, msg else: for twms_config_file in glob.glob(configsearch): # Figure out the correct source path source_path = env.reprojectEndpoint_wmts + "/" + twms_config_file.replace( config_base_path, "").replace("twms.config", "") SourcePath = re.compile( r"^%sdefault/(\$\{date\}\/)?GoogleMapsCompatible_Level[0-9]{1,2}$" % source_path) # Add regular expressions to lookup dictionary twms_directive_rules = {} twms_directive_rules["Size"] = Size twms_directive_rules["PageSize"] = PageSize twms_directive_rules["SkippedLevels"] = SkippedLevels twms_directive_rules["BoundingBox"] = BoundingBox twms_directive_rules["SourcePath"] = SourcePath twms_directive_rules["SourcePostfix"] = SourcePostfix if verbose: print "Validating " + twms_config_file apache_parse_obj = parse_config.ParseApacheConfig( apache_config_path=twms_config_file) twms_config = apache_parse_obj.parse_config() # Evaluate directives for directive in twms_config: result = eval_directive( directive, env, service_type, config, sigevent_url, verbose, allowed_apache_directives=allowed_twms_directives, directive_rules=twms_directive_rules) errors += result[0] msg += result[1] return errors, msg
help="Use cached parsed config.", action="store_true") arguments = parser.parse_args() APACHE_CONF_FILE = arguments.input OUTPUT_FILE = arguments.output DESIRED_VHOST = arguments.servername FROM_CACHE = arguments.cache if FROM_CACHE: with open('data.pkl', 'rb') as f: apache_config = pickle.load(f) else: with open('data.pkl', 'wb') as f: apache_parse_obj = parse_config.ParseApacheConfig( apache_config_path=APACHE_CONF_FILE) apache_config = apache_parse_obj.parse_config() pickle.dump(apache_config, f, pickle.HIGHEST_PROTOCOL) def remove_quotes(_str): if _str.startswith('"'): _str = _str[1:] if _str.endswith('"'): _str = _str[:-1] return _str def sepatare_server_names(directive_name, dir_inst, parse_dict): if directive_name in dir_inst.name: parse_dict[directive_name] = dir_inst.args