Esempio n. 1
0
def read_conf(conf_path, section_name=None, defaults=None, use_yaml=False):
    if use_yaml:
        return parse_config(conf_path)
    if defaults is None:
        defaults = {}
    parser = SafeConfigParser(defaults)
    success = parser.read(conf_path)
    if not success:
        print("Unable to read config from %s" % conf_path)
        sys.exit(1)
    if section_name:
        if parser.has_section(section_name):

            # if log_format is set, extract it from the parser
            # to prevent to expand variables which can, in case of
            # log_format throw a ConfigParser.InterpolationMissingOptionError
            log_format = None
            if parser.has_option(section_name, 'log_format'):
                log_format = parser.get(section_name, 'log_format', raw=True)
                # don't use remove_options because it can fail without reason
                parser.set(section_name, 'log_format', '')

            conf = dict(parser.items(section_name))

            # Add log_format again, after parsing
            if log_format:
                conf['log_format'] = log_format

        else:
            print('Unable to find section %s in config %s' %
                  (section_name, conf_path))
            exit(1)
    else:
        conf = {}
        for section in parser.sections():

            # if log_format is set, extract it from the parser
            # to prevent to expand variables which can, in case of
            # log_format throw a ConfigParser.InterpolationMissingOptionError
            log_format = None
            if parser.has_option(section, 'log_format'):
                log_format = parser.get(section, 'log_format', raw=True)
                # don't use remove_options because it can fail without reason
                parser.set(section, 'log_format', '')

            conf.update({section: dict(parser.items(section))})

            # Add log_format again, after parsing
            if log_format:
                conf[section]['log_format'] = log_format

    return conf
Esempio n. 2
0
def getManifest(fp, format, defaults=None):
    """Read the manifest from the given open file pointer according to the
    given ManifestFormat. Pass a dict as ``defaults`` to override the defaults
    from the manifest format.
    """

    if defaults is None:
        defaults = format.defaults

    parser = SafeConfigParser()
    if six.PY2:
        parser.readfp(fp)
    else:
        data = fp.read()
        if isinstance(data, six.binary_type):
            data = data.decode()
        parser.read_string(data)

    results = {}
    for key in format.keys:
        if parser.has_option(format.resourceType, key):
            results[key] = parser.get(format.resourceType, key)
        else:
            results[key] = defaults.get(key, None)

    for key in format.parameterSections:
        sectionName = "%s:%s" % (format.resourceType, key,)
        if parser.has_section(sectionName):
            results[key] = dict(parser.items(sectionName))
        else:
            results[key] = {}

    return results
Esempio n. 3
0
def getManifest(fp, format, defaults=None):
    """Read the manifest from the given open file pointer according to the
    given ManifestFormat. Pass a dict as ``defaults`` to override the defaults
    from the manifest format.
    """

    if defaults is None:
        defaults = format.defaults

    parser = SafeConfigParser()
    parser.readfp(fp)

    results = {}
    for key in format.keys:
        if parser.has_option(format.resourceType, key):
            results[key] = parser.get(format.resourceType, key)
        else:
            results[key] = defaults.get(key, None)

    for key in format.parameterSections:
        sectionName = "%s:%s" % (format.resourceType, key,)
        if parser.has_section(sectionName):
            results[key] = dict(parser.items(sectionName))
        else:
            results[key] = {}

    return results
Esempio n. 4
0
def check_dlm_cfgfile():
    """Parse DLM config file"""
    fname = "/etc/dlm/dlm.conf"

    try:
        with open(fname, 'r') as fp:
            cfgs = '[dlm]\n' + fp.read()
    except (OSError, IOError):
        return False

    cfg = SafeConfigParser()
    try:
        cfg.read_string(cfgs)
    except AttributeError:
        # Python2 ConfigParser doesn't have cfg.read_string
        from cStringIO import StringIO
        cfg.readfp(StringIO(cfgs))

    if not cfg.has_option('dlm', 'protocol'):
        return False

    proto = cfg.get('dlm', 'protocol').lower()
    return proto in ['sctp', 'detect', '1', '2']
Esempio n. 5
0
    if options.zulip_config_file is None:
        config_file = zulip.get_default_config_filename()
    else:
        config_file = options.zulip_config_file

    config = SafeConfigParser()
    try:
        with open(config_file, 'r') as f:
            config.readfp(f, config_file)
    except IOError:
        pass
    for option in ("jid", "jabber_password", "conference_domain", "mode", "zulip_email_suffix",
                   "jabber_server_address", "jabber_server_port"):
        if (getattr(options, option) is None
            and config.has_option("jabber_mirror", option)):
            setattr(options, option, config.get("jabber_mirror", option))

    for option in ("no_use_tls",):
        if getattr(options, option) is None:
            if config.has_option("jabber_mirror", option):
                setattr(options, option, config.getboolean("jabber_mirror", option))
            else:
                setattr(options, option, False)

    if options.mode is None:
        options.mode = "personal"

    if options.zulip_email_suffix is None:
        options.zulip_email_suffix = ''
Esempio n. 6
0
class Config(MutableMapping):
    """Read and write to config file.

    A config object is essentially a string key-value store that can be treated like a dictionary::

        c = Config()
        c['foo'] = 'bar'
        print c['foo']

    The file location may be specified::

        c = Config('~/matt/anotherconfig.cfg')
        c['where'] = 'in a different file'

    If no location is specified, the environment variable CHEMDATAEXTRACTOR_CONFIG is checked and used if available.
    Otherwise, a standard config location is used, which varies depending on the operating system. You can check the
    location using the ``path`` property. For more information see https://github.com/ActiveState/appdirs

    It is possible to edit the file by hand with a text editor.

    Warning: multiple instances of Config() pointing to the same file will not see each others' changes, and will
    overwrite the entire file when any key is changed.

    """

    #: These values will be present in a config object unless they are explicitly defined otherwise in the config file
    default_values = {}

    def __init__(self, path=None):
        """

        :param string path: (Optional) Path to config file location.
        """
        self._path = path

        # Use CHEMDATAEXTRACTOR_CONFIG environment variable if set
        if not self._path:
            self._path = os.environ.get('CHEMDATAEXTRACTOR_CONFIG')
        # Use OS-dependent config directory given by appdirs
        if not self._path:
            self._path = os.path.join(
                appdirs.user_config_dir('ChemDataExtractor'), 'config.cfg')
        self._parser = SafeConfigParser()
        if os.path.isfile(self.path):
            with open(self.path) as f:
                self._parser.readfp(f)
        if not self._parser.has_section('cde'):
            self._parser.add_section('cde')
        for k, v in six.iteritems(self.default_values):
            if not self._parser.has_option('cde', k.encode('utf-8')):
                self._parser.set('cde', k.encode('utf-8'), v.encode('utf-8'))

    @property
    def path(self):
        """The path to the config file."""
        return self._path

    def _flush(self):
        """Save the contents of data to the file on disk. You should not need to call this manually."""
        d = os.path.dirname(self.path)
        if not os.path.isdir(d):
            os.makedirs(d)
        with open(self.path, 'w') as f:
            self._parser.write(f)

    def __contains__(self, k):
        return self._parser.has_option('cde', k.encode('utf-8'))

    def __getitem__(self, k):
        try:
            return self._parser.get('cde', k.encode('utf-8'))
        except NoOptionError:
            raise KeyError(k)

    def __setitem__(self, k, v):
        self._parser.set('cde', k.encode('utf-8'), v.encode('utf-8'))
        self._flush()

    def __delitem__(self, k):
        try:
            self._parser.remove_option('cde', k.encode('utf-8'))
            self._flush()
        except NoOptionError:
            raise KeyError(k)

    def __iter__(self):
        return (k for k, v in self._parser.items('cde'))

    def __len__(self):
        return len(self._parser.items('cde'))

    def __repr__(self):
        return '<Config: %s>' % self.path

    def clear(self):
        """Clear all values from config."""
        self._parser.remove_section('cde')
        self._parser.add_section('cde')
        self._flush()
Esempio n. 7
0
    def __init__(self,
                 email=None,
                 api_key=None,
                 config_file=None,
                 verbose=False,
                 retry_on_errors=True,
                 site=None,
                 client=None,
                 cert_bundle=None,
                 insecure=None,
                 client_cert=None,
                 client_cert_key=None):
        # type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[str], Optional[str]) -> None
        if client is None:
            client = _default_client()

        # Normalize user-specified path
        if config_file is not None:
            config_file = os.path.abspath(os.path.expanduser(config_file))
        # Fill values from Environment Variables if not available in Constructor
        if config_file is None:
            config_file = os.environ.get("ZULIP_CONFIG")
        if api_key is None:
            api_key = os.environ.get("ZULIP_API_KEY")
        if email is None:
            email = os.environ.get("ZULIP_EMAIL")
        if site is None:
            site = os.environ.get("ZULIP_SITE")
        if client_cert is None:
            client_cert = os.environ.get("ZULIP_CERT")
        if client_cert_key is None:
            client_cert_key = os.environ.get("ZULIP_CERT_KEY")
        if cert_bundle is None:
            cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")
        if insecure is None:
            # Be quite strict about what is accepted so that users don't
            # disable security unintentionally.
            insecure_setting = os.environ.get('ZULIP_ALLOW_INSECURE')

            if insecure_setting is not None:
                insecure = validate_boolean_field(insecure_setting)

                if insecure is None:
                    raise ZulipError(
                        "The ZULIP_ALLOW_INSECURE environment "
                        "variable is set to '{}', it must be "
                        "'true' or 'false'".format(insecure_setting))
        if config_file is None:
            config_file = get_default_config_filename()

        if config_file is not None and os.path.exists(config_file):
            config = SafeConfigParser()
            with open(config_file, 'r') as f:
                config.readfp(f, config_file)
            if api_key is None:
                api_key = config.get("api", "key")
            if email is None:
                email = config.get("api", "email")
            if site is None and config.has_option("api", "site"):
                site = config.get("api", "site")
            if client_cert is None and config.has_option("api", "client_cert"):
                client_cert = config.get("api", "client_cert")
            if client_cert_key is None and config.has_option(
                    "api", "client_cert_key"):
                client_cert_key = config.get("api", "client_cert_key")
            if cert_bundle is None and config.has_option("api", "cert_bundle"):
                cert_bundle = config.get("api", "cert_bundle")
            if insecure is None and config.has_option("api", "insecure"):
                # Be quite strict about what is accepted so that users don't
                # disable security unintentionally.
                insecure_setting = config.get('api', 'insecure')

                insecure = validate_boolean_field(insecure_setting)

                if insecure is None:
                    raise ZulipError(
                        "insecure is set to '{}', it must be "
                        "'true' or 'false' if it is used in {}".format(
                            insecure_setting, config_file))

        elif None in (api_key, email):
            raise ConfigNotFoundError(
                "api_key or email not specified and file %s does not exist" %
                (config_file, ))

        assert (api_key is not None and email is not None)
        self.api_key = api_key
        self.email = email
        self.verbose = verbose
        if site is not None:
            if site.startswith("localhost"):
                site = "http://" + site
            elif not site.startswith("http"):
                site = "https://" + site
            # Remove trailing "/"s from site to simplify the below logic for adding "/api"
            site = site.rstrip("/")
            self.base_url = site
        else:
            raise MissingURLError(
                "Missing Zulip server URL; specify via --site or ~/.zuliprc.")

        if not self.base_url.endswith("/api"):
            self.base_url += "/api"
        self.base_url += "/"
        self.retry_on_errors = retry_on_errors
        self.client_name = client

        if insecure:
            logger.warning('Insecure mode enabled. The server\'s SSL/TLS '
                           'certificate will not be validated, making the '
                           'HTTPS connection potentially insecure')
            self.tls_verification = False  # type: Union[bool, str]
        elif cert_bundle is not None:
            if not os.path.isfile(cert_bundle):
                raise ConfigNotFoundError("tls bundle '%s' does not exist" %
                                          (cert_bundle, ))
            self.tls_verification = cert_bundle
        else:
            # Default behavior: verify against system CA certificates
            self.tls_verification = True

        if client_cert is None:
            if client_cert_key is not None:
                raise ConfigNotFoundError(
                    "client cert key '%s' specified, but no client cert public part provided"
                    % (client_cert_key, ))
        else:  # we have a client cert
            if not os.path.isfile(client_cert):
                raise ConfigNotFoundError("client cert '%s' does not exist" %
                                          (client_cert, ))
            if client_cert_key is not None:
                if not os.path.isfile(client_cert_key):
                    raise ConfigNotFoundError(
                        "client cert key '%s' does not exist" %
                        (client_cert_key, ))
        self.client_cert = client_cert
        self.client_cert_key = client_cert_key

        self.session = None  # type: Optional[requests.Session]

        self.has_connected = False
Esempio n. 8
0
    if options.zulip_config_file is None:
        config_file = zulip.get_default_config_filename()
    else:
        config_file = options.zulip_config_file

    config = SafeConfigParser()
    try:
        with open(config_file, 'r') as f:
            config.readfp(f, config_file)
    except IOError:
        pass
    for option in ("jid", "jabber_password", "conference_domain", "mode",
                   "zulip_email_suffix", "jabber_server_address",
                   "jabber_server_port"):
        if (getattr(options, option) is None
                and config.has_option("jabber_mirror", option)):
            setattr(options, option, config.get("jabber_mirror", option))

    for option in ("no_use_tls", ):
        if getattr(options, option) is None:
            if config.has_option("jabber_mirror", option):
                setattr(options, option,
                        config.getboolean("jabber_mirror", option))
            else:
                setattr(options, option, False)

    if options.mode is None:
        options.mode = "personal"

    if options.zulip_email_suffix is None:
        options.zulip_email_suffix = ''
Esempio n. 9
0
def load_conf():

    conf_path = os.path.expanduser(config_path)
    if not os.path.exists(conf_path):
        logger.warn("{conf} couldn't be found, please use \'coscmd config -h\' to learn how to config coscmd!".format(conf=to_printable_str(conf_path)))
        raise IOError
    else:
        logger.debug('{conf} is found'.format(conf=to_printable_str(conf_path)))

    try:
        with open(conf_path, 'r') as f:
            cp = SafeConfigParser()
            cp.readfp(fp=f)
            if not cp.has_section('common'):
                raise Exception("[common] section could't be found, please check your config file.")
            if cp.has_option('common', 'part_size'):
                part_size = cp.getint('common', 'part_size')
            else:
                part_size = 1
            if cp.has_option('common', 'max_thread'):
                max_thread = cp.getint('common', 'max_thread')
            else:
                max_thread = 5
            try:
                secret_id = cp.get('common', 'secret_id')
            except Exception:
                secret_id = cp.get('common', 'access_id')
            try:
                appid = cp.get('common', 'appid')
                bucket = cp.get('common', 'bucket')
                if bucket.endswith("-"+str(appid)):
                    bucket = bucket.rstrip(appid)
                    bucket = bucket[:-1]
            except Exception:
                try:
                    bucket = cp.get('common', 'bucket')
                    appid = bucket.split('-')[-1]
                    bucket = bucket.rstrip(appid)
                    bucket = bucket[:-1]
                except Exception:
                    # check if user use -b bucket
                    if (pre_bucket == ""):
                        logger.error("The configuration file is wrong. Please reconfirm")
            try:
                schema = cp.get('common', 'schema')
            except Exception:
                schema = 'https'
            try:
                verify = cp.get('common', 'verify')
            except Exception:
                verify = 'md5'
            try:
                token = cp.get('common', 'token')
            except Exception:
                token = None

            try:
                error = cp.get('common', 'error')
            except Exception:
                error = "cos.err.log"

            try:
                success = cp.get('common', 'success')
            except Exception:
                success = "cos.suc.log"


            try:
                anonymous = cp.get('common', 'anonymous')
                if anonymous == 'True' or anonymous == 'true':
                    anonymous = True
                else:
                    anonymous = False
            except Exception:
                anonymous = False
            try:
                retry = int(cp.get('common', 'retry'))
            except Exception:
                retry = 2
            try:
                timeout = int(cp.get('common', 'timeout'))
            except Exception:
                timeout = 60
            region, endpoint = None, None
            if cp.has_option('common', 'region'):
                region = cp.get('common', 'region')
            if cp.has_option('common', 'endpoint'):
                endpoint = cp.get('common', 'endpoint')
            if pre_appid != "":
                appid = pre_appid
            if pre_bucket != "":
                bucket = pre_bucket
            if pre_region != "":
                region = pre_region
            conf = CoscmdConfig(
                appid=appid,
                secret_id=secret_id,
                secret_key=cp.get('common', 'secret_key'),
                token=token,
                region=compatible(region),
                endpoint=endpoint,
                bucket=bucket,
                part_size=part_size,
                max_thread=max_thread,
                schema=schema,
                anonymous=anonymous,
                verify=verify,
                retry=retry,
                timeout=timeout,
                error=error,
                success=success
            )
            return conf
    except Exception as e:
        raise(e)
Esempio n. 10
0
def createThemeFromTemplate(title, description, baseOn='template'):
    """Create a new theme from the given title and description based on
    another theme resource directory
    """

    source = queryResourceDirectory(THEME_RESOURCE_NAME, baseOn)
    if source is None:
        raise KeyError("Theme {0:s} not found".format(baseOn))

    themeName = getUtility(IURLNormalizer).normalize(title)
    if isinstance(themeName, six.text_type):
        themeName = themeName.encode('utf-8')

    resources = getOrCreatePersistentResourceDirectory()

    resources.makeDirectory(themeName)
    target = resources[themeName]

    cloneResourceDirectory(source, target)

    manifest = SafeConfigParser()

    if MANIFEST_FILENAME in target:
        fp = target.openFile(MANIFEST_FILENAME)
        try:
            manifest.readfp(fp)
        finally:
            fp.close()

    if not manifest.has_section('theme'):
        manifest.add_section('theme')

    if not isinstance(title, str):
        title = title.encode('utf-8')
    if not isinstance(description, str):
        description = description.encode('utf-8')
    manifest.set('theme', 'title', title)
    manifest.set('theme', 'description', description)

    if manifest.has_option('theme', 'prefix'):
        prefix = u"/++%s++%s" % (THEME_RESOURCE_NAME, themeName)
        manifest.set('theme', 'prefix', prefix)

    if manifest.has_option('theme', 'rules'):
        rule = manifest.get('theme', 'rules')
        rule_file_name = rule.split('/')[-1]  # extract real rules file name
        rules = u"/++%s++%s/%s" % (THEME_RESOURCE_NAME, themeName,
                                   rule_file_name)
        manifest.set('theme', 'rules', rules)

    paths_to_fix = [
        'development-css', 'production-css', 'tinymce-content-css',
        'development-js', 'production-js'
    ]
    for var_path in paths_to_fix:
        if not manifest.has_option('theme', var_path):
            continue
        val = manifest.get('theme', var_path)
        if not val:
            continue
        template_prefix = '++%s++%s/' % (THEME_RESOURCE_NAME, baseOn)
        if template_prefix in val:
            # okay, fix
            val = val.replace(template_prefix,
                              '++%s++%s/' % (THEME_RESOURCE_NAME, themeName))
            manifest.set('theme', var_path, val)

    manifestContents = StringIO()
    manifest.write(manifestContents)
    target.writeFile(MANIFEST_FILENAME, manifestContents)

    return themeName
Esempio n. 11
0
class Config(object):
    """
    Manages the configuration file
    """
    def __init__(self):
        """
        DEFAULT VALUES
        """
        self._basescript = None
        self.recentvaults = []
        self.pwlength = 10
        self.search_notes = False
        self.search_passwd = False
        self.use_pwgen = False
        # self.alphabet = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_"
        self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_"
        self.avoid_bigrams = "cl mn nm nn rn vv VV"

        self._fname = self.get_config_filename()
        print("Using config %s" % self._fname)
        self._parser = SafeConfigParser()

        if os.path.exists(self._fname):
            self._parser.read(self._fname)

        if not self._parser.has_section("base"):
            self._parser.add_section("base")

        for num in range(10):
            if (not self._parser.has_option("base",
                                            "recentvaults" + str(num))):
                break
            self.recentvaults.append(
                self._parser.get("base", "recentvaults" + str(num)))

        if self._parser.has_option("base", "pwlength"):
            self.pwlength = int(self._parser.get("base", "pwlength"))

        if self._parser.has_option("base", "search_notes"):
            if self._parser.get("base", "search_notes") == "True":
                self.search_notes = True

        if self._parser.has_option("base", "search_passwd"):
            if self._parser.get("base", "search_passwd") == "True":
                self.search_passwd = True

        if self._parser.has_option("base", "use_pwgen"):
            if self._parser.get("base", "use_pwgen") == "True":
                self.use_pwgen = True

        if self._parser.has_option("base", "alphabet"):
            self.alphabet = self._parser.get("base", "alphabet")

        if self._parser.has_option("base", "avoid_bigrams"):
            self.avoid_bigrams = self._parser.get("base", "avoid_bigrams")

        if not os.path.exists(self._fname):
            self.save()

    def set_basescript(self, basescript):
        self._basescript = basescript

    def get_basescript(self):
        return self._basescript

    def save(self):
        if (not os.path.exists(os.path.dirname(self._fname))):
            os.mkdir(os.path.dirname(self._fname))

        # remove duplicates and trim to 10 items
        _saved_recentvaults = []
        for item in self.recentvaults:
            if item in _saved_recentvaults:
                continue
            self._parser.set("base",
                             "recentvaults" + str(len(_saved_recentvaults)),
                             item)
            _saved_recentvaults.append(item)
            if (len(_saved_recentvaults) >= 10):
                break

        self._parser.set("base", "pwlength", str(self.pwlength))
        self._parser.set("base", "search_notes", str(self.search_notes))
        self._parser.set("base", "search_passwd", str(self.search_passwd))
        self._parser.set("base", "use_pwgen", str(self.use_pwgen))
        self._parser.set("base", "alphabet", str(self.alphabet))
        self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams))
        filehandle = open(self._fname, 'w')
        self._parser.write(filehandle)
        filehandle.close()

    @staticmethod
    def get_config_filename():
        """
        Returns the full filename of the config file
        """
        base_fname = "loxodo"

        # On Mac OS X, config files go to ~/Library/Application Support/foo/
        if platform.system() == "Darwin":
            base_path = os.path.join(os.path.expanduser("~"), "Library",
                                     "Application Support")
            if os.path.isdir(base_path):
                return os.path.join(base_path, base_fname, base_fname + ".ini")

        # On Microsoft Windows, config files go to $APPDATA/foo/
        if platform.system() in ("Windows", "Microsoft"):
            if ("APPDATA" in os.environ):
                base_path = os.environ["APPDATA"]
                if os.path.isdir(base_path):
                    return os.path.join(base_path, base_fname,
                                        base_fname + ".ini")

        # Allow config directory override as per freedesktop.org XDG Base Directory Specification
        if ("XDG_CONFIG_HOME" in os.environ):
            base_path = os.environ["XDG_CONFIG_HOME"]
            if os.path.isdir(base_path):
                return os.path.join(base_path, base_fname, base_fname + ".ini")

        # Default configuration path is ~/.config/foo/
        base_path = os.path.join(os.path.expanduser("~"), ".config")
        if os.path.isdir(base_path):
            return os.path.join(base_path, base_fname, base_fname + ".ini")
        else:
            return os.path.join(os.path.expanduser("~"),
                                "." + base_fname + ".ini")
Esempio n. 12
0
    def loadConfig(self):
        """Load build configuration from the spec file."""
        specfiles = [
            spec for spec in os.listdir(self.spec_dir) if spec.endswith('.ini')
        ]
        if len(specfiles) == 0:
            raise BuildError('No .ini file found')
        elif len(specfiles) > 1:
            raise BuildError('Multiple .ini files found')

        if six.PY2:
            conf = SafeConfigParser()
        else:
            conf = ConfigParser()
        conf.read(os.path.join(self.spec_dir, specfiles[0]))

        # [naming] section
        for entry in ('name', 'version', 'release', 'description'):
            setattr(self, entry, conf.get('naming', entry))
        if conf.has_option('naming', 'epoch'):
            self.epoch = conf.get('naming', 'epoch')

        # [building] section
        self.platform = conf.get('building', 'platform')

        # preinstalled are paths to files or directories that must exist
        # in the VM for it to execute the build.
        # If the path ends in / or \ it must be a directory, otherwise it must
        # be a file.
        # They may be specified as Cygwin (/cygdrive/c/...) or Windows (C:\...)
        # absolute paths, or without a path in which case it is searched for
        # on the PATH.
        if conf.has_option('building', 'preinstalled'):
            self.preinstalled.extend([
                e.strip()
                for e in conf.get('building', 'preinstalled').split('\n') if e
            ])

        # buildrequires and provides are multi-valued (space-separated)
        for br in conf.get('building', 'buildrequires').split():
            # buildrequires is a space-separated list
            # each item in the list is in the format:
            # pkgname[:opt1:opt2=val2:...]
            # the options are put into a dict
            # if the option has no =val, the value in the dict will be None
            if br:
                br = br.split(':')
                bropts = {}
                for opt in br[1:]:
                    if '=' in opt:
                        key, val = opt.split('=', 1)
                    else:
                        key = opt
                        val = None
                    bropts[key] = val
                self.buildrequires.append((br[0], bropts))
        for prov in conf.get('building', 'provides').split():
            if prov:
                self.provides.append(prov)
        # optionally specify a shell to use (defaults to bash)
        # valid values are: cmd, cmd.exe (alias for cmd), and bash
        if conf.has_option('building', 'shell'):
            self.shell = conf.get('building', 'shell')
        else:
            self.shell = 'bash'
        # execute is multi-valued (newline-separated)
        self.execute.extend([
            e.strip() for e in conf.get('building', 'execute').split('\n') if e
        ])

        # postbuild are files or directories that must exist after the build is
        # complete, but are not included in the build output
        # they are specified as paths relative the source directory, and may be
        # in Unix or Windows format
        # each entry may contain shell-style globs, and one or more files
        # matching the glob is considered valid
        if conf.has_option('building', 'postbuild'):
            for entry in conf.get('building', 'postbuild').split('\n'):
                entry = entry.strip()
                if not entry:
                    continue
                for var in ('name', 'version', 'release'):
                    entry = entry.replace('$' + var, getattr(self, var))
                self.postbuild.append(entry)

        # [files] section
        for entry in conf.get('files', 'output').split('\n'):
            entry = entry.strip()
            if not entry:
                continue
            tokens = entry.split(':')
            filename = tokens[0]
            for var in ('name', 'version', 'release'):
                filename = filename.replace('$' + var, getattr(self, var))
            metadata = {}
            metadata['platforms'] = tokens[1].split(',')
            if len(tokens) > 2:
                metadata['flags'] = tokens[2].split(',')
            else:
                metadata['flags'] = []
            self.output[filename] = metadata
        self.logs.extend(
            [e.strip() for e in conf.get('files', 'logs').split('\n') if e])
Esempio n. 13
0
    def __init__(self, email=None, api_key=None, config_file=None,
                 verbose=False, retry_on_errors=True,
                 site=None, client=None,
                 cert_bundle=None, insecure=None,
                 client_cert=None, client_cert_key=None):
        # type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[str], Optional[str]) -> None
        if client is None:
            client = _default_client()

        # Normalize user-specified path
        if config_file is not None:
            config_file = os.path.abspath(os.path.expanduser(config_file))
        # Fill values from Environment Variables if not available in Constructor
        if config_file is None:
            config_file = os.environ.get("ZULIP_CONFIG")
        if api_key is None:
            api_key = os.environ.get("ZULIP_API_KEY")
        if email is None:
            email = os.environ.get("ZULIP_EMAIL")
        if site is None:
            site = os.environ.get("ZULIP_SITE")
        if client_cert is None:
            client_cert = os.environ.get("ZULIP_CERT")
        if client_cert_key is None:
            client_cert_key = os.environ.get("ZULIP_CERT_KEY")
        if cert_bundle is None:
            cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")
        if insecure is None:
            # Be quite strict about what is accepted so that users don't
            # disable security unintentionally.
            insecure_setting = os.environ.get('ZULIP_ALLOW_INSECURE')

            if insecure_setting is not None:
                insecure = validate_boolean_field(insecure_setting)

                if insecure is None:
                    raise ZulipError("The ZULIP_ALLOW_INSECURE environment "
                                     "variable is set to '{}', it must be "
                                     "'true' or 'false'"
                                     .format(insecure_setting))
        if config_file is None:
            config_file = get_default_config_filename()

        if config_file is not None and os.path.exists(config_file):
            config = SafeConfigParser()
            with open(config_file, 'r') as f:
                config.readfp(f, config_file)
            if api_key is None:
                api_key = config.get("api", "key")
            if email is None:
                email = config.get("api", "email")
            if site is None and config.has_option("api", "site"):
                site = config.get("api", "site")
            if client_cert is None and config.has_option("api", "client_cert"):
                client_cert = config.get("api", "client_cert")
            if client_cert_key is None and config.has_option("api", "client_cert_key"):
                client_cert_key = config.get("api", "client_cert_key")
            if cert_bundle is None and config.has_option("api", "cert_bundle"):
                cert_bundle = config.get("api", "cert_bundle")
            if insecure is None and config.has_option("api", "insecure"):
                # Be quite strict about what is accepted so that users don't
                # disable security unintentionally.
                insecure_setting = config.get('api', 'insecure')

                insecure = validate_boolean_field(insecure_setting)

                if insecure is None:
                    raise ZulipError("insecure is set to '{}', it must be "
                                     "'true' or 'false' if it is used in {}"
                                     .format(insecure_setting, config_file))

        elif None in (api_key, email):
            raise ConfigNotFoundError("api_key or email not specified and file %s does not exist"
                                      % (config_file,))

        assert(api_key is not None and email is not None)
        self.api_key = api_key
        self.email = email
        self.verbose = verbose
        if site is not None:
            if site.startswith("localhost"):
                site = "http://" + site
            elif not site.startswith("http"):
                site = "https://" + site
            # Remove trailing "/"s from site to simplify the below logic for adding "/api"
            site = site.rstrip("/")
            self.base_url = site
        else:
            raise MissingURLError("Missing Zulip server URL; specify via --site or ~/.zuliprc.")

        if not self.base_url.endswith("/api"):
            self.base_url += "/api"
        self.base_url += "/"
        self.retry_on_errors = retry_on_errors
        self.client_name = client

        if insecure:
            logger.warning('Insecure mode enabled. The server\'s SSL/TLS '
                           'certificate will not be validated, making the '
                           'HTTPS connection potentially insecure')
            self.tls_verification = False  # type: Union[bool, str]
        elif cert_bundle is not None:
            if not os.path.isfile(cert_bundle):
                raise ConfigNotFoundError("tls bundle '%s' does not exist"
                                          % (cert_bundle,))
            self.tls_verification = cert_bundle
        else:
            # Default behavior: verify against system CA certificates
            self.tls_verification = True

        if client_cert is None:
            if client_cert_key is not None:
                raise ConfigNotFoundError("client cert key '%s' specified, but no client cert public part provided"
                                          % (client_cert_key,))
        else:  # we have a client cert
            if not os.path.isfile(client_cert):
                raise ConfigNotFoundError("client cert '%s' does not exist"
                                          % (client_cert,))
            if client_cert_key is not None:
                if not os.path.isfile(client_cert_key):
                    raise ConfigNotFoundError("client cert key '%s' does not exist"
                                              % (client_cert_key,))
        self.client_cert = client_cert
        self.client_cert_key = client_cert_key

        self.session = None  # type: Optional[requests.Session]

        self.has_connected = False
Esempio n. 14
0
class PackageConfigHandler(object):
    """
    Manager class for packages files for tracking installation of modules
    """

    def __init__(self):
        # noinspection PyUnresolvedReferences
        from six.moves.configparser import SafeConfigParser
        self.package_cfg = os.path.expanduser('~/Documents/site-packages/.pypi_packages')
        if not os.path.isfile(self.package_cfg):
            print('Creating package file')
            with open(self.package_cfg, 'w') as outs:
                outs.close()
        self.parser = SafeConfigParser()
        self.parser.read(self.package_cfg)

    def save(self):
        with open(self.package_cfg, 'w') as outs:
            self.parser.write(outs)

    def add_module(self, pkg_info):
        """

        :param pkg_info: A dict that has name, url, version, summary
        :return:
        """
        if not self.parser.has_section(pkg_info['name']):
            self.parser.add_section(pkg_info['name'])
        self.parser.set(pkg_info['name'], 'url', pkg_info['url'])
        self.parser.set(pkg_info['name'], 'version', pkg_info['version'])
        self.parser.set(pkg_info['name'], 'summary', pkg_info['summary'])
        self.parser.set(pkg_info['name'], 'files', pkg_info['files'])
        self.parser.set(pkg_info['name'], 'dependency', pkg_info['dependency'])
        self.save()

    def list_modules(self):
        return [module for module in self.parser.sections()]

    def module_exists(self, name):
        return self.parser.has_section(name)

    def get_info(self, name):
        if self.parser.has_section(name):
            tbl = {}
            for opt, value in self.parser.items(name):
                tbl[opt] = value
            return tbl

    def remove_module(self, name):
        self.parser.remove_section(name)
        self.save()

    def get_files_installed(self, section_name):
        if self.parser.has_option(section_name, 'files'):
            files = self.parser.get(section_name, 'files').strip()
            return files.split(',')
        else:
            return None

    def get_dependencies(self, section_name):
        if self.parser.has_option(section_name, 'dependency'):
            dependencies = self.parser.get(section_name, 'dependency').strip()
            return set(dependencies.split(',')) if dependencies != '' else set()
        else:
            return None

    def get_all_dependencies(self, exclude_module=()):
        all_dependencies = set()
        for section_name in self.parser.sections():
            if section_name not in exclude_module and self.parser.has_option(section_name, 'dependency'):
                dependencies = self.parser.get(section_name, 'dependency').strip()
                if dependencies != '':
                    for dep in dependencies.split(','):
                        all_dependencies.add(dep)
        return all_dependencies
Esempio n. 15
0
    def parse_config_file(self, config_file):
        config = SafeConfigParser(self.DEFAULT_CONFIG)
        config.readfp(config_file)

        blessconfig = {
            'CLIENT_CONFIG': {
                'domain_regex': config.get('CLIENT', 'domain_regex'),
                'cache_dir': config.get('CLIENT', 'cache_dir'),
                'cache_file': config.get('CLIENT', 'cache_file'),
                'mfa_cache_dir': config.get('CLIENT', 'mfa_cache_dir'),
                'mfa_cache_file': config.get('CLIENT', 'mfa_cache_file'),
                'ip_urls': [s.strip() for s in config.get('CLIENT', 'ip_urls').split(",")],
                'update_script': config.get('CLIENT', 'update_script'),
                'user_session_length': int(config.get('CLIENT', 'user_session_length')),
                'usebless_role_session_length': int(config.get('CLIENT', 'usebless_role_session_length')),
                'update_sshagent': config.getboolean('CLIENT', 'update_sshagent'),
                'use_env_creds': config.getboolean('CLIENT', 'use_env_creds'),
            },
            'BLESS_CONFIG': {
                'ca_backend': config.get('MAIN', 'ca_backend'),
                'userrole': config.get('LAMBDA', 'user_role'),
                'accountid': config.get('LAMBDA', 'account_id'),
                'functionname': config.get('LAMBDA', 'functionname'),
                'functionversion': config.get('LAMBDA', 'functionversion'),
                'certlifetime': config.getint('LAMBDA', 'certlifetime'),
                'ipcachelifetime': config.getint('LAMBDA', 'ipcachelifetime'),
                'timeoutconfig': {
                    'connect': config.getint('LAMBDA', 'timeout_connect'),
                    'read': config.getint('LAMBDA', 'timeout_read')
                }
            },
            'AWS_CONFIG': {
                'remote_user': config.get('MAIN', 'remote_user')
            },
            'REGION_ALIAS': {}
        }

        if config.has_option('MAIN', 'bastion_ips'):
            blessconfig['AWS_CONFIG']['bastion_ips'] = config.get('MAIN', 'bastion_ips')
        if blessconfig['BLESS_CONFIG']['ca_backend'].lower() == 'hashicorp-vault':
            blessconfig['VAULT_CONFIG'] = {
                'vault_addr': config.get('VAULT', 'vault_addr'),
                'auth_mount': config.get('VAULT', 'auth_mount'),
                'ssh_backend_mount': config.get('VAULT', 'ssh_backend_mount'),
                'ssh_backend_role': config.get('VAULT', 'ssh_backend_role'),
            }

        regions = config.get('MAIN', 'region_aliases').split(",")
        regions = [region.strip() for region in regions]
        for region in regions:
            region = region.upper()
            kms_region_key = 'KMSAUTH_CONFIG_{}'.format(region)
            blessconfig.update({kms_region_key: self._get_region_kms_config(region, config)})
            blessconfig['REGION_ALIAS'].update({region: blessconfig[kms_region_key]['awsregion']})

            if config.has_section('HOUSEKEEPER'):
                hk_region_key = 'HOUSEKEEPER_CONFIG_{}'.format(region)
                blessconfig.update({hk_region_key: self._get_region_housekeeper_config(region, config)})
                blessconfig['REGION_ALIAS'].update({region: blessconfig[hk_region_key]['awsregion']})

        return blessconfig
Esempio n. 16
0
 def has_option(self, name, option_name):
     section_name = self._get_section_name(name)
     return SafeConfigParser.has_option(self, section_name, option_name)
Esempio n. 17
0
    def onDiscovery(self, theme, settings, dependenciesSettings):
        res = queryResourceDirectory(THEME_RESOURCE_NAME, theme)
        if res is None:
            return

        directoryName = 'views'
        if 'directory' in settings:
            directoryName = settings['directory']

        if res.isDirectory(directoryName):
            viewsDir = res[directoryName]

            layer = getattr(schemata, theme, None)

            if 'layer' in settings:
                layerName = settings['layer']

                try:
                    layer = resolve(layerName)
                except (ImportError, AttributeError):
                    logger.warn("Could not import %s" % layerName)
                    return

            viewConfig = SafeConfigParser()

            if viewsDir.isFile(VIEW_CONFIG_FILENAME):
                fp = viewsDir.openFile(VIEW_CONFIG_FILENAME)
                try:
                    viewConfig.readfp(fp)
                finally:
                    try:
                        fp.close()
                    except AttributeError:
                        pass

            views = []
            configurationMachine = ConfigurationMachine()
            path = viewsDir.directory

            for filename in os.listdir(path):
                if not filename.lower().endswith(EXTENSION):
                    continue

                name = viewName = filename[:-3]
                permission = 'zope2.View'
                for_ = Interface
                class_ = None
                template = os.path.join(path, filename)
                menu = {}

                # Read override options from views.cfg if applicable
                if viewConfig.has_section(name):

                    if viewConfig.has_option(name, 'name'):
                        viewName = viewConfig.get(name, 'name')

                    if viewConfig.has_option(name, 'permission'):
                        permission = viewConfig.get(name, 'permission')

                    if viewConfig.has_option(name, 'for'):
                        forStr = viewConfig.get(name, 'for')
                        if forStr != "*":
                            for_ = resolve(forStr)

                    if viewConfig.has_option(name, 'class'):
                        class_ = resolve(viewConfig.get(name, 'class'))

                    if viewConfig.has_option(name, 'menu'):
                        menu = dict(
                            title=viewConfig.get(name, 'menu'),
                            menu=getattr(
                                zope.browsermenu.metaconfigure.menus,
                                "plone_displayviews",
                            ),
                        )

                Products.Five.browser.metaconfigure.page(
                    configurationMachine,
                    name=viewName,
                    permission=permission,
                    for_=for_,
                    layer=layer,
                    template=template,
                    class_=class_,
                    **menu
                )

                views.append(name)

            if len(views) > 0:
                configurationMachine.execute_actions()

            self.registered[theme] = views
def _parse_args(args_str):
    keystone_auth_parser = SafeConfigParser()
    conf_file = keystone_auth_parser.read(
        '/etc/contrail/contrail-keystone-auth.conf')

    default_keystone_vals = {
        "username": "******",
        "tenant_name": "admin",
        "domain_name": "Default"
    }

    get_vars = (lambda x: keystone_auth_parser.get('KEYSTONE', x)
                if keystone_auth_parser.has_option('KEYSTONE', x) else None)
    if conf_file:
        if keystone_auth_parser.has_section('KEYSTONE'):
            username = get_vars('admin_user')
            if username:
                default_keystone_vals['username'] = username
            password = get_vars('admin_password')
            if password:
                default_keystone_vals['password'] = password
            tenant_name = get_vars('admin_tenant_name')
            if tenant_name:
                default_keystone_vals['tenant_name'] = tenant_name
            domain_name = get_vars('user_domain_name')
            if domain_name:
                default_keystone_vals['domain_name'] = domain_name

    parser = argparse.ArgumentParser(
        formatter_class=argparse.RawTextHelpFormatter, description='')

    parser.add_argument('-v',
                        '--version',
                        action='version',
                        version='%(prog)s ' + __version__)

    parser.add_argument("--debug",
                        help="Run in debug mode, default False",
                        action='store_true',
                        default=False)

    parser.add_argument("--to-json", help="File to dump json to", default=None)

    parser.add_argument("--username",
                        help="Username used to login to API Server")

    parser.add_argument("--password",
                        help="Password used to login to API Server")

    parser.add_argument("--api-server-host", help="IP of API Server")

    parser.add_argument("--tenant-name", help="Name of Tenant")

    parser.add_argument("--domain-name", help="Domain name")

    parser.add_argument("--read-timeout",
                        dest='timeout',
                        help="Timeout for Data reading operations",
                        default=300)

    parser.add_argument(
        "--connection-timeout",
        help="Timeout set for VNC API to connect to API Server",
        default=120)

    ts = calendar.timegm(time.gmtime())
    if os.path.isdir("/var/log/contrail"):
        default_log = "/var/log/contrail/fabric_validation-{0}.log".format(ts)
    else:
        import tempfile
        default_log = '{0}/fabric_validation-{1}.log'.format(
            tempfile.gettempdir(), ts)

    parser.add_argument("--log_file",
                        help="Log file to save output, default '%(default)s'",
                        default=default_log)

    parser.set_defaults(**default_keystone_vals)
    args_obj, _ = parser.parse_known_args(args_str.split())
    _args = args_obj
    return _args
Esempio n. 19
0
    def __init__(self,
                 email=None,
                 api_key=None,
                 config_file=None,
                 verbose=False,
                 retry_on_errors=True,
                 site=None,
                 client=None,
                 cert_bundle=None,
                 insecure=None,
                 client_cert=None,
                 client_cert_key=None):
        if client is None:
            client = _default_client()

        if config_file is None:
            config_file = get_default_config_filename()
        if os.path.exists(config_file):
            config = SafeConfigParser()
            with open(config_file, 'r') as f:
                config.readfp(f, config_file)
            if api_key is None:
                api_key = config.get("api", "key")
            if email is None:
                email = config.get("api", "email")
            if site is None and config.has_option("api", "site"):
                site = config.get("api", "site")
            if client_cert is None and config.has_option("api", "client_cert"):
                client_cert = config.get("api", "client_cert")
            if client_cert_key is None and config.has_option(
                    "api", "client_cert_key"):
                client_cert_key = config.get("api", "client_cert_key")
            if cert_bundle is None and config.has_option("api", "cert_bundle"):
                cert_bundle = config.get("api", "cert_bundle")
            if insecure is None and config.has_option("api", "insecure"):
                # Be quite strict about what is accepted so that users don't
                # disable security unintentionally.
                insecure_setting = config.get("api", "insecure").lower()
                if insecure_setting == "true":
                    insecure = True
                elif insecure_setting == "false":
                    insecure = False
                else:
                    raise RuntimeError(
                        "insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
                        % (insecure_setting, config_file))
        elif None in (api_key, email):
            raise RuntimeError(
                "api_key or email not specified and %s does not exist" %
                (config_file, ))

        self.api_key = api_key
        self.email = email
        self.verbose = verbose
        if site is not None:
            if not site.startswith("http"):
                site = "https://" + site
            # Remove trailing "/"s from site to simplify the below logic for adding "/api"
            site = site.rstrip("/")
            self.base_url = site
        else:
            self.base_url = "https://api.zulip.com"

        if self.base_url != "https://api.zulip.com" and not self.base_url.endswith(
                "/api"):
            self.base_url += "/api"
        self.base_url += "/"
        self.retry_on_errors = retry_on_errors
        self.client_name = client

        if insecure:
            self.tls_verification = False
        elif cert_bundle is not None:
            if not os.path.isfile(cert_bundle):
                raise RuntimeError("tls bundle '%s' does not exist" %
                                   (cert_bundle, ))
            self.tls_verification = cert_bundle
        else:
            # Default behavior: verify against system CA certificates
            self.tls_verification = True

        if client_cert is None:
            if client_cert_key is not None:
                raise RuntimeError(
                    "client cert key '%s' specified, but no client cert public part provided"
                    % (client_cert_key, ))
        else:  # we have a client cert
            if not os.path.isfile(client_cert):
                raise RuntimeError("client cert '%s' does not exist" %
                                   (client_cert, ))
            if client_cert_key is not None:
                if not os.path.isfile(client_cert_key):
                    raise RuntimeError("client cert key '%s' does not exist" %
                                       (client_cert_key, ))
        self.client_cert = client_cert
        self.client_cert_key = client_cert_key
Esempio n. 20
0
File: pip.py Progetto: BBOOXX/stash
 def has_option(self, name, option_name):
     section_name = self._get_section_name(name)
     return SafeConfigParser.has_option(self, section_name, option_name)
Esempio n. 21
0
    def __init__(self, email=None, api_key=None, config_file=None,
                 verbose=False, retry_on_errors=True,
                 site=None, client=None,
                 cert_bundle=None, insecure=None):
        if client is None:
            client = _default_client()

        if config_file is None:
            config_file = get_default_config_filename()
        if os.path.exists(config_file):
            config = SafeConfigParser()
            with open(config_file, 'r') as f:
                config.readfp(f, config_file)
            if api_key is None:
                api_key = config.get("api", "key")
            if email is None:
                email = config.get("api", "email")
            if site is None and config.has_option("api", "site"):
                site = config.get("api", "site")
            if cert_bundle is None and config.has_option("api", "cert_bundle"):
                cert_bundle = config.get("api", "cert_bundle")
            if insecure is None and config.has_option("api", "insecure"):
                # Be quite strict about what is accepted so that users don't
                # disable security unintentionally.
                insecure_setting = config.get("api", "insecure").lower()
                if insecure_setting == "true":
                    insecure = True
                elif insecure_setting == "false":
                    insecure = False
                else:
                    raise RuntimeError("insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
                                       % (insecure_setting, config_file))
        elif None in (api_key, email):
            raise RuntimeError("api_key or email not specified and %s does not exist"
                               % (config_file,))

        self.api_key = api_key
        self.email = email
        self.verbose = verbose
        if site is not None:
            if not site.startswith("http"):
                site = "https://" + site
            # Remove trailing "/"s from site to simplify the below logic for adding "/api"
            site = site.rstrip("/")
            self.base_url = site
        else:
            self.base_url = "https://api.zulip.com"

        if self.base_url != "https://api.zulip.com" and not self.base_url.endswith("/api"):
            self.base_url += "/api"
        self.base_url += "/"
        self.retry_on_errors = retry_on_errors
        self.client_name = client

        if insecure:
            self.tls_verification=False
        elif cert_bundle is not None:
            if not os.path.isfile(cert_bundle):
                raise RuntimeError("tls bundle '%s' does not exist"
                                   %(cert_bundle,))
            self.tls_verification=cert_bundle
        else:
            # Default behavior: verify against system CA certificates
            self.tls_verification=True
Esempio n. 22
0
class Csw(object):
    """ Base CSW server """
    def __init__(self, rtconfig=None, env=None, version='3.0.0'):
        """ Initialize CSW """

        if not env:
            self.environ = os.environ
        else:
            self.environ = env

        self.context = config.StaticContext()

        # Lazy load this when needed
        # (it will permanently update global cfg namespaces)
        self.sruobj = None
        self.opensearchobj = None
        self.oaipmhobj = None

        # init kvp
        self.kvp = {}

        self.mode = 'csw'
        self.asynchronous = False
        self.soap = False
        self.request = None
        self.exception = False
        self.status = 'OK'
        self.profiles = None
        self.manager = False
        self.outputschemas = {}
        self.mimetype = 'application/xml; charset=UTF-8'
        self.encoding = 'UTF-8'
        self.pretty_print = 0
        self.domainquerytype = 'list'
        self.orm = 'django'
        self.language = {'639_code': 'en', 'text': 'english'}
        self.process_time_start = time()

        # define CSW implementation object (default CSW3)
        self.iface = csw3.Csw3(server_csw=self)
        self.request_version = version

        if self.request_version == '2.0.2':
            self.iface = csw2.Csw2(server_csw=self)
            self.context.set_model('csw')

        # load user configuration
        try:
            LOGGER.info('Loading user configuration')
            if isinstance(rtconfig, SafeConfigParser):  # serialized already
                self.config = rtconfig
            else:
                self.config = SafeConfigParser()
                if isinstance(rtconfig, dict):  # dictionary
                    for section, options in rtconfig.items():
                        self.config.add_section(section)
                        for k, v in options.items():
                            self.config.set(section, k, v)
                else:  # configuration file
                    import codecs
                    with codecs.open(rtconfig, encoding='utf-8') as scp:
                        self.config.readfp(scp)
        except Exception as err:
            msg = 'Could not load configuration'
            LOGGER.exception('%s %s: %s', msg, rtconfig, err)
            self.response = self.iface.exceptionreport(
                'NoApplicableCode', 'service', msg)
            return

        # set server.home safely
        # TODO: make this more abstract
        self.config.set(
            'server', 'home',
            os.path.dirname(os.path.join(os.path.dirname(__file__), '..'))
        )

        self.context.pycsw_home = self.config.get('server', 'home')
        self.context.url = self.config.get('server', 'url')

        log.setup_logger(self.config)

        LOGGER.info('running configuration %s', rtconfig)
        LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING'])

        # set OGC schemas location
        if not self.config.has_option('server', 'ogc_schemas_base'):
            self.config.set('server', 'ogc_schemas_base',
                            self.context.ogc_schemas_base)

        # set mimetype
        if self.config.has_option('server', 'mimetype'):
            self.mimetype = self.config.get('server', 'mimetype').encode()

        # set encoding
        if self.config.has_option('server', 'encoding'):
            self.encoding = self.config.get('server', 'encoding')

        # set domainquerytype
        if self.config.has_option('server', 'domainquerytype'):
            self.domainquerytype = self.config.get('server', 'domainquerytype')

        # set XML pretty print
        if (self.config.has_option('server', 'pretty_print') and
                self.config.get('server', 'pretty_print') == 'true'):
            self.pretty_print = 1

        # set Spatial Ranking option
        if (self.config.has_option('server', 'spatial_ranking') and
                self.config.get('server', 'spatial_ranking') == 'true'):
            util.ranking_enabled = True

        # set language default
        if self.config.has_option('server', 'language'):
            try:
                LOGGER.info('Setting language')
                lang_code = self.config.get('server', 'language').split('-')[0]
                self.language['639_code'] = lang_code
                self.language['text'] = self.context.languages[lang_code]
            except Exception as err:
                LOGGER.exception('Could not set language: %s', err)
                pass

        LOGGER.debug('Configuration: %s.', self.config)
        LOGGER.debug('Model: %s.', self.context.model)

        # load user-defined mappings if they exist
        if self.config.has_option('repository', 'mappings'):
            # override default repository mappings
            try:
                import imp
                module = self.config.get('repository', 'mappings')
                if os.sep in module:  # filepath
                    modulename = '%s' % os.path.splitext(module)[0].replace(
                        os.sep, '.')
                    mappings = imp.load_source(modulename, module)
                else:  # dotted name
                    mappings = __import__(module, fromlist=[''])
                LOGGER.info('Loading custom repository mappings '
                             'from %s', module)
                self.context.md_core_model = mappings.MD_CORE_MODEL
                self.context.refresh_dc(mappings.MD_CORE_MODEL)
            except Exception as err:
                LOGGER.exception('Could not load custom mappings: %s', err)
                self.response = self.iface.exceptionreport(
                    'NoApplicableCode', 'service',
                    'Could not load repository.mappings')

        # load outputschemas
        LOGGER.info('Loading outputschemas')

        for osch in pycsw.plugins.outputschemas.__all__:
            output_schema_module = __import__(
                'pycsw.plugins.outputschemas.%s' % osch)
            mod = getattr(output_schema_module.plugins.outputschemas, osch)
            self.outputschemas[mod.NAMESPACE] = mod

        LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas)
        LOGGER.debug('Namespaces: %s', self.context.namespaces)

    def expand_path(self, path):
        """ return safe path for WSGI environments """
        if 'local.app_root' in self.environ and not os.path.isabs(path):
            return os.path.join(self.environ['local.app_root'], path)
        else:
            return path

    def dispatch_wsgi(self):
        """ WSGI handler """

        if hasattr(self, 'response'):
            return self._write_response()

        LOGGER.debug('WSGI mode detected')

        if self.environ['REQUEST_METHOD'] == 'POST':
            try:
                request_body_size = int(self.environ.get('CONTENT_LENGTH', 0))
            except (ValueError):
                request_body_size = 0

            self.requesttype = 'POST'
            self.request = self.environ['wsgi.input'].read(request_body_size)
            LOGGER.debug('Request type: POST.  Request:\n%s\n', self.request)

        else:  # it's a GET request
            self.requesttype = 'GET'
            self.request = wsgiref.util.request_uri(self.environ)
            try:
                query_part = splitquery(self.request)[-1]
                self.kvp = dict(parse_qsl(query_part, keep_blank_values=True))
            except AttributeError as err:
                LOGGER.exception('Could not parse query string')
                self.kvp = {}
            LOGGER.debug('Request type: GET.  Request:\n%s\n', self.request)
        return self.dispatch()

    def opensearch(self):
        """ enable OpenSearch """
        if not self.opensearchobj:
            self.opensearchobj = opensearch.OpenSearch(self.context)

        return self.opensearchobj

    def sru(self):
        """ enable SRU """
        if not self.sruobj:
            self.sruobj = sru.Sru(self.context)

        return self.sruobj

    def oaipmh(self):
        """ enable OAI-PMH """
        if not self.oaipmhobj:
            self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config)
        return self.oaipmhobj

    def dispatch(self, writer=sys.stdout, write_headers=True):
        """ Handle incoming HTTP request """

        error = 0
        if self.requesttype == 'GET':
            self.kvp = self.normalize_kvp(self.kvp)
            version_202 = ('version' in self.kvp and
                           self.kvp['version'] == '2.0.2')
            accept_version_202 = ('acceptversions' in self.kvp and
                                  '2.0.2' in self.kvp['acceptversions'])
            if version_202 or accept_version_202:
                self.request_version = '2.0.2'
        elif self.requesttype == 'POST':
            if self.request.find(b'cat/csw/2.0.2') != -1:
                self.request_version = '2.0.2'
            elif self.request.find(b'cat/csw/3.0') != -1:
                self.request_version = '3.0.0'

        if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
                self.kvp['mode'] == 'sru'):
            self.mode = 'sru'
            self.request_version = '2.0.2'
            LOGGER.info('SRU mode detected; processing request')
            self.kvp = self.sru().request_sru2csw(self.kvp)

        if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
                self.kvp['mode'] == 'oaipmh'):
            self.mode = 'oaipmh'
            self.request_version = '2.0.2'
            LOGGER.info('OAI-PMH mode detected; processing request.')
            self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k)
            self.kvp = self.oaipmh().request(self.kvp)

        if self.request_version == '2.0.2':
            self.iface = csw2.Csw2(server_csw=self)
            self.context.set_model('csw')

        # configure transaction support, if specified in config
        self._gen_manager()

        namespaces = self.context.namespaces
        ops = self.context.model['operations']
        constraints = self.context.model['constraints']
        # generate domain model
        # NOTE: We should probably avoid this sort of mutable state for WSGI
        if 'GetDomain' not in ops:
            ops['GetDomain'] = self.context.gen_domains()

        # generate distributed search model, if specified in config
        if self.config.has_option('server', 'federatedcatalogues'):
            LOGGER.info('Configuring distributed search')

            constraints['FederatedCatalogues'] = {'values': []}

            for fedcat in self.config.get('server',
                                          'federatedcatalogues').split(','):
                LOGGER.debug('federated catalogue: %s', fedcat)
                constraints['FederatedCatalogues']['values'].append(fedcat)

        for key, value in self.outputschemas.items():
            get_records_params = ops['GetRecords']['parameters']
            get_records_params['outputSchema']['values'].append(
                value.NAMESPACE)
            get_records_by_id_params = ops['GetRecordById']['parameters']
            get_records_by_id_params['outputSchema']['values'].append(
                value.NAMESPACE)
            if 'Harvest' in ops:
                harvest_params = ops['Harvest']['parameters']
                harvest_params['ResourceType']['values'].append(
                    value.NAMESPACE)

        LOGGER.info('Setting MaxRecordDefault')
        if self.config.has_option('server', 'maxrecords'):
            constraints['MaxRecordDefault']['values'] = [
                self.config.get('server', 'maxrecords')]

        # load profiles
        if self.config.has_option('server', 'profiles'):
            self.profiles = pprofile.load_profiles(
                os.path.join('pycsw', 'plugins', 'profiles'),
                pprofile.Profile,
                self.config.get('server', 'profiles')
            )

            for prof in self.profiles['plugins'].keys():
                tmp = self.profiles['plugins'][prof](self.context.model,
                                                     namespaces,
                                                     self.context)

                key = tmp.outputschema  # to ref by outputschema
                self.profiles['loaded'][key] = tmp
                self.profiles['loaded'][key].extend_core(self.context.model,
                                                         namespaces,
                                                         self.config)

            LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys()))

        # init repository
        # look for tablename, set 'records' as default
        if not self.config.has_option('repository', 'table'):
            self.config.set('repository', 'table', 'records')

        repo_filter = None
        if self.config.has_option('repository', 'filter'):
            repo_filter = self.config.get('repository', 'filter')

        if self.config.has_option('repository', 'source'):  # load custom repository
            rs = self.config.get('repository', 'source')
            rs_modname, rs_clsname = rs.rsplit('.', 1)

            rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname])
            rs_cls = getattr(rs_mod, rs_clsname)

            try:
                self.repository = rs_cls(self.context, repo_filter)
                LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype)
            except Exception as err:
                msg = 'Could not load custom repository %s: %s' % (rs, err)
                LOGGER.exception(msg)
                error = 1
                code = 'NoApplicableCode'
                locator = 'service'
                text = 'Could not initialize repository. Check server logs'

        else:  # load default repository
            self.orm = 'sqlalchemy'
            from pycsw.core import repository
            try:
                LOGGER.info('Loading default repository')
                self.repository = repository.Repository(
                    self.config.get('repository', 'database'),
                    self.context,
                    self.environ.get('local.app_root', None),
                    self.config.get('repository', 'table'),
                    repo_filter
                )
                LOGGER.debug(
                    'Repository loaded (local): %s.' % self.repository.dbtype)
            except Exception as err:
                msg = 'Could not load repository (local): %s' % err
                LOGGER.exception(msg)
                error = 1
                code = 'NoApplicableCode'
                locator = 'service'
                text = 'Could not initialize repository. Check server logs'

        if self.requesttype == 'POST':
            LOGGER.debug('HTTP POST request')
            LOGGER.debug('CSW version: %s', self.iface.version)
            self.kvp = self.iface.parse_postdata(self.request)

        if isinstance(self.kvp, str):  # it's an exception
            error = 1
            locator = 'service'
            text = self.kvp
            if (self.kvp.find('the document is not valid') != -1 or
                    self.kvp.find('document not well-formed') != -1):
                code = 'NoApplicableCode'
            else:
                code = 'InvalidParameterValue'

        LOGGER.debug('HTTP Headers:\n%s.', self.environ)
        LOGGER.debug('Parsed request parameters: %s', self.kvp)

        if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
                self.kvp['mode'] == 'opensearch'):
            self.mode = 'opensearch'
            LOGGER.info('OpenSearch mode detected; processing request.')
            self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'

        if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or
                (len(self.kvp) == 1 and 'config' in self.kvp)):
            LOGGER.info('Turning on default csw30:Capabilities for base URL')
            self.kvp = {
                'service': 'CSW',
                'acceptversions': '3.0.0',
                'request': 'GetCapabilities'
            }
            http_accept = self.environ.get('HTTP_ACCEPT', '')
            if 'application/opensearchdescription+xml' in http_accept:
                self.mode = 'opensearch'
                self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'

        if error == 0:
            # test for the basic keyword values (service, version, request)
            basic_options = ['service', 'request']
            request = self.kvp.get('request', '')
            own_version_integer = util.get_version_integer(
                self.request_version)
            if self.request_version == '2.0.2':
                basic_options.append('version')
            if self.request_version == '3.0.0' and 'version' not in self.kvp and self.requesttype == 'POST':
                if 'service' not in self.kvp:
                    self.kvp['service'] = 'CSW'
                    basic_options.append('service')
                self.kvp['version'] = self.request_version
                basic_options.append('version')

            for k in basic_options:
                if k not in self.kvp:
                    if (k in ['version', 'acceptversions'] and
                            request == 'GetCapabilities'):
                        pass
                    else:
                        error = 1
                        locator = k
                        code = 'MissingParameterValue'
                        text = 'Missing keyword: %s' % k
                        break

            # test each of the basic keyword values
            if error == 0:
                # test service
                if self.kvp['service'] != 'CSW':
                    error = 1
                    locator = 'service'
                    code = 'InvalidParameterValue'
                    text = 'Invalid value for service: %s.\
                    Value MUST be CSW' % self.kvp['service']

                # test version
                kvp_version = self.kvp.get('version', '')
                try:
                    kvp_version_integer = util.get_version_integer(kvp_version)
                except Exception as err:
                    kvp_version_integer = 'invalid_value'
                if (request != 'GetCapabilities' and
                        kvp_version_integer != own_version_integer):
                    error = 1
                    locator = 'version'
                    code = 'InvalidParameterValue'
                    text = ('Invalid value for version: %s. Value MUST be '
                            '2.0.2 or 3.0.0' % kvp_version)

                # check for GetCapabilities acceptversions
                if 'acceptversions' in self.kvp:
                    for vers in self.kvp['acceptversions'].split(','):
                        vers_integer = util.get_version_integer(vers)
                        if vers_integer == own_version_integer:
                            break
                        else:
                            error = 1
                            locator = 'acceptversions'
                            code = 'VersionNegotiationFailed'
                            text = ('Invalid parameter value in '
                                    'acceptversions: %s. Value MUST be '
                                    '2.0.2 or 3.0.0' %
                                    self.kvp['acceptversions'])

                # test request
                if self.kvp['request'] not in \
                    self.context.model['operations']:
                    error = 1
                    locator = 'request'
                    if request in ['Transaction', 'Harvest']:
                        code = 'OperationNotSupported'
                        text = '%s operations are not supported' % request
                    else:
                        code = 'InvalidParameterValue'
                        text = 'Invalid value for request: %s' % request

        if error == 1:  # return an ExceptionReport
            LOGGER.error('basic service options error: %s, %s, %s', code, locator, text)
            self.response = self.iface.exceptionreport(code, locator, text)

        else:  # process per the request value

            if 'responsehandler' in self.kvp:
                # set flag to process asynchronously
                import threading
                self.asynchronous = True
                request_id = self.kvp.get('requestid', None)
                if request_id is None:
                    import uuid
                    self.kvp['requestid'] = str(uuid.uuid4())

            if self.kvp['request'] == 'GetCapabilities':
                self.response = self.iface.getcapabilities()
            elif self.kvp['request'] == 'DescribeRecord':
                self.response = self.iface.describerecord()
            elif self.kvp['request'] == 'GetDomain':
                self.response = self.iface.getdomain()
            elif self.kvp['request'] == 'GetRecords':
                if self.asynchronous:  # process asynchronously
                    threading.Thread(target=self.iface.getrecords).start()
                    self.response = self.iface._write_acknowledgement()
                else:
                    self.response = self.iface.getrecords()
            elif self.kvp['request'] == 'GetRecordById':
                self.response = self.iface.getrecordbyid()
            elif self.kvp['request'] == 'GetRepositoryItem':
                self.response = self.iface.getrepositoryitem()
            elif self.kvp['request'] == 'Transaction':
                self.response = self.iface.transaction()
            elif self.kvp['request'] == 'Harvest':
                if self.asynchronous:  # process asynchronously
                    threading.Thread(target=self.iface.harvest).start()
                    self.response = self.iface._write_acknowledgement()
                else:
                    self.response = self.iface.harvest()
            else:
                self.response = self.iface.exceptionreport(
                    'InvalidParameterValue', 'request',
                    'Invalid request parameter: %s' % self.kvp['request']
                )

        LOGGER.info('Request processed')
        if self.mode == 'sru':
            LOGGER.info('SRU mode detected; processing response.')
            self.response = self.sru().response_csw2sru(self.response,
                                                        self.environ)
        elif self.mode == 'opensearch':
            LOGGER.info('OpenSearch mode detected; processing response.')
            self.response = self.opensearch().response_csw2opensearch(
                self.response, self.config)

        elif self.mode == 'oaipmh':
            LOGGER.info('OAI-PMH mode detected; processing response.')
            self.response = self.oaipmh().response(
                self.response, self.oaiargs, self.repository,
                self.config.get('server', 'url')
            )

        return self._write_response()

    def getcapabilities(self):
        """ Handle GetCapabilities request """
        return self.iface.getcapabilities()

    def describerecord(self):
        """ Handle DescribeRecord request """
        return self.iface.describerecord()

    def getdomain(self):
        """ Handle GetDomain request """
        return self.iface.getdomain()

    def getrecords(self):
        """ Handle GetRecords request """
        return self.iface.getrecords()

    def getrecordbyid(self, raw=False):
        """ Handle GetRecordById request """
        return self.iface.getrecordbyid(raw)

    def getrepositoryitem(self):
        """ Handle GetRepositoryItem request """
        return self.iface.getrepositoryitem()

    def transaction(self):
        """ Handle Transaction request """
        return self.iface.transaction()

    def harvest(self):
        """ Handle Harvest request """
        return self.iface.harvest()

    def _write_response(self):
        """ Generate response """
        # set HTTP response headers and XML declaration

        xmldecl = ''
        appinfo = ''

        LOGGER.info('Writing response.')

        if hasattr(self, 'soap') and self.soap:
            self._gen_soap_wrapper()

        if etree.__version__ >= '3.5.0':  # remove superfluous namespaces
            etree.cleanup_namespaces(self.response,
                                     keep_ns_prefixes=self.context.keep_ns_prefixes)

        response = etree.tostring(self.response,
                                  pretty_print=self.pretty_print,
                                  encoding='unicode')

        if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and
                self.kvp['outputformat'] == 'application/json'):
            self.contenttype = self.kvp['outputformat']
            from pycsw.core.formats import fmt_json
            response = fmt_json.xml2json(response,
                                         self.context.namespaces,
                                         self.pretty_print)
        else:  # it's XML
            if 'outputformat' in self.kvp:
                self.contenttype = self.kvp['outputformat']
            else:
                self.contenttype = self.mimetype

            xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>'
                       '\n' % self.encoding)
            appinfo = '<!-- pycsw %s -->\n' % self.context.version

        if isinstance(self.contenttype, bytes):
            self.contenttype = self.contenttype.decode()

        s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding)
        LOGGER.debug('Response code: %s',
                     self.context.response_codes[self.status])
        LOGGER.debug('Response:\n%s', s)
        return [self.context.response_codes[self.status], s]

    def _gen_soap_wrapper(self):
        """ Generate SOAP wrapper """
        LOGGER.info('Writing SOAP wrapper.')
        node = etree.Element(
            util.nspath_eval('soapenv:Envelope', self.context.namespaces),
            nsmap=self.context.namespaces
        )

        schema_location_ns = util.nspath_eval('xsi:schemaLocation',
                                              self.context.namespaces)
        node.attrib[schema_location_ns] = '%s %s' % (
            self.context.namespaces['soapenv'],
            self.context.namespaces['soapenv']
        )

        node2 = etree.SubElement(
            node, util.nspath_eval('soapenv:Body', self.context.namespaces))

        if self.exception:
            node3 = etree.SubElement(
                node2,
                util.nspath_eval('soapenv:Fault', self.context.namespaces)
            )
            node4 = etree.SubElement(
                node3,
                util.nspath_eval('soapenv:Code', self.context.namespaces)
            )

            etree.SubElement(
                node4,
                util.nspath_eval('soapenv:Value', self.context.namespaces)
            ).text = 'soap:Server'

            node4 = etree.SubElement(
                node3,
                util.nspath_eval('soapenv:Reason', self.context.namespaces)
            )

            etree.SubElement(
                node4,
                util.nspath_eval('soapenv:Text', self.context.namespaces)
            ).text = 'A server exception was encountered.'

            node4 = etree.SubElement(
                node3,
                util.nspath_eval('soapenv:Detail', self.context.namespaces)
            )
            node4.append(self.response)
        else:
            node2.append(self.response)

        self.response = node

    def _gen_manager(self):
        """ Update self.context.model with CSW-T advertising """
        if (self.config.has_option('manager', 'transactions') and
                self.config.get('manager', 'transactions') == 'true'):

            self.manager = True

            self.context.model['operations_order'].append('Transaction')

            self.context.model['operations']['Transaction'] = {
                'methods': {'get': False, 'post': True},
                'parameters': {}
            }

            schema_values = [
                'http://www.opengis.net/cat/csw/2.0.2',
                'http://www.opengis.net/cat/csw/3.0',
                'http://www.opengis.net/wms',
                'http://www.opengis.net/wmts/1.0',
                'http://www.opengis.net/wfs',
                'http://www.opengis.net/wfs/2.0',
                'http://www.opengis.net/wcs',
                'http://www.opengis.net/wps/1.0.0',
                'http://www.opengis.net/sos/1.0',
                'http://www.opengis.net/sos/2.0',
                'http://www.isotc211.org/2005/gmi',
                'urn:geoss:waf',
            ]

            self.context.model['operations_order'].append('Harvest')

            self.context.model['operations']['Harvest'] = {
                'methods': {'get': False, 'post': True},
                'parameters': {
                    'ResourceType': {'values': schema_values}
                }
            }

            self.context.model['operations']['Transaction'] = {
                'methods': {'get': False, 'post': True},
                'parameters': {
                    'TransactionSchemas': {'values': sorted(schema_values)}
                }
            }

            self.csw_harvest_pagesize = 10
            if self.config.has_option('manager', 'csw_harvest_pagesize'):
                self.csw_harvest_pagesize = int(
                    self.config.get('manager', 'csw_harvest_pagesize'))

    def _test_manager(self):
        """ Verify that transactions are allowed """

        if self.config.get('manager', 'transactions') != 'true':
            raise RuntimeError('CSW-T interface is disabled')

        """ get the client first forwarded ip """
        if 'HTTP_X_FORWARDED_FOR' in self.environ:
            ipaddress = self.environ['HTTP_X_FORWARDED_FOR'].split(',')[0].strip()
        else:
            ipaddress = self.environ['REMOTE_ADDR']

        if not self.config.has_option('manager', 'allowed_ips') or \
        (self.config.has_option('manager', 'allowed_ips') and not
         util.ipaddress_in_whitelist(ipaddress,
                        self.config.get('manager', 'allowed_ips').split(','))):
            raise RuntimeError(
            'CSW-T operations not allowed for this IP address: %s' % ipaddress)

    def _cql_update_queryables_mappings(self, cql, mappings):
        """ Transform CQL query's properties to underlying DB columns """
        LOGGER.debug('Raw CQL text = %s', cql)
        LOGGER.debug(str(list(mappings.keys())))
        if cql is not None:
            for key in mappings.keys():
                try:
                    cql = cql.replace(key, mappings[key]['dbcol'])
                except:
                    cql = cql.replace(key, mappings[key])
            LOGGER.debug('Interpolated CQL text = %s.', cql)
            return cql

    def _process_responsehandler(self, xml):
        """ Process response handler """

        if self.kvp['responsehandler'] is not None:
            LOGGER.info('Processing responsehandler %s' %
                         self.kvp['responsehandler'])

            uprh = urlparse(self.kvp['responsehandler'])

            if uprh.scheme == 'mailto':  # email
                import smtplib

                LOGGER.debug('Email detected')

                smtp_host = 'localhost'
                if self.config.has_option('server', 'smtp_host'):
                    smtp_host = self.config.get('server', 'smtp_host')

                body = ('Subject: pycsw %s results\n\n%s' %
                        (self.kvp['request'], xml))

                try:
                    LOGGER.info('Sending email')
                    msg = smtplib.SMTP(smtp_host)
                    msg.sendmail(
                        self.config.get('metadata:main', 'contact_email'),
                        uprh.path, body
                    )
                    msg.quit()
                    LOGGER.debug('Email sent successfully.')
                except Exception as err:
                    LOGGER.exception('Error processing email')

            elif uprh.scheme == 'ftp':
                import ftplib

                LOGGER.debug('FTP detected.')

                try:
                    LOGGER.info('Sending to FTP server.')
                    ftp = ftplib.FTP(uprh.hostname)
                    if uprh.username is not None:
                        ftp.login(uprh.username, uprh.password)
                    ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml))
                    ftp.quit()
                    LOGGER.debug('FTP sent successfully.')
                except Exception as err:
                    LOGGER.exception('Error processing FTP')

    @staticmethod
    def normalize_kvp(kvp):
        """Normalize Key Value Pairs.

        This method will transform all keys to lowercase and leave values
        unchanged, as specified in the CSW standard (see for example note
        C on Table 62 - KVP Encoding for DescribeRecord operation request
        of the CSW standard version 2.0.2)

        :arg kvp: a mapping with Key Value Pairs
        :type kvp: dict
        :returns: A new dictionary with normalized parameters
        """

        result = dict()
        for name, value in kvp.items():
            result[name.lower()] = value
        return result
Esempio n. 23
0
    def __init__(self, email=None, api_key=None, config_file=None,
                 verbose=False, retry_on_errors=True,
                 site=None, client=None,
                 cert_bundle=None, insecure=None,
                 client_cert=None, client_cert_key=None):
        # type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], bool, Optional[str], Optional[str]) -> None
        if client is None:
            client = _default_client()

        # Fill values from Environment Variables if not available in Constructor
        if config_file is None:
            config_file = os.environ.get("ZULIP_CONFIG")
        if api_key is None:
            api_key = os.environ.get("ZULIP_API_KEY")
        if email is None:
            email = os.environ.get("ZULIP_EMAIL")
        if site is None:
            site = os.environ.get("ZULIP_SITE")
        if client_cert is None:
            client_cert = os.environ.get("ZULIP_CERT")
        if client_cert_key is None:
            client_cert_key = os.environ.get("ZULIP_CERT_KEY")
        if cert_bundle is None:
            cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")

        if config_file is None:
            config_file = get_default_config_filename()

        if config_file is not None and os.path.exists(config_file):
            config = SafeConfigParser()
            with open(config_file, 'r') as f:
                config.readfp(f, config_file)
            if api_key is None:
                api_key = config.get("api", "key")
            if email is None:
                email = config.get("api", "email")
            if site is None and config.has_option("api", "site"):
                site = config.get("api", "site")
            if client_cert is None and config.has_option("api", "client_cert"):
                client_cert = config.get("api", "client_cert")
            if client_cert_key is None and config.has_option("api", "client_cert_key"):
                client_cert_key = config.get("api", "client_cert_key")
            if cert_bundle is None and config.has_option("api", "cert_bundle"):
                cert_bundle = config.get("api", "cert_bundle")
            if insecure is None and config.has_option("api", "insecure"):
                # Be quite strict about what is accepted so that users don't
                # disable security unintentionally.
                insecure_setting = config.get("api", "insecure").lower()
                if insecure_setting == "true":
                    insecure = True
                elif insecure_setting == "false":
                    insecure = False
                else:
                    raise RuntimeError("insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
                                       % (insecure_setting, config_file))
        elif None in (api_key, email):
            raise RuntimeError("api_key or email not specified and %s does not exist"
                               % (config_file,))

        self.api_key = api_key
        self.email = email
        self.verbose = verbose
        if site is not None:
            if site.startswith("localhost"):
                site = "http://" + site
            elif not site.startswith("http"):
                site = "https://" + site
            # Remove trailing "/"s from site to simplify the below logic for adding "/api"
            site = site.rstrip("/")
            self.base_url = site
        else:
            raise RuntimeError("Missing Zulip server URL; specify via --site or ~/.zuliprc.")

        if not self.base_url.endswith("/api"):
            self.base_url += "/api"
        self.base_url += "/"
        self.retry_on_errors = retry_on_errors
        self.client_name = client

        if insecure:
            self.tls_verification = False  # type: Union[bool, str]
        elif cert_bundle is not None:
            if not os.path.isfile(cert_bundle):
                raise RuntimeError("tls bundle '%s' does not exist"
                                   % (cert_bundle,))
            self.tls_verification = cert_bundle
        else:
            # Default behavior: verify against system CA certificates
            self.tls_verification = True

        if client_cert is None:
            if client_cert_key is not None:
                raise RuntimeError("client cert key '%s' specified, but no client cert public part provided"
                                   % (client_cert_key,))
        else:  # we have a client cert
            if not os.path.isfile(client_cert):
                raise RuntimeError("client cert '%s' does not exist"
                                   % (client_cert,))
            if client_cert_key is not None:
                if not os.path.isfile(client_cert_key):
                    raise RuntimeError("client cert key '%s' does not exist"
                                       % (client_cert_key,))
        self.client_cert = client_cert
        self.client_cert_key = client_cert_key
Esempio n. 24
0
class Config(object):
    """
    Manages the configuration file
    """
    def __init__(self):
        """
        DEFAULT VALUES
        """
        self._basescript = None
        self.recentvaults = []
        self.pwlength = 10
        self.search_notes = False
        self.search_passwd = False
        self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_"
        self.avoid_bigrams = "cl mn nm nn rn vv VV"

        self._fname = self.get_config_filename()
        self._parser = SafeConfigParser()

        if os.path.exists(self._fname):
            self._parser.read(self._fname)

        if not self._parser.has_section("base"):
            self._parser.add_section("base")

        for num in range(10):
            if (not self._parser.has_option("base", "recentvaults" + str(num))):
                break
            self.recentvaults.append(self._parser.get("base", "recentvaults" + str(num)))

        if self._parser.has_option("base", "pwlength"):
            self.pwlength = int(self._parser.get("base", "pwlength"))

        if self._parser.has_option("base", "search_notes"):
            if self._parser.get("base", "search_notes") == "True":
                self.search_notes = True

        if self._parser.has_option("base", "search_passwd"):
            if self._parser.get("base", "search_passwd") == "True":
                self.search_passwd = True

        if self._parser.has_option("base", "alphabet"):
            self.alphabet = self._parser.get("base", "alphabet")

        if self._parser.has_option("base", "avoid_bigrams"):
            self.avoid_bigrams = self._parser.get("base", "avoid_bigrams")

        if not os.path.exists(self._fname):
            self.save()

    def set_basescript(self, basescript):
        self._basescript = basescript

    def get_basescript(self):
        return self._basescript

    def save(self):
        if (not os.path.exists(os.path.dirname(self._fname))):
            os.mkdir(os.path.dirname(self._fname))

        # remove duplicates and trim to 10 items
        _saved_recentvaults = []
        for item in self.recentvaults:
            if item in _saved_recentvaults:
                continue
            self._parser.set("base", "recentvaults" + str(len(_saved_recentvaults)), item)
            _saved_recentvaults.append(item)
            if (len(_saved_recentvaults) >= 10):
                break

        self._parser.set("base", "pwlength", str(self.pwlength))
        self._parser.set("base", "search_notes", str(self.search_notes))
        self._parser.set("base", "search_passwd", str(self.search_passwd))
        self._parser.set("base", "alphabet", str(self.alphabet))
        self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams))
        filehandle = open(self._fname, 'w')
        self._parser.write(filehandle)
        filehandle.close()

    @staticmethod
    def get_config_filename():
        """
        Returns the full filename of the config file
        """
        base_fname = "loxodo"

        # On Mac OS X, config files go to ~/Library/Application Support/foo/
        if platform.system() == "Darwin":
            base_path = os.path.join(os.path.expanduser("~"), "Library", "Application Support")
            if os.path.isdir(base_path):
                return os.path.join(base_path, base_fname, base_fname + ".ini")

        # On Microsoft Windows, config files go to $APPDATA/foo/
        if platform.system() in ("Windows", "Microsoft"):
            if ("APPDATA" in os.environ):
                base_path = os.environ["APPDATA"]
                if os.path.isdir(base_path):
                    return os.path.join(base_path, base_fname, base_fname + ".ini")

        # Allow config directory override as per freedesktop.org XDG Base Directory Specification
        if ("XDG_CONFIG_HOME" in os.environ):
            base_path = os.environ["XDG_CONFIG_HOME"]
            if os.path.isdir(base_path):
                return os.path.join(base_path, base_fname, base_fname + ".ini")

        # Default configuration path is ~/.config/foo/
        base_path = os.path.join(os.path.expanduser("~"), ".config")
        if os.path.isdir(base_path):
            return os.path.join(base_path, base_fname, base_fname + ".ini")
        else:
            return os.path.join(os.path.expanduser("~"),"."+ base_fname + ".ini")
Esempio n. 25
0
class Csw(object):
    """ Base CSW server """
    def __init__(self, rtconfig=None, env=None, version='3.0.0'):
        """ Initialize CSW """

        if not env:
            self.environ = os.environ
        else:
            self.environ = env

        self.context = config.StaticContext()

        # Lazy load this when needed
        # (it will permanently update global cfg namespaces)
        self.sruobj = None
        self.opensearchobj = None
        self.oaipmhobj = None

        # init kvp
        self.kvp = {}

        self.mode = 'csw'
        self.async = False
        self.soap = False
        self.request = None
        self.exception = False
        self.status = 'OK'
        self.profiles = None
        self.manager = False
        self.outputschemas = {}
        self.mimetype = 'application/xml; charset=UTF-8'
        self.encoding = 'UTF-8'
        self.pretty_print = 0
        self.domainquerytype = 'list'
        self.orm = 'django'
        self.language = {'639_code': 'en', 'text': 'english'}
        self.process_time_start = time()

        # define CSW implementation object (default CSW3)
        self.iface = csw3.Csw3(server_csw=self)
        self.request_version = version

        if self.request_version == '2.0.2':
            self.iface = csw2.Csw2(server_csw=self)
            self.context.set_model('csw')

        # load user configuration
        try:
            LOGGER.info('Loading user configuration')
            if isinstance(rtconfig, SafeConfigParser):  # serialized already
                self.config = rtconfig
            else:
                self.config = SafeConfigParser()
                if isinstance(rtconfig, dict):  # dictionary
                    for section, options in rtconfig.items():
                        self.config.add_section(section)
                        for k, v in options.items():
                            self.config.set(section, k, v)
                else:  # configuration file
                    import codecs
                    with codecs.open(rtconfig, encoding='utf-8') as scp:
                        self.config.readfp(scp)
        except Exception as err:
            LOGGER.exception('Could not load user configuration: %s', err)
            self.response = self.iface.exceptionreport(
                'NoApplicableCode', 'service',
                'Error opening configuration %s' % rtconfig
            )
            return

        # set server.home safely
        # TODO: make this more abstract
        self.config.set(
            'server', 'home',
            os.path.dirname(os.path.join(os.path.dirname(__file__), '..'))
        )

        self.context.pycsw_home = self.config.get('server', 'home')
        self.context.url = self.config.get('server', 'url')

        log.setup_logger(self.config)

        LOGGER.info('running configuration %s', rtconfig)
        LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING'])

        # set OGC schemas location
        if not self.config.has_option('server', 'ogc_schemas_base'):
            self.config.set('server', 'ogc_schemas_base',
                            self.context.ogc_schemas_base)

        # set mimetype
        if self.config.has_option('server', 'mimetype'):
            self.mimetype = self.config.get('server', 'mimetype').encode()

        # set encoding
        if self.config.has_option('server', 'encoding'):
            self.encoding = self.config.get('server', 'encoding')

        # set domainquerytype
        if self.config.has_option('server', 'domainquerytype'):
            self.domainquerytype = self.config.get('server', 'domainquerytype')

        # set XML pretty print
        if (self.config.has_option('server', 'pretty_print') and
                self.config.get('server', 'pretty_print') == 'true'):
            self.pretty_print = 1

        # set Spatial Ranking option
        if (self.config.has_option('server', 'spatial_ranking') and
                self.config.get('server', 'spatial_ranking') == 'true'):
            util.ranking_enabled = True

        # set language default
        if self.config.has_option('server', 'language'):
            try:
                LOGGER.info('Setting language')
                lang_code = self.config.get('server', 'language').split('-')[0]
                self.language['639_code'] = lang_code
                self.language['text'] = self.context.languages[lang_code]
            except Exception as err:
                LOGGER.exception('Could not set language: %s', err)
                pass

        LOGGER.debug('Configuration: %s.', self.config)
        LOGGER.debug('Model: %s.', self.context.model)

        # load user-defined mappings if they exist
        if self.config.has_option('repository', 'mappings'):
            # override default repository mappings
            try:
                import imp
                module = self.config.get('repository', 'mappings')
                if '/' in module:  # filepath
                    modulename = '%s' % os.path.splitext(module)[0].replace(
                        os.sep, '.')
                    mappings = imp.load_source(modulename, module)
                else:  # dotted name
                    mappings = __import__(module, fromlist=[''])
                LOGGER.info('Loading custom repository mappings '
                             'from %s', module)
                self.context.md_core_model = mappings.MD_CORE_MODEL
                self.context.refresh_dc(mappings.MD_CORE_MODEL)
            except Exception as err:
                LOGGER.exception('Could not load custom mappings: %s', err)
                self.response = self.iface.exceptionreport(
                    'NoApplicableCode', 'service',
                    'Could not load repository.mappings')

        # load outputschemas
        LOGGER.info('Loading outputschemas')

        for osch in pycsw.plugins.outputschemas.__all__:
            output_schema_module = __import__(
                'pycsw.plugins.outputschemas.%s' % osch)
            mod = getattr(output_schema_module.plugins.outputschemas, osch)
            self.outputschemas[mod.NAMESPACE] = mod

        LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas)
        LOGGER.debug('Namespaces: %s', self.context.namespaces)

    def expand_path(self, path):
        """ return safe path for WSGI environments """
        if 'local.app_root' in self.environ and not os.path.isabs(path):
            return os.path.join(self.environ['local.app_root'], path)
        else:
            return path

    def dispatch_wsgi(self):
        """ WSGI handler """

        if hasattr(self, 'response'):
            return self._write_response()

        LOGGER.debug('WSGI mode detected')

        if self.environ['REQUEST_METHOD'] == 'POST':
            try:
                request_body_size = int(self.environ.get('CONTENT_LENGTH', 0))
            except (ValueError):
                request_body_size = 0

            self.requesttype = 'POST'
            self.request = self.environ['wsgi.input'].read(request_body_size)
            LOGGER.debug('Request type: POST.  Request:\n%s\n', self.request)

        else:  # it's a GET request
            self.requesttype = 'GET'
            self.request = wsgiref.util.request_uri(self.environ)
            try:
                query_part = splitquery(self.request)[-1]
                self.kvp = dict(parse_qsl(query_part, keep_blank_values=True))
            except AttributeError as err:
                LOGGER.exception('Could not parse query string')
                self.kvp = {}
            LOGGER.debug('Request type: GET.  Request:\n%s\n', self.request)
        return self.dispatch()

    def opensearch(self):
        """ enable OpenSearch """
        if not self.opensearchobj:
            self.opensearchobj = opensearch.OpenSearch(self.context)

        return self.opensearchobj

    def sru(self):
        """ enable SRU """
        if not self.sruobj:
            self.sruobj = sru.Sru(self.context)

        return self.sruobj

    def oaipmh(self):
        """ enable OAI-PMH """
        if not self.oaipmhobj:
            self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config)
        return self.oaipmhobj

    def dispatch(self, writer=sys.stdout, write_headers=True):
        """ Handle incoming HTTP request """

        if self.requesttype == 'GET':
            self.kvp = self.normalize_kvp(self.kvp)
            version_202 = ('version' in self.kvp and
                           self.kvp['version'] == '2.0.2')
            accept_version_202 = ('acceptversions' in self.kvp and
                                  '2.0.2' in self.kvp['acceptversions'])
            if version_202 or accept_version_202:
                self.request_version = '2.0.2'
        elif self.requesttype == 'POST':
            if self.request.find(b'2.0.2') != -1:
                self.request_version = '2.0.2'

        if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
                self.kvp['mode'] == 'sru'):
            self.mode = 'sru'
            self.request_version = '2.0.2'
            LOGGER.info('SRU mode detected; processing request')
            self.kvp = self.sru().request_sru2csw(self.kvp)

        if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
                self.kvp['mode'] == 'oaipmh'):
            self.mode = 'oaipmh'
            self.request_version = '2.0.2'
            LOGGER.info('OAI-PMH mode detected; processing request.')
            self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k)
            self.kvp = self.oaipmh().request(self.kvp)

        if self.request_version == '2.0.2':
            self.iface = csw2.Csw2(server_csw=self)
            self.context.set_model('csw')

        # configure transaction support, if specified in config
        self._gen_manager()

        namespaces = self.context.namespaces
        ops = self.context.model['operations']
        constraints = self.context.model['constraints']
        # generate domain model
        # NOTE: We should probably avoid this sort of mutable state for WSGI
        if 'GetDomain' not in ops:
            ops['GetDomain'] = self.context.gen_domains()

        # generate distributed search model, if specified in config
        if self.config.has_option('server', 'federatedcatalogues'):
            LOGGER.info('Configuring distributed search')

            constraints['FederatedCatalogues'] = {'values': []}

            for fedcat in self.config.get('server',
                                          'federatedcatalogues').split(','):
                LOGGER.debug('federated catalogue: %s', fedcat)
                constraints['FederatedCatalogues']['values'].append(fedcat)

        for key, value in self.outputschemas.items():
            get_records_params = ops['GetRecords']['parameters']
            get_records_params['outputSchema']['values'].append(
                value.NAMESPACE)
            get_records_by_id_params = ops['GetRecordById']['parameters']
            get_records_by_id_params['outputSchema']['values'].append(
                value.NAMESPACE)
            if 'Harvest' in ops:
                harvest_params = ops['Harvest']['parameters']
                harvest_params['ResourceType']['values'].append(
                    value.NAMESPACE)

        LOGGER.info('Setting MaxRecordDefault')
        if self.config.has_option('server', 'maxrecords'):
            constraints['MaxRecordDefault']['values'] = [
                self.config.get('server', 'maxrecords')]

        # load profiles
        if self.config.has_option('server', 'profiles'):
            self.profiles = pprofile.load_profiles(
                os.path.join('pycsw', 'plugins', 'profiles'),
                pprofile.Profile,
                self.config.get('server', 'profiles')
            )

            for prof in self.profiles['plugins'].keys():
                tmp = self.profiles['plugins'][prof](self.context.model,
                                                     namespaces,
                                                     self.context)

                key = tmp.outputschema  # to ref by outputschema
                self.profiles['loaded'][key] = tmp
                self.profiles['loaded'][key].extend_core(self.context.model,
                                                         namespaces,
                                                         self.config)

            LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys()))

        # init repository
        # look for tablename, set 'records' as default
        if not self.config.has_option('repository', 'table'):
            self.config.set('repository', 'table', 'records')

        repo_filter = None
        if self.config.has_option('repository', 'filter'):
            repo_filter = self.config.get('repository', 'filter')

        if self.config.has_option('repository', 'source'):  # load custom repository
            rs = self.config.get('repository', 'source')
            rs_modname, rs_clsname = rs.rsplit('.', 1)

            rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname])
            rs_cls = getattr(rs_mod, rs_clsname)

            try:
                self.repository = rs_cls(self.context, repo_filter)
                LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype)
            except Exception as err:
                msg = 'Could not load custom repository'
                LOGGER.exception(msg)
                self.response = self.iface.exceptionreport(
                    'NoApplicableCode', 'service', msg)

        else:  # load default repository
            self.orm = 'sqlalchemy'
            from pycsw.core import repository
            try:
                LOGGER.info('Loading default repository')
                self.repository = repository.Repository(
                    self.config.get('repository', 'database'),
                    self.context,
                    self.environ.get('local.app_root', None),
                    self.config.get('repository', 'table'),
                    repo_filter
                )
                LOGGER.debug(
                    'Repository loaded (local): %s.' % self.repository.dbtype)
            except Exception as err:
                msg = 'Could not load repository (local)'
                LOGGER.exception(msg)
                self.response = self.iface.exceptionreport(
                    'NoApplicableCode', 'service', msg)

        if self.requesttype == 'POST':
            LOGGER.debug('HTTP POST request')
            LOGGER.debug('CSW version: %s', self.iface.version)
            self.kvp = self.iface.parse_postdata(self.request)

        error = 0

        if isinstance(self.kvp, str):  # it's an exception
            error = 1
            locator = 'service'
            text = self.kvp
            if (self.kvp.find('the document is not valid') != -1 or
                    self.kvp.find('document not well-formed') != -1):
                code = 'NoApplicableCode'
            else:
                code = 'InvalidParameterValue'

        LOGGER.debug('HTTP Headers:\n%s.', self.environ)
        LOGGER.debug('Parsed request parameters: %s', self.kvp)

        if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
                self.kvp['mode'] == 'opensearch'):
            self.mode = 'opensearch'
            LOGGER.info('OpenSearch mode detected; processing request.')
            self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'

        if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or
                (len(self.kvp) == 1 and 'config' in self.kvp)):
            LOGGER.info('Turning on default csw30:Capabilities for base URL')
            self.kvp = {
                'service': 'CSW',
                'acceptversions': '3.0.0',
                'request': 'GetCapabilities'
            }
            http_accept = self.environ.get('HTTP_ACCEPT', '')
            if 'application/opensearchdescription+xml' in http_accept:
                self.mode = 'opensearch'
                self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'

        if error == 0:
            # test for the basic keyword values (service, version, request)
            basic_options = ['service', 'request']
            request = self.kvp.get('request', '')
            own_version_integer = util.get_version_integer(
                self.request_version)
            if self.request_version == '2.0.2':
                basic_options.append('version')

            for k in basic_options:
                if k not in self.kvp:
                    if (k in ['version', 'acceptversions'] and
                            request == 'GetCapabilities'):
                        pass
                    else:
                        error = 1
                        locator = k
                        code = 'MissingParameterValue'
                        text = 'Missing keyword: %s' % k
                        break

            # test each of the basic keyword values
            if error == 0:
                # test service
                if self.kvp['service'] != 'CSW':
                    error = 1
                    locator = 'service'
                    code = 'InvalidParameterValue'
                    text = 'Invalid value for service: %s.\
                    Value MUST be CSW' % self.kvp['service']

                # test version
                kvp_version = self.kvp.get('version', '')
                try:
                    kvp_version_integer = util.get_version_integer(kvp_version)
                except Exception as err:
                    kvp_version_integer = 'invalid_value'
                if (request != 'GetCapabilities' and
                        kvp_version_integer != own_version_integer):
                    error = 1
                    locator = 'version'
                    code = 'InvalidParameterValue'
                    text = ('Invalid value for version: %s. Value MUST be '
                            '2.0.2 or 3.0.0' % kvp_version)

                # check for GetCapabilities acceptversions
                if 'acceptversions' in self.kvp:
                    for vers in self.kvp['acceptversions'].split(','):
                        vers_integer = util.get_version_integer(vers)
                        if vers_integer == own_version_integer:
                            break
                        else:
                            error = 1
                            locator = 'acceptversions'
                            code = 'VersionNegotiationFailed'
                            text = ('Invalid parameter value in '
                                    'acceptversions: %s. Value MUST be '
                                    '2.0.2 or 3.0.0' %
                                    self.kvp['acceptversions'])

                # test request
                if self.kvp['request'] not in \
                    self.context.model['operations']:
                    error = 1
                    locator = 'request'
                    if request in ['Transaction', 'Harvest']:
                        code = 'OperationNotSupported'
                        text = '%s operations are not supported' % request
                    else:
                        code = 'InvalidParameterValue'
                        text = 'Invalid value for request: %s' % request

        if error == 1:  # return an ExceptionReport
            LOGGER.error('basic service options error: %s, %s, %s', code, locator, text)
            self.response = self.iface.exceptionreport(code, locator, text)

        else:  # process per the request value

            if 'responsehandler' in self.kvp:
                # set flag to process asynchronously
                import threading
                self.async = True
                request_id = self.kvp.get('requestid', None)
                if request_id is None:
                    import uuid
                    self.kvp['requestid'] = str(uuid.uuid4())

            if self.kvp['request'] == 'GetCapabilities':
                self.response = self.iface.getcapabilities()
            elif self.kvp['request'] == 'DescribeRecord':
                self.response = self.iface.describerecord()
            elif self.kvp['request'] == 'GetDomain':
                self.response = self.iface.getdomain()
            elif self.kvp['request'] == 'GetRecords':
                if self.async:  # process asynchronously
                    threading.Thread(target=self.iface.getrecords).start()
                    self.response = self.iface._write_acknowledgement()
                else:
                    self.response = self.iface.getrecords()
            elif self.kvp['request'] == 'GetRecordById':
                self.response = self.iface.getrecordbyid()
            elif self.kvp['request'] == 'GetRepositoryItem':
                self.response = self.iface.getrepositoryitem()
            elif self.kvp['request'] == 'Transaction':
                self.response = self.iface.transaction()
            elif self.kvp['request'] == 'Harvest':
                if self.async:  # process asynchronously
                    threading.Thread(target=self.iface.harvest).start()
                    self.response = self.iface._write_acknowledgement()
                else:
                    self.response = self.iface.harvest()
            else:
                self.response = self.iface.exceptionreport(
                    'InvalidParameterValue', 'request',
                    'Invalid request parameter: %s' % self.kvp['request']
                )

        LOGGER.info('Request processed')
        if self.mode == 'sru':
            LOGGER.info('SRU mode detected; processing response.')
            self.response = self.sru().response_csw2sru(self.response,
                                                        self.environ)
        elif self.mode == 'opensearch':
            LOGGER.info('OpenSearch mode detected; processing response.')
            self.response = self.opensearch().response_csw2opensearch(
                self.response, self.config)

        elif self.mode == 'oaipmh':
            LOGGER.info('OAI-PMH mode detected; processing response.')
            self.response = self.oaipmh().response(
                self.response, self.oaiargs, self.repository,
                self.config.get('server', 'url')
            )

        return self._write_response()

    def getcapabilities(self):
        """ Handle GetCapabilities request """
        return self.iface.getcapabilities()

    def describerecord(self):
        """ Handle DescribeRecord request """
        return self.iface.describerecord()

    def getdomain(self):
        """ Handle GetDomain request """
        return self.iface.getdomain()

    def getrecords(self):
        """ Handle GetRecords request """
        return self.iface.getrecords()

    def getrecordbyid(self, raw=False):
        """ Handle GetRecordById request """
        return self.iface.getrecordbyid(raw)

    def getrepositoryitem(self):
        """ Handle GetRepositoryItem request """
        return self.iface.getrepositoryitem()

    def transaction(self):
        """ Handle Transaction request """
        return self.iface.transaction()

    def harvest(self):
        """ Handle Harvest request """
        return self.iface.harvest()

    def _write_response(self):
        """ Generate response """
        # set HTTP response headers and XML declaration

        xmldecl = ''
        appinfo = ''

        LOGGER.info('Writing response.')

        if hasattr(self, 'soap') and self.soap:
            self._gen_soap_wrapper()

        if etree.__version__ >= '3.5.0':  # remove superfluous namespaces
            etree.cleanup_namespaces(self.response,
                                     keep_ns_prefixes=self.context.keep_ns_prefixes)

        response = etree.tostring(self.response,
                                  pretty_print=self.pretty_print,
                                  encoding='unicode')

        if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and
                self.kvp['outputformat'] == 'application/json'):
            self.contenttype = self.kvp['outputformat']
            from pycsw.core.formats import fmt_json
            response = fmt_json.xml2json(response,
                                         self.context.namespaces,
                                         self.pretty_print)
        else:  # it's XML
            if 'outputformat' in self.kvp:
                self.contenttype = self.kvp['outputformat']
            else:
                self.contenttype = self.mimetype

            xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>'
                       '\n' % self.encoding)
            appinfo = '<!-- pycsw %s -->\n' % self.context.version

        if isinstance(self.contenttype, bytes):
            self.contenttype = self.contenttype.decode()

        s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding)
        LOGGER.debug('Response code: %s',
                     self.context.response_codes[self.status])
        LOGGER.debug('Response:\n%s', s)
        return [self.context.response_codes[self.status], s]

    def _gen_soap_wrapper(self):
        """ Generate SOAP wrapper """
        LOGGER.info('Writing SOAP wrapper.')
        node = etree.Element(
            util.nspath_eval('soapenv:Envelope', self.context.namespaces),
            nsmap=self.context.namespaces
        )

        schema_location_ns = util.nspath_eval('xsi:schemaLocation',
                                              self.context.namespaces)
        node.attrib[schema_location_ns] = '%s %s' % (
            self.context.namespaces['soapenv'],
            self.context.namespaces['soapenv']
        )

        node2 = etree.SubElement(
            node, util.nspath_eval('soapenv:Body', self.context.namespaces))

        if self.exception:
            node3 = etree.SubElement(
                node2,
                util.nspath_eval('soapenv:Fault', self.context.namespaces)
            )
            node4 = etree.SubElement(
                node3,
                util.nspath_eval('soapenv:Code', self.context.namespaces)
            )

            etree.SubElement(
                node4,
                util.nspath_eval('soapenv:Value', self.context.namespaces)
            ).text = 'soap:Server'

            node4 = etree.SubElement(
                node3,
                util.nspath_eval('soapenv:Reason', self.context.namespaces)
            )

            etree.SubElement(
                node4,
                util.nspath_eval('soapenv:Text', self.context.namespaces)
            ).text = 'A server exception was encountered.'

            node4 = etree.SubElement(
                node3,
                util.nspath_eval('soapenv:Detail', self.context.namespaces)
            )
            node4.append(self.response)
        else:
            node2.append(self.response)

        self.response = node

    def _gen_manager(self):
        """ Update self.context.model with CSW-T advertising """
        if (self.config.has_option('manager', 'transactions') and
                self.config.get('manager', 'transactions') == 'true'):

            self.manager = True

            self.context.model['operations_order'].append('Transaction')

            self.context.model['operations']['Transaction'] = {
                'methods': {'get': False, 'post': True},
                'parameters': {}
            }

            schema_values = [
                'http://www.opengis.net/cat/csw/2.0.2',
                'http://www.opengis.net/cat/csw/3.0',
                'http://www.opengis.net/wms',
                'http://www.opengis.net/wmts/1.0',
                'http://www.opengis.net/wfs',
                'http://www.opengis.net/wfs/2.0',
                'http://www.opengis.net/wcs',
                'http://www.opengis.net/wps/1.0.0',
                'http://www.opengis.net/sos/1.0',
                'http://www.opengis.net/sos/2.0',
                'http://www.isotc211.org/2005/gmi',
                'urn:geoss:waf',
            ]

            self.context.model['operations_order'].append('Harvest')

            self.context.model['operations']['Harvest'] = {
                'methods': {'get': False, 'post': True},
                'parameters': {
                    'ResourceType': {'values': schema_values}
                }
            }

            self.context.model['operations']['Transaction'] = {
                'methods': {'get': False, 'post': True},
                'parameters': {
                    'TransactionSchemas': {'values': sorted(schema_values)}
                }
            }

            self.csw_harvest_pagesize = 10
            if self.config.has_option('manager', 'csw_harvest_pagesize'):
                self.csw_harvest_pagesize = int(
                    self.config.get('manager', 'csw_harvest_pagesize'))

    def _test_manager(self):
        """ Verify that transactions are allowed """

        if self.config.get('manager', 'transactions') != 'true':
            raise RuntimeError('CSW-T interface is disabled')

        ipaddress = self.environ['REMOTE_ADDR']

        if not self.config.has_option('manager', 'allowed_ips') or \
        (self.config.has_option('manager', 'allowed_ips') and not
         util.ipaddress_in_whitelist(ipaddress,
                        self.config.get('manager', 'allowed_ips').split(','))):
            raise RuntimeError(
            'CSW-T operations not allowed for this IP address: %s' % ipaddress)

    def _cql_update_queryables_mappings(self, cql, mappings):
        """ Transform CQL query's properties to underlying DB columns """
        LOGGER.debug('Raw CQL text = %s', cql)
        LOGGER.debug(str(list(mappings.keys())))
        if cql is not None:
            for key in mappings.keys():
                try:
                    cql = cql.replace(key, mappings[key]['dbcol'])
                except:
                    cql = cql.replace(key, mappings[key])
            LOGGER.debug('Interpolated CQL text = %s.', cql)
            return cql

    def _process_responsehandler(self, xml):
        """ Process response handler """

        if self.kvp['responsehandler'] is not None:
            LOGGER.info('Processing responsehandler %s' %
                         self.kvp['responsehandler'])

            uprh = urlparse(self.kvp['responsehandler'])

            if uprh.scheme == 'mailto':  # email
                import smtplib

                LOGGER.debug('Email detected')

                smtp_host = 'localhost'
                if self.config.has_option('server', 'smtp_host'):
                    smtp_host = self.config.get('server', 'smtp_host')

                body = ('Subject: pycsw %s results\n\n%s' %
                        (self.kvp['request'], xml))

                try:
                    LOGGER.info('Sending email')
                    msg = smtplib.SMTP(smtp_host)
                    msg.sendmail(
                        self.config.get('metadata:main', 'contact_email'),
                        uprh.path, body
                    )
                    msg.quit()
                    LOGGER.debug('Email sent successfully.')
                except Exception as err:
                    LOGGER.exception('Error processing email')

            elif uprh.scheme == 'ftp':
                import ftplib

                LOGGER.debug('FTP detected.')

                try:
                    LOGGER.info('Sending to FTP server.')
                    ftp = ftplib.FTP(uprh.hostname)
                    if uprh.username is not None:
                        ftp.login(uprh.username, uprh.password)
                    ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml))
                    ftp.quit()
                    LOGGER.debug('FTP sent successfully.')
                except Exception as err:
                    LOGGER.exception('Error processing FTP')

    @staticmethod
    def normalize_kvp(kvp):
        """Normalize Key Value Pairs.

        This method will transform all keys to lowercase and leave values
        unchanged, as specified in the CSW standard (see for example note
        C on Table 62 - KVP Encoding for DescribeRecord operation request
        of the CSW standard version 2.0.2)

        :arg kvp: a mapping with Key Value Pairs
        :type kvp: dict
        :returns: A new dictionary with normalized parameters
        """

        result = dict()
        for name, value in kvp.items():
            result[name.lower()] = value
        return result
Esempio n. 26
0
File: pip.py Progetto: bron84/stash
class PackageConfigHandler(object):
    """
    Manager class for packages files for tracking installation of modules
    """

    def __init__(self):
        # noinspection PyUnresolvedReferences
        from six.moves.configparser import SafeConfigParser
        self.package_cfg = os.path.expanduser('~/Documents/site-packages/.pypi_packages')
        if not os.path.isfile(self.package_cfg):
            print('Creating package file')
            with open(self.package_cfg, 'w') as outs:
                outs.close()
        self.parser = SafeConfigParser()
        self.parser.read(self.package_cfg)

    def save(self):
        with open(self.package_cfg, 'w') as outs:
            self.parser.write(outs)

    def add_module(self, pkg_info):
        """

        :param pkg_info: A dict that has name, url, version, summary
        :return:
        """
        if not self.parser.has_section(pkg_info['name']):
            self.parser.add_section(pkg_info['name'])
        self.parser.set(pkg_info['name'], 'url', pkg_info['url'])
        self.parser.set(pkg_info['name'], 'version', pkg_info['version'])
        self.parser.set(pkg_info['name'], 'summary', pkg_info['summary'])
        self.parser.set(pkg_info['name'], 'files', pkg_info['files'])
        self.parser.set(pkg_info['name'], 'dependency', pkg_info['dependency'])
        self.save()

    def list_modules(self):
        return [module for module in self.parser.sections()]

    def module_exists(self, name):
        return self.parser.has_section(name)

    def get_info(self, name):
        if self.parser.has_section(name):
            tbl = {}
            for opt, value in self.parser.items(name):
                tbl[opt] = value
            return tbl

    def remove_module(self, name):
        self.parser.remove_section(name)
        self.save()

    def get_files_installed(self, section_name):
        if self.parser.has_option(section_name, 'files'):
            files = self.parser.get(section_name, 'files').strip()
            return files.split(',')
        else:
            return None

    def get_dependencies(self, section_name):
        if self.parser.has_option(section_name, 'dependency'):
            dependencies = self.parser.get(section_name, 'dependency').strip()
            return set(dependencies.split(',')) if dependencies != '' else set()
        else:
            return None

    def get_all_dependencies(self, exclude_module=()):
        all_dependencies = set()
        for section_name in self.parser.sections():
            if section_name not in exclude_module and self.parser.has_option(section_name, 'dependency'):
                dependencies = self.parser.get(section_name, 'dependency').strip()
                if dependencies != '':
                    for dep in dependencies.split(','):
                        all_dependencies.add(dep)
        return all_dependencies
Esempio n. 27
0
    def __init__(self,
                 email=None,
                 api_key=None,
                 config_file=None,
                 verbose=False,
                 retry_on_errors=True,
                 site=None,
                 client=None,
                 cert_bundle=None,
                 insecure=None,
                 client_cert=None,
                 client_cert_key=None):
        # type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], bool, Optional[str], Optional[str]) -> None
        if client is None:
            client = _default_client()

        # Fill values from Environment Variables if not available in Constructor
        if config_file is None:
            config_file = os.environ.get("ZULIP_CONFIG")
        if api_key is None:
            api_key = os.environ.get("ZULIP_API_KEY")
        if email is None:
            email = os.environ.get("ZULIP_EMAIL")
        if site is None:
            site = os.environ.get("ZULIP_SITE")
        if client_cert is None:
            client_cert = os.environ.get("ZULIP_CERT")
        if client_cert_key is None:
            client_cert_key = os.environ.get("ZULIP_CERT_KEY")
        if cert_bundle is None:
            cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")

        if config_file is None:
            config_file = get_default_config_filename()

        if os.path.exists(config_file):
            config = SafeConfigParser()
            with open(config_file, 'r') as f:
                config.readfp(f, config_file)
            if api_key is None:
                api_key = config.get("api", "key")
            if email is None:
                email = config.get("api", "email")
            if site is None and config.has_option("api", "site"):
                site = config.get("api", "site")
            if client_cert is None and config.has_option("api", "client_cert"):
                client_cert = config.get("api", "client_cert")
            if client_cert_key is None and config.has_option(
                    "api", "client_cert_key"):
                client_cert_key = config.get("api", "client_cert_key")
            if cert_bundle is None and config.has_option("api", "cert_bundle"):
                cert_bundle = config.get("api", "cert_bundle")
            if insecure is None and config.has_option("api", "insecure"):
                # Be quite strict about what is accepted so that users don't
                # disable security unintentionally.
                insecure_setting = config.get("api", "insecure").lower()
                if insecure_setting == "true":
                    insecure = True
                elif insecure_setting == "false":
                    insecure = False
                else:
                    raise RuntimeError(
                        "insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
                        % (insecure_setting, config_file))
        elif None in (api_key, email):
            raise RuntimeError(
                "api_key or email not specified and %s does not exist" %
                (config_file, ))

        self.api_key = api_key
        self.email = email
        self.verbose = verbose
        if site is not None:
            if site.startswith("localhost"):
                site = "http://" + site
            elif not site.startswith("http"):
                site = "https://" + site
            # Remove trailing "/"s from site to simplify the below logic for adding "/api"
            site = site.rstrip("/")
            self.base_url = site
        else:
            raise RuntimeError(
                "Missing Zulip server URL; specify via --site or ~/.zuliprc.")

        if not self.base_url.endswith("/api"):
            self.base_url += "/api"
        self.base_url += "/"
        self.retry_on_errors = retry_on_errors
        self.client_name = client

        if insecure:
            self.tls_verification = False  # type: Union[bool, str]
        elif cert_bundle is not None:
            if not os.path.isfile(cert_bundle):
                raise RuntimeError("tls bundle '%s' does not exist" %
                                   (cert_bundle, ))
            self.tls_verification = cert_bundle
        else:
            # Default behavior: verify against system CA certificates
            self.tls_verification = True

        if client_cert is None:
            if client_cert_key is not None:
                raise RuntimeError(
                    "client cert key '%s' specified, but no client cert public part provided"
                    % (client_cert_key, ))
        else:  # we have a client cert
            if not os.path.isfile(client_cert):
                raise RuntimeError("client cert '%s' does not exist" %
                                   (client_cert, ))
            if client_cert_key is not None:
                if not os.path.isfile(client_cert_key):
                    raise RuntimeError("client cert key '%s' does not exist" %
                                       (client_cert_key, ))
        self.client_cert = client_cert
        self.client_cert_key = client_cert_key
Esempio n. 28
0
def load_conf():

    conf_path = os.path.expanduser(config_path)
    if not os.path.exists(conf_path):
        logger.warn(
            "{conf} couldn't be found, please use \'coscmd config -h\' to learn how to config coscmd!"
            .format(conf=to_printable_str(conf_path)))
        raise IOError
    else:
        logger.debug(
            '{conf} is found'.format(conf=to_printable_str(conf_path)))

    with open(conf_path, 'r') as f:
        cp = SafeConfigParser()
        cp.readfp(fp=f)
        if cp.has_option('common', 'part_size'):
            part_size = cp.getint('common', 'part_size')
        else:
            part_size = 1

        if cp.has_option('common', 'max_thread'):
            max_thread = cp.getint('common', 'max_thread')
        else:
            max_thread = 5
        try:
            secret_id = cp.get('common', 'secret_id')
        except Exception:
            secret_id = cp.get('common', 'access_id')
        try:
            appid = cp.get('common', 'appid')
            bucket = cp.get('common', 'bucket')
            if bucket.endswith("-" + str(appid)):
                bucket = bucket.rstrip(appid)
                bucket = bucket[:-1]
        except Exception:
            try:
                bucket = cp.get('common', 'bucket')
                appid = bucket.split('-')[-1]
                bucket = bucket.rstrip(appid)
                bucket = bucket[:-1]
            except Exception:
                logger.error(
                    "The configuration file is wrong. Please reconfirm")
        try:
            schema = cp.get('common', 'schema')
        except:
            schema = 'https'
        try:
            anonymous = cp.get('common', 'anonymous')
        except:
            anonymous = 'False'
        region, endpoint = None, None
        if cp.has_option('common', 'region'):
            region = cp.get('common', 'region')
        else:
            endpoint = cp.get('common', 'endpoint')

        if pre_appid != "":
            appid = pre_appid
        if pre_bucket != "":
            bucket = pre_bucket
        if pre_region != "":
            region = pre_region
        conf = CosConfig(appid=appid,
                         secret_id=secret_id,
                         secret_key=cp.get('common', 'secret_key'),
                         region=compatible(region),
                         endpoint=endpoint,
                         bucket=bucket,
                         part_size=part_size,
                         max_thread=max_thread,
                         schema=schema,
                         anonymous=anonymous)
        return conf