def read(self): """Override base class version to avoid parsing error with the first 'RANDFILE = ...' part of the openssl file. Also, reformat _sections to allow for the style of SSL config files where section headings can have spaces either side of the brackets e.g. [ sectionName ] and comments can occur on the same line as an option e.g. option = blah # This is option blah Reformat _sections to """ try: config_file = open(self._filePath) fileTxt = config_file.read() except Exception as e: raise OpenSSLConfigError('Reading OpenSSL config file "%s": %s' % (self._filePath, e)) idx = re.search('\[\s*\w*\s*\]', fileTxt).span()[0] config_file.seek(idx) SafeConfigParser.readfp(self, config_file) # Filter section names and remove comments from options for section, val in list(self._sections.items()): newSection = section self._sections[newSection.strip()] = \ dict([(opt, self._filtOptVal(optVal)) for opt, optVal in list(val.items())]) del self._sections[section] self._set_required_dn_params()
class Config(object): """A ConfigParser wrapper to support defaults when calling instance methods, and also tied to a single section""" SECTION = 'silkyy' def __init__(self, values=None, extra_sources=()): if values is None: sources = self._getsources() self.cp = ConfigParser() if __package__: default_config = ensure_str(get_data(__package__, 'default.conf')) self._load_config_file(StringIO(default_config)) for source in sources: if os.path.exists(source): self._load_config_file(open(source)) else: self.cp = SafeConfigParser(values) self.cp.add_section(self.SECTION) def _load_config_file(self, fp): config = StringIO() config.write('[' + self.SECTION + ']' + os.linesep) config.write(fp.read()) config.seek(0, os.SEEK_SET) self.cp.readfp(config) def _getsources(self): sources = ['conf/silkyy.conf'] return sources def get(self, option, default=None): env_key = 'SILKYY_' + option.replace('.', '_').upper() try: return os.getenv(env_key) or self.cp.get(self.SECTION, option) except (NoSectionError, NoOptionError): if default is not None: return default raise def _get(self, option, conv, default=None): return conv(self.get(option, default)) def getint(self, option, default=None): return self._get(option, int, default) def getboolean(self, option, default=None): return self._get(option, str2bool, default) def getfloat(self, option, default=None): return self._get(option, float, default) def items(self, section, default=None): try: return self.cp.items(section) except (NoSectionError, NoOptionError): if default is not None: return default raise
def getManifest(fp, format, defaults=None): """Read the manifest from the given open file pointer according to the given ManifestFormat. Pass a dict as ``defaults`` to override the defaults from the manifest format. """ if defaults is None: defaults = format.defaults parser = SafeConfigParser() if six.PY2: parser.readfp(fp) else: data = fp.read() if isinstance(data, six.binary_type): data = data.decode() parser.read_string(data) results = {} for key in format.keys: if parser.has_option(format.resourceType, key): results[key] = parser.get(format.resourceType, key) else: results[key] = defaults.get(key, None) for key in format.parameterSections: sectionName = "%s:%s" % (format.resourceType, key,) if parser.has_section(sectionName): results[key] = dict(parser.items(sectionName)) else: results[key] = {} return results
def read_config_file(path): config = SafeConfigParser() cfp = open(path, 'r') config.readfp(cfp) cfp.close() return config
def getPluginSettings(themeDirectory, plugins=None): """Given an IResourceDirectory for a theme, return the settings for the given list of plugins (or all plugins, if not given) provided as a list of (name, plugin) pairs. Returns a dict of dicts, with the outer dict having plugin names as keys and containing plugins settings (key/value pairs) as values. """ if plugins is None: plugins = getPlugins() manifestContents = {} if themeDirectory.isFile(MANIFEST_FILENAME): parser = SafeConfigParser() fp = themeDirectory.openFile(MANIFEST_FILENAME) try: parser.readfp(fp) for section in parser.sections(): manifestContents[section] = {} for name, value in parser.items(section): manifestContents[section][name] = value finally: try: fp.close() except AttributeError: pass pluginSettings = {} for name, plugin in plugins: pluginSettings[name] = manifestContents.get( "{0:s}:{1:s}".format(THEME_RESOURCE_NAME, name), {} ) return pluginSettings
class Config(object): """A ConfigParser wrapper to support defaults when calling instance methods, and also tied to a single section""" SECTION = 'scrapyd' def __init__(self, values=None, extra_sources=()): if values is None: sources = self._getsources() default_config = get_data(__package__, 'default_scrapyd.conf').decode('utf8') self.cp = SafeConfigParser() self.cp.readfp(io.StringIO(default_config)) sources.extend(extra_sources) for fname in sources: try: with io.open(fname) as fp: self.cp.readfp(fp) except (IOError, OSError): pass else: self.cp = SafeConfigParser(values) self.cp.add_section(self.SECTION) def _getsources(self): sources = ['/etc/scrapyd/scrapyd.conf', r'c:\scrapyd\scrapyd.conf'] sources += sorted(glob.glob('/etc/scrapyd/conf.d/*')) sources += ['scrapyd.conf'] sources += [expanduser('~/.scrapyd.conf')] scrapy_cfg = closest_scrapy_cfg() if scrapy_cfg: sources.append(scrapy_cfg) return sources def _getany(self, method, option, default): try: return method(self.SECTION, option) except (NoSectionError, NoOptionError): if default is not None: return default raise def get(self, option, default=None): return self._getany(self.cp.get, option, default) def getint(self, option, default=None): return self._getany(self.cp.getint, option, default) def getfloat(self, option, default=None): return self._getany(self.cp.getfloat, option, default) def getboolean(self, option, default=None): return self._getany(self.cp.getboolean, option, default) def items(self, section, default=None): try: return self.cp.items(section) except (NoSectionError, NoOptionError): if default is not None: return default raise
def getManifest(fp, format, defaults=None): """Read the manifest from the given open file pointer according to the given ManifestFormat. Pass a dict as ``defaults`` to override the defaults from the manifest format. """ if defaults is None: defaults = format.defaults parser = SafeConfigParser() parser.readfp(fp) results = {} for key in format.keys: if parser.has_option(format.resourceType, key): results[key] = parser.get(format.resourceType, key) else: results[key] = defaults.get(key, None) for key in format.parameterSections: sectionName = "%s:%s" % (format.resourceType, key,) if parser.has_section(sectionName): results[key] = dict(parser.items(sectionName)) else: results[key] = {} return results
class Config(object): """A ConfigParser wrapper to support defaults when calling instance methods, and also tied to a single section""" SECTION = 'scrapydartx' def __init__(self, values=None, extra_sources=()): if values is None: sources = self._getsources() default_config = get_data(__package__, 'default_scrapyd.conf').decode('utf8') self.cp = SafeConfigParser() self.cp.readfp(StringIO(default_config)) self.cp.read(sources) for fp in extra_sources: self.cp.readfp(fp) else: self.cp = SafeConfigParser(values) self.cp.add_section(self.SECTION) def _getsources(self): sources = [ '/etc/scrapydartx/scrapydartx.conf', r'c:\scrapyd\scrapyd.conf' ] sources += sorted(glob.glob('/etc/scrapydartx/conf.d/*')) sources += ['scrapydartx.conf'] sources += [expanduser('~/.scrapydartx.conf')] scrapy_cfg = closest_scrapy_cfg() if scrapy_cfg: sources.append(scrapy_cfg) return sources def _getany(self, method, option, default): try: return method(self.SECTION, option) except (NoSectionError, NoOptionError): if default is not None: return default raise def get(self, option, default=None): return self._getany(self.cp.get, option, default) def getint(self, option, default=None): return self._getany(self.cp.getint, option, default) def getfloat(self, option, default=None): return self._getany(self.cp.getfloat, option, default) def getboolean(self, option, default=None): return self._getany(self.cp.getboolean, option, default) def items(self, section, default=None): try: return self.cp.items(section) except (NoSectionError, NoOptionError): if default is not None: return default raise
def parse_config_file(self, config_file): config = SafeConfigParser(self.DEFAULT_CONFIG) config.readfp(config_file) blessconfig = { 'CLIENT_CONFIG': { 'domain_regex': config.get('CLIENT', 'domain_regex'), 'cache_dir': config.get('CLIENT', 'cache_dir'), 'cache_file': config.get('CLIENT', 'cache_file'), 'mfa_cache_dir': config.get('CLIENT', 'mfa_cache_dir'), 'mfa_cache_file': config.get('CLIENT', 'mfa_cache_file'), 'ip_urls': [ s.strip() for s in config.get('CLIENT', 'ip_urls').split(",") ], 'update_script': config.get('CLIENT', 'update_script'), 'user_session_length': int(config.get('CLIENT', 'user_session_length')), 'usebless_role_session_length': int(config.get('CLIENT', 'usebless_role_session_length')), 'update_sshagent': config.getboolean('CLIENT', 'update_sshagent'), }, 'BLESS_CONFIG': { 'userrole': config.get('LAMBDA', 'user_role'), 'accountid': config.get('LAMBDA', 'account_id'), 'functionname': config.get('LAMBDA', 'functionname'), 'functionversion': config.get('LAMBDA', 'functionversion'), 'certlifetime': config.getint('LAMBDA', 'certlifetime'), 'ipcachelifetime': config.getint('LAMBDA', 'ipcachelifetime'), 'timeoutconfig': { 'connect': config.getint('LAMBDA', 'timeout_connect'), 'read': config.getint('LAMBDA', 'timeout_read') } }, 'AWS_CONFIG': { 'bastion_ips': config.get('MAIN', 'bastion_ips'), 'remote_user': config.get('MAIN', 'remote_user') }, 'REGION_ALIAS': {} } regions = config.get('MAIN', 'region_aliases').split(",") regions = [region.strip() for region in regions] for region in regions: region = region.upper() kms_region_key = 'KMSAUTH_CONFIG_{}'.format(region) blessconfig.update( {kms_region_key: self._get_region_kms_config(region, config)}) blessconfig['REGION_ALIAS'].update( {region: blessconfig[kms_region_key]['awsregion']}) return blessconfig
def init_ini_file(file=args.ini_file): cp = SafeConfigParser() fp = open(file) cp.optionxform = str cp.readfp(fp) fp.close() cp.set('condor', 'lalsuite-install', lalinf_prefix) cp.set('analysis', 'engine', args.engine) cp.remove_option('analysis', 'nparallel') return cp
def init_ini_file(file=args.ini_file): cp=SafeConfigParser() fp=open(file) cp.optionxform = str cp.readfp(fp) fp.close() cp.set('condor','lalsuite-install',lalinf_prefix) cp.set('analysis','engine',args.engine) cp.remove_option('analysis','nparallel') return cp
def setUp(self): confdata = six.StringIO("""[broker] urls = amqps://broker1.example.com:5671 amqps://broker2.example.com:5671 cert = /etc/koji-hub/plugins/client.pem cacert = /etc/koji-hub/plugins/ca.pem topic_prefix = koji connect_timeout = 10 send_timeout = 60 """) conf = SafeConfigParser() conf.readfp(confdata) self.handler = protonmsg.TimeoutHandler( 'amqps://broker1.example.com:5671', [], conf)
def parse(self): parser = SafeConfigParser() parser.readfp(StringIO(self.obj.content)) for section in parser.values(): if section.name == 'options': options = 'install_requires', 'setup_requires', 'test_require' for name in options: content = section.get(name) if not content: continue self._parse_content(content) elif section.name == 'options.extras_require': for content in section.values(): self._parse_content(content)
def get_vrouter_tor_agent_name(self, conf_file): tor_agent_name = None if conf_file: try: data = StringIO('\n'.join(line.strip() for line in open(conf_file))) Config = SafeConfigParser() Config.readfp(data) except Exception as err: self.msg_log("Error reading file : " + conf_file + " Error : " + str(err), SandeshLevel.SYS_ERR) return tor_agent_name tor_agent_name = Config.get("DEFAULT", "agent_name") return tor_agent_name
def read_header(self): ''' Read the backup file header that contains the meta data about this particular backup. ''' with open(self.header) as fd: config = SafeConfigParser() config.readfp(fd) self.backup_type = config.get('ipa', 'type') self.backup_time = config.get('ipa', 'time') self.backup_host = config.get('ipa', 'host') self.backup_ipa_version = config.get('ipa', 'ipa_version') self.backup_version = config.get('ipa', 'version') self.backup_services = config.get('ipa', 'services').split(',')
def read_ini(cls, path, section=None): """read preferences from an .ini file""" parser = ConfigParser() parser.optionxform = str parser.readfp(mozfile.load(path)) if section: if section not in parser.sections(): raise PreferencesReadError("No section '%s' in %s" % (section, path)) retval = parser.items(section, raw=True) else: retval = parser.defaults().items() # cast the preferences since .ini is just strings return [(i, cls.cast(j)) for i, j in retval]
def setUp(self): super(FunctionalTestBase, self).setUp() if not os.path.exists(TEST_CFG): raise Exception("Unable to run the write tests without a test.ini in that defines an access_token with write privs.") cfg = SafeConfigParser() with open(TEST_CFG) as fp: cfg.readfp(fp, 'test.ini') access_token = cfg.get('write_tests', 'access_token') try: activity_id = cfg.get('activity_tests', 'activity_id') except NoOptionError: activity_id = None self.client = Client(access_token=access_token) self.activity_id = activity_id
def setUp(self): confdata = six.StringIO("""[broker] urls = amqps://broker1.example.com:5671 amqps://broker2.example.com:5671 cert = /etc/koji-hub/plugins/client.pem cacert = /etc/koji-hub/plugins/ca.pem topic_prefix = koji connect_timeout = 10 send_timeout = 60 """) if six.PY2: conf = SafeConfigParser() conf.readfp(confdata) else: conf = ConfigParser() conf.read_file(confdata) self.handler = protonmsg.TimeoutHandler('amqps://broker1.example.com:5671', [], conf)
def get_vrouter_process_info(self, proc_name): vrouter_file = "/etc/contrail/supervisord_vrouter_files" for root, dirs, files in os.walk(vrouter_file): for file in files: if file.endswith(".ini"): filename = \ '/etc/contrail/supervisord_vrouter_files/' + file try: data = StringIO('\n'.join(line.strip() for line in open(filename))) except IOError: msg = "This file does not exist anymore so continuing: " self.msg_log(msg + filename, SandeshLevel.SYS_ERR) continue Config = SafeConfigParser() Config.readfp(data) sections = Config.sections() if not sections[0]: msg = "Section not present in the ini file : " self.msg_log(msg + filename, SandeshLevel.SYS_ERR) continue name = sections[0].split(':') if len(name) < 2: msg = "Incorrect section name in the ini file : " self.msg_log(msg + filename, SandeshLevel.SYS_ERR) continue if name[1] == proc_name: command = Config.get(sections[0], "command") if not command: msg = "Command not present in the ini file : " self.msg_log(msg + filename, SandeshLevel.SYS_ERR) continue args = command.split() if (args[0] == '/usr/bin/contrail-tor-agent'): try: index = args.index('--config_file') args_val = args[index + 1] agent_name = \ self.get_vrouter_tor_agent_name(args_val) return (proc_name, agent_name) except Exception as err: msg = "Tor Agent command does " + \ "not have config file : " self.msg_log(msg + command, SandeshLevel.SYS_ERR) return ('vrouter_group', socket.getfqdn(self.host_ip) if self.hostname is None else self.hostname)
def handle(self, *args, **options): # type: (*Any, **Any) -> None config_file = os.path.join(os.environ["HOME"], ".zuliprc") if not os.path.exists(config_file): raise RuntimeError("No ~/.zuliprc found") config = SafeConfigParser() with open(config_file, 'r') as f: config.readfp(f, config_file) api_key = config.get("api", "key") email = config.get("api", "email") try: user_profile = get_user_profile_by_email(email) user_profile.api_key = api_key user_profile.save(update_fields=["api_key"]) except UserProfile.DoesNotExist: print("User %s does not exist; not syncing API key" % (email,))
def test_on_start_no_ssl(self, SSLDomain): confdata = six.StringIO("""[broker] urls = amqp://broker1.example.com:5672 amqp://broker2.example.com:5672 topic_prefix = koji connect_timeout = 10 send_timeout = 60 """) conf = SafeConfigParser() conf.readfp(confdata) handler = protonmsg.TimeoutHandler('amqp://broker1.example.com:5672', [], conf) event = MagicMock() handler.on_start(event) event.container.connect.assert_called_once_with( url='amqp://broker1.example.com:5672', reconnect=False, ssl_domain=None) self.assertEqual(SSLDomain.call_count, 0)
def setUp(self): super(FunctionalTestBase, self).setUp() if not os.path.exists(TEST_CFG): raise Exception( "Unable to run the write tests without a test.ini in that defines an access_token with write privs." ) cfg = SafeConfigParser() with open(TEST_CFG) as fp: cfg.readfp(fp, 'test.ini') access_token = cfg.get('write_tests', 'access_token') try: activity_id = cfg.get('activity_tests', 'activity_id') except NoOptionError: activity_id = None self.client = Client(access_token=access_token) self.activity_id = activity_id
def parse_config_file(filename): """Parse INI files containing IMAP connection details. Used by livetest.py and interact.py """ parser = SafeConfigParser(get_string_config_defaults()) with open(filename, 'r') as fh: parser.readfp(fh) conf = _read_config_section(parser, "DEFAULT") if conf.expect_failure: raise ValueError("expect_failure should not be set for the DEFAULT section") conf.alternates = {} for section in parser.sections(): conf.alternates[section] = _read_config_section(parser, section) return conf
def parse(self): """ :return: """ parser = SafeConfigParser() parser.readfp(StringIO(self.obj.content)) for section in parser.sections(): try: content = parser.get(section=section, option="deps") for n, line in enumerate(content.splitlines()): if self.is_marked_line(line): continue if line: req = RequirementsTXTLineParser.parse(line) if req: req.dependency_type = self.obj.file_type self.obj.dependencies.append(req) except NoOptionError: pass
def test_on_start_no_ssl(self, SSLDomain): confdata = six.StringIO("""[broker] urls = amqp://broker1.example.com:5672 amqp://broker2.example.com:5672 topic_prefix = koji connect_timeout = 10 send_timeout = 60 """) if six.PY2: conf = SafeConfigParser() conf.readfp(confdata) else: conf = ConfigParser() conf.read_file(confdata) handler = protonmsg.TimeoutHandler('amqp://broker1.example.com:5672', [], conf) event = MagicMock() handler.on_start(event) event.container.connect.assert_called_once_with(url='amqp://broker1.example.com:5672', reconnect=False, ssl_domain=None) self.assertEqual(SSLDomain.call_count, 0)
def configure(filename=None): """This function gives to the user application a chance to define where configuration file should live. Subsequent calls to this function will have no effect, unless you call :func:`reconfigure`. :param str filename: Full path to configuration file. """ global retry if getattr(configure, '_configured', False): return filename = filename or DEFAULT_CONFIG_FILENAME _ensure_directory(filename) parser = SafeConfigParser() if os.path.isfile(filename): with open(filename, 'rt') as fp: parser.readfp(fp) if not parser.has_section(RETRY_SECTION): parser.add_section(RETRY_SECTION) parser.set(RETRY_SECTION, 'max_tries', str(constants.BACKOFF_DEFAULT_MAXTRIES)) parser.set(RETRY_SECTION, 'delay', str(constants.BACKOFF_DEFAULT_DELAY)) parser.set(RETRY_SECTION, 'factor', str(constants.BACKOFF_DEFAULT_FACTOR)) with open(filename, 'wt') as fp: parser.write(fp) retry = RetrySettings(max_tries=parser.getint(RETRY_SECTION, 'max_tries'), delay=parser.getint(RETRY_SECTION, 'delay'), factor=parser.getint(RETRY_SECTION, 'factor')) setattr(configure, '_configured', True) setattr(configure, '_configured_filename', filename)
def _update_settings_from_file(section, settings): tries = 0 current_directory = os.path.normpath(os.getcwd()) config_file = None while current_directory and tries < MAX_CONFIG_SEARCH_DEPTH: potential_path = os.path.join(current_directory, 'setup.cfg') if os.path.exists(potential_path): config_file = potential_path break new_directory = os.path.split(current_directory)[0] if current_directory == new_directory: break current_directory = new_directory tries += 1 if config_file and os.path.exists(config_file): with open(config_file, 'rU') as fp: config = SafeConfigParser() config.readfp(fp) if config.has_section('tool:multilint'): settings.update(sanitize(config.items('tool:multilint')))
def check_dlm_cfgfile(): """Parse DLM config file""" fname = "/etc/dlm/dlm.conf" try: with open(fname, 'r') as fp: cfgs = '[dlm]\n' + fp.read() except (OSError, IOError): return False cfg = SafeConfigParser() try: cfg.read_string(cfgs) except AttributeError: # Python2 ConfigParser doesn't have cfg.read_string from cStringIO import StringIO cfg.readfp(StringIO(cfgs)) if not cfg.has_option('dlm', 'protocol'): return False proto = cfg.get('dlm', 'protocol').lower() return proto in ['sctp', 'detect', '1', '2']
def parse_config_file(filename): """ Parses a configuration file and returns a settings dictionary. Args: filename (str): File to read configuration settings from. Returns: dict: A dictionary of settings options. """ parser = SafeConfigParser() with open(filename) as fp: parser.readfp(fp) settings = { section: { item[0]: _parse_config_val(item[1]) for item in parser.items(section) } for section in parser.sections() } return settings
def __init__(self, email=None, api_key=None, config_file=None, verbose=False, retry_on_errors=True, site=None, client=None, cert_bundle=None, insecure=None): if client is None: client = _default_client() if config_file is None: config_file = get_default_config_filename() if os.path.exists(config_file): config = SafeConfigParser() with open(config_file, 'r') as f: config.readfp(f, config_file) if api_key is None: api_key = config.get("api", "key") if email is None: email = config.get("api", "email") if site is None and config.has_option("api", "site"): site = config.get("api", "site") if cert_bundle is None and config.has_option("api", "cert_bundle"): cert_bundle = config.get("api", "cert_bundle") if insecure is None and config.has_option("api", "insecure"): # Be quite strict about what is accepted so that users don't # disable security unintentionally. insecure_setting = config.get("api", "insecure").lower() if insecure_setting == "true": insecure = True elif insecure_setting == "false": insecure = False else: raise RuntimeError("insecure is set to '%s', it must be 'true' or 'false' if it is used in %s" % (insecure_setting, config_file)) elif None in (api_key, email): raise RuntimeError("api_key or email not specified and %s does not exist" % (config_file,)) self.api_key = api_key self.email = email self.verbose = verbose if site is not None: if not site.startswith("http"): site = "https://" + site # Remove trailing "/"s from site to simplify the below logic for adding "/api" site = site.rstrip("/") self.base_url = site else: self.base_url = "https://api.zulip.com" if self.base_url != "https://api.zulip.com" and not self.base_url.endswith("/api"): self.base_url += "/api" self.base_url += "/" self.retry_on_errors = retry_on_errors self.client_name = client if insecure: self.tls_verification=False elif cert_bundle is not None: if not os.path.isfile(cert_bundle): raise RuntimeError("tls bundle '%s' does not exist" %(cert_bundle,)) self.tls_verification=cert_bundle else: # Default behavior: verify against system CA certificates self.tls_verification=True
class Csw(object): """ Base CSW server """ def __init__(self, rtconfig=None, env=None, version='3.0.0'): """ Initialize CSW """ if not env: self.environ = os.environ else: self.environ = env self.context = config.StaticContext() # Lazy load this when needed # (it will permanently update global cfg namespaces) self.sruobj = None self.opensearchobj = None self.oaipmhobj = None # init kvp self.kvp = {} self.mode = 'csw' self.asynchronous = False self.soap = False self.request = None self.exception = False self.status = 'OK' self.profiles = None self.manager = False self.outputschemas = {} self.mimetype = 'application/xml; charset=UTF-8' self.encoding = 'UTF-8' self.pretty_print = 0 self.domainquerytype = 'list' self.orm = 'django' self.language = {'639_code': 'en', 'text': 'english'} self.process_time_start = time() # define CSW implementation object (default CSW3) self.iface = csw3.Csw3(server_csw=self) self.request_version = version if self.request_version == '2.0.2': self.iface = csw2.Csw2(server_csw=self) self.context.set_model('csw') # load user configuration try: LOGGER.info('Loading user configuration') if isinstance(rtconfig, SafeConfigParser): # serialized already self.config = rtconfig else: self.config = SafeConfigParser() if isinstance(rtconfig, dict): # dictionary for section, options in rtconfig.items(): self.config.add_section(section) for k, v in options.items(): self.config.set(section, k, v) else: # configuration file import codecs with codecs.open(rtconfig, encoding='utf-8') as scp: self.config.readfp(scp) except Exception as err: msg = 'Could not load configuration' LOGGER.exception('%s %s: %s', msg, rtconfig, err) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', msg) return # set server.home safely # TODO: make this more abstract self.config.set( 'server', 'home', os.path.dirname(os.path.join(os.path.dirname(__file__), '..')) ) self.context.pycsw_home = self.config.get('server', 'home') self.context.url = self.config.get('server', 'url') log.setup_logger(self.config) LOGGER.info('running configuration %s', rtconfig) LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING']) # set OGC schemas location if not self.config.has_option('server', 'ogc_schemas_base'): self.config.set('server', 'ogc_schemas_base', self.context.ogc_schemas_base) # set mimetype if self.config.has_option('server', 'mimetype'): self.mimetype = self.config.get('server', 'mimetype').encode() # set encoding if self.config.has_option('server', 'encoding'): self.encoding = self.config.get('server', 'encoding') # set domainquerytype if self.config.has_option('server', 'domainquerytype'): self.domainquerytype = self.config.get('server', 'domainquerytype') # set XML pretty print if (self.config.has_option('server', 'pretty_print') and self.config.get('server', 'pretty_print') == 'true'): self.pretty_print = 1 # set Spatial Ranking option if (self.config.has_option('server', 'spatial_ranking') and self.config.get('server', 'spatial_ranking') == 'true'): util.ranking_enabled = True # set language default if self.config.has_option('server', 'language'): try: LOGGER.info('Setting language') lang_code = self.config.get('server', 'language').split('-')[0] self.language['639_code'] = lang_code self.language['text'] = self.context.languages[lang_code] except Exception as err: LOGGER.exception('Could not set language: %s', err) pass LOGGER.debug('Configuration: %s.', self.config) LOGGER.debug('Model: %s.', self.context.model) # load user-defined mappings if they exist if self.config.has_option('repository', 'mappings'): # override default repository mappings try: import imp module = self.config.get('repository', 'mappings') if os.sep in module: # filepath modulename = '%s' % os.path.splitext(module)[0].replace( os.sep, '.') mappings = imp.load_source(modulename, module) else: # dotted name mappings = __import__(module, fromlist=['']) LOGGER.info('Loading custom repository mappings ' 'from %s', module) self.context.md_core_model = mappings.MD_CORE_MODEL self.context.refresh_dc(mappings.MD_CORE_MODEL) except Exception as err: LOGGER.exception('Could not load custom mappings: %s', err) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', 'Could not load repository.mappings') # load outputschemas LOGGER.info('Loading outputschemas') for osch in pycsw.plugins.outputschemas.__all__: output_schema_module = __import__( 'pycsw.plugins.outputschemas.%s' % osch) mod = getattr(output_schema_module.plugins.outputschemas, osch) self.outputschemas[mod.NAMESPACE] = mod LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas) LOGGER.debug('Namespaces: %s', self.context.namespaces) def expand_path(self, path): """ return safe path for WSGI environments """ if 'local.app_root' in self.environ and not os.path.isabs(path): return os.path.join(self.environ['local.app_root'], path) else: return path def dispatch_wsgi(self): """ WSGI handler """ if hasattr(self, 'response'): return self._write_response() LOGGER.debug('WSGI mode detected') if self.environ['REQUEST_METHOD'] == 'POST': try: request_body_size = int(self.environ.get('CONTENT_LENGTH', 0)) except (ValueError): request_body_size = 0 self.requesttype = 'POST' self.request = self.environ['wsgi.input'].read(request_body_size) LOGGER.debug('Request type: POST. Request:\n%s\n', self.request) else: # it's a GET request self.requesttype = 'GET' self.request = wsgiref.util.request_uri(self.environ) try: query_part = splitquery(self.request)[-1] self.kvp = dict(parse_qsl(query_part, keep_blank_values=True)) except AttributeError as err: LOGGER.exception('Could not parse query string') self.kvp = {} LOGGER.debug('Request type: GET. Request:\n%s\n', self.request) return self.dispatch() def opensearch(self): """ enable OpenSearch """ if not self.opensearchobj: self.opensearchobj = opensearch.OpenSearch(self.context) return self.opensearchobj def sru(self): """ enable SRU """ if not self.sruobj: self.sruobj = sru.Sru(self.context) return self.sruobj def oaipmh(self): """ enable OAI-PMH """ if not self.oaipmhobj: self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config) return self.oaipmhobj def dispatch(self, writer=sys.stdout, write_headers=True): """ Handle incoming HTTP request """ error = 0 if self.requesttype == 'GET': self.kvp = self.normalize_kvp(self.kvp) version_202 = ('version' in self.kvp and self.kvp['version'] == '2.0.2') accept_version_202 = ('acceptversions' in self.kvp and '2.0.2' in self.kvp['acceptversions']) if version_202 or accept_version_202: self.request_version = '2.0.2' elif self.requesttype == 'POST': if self.request.find(b'cat/csw/2.0.2') != -1: self.request_version = '2.0.2' elif self.request.find(b'cat/csw/3.0') != -1: self.request_version = '3.0.0' if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'sru'): self.mode = 'sru' self.request_version = '2.0.2' LOGGER.info('SRU mode detected; processing request') self.kvp = self.sru().request_sru2csw(self.kvp) if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'oaipmh'): self.mode = 'oaipmh' self.request_version = '2.0.2' LOGGER.info('OAI-PMH mode detected; processing request.') self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k) self.kvp = self.oaipmh().request(self.kvp) if self.request_version == '2.0.2': self.iface = csw2.Csw2(server_csw=self) self.context.set_model('csw') # configure transaction support, if specified in config self._gen_manager() namespaces = self.context.namespaces ops = self.context.model['operations'] constraints = self.context.model['constraints'] # generate domain model # NOTE: We should probably avoid this sort of mutable state for WSGI if 'GetDomain' not in ops: ops['GetDomain'] = self.context.gen_domains() # generate distributed search model, if specified in config if self.config.has_option('server', 'federatedcatalogues'): LOGGER.info('Configuring distributed search') constraints['FederatedCatalogues'] = {'values': []} for fedcat in self.config.get('server', 'federatedcatalogues').split(','): LOGGER.debug('federated catalogue: %s', fedcat) constraints['FederatedCatalogues']['values'].append(fedcat) for key, value in self.outputschemas.items(): get_records_params = ops['GetRecords']['parameters'] get_records_params['outputSchema']['values'].append( value.NAMESPACE) get_records_by_id_params = ops['GetRecordById']['parameters'] get_records_by_id_params['outputSchema']['values'].append( value.NAMESPACE) if 'Harvest' in ops: harvest_params = ops['Harvest']['parameters'] harvest_params['ResourceType']['values'].append( value.NAMESPACE) LOGGER.info('Setting MaxRecordDefault') if self.config.has_option('server', 'maxrecords'): constraints['MaxRecordDefault']['values'] = [ self.config.get('server', 'maxrecords')] # load profiles if self.config.has_option('server', 'profiles'): self.profiles = pprofile.load_profiles( os.path.join('pycsw', 'plugins', 'profiles'), pprofile.Profile, self.config.get('server', 'profiles') ) for prof in self.profiles['plugins'].keys(): tmp = self.profiles['plugins'][prof](self.context.model, namespaces, self.context) key = tmp.outputschema # to ref by outputschema self.profiles['loaded'][key] = tmp self.profiles['loaded'][key].extend_core(self.context.model, namespaces, self.config) LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys())) # init repository # look for tablename, set 'records' as default if not self.config.has_option('repository', 'table'): self.config.set('repository', 'table', 'records') repo_filter = None if self.config.has_option('repository', 'filter'): repo_filter = self.config.get('repository', 'filter') if self.config.has_option('repository', 'source'): # load custom repository rs = self.config.get('repository', 'source') rs_modname, rs_clsname = rs.rsplit('.', 1) rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname]) rs_cls = getattr(rs_mod, rs_clsname) try: self.repository = rs_cls(self.context, repo_filter) LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype) except Exception as err: msg = 'Could not load custom repository %s: %s' % (rs, err) LOGGER.exception(msg) error = 1 code = 'NoApplicableCode' locator = 'service' text = 'Could not initialize repository. Check server logs' else: # load default repository self.orm = 'sqlalchemy' from pycsw.core import repository try: LOGGER.info('Loading default repository') self.repository = repository.Repository( self.config.get('repository', 'database'), self.context, self.environ.get('local.app_root', None), self.config.get('repository', 'table'), repo_filter ) LOGGER.debug( 'Repository loaded (local): %s.' % self.repository.dbtype) except Exception as err: msg = 'Could not load repository (local): %s' % err LOGGER.exception(msg) error = 1 code = 'NoApplicableCode' locator = 'service' text = 'Could not initialize repository. Check server logs' if self.requesttype == 'POST': LOGGER.debug('HTTP POST request') LOGGER.debug('CSW version: %s', self.iface.version) self.kvp = self.iface.parse_postdata(self.request) if isinstance(self.kvp, str): # it's an exception error = 1 locator = 'service' text = self.kvp if (self.kvp.find('the document is not valid') != -1 or self.kvp.find('document not well-formed') != -1): code = 'NoApplicableCode' else: code = 'InvalidParameterValue' LOGGER.debug('HTTP Headers:\n%s.', self.environ) LOGGER.debug('Parsed request parameters: %s', self.kvp) if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'opensearch'): self.mode = 'opensearch' LOGGER.info('OpenSearch mode detected; processing request.') self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom' if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or (len(self.kvp) == 1 and 'config' in self.kvp)): LOGGER.info('Turning on default csw30:Capabilities for base URL') self.kvp = { 'service': 'CSW', 'acceptversions': '3.0.0', 'request': 'GetCapabilities' } http_accept = self.environ.get('HTTP_ACCEPT', '') if 'application/opensearchdescription+xml' in http_accept: self.mode = 'opensearch' self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom' if error == 0: # test for the basic keyword values (service, version, request) basic_options = ['service', 'request'] request = self.kvp.get('request', '') own_version_integer = util.get_version_integer( self.request_version) if self.request_version == '2.0.2': basic_options.append('version') if self.request_version == '3.0.0' and 'version' not in self.kvp and self.requesttype == 'POST': if 'service' not in self.kvp: self.kvp['service'] = 'CSW' basic_options.append('service') self.kvp['version'] = self.request_version basic_options.append('version') for k in basic_options: if k not in self.kvp: if (k in ['version', 'acceptversions'] and request == 'GetCapabilities'): pass else: error = 1 locator = k code = 'MissingParameterValue' text = 'Missing keyword: %s' % k break # test each of the basic keyword values if error == 0: # test service if self.kvp['service'] != 'CSW': error = 1 locator = 'service' code = 'InvalidParameterValue' text = 'Invalid value for service: %s.\ Value MUST be CSW' % self.kvp['service'] # test version kvp_version = self.kvp.get('version', '') try: kvp_version_integer = util.get_version_integer(kvp_version) except Exception as err: kvp_version_integer = 'invalid_value' if (request != 'GetCapabilities' and kvp_version_integer != own_version_integer): error = 1 locator = 'version' code = 'InvalidParameterValue' text = ('Invalid value for version: %s. Value MUST be ' '2.0.2 or 3.0.0' % kvp_version) # check for GetCapabilities acceptversions if 'acceptversions' in self.kvp: for vers in self.kvp['acceptversions'].split(','): vers_integer = util.get_version_integer(vers) if vers_integer == own_version_integer: break else: error = 1 locator = 'acceptversions' code = 'VersionNegotiationFailed' text = ('Invalid parameter value in ' 'acceptversions: %s. Value MUST be ' '2.0.2 or 3.0.0' % self.kvp['acceptversions']) # test request if self.kvp['request'] not in \ self.context.model['operations']: error = 1 locator = 'request' if request in ['Transaction', 'Harvest']: code = 'OperationNotSupported' text = '%s operations are not supported' % request else: code = 'InvalidParameterValue' text = 'Invalid value for request: %s' % request if error == 1: # return an ExceptionReport LOGGER.error('basic service options error: %s, %s, %s', code, locator, text) self.response = self.iface.exceptionreport(code, locator, text) else: # process per the request value if 'responsehandler' in self.kvp: # set flag to process asynchronously import threading self.asynchronous = True request_id = self.kvp.get('requestid', None) if request_id is None: import uuid self.kvp['requestid'] = str(uuid.uuid4()) if self.kvp['request'] == 'GetCapabilities': self.response = self.iface.getcapabilities() elif self.kvp['request'] == 'DescribeRecord': self.response = self.iface.describerecord() elif self.kvp['request'] == 'GetDomain': self.response = self.iface.getdomain() elif self.kvp['request'] == 'GetRecords': if self.asynchronous: # process asynchronously threading.Thread(target=self.iface.getrecords).start() self.response = self.iface._write_acknowledgement() else: self.response = self.iface.getrecords() elif self.kvp['request'] == 'GetRecordById': self.response = self.iface.getrecordbyid() elif self.kvp['request'] == 'GetRepositoryItem': self.response = self.iface.getrepositoryitem() elif self.kvp['request'] == 'Transaction': self.response = self.iface.transaction() elif self.kvp['request'] == 'Harvest': if self.asynchronous: # process asynchronously threading.Thread(target=self.iface.harvest).start() self.response = self.iface._write_acknowledgement() else: self.response = self.iface.harvest() else: self.response = self.iface.exceptionreport( 'InvalidParameterValue', 'request', 'Invalid request parameter: %s' % self.kvp['request'] ) LOGGER.info('Request processed') if self.mode == 'sru': LOGGER.info('SRU mode detected; processing response.') self.response = self.sru().response_csw2sru(self.response, self.environ) elif self.mode == 'opensearch': LOGGER.info('OpenSearch mode detected; processing response.') self.response = self.opensearch().response_csw2opensearch( self.response, self.config) elif self.mode == 'oaipmh': LOGGER.info('OAI-PMH mode detected; processing response.') self.response = self.oaipmh().response( self.response, self.oaiargs, self.repository, self.config.get('server', 'url') ) return self._write_response() def getcapabilities(self): """ Handle GetCapabilities request """ return self.iface.getcapabilities() def describerecord(self): """ Handle DescribeRecord request """ return self.iface.describerecord() def getdomain(self): """ Handle GetDomain request """ return self.iface.getdomain() def getrecords(self): """ Handle GetRecords request """ return self.iface.getrecords() def getrecordbyid(self, raw=False): """ Handle GetRecordById request """ return self.iface.getrecordbyid(raw) def getrepositoryitem(self): """ Handle GetRepositoryItem request """ return self.iface.getrepositoryitem() def transaction(self): """ Handle Transaction request """ return self.iface.transaction() def harvest(self): """ Handle Harvest request """ return self.iface.harvest() def _write_response(self): """ Generate response """ # set HTTP response headers and XML declaration xmldecl = '' appinfo = '' LOGGER.info('Writing response.') if hasattr(self, 'soap') and self.soap: self._gen_soap_wrapper() if etree.__version__ >= '3.5.0': # remove superfluous namespaces etree.cleanup_namespaces(self.response, keep_ns_prefixes=self.context.keep_ns_prefixes) response = etree.tostring(self.response, pretty_print=self.pretty_print, encoding='unicode') if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and self.kvp['outputformat'] == 'application/json'): self.contenttype = self.kvp['outputformat'] from pycsw.core.formats import fmt_json response = fmt_json.xml2json(response, self.context.namespaces, self.pretty_print) else: # it's XML if 'outputformat' in self.kvp: self.contenttype = self.kvp['outputformat'] else: self.contenttype = self.mimetype xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>' '\n' % self.encoding) appinfo = '<!-- pycsw %s -->\n' % self.context.version if isinstance(self.contenttype, bytes): self.contenttype = self.contenttype.decode() s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding) LOGGER.debug('Response code: %s', self.context.response_codes[self.status]) LOGGER.debug('Response:\n%s', s) return [self.context.response_codes[self.status], s] def _gen_soap_wrapper(self): """ Generate SOAP wrapper """ LOGGER.info('Writing SOAP wrapper.') node = etree.Element( util.nspath_eval('soapenv:Envelope', self.context.namespaces), nsmap=self.context.namespaces ) schema_location_ns = util.nspath_eval('xsi:schemaLocation', self.context.namespaces) node.attrib[schema_location_ns] = '%s %s' % ( self.context.namespaces['soapenv'], self.context.namespaces['soapenv'] ) node2 = etree.SubElement( node, util.nspath_eval('soapenv:Body', self.context.namespaces)) if self.exception: node3 = etree.SubElement( node2, util.nspath_eval('soapenv:Fault', self.context.namespaces) ) node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Code', self.context.namespaces) ) etree.SubElement( node4, util.nspath_eval('soapenv:Value', self.context.namespaces) ).text = 'soap:Server' node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Reason', self.context.namespaces) ) etree.SubElement( node4, util.nspath_eval('soapenv:Text', self.context.namespaces) ).text = 'A server exception was encountered.' node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Detail', self.context.namespaces) ) node4.append(self.response) else: node2.append(self.response) self.response = node def _gen_manager(self): """ Update self.context.model with CSW-T advertising """ if (self.config.has_option('manager', 'transactions') and self.config.get('manager', 'transactions') == 'true'): self.manager = True self.context.model['operations_order'].append('Transaction') self.context.model['operations']['Transaction'] = { 'methods': {'get': False, 'post': True}, 'parameters': {} } schema_values = [ 'http://www.opengis.net/cat/csw/2.0.2', 'http://www.opengis.net/cat/csw/3.0', 'http://www.opengis.net/wms', 'http://www.opengis.net/wmts/1.0', 'http://www.opengis.net/wfs', 'http://www.opengis.net/wfs/2.0', 'http://www.opengis.net/wcs', 'http://www.opengis.net/wps/1.0.0', 'http://www.opengis.net/sos/1.0', 'http://www.opengis.net/sos/2.0', 'http://www.isotc211.org/2005/gmi', 'urn:geoss:waf', ] self.context.model['operations_order'].append('Harvest') self.context.model['operations']['Harvest'] = { 'methods': {'get': False, 'post': True}, 'parameters': { 'ResourceType': {'values': schema_values} } } self.context.model['operations']['Transaction'] = { 'methods': {'get': False, 'post': True}, 'parameters': { 'TransactionSchemas': {'values': sorted(schema_values)} } } self.csw_harvest_pagesize = 10 if self.config.has_option('manager', 'csw_harvest_pagesize'): self.csw_harvest_pagesize = int( self.config.get('manager', 'csw_harvest_pagesize')) def _test_manager(self): """ Verify that transactions are allowed """ if self.config.get('manager', 'transactions') != 'true': raise RuntimeError('CSW-T interface is disabled') """ get the client first forwarded ip """ if 'HTTP_X_FORWARDED_FOR' in self.environ: ipaddress = self.environ['HTTP_X_FORWARDED_FOR'].split(',')[0].strip() else: ipaddress = self.environ['REMOTE_ADDR'] if not self.config.has_option('manager', 'allowed_ips') or \ (self.config.has_option('manager', 'allowed_ips') and not util.ipaddress_in_whitelist(ipaddress, self.config.get('manager', 'allowed_ips').split(','))): raise RuntimeError( 'CSW-T operations not allowed for this IP address: %s' % ipaddress) def _cql_update_queryables_mappings(self, cql, mappings): """ Transform CQL query's properties to underlying DB columns """ LOGGER.debug('Raw CQL text = %s', cql) LOGGER.debug(str(list(mappings.keys()))) if cql is not None: for key in mappings.keys(): try: cql = cql.replace(key, mappings[key]['dbcol']) except: cql = cql.replace(key, mappings[key]) LOGGER.debug('Interpolated CQL text = %s.', cql) return cql def _process_responsehandler(self, xml): """ Process response handler """ if self.kvp['responsehandler'] is not None: LOGGER.info('Processing responsehandler %s' % self.kvp['responsehandler']) uprh = urlparse(self.kvp['responsehandler']) if uprh.scheme == 'mailto': # email import smtplib LOGGER.debug('Email detected') smtp_host = 'localhost' if self.config.has_option('server', 'smtp_host'): smtp_host = self.config.get('server', 'smtp_host') body = ('Subject: pycsw %s results\n\n%s' % (self.kvp['request'], xml)) try: LOGGER.info('Sending email') msg = smtplib.SMTP(smtp_host) msg.sendmail( self.config.get('metadata:main', 'contact_email'), uprh.path, body ) msg.quit() LOGGER.debug('Email sent successfully.') except Exception as err: LOGGER.exception('Error processing email') elif uprh.scheme == 'ftp': import ftplib LOGGER.debug('FTP detected.') try: LOGGER.info('Sending to FTP server.') ftp = ftplib.FTP(uprh.hostname) if uprh.username is not None: ftp.login(uprh.username, uprh.password) ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml)) ftp.quit() LOGGER.debug('FTP sent successfully.') except Exception as err: LOGGER.exception('Error processing FTP') @staticmethod def normalize_kvp(kvp): """Normalize Key Value Pairs. This method will transform all keys to lowercase and leave values unchanged, as specified in the CSW standard (see for example note C on Table 62 - KVP Encoding for DescribeRecord operation request of the CSW standard version 2.0.2) :arg kvp: a mapping with Key Value Pairs :type kvp: dict :returns: A new dictionary with normalized parameters """ result = dict() for name, value in kvp.items(): result[name.lower()] = value return result
def __init__(self, email=None, api_key=None, config_file=None, verbose=False, retry_on_errors=True, site=None, client=None, cert_bundle=None, insecure=None, client_cert=None, client_cert_key=None): # type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[str], Optional[str]) -> None if client is None: client = _default_client() # Normalize user-specified path if config_file is not None: config_file = os.path.abspath(os.path.expanduser(config_file)) # Fill values from Environment Variables if not available in Constructor if config_file is None: config_file = os.environ.get("ZULIP_CONFIG") if api_key is None: api_key = os.environ.get("ZULIP_API_KEY") if email is None: email = os.environ.get("ZULIP_EMAIL") if site is None: site = os.environ.get("ZULIP_SITE") if client_cert is None: client_cert = os.environ.get("ZULIP_CERT") if client_cert_key is None: client_cert_key = os.environ.get("ZULIP_CERT_KEY") if cert_bundle is None: cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE") if insecure is None: # Be quite strict about what is accepted so that users don't # disable security unintentionally. insecure_setting = os.environ.get('ZULIP_ALLOW_INSECURE') if insecure_setting is not None: insecure = validate_boolean_field(insecure_setting) if insecure is None: raise ZulipError( "The ZULIP_ALLOW_INSECURE environment " "variable is set to '{}', it must be " "'true' or 'false'".format(insecure_setting)) if config_file is None: config_file = get_default_config_filename() if config_file is not None and os.path.exists(config_file): config = SafeConfigParser() with open(config_file, 'r') as f: config.readfp(f, config_file) if api_key is None: api_key = config.get("api", "key") if email is None: email = config.get("api", "email") if site is None and config.has_option("api", "site"): site = config.get("api", "site") if client_cert is None and config.has_option("api", "client_cert"): client_cert = config.get("api", "client_cert") if client_cert_key is None and config.has_option( "api", "client_cert_key"): client_cert_key = config.get("api", "client_cert_key") if cert_bundle is None and config.has_option("api", "cert_bundle"): cert_bundle = config.get("api", "cert_bundle") if insecure is None and config.has_option("api", "insecure"): # Be quite strict about what is accepted so that users don't # disable security unintentionally. insecure_setting = config.get('api', 'insecure') insecure = validate_boolean_field(insecure_setting) if insecure is None: raise ZulipError( "insecure is set to '{}', it must be " "'true' or 'false' if it is used in {}".format( insecure_setting, config_file)) elif None in (api_key, email): raise ConfigNotFoundError( "api_key or email not specified and file %s does not exist" % (config_file, )) assert (api_key is not None and email is not None) self.api_key = api_key self.email = email self.verbose = verbose if site is not None: if site.startswith("localhost"): site = "http://" + site elif not site.startswith("http"): site = "https://" + site # Remove trailing "/"s from site to simplify the below logic for adding "/api" site = site.rstrip("/") self.base_url = site else: raise MissingURLError( "Missing Zulip server URL; specify via --site or ~/.zuliprc.") if not self.base_url.endswith("/api"): self.base_url += "/api" self.base_url += "/" self.retry_on_errors = retry_on_errors self.client_name = client if insecure: logger.warning('Insecure mode enabled. The server\'s SSL/TLS ' 'certificate will not be validated, making the ' 'HTTPS connection potentially insecure') self.tls_verification = False # type: Union[bool, str] elif cert_bundle is not None: if not os.path.isfile(cert_bundle): raise ConfigNotFoundError("tls bundle '%s' does not exist" % (cert_bundle, )) self.tls_verification = cert_bundle else: # Default behavior: verify against system CA certificates self.tls_verification = True if client_cert is None: if client_cert_key is not None: raise ConfigNotFoundError( "client cert key '%s' specified, but no client cert public part provided" % (client_cert_key, )) else: # we have a client cert if not os.path.isfile(client_cert): raise ConfigNotFoundError("client cert '%s' does not exist" % (client_cert, )) if client_cert_key is not None: if not os.path.isfile(client_cert_key): raise ConfigNotFoundError( "client cert key '%s' does not exist" % (client_cert_key, )) self.client_cert = client_cert self.client_cert_key = client_cert_key self.session = None # type: Optional[requests.Session] self.has_connected = False
class Csw(object): """ Base CSW server """ def __init__(self, rtconfig=None, env=None, version='3.0.0'): """ Initialize CSW """ if not env: self.environ = os.environ else: self.environ = env self.context = config.StaticContext() # Lazy load this when needed # (it will permanently update global cfg namespaces) self.sruobj = None self.opensearchobj = None self.oaipmhobj = None # init kvp self.kvp = {} self.mode = 'csw' self.async = False self.soap = False self.request = None self.exception = False self.status = 'OK' self.profiles = None self.manager = False self.outputschemas = {} self.mimetype = 'application/xml; charset=UTF-8' self.encoding = 'UTF-8' self.pretty_print = 0 self.domainquerytype = 'list' self.orm = 'django' self.language = {'639_code': 'en', 'text': 'english'} self.process_time_start = time() # define CSW implementation object (default CSW3) self.iface = csw3.Csw3(server_csw=self) self.request_version = version if self.request_version == '2.0.2': self.iface = csw2.Csw2(server_csw=self) self.context.set_model('csw') # load user configuration try: LOGGER.info('Loading user configuration') if isinstance(rtconfig, SafeConfigParser): # serialized already self.config = rtconfig else: self.config = SafeConfigParser() if isinstance(rtconfig, dict): # dictionary for section, options in rtconfig.items(): self.config.add_section(section) for k, v in options.items(): self.config.set(section, k, v) else: # configuration file import codecs with codecs.open(rtconfig, encoding='utf-8') as scp: self.config.readfp(scp) except Exception as err: LOGGER.exception('Could not load user configuration: %s', err) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', 'Error opening configuration %s' % rtconfig ) return # set server.home safely # TODO: make this more abstract self.config.set( 'server', 'home', os.path.dirname(os.path.join(os.path.dirname(__file__), '..')) ) self.context.pycsw_home = self.config.get('server', 'home') self.context.url = self.config.get('server', 'url') log.setup_logger(self.config) LOGGER.info('running configuration %s', rtconfig) LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING']) # set OGC schemas location if not self.config.has_option('server', 'ogc_schemas_base'): self.config.set('server', 'ogc_schemas_base', self.context.ogc_schemas_base) # set mimetype if self.config.has_option('server', 'mimetype'): self.mimetype = self.config.get('server', 'mimetype').encode() # set encoding if self.config.has_option('server', 'encoding'): self.encoding = self.config.get('server', 'encoding') # set domainquerytype if self.config.has_option('server', 'domainquerytype'): self.domainquerytype = self.config.get('server', 'domainquerytype') # set XML pretty print if (self.config.has_option('server', 'pretty_print') and self.config.get('server', 'pretty_print') == 'true'): self.pretty_print = 1 # set Spatial Ranking option if (self.config.has_option('server', 'spatial_ranking') and self.config.get('server', 'spatial_ranking') == 'true'): util.ranking_enabled = True # set language default if self.config.has_option('server', 'language'): try: LOGGER.info('Setting language') lang_code = self.config.get('server', 'language').split('-')[0] self.language['639_code'] = lang_code self.language['text'] = self.context.languages[lang_code] except Exception as err: LOGGER.exception('Could not set language: %s', err) pass LOGGER.debug('Configuration: %s.', self.config) LOGGER.debug('Model: %s.', self.context.model) # load user-defined mappings if they exist if self.config.has_option('repository', 'mappings'): # override default repository mappings try: import imp module = self.config.get('repository', 'mappings') if '/' in module: # filepath modulename = '%s' % os.path.splitext(module)[0].replace( os.sep, '.') mappings = imp.load_source(modulename, module) else: # dotted name mappings = __import__(module, fromlist=['']) LOGGER.info('Loading custom repository mappings ' 'from %s', module) self.context.md_core_model = mappings.MD_CORE_MODEL self.context.refresh_dc(mappings.MD_CORE_MODEL) except Exception as err: LOGGER.exception('Could not load custom mappings: %s', err) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', 'Could not load repository.mappings') # load outputschemas LOGGER.info('Loading outputschemas') for osch in pycsw.plugins.outputschemas.__all__: output_schema_module = __import__( 'pycsw.plugins.outputschemas.%s' % osch) mod = getattr(output_schema_module.plugins.outputschemas, osch) self.outputschemas[mod.NAMESPACE] = mod LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas) LOGGER.debug('Namespaces: %s', self.context.namespaces) def expand_path(self, path): """ return safe path for WSGI environments """ if 'local.app_root' in self.environ and not os.path.isabs(path): return os.path.join(self.environ['local.app_root'], path) else: return path def dispatch_wsgi(self): """ WSGI handler """ if hasattr(self, 'response'): return self._write_response() LOGGER.debug('WSGI mode detected') if self.environ['REQUEST_METHOD'] == 'POST': try: request_body_size = int(self.environ.get('CONTENT_LENGTH', 0)) except (ValueError): request_body_size = 0 self.requesttype = 'POST' self.request = self.environ['wsgi.input'].read(request_body_size) LOGGER.debug('Request type: POST. Request:\n%s\n', self.request) else: # it's a GET request self.requesttype = 'GET' self.request = wsgiref.util.request_uri(self.environ) try: query_part = splitquery(self.request)[-1] self.kvp = dict(parse_qsl(query_part, keep_blank_values=True)) except AttributeError as err: LOGGER.exception('Could not parse query string') self.kvp = {} LOGGER.debug('Request type: GET. Request:\n%s\n', self.request) return self.dispatch() def opensearch(self): """ enable OpenSearch """ if not self.opensearchobj: self.opensearchobj = opensearch.OpenSearch(self.context) return self.opensearchobj def sru(self): """ enable SRU """ if not self.sruobj: self.sruobj = sru.Sru(self.context) return self.sruobj def oaipmh(self): """ enable OAI-PMH """ if not self.oaipmhobj: self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config) return self.oaipmhobj def dispatch(self, writer=sys.stdout, write_headers=True): """ Handle incoming HTTP request """ if self.requesttype == 'GET': self.kvp = self.normalize_kvp(self.kvp) version_202 = ('version' in self.kvp and self.kvp['version'] == '2.0.2') accept_version_202 = ('acceptversions' in self.kvp and '2.0.2' in self.kvp['acceptversions']) if version_202 or accept_version_202: self.request_version = '2.0.2' elif self.requesttype == 'POST': if self.request.find(b'2.0.2') != -1: self.request_version = '2.0.2' if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'sru'): self.mode = 'sru' self.request_version = '2.0.2' LOGGER.info('SRU mode detected; processing request') self.kvp = self.sru().request_sru2csw(self.kvp) if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'oaipmh'): self.mode = 'oaipmh' self.request_version = '2.0.2' LOGGER.info('OAI-PMH mode detected; processing request.') self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k) self.kvp = self.oaipmh().request(self.kvp) if self.request_version == '2.0.2': self.iface = csw2.Csw2(server_csw=self) self.context.set_model('csw') # configure transaction support, if specified in config self._gen_manager() namespaces = self.context.namespaces ops = self.context.model['operations'] constraints = self.context.model['constraints'] # generate domain model # NOTE: We should probably avoid this sort of mutable state for WSGI if 'GetDomain' not in ops: ops['GetDomain'] = self.context.gen_domains() # generate distributed search model, if specified in config if self.config.has_option('server', 'federatedcatalogues'): LOGGER.info('Configuring distributed search') constraints['FederatedCatalogues'] = {'values': []} for fedcat in self.config.get('server', 'federatedcatalogues').split(','): LOGGER.debug('federated catalogue: %s', fedcat) constraints['FederatedCatalogues']['values'].append(fedcat) for key, value in self.outputschemas.items(): get_records_params = ops['GetRecords']['parameters'] get_records_params['outputSchema']['values'].append( value.NAMESPACE) get_records_by_id_params = ops['GetRecordById']['parameters'] get_records_by_id_params['outputSchema']['values'].append( value.NAMESPACE) if 'Harvest' in ops: harvest_params = ops['Harvest']['parameters'] harvest_params['ResourceType']['values'].append( value.NAMESPACE) LOGGER.info('Setting MaxRecordDefault') if self.config.has_option('server', 'maxrecords'): constraints['MaxRecordDefault']['values'] = [ self.config.get('server', 'maxrecords')] # load profiles if self.config.has_option('server', 'profiles'): self.profiles = pprofile.load_profiles( os.path.join('pycsw', 'plugins', 'profiles'), pprofile.Profile, self.config.get('server', 'profiles') ) for prof in self.profiles['plugins'].keys(): tmp = self.profiles['plugins'][prof](self.context.model, namespaces, self.context) key = tmp.outputschema # to ref by outputschema self.profiles['loaded'][key] = tmp self.profiles['loaded'][key].extend_core(self.context.model, namespaces, self.config) LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys())) # init repository # look for tablename, set 'records' as default if not self.config.has_option('repository', 'table'): self.config.set('repository', 'table', 'records') repo_filter = None if self.config.has_option('repository', 'filter'): repo_filter = self.config.get('repository', 'filter') if self.config.has_option('repository', 'source'): # load custom repository rs = self.config.get('repository', 'source') rs_modname, rs_clsname = rs.rsplit('.', 1) rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname]) rs_cls = getattr(rs_mod, rs_clsname) try: self.repository = rs_cls(self.context, repo_filter) LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype) except Exception as err: msg = 'Could not load custom repository' LOGGER.exception(msg) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', msg) else: # load default repository self.orm = 'sqlalchemy' from pycsw.core import repository try: LOGGER.info('Loading default repository') self.repository = repository.Repository( self.config.get('repository', 'database'), self.context, self.environ.get('local.app_root', None), self.config.get('repository', 'table'), repo_filter ) LOGGER.debug( 'Repository loaded (local): %s.' % self.repository.dbtype) except Exception as err: msg = 'Could not load repository (local)' LOGGER.exception(msg) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', msg) if self.requesttype == 'POST': LOGGER.debug('HTTP POST request') LOGGER.debug('CSW version: %s', self.iface.version) self.kvp = self.iface.parse_postdata(self.request) error = 0 if isinstance(self.kvp, str): # it's an exception error = 1 locator = 'service' text = self.kvp if (self.kvp.find('the document is not valid') != -1 or self.kvp.find('document not well-formed') != -1): code = 'NoApplicableCode' else: code = 'InvalidParameterValue' LOGGER.debug('HTTP Headers:\n%s.', self.environ) LOGGER.debug('Parsed request parameters: %s', self.kvp) if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'opensearch'): self.mode = 'opensearch' LOGGER.info('OpenSearch mode detected; processing request.') self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom' if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or (len(self.kvp) == 1 and 'config' in self.kvp)): LOGGER.info('Turning on default csw30:Capabilities for base URL') self.kvp = { 'service': 'CSW', 'acceptversions': '3.0.0', 'request': 'GetCapabilities' } http_accept = self.environ.get('HTTP_ACCEPT', '') if 'application/opensearchdescription+xml' in http_accept: self.mode = 'opensearch' self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom' if error == 0: # test for the basic keyword values (service, version, request) basic_options = ['service', 'request'] request = self.kvp.get('request', '') own_version_integer = util.get_version_integer( self.request_version) if self.request_version == '2.0.2': basic_options.append('version') for k in basic_options: if k not in self.kvp: if (k in ['version', 'acceptversions'] and request == 'GetCapabilities'): pass else: error = 1 locator = k code = 'MissingParameterValue' text = 'Missing keyword: %s' % k break # test each of the basic keyword values if error == 0: # test service if self.kvp['service'] != 'CSW': error = 1 locator = 'service' code = 'InvalidParameterValue' text = 'Invalid value for service: %s.\ Value MUST be CSW' % self.kvp['service'] # test version kvp_version = self.kvp.get('version', '') try: kvp_version_integer = util.get_version_integer(kvp_version) except Exception as err: kvp_version_integer = 'invalid_value' if (request != 'GetCapabilities' and kvp_version_integer != own_version_integer): error = 1 locator = 'version' code = 'InvalidParameterValue' text = ('Invalid value for version: %s. Value MUST be ' '2.0.2 or 3.0.0' % kvp_version) # check for GetCapabilities acceptversions if 'acceptversions' in self.kvp: for vers in self.kvp['acceptversions'].split(','): vers_integer = util.get_version_integer(vers) if vers_integer == own_version_integer: break else: error = 1 locator = 'acceptversions' code = 'VersionNegotiationFailed' text = ('Invalid parameter value in ' 'acceptversions: %s. Value MUST be ' '2.0.2 or 3.0.0' % self.kvp['acceptversions']) # test request if self.kvp['request'] not in \ self.context.model['operations']: error = 1 locator = 'request' if request in ['Transaction', 'Harvest']: code = 'OperationNotSupported' text = '%s operations are not supported' % request else: code = 'InvalidParameterValue' text = 'Invalid value for request: %s' % request if error == 1: # return an ExceptionReport LOGGER.error('basic service options error: %s, %s, %s', code, locator, text) self.response = self.iface.exceptionreport(code, locator, text) else: # process per the request value if 'responsehandler' in self.kvp: # set flag to process asynchronously import threading self.async = True request_id = self.kvp.get('requestid', None) if request_id is None: import uuid self.kvp['requestid'] = str(uuid.uuid4()) if self.kvp['request'] == 'GetCapabilities': self.response = self.iface.getcapabilities() elif self.kvp['request'] == 'DescribeRecord': self.response = self.iface.describerecord() elif self.kvp['request'] == 'GetDomain': self.response = self.iface.getdomain() elif self.kvp['request'] == 'GetRecords': if self.async: # process asynchronously threading.Thread(target=self.iface.getrecords).start() self.response = self.iface._write_acknowledgement() else: self.response = self.iface.getrecords() elif self.kvp['request'] == 'GetRecordById': self.response = self.iface.getrecordbyid() elif self.kvp['request'] == 'GetRepositoryItem': self.response = self.iface.getrepositoryitem() elif self.kvp['request'] == 'Transaction': self.response = self.iface.transaction() elif self.kvp['request'] == 'Harvest': if self.async: # process asynchronously threading.Thread(target=self.iface.harvest).start() self.response = self.iface._write_acknowledgement() else: self.response = self.iface.harvest() else: self.response = self.iface.exceptionreport( 'InvalidParameterValue', 'request', 'Invalid request parameter: %s' % self.kvp['request'] ) LOGGER.info('Request processed') if self.mode == 'sru': LOGGER.info('SRU mode detected; processing response.') self.response = self.sru().response_csw2sru(self.response, self.environ) elif self.mode == 'opensearch': LOGGER.info('OpenSearch mode detected; processing response.') self.response = self.opensearch().response_csw2opensearch( self.response, self.config) elif self.mode == 'oaipmh': LOGGER.info('OAI-PMH mode detected; processing response.') self.response = self.oaipmh().response( self.response, self.oaiargs, self.repository, self.config.get('server', 'url') ) return self._write_response() def getcapabilities(self): """ Handle GetCapabilities request """ return self.iface.getcapabilities() def describerecord(self): """ Handle DescribeRecord request """ return self.iface.describerecord() def getdomain(self): """ Handle GetDomain request """ return self.iface.getdomain() def getrecords(self): """ Handle GetRecords request """ return self.iface.getrecords() def getrecordbyid(self, raw=False): """ Handle GetRecordById request """ return self.iface.getrecordbyid(raw) def getrepositoryitem(self): """ Handle GetRepositoryItem request """ return self.iface.getrepositoryitem() def transaction(self): """ Handle Transaction request """ return self.iface.transaction() def harvest(self): """ Handle Harvest request """ return self.iface.harvest() def _write_response(self): """ Generate response """ # set HTTP response headers and XML declaration xmldecl = '' appinfo = '' LOGGER.info('Writing response.') if hasattr(self, 'soap') and self.soap: self._gen_soap_wrapper() if etree.__version__ >= '3.5.0': # remove superfluous namespaces etree.cleanup_namespaces(self.response, keep_ns_prefixes=self.context.keep_ns_prefixes) response = etree.tostring(self.response, pretty_print=self.pretty_print, encoding='unicode') if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and self.kvp['outputformat'] == 'application/json'): self.contenttype = self.kvp['outputformat'] from pycsw.core.formats import fmt_json response = fmt_json.xml2json(response, self.context.namespaces, self.pretty_print) else: # it's XML if 'outputformat' in self.kvp: self.contenttype = self.kvp['outputformat'] else: self.contenttype = self.mimetype xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>' '\n' % self.encoding) appinfo = '<!-- pycsw %s -->\n' % self.context.version if isinstance(self.contenttype, bytes): self.contenttype = self.contenttype.decode() s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding) LOGGER.debug('Response code: %s', self.context.response_codes[self.status]) LOGGER.debug('Response:\n%s', s) return [self.context.response_codes[self.status], s] def _gen_soap_wrapper(self): """ Generate SOAP wrapper """ LOGGER.info('Writing SOAP wrapper.') node = etree.Element( util.nspath_eval('soapenv:Envelope', self.context.namespaces), nsmap=self.context.namespaces ) schema_location_ns = util.nspath_eval('xsi:schemaLocation', self.context.namespaces) node.attrib[schema_location_ns] = '%s %s' % ( self.context.namespaces['soapenv'], self.context.namespaces['soapenv'] ) node2 = etree.SubElement( node, util.nspath_eval('soapenv:Body', self.context.namespaces)) if self.exception: node3 = etree.SubElement( node2, util.nspath_eval('soapenv:Fault', self.context.namespaces) ) node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Code', self.context.namespaces) ) etree.SubElement( node4, util.nspath_eval('soapenv:Value', self.context.namespaces) ).text = 'soap:Server' node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Reason', self.context.namespaces) ) etree.SubElement( node4, util.nspath_eval('soapenv:Text', self.context.namespaces) ).text = 'A server exception was encountered.' node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Detail', self.context.namespaces) ) node4.append(self.response) else: node2.append(self.response) self.response = node def _gen_manager(self): """ Update self.context.model with CSW-T advertising """ if (self.config.has_option('manager', 'transactions') and self.config.get('manager', 'transactions') == 'true'): self.manager = True self.context.model['operations_order'].append('Transaction') self.context.model['operations']['Transaction'] = { 'methods': {'get': False, 'post': True}, 'parameters': {} } schema_values = [ 'http://www.opengis.net/cat/csw/2.0.2', 'http://www.opengis.net/cat/csw/3.0', 'http://www.opengis.net/wms', 'http://www.opengis.net/wmts/1.0', 'http://www.opengis.net/wfs', 'http://www.opengis.net/wfs/2.0', 'http://www.opengis.net/wcs', 'http://www.opengis.net/wps/1.0.0', 'http://www.opengis.net/sos/1.0', 'http://www.opengis.net/sos/2.0', 'http://www.isotc211.org/2005/gmi', 'urn:geoss:waf', ] self.context.model['operations_order'].append('Harvest') self.context.model['operations']['Harvest'] = { 'methods': {'get': False, 'post': True}, 'parameters': { 'ResourceType': {'values': schema_values} } } self.context.model['operations']['Transaction'] = { 'methods': {'get': False, 'post': True}, 'parameters': { 'TransactionSchemas': {'values': sorted(schema_values)} } } self.csw_harvest_pagesize = 10 if self.config.has_option('manager', 'csw_harvest_pagesize'): self.csw_harvest_pagesize = int( self.config.get('manager', 'csw_harvest_pagesize')) def _test_manager(self): """ Verify that transactions are allowed """ if self.config.get('manager', 'transactions') != 'true': raise RuntimeError('CSW-T interface is disabled') ipaddress = self.environ['REMOTE_ADDR'] if not self.config.has_option('manager', 'allowed_ips') or \ (self.config.has_option('manager', 'allowed_ips') and not util.ipaddress_in_whitelist(ipaddress, self.config.get('manager', 'allowed_ips').split(','))): raise RuntimeError( 'CSW-T operations not allowed for this IP address: %s' % ipaddress) def _cql_update_queryables_mappings(self, cql, mappings): """ Transform CQL query's properties to underlying DB columns """ LOGGER.debug('Raw CQL text = %s', cql) LOGGER.debug(str(list(mappings.keys()))) if cql is not None: for key in mappings.keys(): try: cql = cql.replace(key, mappings[key]['dbcol']) except: cql = cql.replace(key, mappings[key]) LOGGER.debug('Interpolated CQL text = %s.', cql) return cql def _process_responsehandler(self, xml): """ Process response handler """ if self.kvp['responsehandler'] is not None: LOGGER.info('Processing responsehandler %s' % self.kvp['responsehandler']) uprh = urlparse(self.kvp['responsehandler']) if uprh.scheme == 'mailto': # email import smtplib LOGGER.debug('Email detected') smtp_host = 'localhost' if self.config.has_option('server', 'smtp_host'): smtp_host = self.config.get('server', 'smtp_host') body = ('Subject: pycsw %s results\n\n%s' % (self.kvp['request'], xml)) try: LOGGER.info('Sending email') msg = smtplib.SMTP(smtp_host) msg.sendmail( self.config.get('metadata:main', 'contact_email'), uprh.path, body ) msg.quit() LOGGER.debug('Email sent successfully.') except Exception as err: LOGGER.exception('Error processing email') elif uprh.scheme == 'ftp': import ftplib LOGGER.debug('FTP detected.') try: LOGGER.info('Sending to FTP server.') ftp = ftplib.FTP(uprh.hostname) if uprh.username is not None: ftp.login(uprh.username, uprh.password) ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml)) ftp.quit() LOGGER.debug('FTP sent successfully.') except Exception as err: LOGGER.exception('Error processing FTP') @staticmethod def normalize_kvp(kvp): """Normalize Key Value Pairs. This method will transform all keys to lowercase and leave values unchanged, as specified in the CSW standard (see for example note C on Table 62 - KVP Encoding for DescribeRecord operation request of the CSW standard version 2.0.2) :arg kvp: a mapping with Key Value Pairs :type kvp: dict :returns: A new dictionary with normalized parameters """ result = dict() for name, value in kvp.items(): result[name.lower()] = value return result
parser.add_option_group(jabber_group) parser.add_option_group(zulip.generate_option_group(parser, "zulip-")) (options, args) = parser.parse_args() logging.basicConfig(level=options.log_level, format='%(levelname)-8s %(message)s') if options.zulip_config_file is None: config_file = zulip.get_default_config_filename() else: config_file = options.zulip_config_file config = SafeConfigParser() try: with open(config_file, 'r') as f: config.readfp(f, config_file) except IOError: pass for option in ("jid", "jabber_password", "conference_domain", "mode", "zulip_email_suffix", "jabber_server_address", "jabber_server_port"): if (getattr(options, option) is None and config.has_option("jabber_mirror", option)): setattr(options, option, config.get("jabber_mirror", option)) for option in ("no_use_tls", ): if getattr(options, option) is None: if config.has_option("jabber_mirror", option): setattr(options, option, config.getboolean("jabber_mirror", option)) else:
def get_giphy_api_key_from_config(): config = SafeConfigParser() with open(os.environ['HOME'] + '/.giphy_config', 'r') as config_file: config.readfp(config_file) return config.get("giphy-config", "key")
def get_weather_api_key_from_config(directory, config_name): config = SafeConfigParser() with open(directory + config_name, 'r') as config_file: config.readfp(config_file) return config.get("weather-config", "key")
def load_conf(): conf_path = os.path.expanduser(config_path) if not os.path.exists(conf_path): logger.warn("{conf} couldn't be found, please use \'coscmd config -h\' to learn how to config coscmd!".format(conf=to_printable_str(conf_path))) raise IOError else: logger.debug('{conf} is found'.format(conf=to_printable_str(conf_path))) try: with open(conf_path, 'r') as f: cp = SafeConfigParser() cp.readfp(fp=f) if not cp.has_section('common'): raise Exception("[common] section could't be found, please check your config file.") if cp.has_option('common', 'part_size'): part_size = cp.getint('common', 'part_size') else: part_size = 1 if cp.has_option('common', 'max_thread'): max_thread = cp.getint('common', 'max_thread') else: max_thread = 5 try: secret_id = cp.get('common', 'secret_id') except Exception: secret_id = cp.get('common', 'access_id') try: appid = cp.get('common', 'appid') bucket = cp.get('common', 'bucket') if bucket.endswith("-"+str(appid)): bucket = bucket.rstrip(appid) bucket = bucket[:-1] except Exception: try: bucket = cp.get('common', 'bucket') appid = bucket.split('-')[-1] bucket = bucket.rstrip(appid) bucket = bucket[:-1] except Exception: # check if user use -b bucket if (pre_bucket == ""): logger.error("The configuration file is wrong. Please reconfirm") try: schema = cp.get('common', 'schema') except Exception: schema = 'https' try: verify = cp.get('common', 'verify') except Exception: verify = 'md5' try: token = cp.get('common', 'token') except Exception: token = None try: error = cp.get('common', 'error') except Exception: error = "cos.err.log" try: success = cp.get('common', 'success') except Exception: success = "cos.suc.log" try: anonymous = cp.get('common', 'anonymous') if anonymous == 'True' or anonymous == 'true': anonymous = True else: anonymous = False except Exception: anonymous = False try: retry = int(cp.get('common', 'retry')) except Exception: retry = 2 try: timeout = int(cp.get('common', 'timeout')) except Exception: timeout = 60 region, endpoint = None, None if cp.has_option('common', 'region'): region = cp.get('common', 'region') if cp.has_option('common', 'endpoint'): endpoint = cp.get('common', 'endpoint') if pre_appid != "": appid = pre_appid if pre_bucket != "": bucket = pre_bucket if pre_region != "": region = pre_region conf = CoscmdConfig( appid=appid, secret_id=secret_id, secret_key=cp.get('common', 'secret_key'), token=token, region=compatible(region), endpoint=endpoint, bucket=bucket, part_size=part_size, max_thread=max_thread, schema=schema, anonymous=anonymous, verify=verify, retry=retry, timeout=timeout, error=error, success=success ) return conf except Exception as e: raise(e)
def onDiscovery(self, theme, settings, dependenciesSettings): res = queryResourceDirectory(THEME_RESOURCE_NAME, theme) if res is None: return directoryName = 'views' if 'directory' in settings: directoryName = settings['directory'] if res.isDirectory(directoryName): viewsDir = res[directoryName] layer = getattr(schemata, theme, None) if 'layer' in settings: layerName = settings['layer'] try: layer = resolve(layerName) except (ImportError, AttributeError): logger.warn("Could not import %s" % layerName) return viewConfig = SafeConfigParser() if viewsDir.isFile(VIEW_CONFIG_FILENAME): fp = viewsDir.openFile(VIEW_CONFIG_FILENAME) try: viewConfig.readfp(fp) finally: try: fp.close() except AttributeError: pass views = [] configurationMachine = ConfigurationMachine() path = viewsDir.directory for filename in os.listdir(path): if not filename.lower().endswith(EXTENSION): continue name = viewName = filename[:-3] permission = 'zope2.View' for_ = Interface class_ = None template = os.path.join(path, filename) menu = {} # Read override options from views.cfg if applicable if viewConfig.has_section(name): if viewConfig.has_option(name, 'name'): viewName = viewConfig.get(name, 'name') if viewConfig.has_option(name, 'permission'): permission = viewConfig.get(name, 'permission') if viewConfig.has_option(name, 'for'): forStr = viewConfig.get(name, 'for') if forStr != "*": for_ = resolve(forStr) if viewConfig.has_option(name, 'class'): class_ = resolve(viewConfig.get(name, 'class')) if viewConfig.has_option(name, 'menu'): menu = dict( title=viewConfig.get(name, 'menu'), menu=getattr( zope.browsermenu.metaconfigure.menus, "plone_displayviews", ), ) Products.Five.browser.metaconfigure.page( configurationMachine, name=viewName, permission=permission, for_=for_, layer=layer, template=template, class_=class_, **menu ) views.append(name) if len(views) > 0: configurationMachine.execute_actions() self.registered[theme] = views
def deploy_django(proj): """ Deploy a Django project """ wsgi_base_path = os.environ.get('WSGI_BASE_PATH', '/var/www/wsgi') httpd_conf_dir = os.environ.get('HTTPD_CONF_DIR', '/etc/httpd/locations.d') httpd_host = os.environ.get('HTTPD_HOST', platform.node()) httpd_media_base = os.environ.get('HTTPD_MEDIA_BASE', '/var/www/html/media') httpd_static_base = os.environ.get('HTTPD_STATIC_BASE', '/var/www/html/static') secret_key_gen = os.environ.get('SECRET_KEY_GEN', '/usr/bin/pwgen -c -n -y 78 1') proj_base = os.path.join(wsgi_base_path, proj) path = lambda p: os.path.join(proj_base, p) proj_defaults = { 'name': proj, 'proj_base': proj_base, 'dst': '%(name)s-project', 'settings': '%(name)s_production', 'url': '/%(name)s', 'build': 'build/build.sh', 'wsgi': 'wsgi.py', 'allowed_hosts': httpd_host, 'secret_key': subprocess.check_output(secret_key_gen.split() ).strip().replace("'", "-"), 'media_root': os.path.join(httpd_media_base, proj), 'static_root': os.path.join(httpd_static_base, proj), 'scm': '/usr/bin/git', 'settings_append': DEFAULT_SETTINGS_APPEND, 'deploy_requires': None, 'deploy_commands': ['migrate'] } # Protect '%' from interpolation proj_defaults['secret_key'] = re.sub(r'%', r'', proj_defaults['secret_key']) # Choose clone command proj_defaults['scm_clone'] = SCM_DEFAULT_CHECKOUT[os.path.split( proj_defaults['scm'])[-1]] # Load defaults cfg = SafeConfigParser(proj_defaults) # Force read cfg.readfp(open(proj+'.cfg', 'r')) #logger.debug('Final configuration:') #for k,v in cfg.items(CFG_SECTION): # logger.debug('\t%s: %s', k, v) # Create directory os.mkdir(proj_base) # Virtualenv virtualenv.create_environment(proj_base) # Checkout subprocess.check_call([ cfg.get(CFG_SECTION, 'scm'), cfg.get(CFG_SECTION, 'scm_clone'), cfg.get(CFG_SECTION, 'src'), path(cfg.get(CFG_SECTION, 'dst')), ]) # Build activate = path('bin/activate') build = os.path.join( cfg.get(CFG_SECTION, 'dst'), cfg.get(CFG_SECTION, 'build') ) subprocess.check_call([build], cwd=proj_base, env={'BASH_ENV': activate}) # Install Deploy Requiremts deploy_requires = cfg.get(CFG_SECTION, 'deploy_requires') if deploy_requires: logger.debug('Installing: %s', deploy_requires) cmd = [os.path.join(virtualenv.path_locations(proj_base)[-1], 'pip') , 'install'] cmd.extend(parse_list(deploy_requires)) subprocess.check_call(cmd) # Create settings settings_file = path(cfg.get(CFG_SECTION, 'settings'))+'.py' slock = LockFile(settings_file) slock.acquire() if os.path.exists(settings_file): slock.release() raise IOError([17, 'File exists']) try: sfp = open(settings_file, 'w') print(DJANGO_SETTINGS_TEMPLATE % dict(cfg.items(CFG_SECTION)), file=sfp) sfp.close() finally: slock.release() # Create wsgi script wsgi_file = path(cfg.get(CFG_SECTION, 'wsgi')) slock = LockFile(wsgi_file) slock.acquire() if os.path.exists(wsgi_file): slock.release() raise IOError([17, 'File exists']) try: wfp = open(wsgi_file, 'w') print(WSGI_TEMPLATE % dict(cfg.items(CFG_SECTION)), file=wfp) wfp.close() finally: slock.release() # Create apache conf conf_file = os.path.join(httpd_conf_dir, cfg.get(CFG_SECTION, 'name'))+'.conf' slock = LockFile(conf_file) slock.acquire() if os.path.exists(conf_file): slock.release() raise IOError([17, 'File exists']) try: sfp = open(conf_file, 'w') conf = dict(cfg.items(CFG_SECTION)) conf['site_libs'] = os.path.join( virtualenv.path_locations(proj_base)[1], 'site-packages') http_conf = HTTPD_CONF_TEMPLATE % conf print(http_conf, file=sfp) sfp.close() finally: slock.release() # Perform django commands deploy_commands = cfg.get(CFG_SECTION, 'deploy_commands') if deploy_commands: manage = [os.path.join(virtualenv.path_locations(proj_base)[-1], virtualenv.expected_exe) , 'manage.py'] os.chdir(path(cfg.get(CFG_SECTION, 'dst'))) # Deployment django environment dep_env = os.environ.copy() dep_env['DJANGO_SETTINGS_MODULE'] = cfg.get(CFG_SECTION, 'settings') dep_env['PYTHONPATH'] = path('.') logger.debug('Environment for commands: PYTHONPATH=%s', dep_env['PYTHONPATH']) logger.debug(' Django settings: %s', dep_env['DJANGO_SETTINGS_MODULE']) for cmd in parse_list(deploy_commands): logger.debug("Executing '%s'", ' '.join(manage+[cmd])) subprocess.check_call(manage+cmd.split(), env=dep_env) # That's it. Remember to reload apache print('You should reload apache:\n', '\t', 'systemctl reload httpd') return True
def __init__(self, email=None, api_key=None, config_file=None, verbose=False, retry_on_errors=True, site=None, client=None, cert_bundle=None, insecure=None, client_cert=None, client_cert_key=None): # type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], bool, Optional[str], Optional[str]) -> None if client is None: client = _default_client() # Fill values from Environment Variables if not available in Constructor if config_file is None: config_file = os.environ.get("ZULIP_CONFIG") if api_key is None: api_key = os.environ.get("ZULIP_API_KEY") if email is None: email = os.environ.get("ZULIP_EMAIL") if site is None: site = os.environ.get("ZULIP_SITE") if client_cert is None: client_cert = os.environ.get("ZULIP_CERT") if client_cert_key is None: client_cert_key = os.environ.get("ZULIP_CERT_KEY") if cert_bundle is None: cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE") if config_file is None: config_file = get_default_config_filename() if config_file is not None and os.path.exists(config_file): config = SafeConfigParser() with open(config_file, 'r') as f: config.readfp(f, config_file) if api_key is None: api_key = config.get("api", "key") if email is None: email = config.get("api", "email") if site is None and config.has_option("api", "site"): site = config.get("api", "site") if client_cert is None and config.has_option("api", "client_cert"): client_cert = config.get("api", "client_cert") if client_cert_key is None and config.has_option("api", "client_cert_key"): client_cert_key = config.get("api", "client_cert_key") if cert_bundle is None and config.has_option("api", "cert_bundle"): cert_bundle = config.get("api", "cert_bundle") if insecure is None and config.has_option("api", "insecure"): # Be quite strict about what is accepted so that users don't # disable security unintentionally. insecure_setting = config.get("api", "insecure").lower() if insecure_setting == "true": insecure = True elif insecure_setting == "false": insecure = False else: raise RuntimeError("insecure is set to '%s', it must be 'true' or 'false' if it is used in %s" % (insecure_setting, config_file)) elif None in (api_key, email): raise RuntimeError("api_key or email not specified and %s does not exist" % (config_file,)) self.api_key = api_key self.email = email self.verbose = verbose if site is not None: if site.startswith("localhost"): site = "http://" + site elif not site.startswith("http"): site = "https://" + site # Remove trailing "/"s from site to simplify the below logic for adding "/api" site = site.rstrip("/") self.base_url = site else: raise RuntimeError("Missing Zulip server URL; specify via --site or ~/.zuliprc.") if not self.base_url.endswith("/api"): self.base_url += "/api" self.base_url += "/" self.retry_on_errors = retry_on_errors self.client_name = client if insecure: self.tls_verification = False # type: Union[bool, str] elif cert_bundle is not None: if not os.path.isfile(cert_bundle): raise RuntimeError("tls bundle '%s' does not exist" % (cert_bundle,)) self.tls_verification = cert_bundle else: # Default behavior: verify against system CA certificates self.tls_verification = True if client_cert is None: if client_cert_key is not None: raise RuntimeError("client cert key '%s' specified, but no client cert public part provided" % (client_cert_key,)) else: # we have a client cert if not os.path.isfile(client_cert): raise RuntimeError("client cert '%s' does not exist" % (client_cert,)) if client_cert_key is not None: if not os.path.isfile(client_cert_key): raise RuntimeError("client cert key '%s' does not exist" % (client_cert_key,)) self.client_cert = client_cert self.client_cert_key = client_cert_key
def __init__(self, email=None, api_key=None, config_file=None, verbose=False, retry_on_errors=True, site=None, client=None, cert_bundle=None, insecure=None, client_cert=None, client_cert_key=None): if client is None: client = _default_client() if config_file is None: config_file = get_default_config_filename() if os.path.exists(config_file): config = SafeConfigParser() with open(config_file, 'r') as f: config.readfp(f, config_file) if api_key is None: api_key = config.get("api", "key") if email is None: email = config.get("api", "email") if site is None and config.has_option("api", "site"): site = config.get("api", "site") if client_cert is None and config.has_option("api", "client_cert"): client_cert = config.get("api", "client_cert") if client_cert_key is None and config.has_option( "api", "client_cert_key"): client_cert_key = config.get("api", "client_cert_key") if cert_bundle is None and config.has_option("api", "cert_bundle"): cert_bundle = config.get("api", "cert_bundle") if insecure is None and config.has_option("api", "insecure"): # Be quite strict about what is accepted so that users don't # disable security unintentionally. insecure_setting = config.get("api", "insecure").lower() if insecure_setting == "true": insecure = True elif insecure_setting == "false": insecure = False else: raise RuntimeError( "insecure is set to '%s', it must be 'true' or 'false' if it is used in %s" % (insecure_setting, config_file)) elif None in (api_key, email): raise RuntimeError( "api_key or email not specified and %s does not exist" % (config_file, )) self.api_key = api_key self.email = email self.verbose = verbose if site is not None: if not site.startswith("http"): site = "https://" + site # Remove trailing "/"s from site to simplify the below logic for adding "/api" site = site.rstrip("/") self.base_url = site else: self.base_url = "https://api.zulip.com" if self.base_url != "https://api.zulip.com" and not self.base_url.endswith( "/api"): self.base_url += "/api" self.base_url += "/" self.retry_on_errors = retry_on_errors self.client_name = client if insecure: self.tls_verification = False elif cert_bundle is not None: if not os.path.isfile(cert_bundle): raise RuntimeError("tls bundle '%s' does not exist" % (cert_bundle, )) self.tls_verification = cert_bundle else: # Default behavior: verify against system CA certificates self.tls_verification = True if client_cert is None: if client_cert_key is not None: raise RuntimeError( "client cert key '%s' specified, but no client cert public part provided" % (client_cert_key, )) else: # we have a client cert if not os.path.isfile(client_cert): raise RuntimeError("client cert '%s' does not exist" % (client_cert, )) if client_cert_key is not None: if not os.path.isfile(client_cert_key): raise RuntimeError("client cert key '%s' does not exist" % (client_cert_key, )) self.client_cert = client_cert self.client_cert_key = client_cert_key
def __init__(self, email=None, api_key=None, config_file=None, verbose=False, retry_on_errors=True, site=None, client=None, cert_bundle=None, insecure=None, client_cert=None, client_cert_key=None): # type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[str], Optional[str]) -> None if client is None: client = _default_client() # Normalize user-specified path if config_file is not None: config_file = os.path.abspath(os.path.expanduser(config_file)) # Fill values from Environment Variables if not available in Constructor if config_file is None: config_file = os.environ.get("ZULIP_CONFIG") if api_key is None: api_key = os.environ.get("ZULIP_API_KEY") if email is None: email = os.environ.get("ZULIP_EMAIL") if site is None: site = os.environ.get("ZULIP_SITE") if client_cert is None: client_cert = os.environ.get("ZULIP_CERT") if client_cert_key is None: client_cert_key = os.environ.get("ZULIP_CERT_KEY") if cert_bundle is None: cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE") if insecure is None: # Be quite strict about what is accepted so that users don't # disable security unintentionally. insecure_setting = os.environ.get('ZULIP_ALLOW_INSECURE') if insecure_setting is not None: insecure = validate_boolean_field(insecure_setting) if insecure is None: raise ZulipError("The ZULIP_ALLOW_INSECURE environment " "variable is set to '{}', it must be " "'true' or 'false'" .format(insecure_setting)) if config_file is None: config_file = get_default_config_filename() if config_file is not None and os.path.exists(config_file): config = SafeConfigParser() with open(config_file, 'r') as f: config.readfp(f, config_file) if api_key is None: api_key = config.get("api", "key") if email is None: email = config.get("api", "email") if site is None and config.has_option("api", "site"): site = config.get("api", "site") if client_cert is None and config.has_option("api", "client_cert"): client_cert = config.get("api", "client_cert") if client_cert_key is None and config.has_option("api", "client_cert_key"): client_cert_key = config.get("api", "client_cert_key") if cert_bundle is None and config.has_option("api", "cert_bundle"): cert_bundle = config.get("api", "cert_bundle") if insecure is None and config.has_option("api", "insecure"): # Be quite strict about what is accepted so that users don't # disable security unintentionally. insecure_setting = config.get('api', 'insecure') insecure = validate_boolean_field(insecure_setting) if insecure is None: raise ZulipError("insecure is set to '{}', it must be " "'true' or 'false' if it is used in {}" .format(insecure_setting, config_file)) elif None in (api_key, email): raise ConfigNotFoundError("api_key or email not specified and file %s does not exist" % (config_file,)) assert(api_key is not None and email is not None) self.api_key = api_key self.email = email self.verbose = verbose if site is not None: if site.startswith("localhost"): site = "http://" + site elif not site.startswith("http"): site = "https://" + site # Remove trailing "/"s from site to simplify the below logic for adding "/api" site = site.rstrip("/") self.base_url = site else: raise MissingURLError("Missing Zulip server URL; specify via --site or ~/.zuliprc.") if not self.base_url.endswith("/api"): self.base_url += "/api" self.base_url += "/" self.retry_on_errors = retry_on_errors self.client_name = client if insecure: logger.warning('Insecure mode enabled. The server\'s SSL/TLS ' 'certificate will not be validated, making the ' 'HTTPS connection potentially insecure') self.tls_verification = False # type: Union[bool, str] elif cert_bundle is not None: if not os.path.isfile(cert_bundle): raise ConfigNotFoundError("tls bundle '%s' does not exist" % (cert_bundle,)) self.tls_verification = cert_bundle else: # Default behavior: verify against system CA certificates self.tls_verification = True if client_cert is None: if client_cert_key is not None: raise ConfigNotFoundError("client cert key '%s' specified, but no client cert public part provided" % (client_cert_key,)) else: # we have a client cert if not os.path.isfile(client_cert): raise ConfigNotFoundError("client cert '%s' does not exist" % (client_cert,)) if client_cert_key is not None: if not os.path.isfile(client_cert_key): raise ConfigNotFoundError("client cert key '%s' does not exist" % (client_cert_key,)) self.client_cert = client_cert self.client_cert_key = client_cert_key self.session = None # type: Optional[requests.Session] self.has_connected = False
def configparser2yaml(cpfile): dict_ = {} cp = ConfigParser() with codecs.open(cpfile, encoding='utf-8') as fh: cp.readfp(fh) for section in cp.sections(): if section.startswith('contact:'): # contacts are now nested if 'contact' not in dict_: dict_['contact'] = {} section2 = dict_['contact'][section.split(':')[1]] = {} elif section.startswith('distribution:'): # distributions now nested if 'distribution' not in dict_: dict_['distribution'] = {} section2 = dict_['distribution'][section.split(':')[1]] = {} else: section2 = dict_[section] = {} for k, v in cp.items(section): if section == 'identification': # keywords are now nested if 'keywords' not in section2: section2['keywords'] = {} if 'default' not in section2['keywords']: section2['keywords']['default'] = {} if 'gc_cst' not in section2['keywords']: section2['keywords']['gc_cst'] = {} if 'wmo' not in section2['keywords']: section2['keywords']['wmo'] = {} if 'hnap_category_information' not in section2['keywords']: section2['keywords']['hnap_category_information'] = {} if 'hnap_category_geography' not in section2['keywords']: section2['keywords']['hnap_category_geography'] = {} if 'hnap_category_content' not in section2['keywords']: section2['keywords']['hnap_category_content'] = {} if k in ['topiccategory']: section2['topiccategory'] = [v] if k in ['keywords_en', 'keywords_fr']: section2['keywords']['default'][k] = [k2.strip() for k2 in v.split(',')] # noqa if k in ['keywords_gc_cst_en']: section2['keywords']['gc_cst']['keywords_en'] = [k2.strip() for k2 in v.split(',')] # noqa if k in ['keywords_gc_cst_fr']: section2['keywords']['gc_cst']['keywords_fr'] = [k2.strip() for k2 in v.split(',')] # noqa if k in ['keywords_wmo']: section2['keywords']['wmo']['keywords_en'] = [k2.strip() for k2 in v.split(',')] # noqa if k in ['hnap_category_information_en']: section2['keywords']['hnap_category_information']['keywords_en'] = [v] # noqa section2['keywords']['hnap_category_information']['keywords_fr'] = [v] # noqa if k in ['hnap_category_geography_en']: section2['keywords']['hnap_category_geography']['keywords_en'] = [v] # noqa if k in ['hnap_category_geography_fr']: section2['keywords']['hnap_category_geography']['keywords_fr'] = [v] # noqa if k in ['hnap_category_content_en']: section2['keywords']['hnap_category_content']['keywords_en'] = [v] # noqa if k in ['hnap_category_content_fr']: section2['keywords']['hnap_category_content']['keywords_fr'] = [v] # noqa if k == 'keywords_type': section2['keywords']['default'][k] = v section2['keywords']['gc_cst'][k] = v section2['keywords']['wmo'][k] = v section2['keywords']['hnap_category_geography'][k] = v section2['keywords']['hnap_category_information'][k] = v section2['keywords']['hnap_category_content'][k] = v else: section2[k] = v return yaml.safe_dump(dict_, default_flow_style=False, allow_unicode=True)
def config_from_string(value): config = SafeConfigParser() config.readfp(StringIO(value)) return Config(config)
from __future__ import print_function import random, unittest from ofs.remote import swiftstore as store from ofs.remote.swiftstore import SwiftOFS from ofs import OFSException from six import StringIO from six.moves.configparser import SafeConfigParser TEST_TEXT = """I am a banana""" cfg = SafeConfigParser() cfg.readfp(open('test.ini')) class TestSwiftOFS(unittest.TestCase): def setUp(self): self.bucket_name = 'ofs-test-bucket' auth_url = cfg.get('ofs', 'ofs.os_auth_url') user = cfg.get('ofs', 'ofs.os_user') passwd = cfg.get('ofs', 'ofs.os_passwd') tenant = cfg.get('ofs', 'ofs.os_tenant') self.ofs = SwiftOFS(os_auth_url=auth_url, os_user=user, os_passwd=passwd, os_tenant=tenant) self.bucket = self.ofs.connection.put_container(self.bucket_name, headers=store.PUBLIC_HEADER) def tearDown(self): self._del_bucket(self.bucket_name) def _makefp(self):
def __init__(self, email=None, api_key=None, config_file=None, verbose=False, retry_on_errors=True, site=None, client=None, cert_bundle=None, insecure=None, client_cert=None, client_cert_key=None): # type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], bool, Optional[str], Optional[str]) -> None if client is None: client = _default_client() # Fill values from Environment Variables if not available in Constructor if config_file is None: config_file = os.environ.get("ZULIP_CONFIG") if api_key is None: api_key = os.environ.get("ZULIP_API_KEY") if email is None: email = os.environ.get("ZULIP_EMAIL") if site is None: site = os.environ.get("ZULIP_SITE") if client_cert is None: client_cert = os.environ.get("ZULIP_CERT") if client_cert_key is None: client_cert_key = os.environ.get("ZULIP_CERT_KEY") if cert_bundle is None: cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE") if config_file is None: config_file = get_default_config_filename() if os.path.exists(config_file): config = SafeConfigParser() with open(config_file, 'r') as f: config.readfp(f, config_file) if api_key is None: api_key = config.get("api", "key") if email is None: email = config.get("api", "email") if site is None and config.has_option("api", "site"): site = config.get("api", "site") if client_cert is None and config.has_option("api", "client_cert"): client_cert = config.get("api", "client_cert") if client_cert_key is None and config.has_option( "api", "client_cert_key"): client_cert_key = config.get("api", "client_cert_key") if cert_bundle is None and config.has_option("api", "cert_bundle"): cert_bundle = config.get("api", "cert_bundle") if insecure is None and config.has_option("api", "insecure"): # Be quite strict about what is accepted so that users don't # disable security unintentionally. insecure_setting = config.get("api", "insecure").lower() if insecure_setting == "true": insecure = True elif insecure_setting == "false": insecure = False else: raise RuntimeError( "insecure is set to '%s', it must be 'true' or 'false' if it is used in %s" % (insecure_setting, config_file)) elif None in (api_key, email): raise RuntimeError( "api_key or email not specified and %s does not exist" % (config_file, )) self.api_key = api_key self.email = email self.verbose = verbose if site is not None: if site.startswith("localhost"): site = "http://" + site elif not site.startswith("http"): site = "https://" + site # Remove trailing "/"s from site to simplify the below logic for adding "/api" site = site.rstrip("/") self.base_url = site else: raise RuntimeError( "Missing Zulip server URL; specify via --site or ~/.zuliprc.") if not self.base_url.endswith("/api"): self.base_url += "/api" self.base_url += "/" self.retry_on_errors = retry_on_errors self.client_name = client if insecure: self.tls_verification = False # type: Union[bool, str] elif cert_bundle is not None: if not os.path.isfile(cert_bundle): raise RuntimeError("tls bundle '%s' does not exist" % (cert_bundle, )) self.tls_verification = cert_bundle else: # Default behavior: verify against system CA certificates self.tls_verification = True if client_cert is None: if client_cert_key is not None: raise RuntimeError( "client cert key '%s' specified, but no client cert public part provided" % (client_cert_key, )) else: # we have a client cert if not os.path.isfile(client_cert): raise RuntimeError("client cert '%s' does not exist" % (client_cert, )) if client_cert_key is not None: if not os.path.isfile(client_cert_key): raise RuntimeError("client cert key '%s' does not exist" % (client_cert_key, )) self.client_cert = client_cert self.client_cert_key = client_cert_key
class GlobusConfigParser(object): """ Wraps a SafeConfigParser to do modified get()s and config file loading. """ _GENERAL_CONF_SECTION = 'general' def __init__(self): self._parser = SafeConfigParser() self._load_config() def _load_config(self): # TODO: /etc is not windows friendly, not sure about expanduser self._read([_get_lib_config_path(), "/etc/globus.cfg", os.path.expanduser("~/.globus.cfg")]) def _read(self, filenames): """ Wraps up self._parser.read() to inject '[general]\n' at the beginning of file contents. Originally, this was implemented by catching MissingSectionHeaderErrors, but that's actually overcomplicated and unnecessary. Always inserting it, uniformly, is simpler and doesn't change the semantics of config parsing at all. """ for fname in filenames: try: with open(fname) as f: # wrap the file-like object in a StringIO so that we can # pass it to the SafeConfigParser as a file like object wrapped_file = StringIO( '[{}]\n'.format(self._GENERAL_CONF_SECTION) + f.read()) try: self._parser.readfp(wrapped_file, fname) except DuplicateSectionError: f.seek(0) self._parser.readfp(f, fname) except IOError: continue def get(self, option, section=None, environment=None, failover_to_general=False, check_env=False, type_cast=str): """ Attempt to lookup an option in the config file. Optionally failover to the general section if the option is not found. Also optionally, check for a relevant environment variable, which is named always as GLOBUS_SDK_{option.upper()}. Note that 'section' doesn't slot into the naming at all. Otherwise, we'd have to contend with GLOBUS_SDK_GENERAL_... for almost everything, and GLOBUS_SDK_ENVIRONMENT\ PROD_... which is awful. Returns None for an unfound key, rather than raising a NoOptionError. """ # envrionment is just a fancy name for sections that start with # 'environment ' if environment: section = 'environment ' + environment # if you don't specify a section or an environment, assume it's the # general conf section if section is None: section = self._GENERAL_CONF_SECTION # if this is a config option which checks the shell env, look there # *first* for a value -- env values have higher precedence than config # files so that you can locally override the behavior of a command in a # given shell or subshell env_option_name = 'GLOBUS_SDK_{}'.format(option.upper()) value = None if check_env and env_option_name in os.environ: value = os.environ[env_option_name] else: try: value = self._parser.get(section, option) except (NoOptionError, NoSectionError): if failover_to_general: value = self.get(option, section=self._GENERAL_CONF_SECTION) if value is not None: value = type_cast(value) return value
parser.add_option_group(jabber_group) parser.add_option_group(zulip.generate_option_group(parser, "zulip-")) (options, args) = parser.parse_args() logging.basicConfig(level=options.log_level, format='%(levelname)-8s %(message)s') if options.zulip_config_file is None: config_file = zulip.get_default_config_filename() else: config_file = options.zulip_config_file config = SafeConfigParser() try: with open(config_file, 'r') as f: config.readfp(f, config_file) except IOError: pass for option in ("jid", "jabber_password", "conference_domain", "mode", "zulip_email_suffix", "jabber_server_address", "jabber_server_port"): if (getattr(options, option) is None and config.has_option("jabber_mirror", option)): setattr(options, option, config.get("jabber_mirror", option)) for option in ("no_use_tls",): if getattr(options, option) is None: if config.has_option("jabber_mirror", option): setattr(options, option, config.getboolean("jabber_mirror", option)) else: setattr(options, option, False)