def read(self):
"""Override base class version to avoid parsing error with the first
'RANDFILE = ...' part of the openssl file. Also, reformat _sections
to allow for the style of SSL config files where section headings can
have spaces either side of the brackets e.g.
[ sectionName ]
and comments can occur on the same line as an option e.g.
option = blah # This is option blah
Reformat _sections to """
try:
config_file = open(self._filePath)
fileTxt = config_file.read()
except Exception as e:
raise OpenSSLConfigError('Reading OpenSSL config file "%s": %s' %
(self._filePath, e))
idx = re.search('\[\s*\w*\s*\]', fileTxt).span()[0]
config_file.seek(idx)
SafeConfigParser.readfp(self, config_file)
# Filter section names and remove comments from options
for section, val in list(self._sections.items()):
newSection = section
self._sections[newSection.strip()] = \
dict([(opt, self._filtOptVal(optVal))
for opt, optVal in list(val.items())])
del self._sections[section]
self._set_required_dn_params()
class Config(object):
"""A ConfigParser wrapper to support defaults when calling instance
methods, and also tied to a single section"""
SECTION = 'silkyy'
def __init__(self, values=None, extra_sources=()):
if values is None:
sources = self._getsources()
self.cp = ConfigParser()
if __package__:
default_config = ensure_str(get_data(__package__, 'default.conf'))
self._load_config_file(StringIO(default_config))
for source in sources:
if os.path.exists(source):
self._load_config_file(open(source))
else:
self.cp = SafeConfigParser(values)
self.cp.add_section(self.SECTION)
def _load_config_file(self, fp):
config = StringIO()
config.write('[' + self.SECTION + ']' + os.linesep)
config.write(fp.read())
config.seek(0, os.SEEK_SET)
self.cp.readfp(config)
def _getsources(self):
sources = ['conf/silkyy.conf']
return sources
def get(self, option, default=None):
env_key = 'SILKYY_' + option.replace('.', '_').upper()
try:
return os.getenv(env_key) or self.cp.get(self.SECTION, option)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def _get(self, option, conv, default=None):
return conv(self.get(option, default))
def getint(self, option, default=None):
return self._get(option, int, default)
def getboolean(self, option, default=None):
return self._get(option, str2bool, default)
def getfloat(self, option, default=None):
return self._get(option, float, default)
def items(self, section, default=None):
try:
return self.cp.items(section)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def getManifest(fp, format, defaults=None):
"""Read the manifest from the given open file pointer according to the
given ManifestFormat. Pass a dict as ``defaults`` to override the defaults
from the manifest format.
"""
if defaults is None:
defaults = format.defaults
parser = SafeConfigParser()
if six.PY2:
parser.readfp(fp)
else:
data = fp.read()
if isinstance(data, six.binary_type):
data = data.decode()
parser.read_string(data)
results = {}
for key in format.keys:
if parser.has_option(format.resourceType, key):
results[key] = parser.get(format.resourceType, key)
else:
results[key] = defaults.get(key, None)
for key in format.parameterSections:
sectionName = "%s:%s" % (format.resourceType, key,)
if parser.has_section(sectionName):
results[key] = dict(parser.items(sectionName))
else:
results[key] = {}
return results
def read_config_file(path):
config = SafeConfigParser()
cfp = open(path, 'r')
config.readfp(cfp)
cfp.close()
return config
def read_config_file(path):
config = SafeConfigParser()
cfp = open(path, 'r')
config.readfp(cfp)
cfp.close()
return config
def getPluginSettings(themeDirectory, plugins=None):
"""Given an IResourceDirectory for a theme, return the settings for the
given list of plugins (or all plugins, if not given) provided as a list
of (name, plugin) pairs.
Returns a dict of dicts, with the outer dict having plugin names as keys
and containing plugins settings (key/value pairs) as values.
"""
if plugins is None:
plugins = getPlugins()
manifestContents = {}
if themeDirectory.isFile(MANIFEST_FILENAME):
parser = SafeConfigParser()
fp = themeDirectory.openFile(MANIFEST_FILENAME)
try:
parser.readfp(fp)
for section in parser.sections():
manifestContents[section] = {}
for name, value in parser.items(section):
manifestContents[section][name] = value
finally:
try:
fp.close()
except AttributeError:
pass
pluginSettings = {}
for name, plugin in plugins:
pluginSettings[name] = manifestContents.get(
"{0:s}:{1:s}".format(THEME_RESOURCE_NAME, name),
{}
)
return pluginSettings
class Config(object):
"""A ConfigParser wrapper to support defaults when calling instance
methods, and also tied to a single section"""
SECTION = 'scrapyd'
def __init__(self, values=None, extra_sources=()):
if values is None:
sources = self._getsources()
default_config = get_data(__package__, 'default_scrapyd.conf').decode('utf8')
self.cp = SafeConfigParser()
self.cp.readfp(io.StringIO(default_config))
sources.extend(extra_sources)
for fname in sources:
try:
with io.open(fname) as fp:
self.cp.readfp(fp)
except (IOError, OSError):
pass
else:
self.cp = SafeConfigParser(values)
self.cp.add_section(self.SECTION)
def _getsources(self):
sources = ['/etc/scrapyd/scrapyd.conf', r'c:\scrapyd\scrapyd.conf']
sources += sorted(glob.glob('/etc/scrapyd/conf.d/*'))
sources += ['scrapyd.conf']
sources += [expanduser('~/.scrapyd.conf')]
scrapy_cfg = closest_scrapy_cfg()
if scrapy_cfg:
sources.append(scrapy_cfg)
return sources
def _getany(self, method, option, default):
try:
return method(self.SECTION, option)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def get(self, option, default=None):
return self._getany(self.cp.get, option, default)
def getint(self, option, default=None):
return self._getany(self.cp.getint, option, default)
def getfloat(self, option, default=None):
return self._getany(self.cp.getfloat, option, default)
def getboolean(self, option, default=None):
return self._getany(self.cp.getboolean, option, default)
def items(self, section, default=None):
try:
return self.cp.items(section)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def getManifest(fp, format, defaults=None):
"""Read the manifest from the given open file pointer according to the
given ManifestFormat. Pass a dict as ``defaults`` to override the defaults
from the manifest format.
"""
if defaults is None:
defaults = format.defaults
parser = SafeConfigParser()
parser.readfp(fp)
results = {}
for key in format.keys:
if parser.has_option(format.resourceType, key):
results[key] = parser.get(format.resourceType, key)
else:
results[key] = defaults.get(key, None)
for key in format.parameterSections:
sectionName = "%s:%s" % (format.resourceType, key,)
if parser.has_section(sectionName):
results[key] = dict(parser.items(sectionName))
else:
results[key] = {}
return results
class Config(object):
"""A ConfigParser wrapper to support defaults when calling instance
methods, and also tied to a single section"""
SECTION = 'scrapydartx'
def __init__(self, values=None, extra_sources=()):
if values is None:
sources = self._getsources()
default_config = get_data(__package__,
'default_scrapyd.conf').decode('utf8')
self.cp = SafeConfigParser()
self.cp.readfp(StringIO(default_config))
self.cp.read(sources)
for fp in extra_sources:
self.cp.readfp(fp)
else:
self.cp = SafeConfigParser(values)
self.cp.add_section(self.SECTION)
def _getsources(self):
sources = [
'/etc/scrapydartx/scrapydartx.conf', r'c:\scrapyd\scrapyd.conf'
]
sources += sorted(glob.glob('/etc/scrapydartx/conf.d/*'))
sources += ['scrapydartx.conf']
sources += [expanduser('~/.scrapydartx.conf')]
scrapy_cfg = closest_scrapy_cfg()
if scrapy_cfg:
sources.append(scrapy_cfg)
return sources
def _getany(self, method, option, default):
try:
return method(self.SECTION, option)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def get(self, option, default=None):
return self._getany(self.cp.get, option, default)
def getint(self, option, default=None):
return self._getany(self.cp.getint, option, default)
def getfloat(self, option, default=None):
return self._getany(self.cp.getfloat, option, default)
def getboolean(self, option, default=None):
return self._getany(self.cp.getboolean, option, default)
def items(self, section, default=None):
try:
return self.cp.items(section)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def parse_config_file(self, config_file):
config = SafeConfigParser(self.DEFAULT_CONFIG)
config.readfp(config_file)
blessconfig = {
'CLIENT_CONFIG': {
'domain_regex':
config.get('CLIENT', 'domain_regex'),
'cache_dir':
config.get('CLIENT', 'cache_dir'),
'cache_file':
config.get('CLIENT', 'cache_file'),
'mfa_cache_dir':
config.get('CLIENT', 'mfa_cache_dir'),
'mfa_cache_file':
config.get('CLIENT', 'mfa_cache_file'),
'ip_urls': [
s.strip()
for s in config.get('CLIENT', 'ip_urls').split(",")
],
'update_script':
config.get('CLIENT', 'update_script'),
'user_session_length':
int(config.get('CLIENT', 'user_session_length')),
'usebless_role_session_length':
int(config.get('CLIENT', 'usebless_role_session_length')),
'update_sshagent':
config.getboolean('CLIENT', 'update_sshagent'),
},
'BLESS_CONFIG': {
'userrole': config.get('LAMBDA', 'user_role'),
'accountid': config.get('LAMBDA', 'account_id'),
'functionname': config.get('LAMBDA', 'functionname'),
'functionversion': config.get('LAMBDA', 'functionversion'),
'certlifetime': config.getint('LAMBDA', 'certlifetime'),
'ipcachelifetime': config.getint('LAMBDA', 'ipcachelifetime'),
'timeoutconfig': {
'connect': config.getint('LAMBDA', 'timeout_connect'),
'read': config.getint('LAMBDA', 'timeout_read')
}
},
'AWS_CONFIG': {
'bastion_ips': config.get('MAIN', 'bastion_ips'),
'remote_user': config.get('MAIN', 'remote_user')
},
'REGION_ALIAS': {}
}
regions = config.get('MAIN', 'region_aliases').split(",")
regions = [region.strip() for region in regions]
for region in regions:
region = region.upper()
kms_region_key = 'KMSAUTH_CONFIG_{}'.format(region)
blessconfig.update(
{kms_region_key: self._get_region_kms_config(region, config)})
blessconfig['REGION_ALIAS'].update(
{region: blessconfig[kms_region_key]['awsregion']})
return blessconfig
def init_ini_file(file=args.ini_file):
cp = SafeConfigParser()
fp = open(file)
cp.optionxform = str
cp.readfp(fp)
fp.close()
cp.set('condor', 'lalsuite-install', lalinf_prefix)
cp.set('analysis', 'engine', args.engine)
cp.remove_option('analysis', 'nparallel')
return cp
def init_ini_file(file=args.ini_file):
cp=SafeConfigParser()
fp=open(file)
cp.optionxform = str
cp.readfp(fp)
fp.close()
cp.set('condor','lalsuite-install',lalinf_prefix)
cp.set('analysis','engine',args.engine)
cp.remove_option('analysis','nparallel')
return cp
def setUp(self):
confdata = six.StringIO("""[broker]
urls = amqps://broker1.example.com:5671 amqps://broker2.example.com:5671
cert = /etc/koji-hub/plugins/client.pem
cacert = /etc/koji-hub/plugins/ca.pem
topic_prefix = koji
connect_timeout = 10
send_timeout = 60
""")
conf = SafeConfigParser()
conf.readfp(confdata)
self.handler = protonmsg.TimeoutHandler(
'amqps://broker1.example.com:5671', [], conf)
def parse(self):
parser = SafeConfigParser()
parser.readfp(StringIO(self.obj.content))
for section in parser.values():
if section.name == 'options':
options = 'install_requires', 'setup_requires', 'test_require'
for name in options:
content = section.get(name)
if not content:
continue
self._parse_content(content)
elif section.name == 'options.extras_require':
for content in section.values():
self._parse_content(content)
def get_vrouter_tor_agent_name(self, conf_file):
tor_agent_name = None
if conf_file:
try:
data = StringIO('\n'.join(line.strip()
for line in open(conf_file)))
Config = SafeConfigParser()
Config.readfp(data)
except Exception as err:
self.msg_log("Error reading file : " + conf_file + " Error : " + str(err),
SandeshLevel.SYS_ERR)
return tor_agent_name
tor_agent_name = Config.get("DEFAULT", "agent_name")
return tor_agent_name
def read_header(self):
'''
Read the backup file header that contains the meta data about
this particular backup.
'''
with open(self.header) as fd:
config = SafeConfigParser()
config.readfp(fd)
self.backup_type = config.get('ipa', 'type')
self.backup_time = config.get('ipa', 'time')
self.backup_host = config.get('ipa', 'host')
self.backup_ipa_version = config.get('ipa', 'ipa_version')
self.backup_version = config.get('ipa', 'version')
self.backup_services = config.get('ipa', 'services').split(',')
def read_ini(cls, path, section=None):
"""read preferences from an .ini file"""
parser = ConfigParser()
parser.optionxform = str
parser.readfp(mozfile.load(path))
if section:
if section not in parser.sections():
raise PreferencesReadError("No section '%s' in %s" % (section, path))
retval = parser.items(section, raw=True)
else:
retval = parser.defaults().items()
# cast the preferences since .ini is just strings
return [(i, cls.cast(j)) for i, j in retval]
def setUp(self):
super(FunctionalTestBase, self).setUp()
if not os.path.exists(TEST_CFG):
raise Exception("Unable to run the write tests without a test.ini in that defines an access_token with write privs.")
cfg = SafeConfigParser()
with open(TEST_CFG) as fp:
cfg.readfp(fp, 'test.ini')
access_token = cfg.get('write_tests', 'access_token')
try:
activity_id = cfg.get('activity_tests', 'activity_id')
except NoOptionError:
activity_id = None
self.client = Client(access_token=access_token)
self.activity_id = activity_id
def setUp(self):
confdata = six.StringIO("""[broker]
urls = amqps://broker1.example.com:5671 amqps://broker2.example.com:5671
cert = /etc/koji-hub/plugins/client.pem
cacert = /etc/koji-hub/plugins/ca.pem
topic_prefix = koji
connect_timeout = 10
send_timeout = 60
""")
if six.PY2:
conf = SafeConfigParser()
conf.readfp(confdata)
else:
conf = ConfigParser()
conf.read_file(confdata)
self.handler = protonmsg.TimeoutHandler('amqps://broker1.example.com:5671', [], conf)
def read_ini(cls, path, section=None):
"""read preferences from an .ini file"""
parser = ConfigParser()
parser.optionxform = str
parser.readfp(mozfile.load(path))
if section:
if section not in parser.sections():
raise PreferencesReadError("No section '%s' in %s" % (section, path))
retval = parser.items(section, raw=True)
else:
retval = parser.defaults().items()
# cast the preferences since .ini is just strings
return [(i, cls.cast(j)) for i, j in retval]
def get_vrouter_process_info(self, proc_name):
vrouter_file = "/etc/contrail/supervisord_vrouter_files"
for root, dirs, files in os.walk(vrouter_file):
for file in files:
if file.endswith(".ini"):
filename = \
'/etc/contrail/supervisord_vrouter_files/' + file
try:
data = StringIO('\n'.join(line.strip()
for line in open(filename)))
except IOError:
msg = "This file does not exist anymore so continuing: "
self.msg_log(msg + filename, SandeshLevel.SYS_ERR)
continue
Config = SafeConfigParser()
Config.readfp(data)
sections = Config.sections()
if not sections[0]:
msg = "Section not present in the ini file : "
self.msg_log(msg + filename, SandeshLevel.SYS_ERR)
continue
name = sections[0].split(':')
if len(name) < 2:
msg = "Incorrect section name in the ini file : "
self.msg_log(msg + filename, SandeshLevel.SYS_ERR)
continue
if name[1] == proc_name:
command = Config.get(sections[0], "command")
if not command:
msg = "Command not present in the ini file : "
self.msg_log(msg + filename, SandeshLevel.SYS_ERR)
continue
args = command.split()
if (args[0] == '/usr/bin/contrail-tor-agent'):
try:
index = args.index('--config_file')
args_val = args[index + 1]
agent_name = \
self.get_vrouter_tor_agent_name(args_val)
return (proc_name, agent_name)
except Exception as err:
msg = "Tor Agent command does " + \
"not have config file : "
self.msg_log(msg + command, SandeshLevel.SYS_ERR)
return ('vrouter_group', socket.getfqdn(self.host_ip) if self.hostname
is None else self.hostname)
def handle(self, *args, **options):
# type: (*Any, **Any) -> None
config_file = os.path.join(os.environ["HOME"], ".zuliprc")
if not os.path.exists(config_file):
raise RuntimeError("No ~/.zuliprc found")
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
api_key = config.get("api", "key")
email = config.get("api", "email")
try:
user_profile = get_user_profile_by_email(email)
user_profile.api_key = api_key
user_profile.save(update_fields=["api_key"])
except UserProfile.DoesNotExist:
print("User %s does not exist; not syncing API key" % (email,))
def test_on_start_no_ssl(self, SSLDomain):
confdata = six.StringIO("""[broker]
urls = amqp://broker1.example.com:5672 amqp://broker2.example.com:5672
topic_prefix = koji
connect_timeout = 10
send_timeout = 60
""")
conf = SafeConfigParser()
conf.readfp(confdata)
handler = protonmsg.TimeoutHandler('amqp://broker1.example.com:5672',
[], conf)
event = MagicMock()
handler.on_start(event)
event.container.connect.assert_called_once_with(
url='amqp://broker1.example.com:5672',
reconnect=False,
ssl_domain=None)
self.assertEqual(SSLDomain.call_count, 0)
def setUp(self):
super(FunctionalTestBase, self).setUp()
if not os.path.exists(TEST_CFG):
raise Exception(
"Unable to run the write tests without a test.ini in that defines an access_token with write privs."
)
cfg = SafeConfigParser()
with open(TEST_CFG) as fp:
cfg.readfp(fp, 'test.ini')
access_token = cfg.get('write_tests', 'access_token')
try:
activity_id = cfg.get('activity_tests', 'activity_id')
except NoOptionError:
activity_id = None
self.client = Client(access_token=access_token)
self.activity_id = activity_id
def parse_config_file(filename):
"""Parse INI files containing IMAP connection details.
Used by livetest.py and interact.py
"""
parser = SafeConfigParser(get_string_config_defaults())
with open(filename, 'r') as fh:
parser.readfp(fh)
conf = _read_config_section(parser, "DEFAULT")
if conf.expect_failure:
raise ValueError("expect_failure should not be set for the DEFAULT section")
conf.alternates = {}
for section in parser.sections():
conf.alternates[section] = _read_config_section(parser, section)
return conf
def parse_config_file(filename):
"""Parse INI files containing IMAP connection details.
Used by livetest.py and interact.py
"""
parser = SafeConfigParser(get_string_config_defaults())
with open(filename, 'r') as fh:
parser.readfp(fh)
conf = _read_config_section(parser, "DEFAULT")
if conf.expect_failure:
raise ValueError("expect_failure should not be set for the DEFAULT section")
conf.alternates = {}
for section in parser.sections():
conf.alternates[section] = _read_config_section(parser, section)
return conf
def parse(self):
"""
:return:
"""
parser = SafeConfigParser()
parser.readfp(StringIO(self.obj.content))
for section in parser.sections():
try:
content = parser.get(section=section, option="deps")
for n, line in enumerate(content.splitlines()):
if self.is_marked_line(line):
continue
if line:
req = RequirementsTXTLineParser.parse(line)
if req:
req.dependency_type = self.obj.file_type
self.obj.dependencies.append(req)
except NoOptionError:
pass
def test_on_start_no_ssl(self, SSLDomain):
confdata = six.StringIO("""[broker]
urls = amqp://broker1.example.com:5672 amqp://broker2.example.com:5672
topic_prefix = koji
connect_timeout = 10
send_timeout = 60
""")
if six.PY2:
conf = SafeConfigParser()
conf.readfp(confdata)
else:
conf = ConfigParser()
conf.read_file(confdata)
handler = protonmsg.TimeoutHandler('amqp://broker1.example.com:5672', [], conf)
event = MagicMock()
handler.on_start(event)
event.container.connect.assert_called_once_with(url='amqp://broker1.example.com:5672',
reconnect=False,
ssl_domain=None)
self.assertEqual(SSLDomain.call_count, 0)
def configure(filename=None):
"""This function gives to the user application a chance to define where
configuration file should live. Subsequent calls to this function will have
no effect, unless you call :func:`reconfigure`.
:param str filename: Full path to configuration file.
"""
global retry
if getattr(configure, '_configured', False):
return
filename = filename or DEFAULT_CONFIG_FILENAME
_ensure_directory(filename)
parser = SafeConfigParser()
if os.path.isfile(filename):
with open(filename, 'rt') as fp:
parser.readfp(fp)
if not parser.has_section(RETRY_SECTION):
parser.add_section(RETRY_SECTION)
parser.set(RETRY_SECTION, 'max_tries',
str(constants.BACKOFF_DEFAULT_MAXTRIES))
parser.set(RETRY_SECTION, 'delay',
str(constants.BACKOFF_DEFAULT_DELAY))
parser.set(RETRY_SECTION, 'factor',
str(constants.BACKOFF_DEFAULT_FACTOR))
with open(filename, 'wt') as fp:
parser.write(fp)
retry = RetrySettings(max_tries=parser.getint(RETRY_SECTION, 'max_tries'),
delay=parser.getint(RETRY_SECTION, 'delay'),
factor=parser.getint(RETRY_SECTION, 'factor'))
setattr(configure, '_configured', True)
setattr(configure, '_configured_filename', filename)
def _update_settings_from_file(section, settings):
tries = 0
current_directory = os.path.normpath(os.getcwd())
config_file = None
while current_directory and tries < MAX_CONFIG_SEARCH_DEPTH:
potential_path = os.path.join(current_directory, 'setup.cfg')
if os.path.exists(potential_path):
config_file = potential_path
break
new_directory = os.path.split(current_directory)[0]
if current_directory == new_directory:
break
current_directory = new_directory
tries += 1
if config_file and os.path.exists(config_file):
with open(config_file, 'rU') as fp:
config = SafeConfigParser()
config.readfp(fp)
if config.has_section('tool:multilint'):
settings.update(sanitize(config.items('tool:multilint')))
def _update_settings_from_file(section, settings):
tries = 0
current_directory = os.path.normpath(os.getcwd())
config_file = None
while current_directory and tries < MAX_CONFIG_SEARCH_DEPTH:
potential_path = os.path.join(current_directory, 'setup.cfg')
if os.path.exists(potential_path):
config_file = potential_path
break
new_directory = os.path.split(current_directory)[0]
if current_directory == new_directory:
break
current_directory = new_directory
tries += 1
if config_file and os.path.exists(config_file):
with open(config_file, 'rU') as fp:
config = SafeConfigParser()
config.readfp(fp)
if config.has_section('tool:multilint'):
settings.update(sanitize(config.items('tool:multilint')))
def check_dlm_cfgfile():
"""Parse DLM config file"""
fname = "/etc/dlm/dlm.conf"
try:
with open(fname, 'r') as fp:
cfgs = '[dlm]\n' + fp.read()
except (OSError, IOError):
return False
cfg = SafeConfigParser()
try:
cfg.read_string(cfgs)
except AttributeError:
# Python2 ConfigParser doesn't have cfg.read_string
from cStringIO import StringIO
cfg.readfp(StringIO(cfgs))
if not cfg.has_option('dlm', 'protocol'):
return False
proto = cfg.get('dlm', 'protocol').lower()
return proto in ['sctp', 'detect', '1', '2']
def parse_config_file(filename):
"""
Parses a configuration file and returns a settings dictionary.
Args:
filename (str): File to read configuration settings from.
Returns:
dict: A dictionary of settings options.
"""
parser = SafeConfigParser()
with open(filename) as fp:
parser.readfp(fp)
settings = {
section: {
item[0]: _parse_config_val(item[1])
for item in parser.items(section)
}
for section in parser.sections()
}
return settings
def __init__(self, email=None, api_key=None, config_file=None,
verbose=False, retry_on_errors=True,
site=None, client=None,
cert_bundle=None, insecure=None):
if client is None:
client = _default_client()
if config_file is None:
config_file = get_default_config_filename()
if os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get("api", "insecure").lower()
if insecure_setting == "true":
insecure = True
elif insecure_setting == "false":
insecure = False
else:
raise RuntimeError("insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
% (insecure_setting, config_file))
elif None in (api_key, email):
raise RuntimeError("api_key or email not specified and %s does not exist"
% (config_file,))
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
self.base_url = "https://api.zulip.com"
if self.base_url != "https://api.zulip.com" and not self.base_url.endswith("/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
self.tls_verification=False
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise RuntimeError("tls bundle '%s' does not exist"
%(cert_bundle,))
self.tls_verification=cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification=True
class Csw(object):
""" Base CSW server """
def __init__(self, rtconfig=None, env=None, version='3.0.0'):
""" Initialize CSW """
if not env:
self.environ = os.environ
else:
self.environ = env
self.context = config.StaticContext()
# Lazy load this when needed
# (it will permanently update global cfg namespaces)
self.sruobj = None
self.opensearchobj = None
self.oaipmhobj = None
# init kvp
self.kvp = {}
self.mode = 'csw'
self.asynchronous = False
self.soap = False
self.request = None
self.exception = False
self.status = 'OK'
self.profiles = None
self.manager = False
self.outputschemas = {}
self.mimetype = 'application/xml; charset=UTF-8'
self.encoding = 'UTF-8'
self.pretty_print = 0
self.domainquerytype = 'list'
self.orm = 'django'
self.language = {'639_code': 'en', 'text': 'english'}
self.process_time_start = time()
# define CSW implementation object (default CSW3)
self.iface = csw3.Csw3(server_csw=self)
self.request_version = version
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# load user configuration
try:
LOGGER.info('Loading user configuration')
if isinstance(rtconfig, SafeConfigParser): # serialized already
self.config = rtconfig
else:
self.config = SafeConfigParser()
if isinstance(rtconfig, dict): # dictionary
for section, options in rtconfig.items():
self.config.add_section(section)
for k, v in options.items():
self.config.set(section, k, v)
else: # configuration file
import codecs
with codecs.open(rtconfig, encoding='utf-8') as scp:
self.config.readfp(scp)
except Exception as err:
msg = 'Could not load configuration'
LOGGER.exception('%s %s: %s', msg, rtconfig, err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service', msg)
return
# set server.home safely
# TODO: make this more abstract
self.config.set(
'server', 'home',
os.path.dirname(os.path.join(os.path.dirname(__file__), '..'))
)
self.context.pycsw_home = self.config.get('server', 'home')
self.context.url = self.config.get('server', 'url')
log.setup_logger(self.config)
LOGGER.info('running configuration %s', rtconfig)
LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING'])
# set OGC schemas location
if not self.config.has_option('server', 'ogc_schemas_base'):
self.config.set('server', 'ogc_schemas_base',
self.context.ogc_schemas_base)
# set mimetype
if self.config.has_option('server', 'mimetype'):
self.mimetype = self.config.get('server', 'mimetype').encode()
# set encoding
if self.config.has_option('server', 'encoding'):
self.encoding = self.config.get('server', 'encoding')
# set domainquerytype
if self.config.has_option('server', 'domainquerytype'):
self.domainquerytype = self.config.get('server', 'domainquerytype')
# set XML pretty print
if (self.config.has_option('server', 'pretty_print') and
self.config.get('server', 'pretty_print') == 'true'):
self.pretty_print = 1
# set Spatial Ranking option
if (self.config.has_option('server', 'spatial_ranking') and
self.config.get('server', 'spatial_ranking') == 'true'):
util.ranking_enabled = True
# set language default
if self.config.has_option('server', 'language'):
try:
LOGGER.info('Setting language')
lang_code = self.config.get('server', 'language').split('-')[0]
self.language['639_code'] = lang_code
self.language['text'] = self.context.languages[lang_code]
except Exception as err:
LOGGER.exception('Could not set language: %s', err)
pass
LOGGER.debug('Configuration: %s.', self.config)
LOGGER.debug('Model: %s.', self.context.model)
# load user-defined mappings if they exist
if self.config.has_option('repository', 'mappings'):
# override default repository mappings
try:
import imp
module = self.config.get('repository', 'mappings')
if os.sep in module: # filepath
modulename = '%s' % os.path.splitext(module)[0].replace(
os.sep, '.')
mappings = imp.load_source(modulename, module)
else: # dotted name
mappings = __import__(module, fromlist=[''])
LOGGER.info('Loading custom repository mappings '
'from %s', module)
self.context.md_core_model = mappings.MD_CORE_MODEL
self.context.refresh_dc(mappings.MD_CORE_MODEL)
except Exception as err:
LOGGER.exception('Could not load custom mappings: %s', err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service',
'Could not load repository.mappings')
# load outputschemas
LOGGER.info('Loading outputschemas')
for osch in pycsw.plugins.outputschemas.__all__:
output_schema_module = __import__(
'pycsw.plugins.outputschemas.%s' % osch)
mod = getattr(output_schema_module.plugins.outputschemas, osch)
self.outputschemas[mod.NAMESPACE] = mod
LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas)
LOGGER.debug('Namespaces: %s', self.context.namespaces)
def expand_path(self, path):
""" return safe path for WSGI environments """
if 'local.app_root' in self.environ and not os.path.isabs(path):
return os.path.join(self.environ['local.app_root'], path)
else:
return path
def dispatch_wsgi(self):
""" WSGI handler """
if hasattr(self, 'response'):
return self._write_response()
LOGGER.debug('WSGI mode detected')
if self.environ['REQUEST_METHOD'] == 'POST':
try:
request_body_size = int(self.environ.get('CONTENT_LENGTH', 0))
except (ValueError):
request_body_size = 0
self.requesttype = 'POST'
self.request = self.environ['wsgi.input'].read(request_body_size)
LOGGER.debug('Request type: POST. Request:\n%s\n', self.request)
else: # it's a GET request
self.requesttype = 'GET'
self.request = wsgiref.util.request_uri(self.environ)
try:
query_part = splitquery(self.request)[-1]
self.kvp = dict(parse_qsl(query_part, keep_blank_values=True))
except AttributeError as err:
LOGGER.exception('Could not parse query string')
self.kvp = {}
LOGGER.debug('Request type: GET. Request:\n%s\n', self.request)
return self.dispatch()
def opensearch(self):
""" enable OpenSearch """
if not self.opensearchobj:
self.opensearchobj = opensearch.OpenSearch(self.context)
return self.opensearchobj
def sru(self):
""" enable SRU """
if not self.sruobj:
self.sruobj = sru.Sru(self.context)
return self.sruobj
def oaipmh(self):
""" enable OAI-PMH """
if not self.oaipmhobj:
self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config)
return self.oaipmhobj
def dispatch(self, writer=sys.stdout, write_headers=True):
""" Handle incoming HTTP request """
error = 0
if self.requesttype == 'GET':
self.kvp = self.normalize_kvp(self.kvp)
version_202 = ('version' in self.kvp and
self.kvp['version'] == '2.0.2')
accept_version_202 = ('acceptversions' in self.kvp and
'2.0.2' in self.kvp['acceptversions'])
if version_202 or accept_version_202:
self.request_version = '2.0.2'
elif self.requesttype == 'POST':
if self.request.find(b'cat/csw/2.0.2') != -1:
self.request_version = '2.0.2'
elif self.request.find(b'cat/csw/3.0') != -1:
self.request_version = '3.0.0'
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'sru'):
self.mode = 'sru'
self.request_version = '2.0.2'
LOGGER.info('SRU mode detected; processing request')
self.kvp = self.sru().request_sru2csw(self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'oaipmh'):
self.mode = 'oaipmh'
self.request_version = '2.0.2'
LOGGER.info('OAI-PMH mode detected; processing request.')
self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k)
self.kvp = self.oaipmh().request(self.kvp)
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# configure transaction support, if specified in config
self._gen_manager()
namespaces = self.context.namespaces
ops = self.context.model['operations']
constraints = self.context.model['constraints']
# generate domain model
# NOTE: We should probably avoid this sort of mutable state for WSGI
if 'GetDomain' not in ops:
ops['GetDomain'] = self.context.gen_domains()
# generate distributed search model, if specified in config
if self.config.has_option('server', 'federatedcatalogues'):
LOGGER.info('Configuring distributed search')
constraints['FederatedCatalogues'] = {'values': []}
for fedcat in self.config.get('server',
'federatedcatalogues').split(','):
LOGGER.debug('federated catalogue: %s', fedcat)
constraints['FederatedCatalogues']['values'].append(fedcat)
for key, value in self.outputschemas.items():
get_records_params = ops['GetRecords']['parameters']
get_records_params['outputSchema']['values'].append(
value.NAMESPACE)
get_records_by_id_params = ops['GetRecordById']['parameters']
get_records_by_id_params['outputSchema']['values'].append(
value.NAMESPACE)
if 'Harvest' in ops:
harvest_params = ops['Harvest']['parameters']
harvest_params['ResourceType']['values'].append(
value.NAMESPACE)
LOGGER.info('Setting MaxRecordDefault')
if self.config.has_option('server', 'maxrecords'):
constraints['MaxRecordDefault']['values'] = [
self.config.get('server', 'maxrecords')]
# load profiles
if self.config.has_option('server', 'profiles'):
self.profiles = pprofile.load_profiles(
os.path.join('pycsw', 'plugins', 'profiles'),
pprofile.Profile,
self.config.get('server', 'profiles')
)
for prof in self.profiles['plugins'].keys():
tmp = self.profiles['plugins'][prof](self.context.model,
namespaces,
self.context)
key = tmp.outputschema # to ref by outputschema
self.profiles['loaded'][key] = tmp
self.profiles['loaded'][key].extend_core(self.context.model,
namespaces,
self.config)
LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys()))
# init repository
# look for tablename, set 'records' as default
if not self.config.has_option('repository', 'table'):
self.config.set('repository', 'table', 'records')
repo_filter = None
if self.config.has_option('repository', 'filter'):
repo_filter = self.config.get('repository', 'filter')
if self.config.has_option('repository', 'source'): # load custom repository
rs = self.config.get('repository', 'source')
rs_modname, rs_clsname = rs.rsplit('.', 1)
rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname])
rs_cls = getattr(rs_mod, rs_clsname)
try:
self.repository = rs_cls(self.context, repo_filter)
LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype)
except Exception as err:
msg = 'Could not load custom repository %s: %s' % (rs, err)
LOGGER.exception(msg)
error = 1
code = 'NoApplicableCode'
locator = 'service'
text = 'Could not initialize repository. Check server logs'
else: # load default repository
self.orm = 'sqlalchemy'
from pycsw.core import repository
try:
LOGGER.info('Loading default repository')
self.repository = repository.Repository(
self.config.get('repository', 'database'),
self.context,
self.environ.get('local.app_root', None),
self.config.get('repository', 'table'),
repo_filter
)
LOGGER.debug(
'Repository loaded (local): %s.' % self.repository.dbtype)
except Exception as err:
msg = 'Could not load repository (local): %s' % err
LOGGER.exception(msg)
error = 1
code = 'NoApplicableCode'
locator = 'service'
text = 'Could not initialize repository. Check server logs'
if self.requesttype == 'POST':
LOGGER.debug('HTTP POST request')
LOGGER.debug('CSW version: %s', self.iface.version)
self.kvp = self.iface.parse_postdata(self.request)
if isinstance(self.kvp, str): # it's an exception
error = 1
locator = 'service'
text = self.kvp
if (self.kvp.find('the document is not valid') != -1 or
self.kvp.find('document not well-formed') != -1):
code = 'NoApplicableCode'
else:
code = 'InvalidParameterValue'
LOGGER.debug('HTTP Headers:\n%s.', self.environ)
LOGGER.debug('Parsed request parameters: %s', self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'opensearch'):
self.mode = 'opensearch'
LOGGER.info('OpenSearch mode detected; processing request.')
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or
(len(self.kvp) == 1 and 'config' in self.kvp)):
LOGGER.info('Turning on default csw30:Capabilities for base URL')
self.kvp = {
'service': 'CSW',
'acceptversions': '3.0.0',
'request': 'GetCapabilities'
}
http_accept = self.environ.get('HTTP_ACCEPT', '')
if 'application/opensearchdescription+xml' in http_accept:
self.mode = 'opensearch'
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if error == 0:
# test for the basic keyword values (service, version, request)
basic_options = ['service', 'request']
request = self.kvp.get('request', '')
own_version_integer = util.get_version_integer(
self.request_version)
if self.request_version == '2.0.2':
basic_options.append('version')
if self.request_version == '3.0.0' and 'version' not in self.kvp and self.requesttype == 'POST':
if 'service' not in self.kvp:
self.kvp['service'] = 'CSW'
basic_options.append('service')
self.kvp['version'] = self.request_version
basic_options.append('version')
for k in basic_options:
if k not in self.kvp:
if (k in ['version', 'acceptversions'] and
request == 'GetCapabilities'):
pass
else:
error = 1
locator = k
code = 'MissingParameterValue'
text = 'Missing keyword: %s' % k
break
# test each of the basic keyword values
if error == 0:
# test service
if self.kvp['service'] != 'CSW':
error = 1
locator = 'service'
code = 'InvalidParameterValue'
text = 'Invalid value for service: %s.\
Value MUST be CSW' % self.kvp['service']
# test version
kvp_version = self.kvp.get('version', '')
try:
kvp_version_integer = util.get_version_integer(kvp_version)
except Exception as err:
kvp_version_integer = 'invalid_value'
if (request != 'GetCapabilities' and
kvp_version_integer != own_version_integer):
error = 1
locator = 'version'
code = 'InvalidParameterValue'
text = ('Invalid value for version: %s. Value MUST be '
'2.0.2 or 3.0.0' % kvp_version)
# check for GetCapabilities acceptversions
if 'acceptversions' in self.kvp:
for vers in self.kvp['acceptversions'].split(','):
vers_integer = util.get_version_integer(vers)
if vers_integer == own_version_integer:
break
else:
error = 1
locator = 'acceptversions'
code = 'VersionNegotiationFailed'
text = ('Invalid parameter value in '
'acceptversions: %s. Value MUST be '
'2.0.2 or 3.0.0' %
self.kvp['acceptversions'])
# test request
if self.kvp['request'] not in \
self.context.model['operations']:
error = 1
locator = 'request'
if request in ['Transaction', 'Harvest']:
code = 'OperationNotSupported'
text = '%s operations are not supported' % request
else:
code = 'InvalidParameterValue'
text = 'Invalid value for request: %s' % request
if error == 1: # return an ExceptionReport
LOGGER.error('basic service options error: %s, %s, %s', code, locator, text)
self.response = self.iface.exceptionreport(code, locator, text)
else: # process per the request value
if 'responsehandler' in self.kvp:
# set flag to process asynchronously
import threading
self.asynchronous = True
request_id = self.kvp.get('requestid', None)
if request_id is None:
import uuid
self.kvp['requestid'] = str(uuid.uuid4())
if self.kvp['request'] == 'GetCapabilities':
self.response = self.iface.getcapabilities()
elif self.kvp['request'] == 'DescribeRecord':
self.response = self.iface.describerecord()
elif self.kvp['request'] == 'GetDomain':
self.response = self.iface.getdomain()
elif self.kvp['request'] == 'GetRecords':
if self.asynchronous: # process asynchronously
threading.Thread(target=self.iface.getrecords).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.getrecords()
elif self.kvp['request'] == 'GetRecordById':
self.response = self.iface.getrecordbyid()
elif self.kvp['request'] == 'GetRepositoryItem':
self.response = self.iface.getrepositoryitem()
elif self.kvp['request'] == 'Transaction':
self.response = self.iface.transaction()
elif self.kvp['request'] == 'Harvest':
if self.asynchronous: # process asynchronously
threading.Thread(target=self.iface.harvest).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.harvest()
else:
self.response = self.iface.exceptionreport(
'InvalidParameterValue', 'request',
'Invalid request parameter: %s' % self.kvp['request']
)
LOGGER.info('Request processed')
if self.mode == 'sru':
LOGGER.info('SRU mode detected; processing response.')
self.response = self.sru().response_csw2sru(self.response,
self.environ)
elif self.mode == 'opensearch':
LOGGER.info('OpenSearch mode detected; processing response.')
self.response = self.opensearch().response_csw2opensearch(
self.response, self.config)
elif self.mode == 'oaipmh':
LOGGER.info('OAI-PMH mode detected; processing response.')
self.response = self.oaipmh().response(
self.response, self.oaiargs, self.repository,
self.config.get('server', 'url')
)
return self._write_response()
def getcapabilities(self):
""" Handle GetCapabilities request """
return self.iface.getcapabilities()
def describerecord(self):
""" Handle DescribeRecord request """
return self.iface.describerecord()
def getdomain(self):
""" Handle GetDomain request """
return self.iface.getdomain()
def getrecords(self):
""" Handle GetRecords request """
return self.iface.getrecords()
def getrecordbyid(self, raw=False):
""" Handle GetRecordById request """
return self.iface.getrecordbyid(raw)
def getrepositoryitem(self):
""" Handle GetRepositoryItem request """
return self.iface.getrepositoryitem()
def transaction(self):
""" Handle Transaction request """
return self.iface.transaction()
def harvest(self):
""" Handle Harvest request """
return self.iface.harvest()
def _write_response(self):
""" Generate response """
# set HTTP response headers and XML declaration
xmldecl = ''
appinfo = ''
LOGGER.info('Writing response.')
if hasattr(self, 'soap') and self.soap:
self._gen_soap_wrapper()
if etree.__version__ >= '3.5.0': # remove superfluous namespaces
etree.cleanup_namespaces(self.response,
keep_ns_prefixes=self.context.keep_ns_prefixes)
response = etree.tostring(self.response,
pretty_print=self.pretty_print,
encoding='unicode')
if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and
self.kvp['outputformat'] == 'application/json'):
self.contenttype = self.kvp['outputformat']
from pycsw.core.formats import fmt_json
response = fmt_json.xml2json(response,
self.context.namespaces,
self.pretty_print)
else: # it's XML
if 'outputformat' in self.kvp:
self.contenttype = self.kvp['outputformat']
else:
self.contenttype = self.mimetype
xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>'
'\n' % self.encoding)
appinfo = '<!-- pycsw %s -->\n' % self.context.version
if isinstance(self.contenttype, bytes):
self.contenttype = self.contenttype.decode()
s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding)
LOGGER.debug('Response code: %s',
self.context.response_codes[self.status])
LOGGER.debug('Response:\n%s', s)
return [self.context.response_codes[self.status], s]
def _gen_soap_wrapper(self):
""" Generate SOAP wrapper """
LOGGER.info('Writing SOAP wrapper.')
node = etree.Element(
util.nspath_eval('soapenv:Envelope', self.context.namespaces),
nsmap=self.context.namespaces
)
schema_location_ns = util.nspath_eval('xsi:schemaLocation',
self.context.namespaces)
node.attrib[schema_location_ns] = '%s %s' % (
self.context.namespaces['soapenv'],
self.context.namespaces['soapenv']
)
node2 = etree.SubElement(
node, util.nspath_eval('soapenv:Body', self.context.namespaces))
if self.exception:
node3 = etree.SubElement(
node2,
util.nspath_eval('soapenv:Fault', self.context.namespaces)
)
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Code', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Value', self.context.namespaces)
).text = 'soap:Server'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Reason', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Text', self.context.namespaces)
).text = 'A server exception was encountered.'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Detail', self.context.namespaces)
)
node4.append(self.response)
else:
node2.append(self.response)
self.response = node
def _gen_manager(self):
""" Update self.context.model with CSW-T advertising """
if (self.config.has_option('manager', 'transactions') and
self.config.get('manager', 'transactions') == 'true'):
self.manager = True
self.context.model['operations_order'].append('Transaction')
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {}
}
schema_values = [
'http://www.opengis.net/cat/csw/2.0.2',
'http://www.opengis.net/cat/csw/3.0',
'http://www.opengis.net/wms',
'http://www.opengis.net/wmts/1.0',
'http://www.opengis.net/wfs',
'http://www.opengis.net/wfs/2.0',
'http://www.opengis.net/wcs',
'http://www.opengis.net/wps/1.0.0',
'http://www.opengis.net/sos/1.0',
'http://www.opengis.net/sos/2.0',
'http://www.isotc211.org/2005/gmi',
'urn:geoss:waf',
]
self.context.model['operations_order'].append('Harvest')
self.context.model['operations']['Harvest'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'ResourceType': {'values': schema_values}
}
}
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'TransactionSchemas': {'values': sorted(schema_values)}
}
}
self.csw_harvest_pagesize = 10
if self.config.has_option('manager', 'csw_harvest_pagesize'):
self.csw_harvest_pagesize = int(
self.config.get('manager', 'csw_harvest_pagesize'))
def _test_manager(self):
""" Verify that transactions are allowed """
if self.config.get('manager', 'transactions') != 'true':
raise RuntimeError('CSW-T interface is disabled')
""" get the client first forwarded ip """
if 'HTTP_X_FORWARDED_FOR' in self.environ:
ipaddress = self.environ['HTTP_X_FORWARDED_FOR'].split(',')[0].strip()
else:
ipaddress = self.environ['REMOTE_ADDR']
if not self.config.has_option('manager', 'allowed_ips') or \
(self.config.has_option('manager', 'allowed_ips') and not
util.ipaddress_in_whitelist(ipaddress,
self.config.get('manager', 'allowed_ips').split(','))):
raise RuntimeError(
'CSW-T operations not allowed for this IP address: %s' % ipaddress)
def _cql_update_queryables_mappings(self, cql, mappings):
""" Transform CQL query's properties to underlying DB columns """
LOGGER.debug('Raw CQL text = %s', cql)
LOGGER.debug(str(list(mappings.keys())))
if cql is not None:
for key in mappings.keys():
try:
cql = cql.replace(key, mappings[key]['dbcol'])
except:
cql = cql.replace(key, mappings[key])
LOGGER.debug('Interpolated CQL text = %s.', cql)
return cql
def _process_responsehandler(self, xml):
""" Process response handler """
if self.kvp['responsehandler'] is not None:
LOGGER.info('Processing responsehandler %s' %
self.kvp['responsehandler'])
uprh = urlparse(self.kvp['responsehandler'])
if uprh.scheme == 'mailto': # email
import smtplib
LOGGER.debug('Email detected')
smtp_host = 'localhost'
if self.config.has_option('server', 'smtp_host'):
smtp_host = self.config.get('server', 'smtp_host')
body = ('Subject: pycsw %s results\n\n%s' %
(self.kvp['request'], xml))
try:
LOGGER.info('Sending email')
msg = smtplib.SMTP(smtp_host)
msg.sendmail(
self.config.get('metadata:main', 'contact_email'),
uprh.path, body
)
msg.quit()
LOGGER.debug('Email sent successfully.')
except Exception as err:
LOGGER.exception('Error processing email')
elif uprh.scheme == 'ftp':
import ftplib
LOGGER.debug('FTP detected.')
try:
LOGGER.info('Sending to FTP server.')
ftp = ftplib.FTP(uprh.hostname)
if uprh.username is not None:
ftp.login(uprh.username, uprh.password)
ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml))
ftp.quit()
LOGGER.debug('FTP sent successfully.')
except Exception as err:
LOGGER.exception('Error processing FTP')
@staticmethod
def normalize_kvp(kvp):
"""Normalize Key Value Pairs.
This method will transform all keys to lowercase and leave values
unchanged, as specified in the CSW standard (see for example note
C on Table 62 - KVP Encoding for DescribeRecord operation request
of the CSW standard version 2.0.2)
:arg kvp: a mapping with Key Value Pairs
:type kvp: dict
:returns: A new dictionary with normalized parameters
"""
result = dict()
for name, value in kvp.items():
result[name.lower()] = value
return result
def __init__(self,
email=None,
api_key=None,
config_file=None,
verbose=False,
retry_on_errors=True,
site=None,
client=None,
cert_bundle=None,
insecure=None,
client_cert=None,
client_cert_key=None):
# type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[str], Optional[str]) -> None
if client is None:
client = _default_client()
# Normalize user-specified path
if config_file is not None:
config_file = os.path.abspath(os.path.expanduser(config_file))
# Fill values from Environment Variables if not available in Constructor
if config_file is None:
config_file = os.environ.get("ZULIP_CONFIG")
if api_key is None:
api_key = os.environ.get("ZULIP_API_KEY")
if email is None:
email = os.environ.get("ZULIP_EMAIL")
if site is None:
site = os.environ.get("ZULIP_SITE")
if client_cert is None:
client_cert = os.environ.get("ZULIP_CERT")
if client_cert_key is None:
client_cert_key = os.environ.get("ZULIP_CERT_KEY")
if cert_bundle is None:
cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")
if insecure is None:
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = os.environ.get('ZULIP_ALLOW_INSECURE')
if insecure_setting is not None:
insecure = validate_boolean_field(insecure_setting)
if insecure is None:
raise ZulipError(
"The ZULIP_ALLOW_INSECURE environment "
"variable is set to '{}', it must be "
"'true' or 'false'".format(insecure_setting))
if config_file is None:
config_file = get_default_config_filename()
if config_file is not None and os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if client_cert is None and config.has_option("api", "client_cert"):
client_cert = config.get("api", "client_cert")
if client_cert_key is None and config.has_option(
"api", "client_cert_key"):
client_cert_key = config.get("api", "client_cert_key")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get('api', 'insecure')
insecure = validate_boolean_field(insecure_setting)
if insecure is None:
raise ZulipError(
"insecure is set to '{}', it must be "
"'true' or 'false' if it is used in {}".format(
insecure_setting, config_file))
elif None in (api_key, email):
raise ConfigNotFoundError(
"api_key or email not specified and file %s does not exist" %
(config_file, ))
assert (api_key is not None and email is not None)
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if site.startswith("localhost"):
site = "http://" + site
elif not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
raise MissingURLError(
"Missing Zulip server URL; specify via --site or ~/.zuliprc.")
if not self.base_url.endswith("/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
logger.warning('Insecure mode enabled. The server\'s SSL/TLS '
'certificate will not be validated, making the '
'HTTPS connection potentially insecure')
self.tls_verification = False # type: Union[bool, str]
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise ConfigNotFoundError("tls bundle '%s' does not exist" %
(cert_bundle, ))
self.tls_verification = cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification = True
if client_cert is None:
if client_cert_key is not None:
raise ConfigNotFoundError(
"client cert key '%s' specified, but no client cert public part provided"
% (client_cert_key, ))
else: # we have a client cert
if not os.path.isfile(client_cert):
raise ConfigNotFoundError("client cert '%s' does not exist" %
(client_cert, ))
if client_cert_key is not None:
if not os.path.isfile(client_cert_key):
raise ConfigNotFoundError(
"client cert key '%s' does not exist" %
(client_cert_key, ))
self.client_cert = client_cert
self.client_cert_key = client_cert_key
self.session = None # type: Optional[requests.Session]
self.has_connected = False
class Csw(object):
""" Base CSW server """
def __init__(self, rtconfig=None, env=None, version='3.0.0'):
""" Initialize CSW """
if not env:
self.environ = os.environ
else:
self.environ = env
self.context = config.StaticContext()
# Lazy load this when needed
# (it will permanently update global cfg namespaces)
self.sruobj = None
self.opensearchobj = None
self.oaipmhobj = None
# init kvp
self.kvp = {}
self.mode = 'csw'
self.async = False
self.soap = False
self.request = None
self.exception = False
self.status = 'OK'
self.profiles = None
self.manager = False
self.outputschemas = {}
self.mimetype = 'application/xml; charset=UTF-8'
self.encoding = 'UTF-8'
self.pretty_print = 0
self.domainquerytype = 'list'
self.orm = 'django'
self.language = {'639_code': 'en', 'text': 'english'}
self.process_time_start = time()
# define CSW implementation object (default CSW3)
self.iface = csw3.Csw3(server_csw=self)
self.request_version = version
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# load user configuration
try:
LOGGER.info('Loading user configuration')
if isinstance(rtconfig, SafeConfigParser): # serialized already
self.config = rtconfig
else:
self.config = SafeConfigParser()
if isinstance(rtconfig, dict): # dictionary
for section, options in rtconfig.items():
self.config.add_section(section)
for k, v in options.items():
self.config.set(section, k, v)
else: # configuration file
import codecs
with codecs.open(rtconfig, encoding='utf-8') as scp:
self.config.readfp(scp)
except Exception as err:
LOGGER.exception('Could not load user configuration: %s', err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service',
'Error opening configuration %s' % rtconfig
)
return
# set server.home safely
# TODO: make this more abstract
self.config.set(
'server', 'home',
os.path.dirname(os.path.join(os.path.dirname(__file__), '..'))
)
self.context.pycsw_home = self.config.get('server', 'home')
self.context.url = self.config.get('server', 'url')
log.setup_logger(self.config)
LOGGER.info('running configuration %s', rtconfig)
LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING'])
# set OGC schemas location
if not self.config.has_option('server', 'ogc_schemas_base'):
self.config.set('server', 'ogc_schemas_base',
self.context.ogc_schemas_base)
# set mimetype
if self.config.has_option('server', 'mimetype'):
self.mimetype = self.config.get('server', 'mimetype').encode()
# set encoding
if self.config.has_option('server', 'encoding'):
self.encoding = self.config.get('server', 'encoding')
# set domainquerytype
if self.config.has_option('server', 'domainquerytype'):
self.domainquerytype = self.config.get('server', 'domainquerytype')
# set XML pretty print
if (self.config.has_option('server', 'pretty_print') and
self.config.get('server', 'pretty_print') == 'true'):
self.pretty_print = 1
# set Spatial Ranking option
if (self.config.has_option('server', 'spatial_ranking') and
self.config.get('server', 'spatial_ranking') == 'true'):
util.ranking_enabled = True
# set language default
if self.config.has_option('server', 'language'):
try:
LOGGER.info('Setting language')
lang_code = self.config.get('server', 'language').split('-')[0]
self.language['639_code'] = lang_code
self.language['text'] = self.context.languages[lang_code]
except Exception as err:
LOGGER.exception('Could not set language: %s', err)
pass
LOGGER.debug('Configuration: %s.', self.config)
LOGGER.debug('Model: %s.', self.context.model)
# load user-defined mappings if they exist
if self.config.has_option('repository', 'mappings'):
# override default repository mappings
try:
import imp
module = self.config.get('repository', 'mappings')
if '/' in module: # filepath
modulename = '%s' % os.path.splitext(module)[0].replace(
os.sep, '.')
mappings = imp.load_source(modulename, module)
else: # dotted name
mappings = __import__(module, fromlist=[''])
LOGGER.info('Loading custom repository mappings '
'from %s', module)
self.context.md_core_model = mappings.MD_CORE_MODEL
self.context.refresh_dc(mappings.MD_CORE_MODEL)
except Exception as err:
LOGGER.exception('Could not load custom mappings: %s', err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service',
'Could not load repository.mappings')
# load outputschemas
LOGGER.info('Loading outputschemas')
for osch in pycsw.plugins.outputschemas.__all__:
output_schema_module = __import__(
'pycsw.plugins.outputschemas.%s' % osch)
mod = getattr(output_schema_module.plugins.outputschemas, osch)
self.outputschemas[mod.NAMESPACE] = mod
LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas)
LOGGER.debug('Namespaces: %s', self.context.namespaces)
def expand_path(self, path):
""" return safe path for WSGI environments """
if 'local.app_root' in self.environ and not os.path.isabs(path):
return os.path.join(self.environ['local.app_root'], path)
else:
return path
def dispatch_wsgi(self):
""" WSGI handler """
if hasattr(self, 'response'):
return self._write_response()
LOGGER.debug('WSGI mode detected')
if self.environ['REQUEST_METHOD'] == 'POST':
try:
request_body_size = int(self.environ.get('CONTENT_LENGTH', 0))
except (ValueError):
request_body_size = 0
self.requesttype = 'POST'
self.request = self.environ['wsgi.input'].read(request_body_size)
LOGGER.debug('Request type: POST. Request:\n%s\n', self.request)
else: # it's a GET request
self.requesttype = 'GET'
self.request = wsgiref.util.request_uri(self.environ)
try:
query_part = splitquery(self.request)[-1]
self.kvp = dict(parse_qsl(query_part, keep_blank_values=True))
except AttributeError as err:
LOGGER.exception('Could not parse query string')
self.kvp = {}
LOGGER.debug('Request type: GET. Request:\n%s\n', self.request)
return self.dispatch()
def opensearch(self):
""" enable OpenSearch """
if not self.opensearchobj:
self.opensearchobj = opensearch.OpenSearch(self.context)
return self.opensearchobj
def sru(self):
""" enable SRU """
if not self.sruobj:
self.sruobj = sru.Sru(self.context)
return self.sruobj
def oaipmh(self):
""" enable OAI-PMH """
if not self.oaipmhobj:
self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config)
return self.oaipmhobj
def dispatch(self, writer=sys.stdout, write_headers=True):
""" Handle incoming HTTP request """
if self.requesttype == 'GET':
self.kvp = self.normalize_kvp(self.kvp)
version_202 = ('version' in self.kvp and
self.kvp['version'] == '2.0.2')
accept_version_202 = ('acceptversions' in self.kvp and
'2.0.2' in self.kvp['acceptversions'])
if version_202 or accept_version_202:
self.request_version = '2.0.2'
elif self.requesttype == 'POST':
if self.request.find(b'2.0.2') != -1:
self.request_version = '2.0.2'
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'sru'):
self.mode = 'sru'
self.request_version = '2.0.2'
LOGGER.info('SRU mode detected; processing request')
self.kvp = self.sru().request_sru2csw(self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'oaipmh'):
self.mode = 'oaipmh'
self.request_version = '2.0.2'
LOGGER.info('OAI-PMH mode detected; processing request.')
self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k)
self.kvp = self.oaipmh().request(self.kvp)
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# configure transaction support, if specified in config
self._gen_manager()
namespaces = self.context.namespaces
ops = self.context.model['operations']
constraints = self.context.model['constraints']
# generate domain model
# NOTE: We should probably avoid this sort of mutable state for WSGI
if 'GetDomain' not in ops:
ops['GetDomain'] = self.context.gen_domains()
# generate distributed search model, if specified in config
if self.config.has_option('server', 'federatedcatalogues'):
LOGGER.info('Configuring distributed search')
constraints['FederatedCatalogues'] = {'values': []}
for fedcat in self.config.get('server',
'federatedcatalogues').split(','):
LOGGER.debug('federated catalogue: %s', fedcat)
constraints['FederatedCatalogues']['values'].append(fedcat)
for key, value in self.outputschemas.items():
get_records_params = ops['GetRecords']['parameters']
get_records_params['outputSchema']['values'].append(
value.NAMESPACE)
get_records_by_id_params = ops['GetRecordById']['parameters']
get_records_by_id_params['outputSchema']['values'].append(
value.NAMESPACE)
if 'Harvest' in ops:
harvest_params = ops['Harvest']['parameters']
harvest_params['ResourceType']['values'].append(
value.NAMESPACE)
LOGGER.info('Setting MaxRecordDefault')
if self.config.has_option('server', 'maxrecords'):
constraints['MaxRecordDefault']['values'] = [
self.config.get('server', 'maxrecords')]
# load profiles
if self.config.has_option('server', 'profiles'):
self.profiles = pprofile.load_profiles(
os.path.join('pycsw', 'plugins', 'profiles'),
pprofile.Profile,
self.config.get('server', 'profiles')
)
for prof in self.profiles['plugins'].keys():
tmp = self.profiles['plugins'][prof](self.context.model,
namespaces,
self.context)
key = tmp.outputschema # to ref by outputschema
self.profiles['loaded'][key] = tmp
self.profiles['loaded'][key].extend_core(self.context.model,
namespaces,
self.config)
LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys()))
# init repository
# look for tablename, set 'records' as default
if not self.config.has_option('repository', 'table'):
self.config.set('repository', 'table', 'records')
repo_filter = None
if self.config.has_option('repository', 'filter'):
repo_filter = self.config.get('repository', 'filter')
if self.config.has_option('repository', 'source'): # load custom repository
rs = self.config.get('repository', 'source')
rs_modname, rs_clsname = rs.rsplit('.', 1)
rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname])
rs_cls = getattr(rs_mod, rs_clsname)
try:
self.repository = rs_cls(self.context, repo_filter)
LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype)
except Exception as err:
msg = 'Could not load custom repository'
LOGGER.exception(msg)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service', msg)
else: # load default repository
self.orm = 'sqlalchemy'
from pycsw.core import repository
try:
LOGGER.info('Loading default repository')
self.repository = repository.Repository(
self.config.get('repository', 'database'),
self.context,
self.environ.get('local.app_root', None),
self.config.get('repository', 'table'),
repo_filter
)
LOGGER.debug(
'Repository loaded (local): %s.' % self.repository.dbtype)
except Exception as err:
msg = 'Could not load repository (local)'
LOGGER.exception(msg)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service', msg)
if self.requesttype == 'POST':
LOGGER.debug('HTTP POST request')
LOGGER.debug('CSW version: %s', self.iface.version)
self.kvp = self.iface.parse_postdata(self.request)
error = 0
if isinstance(self.kvp, str): # it's an exception
error = 1
locator = 'service'
text = self.kvp
if (self.kvp.find('the document is not valid') != -1 or
self.kvp.find('document not well-formed') != -1):
code = 'NoApplicableCode'
else:
code = 'InvalidParameterValue'
LOGGER.debug('HTTP Headers:\n%s.', self.environ)
LOGGER.debug('Parsed request parameters: %s', self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'opensearch'):
self.mode = 'opensearch'
LOGGER.info('OpenSearch mode detected; processing request.')
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or
(len(self.kvp) == 1 and 'config' in self.kvp)):
LOGGER.info('Turning on default csw30:Capabilities for base URL')
self.kvp = {
'service': 'CSW',
'acceptversions': '3.0.0',
'request': 'GetCapabilities'
}
http_accept = self.environ.get('HTTP_ACCEPT', '')
if 'application/opensearchdescription+xml' in http_accept:
self.mode = 'opensearch'
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if error == 0:
# test for the basic keyword values (service, version, request)
basic_options = ['service', 'request']
request = self.kvp.get('request', '')
own_version_integer = util.get_version_integer(
self.request_version)
if self.request_version == '2.0.2':
basic_options.append('version')
for k in basic_options:
if k not in self.kvp:
if (k in ['version', 'acceptversions'] and
request == 'GetCapabilities'):
pass
else:
error = 1
locator = k
code = 'MissingParameterValue'
text = 'Missing keyword: %s' % k
break
# test each of the basic keyword values
if error == 0:
# test service
if self.kvp['service'] != 'CSW':
error = 1
locator = 'service'
code = 'InvalidParameterValue'
text = 'Invalid value for service: %s.\
Value MUST be CSW' % self.kvp['service']
# test version
kvp_version = self.kvp.get('version', '')
try:
kvp_version_integer = util.get_version_integer(kvp_version)
except Exception as err:
kvp_version_integer = 'invalid_value'
if (request != 'GetCapabilities' and
kvp_version_integer != own_version_integer):
error = 1
locator = 'version'
code = 'InvalidParameterValue'
text = ('Invalid value for version: %s. Value MUST be '
'2.0.2 or 3.0.0' % kvp_version)
# check for GetCapabilities acceptversions
if 'acceptversions' in self.kvp:
for vers in self.kvp['acceptversions'].split(','):
vers_integer = util.get_version_integer(vers)
if vers_integer == own_version_integer:
break
else:
error = 1
locator = 'acceptversions'
code = 'VersionNegotiationFailed'
text = ('Invalid parameter value in '
'acceptversions: %s. Value MUST be '
'2.0.2 or 3.0.0' %
self.kvp['acceptversions'])
# test request
if self.kvp['request'] not in \
self.context.model['operations']:
error = 1
locator = 'request'
if request in ['Transaction', 'Harvest']:
code = 'OperationNotSupported'
text = '%s operations are not supported' % request
else:
code = 'InvalidParameterValue'
text = 'Invalid value for request: %s' % request
if error == 1: # return an ExceptionReport
LOGGER.error('basic service options error: %s, %s, %s', code, locator, text)
self.response = self.iface.exceptionreport(code, locator, text)
else: # process per the request value
if 'responsehandler' in self.kvp:
# set flag to process asynchronously
import threading
self.async = True
request_id = self.kvp.get('requestid', None)
if request_id is None:
import uuid
self.kvp['requestid'] = str(uuid.uuid4())
if self.kvp['request'] == 'GetCapabilities':
self.response = self.iface.getcapabilities()
elif self.kvp['request'] == 'DescribeRecord':
self.response = self.iface.describerecord()
elif self.kvp['request'] == 'GetDomain':
self.response = self.iface.getdomain()
elif self.kvp['request'] == 'GetRecords':
if self.async: # process asynchronously
threading.Thread(target=self.iface.getrecords).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.getrecords()
elif self.kvp['request'] == 'GetRecordById':
self.response = self.iface.getrecordbyid()
elif self.kvp['request'] == 'GetRepositoryItem':
self.response = self.iface.getrepositoryitem()
elif self.kvp['request'] == 'Transaction':
self.response = self.iface.transaction()
elif self.kvp['request'] == 'Harvest':
if self.async: # process asynchronously
threading.Thread(target=self.iface.harvest).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.harvest()
else:
self.response = self.iface.exceptionreport(
'InvalidParameterValue', 'request',
'Invalid request parameter: %s' % self.kvp['request']
)
LOGGER.info('Request processed')
if self.mode == 'sru':
LOGGER.info('SRU mode detected; processing response.')
self.response = self.sru().response_csw2sru(self.response,
self.environ)
elif self.mode == 'opensearch':
LOGGER.info('OpenSearch mode detected; processing response.')
self.response = self.opensearch().response_csw2opensearch(
self.response, self.config)
elif self.mode == 'oaipmh':
LOGGER.info('OAI-PMH mode detected; processing response.')
self.response = self.oaipmh().response(
self.response, self.oaiargs, self.repository,
self.config.get('server', 'url')
)
return self._write_response()
def getcapabilities(self):
""" Handle GetCapabilities request """
return self.iface.getcapabilities()
def describerecord(self):
""" Handle DescribeRecord request """
return self.iface.describerecord()
def getdomain(self):
""" Handle GetDomain request """
return self.iface.getdomain()
def getrecords(self):
""" Handle GetRecords request """
return self.iface.getrecords()
def getrecordbyid(self, raw=False):
""" Handle GetRecordById request """
return self.iface.getrecordbyid(raw)
def getrepositoryitem(self):
""" Handle GetRepositoryItem request """
return self.iface.getrepositoryitem()
def transaction(self):
""" Handle Transaction request """
return self.iface.transaction()
def harvest(self):
""" Handle Harvest request """
return self.iface.harvest()
def _write_response(self):
""" Generate response """
# set HTTP response headers and XML declaration
xmldecl = ''
appinfo = ''
LOGGER.info('Writing response.')
if hasattr(self, 'soap') and self.soap:
self._gen_soap_wrapper()
if etree.__version__ >= '3.5.0': # remove superfluous namespaces
etree.cleanup_namespaces(self.response,
keep_ns_prefixes=self.context.keep_ns_prefixes)
response = etree.tostring(self.response,
pretty_print=self.pretty_print,
encoding='unicode')
if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and
self.kvp['outputformat'] == 'application/json'):
self.contenttype = self.kvp['outputformat']
from pycsw.core.formats import fmt_json
response = fmt_json.xml2json(response,
self.context.namespaces,
self.pretty_print)
else: # it's XML
if 'outputformat' in self.kvp:
self.contenttype = self.kvp['outputformat']
else:
self.contenttype = self.mimetype
xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>'
'\n' % self.encoding)
appinfo = '<!-- pycsw %s -->\n' % self.context.version
if isinstance(self.contenttype, bytes):
self.contenttype = self.contenttype.decode()
s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding)
LOGGER.debug('Response code: %s',
self.context.response_codes[self.status])
LOGGER.debug('Response:\n%s', s)
return [self.context.response_codes[self.status], s]
def _gen_soap_wrapper(self):
""" Generate SOAP wrapper """
LOGGER.info('Writing SOAP wrapper.')
node = etree.Element(
util.nspath_eval('soapenv:Envelope', self.context.namespaces),
nsmap=self.context.namespaces
)
schema_location_ns = util.nspath_eval('xsi:schemaLocation',
self.context.namespaces)
node.attrib[schema_location_ns] = '%s %s' % (
self.context.namespaces['soapenv'],
self.context.namespaces['soapenv']
)
node2 = etree.SubElement(
node, util.nspath_eval('soapenv:Body', self.context.namespaces))
if self.exception:
node3 = etree.SubElement(
node2,
util.nspath_eval('soapenv:Fault', self.context.namespaces)
)
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Code', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Value', self.context.namespaces)
).text = 'soap:Server'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Reason', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Text', self.context.namespaces)
).text = 'A server exception was encountered.'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Detail', self.context.namespaces)
)
node4.append(self.response)
else:
node2.append(self.response)
self.response = node
def _gen_manager(self):
""" Update self.context.model with CSW-T advertising """
if (self.config.has_option('manager', 'transactions') and
self.config.get('manager', 'transactions') == 'true'):
self.manager = True
self.context.model['operations_order'].append('Transaction')
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {}
}
schema_values = [
'http://www.opengis.net/cat/csw/2.0.2',
'http://www.opengis.net/cat/csw/3.0',
'http://www.opengis.net/wms',
'http://www.opengis.net/wmts/1.0',
'http://www.opengis.net/wfs',
'http://www.opengis.net/wfs/2.0',
'http://www.opengis.net/wcs',
'http://www.opengis.net/wps/1.0.0',
'http://www.opengis.net/sos/1.0',
'http://www.opengis.net/sos/2.0',
'http://www.isotc211.org/2005/gmi',
'urn:geoss:waf',
]
self.context.model['operations_order'].append('Harvest')
self.context.model['operations']['Harvest'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'ResourceType': {'values': schema_values}
}
}
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'TransactionSchemas': {'values': sorted(schema_values)}
}
}
self.csw_harvest_pagesize = 10
if self.config.has_option('manager', 'csw_harvest_pagesize'):
self.csw_harvest_pagesize = int(
self.config.get('manager', 'csw_harvest_pagesize'))
def _test_manager(self):
""" Verify that transactions are allowed """
if self.config.get('manager', 'transactions') != 'true':
raise RuntimeError('CSW-T interface is disabled')
ipaddress = self.environ['REMOTE_ADDR']
if not self.config.has_option('manager', 'allowed_ips') or \
(self.config.has_option('manager', 'allowed_ips') and not
util.ipaddress_in_whitelist(ipaddress,
self.config.get('manager', 'allowed_ips').split(','))):
raise RuntimeError(
'CSW-T operations not allowed for this IP address: %s' % ipaddress)
def _cql_update_queryables_mappings(self, cql, mappings):
""" Transform CQL query's properties to underlying DB columns """
LOGGER.debug('Raw CQL text = %s', cql)
LOGGER.debug(str(list(mappings.keys())))
if cql is not None:
for key in mappings.keys():
try:
cql = cql.replace(key, mappings[key]['dbcol'])
except:
cql = cql.replace(key, mappings[key])
LOGGER.debug('Interpolated CQL text = %s.', cql)
return cql
def _process_responsehandler(self, xml):
""" Process response handler """
if self.kvp['responsehandler'] is not None:
LOGGER.info('Processing responsehandler %s' %
self.kvp['responsehandler'])
uprh = urlparse(self.kvp['responsehandler'])
if uprh.scheme == 'mailto': # email
import smtplib
LOGGER.debug('Email detected')
smtp_host = 'localhost'
if self.config.has_option('server', 'smtp_host'):
smtp_host = self.config.get('server', 'smtp_host')
body = ('Subject: pycsw %s results\n\n%s' %
(self.kvp['request'], xml))
try:
LOGGER.info('Sending email')
msg = smtplib.SMTP(smtp_host)
msg.sendmail(
self.config.get('metadata:main', 'contact_email'),
uprh.path, body
)
msg.quit()
LOGGER.debug('Email sent successfully.')
except Exception as err:
LOGGER.exception('Error processing email')
elif uprh.scheme == 'ftp':
import ftplib
LOGGER.debug('FTP detected.')
try:
LOGGER.info('Sending to FTP server.')
ftp = ftplib.FTP(uprh.hostname)
if uprh.username is not None:
ftp.login(uprh.username, uprh.password)
ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml))
ftp.quit()
LOGGER.debug('FTP sent successfully.')
except Exception as err:
LOGGER.exception('Error processing FTP')
@staticmethod
def normalize_kvp(kvp):
"""Normalize Key Value Pairs.
This method will transform all keys to lowercase and leave values
unchanged, as specified in the CSW standard (see for example note
C on Table 62 - KVP Encoding for DescribeRecord operation request
of the CSW standard version 2.0.2)
:arg kvp: a mapping with Key Value Pairs
:type kvp: dict
:returns: A new dictionary with normalized parameters
"""
result = dict()
for name, value in kvp.items():
result[name.lower()] = value
return result
parser.add_option_group(jabber_group)
parser.add_option_group(zulip.generate_option_group(parser, "zulip-"))
(options, args) = parser.parse_args()
logging.basicConfig(level=options.log_level,
format='%(levelname)-8s %(message)s')
if options.zulip_config_file is None:
config_file = zulip.get_default_config_filename()
else:
config_file = options.zulip_config_file
config = SafeConfigParser()
try:
with open(config_file, 'r') as f:
config.readfp(f, config_file)
except IOError:
pass
for option in ("jid", "jabber_password", "conference_domain", "mode",
"zulip_email_suffix", "jabber_server_address",
"jabber_server_port"):
if (getattr(options, option) is None
and config.has_option("jabber_mirror", option)):
setattr(options, option, config.get("jabber_mirror", option))
for option in ("no_use_tls", ):
if getattr(options, option) is None:
if config.has_option("jabber_mirror", option):
setattr(options, option,
config.getboolean("jabber_mirror", option))
else:
def get_giphy_api_key_from_config():
config = SafeConfigParser()
with open(os.environ['HOME'] + '/.giphy_config', 'r') as config_file:
config.readfp(config_file)
return config.get("giphy-config", "key")
def get_weather_api_key_from_config(directory, config_name):
config = SafeConfigParser()
with open(directory + config_name, 'r') as config_file:
config.readfp(config_file)
return config.get("weather-config", "key")
def load_conf():
conf_path = os.path.expanduser(config_path)
if not os.path.exists(conf_path):
logger.warn("{conf} couldn't be found, please use \'coscmd config -h\' to learn how to config coscmd!".format(conf=to_printable_str(conf_path)))
raise IOError
else:
logger.debug('{conf} is found'.format(conf=to_printable_str(conf_path)))
try:
with open(conf_path, 'r') as f:
cp = SafeConfigParser()
cp.readfp(fp=f)
if not cp.has_section('common'):
raise Exception("[common] section could't be found, please check your config file.")
if cp.has_option('common', 'part_size'):
part_size = cp.getint('common', 'part_size')
else:
part_size = 1
if cp.has_option('common', 'max_thread'):
max_thread = cp.getint('common', 'max_thread')
else:
max_thread = 5
try:
secret_id = cp.get('common', 'secret_id')
except Exception:
secret_id = cp.get('common', 'access_id')
try:
appid = cp.get('common', 'appid')
bucket = cp.get('common', 'bucket')
if bucket.endswith("-"+str(appid)):
bucket = bucket.rstrip(appid)
bucket = bucket[:-1]
except Exception:
try:
bucket = cp.get('common', 'bucket')
appid = bucket.split('-')[-1]
bucket = bucket.rstrip(appid)
bucket = bucket[:-1]
except Exception:
# check if user use -b bucket
if (pre_bucket == ""):
logger.error("The configuration file is wrong. Please reconfirm")
try:
schema = cp.get('common', 'schema')
except Exception:
schema = 'https'
try:
verify = cp.get('common', 'verify')
except Exception:
verify = 'md5'
try:
token = cp.get('common', 'token')
except Exception:
token = None
try:
error = cp.get('common', 'error')
except Exception:
error = "cos.err.log"
try:
success = cp.get('common', 'success')
except Exception:
success = "cos.suc.log"
try:
anonymous = cp.get('common', 'anonymous')
if anonymous == 'True' or anonymous == 'true':
anonymous = True
else:
anonymous = False
except Exception:
anonymous = False
try:
retry = int(cp.get('common', 'retry'))
except Exception:
retry = 2
try:
timeout = int(cp.get('common', 'timeout'))
except Exception:
timeout = 60
region, endpoint = None, None
if cp.has_option('common', 'region'):
region = cp.get('common', 'region')
if cp.has_option('common', 'endpoint'):
endpoint = cp.get('common', 'endpoint')
if pre_appid != "":
appid = pre_appid
if pre_bucket != "":
bucket = pre_bucket
if pre_region != "":
region = pre_region
conf = CoscmdConfig(
appid=appid,
secret_id=secret_id,
secret_key=cp.get('common', 'secret_key'),
token=token,
region=compatible(region),
endpoint=endpoint,
bucket=bucket,
part_size=part_size,
max_thread=max_thread,
schema=schema,
anonymous=anonymous,
verify=verify,
retry=retry,
timeout=timeout,
error=error,
success=success
)
return conf
except Exception as e:
raise(e)
def onDiscovery(self, theme, settings, dependenciesSettings):
res = queryResourceDirectory(THEME_RESOURCE_NAME, theme)
if res is None:
return
directoryName = 'views'
if 'directory' in settings:
directoryName = settings['directory']
if res.isDirectory(directoryName):
viewsDir = res[directoryName]
layer = getattr(schemata, theme, None)
if 'layer' in settings:
layerName = settings['layer']
try:
layer = resolve(layerName)
except (ImportError, AttributeError):
logger.warn("Could not import %s" % layerName)
return
viewConfig = SafeConfigParser()
if viewsDir.isFile(VIEW_CONFIG_FILENAME):
fp = viewsDir.openFile(VIEW_CONFIG_FILENAME)
try:
viewConfig.readfp(fp)
finally:
try:
fp.close()
except AttributeError:
pass
views = []
configurationMachine = ConfigurationMachine()
path = viewsDir.directory
for filename in os.listdir(path):
if not filename.lower().endswith(EXTENSION):
continue
name = viewName = filename[:-3]
permission = 'zope2.View'
for_ = Interface
class_ = None
template = os.path.join(path, filename)
menu = {}
# Read override options from views.cfg if applicable
if viewConfig.has_section(name):
if viewConfig.has_option(name, 'name'):
viewName = viewConfig.get(name, 'name')
if viewConfig.has_option(name, 'permission'):
permission = viewConfig.get(name, 'permission')
if viewConfig.has_option(name, 'for'):
forStr = viewConfig.get(name, 'for')
if forStr != "*":
for_ = resolve(forStr)
if viewConfig.has_option(name, 'class'):
class_ = resolve(viewConfig.get(name, 'class'))
if viewConfig.has_option(name, 'menu'):
menu = dict(
title=viewConfig.get(name, 'menu'),
menu=getattr(
zope.browsermenu.metaconfigure.menus,
"plone_displayviews",
),
)
Products.Five.browser.metaconfigure.page(
configurationMachine,
name=viewName,
permission=permission,
for_=for_,
layer=layer,
template=template,
class_=class_,
**menu
)
views.append(name)
if len(views) > 0:
configurationMachine.execute_actions()
self.registered[theme] = views
def deploy_django(proj):
"""
Deploy a Django project
"""
wsgi_base_path = os.environ.get('WSGI_BASE_PATH',
'/var/www/wsgi')
httpd_conf_dir = os.environ.get('HTTPD_CONF_DIR',
'/etc/httpd/locations.d')
httpd_host = os.environ.get('HTTPD_HOST',
platform.node())
httpd_media_base = os.environ.get('HTTPD_MEDIA_BASE',
'/var/www/html/media')
httpd_static_base = os.environ.get('HTTPD_STATIC_BASE',
'/var/www/html/static')
secret_key_gen = os.environ.get('SECRET_KEY_GEN',
'/usr/bin/pwgen -c -n -y 78 1')
proj_base = os.path.join(wsgi_base_path, proj)
path = lambda p: os.path.join(proj_base, p)
proj_defaults = {
'name': proj,
'proj_base': proj_base,
'dst': '%(name)s-project',
'settings': '%(name)s_production',
'url': '/%(name)s',
'build': 'build/build.sh',
'wsgi': 'wsgi.py',
'allowed_hosts': httpd_host,
'secret_key': subprocess.check_output(secret_key_gen.split()
).strip().replace("'", "-"),
'media_root': os.path.join(httpd_media_base, proj),
'static_root': os.path.join(httpd_static_base, proj),
'scm': '/usr/bin/git',
'settings_append': DEFAULT_SETTINGS_APPEND,
'deploy_requires': None,
'deploy_commands': ['migrate']
}
# Protect '%' from interpolation
proj_defaults['secret_key'] = re.sub(r'%', r'',
proj_defaults['secret_key'])
# Choose clone command
proj_defaults['scm_clone'] = SCM_DEFAULT_CHECKOUT[os.path.split(
proj_defaults['scm'])[-1]]
# Load defaults
cfg = SafeConfigParser(proj_defaults)
# Force read
cfg.readfp(open(proj+'.cfg', 'r'))
#logger.debug('Final configuration:')
#for k,v in cfg.items(CFG_SECTION):
# logger.debug('\t%s: %s', k, v)
# Create directory
os.mkdir(proj_base)
# Virtualenv
virtualenv.create_environment(proj_base)
# Checkout
subprocess.check_call([
cfg.get(CFG_SECTION, 'scm'),
cfg.get(CFG_SECTION, 'scm_clone'),
cfg.get(CFG_SECTION, 'src'),
path(cfg.get(CFG_SECTION, 'dst')),
])
# Build
activate = path('bin/activate')
build = os.path.join(
cfg.get(CFG_SECTION, 'dst'),
cfg.get(CFG_SECTION, 'build')
)
subprocess.check_call([build],
cwd=proj_base,
env={'BASH_ENV': activate})
# Install Deploy Requiremts
deploy_requires = cfg.get(CFG_SECTION, 'deploy_requires')
if deploy_requires:
logger.debug('Installing: %s', deploy_requires)
cmd = [os.path.join(virtualenv.path_locations(proj_base)[-1],
'pip')
, 'install']
cmd.extend(parse_list(deploy_requires))
subprocess.check_call(cmd)
# Create settings
settings_file = path(cfg.get(CFG_SECTION, 'settings'))+'.py'
slock = LockFile(settings_file)
slock.acquire()
if os.path.exists(settings_file):
slock.release()
raise IOError([17, 'File exists'])
try:
sfp = open(settings_file, 'w')
print(DJANGO_SETTINGS_TEMPLATE % dict(cfg.items(CFG_SECTION)),
file=sfp)
sfp.close()
finally:
slock.release()
# Create wsgi script
wsgi_file = path(cfg.get(CFG_SECTION, 'wsgi'))
slock = LockFile(wsgi_file)
slock.acquire()
if os.path.exists(wsgi_file):
slock.release()
raise IOError([17, 'File exists'])
try:
wfp = open(wsgi_file, 'w')
print(WSGI_TEMPLATE % dict(cfg.items(CFG_SECTION)),
file=wfp)
wfp.close()
finally:
slock.release()
# Create apache conf
conf_file = os.path.join(httpd_conf_dir,
cfg.get(CFG_SECTION, 'name'))+'.conf'
slock = LockFile(conf_file)
slock.acquire()
if os.path.exists(conf_file):
slock.release()
raise IOError([17, 'File exists'])
try:
sfp = open(conf_file, 'w')
conf = dict(cfg.items(CFG_SECTION))
conf['site_libs'] = os.path.join(
virtualenv.path_locations(proj_base)[1],
'site-packages')
http_conf = HTTPD_CONF_TEMPLATE % conf
print(http_conf,
file=sfp)
sfp.close()
finally:
slock.release()
# Perform django commands
deploy_commands = cfg.get(CFG_SECTION, 'deploy_commands')
if deploy_commands:
manage = [os.path.join(virtualenv.path_locations(proj_base)[-1],
virtualenv.expected_exe)
, 'manage.py']
os.chdir(path(cfg.get(CFG_SECTION, 'dst')))
# Deployment django environment
dep_env = os.environ.copy()
dep_env['DJANGO_SETTINGS_MODULE'] = cfg.get(CFG_SECTION, 'settings')
dep_env['PYTHONPATH'] = path('.')
logger.debug('Environment for commands: PYTHONPATH=%s',
dep_env['PYTHONPATH'])
logger.debug(' Django settings: %s', dep_env['DJANGO_SETTINGS_MODULE'])
for cmd in parse_list(deploy_commands):
logger.debug("Executing '%s'", ' '.join(manage+[cmd]))
subprocess.check_call(manage+cmd.split(), env=dep_env)
# That's it. Remember to reload apache
print('You should reload apache:\n', '\t', 'systemctl reload httpd')
return True
def __init__(self, email=None, api_key=None, config_file=None,
verbose=False, retry_on_errors=True,
site=None, client=None,
cert_bundle=None, insecure=None,
client_cert=None, client_cert_key=None):
# type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], bool, Optional[str], Optional[str]) -> None
if client is None:
client = _default_client()
# Fill values from Environment Variables if not available in Constructor
if config_file is None:
config_file = os.environ.get("ZULIP_CONFIG")
if api_key is None:
api_key = os.environ.get("ZULIP_API_KEY")
if email is None:
email = os.environ.get("ZULIP_EMAIL")
if site is None:
site = os.environ.get("ZULIP_SITE")
if client_cert is None:
client_cert = os.environ.get("ZULIP_CERT")
if client_cert_key is None:
client_cert_key = os.environ.get("ZULIP_CERT_KEY")
if cert_bundle is None:
cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")
if config_file is None:
config_file = get_default_config_filename()
if config_file is not None and os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if client_cert is None and config.has_option("api", "client_cert"):
client_cert = config.get("api", "client_cert")
if client_cert_key is None and config.has_option("api", "client_cert_key"):
client_cert_key = config.get("api", "client_cert_key")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get("api", "insecure").lower()
if insecure_setting == "true":
insecure = True
elif insecure_setting == "false":
insecure = False
else:
raise RuntimeError("insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
% (insecure_setting, config_file))
elif None in (api_key, email):
raise RuntimeError("api_key or email not specified and %s does not exist"
% (config_file,))
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if site.startswith("localhost"):
site = "http://" + site
elif not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
raise RuntimeError("Missing Zulip server URL; specify via --site or ~/.zuliprc.")
if not self.base_url.endswith("/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
self.tls_verification = False # type: Union[bool, str]
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise RuntimeError("tls bundle '%s' does not exist"
% (cert_bundle,))
self.tls_verification = cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification = True
if client_cert is None:
if client_cert_key is not None:
raise RuntimeError("client cert key '%s' specified, but no client cert public part provided"
% (client_cert_key,))
else: # we have a client cert
if not os.path.isfile(client_cert):
raise RuntimeError("client cert '%s' does not exist"
% (client_cert,))
if client_cert_key is not None:
if not os.path.isfile(client_cert_key):
raise RuntimeError("client cert key '%s' does not exist"
% (client_cert_key,))
self.client_cert = client_cert
self.client_cert_key = client_cert_key
def __init__(self,
email=None,
api_key=None,
config_file=None,
verbose=False,
retry_on_errors=True,
site=None,
client=None,
cert_bundle=None,
insecure=None,
client_cert=None,
client_cert_key=None):
if client is None:
client = _default_client()
if config_file is None:
config_file = get_default_config_filename()
if os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if client_cert is None and config.has_option("api", "client_cert"):
client_cert = config.get("api", "client_cert")
if client_cert_key is None and config.has_option(
"api", "client_cert_key"):
client_cert_key = config.get("api", "client_cert_key")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get("api", "insecure").lower()
if insecure_setting == "true":
insecure = True
elif insecure_setting == "false":
insecure = False
else:
raise RuntimeError(
"insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
% (insecure_setting, config_file))
elif None in (api_key, email):
raise RuntimeError(
"api_key or email not specified and %s does not exist" %
(config_file, ))
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
self.base_url = "https://api.zulip.com"
if self.base_url != "https://api.zulip.com" and not self.base_url.endswith(
"/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
self.tls_verification = False
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise RuntimeError("tls bundle '%s' does not exist" %
(cert_bundle, ))
self.tls_verification = cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification = True
if client_cert is None:
if client_cert_key is not None:
raise RuntimeError(
"client cert key '%s' specified, but no client cert public part provided"
% (client_cert_key, ))
else: # we have a client cert
if not os.path.isfile(client_cert):
raise RuntimeError("client cert '%s' does not exist" %
(client_cert, ))
if client_cert_key is not None:
if not os.path.isfile(client_cert_key):
raise RuntimeError("client cert key '%s' does not exist" %
(client_cert_key, ))
self.client_cert = client_cert
self.client_cert_key = client_cert_key
def __init__(self, email=None, api_key=None, config_file=None,
verbose=False, retry_on_errors=True,
site=None, client=None,
cert_bundle=None, insecure=None,
client_cert=None, client_cert_key=None):
# type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[str], Optional[str]) -> None
if client is None:
client = _default_client()
# Normalize user-specified path
if config_file is not None:
config_file = os.path.abspath(os.path.expanduser(config_file))
# Fill values from Environment Variables if not available in Constructor
if config_file is None:
config_file = os.environ.get("ZULIP_CONFIG")
if api_key is None:
api_key = os.environ.get("ZULIP_API_KEY")
if email is None:
email = os.environ.get("ZULIP_EMAIL")
if site is None:
site = os.environ.get("ZULIP_SITE")
if client_cert is None:
client_cert = os.environ.get("ZULIP_CERT")
if client_cert_key is None:
client_cert_key = os.environ.get("ZULIP_CERT_KEY")
if cert_bundle is None:
cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")
if insecure is None:
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = os.environ.get('ZULIP_ALLOW_INSECURE')
if insecure_setting is not None:
insecure = validate_boolean_field(insecure_setting)
if insecure is None:
raise ZulipError("The ZULIP_ALLOW_INSECURE environment "
"variable is set to '{}', it must be "
"'true' or 'false'"
.format(insecure_setting))
if config_file is None:
config_file = get_default_config_filename()
if config_file is not None and os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if client_cert is None and config.has_option("api", "client_cert"):
client_cert = config.get("api", "client_cert")
if client_cert_key is None and config.has_option("api", "client_cert_key"):
client_cert_key = config.get("api", "client_cert_key")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get('api', 'insecure')
insecure = validate_boolean_field(insecure_setting)
if insecure is None:
raise ZulipError("insecure is set to '{}', it must be "
"'true' or 'false' if it is used in {}"
.format(insecure_setting, config_file))
elif None in (api_key, email):
raise ConfigNotFoundError("api_key or email not specified and file %s does not exist"
% (config_file,))
assert(api_key is not None and email is not None)
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if site.startswith("localhost"):
site = "http://" + site
elif not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
raise MissingURLError("Missing Zulip server URL; specify via --site or ~/.zuliprc.")
if not self.base_url.endswith("/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
logger.warning('Insecure mode enabled. The server\'s SSL/TLS '
'certificate will not be validated, making the '
'HTTPS connection potentially insecure')
self.tls_verification = False # type: Union[bool, str]
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise ConfigNotFoundError("tls bundle '%s' does not exist"
% (cert_bundle,))
self.tls_verification = cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification = True
if client_cert is None:
if client_cert_key is not None:
raise ConfigNotFoundError("client cert key '%s' specified, but no client cert public part provided"
% (client_cert_key,))
else: # we have a client cert
if not os.path.isfile(client_cert):
raise ConfigNotFoundError("client cert '%s' does not exist"
% (client_cert,))
if client_cert_key is not None:
if not os.path.isfile(client_cert_key):
raise ConfigNotFoundError("client cert key '%s' does not exist"
% (client_cert_key,))
self.client_cert = client_cert
self.client_cert_key = client_cert_key
self.session = None # type: Optional[requests.Session]
self.has_connected = False
def configparser2yaml(cpfile):
dict_ = {}
cp = ConfigParser()
with codecs.open(cpfile, encoding='utf-8') as fh:
cp.readfp(fh)
for section in cp.sections():
if section.startswith('contact:'): # contacts are now nested
if 'contact' not in dict_:
dict_['contact'] = {}
section2 = dict_['contact'][section.split(':')[1]] = {}
elif section.startswith('distribution:'): # distributions now nested
if 'distribution' not in dict_:
dict_['distribution'] = {}
section2 = dict_['distribution'][section.split(':')[1]] = {}
else:
section2 = dict_[section] = {}
for k, v in cp.items(section):
if section == 'identification': # keywords are now nested
if 'keywords' not in section2:
section2['keywords'] = {}
if 'default' not in section2['keywords']:
section2['keywords']['default'] = {}
if 'gc_cst' not in section2['keywords']:
section2['keywords']['gc_cst'] = {}
if 'wmo' not in section2['keywords']:
section2['keywords']['wmo'] = {}
if 'hnap_category_information' not in section2['keywords']:
section2['keywords']['hnap_category_information'] = {}
if 'hnap_category_geography' not in section2['keywords']:
section2['keywords']['hnap_category_geography'] = {}
if 'hnap_category_content' not in section2['keywords']:
section2['keywords']['hnap_category_content'] = {}
if k in ['topiccategory']:
section2['topiccategory'] = [v]
if k in ['keywords_en', 'keywords_fr']:
section2['keywords']['default'][k] = [k2.strip() for k2 in v.split(',')] # noqa
if k in ['keywords_gc_cst_en']:
section2['keywords']['gc_cst']['keywords_en'] = [k2.strip() for k2 in v.split(',')] # noqa
if k in ['keywords_gc_cst_fr']:
section2['keywords']['gc_cst']['keywords_fr'] = [k2.strip() for k2 in v.split(',')] # noqa
if k in ['keywords_wmo']:
section2['keywords']['wmo']['keywords_en'] = [k2.strip() for k2 in v.split(',')] # noqa
if k in ['hnap_category_information_en']:
section2['keywords']['hnap_category_information']['keywords_en'] = [v] # noqa
section2['keywords']['hnap_category_information']['keywords_fr'] = [v] # noqa
if k in ['hnap_category_geography_en']:
section2['keywords']['hnap_category_geography']['keywords_en'] = [v] # noqa
if k in ['hnap_category_geography_fr']:
section2['keywords']['hnap_category_geography']['keywords_fr'] = [v] # noqa
if k in ['hnap_category_content_en']:
section2['keywords']['hnap_category_content']['keywords_en'] = [v] # noqa
if k in ['hnap_category_content_fr']:
section2['keywords']['hnap_category_content']['keywords_fr'] = [v] # noqa
if k == 'keywords_type':
section2['keywords']['default'][k] = v
section2['keywords']['gc_cst'][k] = v
section2['keywords']['wmo'][k] = v
section2['keywords']['hnap_category_geography'][k] = v
section2['keywords']['hnap_category_information'][k] = v
section2['keywords']['hnap_category_content'][k] = v
else:
section2[k] = v
return yaml.safe_dump(dict_, default_flow_style=False, allow_unicode=True)
def config_from_string(value):
config = SafeConfigParser()
config.readfp(StringIO(value))
return Config(config)
from __future__ import print_function
import random, unittest
from ofs.remote import swiftstore as store
from ofs.remote.swiftstore import SwiftOFS
from ofs import OFSException
from six import StringIO
from six.moves.configparser import SafeConfigParser
TEST_TEXT = """I am a banana"""
cfg = SafeConfigParser()
cfg.readfp(open('test.ini'))
class TestSwiftOFS(unittest.TestCase):
def setUp(self):
self.bucket_name = 'ofs-test-bucket'
auth_url = cfg.get('ofs', 'ofs.os_auth_url')
user = cfg.get('ofs', 'ofs.os_user')
passwd = cfg.get('ofs', 'ofs.os_passwd')
tenant = cfg.get('ofs', 'ofs.os_tenant')
self.ofs = SwiftOFS(os_auth_url=auth_url, os_user=user,
os_passwd=passwd, os_tenant=tenant)
self.bucket = self.ofs.connection.put_container(self.bucket_name,
headers=store.PUBLIC_HEADER)
def tearDown(self):
self._del_bucket(self.bucket_name)
def _makefp(self):
def __init__(self,
email=None,
api_key=None,
config_file=None,
verbose=False,
retry_on_errors=True,
site=None,
client=None,
cert_bundle=None,
insecure=None,
client_cert=None,
client_cert_key=None):
# type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], bool, Optional[str], Optional[str]) -> None
if client is None:
client = _default_client()
# Fill values from Environment Variables if not available in Constructor
if config_file is None:
config_file = os.environ.get("ZULIP_CONFIG")
if api_key is None:
api_key = os.environ.get("ZULIP_API_KEY")
if email is None:
email = os.environ.get("ZULIP_EMAIL")
if site is None:
site = os.environ.get("ZULIP_SITE")
if client_cert is None:
client_cert = os.environ.get("ZULIP_CERT")
if client_cert_key is None:
client_cert_key = os.environ.get("ZULIP_CERT_KEY")
if cert_bundle is None:
cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")
if config_file is None:
config_file = get_default_config_filename()
if os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if client_cert is None and config.has_option("api", "client_cert"):
client_cert = config.get("api", "client_cert")
if client_cert_key is None and config.has_option(
"api", "client_cert_key"):
client_cert_key = config.get("api", "client_cert_key")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get("api", "insecure").lower()
if insecure_setting == "true":
insecure = True
elif insecure_setting == "false":
insecure = False
else:
raise RuntimeError(
"insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
% (insecure_setting, config_file))
elif None in (api_key, email):
raise RuntimeError(
"api_key or email not specified and %s does not exist" %
(config_file, ))
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if site.startswith("localhost"):
site = "http://" + site
elif not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
raise RuntimeError(
"Missing Zulip server URL; specify via --site or ~/.zuliprc.")
if not self.base_url.endswith("/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
self.tls_verification = False # type: Union[bool, str]
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise RuntimeError("tls bundle '%s' does not exist" %
(cert_bundle, ))
self.tls_verification = cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification = True
if client_cert is None:
if client_cert_key is not None:
raise RuntimeError(
"client cert key '%s' specified, but no client cert public part provided"
% (client_cert_key, ))
else: # we have a client cert
if not os.path.isfile(client_cert):
raise RuntimeError("client cert '%s' does not exist" %
(client_cert, ))
if client_cert_key is not None:
if not os.path.isfile(client_cert_key):
raise RuntimeError("client cert key '%s' does not exist" %
(client_cert_key, ))
self.client_cert = client_cert
self.client_cert_key = client_cert_key
def get_giphy_api_key_from_config():
config = SafeConfigParser()
with open(os.environ['HOME'] + '/.giphy_config', 'r') as config_file:
config.readfp(config_file)
return config.get("giphy-config", "key")
class GlobusConfigParser(object):
"""
Wraps a SafeConfigParser to do modified get()s and config file loading.
"""
_GENERAL_CONF_SECTION = 'general'
def __init__(self):
self._parser = SafeConfigParser()
self._load_config()
def _load_config(self):
# TODO: /etc is not windows friendly, not sure about expanduser
self._read([_get_lib_config_path(), "/etc/globus.cfg",
os.path.expanduser("~/.globus.cfg")])
def _read(self, filenames):
"""
Wraps up self._parser.read() to inject '[general]\n' at the beginning
of file contents.
Originally, this was implemented by catching
MissingSectionHeaderErrors, but that's actually overcomplicated and
unnecessary. Always inserting it, uniformly, is simpler and doesn't
change the semantics of config parsing at all.
"""
for fname in filenames:
try:
with open(fname) as f:
# wrap the file-like object in a StringIO so that we can
# pass it to the SafeConfigParser as a file like object
wrapped_file = StringIO(
'[{}]\n'.format(self._GENERAL_CONF_SECTION) + f.read())
try:
self._parser.readfp(wrapped_file, fname)
except DuplicateSectionError:
f.seek(0)
self._parser.readfp(f, fname)
except IOError:
continue
def get(self, option,
section=None, environment=None,
failover_to_general=False, check_env=False,
type_cast=str):
"""
Attempt to lookup an option in the config file. Optionally failover to
the general section if the option is not found.
Also optionally, check for a relevant environment variable, which is
named always as GLOBUS_SDK_{option.upper()}. Note that 'section'
doesn't slot into the naming at all. Otherwise, we'd have to contend
with GLOBUS_SDK_GENERAL_... for almost everything, and
GLOBUS_SDK_ENVIRONMENT\ PROD_... which is awful.
Returns None for an unfound key, rather than raising a NoOptionError.
"""
# envrionment is just a fancy name for sections that start with
# 'environment '
if environment:
section = 'environment ' + environment
# if you don't specify a section or an environment, assume it's the
# general conf section
if section is None:
section = self._GENERAL_CONF_SECTION
# if this is a config option which checks the shell env, look there
# *first* for a value -- env values have higher precedence than config
# files so that you can locally override the behavior of a command in a
# given shell or subshell
env_option_name = 'GLOBUS_SDK_{}'.format(option.upper())
value = None
if check_env and env_option_name in os.environ:
value = os.environ[env_option_name]
else:
try:
value = self._parser.get(section, option)
except (NoOptionError, NoSectionError):
if failover_to_general:
value = self.get(option,
section=self._GENERAL_CONF_SECTION)
if value is not None:
value = type_cast(value)
return value
parser.add_option_group(jabber_group)
parser.add_option_group(zulip.generate_option_group(parser, "zulip-"))
(options, args) = parser.parse_args()
logging.basicConfig(level=options.log_level,
format='%(levelname)-8s %(message)s')
if options.zulip_config_file is None:
config_file = zulip.get_default_config_filename()
else:
config_file = options.zulip_config_file
config = SafeConfigParser()
try:
with open(config_file, 'r') as f:
config.readfp(f, config_file)
except IOError:
pass
for option in ("jid", "jabber_password", "conference_domain", "mode", "zulip_email_suffix",
"jabber_server_address", "jabber_server_port"):
if (getattr(options, option) is None
and config.has_option("jabber_mirror", option)):
setattr(options, option, config.get("jabber_mirror", option))
for option in ("no_use_tls",):
if getattr(options, option) is None:
if config.has_option("jabber_mirror", option):
setattr(options, option, config.getboolean("jabber_mirror", option))
else:
setattr(options, option, False)
