def read_conf(conf_path, section_name=None, defaults=None, use_yaml=False):
if use_yaml:
return parse_config(conf_path)
if defaults is None:
defaults = {}
parser = SafeConfigParser(defaults)
success = parser.read(conf_path)
if not success:
print("Unable to read config from %s" % conf_path)
sys.exit(1)
if section_name:
if parser.has_section(section_name):
# if log_format is set, extract it from the parser
# to prevent to expand variables which can, in case of
# log_format throw a ConfigParser.InterpolationMissingOptionError
log_format = None
if parser.has_option(section_name, 'log_format'):
log_format = parser.get(section_name, 'log_format', raw=True)
# don't use remove_options because it can fail without reason
parser.set(section_name, 'log_format', '')
conf = dict(parser.items(section_name))
# Add log_format again, after parsing
if log_format:
conf['log_format'] = log_format
else:
print('Unable to find section %s in config %s' %
(section_name, conf_path))
exit(1)
else:
conf = {}
for section in parser.sections():
# if log_format is set, extract it from the parser
# to prevent to expand variables which can, in case of
# log_format throw a ConfigParser.InterpolationMissingOptionError
log_format = None
if parser.has_option(section, 'log_format'):
log_format = parser.get(section, 'log_format', raw=True)
# don't use remove_options because it can fail without reason
parser.set(section, 'log_format', '')
conf.update({section: dict(parser.items(section))})
# Add log_format again, after parsing
if log_format:
conf[section]['log_format'] = log_format
return conf
def getManifest(fp, format, defaults=None):
"""Read the manifest from the given open file pointer according to the
given ManifestFormat. Pass a dict as ``defaults`` to override the defaults
from the manifest format.
"""
if defaults is None:
defaults = format.defaults
parser = SafeConfigParser()
if six.PY2:
parser.readfp(fp)
else:
data = fp.read()
if isinstance(data, six.binary_type):
data = data.decode()
parser.read_string(data)
results = {}
for key in format.keys:
if parser.has_option(format.resourceType, key):
results[key] = parser.get(format.resourceType, key)
else:
results[key] = defaults.get(key, None)
for key in format.parameterSections:
sectionName = "%s:%s" % (format.resourceType, key,)
if parser.has_section(sectionName):
results[key] = dict(parser.items(sectionName))
else:
results[key] = {}
return results
def getManifest(fp, format, defaults=None):
"""Read the manifest from the given open file pointer according to the
given ManifestFormat. Pass a dict as ``defaults`` to override the defaults
from the manifest format.
"""
if defaults is None:
defaults = format.defaults
parser = SafeConfigParser()
parser.readfp(fp)
results = {}
for key in format.keys:
if parser.has_option(format.resourceType, key):
results[key] = parser.get(format.resourceType, key)
else:
results[key] = defaults.get(key, None)
for key in format.parameterSections:
sectionName = "%s:%s" % (format.resourceType, key,)
if parser.has_section(sectionName):
results[key] = dict(parser.items(sectionName))
else:
results[key] = {}
return results
def check_dlm_cfgfile():
"""Parse DLM config file"""
fname = "/etc/dlm/dlm.conf"
try:
with open(fname, 'r') as fp:
cfgs = '[dlm]\n' + fp.read()
except (OSError, IOError):
return False
cfg = SafeConfigParser()
try:
cfg.read_string(cfgs)
except AttributeError:
# Python2 ConfigParser doesn't have cfg.read_string
from cStringIO import StringIO
cfg.readfp(StringIO(cfgs))
if not cfg.has_option('dlm', 'protocol'):
return False
proto = cfg.get('dlm', 'protocol').lower()
return proto in ['sctp', 'detect', '1', '2']
if options.zulip_config_file is None:
config_file = zulip.get_default_config_filename()
else:
config_file = options.zulip_config_file
config = SafeConfigParser()
try:
with open(config_file, 'r') as f:
config.readfp(f, config_file)
except IOError:
pass
for option in ("jid", "jabber_password", "conference_domain", "mode", "zulip_email_suffix",
"jabber_server_address", "jabber_server_port"):
if (getattr(options, option) is None
and config.has_option("jabber_mirror", option)):
setattr(options, option, config.get("jabber_mirror", option))
for option in ("no_use_tls",):
if getattr(options, option) is None:
if config.has_option("jabber_mirror", option):
setattr(options, option, config.getboolean("jabber_mirror", option))
else:
setattr(options, option, False)
if options.mode is None:
options.mode = "personal"
if options.zulip_email_suffix is None:
options.zulip_email_suffix = ''
class Config(MutableMapping):
"""Read and write to config file.
A config object is essentially a string key-value store that can be treated like a dictionary::
c = Config()
c['foo'] = 'bar'
print c['foo']
The file location may be specified::
c = Config('~/matt/anotherconfig.cfg')
c['where'] = 'in a different file'
If no location is specified, the environment variable CHEMDATAEXTRACTOR_CONFIG is checked and used if available.
Otherwise, a standard config location is used, which varies depending on the operating system. You can check the
location using the ``path`` property. For more information see https://github.com/ActiveState/appdirs
It is possible to edit the file by hand with a text editor.
Warning: multiple instances of Config() pointing to the same file will not see each others' changes, and will
overwrite the entire file when any key is changed.
"""
#: These values will be present in a config object unless they are explicitly defined otherwise in the config file
default_values = {}
def __init__(self, path=None):
"""
:param string path: (Optional) Path to config file location.
"""
self._path = path
# Use CHEMDATAEXTRACTOR_CONFIG environment variable if set
if not self._path:
self._path = os.environ.get('CHEMDATAEXTRACTOR_CONFIG')
# Use OS-dependent config directory given by appdirs
if not self._path:
self._path = os.path.join(
appdirs.user_config_dir('ChemDataExtractor'), 'config.cfg')
self._parser = SafeConfigParser()
if os.path.isfile(self.path):
with open(self.path) as f:
self._parser.readfp(f)
if not self._parser.has_section('cde'):
self._parser.add_section('cde')
for k, v in six.iteritems(self.default_values):
if not self._parser.has_option('cde', k.encode('utf-8')):
self._parser.set('cde', k.encode('utf-8'), v.encode('utf-8'))
@property
def path(self):
"""The path to the config file."""
return self._path
def _flush(self):
"""Save the contents of data to the file on disk. You should not need to call this manually."""
d = os.path.dirname(self.path)
if not os.path.isdir(d):
os.makedirs(d)
with open(self.path, 'w') as f:
self._parser.write(f)
def __contains__(self, k):
return self._parser.has_option('cde', k.encode('utf-8'))
def __getitem__(self, k):
try:
return self._parser.get('cde', k.encode('utf-8'))
except NoOptionError:
raise KeyError(k)
def __setitem__(self, k, v):
self._parser.set('cde', k.encode('utf-8'), v.encode('utf-8'))
self._flush()
def __delitem__(self, k):
try:
self._parser.remove_option('cde', k.encode('utf-8'))
self._flush()
except NoOptionError:
raise KeyError(k)
def __iter__(self):
return (k for k, v in self._parser.items('cde'))
def __len__(self):
return len(self._parser.items('cde'))
def __repr__(self):
return '<Config: %s>' % self.path
def clear(self):
"""Clear all values from config."""
self._parser.remove_section('cde')
self._parser.add_section('cde')
self._flush()
def __init__(self,
email=None,
api_key=None,
config_file=None,
verbose=False,
retry_on_errors=True,
site=None,
client=None,
cert_bundle=None,
insecure=None,
client_cert=None,
client_cert_key=None):
# type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[str], Optional[str]) -> None
if client is None:
client = _default_client()
# Normalize user-specified path
if config_file is not None:
config_file = os.path.abspath(os.path.expanduser(config_file))
# Fill values from Environment Variables if not available in Constructor
if config_file is None:
config_file = os.environ.get("ZULIP_CONFIG")
if api_key is None:
api_key = os.environ.get("ZULIP_API_KEY")
if email is None:
email = os.environ.get("ZULIP_EMAIL")
if site is None:
site = os.environ.get("ZULIP_SITE")
if client_cert is None:
client_cert = os.environ.get("ZULIP_CERT")
if client_cert_key is None:
client_cert_key = os.environ.get("ZULIP_CERT_KEY")
if cert_bundle is None:
cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")
if insecure is None:
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = os.environ.get('ZULIP_ALLOW_INSECURE')
if insecure_setting is not None:
insecure = validate_boolean_field(insecure_setting)
if insecure is None:
raise ZulipError(
"The ZULIP_ALLOW_INSECURE environment "
"variable is set to '{}', it must be "
"'true' or 'false'".format(insecure_setting))
if config_file is None:
config_file = get_default_config_filename()
if config_file is not None and os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if client_cert is None and config.has_option("api", "client_cert"):
client_cert = config.get("api", "client_cert")
if client_cert_key is None and config.has_option(
"api", "client_cert_key"):
client_cert_key = config.get("api", "client_cert_key")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get('api', 'insecure')
insecure = validate_boolean_field(insecure_setting)
if insecure is None:
raise ZulipError(
"insecure is set to '{}', it must be "
"'true' or 'false' if it is used in {}".format(
insecure_setting, config_file))
elif None in (api_key, email):
raise ConfigNotFoundError(
"api_key or email not specified and file %s does not exist" %
(config_file, ))
assert (api_key is not None and email is not None)
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if site.startswith("localhost"):
site = "http://" + site
elif not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
raise MissingURLError(
"Missing Zulip server URL; specify via --site or ~/.zuliprc.")
if not self.base_url.endswith("/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
logger.warning('Insecure mode enabled. The server\'s SSL/TLS '
'certificate will not be validated, making the '
'HTTPS connection potentially insecure')
self.tls_verification = False # type: Union[bool, str]
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise ConfigNotFoundError("tls bundle '%s' does not exist" %
(cert_bundle, ))
self.tls_verification = cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification = True
if client_cert is None:
if client_cert_key is not None:
raise ConfigNotFoundError(
"client cert key '%s' specified, but no client cert public part provided"
% (client_cert_key, ))
else: # we have a client cert
if not os.path.isfile(client_cert):
raise ConfigNotFoundError("client cert '%s' does not exist" %
(client_cert, ))
if client_cert_key is not None:
if not os.path.isfile(client_cert_key):
raise ConfigNotFoundError(
"client cert key '%s' does not exist" %
(client_cert_key, ))
self.client_cert = client_cert
self.client_cert_key = client_cert_key
self.session = None # type: Optional[requests.Session]
self.has_connected = False
if options.zulip_config_file is None:
config_file = zulip.get_default_config_filename()
else:
config_file = options.zulip_config_file
config = SafeConfigParser()
try:
with open(config_file, 'r') as f:
config.readfp(f, config_file)
except IOError:
pass
for option in ("jid", "jabber_password", "conference_domain", "mode",
"zulip_email_suffix", "jabber_server_address",
"jabber_server_port"):
if (getattr(options, option) is None
and config.has_option("jabber_mirror", option)):
setattr(options, option, config.get("jabber_mirror", option))
for option in ("no_use_tls", ):
if getattr(options, option) is None:
if config.has_option("jabber_mirror", option):
setattr(options, option,
config.getboolean("jabber_mirror", option))
else:
setattr(options, option, False)
if options.mode is None:
options.mode = "personal"
if options.zulip_email_suffix is None:
options.zulip_email_suffix = ''
def load_conf():
conf_path = os.path.expanduser(config_path)
if not os.path.exists(conf_path):
logger.warn("{conf} couldn't be found, please use \'coscmd config -h\' to learn how to config coscmd!".format(conf=to_printable_str(conf_path)))
raise IOError
else:
logger.debug('{conf} is found'.format(conf=to_printable_str(conf_path)))
try:
with open(conf_path, 'r') as f:
cp = SafeConfigParser()
cp.readfp(fp=f)
if not cp.has_section('common'):
raise Exception("[common] section could't be found, please check your config file.")
if cp.has_option('common', 'part_size'):
part_size = cp.getint('common', 'part_size')
else:
part_size = 1
if cp.has_option('common', 'max_thread'):
max_thread = cp.getint('common', 'max_thread')
else:
max_thread = 5
try:
secret_id = cp.get('common', 'secret_id')
except Exception:
secret_id = cp.get('common', 'access_id')
try:
appid = cp.get('common', 'appid')
bucket = cp.get('common', 'bucket')
if bucket.endswith("-"+str(appid)):
bucket = bucket.rstrip(appid)
bucket = bucket[:-1]
except Exception:
try:
bucket = cp.get('common', 'bucket')
appid = bucket.split('-')[-1]
bucket = bucket.rstrip(appid)
bucket = bucket[:-1]
except Exception:
# check if user use -b bucket
if (pre_bucket == ""):
logger.error("The configuration file is wrong. Please reconfirm")
try:
schema = cp.get('common', 'schema')
except Exception:
schema = 'https'
try:
verify = cp.get('common', 'verify')
except Exception:
verify = 'md5'
try:
token = cp.get('common', 'token')
except Exception:
token = None
try:
error = cp.get('common', 'error')
except Exception:
error = "cos.err.log"
try:
success = cp.get('common', 'success')
except Exception:
success = "cos.suc.log"
try:
anonymous = cp.get('common', 'anonymous')
if anonymous == 'True' or anonymous == 'true':
anonymous = True
else:
anonymous = False
except Exception:
anonymous = False
try:
retry = int(cp.get('common', 'retry'))
except Exception:
retry = 2
try:
timeout = int(cp.get('common', 'timeout'))
except Exception:
timeout = 60
region, endpoint = None, None
if cp.has_option('common', 'region'):
region = cp.get('common', 'region')
if cp.has_option('common', 'endpoint'):
endpoint = cp.get('common', 'endpoint')
if pre_appid != "":
appid = pre_appid
if pre_bucket != "":
bucket = pre_bucket
if pre_region != "":
region = pre_region
conf = CoscmdConfig(
appid=appid,
secret_id=secret_id,
secret_key=cp.get('common', 'secret_key'),
token=token,
region=compatible(region),
endpoint=endpoint,
bucket=bucket,
part_size=part_size,
max_thread=max_thread,
schema=schema,
anonymous=anonymous,
verify=verify,
retry=retry,
timeout=timeout,
error=error,
success=success
)
return conf
except Exception as e:
raise(e)
def createThemeFromTemplate(title, description, baseOn='template'):
"""Create a new theme from the given title and description based on
another theme resource directory
"""
source = queryResourceDirectory(THEME_RESOURCE_NAME, baseOn)
if source is None:
raise KeyError("Theme {0:s} not found".format(baseOn))
themeName = getUtility(IURLNormalizer).normalize(title)
if isinstance(themeName, six.text_type):
themeName = themeName.encode('utf-8')
resources = getOrCreatePersistentResourceDirectory()
resources.makeDirectory(themeName)
target = resources[themeName]
cloneResourceDirectory(source, target)
manifest = SafeConfigParser()
if MANIFEST_FILENAME in target:
fp = target.openFile(MANIFEST_FILENAME)
try:
manifest.readfp(fp)
finally:
fp.close()
if not manifest.has_section('theme'):
manifest.add_section('theme')
if not isinstance(title, str):
title = title.encode('utf-8')
if not isinstance(description, str):
description = description.encode('utf-8')
manifest.set('theme', 'title', title)
manifest.set('theme', 'description', description)
if manifest.has_option('theme', 'prefix'):
prefix = u"/++%s++%s" % (THEME_RESOURCE_NAME, themeName)
manifest.set('theme', 'prefix', prefix)
if manifest.has_option('theme', 'rules'):
rule = manifest.get('theme', 'rules')
rule_file_name = rule.split('/')[-1] # extract real rules file name
rules = u"/++%s++%s/%s" % (THEME_RESOURCE_NAME, themeName,
rule_file_name)
manifest.set('theme', 'rules', rules)
paths_to_fix = [
'development-css', 'production-css', 'tinymce-content-css',
'development-js', 'production-js'
]
for var_path in paths_to_fix:
if not manifest.has_option('theme', var_path):
continue
val = manifest.get('theme', var_path)
if not val:
continue
template_prefix = '++%s++%s/' % (THEME_RESOURCE_NAME, baseOn)
if template_prefix in val:
# okay, fix
val = val.replace(template_prefix,
'++%s++%s/' % (THEME_RESOURCE_NAME, themeName))
manifest.set('theme', var_path, val)
manifestContents = StringIO()
manifest.write(manifestContents)
target.writeFile(MANIFEST_FILENAME, manifestContents)
return themeName
class Config(object):
"""
Manages the configuration file
"""
def __init__(self):
"""
DEFAULT VALUES
"""
self._basescript = None
self.recentvaults = []
self.pwlength = 10
self.search_notes = False
self.search_passwd = False
self.use_pwgen = False
# self.alphabet = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_"
self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_"
self.avoid_bigrams = "cl mn nm nn rn vv VV"
self._fname = self.get_config_filename()
print("Using config %s" % self._fname)
self._parser = SafeConfigParser()
if os.path.exists(self._fname):
self._parser.read(self._fname)
if not self._parser.has_section("base"):
self._parser.add_section("base")
for num in range(10):
if (not self._parser.has_option("base",
"recentvaults" + str(num))):
break
self.recentvaults.append(
self._parser.get("base", "recentvaults" + str(num)))
if self._parser.has_option("base", "pwlength"):
self.pwlength = int(self._parser.get("base", "pwlength"))
if self._parser.has_option("base", "search_notes"):
if self._parser.get("base", "search_notes") == "True":
self.search_notes = True
if self._parser.has_option("base", "search_passwd"):
if self._parser.get("base", "search_passwd") == "True":
self.search_passwd = True
if self._parser.has_option("base", "use_pwgen"):
if self._parser.get("base", "use_pwgen") == "True":
self.use_pwgen = True
if self._parser.has_option("base", "alphabet"):
self.alphabet = self._parser.get("base", "alphabet")
if self._parser.has_option("base", "avoid_bigrams"):
self.avoid_bigrams = self._parser.get("base", "avoid_bigrams")
if not os.path.exists(self._fname):
self.save()
def set_basescript(self, basescript):
self._basescript = basescript
def get_basescript(self):
return self._basescript
def save(self):
if (not os.path.exists(os.path.dirname(self._fname))):
os.mkdir(os.path.dirname(self._fname))
# remove duplicates and trim to 10 items
_saved_recentvaults = []
for item in self.recentvaults:
if item in _saved_recentvaults:
continue
self._parser.set("base",
"recentvaults" + str(len(_saved_recentvaults)),
item)
_saved_recentvaults.append(item)
if (len(_saved_recentvaults) >= 10):
break
self._parser.set("base", "pwlength", str(self.pwlength))
self._parser.set("base", "search_notes", str(self.search_notes))
self._parser.set("base", "search_passwd", str(self.search_passwd))
self._parser.set("base", "use_pwgen", str(self.use_pwgen))
self._parser.set("base", "alphabet", str(self.alphabet))
self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams))
filehandle = open(self._fname, 'w')
self._parser.write(filehandle)
filehandle.close()
@staticmethod
def get_config_filename():
"""
Returns the full filename of the config file
"""
base_fname = "loxodo"
# On Mac OS X, config files go to ~/Library/Application Support/foo/
if platform.system() == "Darwin":
base_path = os.path.join(os.path.expanduser("~"), "Library",
"Application Support")
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# On Microsoft Windows, config files go to $APPDATA/foo/
if platform.system() in ("Windows", "Microsoft"):
if ("APPDATA" in os.environ):
base_path = os.environ["APPDATA"]
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname,
base_fname + ".ini")
# Allow config directory override as per freedesktop.org XDG Base Directory Specification
if ("XDG_CONFIG_HOME" in os.environ):
base_path = os.environ["XDG_CONFIG_HOME"]
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# Default configuration path is ~/.config/foo/
base_path = os.path.join(os.path.expanduser("~"), ".config")
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
else:
return os.path.join(os.path.expanduser("~"),
"." + base_fname + ".ini")
def loadConfig(self):
"""Load build configuration from the spec file."""
specfiles = [
spec for spec in os.listdir(self.spec_dir) if spec.endswith('.ini')
]
if len(specfiles) == 0:
raise BuildError('No .ini file found')
elif len(specfiles) > 1:
raise BuildError('Multiple .ini files found')
if six.PY2:
conf = SafeConfigParser()
else:
conf = ConfigParser()
conf.read(os.path.join(self.spec_dir, specfiles[0]))
# [naming] section
for entry in ('name', 'version', 'release', 'description'):
setattr(self, entry, conf.get('naming', entry))
if conf.has_option('naming', 'epoch'):
self.epoch = conf.get('naming', 'epoch')
# [building] section
self.platform = conf.get('building', 'platform')
# preinstalled are paths to files or directories that must exist
# in the VM for it to execute the build.
# If the path ends in / or \ it must be a directory, otherwise it must
# be a file.
# They may be specified as Cygwin (/cygdrive/c/...) or Windows (C:\...)
# absolute paths, or without a path in which case it is searched for
# on the PATH.
if conf.has_option('building', 'preinstalled'):
self.preinstalled.extend([
e.strip()
for e in conf.get('building', 'preinstalled').split('\n') if e
])
# buildrequires and provides are multi-valued (space-separated)
for br in conf.get('building', 'buildrequires').split():
# buildrequires is a space-separated list
# each item in the list is in the format:
# pkgname[:opt1:opt2=val2:...]
# the options are put into a dict
# if the option has no =val, the value in the dict will be None
if br:
br = br.split(':')
bropts = {}
for opt in br[1:]:
if '=' in opt:
key, val = opt.split('=', 1)
else:
key = opt
val = None
bropts[key] = val
self.buildrequires.append((br[0], bropts))
for prov in conf.get('building', 'provides').split():
if prov:
self.provides.append(prov)
# optionally specify a shell to use (defaults to bash)
# valid values are: cmd, cmd.exe (alias for cmd), and bash
if conf.has_option('building', 'shell'):
self.shell = conf.get('building', 'shell')
else:
self.shell = 'bash'
# execute is multi-valued (newline-separated)
self.execute.extend([
e.strip() for e in conf.get('building', 'execute').split('\n') if e
])
# postbuild are files or directories that must exist after the build is
# complete, but are not included in the build output
# they are specified as paths relative the source directory, and may be
# in Unix or Windows format
# each entry may contain shell-style globs, and one or more files
# matching the glob is considered valid
if conf.has_option('building', 'postbuild'):
for entry in conf.get('building', 'postbuild').split('\n'):
entry = entry.strip()
if not entry:
continue
for var in ('name', 'version', 'release'):
entry = entry.replace('$' + var, getattr(self, var))
self.postbuild.append(entry)
# [files] section
for entry in conf.get('files', 'output').split('\n'):
entry = entry.strip()
if not entry:
continue
tokens = entry.split(':')
filename = tokens[0]
for var in ('name', 'version', 'release'):
filename = filename.replace('$' + var, getattr(self, var))
metadata = {}
metadata['platforms'] = tokens[1].split(',')
if len(tokens) > 2:
metadata['flags'] = tokens[2].split(',')
else:
metadata['flags'] = []
self.output[filename] = metadata
self.logs.extend(
[e.strip() for e in conf.get('files', 'logs').split('\n') if e])
def __init__(self, email=None, api_key=None, config_file=None,
verbose=False, retry_on_errors=True,
site=None, client=None,
cert_bundle=None, insecure=None,
client_cert=None, client_cert_key=None):
# type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], Optional[bool], Optional[str], Optional[str]) -> None
if client is None:
client = _default_client()
# Normalize user-specified path
if config_file is not None:
config_file = os.path.abspath(os.path.expanduser(config_file))
# Fill values from Environment Variables if not available in Constructor
if config_file is None:
config_file = os.environ.get("ZULIP_CONFIG")
if api_key is None:
api_key = os.environ.get("ZULIP_API_KEY")
if email is None:
email = os.environ.get("ZULIP_EMAIL")
if site is None:
site = os.environ.get("ZULIP_SITE")
if client_cert is None:
client_cert = os.environ.get("ZULIP_CERT")
if client_cert_key is None:
client_cert_key = os.environ.get("ZULIP_CERT_KEY")
if cert_bundle is None:
cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")
if insecure is None:
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = os.environ.get('ZULIP_ALLOW_INSECURE')
if insecure_setting is not None:
insecure = validate_boolean_field(insecure_setting)
if insecure is None:
raise ZulipError("The ZULIP_ALLOW_INSECURE environment "
"variable is set to '{}', it must be "
"'true' or 'false'"
.format(insecure_setting))
if config_file is None:
config_file = get_default_config_filename()
if config_file is not None and os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if client_cert is None and config.has_option("api", "client_cert"):
client_cert = config.get("api", "client_cert")
if client_cert_key is None and config.has_option("api", "client_cert_key"):
client_cert_key = config.get("api", "client_cert_key")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get('api', 'insecure')
insecure = validate_boolean_field(insecure_setting)
if insecure is None:
raise ZulipError("insecure is set to '{}', it must be "
"'true' or 'false' if it is used in {}"
.format(insecure_setting, config_file))
elif None in (api_key, email):
raise ConfigNotFoundError("api_key or email not specified and file %s does not exist"
% (config_file,))
assert(api_key is not None and email is not None)
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if site.startswith("localhost"):
site = "http://" + site
elif not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
raise MissingURLError("Missing Zulip server URL; specify via --site or ~/.zuliprc.")
if not self.base_url.endswith("/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
logger.warning('Insecure mode enabled. The server\'s SSL/TLS '
'certificate will not be validated, making the '
'HTTPS connection potentially insecure')
self.tls_verification = False # type: Union[bool, str]
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise ConfigNotFoundError("tls bundle '%s' does not exist"
% (cert_bundle,))
self.tls_verification = cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification = True
if client_cert is None:
if client_cert_key is not None:
raise ConfigNotFoundError("client cert key '%s' specified, but no client cert public part provided"
% (client_cert_key,))
else: # we have a client cert
if not os.path.isfile(client_cert):
raise ConfigNotFoundError("client cert '%s' does not exist"
% (client_cert,))
if client_cert_key is not None:
if not os.path.isfile(client_cert_key):
raise ConfigNotFoundError("client cert key '%s' does not exist"
% (client_cert_key,))
self.client_cert = client_cert
self.client_cert_key = client_cert_key
self.session = None # type: Optional[requests.Session]
self.has_connected = False
class PackageConfigHandler(object):
"""
Manager class for packages files for tracking installation of modules
"""
def __init__(self):
# noinspection PyUnresolvedReferences
from six.moves.configparser import SafeConfigParser
self.package_cfg = os.path.expanduser('~/Documents/site-packages/.pypi_packages')
if not os.path.isfile(self.package_cfg):
print('Creating package file')
with open(self.package_cfg, 'w') as outs:
outs.close()
self.parser = SafeConfigParser()
self.parser.read(self.package_cfg)
def save(self):
with open(self.package_cfg, 'w') as outs:
self.parser.write(outs)
def add_module(self, pkg_info):
"""
:param pkg_info: A dict that has name, url, version, summary
:return:
"""
if not self.parser.has_section(pkg_info['name']):
self.parser.add_section(pkg_info['name'])
self.parser.set(pkg_info['name'], 'url', pkg_info['url'])
self.parser.set(pkg_info['name'], 'version', pkg_info['version'])
self.parser.set(pkg_info['name'], 'summary', pkg_info['summary'])
self.parser.set(pkg_info['name'], 'files', pkg_info['files'])
self.parser.set(pkg_info['name'], 'dependency', pkg_info['dependency'])
self.save()
def list_modules(self):
return [module for module in self.parser.sections()]
def module_exists(self, name):
return self.parser.has_section(name)
def get_info(self, name):
if self.parser.has_section(name):
tbl = {}
for opt, value in self.parser.items(name):
tbl[opt] = value
return tbl
def remove_module(self, name):
self.parser.remove_section(name)
self.save()
def get_files_installed(self, section_name):
if self.parser.has_option(section_name, 'files'):
files = self.parser.get(section_name, 'files').strip()
return files.split(',')
else:
return None
def get_dependencies(self, section_name):
if self.parser.has_option(section_name, 'dependency'):
dependencies = self.parser.get(section_name, 'dependency').strip()
return set(dependencies.split(',')) if dependencies != '' else set()
else:
return None
def get_all_dependencies(self, exclude_module=()):
all_dependencies = set()
for section_name in self.parser.sections():
if section_name not in exclude_module and self.parser.has_option(section_name, 'dependency'):
dependencies = self.parser.get(section_name, 'dependency').strip()
if dependencies != '':
for dep in dependencies.split(','):
all_dependencies.add(dep)
return all_dependencies
def parse_config_file(self, config_file):
config = SafeConfigParser(self.DEFAULT_CONFIG)
config.readfp(config_file)
blessconfig = {
'CLIENT_CONFIG': {
'domain_regex': config.get('CLIENT', 'domain_regex'),
'cache_dir': config.get('CLIENT', 'cache_dir'),
'cache_file': config.get('CLIENT', 'cache_file'),
'mfa_cache_dir': config.get('CLIENT', 'mfa_cache_dir'),
'mfa_cache_file': config.get('CLIENT', 'mfa_cache_file'),
'ip_urls': [s.strip() for s in config.get('CLIENT', 'ip_urls').split(",")],
'update_script': config.get('CLIENT', 'update_script'),
'user_session_length': int(config.get('CLIENT', 'user_session_length')),
'usebless_role_session_length': int(config.get('CLIENT', 'usebless_role_session_length')),
'update_sshagent': config.getboolean('CLIENT', 'update_sshagent'),
'use_env_creds': config.getboolean('CLIENT', 'use_env_creds'),
},
'BLESS_CONFIG': {
'ca_backend': config.get('MAIN', 'ca_backend'),
'userrole': config.get('LAMBDA', 'user_role'),
'accountid': config.get('LAMBDA', 'account_id'),
'functionname': config.get('LAMBDA', 'functionname'),
'functionversion': config.get('LAMBDA', 'functionversion'),
'certlifetime': config.getint('LAMBDA', 'certlifetime'),
'ipcachelifetime': config.getint('LAMBDA', 'ipcachelifetime'),
'timeoutconfig': {
'connect': config.getint('LAMBDA', 'timeout_connect'),
'read': config.getint('LAMBDA', 'timeout_read')
}
},
'AWS_CONFIG': {
'remote_user': config.get('MAIN', 'remote_user')
},
'REGION_ALIAS': {}
}
if config.has_option('MAIN', 'bastion_ips'):
blessconfig['AWS_CONFIG']['bastion_ips'] = config.get('MAIN', 'bastion_ips')
if blessconfig['BLESS_CONFIG']['ca_backend'].lower() == 'hashicorp-vault':
blessconfig['VAULT_CONFIG'] = {
'vault_addr': config.get('VAULT', 'vault_addr'),
'auth_mount': config.get('VAULT', 'auth_mount'),
'ssh_backend_mount': config.get('VAULT', 'ssh_backend_mount'),
'ssh_backend_role': config.get('VAULT', 'ssh_backend_role'),
}
regions = config.get('MAIN', 'region_aliases').split(",")
regions = [region.strip() for region in regions]
for region in regions:
region = region.upper()
kms_region_key = 'KMSAUTH_CONFIG_{}'.format(region)
blessconfig.update({kms_region_key: self._get_region_kms_config(region, config)})
blessconfig['REGION_ALIAS'].update({region: blessconfig[kms_region_key]['awsregion']})
if config.has_section('HOUSEKEEPER'):
hk_region_key = 'HOUSEKEEPER_CONFIG_{}'.format(region)
blessconfig.update({hk_region_key: self._get_region_housekeeper_config(region, config)})
blessconfig['REGION_ALIAS'].update({region: blessconfig[hk_region_key]['awsregion']})
return blessconfig
def has_option(self, name, option_name):
section_name = self._get_section_name(name)
return SafeConfigParser.has_option(self, section_name, option_name)
def onDiscovery(self, theme, settings, dependenciesSettings):
res = queryResourceDirectory(THEME_RESOURCE_NAME, theme)
if res is None:
return
directoryName = 'views'
if 'directory' in settings:
directoryName = settings['directory']
if res.isDirectory(directoryName):
viewsDir = res[directoryName]
layer = getattr(schemata, theme, None)
if 'layer' in settings:
layerName = settings['layer']
try:
layer = resolve(layerName)
except (ImportError, AttributeError):
logger.warn("Could not import %s" % layerName)
return
viewConfig = SafeConfigParser()
if viewsDir.isFile(VIEW_CONFIG_FILENAME):
fp = viewsDir.openFile(VIEW_CONFIG_FILENAME)
try:
viewConfig.readfp(fp)
finally:
try:
fp.close()
except AttributeError:
pass
views = []
configurationMachine = ConfigurationMachine()
path = viewsDir.directory
for filename in os.listdir(path):
if not filename.lower().endswith(EXTENSION):
continue
name = viewName = filename[:-3]
permission = 'zope2.View'
for_ = Interface
class_ = None
template = os.path.join(path, filename)
menu = {}
# Read override options from views.cfg if applicable
if viewConfig.has_section(name):
if viewConfig.has_option(name, 'name'):
viewName = viewConfig.get(name, 'name')
if viewConfig.has_option(name, 'permission'):
permission = viewConfig.get(name, 'permission')
if viewConfig.has_option(name, 'for'):
forStr = viewConfig.get(name, 'for')
if forStr != "*":
for_ = resolve(forStr)
if viewConfig.has_option(name, 'class'):
class_ = resolve(viewConfig.get(name, 'class'))
if viewConfig.has_option(name, 'menu'):
menu = dict(
title=viewConfig.get(name, 'menu'),
menu=getattr(
zope.browsermenu.metaconfigure.menus,
"plone_displayviews",
),
)
Products.Five.browser.metaconfigure.page(
configurationMachine,
name=viewName,
permission=permission,
for_=for_,
layer=layer,
template=template,
class_=class_,
**menu
)
views.append(name)
if len(views) > 0:
configurationMachine.execute_actions()
self.registered[theme] = views
def _parse_args(args_str):
keystone_auth_parser = SafeConfigParser()
conf_file = keystone_auth_parser.read(
'/etc/contrail/contrail-keystone-auth.conf')
default_keystone_vals = {
"username": "******",
"tenant_name": "admin",
"domain_name": "Default"
}
get_vars = (lambda x: keystone_auth_parser.get('KEYSTONE', x)
if keystone_auth_parser.has_option('KEYSTONE', x) else None)
if conf_file:
if keystone_auth_parser.has_section('KEYSTONE'):
username = get_vars('admin_user')
if username:
default_keystone_vals['username'] = username
password = get_vars('admin_password')
if password:
default_keystone_vals['password'] = password
tenant_name = get_vars('admin_tenant_name')
if tenant_name:
default_keystone_vals['tenant_name'] = tenant_name
domain_name = get_vars('user_domain_name')
if domain_name:
default_keystone_vals['domain_name'] = domain_name
parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter, description='')
parser.add_argument('-v',
'--version',
action='version',
version='%(prog)s ' + __version__)
parser.add_argument("--debug",
help="Run in debug mode, default False",
action='store_true',
default=False)
parser.add_argument("--to-json", help="File to dump json to", default=None)
parser.add_argument("--username",
help="Username used to login to API Server")
parser.add_argument("--password",
help="Password used to login to API Server")
parser.add_argument("--api-server-host", help="IP of API Server")
parser.add_argument("--tenant-name", help="Name of Tenant")
parser.add_argument("--domain-name", help="Domain name")
parser.add_argument("--read-timeout",
dest='timeout',
help="Timeout for Data reading operations",
default=300)
parser.add_argument(
"--connection-timeout",
help="Timeout set for VNC API to connect to API Server",
default=120)
ts = calendar.timegm(time.gmtime())
if os.path.isdir("/var/log/contrail"):
default_log = "/var/log/contrail/fabric_validation-{0}.log".format(ts)
else:
import tempfile
default_log = '{0}/fabric_validation-{1}.log'.format(
tempfile.gettempdir(), ts)
parser.add_argument("--log_file",
help="Log file to save output, default '%(default)s'",
default=default_log)
parser.set_defaults(**default_keystone_vals)
args_obj, _ = parser.parse_known_args(args_str.split())
_args = args_obj
return _args
def __init__(self,
email=None,
api_key=None,
config_file=None,
verbose=False,
retry_on_errors=True,
site=None,
client=None,
cert_bundle=None,
insecure=None,
client_cert=None,
client_cert_key=None):
if client is None:
client = _default_client()
if config_file is None:
config_file = get_default_config_filename()
if os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if client_cert is None and config.has_option("api", "client_cert"):
client_cert = config.get("api", "client_cert")
if client_cert_key is None and config.has_option(
"api", "client_cert_key"):
client_cert_key = config.get("api", "client_cert_key")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get("api", "insecure").lower()
if insecure_setting == "true":
insecure = True
elif insecure_setting == "false":
insecure = False
else:
raise RuntimeError(
"insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
% (insecure_setting, config_file))
elif None in (api_key, email):
raise RuntimeError(
"api_key or email not specified and %s does not exist" %
(config_file, ))
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
self.base_url = "https://api.zulip.com"
if self.base_url != "https://api.zulip.com" and not self.base_url.endswith(
"/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
self.tls_verification = False
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise RuntimeError("tls bundle '%s' does not exist" %
(cert_bundle, ))
self.tls_verification = cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification = True
if client_cert is None:
if client_cert_key is not None:
raise RuntimeError(
"client cert key '%s' specified, but no client cert public part provided"
% (client_cert_key, ))
else: # we have a client cert
if not os.path.isfile(client_cert):
raise RuntimeError("client cert '%s' does not exist" %
(client_cert, ))
if client_cert_key is not None:
if not os.path.isfile(client_cert_key):
raise RuntimeError("client cert key '%s' does not exist" %
(client_cert_key, ))
self.client_cert = client_cert
self.client_cert_key = client_cert_key
def has_option(self, name, option_name):
section_name = self._get_section_name(name)
return SafeConfigParser.has_option(self, section_name, option_name)
def __init__(self, email=None, api_key=None, config_file=None,
verbose=False, retry_on_errors=True,
site=None, client=None,
cert_bundle=None, insecure=None):
if client is None:
client = _default_client()
if config_file is None:
config_file = get_default_config_filename()
if os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get("api", "insecure").lower()
if insecure_setting == "true":
insecure = True
elif insecure_setting == "false":
insecure = False
else:
raise RuntimeError("insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
% (insecure_setting, config_file))
elif None in (api_key, email):
raise RuntimeError("api_key or email not specified and %s does not exist"
% (config_file,))
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
self.base_url = "https://api.zulip.com"
if self.base_url != "https://api.zulip.com" and not self.base_url.endswith("/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
self.tls_verification=False
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise RuntimeError("tls bundle '%s' does not exist"
%(cert_bundle,))
self.tls_verification=cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification=True
class Csw(object):
""" Base CSW server """
def __init__(self, rtconfig=None, env=None, version='3.0.0'):
""" Initialize CSW """
if not env:
self.environ = os.environ
else:
self.environ = env
self.context = config.StaticContext()
# Lazy load this when needed
# (it will permanently update global cfg namespaces)
self.sruobj = None
self.opensearchobj = None
self.oaipmhobj = None
# init kvp
self.kvp = {}
self.mode = 'csw'
self.asynchronous = False
self.soap = False
self.request = None
self.exception = False
self.status = 'OK'
self.profiles = None
self.manager = False
self.outputschemas = {}
self.mimetype = 'application/xml; charset=UTF-8'
self.encoding = 'UTF-8'
self.pretty_print = 0
self.domainquerytype = 'list'
self.orm = 'django'
self.language = {'639_code': 'en', 'text': 'english'}
self.process_time_start = time()
# define CSW implementation object (default CSW3)
self.iface = csw3.Csw3(server_csw=self)
self.request_version = version
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# load user configuration
try:
LOGGER.info('Loading user configuration')
if isinstance(rtconfig, SafeConfigParser): # serialized already
self.config = rtconfig
else:
self.config = SafeConfigParser()
if isinstance(rtconfig, dict): # dictionary
for section, options in rtconfig.items():
self.config.add_section(section)
for k, v in options.items():
self.config.set(section, k, v)
else: # configuration file
import codecs
with codecs.open(rtconfig, encoding='utf-8') as scp:
self.config.readfp(scp)
except Exception as err:
msg = 'Could not load configuration'
LOGGER.exception('%s %s: %s', msg, rtconfig, err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service', msg)
return
# set server.home safely
# TODO: make this more abstract
self.config.set(
'server', 'home',
os.path.dirname(os.path.join(os.path.dirname(__file__), '..'))
)
self.context.pycsw_home = self.config.get('server', 'home')
self.context.url = self.config.get('server', 'url')
log.setup_logger(self.config)
LOGGER.info('running configuration %s', rtconfig)
LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING'])
# set OGC schemas location
if not self.config.has_option('server', 'ogc_schemas_base'):
self.config.set('server', 'ogc_schemas_base',
self.context.ogc_schemas_base)
# set mimetype
if self.config.has_option('server', 'mimetype'):
self.mimetype = self.config.get('server', 'mimetype').encode()
# set encoding
if self.config.has_option('server', 'encoding'):
self.encoding = self.config.get('server', 'encoding')
# set domainquerytype
if self.config.has_option('server', 'domainquerytype'):
self.domainquerytype = self.config.get('server', 'domainquerytype')
# set XML pretty print
if (self.config.has_option('server', 'pretty_print') and
self.config.get('server', 'pretty_print') == 'true'):
self.pretty_print = 1
# set Spatial Ranking option
if (self.config.has_option('server', 'spatial_ranking') and
self.config.get('server', 'spatial_ranking') == 'true'):
util.ranking_enabled = True
# set language default
if self.config.has_option('server', 'language'):
try:
LOGGER.info('Setting language')
lang_code = self.config.get('server', 'language').split('-')[0]
self.language['639_code'] = lang_code
self.language['text'] = self.context.languages[lang_code]
except Exception as err:
LOGGER.exception('Could not set language: %s', err)
pass
LOGGER.debug('Configuration: %s.', self.config)
LOGGER.debug('Model: %s.', self.context.model)
# load user-defined mappings if they exist
if self.config.has_option('repository', 'mappings'):
# override default repository mappings
try:
import imp
module = self.config.get('repository', 'mappings')
if os.sep in module: # filepath
modulename = '%s' % os.path.splitext(module)[0].replace(
os.sep, '.')
mappings = imp.load_source(modulename, module)
else: # dotted name
mappings = __import__(module, fromlist=[''])
LOGGER.info('Loading custom repository mappings '
'from %s', module)
self.context.md_core_model = mappings.MD_CORE_MODEL
self.context.refresh_dc(mappings.MD_CORE_MODEL)
except Exception as err:
LOGGER.exception('Could not load custom mappings: %s', err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service',
'Could not load repository.mappings')
# load outputschemas
LOGGER.info('Loading outputschemas')
for osch in pycsw.plugins.outputschemas.__all__:
output_schema_module = __import__(
'pycsw.plugins.outputschemas.%s' % osch)
mod = getattr(output_schema_module.plugins.outputschemas, osch)
self.outputschemas[mod.NAMESPACE] = mod
LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas)
LOGGER.debug('Namespaces: %s', self.context.namespaces)
def expand_path(self, path):
""" return safe path for WSGI environments """
if 'local.app_root' in self.environ and not os.path.isabs(path):
return os.path.join(self.environ['local.app_root'], path)
else:
return path
def dispatch_wsgi(self):
""" WSGI handler """
if hasattr(self, 'response'):
return self._write_response()
LOGGER.debug('WSGI mode detected')
if self.environ['REQUEST_METHOD'] == 'POST':
try:
request_body_size = int(self.environ.get('CONTENT_LENGTH', 0))
except (ValueError):
request_body_size = 0
self.requesttype = 'POST'
self.request = self.environ['wsgi.input'].read(request_body_size)
LOGGER.debug('Request type: POST. Request:\n%s\n', self.request)
else: # it's a GET request
self.requesttype = 'GET'
self.request = wsgiref.util.request_uri(self.environ)
try:
query_part = splitquery(self.request)[-1]
self.kvp = dict(parse_qsl(query_part, keep_blank_values=True))
except AttributeError as err:
LOGGER.exception('Could not parse query string')
self.kvp = {}
LOGGER.debug('Request type: GET. Request:\n%s\n', self.request)
return self.dispatch()
def opensearch(self):
""" enable OpenSearch """
if not self.opensearchobj:
self.opensearchobj = opensearch.OpenSearch(self.context)
return self.opensearchobj
def sru(self):
""" enable SRU """
if not self.sruobj:
self.sruobj = sru.Sru(self.context)
return self.sruobj
def oaipmh(self):
""" enable OAI-PMH """
if not self.oaipmhobj:
self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config)
return self.oaipmhobj
def dispatch(self, writer=sys.stdout, write_headers=True):
""" Handle incoming HTTP request """
error = 0
if self.requesttype == 'GET':
self.kvp = self.normalize_kvp(self.kvp)
version_202 = ('version' in self.kvp and
self.kvp['version'] == '2.0.2')
accept_version_202 = ('acceptversions' in self.kvp and
'2.0.2' in self.kvp['acceptversions'])
if version_202 or accept_version_202:
self.request_version = '2.0.2'
elif self.requesttype == 'POST':
if self.request.find(b'cat/csw/2.0.2') != -1:
self.request_version = '2.0.2'
elif self.request.find(b'cat/csw/3.0') != -1:
self.request_version = '3.0.0'
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'sru'):
self.mode = 'sru'
self.request_version = '2.0.2'
LOGGER.info('SRU mode detected; processing request')
self.kvp = self.sru().request_sru2csw(self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'oaipmh'):
self.mode = 'oaipmh'
self.request_version = '2.0.2'
LOGGER.info('OAI-PMH mode detected; processing request.')
self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k)
self.kvp = self.oaipmh().request(self.kvp)
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# configure transaction support, if specified in config
self._gen_manager()
namespaces = self.context.namespaces
ops = self.context.model['operations']
constraints = self.context.model['constraints']
# generate domain model
# NOTE: We should probably avoid this sort of mutable state for WSGI
if 'GetDomain' not in ops:
ops['GetDomain'] = self.context.gen_domains()
# generate distributed search model, if specified in config
if self.config.has_option('server', 'federatedcatalogues'):
LOGGER.info('Configuring distributed search')
constraints['FederatedCatalogues'] = {'values': []}
for fedcat in self.config.get('server',
'federatedcatalogues').split(','):
LOGGER.debug('federated catalogue: %s', fedcat)
constraints['FederatedCatalogues']['values'].append(fedcat)
for key, value in self.outputschemas.items():
get_records_params = ops['GetRecords']['parameters']
get_records_params['outputSchema']['values'].append(
value.NAMESPACE)
get_records_by_id_params = ops['GetRecordById']['parameters']
get_records_by_id_params['outputSchema']['values'].append(
value.NAMESPACE)
if 'Harvest' in ops:
harvest_params = ops['Harvest']['parameters']
harvest_params['ResourceType']['values'].append(
value.NAMESPACE)
LOGGER.info('Setting MaxRecordDefault')
if self.config.has_option('server', 'maxrecords'):
constraints['MaxRecordDefault']['values'] = [
self.config.get('server', 'maxrecords')]
# load profiles
if self.config.has_option('server', 'profiles'):
self.profiles = pprofile.load_profiles(
os.path.join('pycsw', 'plugins', 'profiles'),
pprofile.Profile,
self.config.get('server', 'profiles')
)
for prof in self.profiles['plugins'].keys():
tmp = self.profiles['plugins'][prof](self.context.model,
namespaces,
self.context)
key = tmp.outputschema # to ref by outputschema
self.profiles['loaded'][key] = tmp
self.profiles['loaded'][key].extend_core(self.context.model,
namespaces,
self.config)
LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys()))
# init repository
# look for tablename, set 'records' as default
if not self.config.has_option('repository', 'table'):
self.config.set('repository', 'table', 'records')
repo_filter = None
if self.config.has_option('repository', 'filter'):
repo_filter = self.config.get('repository', 'filter')
if self.config.has_option('repository', 'source'): # load custom repository
rs = self.config.get('repository', 'source')
rs_modname, rs_clsname = rs.rsplit('.', 1)
rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname])
rs_cls = getattr(rs_mod, rs_clsname)
try:
self.repository = rs_cls(self.context, repo_filter)
LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype)
except Exception as err:
msg = 'Could not load custom repository %s: %s' % (rs, err)
LOGGER.exception(msg)
error = 1
code = 'NoApplicableCode'
locator = 'service'
text = 'Could not initialize repository. Check server logs'
else: # load default repository
self.orm = 'sqlalchemy'
from pycsw.core import repository
try:
LOGGER.info('Loading default repository')
self.repository = repository.Repository(
self.config.get('repository', 'database'),
self.context,
self.environ.get('local.app_root', None),
self.config.get('repository', 'table'),
repo_filter
)
LOGGER.debug(
'Repository loaded (local): %s.' % self.repository.dbtype)
except Exception as err:
msg = 'Could not load repository (local): %s' % err
LOGGER.exception(msg)
error = 1
code = 'NoApplicableCode'
locator = 'service'
text = 'Could not initialize repository. Check server logs'
if self.requesttype == 'POST':
LOGGER.debug('HTTP POST request')
LOGGER.debug('CSW version: %s', self.iface.version)
self.kvp = self.iface.parse_postdata(self.request)
if isinstance(self.kvp, str): # it's an exception
error = 1
locator = 'service'
text = self.kvp
if (self.kvp.find('the document is not valid') != -1 or
self.kvp.find('document not well-formed') != -1):
code = 'NoApplicableCode'
else:
code = 'InvalidParameterValue'
LOGGER.debug('HTTP Headers:\n%s.', self.environ)
LOGGER.debug('Parsed request parameters: %s', self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'opensearch'):
self.mode = 'opensearch'
LOGGER.info('OpenSearch mode detected; processing request.')
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or
(len(self.kvp) == 1 and 'config' in self.kvp)):
LOGGER.info('Turning on default csw30:Capabilities for base URL')
self.kvp = {
'service': 'CSW',
'acceptversions': '3.0.0',
'request': 'GetCapabilities'
}
http_accept = self.environ.get('HTTP_ACCEPT', '')
if 'application/opensearchdescription+xml' in http_accept:
self.mode = 'opensearch'
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if error == 0:
# test for the basic keyword values (service, version, request)
basic_options = ['service', 'request']
request = self.kvp.get('request', '')
own_version_integer = util.get_version_integer(
self.request_version)
if self.request_version == '2.0.2':
basic_options.append('version')
if self.request_version == '3.0.0' and 'version' not in self.kvp and self.requesttype == 'POST':
if 'service' not in self.kvp:
self.kvp['service'] = 'CSW'
basic_options.append('service')
self.kvp['version'] = self.request_version
basic_options.append('version')
for k in basic_options:
if k not in self.kvp:
if (k in ['version', 'acceptversions'] and
request == 'GetCapabilities'):
pass
else:
error = 1
locator = k
code = 'MissingParameterValue'
text = 'Missing keyword: %s' % k
break
# test each of the basic keyword values
if error == 0:
# test service
if self.kvp['service'] != 'CSW':
error = 1
locator = 'service'
code = 'InvalidParameterValue'
text = 'Invalid value for service: %s.\
Value MUST be CSW' % self.kvp['service']
# test version
kvp_version = self.kvp.get('version', '')
try:
kvp_version_integer = util.get_version_integer(kvp_version)
except Exception as err:
kvp_version_integer = 'invalid_value'
if (request != 'GetCapabilities' and
kvp_version_integer != own_version_integer):
error = 1
locator = 'version'
code = 'InvalidParameterValue'
text = ('Invalid value for version: %s. Value MUST be '
'2.0.2 or 3.0.0' % kvp_version)
# check for GetCapabilities acceptversions
if 'acceptversions' in self.kvp:
for vers in self.kvp['acceptversions'].split(','):
vers_integer = util.get_version_integer(vers)
if vers_integer == own_version_integer:
break
else:
error = 1
locator = 'acceptversions'
code = 'VersionNegotiationFailed'
text = ('Invalid parameter value in '
'acceptversions: %s. Value MUST be '
'2.0.2 or 3.0.0' %
self.kvp['acceptversions'])
# test request
if self.kvp['request'] not in \
self.context.model['operations']:
error = 1
locator = 'request'
if request in ['Transaction', 'Harvest']:
code = 'OperationNotSupported'
text = '%s operations are not supported' % request
else:
code = 'InvalidParameterValue'
text = 'Invalid value for request: %s' % request
if error == 1: # return an ExceptionReport
LOGGER.error('basic service options error: %s, %s, %s', code, locator, text)
self.response = self.iface.exceptionreport(code, locator, text)
else: # process per the request value
if 'responsehandler' in self.kvp:
# set flag to process asynchronously
import threading
self.asynchronous = True
request_id = self.kvp.get('requestid', None)
if request_id is None:
import uuid
self.kvp['requestid'] = str(uuid.uuid4())
if self.kvp['request'] == 'GetCapabilities':
self.response = self.iface.getcapabilities()
elif self.kvp['request'] == 'DescribeRecord':
self.response = self.iface.describerecord()
elif self.kvp['request'] == 'GetDomain':
self.response = self.iface.getdomain()
elif self.kvp['request'] == 'GetRecords':
if self.asynchronous: # process asynchronously
threading.Thread(target=self.iface.getrecords).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.getrecords()
elif self.kvp['request'] == 'GetRecordById':
self.response = self.iface.getrecordbyid()
elif self.kvp['request'] == 'GetRepositoryItem':
self.response = self.iface.getrepositoryitem()
elif self.kvp['request'] == 'Transaction':
self.response = self.iface.transaction()
elif self.kvp['request'] == 'Harvest':
if self.asynchronous: # process asynchronously
threading.Thread(target=self.iface.harvest).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.harvest()
else:
self.response = self.iface.exceptionreport(
'InvalidParameterValue', 'request',
'Invalid request parameter: %s' % self.kvp['request']
)
LOGGER.info('Request processed')
if self.mode == 'sru':
LOGGER.info('SRU mode detected; processing response.')
self.response = self.sru().response_csw2sru(self.response,
self.environ)
elif self.mode == 'opensearch':
LOGGER.info('OpenSearch mode detected; processing response.')
self.response = self.opensearch().response_csw2opensearch(
self.response, self.config)
elif self.mode == 'oaipmh':
LOGGER.info('OAI-PMH mode detected; processing response.')
self.response = self.oaipmh().response(
self.response, self.oaiargs, self.repository,
self.config.get('server', 'url')
)
return self._write_response()
def getcapabilities(self):
""" Handle GetCapabilities request """
return self.iface.getcapabilities()
def describerecord(self):
""" Handle DescribeRecord request """
return self.iface.describerecord()
def getdomain(self):
""" Handle GetDomain request """
return self.iface.getdomain()
def getrecords(self):
""" Handle GetRecords request """
return self.iface.getrecords()
def getrecordbyid(self, raw=False):
""" Handle GetRecordById request """
return self.iface.getrecordbyid(raw)
def getrepositoryitem(self):
""" Handle GetRepositoryItem request """
return self.iface.getrepositoryitem()
def transaction(self):
""" Handle Transaction request """
return self.iface.transaction()
def harvest(self):
""" Handle Harvest request """
return self.iface.harvest()
def _write_response(self):
""" Generate response """
# set HTTP response headers and XML declaration
xmldecl = ''
appinfo = ''
LOGGER.info('Writing response.')
if hasattr(self, 'soap') and self.soap:
self._gen_soap_wrapper()
if etree.__version__ >= '3.5.0': # remove superfluous namespaces
etree.cleanup_namespaces(self.response,
keep_ns_prefixes=self.context.keep_ns_prefixes)
response = etree.tostring(self.response,
pretty_print=self.pretty_print,
encoding='unicode')
if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and
self.kvp['outputformat'] == 'application/json'):
self.contenttype = self.kvp['outputformat']
from pycsw.core.formats import fmt_json
response = fmt_json.xml2json(response,
self.context.namespaces,
self.pretty_print)
else: # it's XML
if 'outputformat' in self.kvp:
self.contenttype = self.kvp['outputformat']
else:
self.contenttype = self.mimetype
xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>'
'\n' % self.encoding)
appinfo = '<!-- pycsw %s -->\n' % self.context.version
if isinstance(self.contenttype, bytes):
self.contenttype = self.contenttype.decode()
s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding)
LOGGER.debug('Response code: %s',
self.context.response_codes[self.status])
LOGGER.debug('Response:\n%s', s)
return [self.context.response_codes[self.status], s]
def _gen_soap_wrapper(self):
""" Generate SOAP wrapper """
LOGGER.info('Writing SOAP wrapper.')
node = etree.Element(
util.nspath_eval('soapenv:Envelope', self.context.namespaces),
nsmap=self.context.namespaces
)
schema_location_ns = util.nspath_eval('xsi:schemaLocation',
self.context.namespaces)
node.attrib[schema_location_ns] = '%s %s' % (
self.context.namespaces['soapenv'],
self.context.namespaces['soapenv']
)
node2 = etree.SubElement(
node, util.nspath_eval('soapenv:Body', self.context.namespaces))
if self.exception:
node3 = etree.SubElement(
node2,
util.nspath_eval('soapenv:Fault', self.context.namespaces)
)
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Code', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Value', self.context.namespaces)
).text = 'soap:Server'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Reason', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Text', self.context.namespaces)
).text = 'A server exception was encountered.'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Detail', self.context.namespaces)
)
node4.append(self.response)
else:
node2.append(self.response)
self.response = node
def _gen_manager(self):
""" Update self.context.model with CSW-T advertising """
if (self.config.has_option('manager', 'transactions') and
self.config.get('manager', 'transactions') == 'true'):
self.manager = True
self.context.model['operations_order'].append('Transaction')
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {}
}
schema_values = [
'http://www.opengis.net/cat/csw/2.0.2',
'http://www.opengis.net/cat/csw/3.0',
'http://www.opengis.net/wms',
'http://www.opengis.net/wmts/1.0',
'http://www.opengis.net/wfs',
'http://www.opengis.net/wfs/2.0',
'http://www.opengis.net/wcs',
'http://www.opengis.net/wps/1.0.0',
'http://www.opengis.net/sos/1.0',
'http://www.opengis.net/sos/2.0',
'http://www.isotc211.org/2005/gmi',
'urn:geoss:waf',
]
self.context.model['operations_order'].append('Harvest')
self.context.model['operations']['Harvest'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'ResourceType': {'values': schema_values}
}
}
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'TransactionSchemas': {'values': sorted(schema_values)}
}
}
self.csw_harvest_pagesize = 10
if self.config.has_option('manager', 'csw_harvest_pagesize'):
self.csw_harvest_pagesize = int(
self.config.get('manager', 'csw_harvest_pagesize'))
def _test_manager(self):
""" Verify that transactions are allowed """
if self.config.get('manager', 'transactions') != 'true':
raise RuntimeError('CSW-T interface is disabled')
""" get the client first forwarded ip """
if 'HTTP_X_FORWARDED_FOR' in self.environ:
ipaddress = self.environ['HTTP_X_FORWARDED_FOR'].split(',')[0].strip()
else:
ipaddress = self.environ['REMOTE_ADDR']
if not self.config.has_option('manager', 'allowed_ips') or \
(self.config.has_option('manager', 'allowed_ips') and not
util.ipaddress_in_whitelist(ipaddress,
self.config.get('manager', 'allowed_ips').split(','))):
raise RuntimeError(
'CSW-T operations not allowed for this IP address: %s' % ipaddress)
def _cql_update_queryables_mappings(self, cql, mappings):
""" Transform CQL query's properties to underlying DB columns """
LOGGER.debug('Raw CQL text = %s', cql)
LOGGER.debug(str(list(mappings.keys())))
if cql is not None:
for key in mappings.keys():
try:
cql = cql.replace(key, mappings[key]['dbcol'])
except:
cql = cql.replace(key, mappings[key])
LOGGER.debug('Interpolated CQL text = %s.', cql)
return cql
def _process_responsehandler(self, xml):
""" Process response handler """
if self.kvp['responsehandler'] is not None:
LOGGER.info('Processing responsehandler %s' %
self.kvp['responsehandler'])
uprh = urlparse(self.kvp['responsehandler'])
if uprh.scheme == 'mailto': # email
import smtplib
LOGGER.debug('Email detected')
smtp_host = 'localhost'
if self.config.has_option('server', 'smtp_host'):
smtp_host = self.config.get('server', 'smtp_host')
body = ('Subject: pycsw %s results\n\n%s' %
(self.kvp['request'], xml))
try:
LOGGER.info('Sending email')
msg = smtplib.SMTP(smtp_host)
msg.sendmail(
self.config.get('metadata:main', 'contact_email'),
uprh.path, body
)
msg.quit()
LOGGER.debug('Email sent successfully.')
except Exception as err:
LOGGER.exception('Error processing email')
elif uprh.scheme == 'ftp':
import ftplib
LOGGER.debug('FTP detected.')
try:
LOGGER.info('Sending to FTP server.')
ftp = ftplib.FTP(uprh.hostname)
if uprh.username is not None:
ftp.login(uprh.username, uprh.password)
ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml))
ftp.quit()
LOGGER.debug('FTP sent successfully.')
except Exception as err:
LOGGER.exception('Error processing FTP')
@staticmethod
def normalize_kvp(kvp):
"""Normalize Key Value Pairs.
This method will transform all keys to lowercase and leave values
unchanged, as specified in the CSW standard (see for example note
C on Table 62 - KVP Encoding for DescribeRecord operation request
of the CSW standard version 2.0.2)
:arg kvp: a mapping with Key Value Pairs
:type kvp: dict
:returns: A new dictionary with normalized parameters
"""
result = dict()
for name, value in kvp.items():
result[name.lower()] = value
return result
def __init__(self, email=None, api_key=None, config_file=None,
verbose=False, retry_on_errors=True,
site=None, client=None,
cert_bundle=None, insecure=None,
client_cert=None, client_cert_key=None):
# type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], bool, Optional[str], Optional[str]) -> None
if client is None:
client = _default_client()
# Fill values from Environment Variables if not available in Constructor
if config_file is None:
config_file = os.environ.get("ZULIP_CONFIG")
if api_key is None:
api_key = os.environ.get("ZULIP_API_KEY")
if email is None:
email = os.environ.get("ZULIP_EMAIL")
if site is None:
site = os.environ.get("ZULIP_SITE")
if client_cert is None:
client_cert = os.environ.get("ZULIP_CERT")
if client_cert_key is None:
client_cert_key = os.environ.get("ZULIP_CERT_KEY")
if cert_bundle is None:
cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")
if config_file is None:
config_file = get_default_config_filename()
if config_file is not None and os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if client_cert is None and config.has_option("api", "client_cert"):
client_cert = config.get("api", "client_cert")
if client_cert_key is None and config.has_option("api", "client_cert_key"):
client_cert_key = config.get("api", "client_cert_key")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get("api", "insecure").lower()
if insecure_setting == "true":
insecure = True
elif insecure_setting == "false":
insecure = False
else:
raise RuntimeError("insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
% (insecure_setting, config_file))
elif None in (api_key, email):
raise RuntimeError("api_key or email not specified and %s does not exist"
% (config_file,))
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if site.startswith("localhost"):
site = "http://" + site
elif not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
raise RuntimeError("Missing Zulip server URL; specify via --site or ~/.zuliprc.")
if not self.base_url.endswith("/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
self.tls_verification = False # type: Union[bool, str]
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise RuntimeError("tls bundle '%s' does not exist"
% (cert_bundle,))
self.tls_verification = cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification = True
if client_cert is None:
if client_cert_key is not None:
raise RuntimeError("client cert key '%s' specified, but no client cert public part provided"
% (client_cert_key,))
else: # we have a client cert
if not os.path.isfile(client_cert):
raise RuntimeError("client cert '%s' does not exist"
% (client_cert,))
if client_cert_key is not None:
if not os.path.isfile(client_cert_key):
raise RuntimeError("client cert key '%s' does not exist"
% (client_cert_key,))
self.client_cert = client_cert
self.client_cert_key = client_cert_key
class Config(object):
"""
Manages the configuration file
"""
def __init__(self):
"""
DEFAULT VALUES
"""
self._basescript = None
self.recentvaults = []
self.pwlength = 10
self.search_notes = False
self.search_passwd = False
self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_"
self.avoid_bigrams = "cl mn nm nn rn vv VV"
self._fname = self.get_config_filename()
self._parser = SafeConfigParser()
if os.path.exists(self._fname):
self._parser.read(self._fname)
if not self._parser.has_section("base"):
self._parser.add_section("base")
for num in range(10):
if (not self._parser.has_option("base", "recentvaults" + str(num))):
break
self.recentvaults.append(self._parser.get("base", "recentvaults" + str(num)))
if self._parser.has_option("base", "pwlength"):
self.pwlength = int(self._parser.get("base", "pwlength"))
if self._parser.has_option("base", "search_notes"):
if self._parser.get("base", "search_notes") == "True":
self.search_notes = True
if self._parser.has_option("base", "search_passwd"):
if self._parser.get("base", "search_passwd") == "True":
self.search_passwd = True
if self._parser.has_option("base", "alphabet"):
self.alphabet = self._parser.get("base", "alphabet")
if self._parser.has_option("base", "avoid_bigrams"):
self.avoid_bigrams = self._parser.get("base", "avoid_bigrams")
if not os.path.exists(self._fname):
self.save()
def set_basescript(self, basescript):
self._basescript = basescript
def get_basescript(self):
return self._basescript
def save(self):
if (not os.path.exists(os.path.dirname(self._fname))):
os.mkdir(os.path.dirname(self._fname))
# remove duplicates and trim to 10 items
_saved_recentvaults = []
for item in self.recentvaults:
if item in _saved_recentvaults:
continue
self._parser.set("base", "recentvaults" + str(len(_saved_recentvaults)), item)
_saved_recentvaults.append(item)
if (len(_saved_recentvaults) >= 10):
break
self._parser.set("base", "pwlength", str(self.pwlength))
self._parser.set("base", "search_notes", str(self.search_notes))
self._parser.set("base", "search_passwd", str(self.search_passwd))
self._parser.set("base", "alphabet", str(self.alphabet))
self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams))
filehandle = open(self._fname, 'w')
self._parser.write(filehandle)
filehandle.close()
@staticmethod
def get_config_filename():
"""
Returns the full filename of the config file
"""
base_fname = "loxodo"
# On Mac OS X, config files go to ~/Library/Application Support/foo/
if platform.system() == "Darwin":
base_path = os.path.join(os.path.expanduser("~"), "Library", "Application Support")
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# On Microsoft Windows, config files go to $APPDATA/foo/
if platform.system() in ("Windows", "Microsoft"):
if ("APPDATA" in os.environ):
base_path = os.environ["APPDATA"]
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# Allow config directory override as per freedesktop.org XDG Base Directory Specification
if ("XDG_CONFIG_HOME" in os.environ):
base_path = os.environ["XDG_CONFIG_HOME"]
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# Default configuration path is ~/.config/foo/
base_path = os.path.join(os.path.expanduser("~"), ".config")
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
else:
return os.path.join(os.path.expanduser("~"),"."+ base_fname + ".ini")
class Csw(object):
""" Base CSW server """
def __init__(self, rtconfig=None, env=None, version='3.0.0'):
""" Initialize CSW """
if not env:
self.environ = os.environ
else:
self.environ = env
self.context = config.StaticContext()
# Lazy load this when needed
# (it will permanently update global cfg namespaces)
self.sruobj = None
self.opensearchobj = None
self.oaipmhobj = None
# init kvp
self.kvp = {}
self.mode = 'csw'
self.async = False
self.soap = False
self.request = None
self.exception = False
self.status = 'OK'
self.profiles = None
self.manager = False
self.outputschemas = {}
self.mimetype = 'application/xml; charset=UTF-8'
self.encoding = 'UTF-8'
self.pretty_print = 0
self.domainquerytype = 'list'
self.orm = 'django'
self.language = {'639_code': 'en', 'text': 'english'}
self.process_time_start = time()
# define CSW implementation object (default CSW3)
self.iface = csw3.Csw3(server_csw=self)
self.request_version = version
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# load user configuration
try:
LOGGER.info('Loading user configuration')
if isinstance(rtconfig, SafeConfigParser): # serialized already
self.config = rtconfig
else:
self.config = SafeConfigParser()
if isinstance(rtconfig, dict): # dictionary
for section, options in rtconfig.items():
self.config.add_section(section)
for k, v in options.items():
self.config.set(section, k, v)
else: # configuration file
import codecs
with codecs.open(rtconfig, encoding='utf-8') as scp:
self.config.readfp(scp)
except Exception as err:
LOGGER.exception('Could not load user configuration: %s', err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service',
'Error opening configuration %s' % rtconfig
)
return
# set server.home safely
# TODO: make this more abstract
self.config.set(
'server', 'home',
os.path.dirname(os.path.join(os.path.dirname(__file__), '..'))
)
self.context.pycsw_home = self.config.get('server', 'home')
self.context.url = self.config.get('server', 'url')
log.setup_logger(self.config)
LOGGER.info('running configuration %s', rtconfig)
LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING'])
# set OGC schemas location
if not self.config.has_option('server', 'ogc_schemas_base'):
self.config.set('server', 'ogc_schemas_base',
self.context.ogc_schemas_base)
# set mimetype
if self.config.has_option('server', 'mimetype'):
self.mimetype = self.config.get('server', 'mimetype').encode()
# set encoding
if self.config.has_option('server', 'encoding'):
self.encoding = self.config.get('server', 'encoding')
# set domainquerytype
if self.config.has_option('server', 'domainquerytype'):
self.domainquerytype = self.config.get('server', 'domainquerytype')
# set XML pretty print
if (self.config.has_option('server', 'pretty_print') and
self.config.get('server', 'pretty_print') == 'true'):
self.pretty_print = 1
# set Spatial Ranking option
if (self.config.has_option('server', 'spatial_ranking') and
self.config.get('server', 'spatial_ranking') == 'true'):
util.ranking_enabled = True
# set language default
if self.config.has_option('server', 'language'):
try:
LOGGER.info('Setting language')
lang_code = self.config.get('server', 'language').split('-')[0]
self.language['639_code'] = lang_code
self.language['text'] = self.context.languages[lang_code]
except Exception as err:
LOGGER.exception('Could not set language: %s', err)
pass
LOGGER.debug('Configuration: %s.', self.config)
LOGGER.debug('Model: %s.', self.context.model)
# load user-defined mappings if they exist
if self.config.has_option('repository', 'mappings'):
# override default repository mappings
try:
import imp
module = self.config.get('repository', 'mappings')
if '/' in module: # filepath
modulename = '%s' % os.path.splitext(module)[0].replace(
os.sep, '.')
mappings = imp.load_source(modulename, module)
else: # dotted name
mappings = __import__(module, fromlist=[''])
LOGGER.info('Loading custom repository mappings '
'from %s', module)
self.context.md_core_model = mappings.MD_CORE_MODEL
self.context.refresh_dc(mappings.MD_CORE_MODEL)
except Exception as err:
LOGGER.exception('Could not load custom mappings: %s', err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service',
'Could not load repository.mappings')
# load outputschemas
LOGGER.info('Loading outputschemas')
for osch in pycsw.plugins.outputschemas.__all__:
output_schema_module = __import__(
'pycsw.plugins.outputschemas.%s' % osch)
mod = getattr(output_schema_module.plugins.outputschemas, osch)
self.outputschemas[mod.NAMESPACE] = mod
LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas)
LOGGER.debug('Namespaces: %s', self.context.namespaces)
def expand_path(self, path):
""" return safe path for WSGI environments """
if 'local.app_root' in self.environ and not os.path.isabs(path):
return os.path.join(self.environ['local.app_root'], path)
else:
return path
def dispatch_wsgi(self):
""" WSGI handler """
if hasattr(self, 'response'):
return self._write_response()
LOGGER.debug('WSGI mode detected')
if self.environ['REQUEST_METHOD'] == 'POST':
try:
request_body_size = int(self.environ.get('CONTENT_LENGTH', 0))
except (ValueError):
request_body_size = 0
self.requesttype = 'POST'
self.request = self.environ['wsgi.input'].read(request_body_size)
LOGGER.debug('Request type: POST. Request:\n%s\n', self.request)
else: # it's a GET request
self.requesttype = 'GET'
self.request = wsgiref.util.request_uri(self.environ)
try:
query_part = splitquery(self.request)[-1]
self.kvp = dict(parse_qsl(query_part, keep_blank_values=True))
except AttributeError as err:
LOGGER.exception('Could not parse query string')
self.kvp = {}
LOGGER.debug('Request type: GET. Request:\n%s\n', self.request)
return self.dispatch()
def opensearch(self):
""" enable OpenSearch """
if not self.opensearchobj:
self.opensearchobj = opensearch.OpenSearch(self.context)
return self.opensearchobj
def sru(self):
""" enable SRU """
if not self.sruobj:
self.sruobj = sru.Sru(self.context)
return self.sruobj
def oaipmh(self):
""" enable OAI-PMH """
if not self.oaipmhobj:
self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config)
return self.oaipmhobj
def dispatch(self, writer=sys.stdout, write_headers=True):
""" Handle incoming HTTP request """
if self.requesttype == 'GET':
self.kvp = self.normalize_kvp(self.kvp)
version_202 = ('version' in self.kvp and
self.kvp['version'] == '2.0.2')
accept_version_202 = ('acceptversions' in self.kvp and
'2.0.2' in self.kvp['acceptversions'])
if version_202 or accept_version_202:
self.request_version = '2.0.2'
elif self.requesttype == 'POST':
if self.request.find(b'2.0.2') != -1:
self.request_version = '2.0.2'
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'sru'):
self.mode = 'sru'
self.request_version = '2.0.2'
LOGGER.info('SRU mode detected; processing request')
self.kvp = self.sru().request_sru2csw(self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'oaipmh'):
self.mode = 'oaipmh'
self.request_version = '2.0.2'
LOGGER.info('OAI-PMH mode detected; processing request.')
self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k)
self.kvp = self.oaipmh().request(self.kvp)
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# configure transaction support, if specified in config
self._gen_manager()
namespaces = self.context.namespaces
ops = self.context.model['operations']
constraints = self.context.model['constraints']
# generate domain model
# NOTE: We should probably avoid this sort of mutable state for WSGI
if 'GetDomain' not in ops:
ops['GetDomain'] = self.context.gen_domains()
# generate distributed search model, if specified in config
if self.config.has_option('server', 'federatedcatalogues'):
LOGGER.info('Configuring distributed search')
constraints['FederatedCatalogues'] = {'values': []}
for fedcat in self.config.get('server',
'federatedcatalogues').split(','):
LOGGER.debug('federated catalogue: %s', fedcat)
constraints['FederatedCatalogues']['values'].append(fedcat)
for key, value in self.outputschemas.items():
get_records_params = ops['GetRecords']['parameters']
get_records_params['outputSchema']['values'].append(
value.NAMESPACE)
get_records_by_id_params = ops['GetRecordById']['parameters']
get_records_by_id_params['outputSchema']['values'].append(
value.NAMESPACE)
if 'Harvest' in ops:
harvest_params = ops['Harvest']['parameters']
harvest_params['ResourceType']['values'].append(
value.NAMESPACE)
LOGGER.info('Setting MaxRecordDefault')
if self.config.has_option('server', 'maxrecords'):
constraints['MaxRecordDefault']['values'] = [
self.config.get('server', 'maxrecords')]
# load profiles
if self.config.has_option('server', 'profiles'):
self.profiles = pprofile.load_profiles(
os.path.join('pycsw', 'plugins', 'profiles'),
pprofile.Profile,
self.config.get('server', 'profiles')
)
for prof in self.profiles['plugins'].keys():
tmp = self.profiles['plugins'][prof](self.context.model,
namespaces,
self.context)
key = tmp.outputschema # to ref by outputschema
self.profiles['loaded'][key] = tmp
self.profiles['loaded'][key].extend_core(self.context.model,
namespaces,
self.config)
LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys()))
# init repository
# look for tablename, set 'records' as default
if not self.config.has_option('repository', 'table'):
self.config.set('repository', 'table', 'records')
repo_filter = None
if self.config.has_option('repository', 'filter'):
repo_filter = self.config.get('repository', 'filter')
if self.config.has_option('repository', 'source'): # load custom repository
rs = self.config.get('repository', 'source')
rs_modname, rs_clsname = rs.rsplit('.', 1)
rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname])
rs_cls = getattr(rs_mod, rs_clsname)
try:
self.repository = rs_cls(self.context, repo_filter)
LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype)
except Exception as err:
msg = 'Could not load custom repository'
LOGGER.exception(msg)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service', msg)
else: # load default repository
self.orm = 'sqlalchemy'
from pycsw.core import repository
try:
LOGGER.info('Loading default repository')
self.repository = repository.Repository(
self.config.get('repository', 'database'),
self.context,
self.environ.get('local.app_root', None),
self.config.get('repository', 'table'),
repo_filter
)
LOGGER.debug(
'Repository loaded (local): %s.' % self.repository.dbtype)
except Exception as err:
msg = 'Could not load repository (local)'
LOGGER.exception(msg)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service', msg)
if self.requesttype == 'POST':
LOGGER.debug('HTTP POST request')
LOGGER.debug('CSW version: %s', self.iface.version)
self.kvp = self.iface.parse_postdata(self.request)
error = 0
if isinstance(self.kvp, str): # it's an exception
error = 1
locator = 'service'
text = self.kvp
if (self.kvp.find('the document is not valid') != -1 or
self.kvp.find('document not well-formed') != -1):
code = 'NoApplicableCode'
else:
code = 'InvalidParameterValue'
LOGGER.debug('HTTP Headers:\n%s.', self.environ)
LOGGER.debug('Parsed request parameters: %s', self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'opensearch'):
self.mode = 'opensearch'
LOGGER.info('OpenSearch mode detected; processing request.')
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or
(len(self.kvp) == 1 and 'config' in self.kvp)):
LOGGER.info('Turning on default csw30:Capabilities for base URL')
self.kvp = {
'service': 'CSW',
'acceptversions': '3.0.0',
'request': 'GetCapabilities'
}
http_accept = self.environ.get('HTTP_ACCEPT', '')
if 'application/opensearchdescription+xml' in http_accept:
self.mode = 'opensearch'
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if error == 0:
# test for the basic keyword values (service, version, request)
basic_options = ['service', 'request']
request = self.kvp.get('request', '')
own_version_integer = util.get_version_integer(
self.request_version)
if self.request_version == '2.0.2':
basic_options.append('version')
for k in basic_options:
if k not in self.kvp:
if (k in ['version', 'acceptversions'] and
request == 'GetCapabilities'):
pass
else:
error = 1
locator = k
code = 'MissingParameterValue'
text = 'Missing keyword: %s' % k
break
# test each of the basic keyword values
if error == 0:
# test service
if self.kvp['service'] != 'CSW':
error = 1
locator = 'service'
code = 'InvalidParameterValue'
text = 'Invalid value for service: %s.\
Value MUST be CSW' % self.kvp['service']
# test version
kvp_version = self.kvp.get('version', '')
try:
kvp_version_integer = util.get_version_integer(kvp_version)
except Exception as err:
kvp_version_integer = 'invalid_value'
if (request != 'GetCapabilities' and
kvp_version_integer != own_version_integer):
error = 1
locator = 'version'
code = 'InvalidParameterValue'
text = ('Invalid value for version: %s. Value MUST be '
'2.0.2 or 3.0.0' % kvp_version)
# check for GetCapabilities acceptversions
if 'acceptversions' in self.kvp:
for vers in self.kvp['acceptversions'].split(','):
vers_integer = util.get_version_integer(vers)
if vers_integer == own_version_integer:
break
else:
error = 1
locator = 'acceptversions'
code = 'VersionNegotiationFailed'
text = ('Invalid parameter value in '
'acceptversions: %s. Value MUST be '
'2.0.2 or 3.0.0' %
self.kvp['acceptversions'])
# test request
if self.kvp['request'] not in \
self.context.model['operations']:
error = 1
locator = 'request'
if request in ['Transaction', 'Harvest']:
code = 'OperationNotSupported'
text = '%s operations are not supported' % request
else:
code = 'InvalidParameterValue'
text = 'Invalid value for request: %s' % request
if error == 1: # return an ExceptionReport
LOGGER.error('basic service options error: %s, %s, %s', code, locator, text)
self.response = self.iface.exceptionreport(code, locator, text)
else: # process per the request value
if 'responsehandler' in self.kvp:
# set flag to process asynchronously
import threading
self.async = True
request_id = self.kvp.get('requestid', None)
if request_id is None:
import uuid
self.kvp['requestid'] = str(uuid.uuid4())
if self.kvp['request'] == 'GetCapabilities':
self.response = self.iface.getcapabilities()
elif self.kvp['request'] == 'DescribeRecord':
self.response = self.iface.describerecord()
elif self.kvp['request'] == 'GetDomain':
self.response = self.iface.getdomain()
elif self.kvp['request'] == 'GetRecords':
if self.async: # process asynchronously
threading.Thread(target=self.iface.getrecords).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.getrecords()
elif self.kvp['request'] == 'GetRecordById':
self.response = self.iface.getrecordbyid()
elif self.kvp['request'] == 'GetRepositoryItem':
self.response = self.iface.getrepositoryitem()
elif self.kvp['request'] == 'Transaction':
self.response = self.iface.transaction()
elif self.kvp['request'] == 'Harvest':
if self.async: # process asynchronously
threading.Thread(target=self.iface.harvest).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.harvest()
else:
self.response = self.iface.exceptionreport(
'InvalidParameterValue', 'request',
'Invalid request parameter: %s' % self.kvp['request']
)
LOGGER.info('Request processed')
if self.mode == 'sru':
LOGGER.info('SRU mode detected; processing response.')
self.response = self.sru().response_csw2sru(self.response,
self.environ)
elif self.mode == 'opensearch':
LOGGER.info('OpenSearch mode detected; processing response.')
self.response = self.opensearch().response_csw2opensearch(
self.response, self.config)
elif self.mode == 'oaipmh':
LOGGER.info('OAI-PMH mode detected; processing response.')
self.response = self.oaipmh().response(
self.response, self.oaiargs, self.repository,
self.config.get('server', 'url')
)
return self._write_response()
def getcapabilities(self):
""" Handle GetCapabilities request """
return self.iface.getcapabilities()
def describerecord(self):
""" Handle DescribeRecord request """
return self.iface.describerecord()
def getdomain(self):
""" Handle GetDomain request """
return self.iface.getdomain()
def getrecords(self):
""" Handle GetRecords request """
return self.iface.getrecords()
def getrecordbyid(self, raw=False):
""" Handle GetRecordById request """
return self.iface.getrecordbyid(raw)
def getrepositoryitem(self):
""" Handle GetRepositoryItem request """
return self.iface.getrepositoryitem()
def transaction(self):
""" Handle Transaction request """
return self.iface.transaction()
def harvest(self):
""" Handle Harvest request """
return self.iface.harvest()
def _write_response(self):
""" Generate response """
# set HTTP response headers and XML declaration
xmldecl = ''
appinfo = ''
LOGGER.info('Writing response.')
if hasattr(self, 'soap') and self.soap:
self._gen_soap_wrapper()
if etree.__version__ >= '3.5.0': # remove superfluous namespaces
etree.cleanup_namespaces(self.response,
keep_ns_prefixes=self.context.keep_ns_prefixes)
response = etree.tostring(self.response,
pretty_print=self.pretty_print,
encoding='unicode')
if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and
self.kvp['outputformat'] == 'application/json'):
self.contenttype = self.kvp['outputformat']
from pycsw.core.formats import fmt_json
response = fmt_json.xml2json(response,
self.context.namespaces,
self.pretty_print)
else: # it's XML
if 'outputformat' in self.kvp:
self.contenttype = self.kvp['outputformat']
else:
self.contenttype = self.mimetype
xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>'
'\n' % self.encoding)
appinfo = '<!-- pycsw %s -->\n' % self.context.version
if isinstance(self.contenttype, bytes):
self.contenttype = self.contenttype.decode()
s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding)
LOGGER.debug('Response code: %s',
self.context.response_codes[self.status])
LOGGER.debug('Response:\n%s', s)
return [self.context.response_codes[self.status], s]
def _gen_soap_wrapper(self):
""" Generate SOAP wrapper """
LOGGER.info('Writing SOAP wrapper.')
node = etree.Element(
util.nspath_eval('soapenv:Envelope', self.context.namespaces),
nsmap=self.context.namespaces
)
schema_location_ns = util.nspath_eval('xsi:schemaLocation',
self.context.namespaces)
node.attrib[schema_location_ns] = '%s %s' % (
self.context.namespaces['soapenv'],
self.context.namespaces['soapenv']
)
node2 = etree.SubElement(
node, util.nspath_eval('soapenv:Body', self.context.namespaces))
if self.exception:
node3 = etree.SubElement(
node2,
util.nspath_eval('soapenv:Fault', self.context.namespaces)
)
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Code', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Value', self.context.namespaces)
).text = 'soap:Server'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Reason', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Text', self.context.namespaces)
).text = 'A server exception was encountered.'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Detail', self.context.namespaces)
)
node4.append(self.response)
else:
node2.append(self.response)
self.response = node
def _gen_manager(self):
""" Update self.context.model with CSW-T advertising """
if (self.config.has_option('manager', 'transactions') and
self.config.get('manager', 'transactions') == 'true'):
self.manager = True
self.context.model['operations_order'].append('Transaction')
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {}
}
schema_values = [
'http://www.opengis.net/cat/csw/2.0.2',
'http://www.opengis.net/cat/csw/3.0',
'http://www.opengis.net/wms',
'http://www.opengis.net/wmts/1.0',
'http://www.opengis.net/wfs',
'http://www.opengis.net/wfs/2.0',
'http://www.opengis.net/wcs',
'http://www.opengis.net/wps/1.0.0',
'http://www.opengis.net/sos/1.0',
'http://www.opengis.net/sos/2.0',
'http://www.isotc211.org/2005/gmi',
'urn:geoss:waf',
]
self.context.model['operations_order'].append('Harvest')
self.context.model['operations']['Harvest'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'ResourceType': {'values': schema_values}
}
}
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'TransactionSchemas': {'values': sorted(schema_values)}
}
}
self.csw_harvest_pagesize = 10
if self.config.has_option('manager', 'csw_harvest_pagesize'):
self.csw_harvest_pagesize = int(
self.config.get('manager', 'csw_harvest_pagesize'))
def _test_manager(self):
""" Verify that transactions are allowed """
if self.config.get('manager', 'transactions') != 'true':
raise RuntimeError('CSW-T interface is disabled')
ipaddress = self.environ['REMOTE_ADDR']
if not self.config.has_option('manager', 'allowed_ips') or \
(self.config.has_option('manager', 'allowed_ips') and not
util.ipaddress_in_whitelist(ipaddress,
self.config.get('manager', 'allowed_ips').split(','))):
raise RuntimeError(
'CSW-T operations not allowed for this IP address: %s' % ipaddress)
def _cql_update_queryables_mappings(self, cql, mappings):
""" Transform CQL query's properties to underlying DB columns """
LOGGER.debug('Raw CQL text = %s', cql)
LOGGER.debug(str(list(mappings.keys())))
if cql is not None:
for key in mappings.keys():
try:
cql = cql.replace(key, mappings[key]['dbcol'])
except:
cql = cql.replace(key, mappings[key])
LOGGER.debug('Interpolated CQL text = %s.', cql)
return cql
def _process_responsehandler(self, xml):
""" Process response handler """
if self.kvp['responsehandler'] is not None:
LOGGER.info('Processing responsehandler %s' %
self.kvp['responsehandler'])
uprh = urlparse(self.kvp['responsehandler'])
if uprh.scheme == 'mailto': # email
import smtplib
LOGGER.debug('Email detected')
smtp_host = 'localhost'
if self.config.has_option('server', 'smtp_host'):
smtp_host = self.config.get('server', 'smtp_host')
body = ('Subject: pycsw %s results\n\n%s' %
(self.kvp['request'], xml))
try:
LOGGER.info('Sending email')
msg = smtplib.SMTP(smtp_host)
msg.sendmail(
self.config.get('metadata:main', 'contact_email'),
uprh.path, body
)
msg.quit()
LOGGER.debug('Email sent successfully.')
except Exception as err:
LOGGER.exception('Error processing email')
elif uprh.scheme == 'ftp':
import ftplib
LOGGER.debug('FTP detected.')
try:
LOGGER.info('Sending to FTP server.')
ftp = ftplib.FTP(uprh.hostname)
if uprh.username is not None:
ftp.login(uprh.username, uprh.password)
ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml))
ftp.quit()
LOGGER.debug('FTP sent successfully.')
except Exception as err:
LOGGER.exception('Error processing FTP')
@staticmethod
def normalize_kvp(kvp):
"""Normalize Key Value Pairs.
This method will transform all keys to lowercase and leave values
unchanged, as specified in the CSW standard (see for example note
C on Table 62 - KVP Encoding for DescribeRecord operation request
of the CSW standard version 2.0.2)
:arg kvp: a mapping with Key Value Pairs
:type kvp: dict
:returns: A new dictionary with normalized parameters
"""
result = dict()
for name, value in kvp.items():
result[name.lower()] = value
return result
class PackageConfigHandler(object):
"""
Manager class for packages files for tracking installation of modules
"""
def __init__(self):
# noinspection PyUnresolvedReferences
from six.moves.configparser import SafeConfigParser
self.package_cfg = os.path.expanduser('~/Documents/site-packages/.pypi_packages')
if not os.path.isfile(self.package_cfg):
print('Creating package file')
with open(self.package_cfg, 'w') as outs:
outs.close()
self.parser = SafeConfigParser()
self.parser.read(self.package_cfg)
def save(self):
with open(self.package_cfg, 'w') as outs:
self.parser.write(outs)
def add_module(self, pkg_info):
"""
:param pkg_info: A dict that has name, url, version, summary
:return:
"""
if not self.parser.has_section(pkg_info['name']):
self.parser.add_section(pkg_info['name'])
self.parser.set(pkg_info['name'], 'url', pkg_info['url'])
self.parser.set(pkg_info['name'], 'version', pkg_info['version'])
self.parser.set(pkg_info['name'], 'summary', pkg_info['summary'])
self.parser.set(pkg_info['name'], 'files', pkg_info['files'])
self.parser.set(pkg_info['name'], 'dependency', pkg_info['dependency'])
self.save()
def list_modules(self):
return [module for module in self.parser.sections()]
def module_exists(self, name):
return self.parser.has_section(name)
def get_info(self, name):
if self.parser.has_section(name):
tbl = {}
for opt, value in self.parser.items(name):
tbl[opt] = value
return tbl
def remove_module(self, name):
self.parser.remove_section(name)
self.save()
def get_files_installed(self, section_name):
if self.parser.has_option(section_name, 'files'):
files = self.parser.get(section_name, 'files').strip()
return files.split(',')
else:
return None
def get_dependencies(self, section_name):
if self.parser.has_option(section_name, 'dependency'):
dependencies = self.parser.get(section_name, 'dependency').strip()
return set(dependencies.split(',')) if dependencies != '' else set()
else:
return None
def get_all_dependencies(self, exclude_module=()):
all_dependencies = set()
for section_name in self.parser.sections():
if section_name not in exclude_module and self.parser.has_option(section_name, 'dependency'):
dependencies = self.parser.get(section_name, 'dependency').strip()
if dependencies != '':
for dep in dependencies.split(','):
all_dependencies.add(dep)
return all_dependencies
def __init__(self,
email=None,
api_key=None,
config_file=None,
verbose=False,
retry_on_errors=True,
site=None,
client=None,
cert_bundle=None,
insecure=None,
client_cert=None,
client_cert_key=None):
# type: (Optional[str], Optional[str], Optional[str], bool, bool, Optional[str], Optional[str], Optional[str], bool, Optional[str], Optional[str]) -> None
if client is None:
client = _default_client()
# Fill values from Environment Variables if not available in Constructor
if config_file is None:
config_file = os.environ.get("ZULIP_CONFIG")
if api_key is None:
api_key = os.environ.get("ZULIP_API_KEY")
if email is None:
email = os.environ.get("ZULIP_EMAIL")
if site is None:
site = os.environ.get("ZULIP_SITE")
if client_cert is None:
client_cert = os.environ.get("ZULIP_CERT")
if client_cert_key is None:
client_cert_key = os.environ.get("ZULIP_CERT_KEY")
if cert_bundle is None:
cert_bundle = os.environ.get("ZULIP_CERT_BUNDLE")
if config_file is None:
config_file = get_default_config_filename()
if os.path.exists(config_file):
config = SafeConfigParser()
with open(config_file, 'r') as f:
config.readfp(f, config_file)
if api_key is None:
api_key = config.get("api", "key")
if email is None:
email = config.get("api", "email")
if site is None and config.has_option("api", "site"):
site = config.get("api", "site")
if client_cert is None and config.has_option("api", "client_cert"):
client_cert = config.get("api", "client_cert")
if client_cert_key is None and config.has_option(
"api", "client_cert_key"):
client_cert_key = config.get("api", "client_cert_key")
if cert_bundle is None and config.has_option("api", "cert_bundle"):
cert_bundle = config.get("api", "cert_bundle")
if insecure is None and config.has_option("api", "insecure"):
# Be quite strict about what is accepted so that users don't
# disable security unintentionally.
insecure_setting = config.get("api", "insecure").lower()
if insecure_setting == "true":
insecure = True
elif insecure_setting == "false":
insecure = False
else:
raise RuntimeError(
"insecure is set to '%s', it must be 'true' or 'false' if it is used in %s"
% (insecure_setting, config_file))
elif None in (api_key, email):
raise RuntimeError(
"api_key or email not specified and %s does not exist" %
(config_file, ))
self.api_key = api_key
self.email = email
self.verbose = verbose
if site is not None:
if site.startswith("localhost"):
site = "http://" + site
elif not site.startswith("http"):
site = "https://" + site
# Remove trailing "/"s from site to simplify the below logic for adding "/api"
site = site.rstrip("/")
self.base_url = site
else:
raise RuntimeError(
"Missing Zulip server URL; specify via --site or ~/.zuliprc.")
if not self.base_url.endswith("/api"):
self.base_url += "/api"
self.base_url += "/"
self.retry_on_errors = retry_on_errors
self.client_name = client
if insecure:
self.tls_verification = False # type: Union[bool, str]
elif cert_bundle is not None:
if not os.path.isfile(cert_bundle):
raise RuntimeError("tls bundle '%s' does not exist" %
(cert_bundle, ))
self.tls_verification = cert_bundle
else:
# Default behavior: verify against system CA certificates
self.tls_verification = True
if client_cert is None:
if client_cert_key is not None:
raise RuntimeError(
"client cert key '%s' specified, but no client cert public part provided"
% (client_cert_key, ))
else: # we have a client cert
if not os.path.isfile(client_cert):
raise RuntimeError("client cert '%s' does not exist" %
(client_cert, ))
if client_cert_key is not None:
if not os.path.isfile(client_cert_key):
raise RuntimeError("client cert key '%s' does not exist" %
(client_cert_key, ))
self.client_cert = client_cert
self.client_cert_key = client_cert_key
def load_conf():
conf_path = os.path.expanduser(config_path)
if not os.path.exists(conf_path):
logger.warn(
"{conf} couldn't be found, please use \'coscmd config -h\' to learn how to config coscmd!"
.format(conf=to_printable_str(conf_path)))
raise IOError
else:
logger.debug(
'{conf} is found'.format(conf=to_printable_str(conf_path)))
with open(conf_path, 'r') as f:
cp = SafeConfigParser()
cp.readfp(fp=f)
if cp.has_option('common', 'part_size'):
part_size = cp.getint('common', 'part_size')
else:
part_size = 1
if cp.has_option('common', 'max_thread'):
max_thread = cp.getint('common', 'max_thread')
else:
max_thread = 5
try:
secret_id = cp.get('common', 'secret_id')
except Exception:
secret_id = cp.get('common', 'access_id')
try:
appid = cp.get('common', 'appid')
bucket = cp.get('common', 'bucket')
if bucket.endswith("-" + str(appid)):
bucket = bucket.rstrip(appid)
bucket = bucket[:-1]
except Exception:
try:
bucket = cp.get('common', 'bucket')
appid = bucket.split('-')[-1]
bucket = bucket.rstrip(appid)
bucket = bucket[:-1]
except Exception:
logger.error(
"The configuration file is wrong. Please reconfirm")
try:
schema = cp.get('common', 'schema')
except:
schema = 'https'
try:
anonymous = cp.get('common', 'anonymous')
except:
anonymous = 'False'
region, endpoint = None, None
if cp.has_option('common', 'region'):
region = cp.get('common', 'region')
else:
endpoint = cp.get('common', 'endpoint')
if pre_appid != "":
appid = pre_appid
if pre_bucket != "":
bucket = pre_bucket
if pre_region != "":
region = pre_region
conf = CosConfig(appid=appid,
secret_id=secret_id,
secret_key=cp.get('common', 'secret_key'),
region=compatible(region),
endpoint=endpoint,
bucket=bucket,
part_size=part_size,
max_thread=max_thread,
schema=schema,
anonymous=anonymous)
return conf
