def config(args):
logger.debug("config: " + str(args))
conf_path = os.path.expanduser(config_path)
with open(conf_path, 'w+') as f:
cp = SafeConfigParser()
cp.add_section("common")
cp.set('common', 'secret_id', args.secret_id)
cp.set('common', 'secret_key', args.secret_key)
if args.token != "":
cp.set('common', 'token', args.token)
cp.set('common', 'bucket', args.bucket)
if args.endpoint:
cp.set('common', 'endpoint', args.endpoint)
else:
cp.set('common', 'region', args.region)
cp.set('common', 'max_thread', str(args.max_thread))
cp.set('common', 'part_size', str(args.part_size))
cp.set('common', 'retry', str(args.retry))
cp.set('common', 'timeout', str(args.timeout))
if args.appid != "":
cp.set('common', 'appid', args.appid)
if args.use_http:
cp.set('common', 'schema', 'http')
else:
cp.set('common', 'schema', 'https')
cp.set('common', 'verify', args.verify)
if args.anonymous:
cp.set('common', 'anonymous', 'True')
else:
cp.set('common', 'anonymous', 'False')
cp.write(f)
logger.info("Created configuration file in {path}".format(path=to_printable_str(conf_path)))
def from_sections(cls, sections):
config = SafeConfigParser()
for section, keyvalues in sections.items():
config.add_section(section)
for key, value in keyvalues.items():
config.set(section, key, value)
return cls(config)
def create_header(self, data_only):
'''
Create the backup file header that contains the meta data about
this particular backup.
'''
config = SafeConfigParser()
config.add_section("ipa")
if data_only:
config.set('ipa', 'type', 'DATA')
else:
config.set('ipa', 'type', 'FULL')
config.set('ipa', 'time',
time.strftime(ISO8601_DATETIME_FMT, time.gmtime()))
config.set('ipa', 'host', api.env.host)
config.set('ipa', 'ipa_version', str(version.VERSION))
config.set('ipa', 'version', '1')
dn = DN(('cn', api.env.host), ('cn', 'masters'), ('cn', 'ipa'),
('cn', 'etc'), api.env.basedn)
services_cns = []
try:
conn = self.get_connection()
services = conn.get_entries(dn, conn.SCOPE_ONELEVEL)
except errors.NetworkError:
self.log.critical(
"Unable to obtain list of master services, continuing anyway")
except Exception as e:
self.log.error("Failed to read services from '%s': %s" %
(conn.host, e))
else:
services_cns = [s.single_value['cn'] for s in services]
config.set('ipa', 'services', ','.join(services_cns))
with open(self.header, 'w') as fd:
config.write(fd)
class Config(object):
def __init__(self):
self.config = SafeConfigParser()
self.name = ''
def parse(self, fname, override):
if override:
override = [x for x in csv.reader(
' '.join(override).split(','), delimiter='.')]
logger.info('Reading configuration file: {}'.format(fname))
if not os.path.isfile(fname):
logger.interrupt('File doesn\'t exist: {}'.format(fname))
self.config.optionxform = str
self.config.read(fname)
for section, option, value in override:
if not self.config.has_section(section):
self.config.add_section(section)
self.config.set(section, option, value)
basename = os.path.basename(fname)
self.name = os.path.splitext(basename)[0]
@safe
def _get_options_as_dict(self, section):
if section in self.config.sections():
return {p: v for p, v in self.config.items(section)}
else:
return {}
class Config(object):
"""A ConfigParser wrapper to support defaults when calling instance
methods, and also tied to a single section"""
SECTION = 'scrapyd'
def __init__(self, values=None, extra_sources=()):
if values is None:
sources = self._getsources()
default_config = get_data(__package__, 'default_scrapyd.conf').decode('utf8')
self.cp = SafeConfigParser()
self.cp.readfp(io.StringIO(default_config))
sources.extend(extra_sources)
for fname in sources:
try:
with io.open(fname) as fp:
self.cp.readfp(fp)
except (IOError, OSError):
pass
else:
self.cp = SafeConfigParser(values)
self.cp.add_section(self.SECTION)
def _getsources(self):
sources = ['/etc/scrapyd/scrapyd.conf', r'c:\scrapyd\scrapyd.conf']
sources += sorted(glob.glob('/etc/scrapyd/conf.d/*'))
sources += ['scrapyd.conf']
sources += [expanduser('~/.scrapyd.conf')]
scrapy_cfg = closest_scrapy_cfg()
if scrapy_cfg:
sources.append(scrapy_cfg)
return sources
def _getany(self, method, option, default):
try:
return method(self.SECTION, option)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def get(self, option, default=None):
return self._getany(self.cp.get, option, default)
def getint(self, option, default=None):
return self._getany(self.cp.getint, option, default)
def getfloat(self, option, default=None):
return self._getany(self.cp.getfloat, option, default)
def getboolean(self, option, default=None):
return self._getany(self.cp.getboolean, option, default)
def items(self, section, default=None):
try:
return self.cp.items(section)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def save(self):
"""Save the modules to @_path. If @modules is an empty
dict, then @_path should be removed.
"""
root_logger.debug("Saving StateFile to '%s'", self._path)
for module in list(self.modules.keys()):
if len(self.modules[module]) == 0:
del self.modules[module]
if len(self.modules) == 0:
root_logger.debug(" -> no modules, removing file")
if os.path.exists(self._path):
os.remove(self._path)
return
p = SafeConfigParser()
p.optionxform = str
for module in self.modules.keys():
p.add_section(module)
for (key, value) in self.modules[module].items():
p.set(module, key, str(value))
with open(self._path, "w") as f:
p.write(f)
class Config(object):
"""A ConfigParser wrapper to support defaults when calling instance
methods, and also tied to a single section"""
SECTION = 'silkyy'
def __init__(self, values=None, extra_sources=()):
if values is None:
sources = self._getsources()
self.cp = ConfigParser()
if __package__:
default_config = ensure_str(get_data(__package__, 'default.conf'))
self._load_config_file(StringIO(default_config))
for source in sources:
if os.path.exists(source):
self._load_config_file(open(source))
else:
self.cp = SafeConfigParser(values)
self.cp.add_section(self.SECTION)
def _load_config_file(self, fp):
config = StringIO()
config.write('[' + self.SECTION + ']' + os.linesep)
config.write(fp.read())
config.seek(0, os.SEEK_SET)
self.cp.readfp(config)
def _getsources(self):
sources = ['conf/silkyy.conf']
return sources
def get(self, option, default=None):
env_key = 'SILKYY_' + option.replace('.', '_').upper()
try:
return os.getenv(env_key) or self.cp.get(self.SECTION, option)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def _get(self, option, conv, default=None):
return conv(self.get(option, default))
def getint(self, option, default=None):
return self._get(option, int, default)
def getboolean(self, option, default=None):
return self._get(option, str2bool, default)
def getfloat(self, option, default=None):
return self._get(option, float, default)
def items(self, section, default=None):
try:
return self.cp.items(section)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def create_header(self, data_only):
'''
Create the backup file header that contains the meta data about
this particular backup.
'''
config = SafeConfigParser()
config.add_section("ipa")
if data_only:
config.set('ipa', 'type', 'DATA')
else:
config.set('ipa', 'type', 'FULL')
config.set('ipa', 'time', time.strftime(ISO8601_DATETIME_FMT, time.gmtime()))
config.set('ipa', 'host', api.env.host)
config.set('ipa', 'ipa_version', str(version.VERSION))
config.set('ipa', 'version', '1')
dn = DN(('cn', api.env.host), ('cn', 'masters'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn)
services_cns = []
try:
conn = self.get_connection()
services = conn.get_entries(dn, conn.SCOPE_ONELEVEL)
except errors.NetworkError:
self.log.critical(
"Unable to obtain list of master services, continuing anyway")
except Exception as e:
self.log.error("Failed to read services from '%s': %s" %
(conn.host, e))
else:
services_cns = [s.single_value['cn'] for s in services]
config.set('ipa', 'services', ','.join(services_cns))
with open(self.header, 'w') as fd:
config.write(fd)
def save(self):
"""Save the modules to @_path. If @modules is an empty
dict, then @_path should be removed.
"""
root_logger.debug("Saving StateFile to '%s'", self._path)
for module in list(self.modules):
if len(self.modules[module]) == 0:
del self.modules[module]
if len(self.modules) == 0:
root_logger.debug(" -> no modules, removing file")
if os.path.exists(self._path):
os.remove(self._path)
return
p = SafeConfigParser()
p.optionxform = str
for module in self.modules:
p.add_section(module)
for (key, value) in self.modules[module].items():
p.set(module, key, str(value))
with open(self._path, "w") as f:
p.write(f)
def from_config(cls, path, **kwargs):
"""
Create a LastFM instance from a config file in the following format:
[lastfm]
api_key = myapikey
api_secret = myapisecret
username = thesquelched
password = plaintext_password
# Can be 'password' or 'hashed_password'
auth_method = password
You can also override config values with keyword arguments.
"""
config = SafeConfigParser()
config.add_section('lastfm')
config.read(os.path.expanduser(os.path.expandvars(path)))
return LastFM(
cls._getoption(config, kwargs, 'api_key'),
cls._getoption(config, kwargs, 'api_secret'),
username=cls._getoption(config, kwargs, 'username', None),
password=cls._getoption(config, kwargs, 'password', None),
auth_method=cls._getoption(config, kwargs, 'auth_method', None),
session_key=cls._getoption(config, kwargs, 'session_key', None),
url=cls._getoption(config, kwargs, 'url', None),
)
def set_namespace_options(namespace, options, remove=None):
"""
Set options in the local namespace configuration file.
Can have nasty effects, be careful, only use in test code.
:param namespace: the namespace to work with
:param options: a dictionary with options to set
:param remove: an iterable of options to remove
:returns: a dictionary with all options of the namespace
"""
parser = SafeConfigParser({})
potential_confs = list()
actual_confs = list()
for p, _ in _config_paths():
potential_confs.append(p)
fdone = parser.read((p,))
actual_confs.extend(fdone)
if not actual_confs:
raise ValueError(
"Could not read configuration from any of %s" % potential_confs)
if not parser.has_section(namespace):
print('Namespace %s was not found in %s' % (namespace, actual_confs))
parser.add_section(namespace)
for key, val in options.items():
parser.set(namespace, key, str(val))
if remove:
for key in remove:
parser.remove_option(namespace, key)
with open(actual_confs[-1], 'w') as outfile:
parser.write(outfile)
return dict(parser.items(namespace))
class Config(object):
"""A ConfigParser wrapper to support defaults when calling instance
methods, and also tied to a single section"""
SECTION = 'scrapydartx'
def __init__(self, values=None, extra_sources=()):
if values is None:
sources = self._getsources()
default_config = get_data(__package__,
'default_scrapyd.conf').decode('utf8')
self.cp = SafeConfigParser()
self.cp.readfp(StringIO(default_config))
self.cp.read(sources)
for fp in extra_sources:
self.cp.readfp(fp)
else:
self.cp = SafeConfigParser(values)
self.cp.add_section(self.SECTION)
def _getsources(self):
sources = [
'/etc/scrapydartx/scrapydartx.conf', r'c:\scrapyd\scrapyd.conf'
]
sources += sorted(glob.glob('/etc/scrapydartx/conf.d/*'))
sources += ['scrapydartx.conf']
sources += [expanduser('~/.scrapydartx.conf')]
scrapy_cfg = closest_scrapy_cfg()
if scrapy_cfg:
sources.append(scrapy_cfg)
return sources
def _getany(self, method, option, default):
try:
return method(self.SECTION, option)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def get(self, option, default=None):
return self._getany(self.cp.get, option, default)
def getint(self, option, default=None):
return self._getany(self.cp.getint, option, default)
def getfloat(self, option, default=None):
return self._getany(self.cp.getfloat, option, default)
def getboolean(self, option, default=None):
return self._getany(self.cp.getboolean, option, default)
def items(self, section, default=None):
try:
return self.cp.items(section)
except (NoSectionError, NoOptionError):
if default is not None:
return default
raise
def load(f):
p = SafeConfigParser()
p.read(f)
if not p.has_section('oauth2'):
p.add_section('oauth2')
if not p.has_section('oauth2-state'):
p.add_section('oauth2-state')
return p
def load(f):
p = SafeConfigParser()
p.read(f)
if not p.has_section('oauth2'):
p.add_section('oauth2')
if not p.has_section('oauth2-state'):
p.add_section('oauth2-state')
return p
def _init_config_parser(self, sections=None):
parser = SafeConfigParser(allow_no_value=True)
if sections:
for section in sections:
parser.add_section(section)
for key, value in sections[section].items():
parser.set(section, key,
self._value_converter.to_strings(value))
return parser
def _init_config_parser(self, sections=None):
parser = SafeConfigParser(allow_no_value=True)
if sections:
for section in sections:
parser.add_section(section)
for key, value in sections[section].items():
parser.set(section, key,
self._value_converter.to_strings(value))
return parser
def generate_default_engine_config(cls):
config = SafeConfigParser()
config.add_section("OPENCAFE_ENGINE")
config.set("OPENCAFE_ENGINE", "config_directory", EngineDirectoryManager.OPENCAFE_SUB_DIRS.CONFIG_DIR)
config.set("OPENCAFE_ENGINE", "data_directory", EngineDirectoryManager.OPENCAFE_SUB_DIRS.DATA_DIR)
config.set("OPENCAFE_ENGINE", "log_directory", EngineDirectoryManager.OPENCAFE_SUB_DIRS.LOG_DIR)
config.set("OPENCAFE_ENGINE", "temp_directory", EngineDirectoryManager.OPENCAFE_SUB_DIRS.TEMP_DIR)
config.set("OPENCAFE_ENGINE", "master_log_file_name", "cafe.master")
config.set("OPENCAFE_ENGINE", "logging_verbosity", "STANDARD")
config.set("OPENCAFE_ENGINE", "default_test_repo", "cloudroast")
return config
def _init_config_parser(self, sections=None):
parser = SafeConfigParser(allow_no_value=True)
if sections:
for section in sections:
parser.add_section(section)
for key, value in sections[section].items():
str_val = StringConverter(
{self._default_value: None}).to_strings(value)
parser.set(section, key,
str(str_val) if str_val is not None
else str_val)
return parser
def save_config(self):
self.log.info("Finalizing configuration")
config = SafeConfigParser()
config.add_section("realm")
config.set("realm", "realm_name", api.env.realm)
config.set("realm", "master_host_name", api.env.host)
config.set("realm", "domain_name", api.env.domain)
config.set("realm", "destination_host", self.replica_fqdn)
config.set("realm", "subject_base", str(self.subject_base))
config.set("realm", "version", str(version.NUM_VERSION))
with open(os.path.join(self.dir, "realm_info"), "w") as fd:
config.write(fd)
def save_config(self):
self.log.info("Finalizing configuration")
config = SafeConfigParser()
config.add_section("realm")
config.set("realm", "realm_name", api.env.realm)
config.set("realm", "master_host_name", api.env.host)
config.set("realm", "domain_name", api.env.domain)
config.set("realm", "destination_host", self.replica_fqdn)
config.set("realm", "subject_base", str(self.subject_base))
config.set("realm", "version", str(version.NUM_VERSION))
with open(os.path.join(self.dir, "realm_info"), "w") as fd:
config.write(fd)
def generate_default_engine_config(cls):
config = SafeConfigParser()
config.add_section('OPENCAFE_ENGINE')
config.set('OPENCAFE_ENGINE', 'config_directory',
EngineDirectoryManager.OPENCAFE_SUB_DIRS.CONFIG_DIR)
config.set('OPENCAFE_ENGINE', 'data_directory',
EngineDirectoryManager.OPENCAFE_SUB_DIRS.DATA_DIR)
config.set('OPENCAFE_ENGINE', 'log_directory',
EngineDirectoryManager.OPENCAFE_SUB_DIRS.LOG_DIR)
config.set('OPENCAFE_ENGINE', 'temp_directory',
EngineDirectoryManager.OPENCAFE_SUB_DIRS.TEMP_DIR)
config.set('OPENCAFE_ENGINE', 'master_log_file_name', 'cafe.master')
config.set('OPENCAFE_ENGINE', 'logging_verbosity', 'STANDARD')
config.set('OPENCAFE_ENGINE', 'default_test_repo', 'cloudroast')
return config
def config(request):
path = request.config.getoption("--config")
config = SafeConfigParser()
for section, values in CONFIG_DEFAULTS.items():
config.add_section(section)
for key, value in values.items():
config.set(section, key, value)
if path:
config.read([path])
with open(path) as f:
LOG.debug("Configuration:\n%s", f.read())
return config
def write(cfg_obj, output_file_path):
"""
Only supports writing out a conflagration object with namespaces that
follow the section.key=value pattern that ConfigFile.parse generates
"""
parser = SafeConfigParser()
for k in cfg_obj.__dict__.keys():
parser.add_section(k)
try:
for sub_k, sub_v in cfg_obj.__dict__[k].__dict__.items():
parser.set(k, sub_k, sub_v)
except Exception:
raise Exception(
"Output to config file not supported for conflagrations"
"nested beyond a one dot namespace.")
with open(output_file_path, 'w') as f:
parser.write(f)
def save_to_config(save_config, config_obj, verbose=False):
"""
Write all the values in the config object to a given file in the user's home folder
:param config_file: Name of config file to store in user's home folder
:param config_obj: The config object to export to config file
:param verbose: Specify if stdout should display a message
:return:
"""
parser = SafeConfigParser()
parser.read(save_config)
if not os.path.exists(save_config):
parser.add_section('settings')
for k, v in viewitems(config_obj._asdict()):
if v is not None and isinstance(v, str):
parser.set('settings', k, v)
with open(save_config, 'w') as f:
parser.write(f)
if verbose:
click.echo("Config file written to {}".format(save_config))
def save(self):
"""Save the file list to @_index. If @files is an empty
dict, then @_index should be removed.
"""
root_logger.debug("Saving Index File to '%s'", self._index)
if len(self.files) == 0:
root_logger.debug(" -> no files, removing file")
if os.path.exists(self._index):
os.remove(self._index)
return
p = SafeConfigParser()
p.optionxform = str
p.add_section('files')
for (key, value) in self.files.items():
p.set('files', key, str(value))
with open(self._index, "w") as f:
p.write(f)
def save(self):
"""Save the file list to @_index. If @files is an empty
dict, then @_index should be removed.
"""
root_logger.debug("Saving Index File to '%s'", self._index)
if len(self.files) == 0:
root_logger.debug(" -> no files, removing file")
if os.path.exists(self._index):
os.remove(self._index)
return
p = SafeConfigParser()
p.optionxform = str
p.add_section('files')
for (key, value) in self.files.items():
p.set('files', key, str(value))
with open(self._index, "w") as f:
p.write(f)
def configure(filename=None):
"""This function gives to the user application a chance to define where
configuration file should live. Subsequent calls to this function will have
no effect, unless you call :func:`reconfigure`.
:param str filename: Full path to configuration file.
"""
global retry
if getattr(configure, '_configured', False):
return
filename = filename or DEFAULT_CONFIG_FILENAME
_ensure_directory(filename)
parser = SafeConfigParser()
if os.path.isfile(filename):
with open(filename, 'rt') as fp:
parser.readfp(fp)
if not parser.has_section(RETRY_SECTION):
parser.add_section(RETRY_SECTION)
parser.set(RETRY_SECTION, 'max_tries',
str(constants.BACKOFF_DEFAULT_MAXTRIES))
parser.set(RETRY_SECTION, 'delay',
str(constants.BACKOFF_DEFAULT_DELAY))
parser.set(RETRY_SECTION, 'factor',
str(constants.BACKOFF_DEFAULT_FACTOR))
with open(filename, 'wt') as fp:
parser.write(fp)
retry = RetrySettings(max_tries=parser.getint(RETRY_SECTION, 'max_tries'),
delay=parser.getint(RETRY_SECTION, 'delay'),
factor=parser.getint(RETRY_SECTION, 'factor'))
setattr(configure, '_configured', True)
setattr(configure, '_configured_filename', filename)
def generate_default_engine_config(cls):
config = SafeConfigParser()
config.add_section('OPENCAFE_ENGINE')
config.set(
'OPENCAFE_ENGINE', 'config_directory',
EngineDirectoryManager.OPENCAFE_SUB_DIRS.CONFIG_DIR)
config.set(
'OPENCAFE_ENGINE', 'data_directory',
EngineDirectoryManager.OPENCAFE_SUB_DIRS.DATA_DIR)
config.set(
'OPENCAFE_ENGINE', 'log_directory',
EngineDirectoryManager.OPENCAFE_SUB_DIRS.LOG_DIR)
config.set(
'OPENCAFE_ENGINE', 'temp_directory',
EngineDirectoryManager.OPENCAFE_SUB_DIRS.TEMP_DIR)
config.set(
'OPENCAFE_ENGINE', 'master_log_file_name', 'cafe.master')
config.set(
'OPENCAFE_ENGINE', 'logging_verbosity', 'STANDARD')
config.set(
'OPENCAFE_ENGINE', 'default_test_repo', 'cloudroast')
return config
def run_register():
key = input('Please input node key:')
key_secret = input('Please input node key_secret:')
config = AgentConfig()
server_base = config.get('server')
if urlparse(server_base).scheme == '':
if config.getint('server_https_port'):
server_base = 'https://%s:%d' % (
server_base, config.getint('server_https_port'))
else:
server_base = 'http://%s:%d' % (server_base,
config.getint('server_port'))
register_url = urljoin(server_base, '/nodes/register')
post_data = {'node_key': key}
request = authenticated_request(url=register_url,
method="POST",
app_key=key,
app_secret=key_secret,
body=urlencode(post_data))
client = HTTPClient()
try:
response = client.fetch(request)
response_data = json.loads(response.body)
if not os.path.exists('conf'):
os.makedirs('conf')
with open('conf/node.conf', 'w') as f:
cp = SafeConfigParser()
cp.add_section('agent')
cp.set('agent', 'node_id', str(response_data['id']))
cp.set('agent', 'node_key', key)
cp.set('agent', 'secret_key', key_secret)
cp.write(f)
print('Register succeed!')
except HTTPError as e:
print("Error when registering.")
print(e.message)
def ConfiguratePaser(configuration_item_name,
userid,
ip_address_or_domain,
userpw=None,
installpath=None,
sshport=None,
config_parser=None):
""" Create or update the configuration file based on the input
secifications.
Parameters
----------
configuration_item_name: str
the name of the configuration item
userid: str
user name on the server side
ip_address_or_domain: str
the ip address or the domain of the server
userpw: str
user password to login the server using ssh.
If you want to use "id_rsa.pub", just leave userpw to None
To copy the public key on the server use ssh-copy-id -i name@server.
installpath: str
soma workflow source path on server
sshport: int (optional)
the ssh port
config_parser: SafeConfigParser (optional)
default None, an empty config parser is created
otherwise, the config parser is updatd with the new item
Returns
-------
config_parser: SafeConfigParser
the configuration item
"""
# Create config generator
if not config_parser:
config_parser = SafeConfigParser()
# Get server info
hostname = ip_address_or_domain
info_queue = GetQueueNamesOnPBSTORQUE(userid, ip_address_or_domain, userpw)
# Add section
if configuration_item_name not in config_parser.sections():
config_parser.add_section(configuration_item_name)
# Fill section
config_parser.set(configuration_item_name,
configuration.CFG_CLUSTER_ADDRESS, hostname)
config_parser.set(configuration_item_name,
configuration.CFG_SUBMITTING_MACHINES, hostname)
config_parser.set(configuration_item_name, configuration.OCFG_QUEUES,
" ".join(info_queue))
config_parser.set(configuration_item_name, configuration.OCFG_LOGIN,
userid)
config_parser.set(configuration_item_name, configuration.OCFG_SSHPort,
str(sshport or 22))
if installpath:
config_parser.set(configuration_item_name,
configuration.OCFG_INSTALLPATH, installpath)
config_parser.set(configuration_item_name,
configuration.OCFG_ALLOWED_PYTHON_VERSIONS,
str(sys.version_info[0]))
return config_parser
class Csw(object):
""" Base CSW server """
def __init__(self, rtconfig=None, env=None, version='3.0.0'):
""" Initialize CSW """
if not env:
self.environ = os.environ
else:
self.environ = env
self.context = config.StaticContext()
# Lazy load this when needed
# (it will permanently update global cfg namespaces)
self.sruobj = None
self.opensearchobj = None
self.oaipmhobj = None
# init kvp
self.kvp = {}
self.mode = 'csw'
self.async = False
self.soap = False
self.request = None
self.exception = False
self.status = 'OK'
self.profiles = None
self.manager = False
self.outputschemas = {}
self.mimetype = 'application/xml; charset=UTF-8'
self.encoding = 'UTF-8'
self.pretty_print = 0
self.domainquerytype = 'list'
self.orm = 'django'
self.language = {'639_code': 'en', 'text': 'english'}
self.process_time_start = time()
# define CSW implementation object (default CSW3)
self.iface = csw3.Csw3(server_csw=self)
self.request_version = version
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# load user configuration
try:
LOGGER.info('Loading user configuration')
if isinstance(rtconfig, SafeConfigParser): # serialized already
self.config = rtconfig
else:
self.config = SafeConfigParser()
if isinstance(rtconfig, dict): # dictionary
for section, options in rtconfig.items():
self.config.add_section(section)
for k, v in options.items():
self.config.set(section, k, v)
else: # configuration file
import codecs
with codecs.open(rtconfig, encoding='utf-8') as scp:
self.config.readfp(scp)
except Exception as err:
LOGGER.exception('Could not load user configuration: %s', err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service',
'Error opening configuration %s' % rtconfig
)
return
# set server.home safely
# TODO: make this more abstract
self.config.set(
'server', 'home',
os.path.dirname(os.path.join(os.path.dirname(__file__), '..'))
)
self.context.pycsw_home = self.config.get('server', 'home')
self.context.url = self.config.get('server', 'url')
log.setup_logger(self.config)
LOGGER.info('running configuration %s', rtconfig)
LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING'])
# set OGC schemas location
if not self.config.has_option('server', 'ogc_schemas_base'):
self.config.set('server', 'ogc_schemas_base',
self.context.ogc_schemas_base)
# set mimetype
if self.config.has_option('server', 'mimetype'):
self.mimetype = self.config.get('server', 'mimetype').encode()
# set encoding
if self.config.has_option('server', 'encoding'):
self.encoding = self.config.get('server', 'encoding')
# set domainquerytype
if self.config.has_option('server', 'domainquerytype'):
self.domainquerytype = self.config.get('server', 'domainquerytype')
# set XML pretty print
if (self.config.has_option('server', 'pretty_print') and
self.config.get('server', 'pretty_print') == 'true'):
self.pretty_print = 1
# set Spatial Ranking option
if (self.config.has_option('server', 'spatial_ranking') and
self.config.get('server', 'spatial_ranking') == 'true'):
util.ranking_enabled = True
# set language default
if self.config.has_option('server', 'language'):
try:
LOGGER.info('Setting language')
lang_code = self.config.get('server', 'language').split('-')[0]
self.language['639_code'] = lang_code
self.language['text'] = self.context.languages[lang_code]
except Exception as err:
LOGGER.exception('Could not set language: %s', err)
pass
LOGGER.debug('Configuration: %s.', self.config)
LOGGER.debug('Model: %s.', self.context.model)
# load user-defined mappings if they exist
if self.config.has_option('repository', 'mappings'):
# override default repository mappings
try:
import imp
module = self.config.get('repository', 'mappings')
if '/' in module: # filepath
modulename = '%s' % os.path.splitext(module)[0].replace(
os.sep, '.')
mappings = imp.load_source(modulename, module)
else: # dotted name
mappings = __import__(module, fromlist=[''])
LOGGER.info('Loading custom repository mappings '
'from %s', module)
self.context.md_core_model = mappings.MD_CORE_MODEL
self.context.refresh_dc(mappings.MD_CORE_MODEL)
except Exception as err:
LOGGER.exception('Could not load custom mappings: %s', err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service',
'Could not load repository.mappings')
# load outputschemas
LOGGER.info('Loading outputschemas')
for osch in pycsw.plugins.outputschemas.__all__:
output_schema_module = __import__(
'pycsw.plugins.outputschemas.%s' % osch)
mod = getattr(output_schema_module.plugins.outputschemas, osch)
self.outputschemas[mod.NAMESPACE] = mod
LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas)
LOGGER.debug('Namespaces: %s', self.context.namespaces)
def expand_path(self, path):
""" return safe path for WSGI environments """
if 'local.app_root' in self.environ and not os.path.isabs(path):
return os.path.join(self.environ['local.app_root'], path)
else:
return path
def dispatch_wsgi(self):
""" WSGI handler """
if hasattr(self, 'response'):
return self._write_response()
LOGGER.debug('WSGI mode detected')
if self.environ['REQUEST_METHOD'] == 'POST':
try:
request_body_size = int(self.environ.get('CONTENT_LENGTH', 0))
except (ValueError):
request_body_size = 0
self.requesttype = 'POST'
self.request = self.environ['wsgi.input'].read(request_body_size)
LOGGER.debug('Request type: POST. Request:\n%s\n', self.request)
else: # it's a GET request
self.requesttype = 'GET'
self.request = wsgiref.util.request_uri(self.environ)
try:
query_part = splitquery(self.request)[-1]
self.kvp = dict(parse_qsl(query_part, keep_blank_values=True))
except AttributeError as err:
LOGGER.exception('Could not parse query string')
self.kvp = {}
LOGGER.debug('Request type: GET. Request:\n%s\n', self.request)
return self.dispatch()
def opensearch(self):
""" enable OpenSearch """
if not self.opensearchobj:
self.opensearchobj = opensearch.OpenSearch(self.context)
return self.opensearchobj
def sru(self):
""" enable SRU """
if not self.sruobj:
self.sruobj = sru.Sru(self.context)
return self.sruobj
def oaipmh(self):
""" enable OAI-PMH """
if not self.oaipmhobj:
self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config)
return self.oaipmhobj
def dispatch(self, writer=sys.stdout, write_headers=True):
""" Handle incoming HTTP request """
if self.requesttype == 'GET':
self.kvp = self.normalize_kvp(self.kvp)
version_202 = ('version' in self.kvp and
self.kvp['version'] == '2.0.2')
accept_version_202 = ('acceptversions' in self.kvp and
'2.0.2' in self.kvp['acceptversions'])
if version_202 or accept_version_202:
self.request_version = '2.0.2'
elif self.requesttype == 'POST':
if self.request.find(b'2.0.2') != -1:
self.request_version = '2.0.2'
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'sru'):
self.mode = 'sru'
self.request_version = '2.0.2'
LOGGER.info('SRU mode detected; processing request')
self.kvp = self.sru().request_sru2csw(self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'oaipmh'):
self.mode = 'oaipmh'
self.request_version = '2.0.2'
LOGGER.info('OAI-PMH mode detected; processing request.')
self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k)
self.kvp = self.oaipmh().request(self.kvp)
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# configure transaction support, if specified in config
self._gen_manager()
namespaces = self.context.namespaces
ops = self.context.model['operations']
constraints = self.context.model['constraints']
# generate domain model
# NOTE: We should probably avoid this sort of mutable state for WSGI
if 'GetDomain' not in ops:
ops['GetDomain'] = self.context.gen_domains()
# generate distributed search model, if specified in config
if self.config.has_option('server', 'federatedcatalogues'):
LOGGER.info('Configuring distributed search')
constraints['FederatedCatalogues'] = {'values': []}
for fedcat in self.config.get('server',
'federatedcatalogues').split(','):
LOGGER.debug('federated catalogue: %s', fedcat)
constraints['FederatedCatalogues']['values'].append(fedcat)
for key, value in self.outputschemas.items():
get_records_params = ops['GetRecords']['parameters']
get_records_params['outputSchema']['values'].append(
value.NAMESPACE)
get_records_by_id_params = ops['GetRecordById']['parameters']
get_records_by_id_params['outputSchema']['values'].append(
value.NAMESPACE)
if 'Harvest' in ops:
harvest_params = ops['Harvest']['parameters']
harvest_params['ResourceType']['values'].append(
value.NAMESPACE)
LOGGER.info('Setting MaxRecordDefault')
if self.config.has_option('server', 'maxrecords'):
constraints['MaxRecordDefault']['values'] = [
self.config.get('server', 'maxrecords')]
# load profiles
if self.config.has_option('server', 'profiles'):
self.profiles = pprofile.load_profiles(
os.path.join('pycsw', 'plugins', 'profiles'),
pprofile.Profile,
self.config.get('server', 'profiles')
)
for prof in self.profiles['plugins'].keys():
tmp = self.profiles['plugins'][prof](self.context.model,
namespaces,
self.context)
key = tmp.outputschema # to ref by outputschema
self.profiles['loaded'][key] = tmp
self.profiles['loaded'][key].extend_core(self.context.model,
namespaces,
self.config)
LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys()))
# init repository
# look for tablename, set 'records' as default
if not self.config.has_option('repository', 'table'):
self.config.set('repository', 'table', 'records')
repo_filter = None
if self.config.has_option('repository', 'filter'):
repo_filter = self.config.get('repository', 'filter')
if self.config.has_option('repository', 'source'): # load custom repository
rs = self.config.get('repository', 'source')
rs_modname, rs_clsname = rs.rsplit('.', 1)
rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname])
rs_cls = getattr(rs_mod, rs_clsname)
try:
self.repository = rs_cls(self.context, repo_filter)
LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype)
except Exception as err:
msg = 'Could not load custom repository'
LOGGER.exception(msg)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service', msg)
else: # load default repository
self.orm = 'sqlalchemy'
from pycsw.core import repository
try:
LOGGER.info('Loading default repository')
self.repository = repository.Repository(
self.config.get('repository', 'database'),
self.context,
self.environ.get('local.app_root', None),
self.config.get('repository', 'table'),
repo_filter
)
LOGGER.debug(
'Repository loaded (local): %s.' % self.repository.dbtype)
except Exception as err:
msg = 'Could not load repository (local)'
LOGGER.exception(msg)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service', msg)
if self.requesttype == 'POST':
LOGGER.debug('HTTP POST request')
LOGGER.debug('CSW version: %s', self.iface.version)
self.kvp = self.iface.parse_postdata(self.request)
error = 0
if isinstance(self.kvp, str): # it's an exception
error = 1
locator = 'service'
text = self.kvp
if (self.kvp.find('the document is not valid') != -1 or
self.kvp.find('document not well-formed') != -1):
code = 'NoApplicableCode'
else:
code = 'InvalidParameterValue'
LOGGER.debug('HTTP Headers:\n%s.', self.environ)
LOGGER.debug('Parsed request parameters: %s', self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'opensearch'):
self.mode = 'opensearch'
LOGGER.info('OpenSearch mode detected; processing request.')
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or
(len(self.kvp) == 1 and 'config' in self.kvp)):
LOGGER.info('Turning on default csw30:Capabilities for base URL')
self.kvp = {
'service': 'CSW',
'acceptversions': '3.0.0',
'request': 'GetCapabilities'
}
http_accept = self.environ.get('HTTP_ACCEPT', '')
if 'application/opensearchdescription+xml' in http_accept:
self.mode = 'opensearch'
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if error == 0:
# test for the basic keyword values (service, version, request)
basic_options = ['service', 'request']
request = self.kvp.get('request', '')
own_version_integer = util.get_version_integer(
self.request_version)
if self.request_version == '2.0.2':
basic_options.append('version')
for k in basic_options:
if k not in self.kvp:
if (k in ['version', 'acceptversions'] and
request == 'GetCapabilities'):
pass
else:
error = 1
locator = k
code = 'MissingParameterValue'
text = 'Missing keyword: %s' % k
break
# test each of the basic keyword values
if error == 0:
# test service
if self.kvp['service'] != 'CSW':
error = 1
locator = 'service'
code = 'InvalidParameterValue'
text = 'Invalid value for service: %s.\
Value MUST be CSW' % self.kvp['service']
# test version
kvp_version = self.kvp.get('version', '')
try:
kvp_version_integer = util.get_version_integer(kvp_version)
except Exception as err:
kvp_version_integer = 'invalid_value'
if (request != 'GetCapabilities' and
kvp_version_integer != own_version_integer):
error = 1
locator = 'version'
code = 'InvalidParameterValue'
text = ('Invalid value for version: %s. Value MUST be '
'2.0.2 or 3.0.0' % kvp_version)
# check for GetCapabilities acceptversions
if 'acceptversions' in self.kvp:
for vers in self.kvp['acceptversions'].split(','):
vers_integer = util.get_version_integer(vers)
if vers_integer == own_version_integer:
break
else:
error = 1
locator = 'acceptversions'
code = 'VersionNegotiationFailed'
text = ('Invalid parameter value in '
'acceptversions: %s. Value MUST be '
'2.0.2 or 3.0.0' %
self.kvp['acceptversions'])
# test request
if self.kvp['request'] not in \
self.context.model['operations']:
error = 1
locator = 'request'
if request in ['Transaction', 'Harvest']:
code = 'OperationNotSupported'
text = '%s operations are not supported' % request
else:
code = 'InvalidParameterValue'
text = 'Invalid value for request: %s' % request
if error == 1: # return an ExceptionReport
LOGGER.error('basic service options error: %s, %s, %s', code, locator, text)
self.response = self.iface.exceptionreport(code, locator, text)
else: # process per the request value
if 'responsehandler' in self.kvp:
# set flag to process asynchronously
import threading
self.async = True
request_id = self.kvp.get('requestid', None)
if request_id is None:
import uuid
self.kvp['requestid'] = str(uuid.uuid4())
if self.kvp['request'] == 'GetCapabilities':
self.response = self.iface.getcapabilities()
elif self.kvp['request'] == 'DescribeRecord':
self.response = self.iface.describerecord()
elif self.kvp['request'] == 'GetDomain':
self.response = self.iface.getdomain()
elif self.kvp['request'] == 'GetRecords':
if self.async: # process asynchronously
threading.Thread(target=self.iface.getrecords).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.getrecords()
elif self.kvp['request'] == 'GetRecordById':
self.response = self.iface.getrecordbyid()
elif self.kvp['request'] == 'GetRepositoryItem':
self.response = self.iface.getrepositoryitem()
elif self.kvp['request'] == 'Transaction':
self.response = self.iface.transaction()
elif self.kvp['request'] == 'Harvest':
if self.async: # process asynchronously
threading.Thread(target=self.iface.harvest).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.harvest()
else:
self.response = self.iface.exceptionreport(
'InvalidParameterValue', 'request',
'Invalid request parameter: %s' % self.kvp['request']
)
LOGGER.info('Request processed')
if self.mode == 'sru':
LOGGER.info('SRU mode detected; processing response.')
self.response = self.sru().response_csw2sru(self.response,
self.environ)
elif self.mode == 'opensearch':
LOGGER.info('OpenSearch mode detected; processing response.')
self.response = self.opensearch().response_csw2opensearch(
self.response, self.config)
elif self.mode == 'oaipmh':
LOGGER.info('OAI-PMH mode detected; processing response.')
self.response = self.oaipmh().response(
self.response, self.oaiargs, self.repository,
self.config.get('server', 'url')
)
return self._write_response()
def getcapabilities(self):
""" Handle GetCapabilities request """
return self.iface.getcapabilities()
def describerecord(self):
""" Handle DescribeRecord request """
return self.iface.describerecord()
def getdomain(self):
""" Handle GetDomain request """
return self.iface.getdomain()
def getrecords(self):
""" Handle GetRecords request """
return self.iface.getrecords()
def getrecordbyid(self, raw=False):
""" Handle GetRecordById request """
return self.iface.getrecordbyid(raw)
def getrepositoryitem(self):
""" Handle GetRepositoryItem request """
return self.iface.getrepositoryitem()
def transaction(self):
""" Handle Transaction request """
return self.iface.transaction()
def harvest(self):
""" Handle Harvest request """
return self.iface.harvest()
def _write_response(self):
""" Generate response """
# set HTTP response headers and XML declaration
xmldecl = ''
appinfo = ''
LOGGER.info('Writing response.')
if hasattr(self, 'soap') and self.soap:
self._gen_soap_wrapper()
if etree.__version__ >= '3.5.0': # remove superfluous namespaces
etree.cleanup_namespaces(self.response,
keep_ns_prefixes=self.context.keep_ns_prefixes)
response = etree.tostring(self.response,
pretty_print=self.pretty_print,
encoding='unicode')
if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and
self.kvp['outputformat'] == 'application/json'):
self.contenttype = self.kvp['outputformat']
from pycsw.core.formats import fmt_json
response = fmt_json.xml2json(response,
self.context.namespaces,
self.pretty_print)
else: # it's XML
if 'outputformat' in self.kvp:
self.contenttype = self.kvp['outputformat']
else:
self.contenttype = self.mimetype
xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>'
'\n' % self.encoding)
appinfo = '<!-- pycsw %s -->\n' % self.context.version
if isinstance(self.contenttype, bytes):
self.contenttype = self.contenttype.decode()
s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding)
LOGGER.debug('Response code: %s',
self.context.response_codes[self.status])
LOGGER.debug('Response:\n%s', s)
return [self.context.response_codes[self.status], s]
def _gen_soap_wrapper(self):
""" Generate SOAP wrapper """
LOGGER.info('Writing SOAP wrapper.')
node = etree.Element(
util.nspath_eval('soapenv:Envelope', self.context.namespaces),
nsmap=self.context.namespaces
)
schema_location_ns = util.nspath_eval('xsi:schemaLocation',
self.context.namespaces)
node.attrib[schema_location_ns] = '%s %s' % (
self.context.namespaces['soapenv'],
self.context.namespaces['soapenv']
)
node2 = etree.SubElement(
node, util.nspath_eval('soapenv:Body', self.context.namespaces))
if self.exception:
node3 = etree.SubElement(
node2,
util.nspath_eval('soapenv:Fault', self.context.namespaces)
)
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Code', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Value', self.context.namespaces)
).text = 'soap:Server'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Reason', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Text', self.context.namespaces)
).text = 'A server exception was encountered.'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Detail', self.context.namespaces)
)
node4.append(self.response)
else:
node2.append(self.response)
self.response = node
def _gen_manager(self):
""" Update self.context.model with CSW-T advertising """
if (self.config.has_option('manager', 'transactions') and
self.config.get('manager', 'transactions') == 'true'):
self.manager = True
self.context.model['operations_order'].append('Transaction')
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {}
}
schema_values = [
'http://www.opengis.net/cat/csw/2.0.2',
'http://www.opengis.net/cat/csw/3.0',
'http://www.opengis.net/wms',
'http://www.opengis.net/wmts/1.0',
'http://www.opengis.net/wfs',
'http://www.opengis.net/wfs/2.0',
'http://www.opengis.net/wcs',
'http://www.opengis.net/wps/1.0.0',
'http://www.opengis.net/sos/1.0',
'http://www.opengis.net/sos/2.0',
'http://www.isotc211.org/2005/gmi',
'urn:geoss:waf',
]
self.context.model['operations_order'].append('Harvest')
self.context.model['operations']['Harvest'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'ResourceType': {'values': schema_values}
}
}
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'TransactionSchemas': {'values': sorted(schema_values)}
}
}
self.csw_harvest_pagesize = 10
if self.config.has_option('manager', 'csw_harvest_pagesize'):
self.csw_harvest_pagesize = int(
self.config.get('manager', 'csw_harvest_pagesize'))
def _test_manager(self):
""" Verify that transactions are allowed """
if self.config.get('manager', 'transactions') != 'true':
raise RuntimeError('CSW-T interface is disabled')
ipaddress = self.environ['REMOTE_ADDR']
if not self.config.has_option('manager', 'allowed_ips') or \
(self.config.has_option('manager', 'allowed_ips') and not
util.ipaddress_in_whitelist(ipaddress,
self.config.get('manager', 'allowed_ips').split(','))):
raise RuntimeError(
'CSW-T operations not allowed for this IP address: %s' % ipaddress)
def _cql_update_queryables_mappings(self, cql, mappings):
""" Transform CQL query's properties to underlying DB columns """
LOGGER.debug('Raw CQL text = %s', cql)
LOGGER.debug(str(list(mappings.keys())))
if cql is not None:
for key in mappings.keys():
try:
cql = cql.replace(key, mappings[key]['dbcol'])
except:
cql = cql.replace(key, mappings[key])
LOGGER.debug('Interpolated CQL text = %s.', cql)
return cql
def _process_responsehandler(self, xml):
""" Process response handler """
if self.kvp['responsehandler'] is not None:
LOGGER.info('Processing responsehandler %s' %
self.kvp['responsehandler'])
uprh = urlparse(self.kvp['responsehandler'])
if uprh.scheme == 'mailto': # email
import smtplib
LOGGER.debug('Email detected')
smtp_host = 'localhost'
if self.config.has_option('server', 'smtp_host'):
smtp_host = self.config.get('server', 'smtp_host')
body = ('Subject: pycsw %s results\n\n%s' %
(self.kvp['request'], xml))
try:
LOGGER.info('Sending email')
msg = smtplib.SMTP(smtp_host)
msg.sendmail(
self.config.get('metadata:main', 'contact_email'),
uprh.path, body
)
msg.quit()
LOGGER.debug('Email sent successfully.')
except Exception as err:
LOGGER.exception('Error processing email')
elif uprh.scheme == 'ftp':
import ftplib
LOGGER.debug('FTP detected.')
try:
LOGGER.info('Sending to FTP server.')
ftp = ftplib.FTP(uprh.hostname)
if uprh.username is not None:
ftp.login(uprh.username, uprh.password)
ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml))
ftp.quit()
LOGGER.debug('FTP sent successfully.')
except Exception as err:
LOGGER.exception('Error processing FTP')
@staticmethod
def normalize_kvp(kvp):
"""Normalize Key Value Pairs.
This method will transform all keys to lowercase and leave values
unchanged, as specified in the CSW standard (see for example note
C on Table 62 - KVP Encoding for DescribeRecord operation request
of the CSW standard version 2.0.2)
:arg kvp: a mapping with Key Value Pairs
:type kvp: dict
:returns: A new dictionary with normalized parameters
"""
result = dict()
for name, value in kvp.items():
result[name.lower()] = value
return result
class Config(MutableMapping):
"""Read and write to config file.
A config object is essentially a string key-value store that can be treated like a dictionary::
c = Config()
c['foo'] = 'bar'
print c['foo']
The file location may be specified::
c = Config('~/matt/anotherconfig.cfg')
c['where'] = 'in a different file'
If no location is specified, the environment variable CHEMDATAEXTRACTOR_CONFIG is checked and used if available.
Otherwise, a standard config location is used, which varies depending on the operating system. You can check the
location using the ``path`` property. For more information see https://github.com/ActiveState/appdirs
It is possible to edit the file by hand with a text editor.
Warning: multiple instances of Config() pointing to the same file will not see each others' changes, and will
overwrite the entire file when any key is changed.
"""
#: These values will be present in a config object unless they are explicitly defined otherwise in the config file
default_values = {}
def __init__(self, path=None):
"""
:param string path: (Optional) Path to config file location.
"""
self._path = path
# Use CHEMDATAEXTRACTOR_CONFIG environment variable if set
if not self._path:
self._path = os.environ.get('CHEMDATAEXTRACTOR_CONFIG')
# Use OS-dependent config directory given by appdirs
if not self._path:
self._path = os.path.join(
appdirs.user_config_dir('ChemDataExtractor'), 'config.cfg')
self._parser = SafeConfigParser()
if os.path.isfile(self.path):
with open(self.path) as f:
self._parser.readfp(f)
if not self._parser.has_section('cde'):
self._parser.add_section('cde')
for k, v in six.iteritems(self.default_values):
if not self._parser.has_option('cde', k.encode('utf-8')):
self._parser.set('cde', k.encode('utf-8'), v.encode('utf-8'))
@property
def path(self):
"""The path to the config file."""
return self._path
def _flush(self):
"""Save the contents of data to the file on disk. You should not need to call this manually."""
d = os.path.dirname(self.path)
if not os.path.isdir(d):
os.makedirs(d)
with open(self.path, 'w') as f:
self._parser.write(f)
def __contains__(self, k):
return self._parser.has_option('cde', k.encode('utf-8'))
def __getitem__(self, k):
try:
return self._parser.get('cde', k.encode('utf-8'))
except NoOptionError:
raise KeyError(k)
def __setitem__(self, k, v):
self._parser.set('cde', k.encode('utf-8'), v.encode('utf-8'))
self._flush()
def __delitem__(self, k):
try:
self._parser.remove_option('cde', k.encode('utf-8'))
self._flush()
except NoOptionError:
raise KeyError(k)
def __iter__(self):
return (k for k, v in self._parser.items('cde'))
def __len__(self):
return len(self._parser.items('cde'))
def __repr__(self):
return '<Config: %s>' % self.path
def clear(self):
"""Clear all values from config."""
self._parser.remove_section('cde')
self._parser.add_section('cde')
self._flush()
class PackageConfigHandler(object):
"""
Manager class for packages files for tracking installation of modules
"""
def __init__(self):
# noinspection PyUnresolvedReferences
from six.moves.configparser import SafeConfigParser
self.package_cfg = os.path.expanduser('~/Documents/site-packages/.pypi_packages')
if not os.path.isfile(self.package_cfg):
print('Creating package file')
with open(self.package_cfg, 'w') as outs:
outs.close()
self.parser = SafeConfigParser()
self.parser.read(self.package_cfg)
def save(self):
with open(self.package_cfg, 'w') as outs:
self.parser.write(outs)
def add_module(self, pkg_info):
"""
:param pkg_info: A dict that has name, url, version, summary
:return:
"""
if not self.parser.has_section(pkg_info['name']):
self.parser.add_section(pkg_info['name'])
self.parser.set(pkg_info['name'], 'url', pkg_info['url'])
self.parser.set(pkg_info['name'], 'version', pkg_info['version'])
self.parser.set(pkg_info['name'], 'summary', pkg_info['summary'])
self.parser.set(pkg_info['name'], 'files', pkg_info['files'])
self.parser.set(pkg_info['name'], 'dependency', pkg_info['dependency'])
self.save()
def list_modules(self):
return [module for module in self.parser.sections()]
def module_exists(self, name):
return self.parser.has_section(name)
def get_info(self, name):
if self.parser.has_section(name):
tbl = {}
for opt, value in self.parser.items(name):
tbl[opt] = value
return tbl
def remove_module(self, name):
self.parser.remove_section(name)
self.save()
def get_files_installed(self, section_name):
if self.parser.has_option(section_name, 'files'):
files = self.parser.get(section_name, 'files').strip()
return files.split(',')
else:
return None
def get_dependencies(self, section_name):
if self.parser.has_option(section_name, 'dependency'):
dependencies = self.parser.get(section_name, 'dependency').strip()
return set(dependencies.split(',')) if dependencies != '' else set()
else:
return None
def get_all_dependencies(self, exclude_module=()):
all_dependencies = set()
for section_name in self.parser.sections():
if section_name not in exclude_module and self.parser.has_option(section_name, 'dependency'):
dependencies = self.parser.get(section_name, 'dependency').strip()
if dependencies != '':
for dep in dependencies.split(','):
all_dependencies.add(dep)
return all_dependencies
def createThemeFromTemplate(title, description, baseOn='template'):
"""Create a new theme from the given title and description based on
another theme resource directory
"""
source = queryResourceDirectory(THEME_RESOURCE_NAME, baseOn)
if source is None:
raise KeyError("Theme {0:s} not found".format(baseOn))
themeName = getUtility(IURLNormalizer).normalize(title)
if isinstance(themeName, six.text_type):
themeName = themeName.encode('utf-8')
resources = getOrCreatePersistentResourceDirectory()
resources.makeDirectory(themeName)
target = resources[themeName]
cloneResourceDirectory(source, target)
manifest = SafeConfigParser()
if MANIFEST_FILENAME in target:
fp = target.openFile(MANIFEST_FILENAME)
try:
manifest.readfp(fp)
finally:
fp.close()
if not manifest.has_section('theme'):
manifest.add_section('theme')
if not isinstance(title, str):
title = title.encode('utf-8')
if not isinstance(description, str):
description = description.encode('utf-8')
manifest.set('theme', 'title', title)
manifest.set('theme', 'description', description)
if manifest.has_option('theme', 'prefix'):
prefix = u"/++%s++%s" % (THEME_RESOURCE_NAME, themeName)
manifest.set('theme', 'prefix', prefix)
if manifest.has_option('theme', 'rules'):
rule = manifest.get('theme', 'rules')
rule_file_name = rule.split('/')[-1] # extract real rules file name
rules = u"/++%s++%s/%s" % (THEME_RESOURCE_NAME, themeName,
rule_file_name)
manifest.set('theme', 'rules', rules)
paths_to_fix = [
'development-css', 'production-css', 'tinymce-content-css',
'development-js', 'production-js'
]
for var_path in paths_to_fix:
if not manifest.has_option('theme', var_path):
continue
val = manifest.get('theme', var_path)
if not val:
continue
template_prefix = '++%s++%s/' % (THEME_RESOURCE_NAME, baseOn)
if template_prefix in val:
# okay, fix
val = val.replace(template_prefix,
'++%s++%s/' % (THEME_RESOURCE_NAME, themeName))
manifest.set('theme', var_path, val)
manifestContents = StringIO()
manifest.write(manifestContents)
target.writeFile(MANIFEST_FILENAME, manifestContents)
return themeName
class Config(object):
"""
Manages the configuration file
"""
def __init__(self):
"""
DEFAULT VALUES
"""
self._basescript = None
self.recentvaults = []
self.pwlength = 10
self.search_notes = False
self.search_passwd = False
self.use_pwgen = False
# self.alphabet = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_"
self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_"
self.avoid_bigrams = "cl mn nm nn rn vv VV"
self._fname = self.get_config_filename()
print("Using config %s" % self._fname)
self._parser = SafeConfigParser()
if os.path.exists(self._fname):
self._parser.read(self._fname)
if not self._parser.has_section("base"):
self._parser.add_section("base")
for num in range(10):
if (not self._parser.has_option("base",
"recentvaults" + str(num))):
break
self.recentvaults.append(
self._parser.get("base", "recentvaults" + str(num)))
if self._parser.has_option("base", "pwlength"):
self.pwlength = int(self._parser.get("base", "pwlength"))
if self._parser.has_option("base", "search_notes"):
if self._parser.get("base", "search_notes") == "True":
self.search_notes = True
if self._parser.has_option("base", "search_passwd"):
if self._parser.get("base", "search_passwd") == "True":
self.search_passwd = True
if self._parser.has_option("base", "use_pwgen"):
if self._parser.get("base", "use_pwgen") == "True":
self.use_pwgen = True
if self._parser.has_option("base", "alphabet"):
self.alphabet = self._parser.get("base", "alphabet")
if self._parser.has_option("base", "avoid_bigrams"):
self.avoid_bigrams = self._parser.get("base", "avoid_bigrams")
if not os.path.exists(self._fname):
self.save()
def set_basescript(self, basescript):
self._basescript = basescript
def get_basescript(self):
return self._basescript
def save(self):
if (not os.path.exists(os.path.dirname(self._fname))):
os.mkdir(os.path.dirname(self._fname))
# remove duplicates and trim to 10 items
_saved_recentvaults = []
for item in self.recentvaults:
if item in _saved_recentvaults:
continue
self._parser.set("base",
"recentvaults" + str(len(_saved_recentvaults)),
item)
_saved_recentvaults.append(item)
if (len(_saved_recentvaults) >= 10):
break
self._parser.set("base", "pwlength", str(self.pwlength))
self._parser.set("base", "search_notes", str(self.search_notes))
self._parser.set("base", "search_passwd", str(self.search_passwd))
self._parser.set("base", "use_pwgen", str(self.use_pwgen))
self._parser.set("base", "alphabet", str(self.alphabet))
self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams))
filehandle = open(self._fname, 'w')
self._parser.write(filehandle)
filehandle.close()
@staticmethod
def get_config_filename():
"""
Returns the full filename of the config file
"""
base_fname = "loxodo"
# On Mac OS X, config files go to ~/Library/Application Support/foo/
if platform.system() == "Darwin":
base_path = os.path.join(os.path.expanduser("~"), "Library",
"Application Support")
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# On Microsoft Windows, config files go to $APPDATA/foo/
if platform.system() in ("Windows", "Microsoft"):
if ("APPDATA" in os.environ):
base_path = os.environ["APPDATA"]
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname,
base_fname + ".ini")
# Allow config directory override as per freedesktop.org XDG Base Directory Specification
if ("XDG_CONFIG_HOME" in os.environ):
base_path = os.environ["XDG_CONFIG_HOME"]
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# Default configuration path is ~/.config/foo/
base_path = os.path.join(os.path.expanduser("~"), ".config")
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
else:
return os.path.join(os.path.expanduser("~"),
"." + base_fname + ".ini")
def main():
config = SafeConfigParser()
config.add_section('main')
for k, v in DEFAULTS.items():
config.set('main', k, v)
config.read([
'.bisque',
os.path.expanduser('~/.bisque'), '/etc/bisque/bisque_config'
])
defaults = dict(config.items('main'))
defaults.update(config.items('bqpath'))
parser = argparse.ArgumentParser(
description=DESCRIPTION,
formatter_class=argparse.ArgumentDefaultsHelpFormatter,
)
#parser.add_argument('--help', action=_HelpAction, help='help for help if you need some help') # add custom help
parser.add_argument('--alias',
help="do action on behalf of user specified")
parser.add_argument('-d',
'--debug',
action="store_true",
default=False,
help="log debugging")
parser.add_argument('-H',
'--host',
default=defaults['bisque_host'],
help="bisque host")
parser.add_argument('-c',
'--credentials',
default="%s:%s" %
(defaults['bisque_user'], defaults["bisque_pass"]),
help="user credentials")
parser.add_argument('-C',
'--compatible',
action="store_true",
help="Make compatible with old script")
parser.add_argument('-V',
'--verbose',
action='store_true',
help='print stuff')
#parser.add_argument('-P', '--permission', default="private", help="Set resource permission (compatibility)")
#parser.add_argument('--hidden', default=None, help="Set resource visibility (hidden)")
#parser.add_argument('command', help="one of ls, cp, mv, rm, ln" )
#parser.add_argument('paths', nargs='+')
sp = parser.add_subparsers()
lsp = sp.add_parser('ls')
lsp.add_argument('-u',
'--unique',
default=None,
action="store_true",
help="return unique codes")
lsp.add_argument('paths', nargs='+')
lsp.set_defaults(func=bisque_list)
lnp = sp.add_parser('ln')
lnp.add_argument('-T',
'--tag_file',
default=None,
help="tag document for insert")
lnp.add_argument('-P',
'--permission',
default="private",
help="Set resource permission (compatibility)")
lnp.add_argument('-R',
'--resource',
default=None,
help='force resource type')
lnp.add_argument('--hidden',
default=None,
help="Set resource visibility (hidden)")
lnp.add_argument('paths', nargs='+')
lnp.set_defaults(func=bisque_link)
cpp = sp.add_parser('cp')
cpp.add_argument('paths', nargs='+')
cpp.add_argument('-T',
'--tag_file',
default=None,
help="tag document for insert")
cpp.add_argument('-R',
'--resource',
default=None,
help='force resource type')
cpp.add_argument('-P',
'--permission',
default="private",
help="Set resource permission (compatibility)")
cpp.add_argument('--hidden',
default=None,
help="Set resource visibility (hidden)")
cpp.set_defaults(func=bisque_copy)
mvp = sp.add_parser('mv')
mvp.add_argument('paths', nargs='+')
mvp.set_defaults(func=bisque_rename)
rmp = sp.add_parser('rm')
rmp.add_argument('paths', nargs='+')
rmp.set_defaults(func=bisque_delete)
logging.basicConfig(
filename=config.get('bqpath', 'logfile'),
level=logging.INFO,
format="%(asctime)s %(levelname)-5.5s [%(name)s] %(message)s")
log = logging.getLogger('rods2bq')
args = parser.parse_args()
if args.debug:
logging.getLogger().setLevel(logging.DEBUG)
#if args.command not in OPERATIONS:
# parser.error("command %s must be one of 'ln', 'ls', 'cp', 'mv', 'rm'" % args.command)
if len(args.paths) > 1:
args.dstpath = args.paths.pop()
args.srcpath = args.paths
else:
args.srcpath = args.paths
if args.compatible:
paths = []
irods_host = defaults.get('irods_host')
for el in args.srcpath:
if not el.startswith('irods://'):
paths.append(irods_host + el)
else:
paths.append(el)
args.srcpath = paths
if args.dstpath and not args.dstpath.startswith('irods://'):
args.dstpath = irods_host + args.dstpath
if args.debug:
six.print_(args, file=sys.stderr)
try:
session = requests.Session()
requests.packages.urllib3.disable_warnings()
session.log = logging.getLogger('rods2bq')
#session.verify = False
session.auth = tuple(args.credentials.split(':'))
#session.headers.update ( {'content-type': 'application/xml'} )
#OPERATIONS[args.command] (session, args)
args.func(session, args)
except requests.exceptions.HTTPError as e:
log.exception("exception occurred %s : %s", e, e.response.text)
six.print_("ERROR:", e.response and e.response.status_code)
sys.exit(0)
class Csw(object):
""" Base CSW server """
def __init__(self, rtconfig=None, env=None, version='3.0.0'):
""" Initialize CSW """
if not env:
self.environ = os.environ
else:
self.environ = env
self.context = config.StaticContext()
# Lazy load this when needed
# (it will permanently update global cfg namespaces)
self.sruobj = None
self.opensearchobj = None
self.oaipmhobj = None
# init kvp
self.kvp = {}
self.mode = 'csw'
self.asynchronous = False
self.soap = False
self.request = None
self.exception = False
self.status = 'OK'
self.profiles = None
self.manager = False
self.outputschemas = {}
self.mimetype = 'application/xml; charset=UTF-8'
self.encoding = 'UTF-8'
self.pretty_print = 0
self.domainquerytype = 'list'
self.orm = 'django'
self.language = {'639_code': 'en', 'text': 'english'}
self.process_time_start = time()
# define CSW implementation object (default CSW3)
self.iface = csw3.Csw3(server_csw=self)
self.request_version = version
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# load user configuration
try:
LOGGER.info('Loading user configuration')
if isinstance(rtconfig, SafeConfigParser): # serialized already
self.config = rtconfig
else:
self.config = SafeConfigParser()
if isinstance(rtconfig, dict): # dictionary
for section, options in rtconfig.items():
self.config.add_section(section)
for k, v in options.items():
self.config.set(section, k, v)
else: # configuration file
import codecs
with codecs.open(rtconfig, encoding='utf-8') as scp:
self.config.readfp(scp)
except Exception as err:
msg = 'Could not load configuration'
LOGGER.exception('%s %s: %s', msg, rtconfig, err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service', msg)
return
# set server.home safely
# TODO: make this more abstract
self.config.set(
'server', 'home',
os.path.dirname(os.path.join(os.path.dirname(__file__), '..'))
)
self.context.pycsw_home = self.config.get('server', 'home')
self.context.url = self.config.get('server', 'url')
log.setup_logger(self.config)
LOGGER.info('running configuration %s', rtconfig)
LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING'])
# set OGC schemas location
if not self.config.has_option('server', 'ogc_schemas_base'):
self.config.set('server', 'ogc_schemas_base',
self.context.ogc_schemas_base)
# set mimetype
if self.config.has_option('server', 'mimetype'):
self.mimetype = self.config.get('server', 'mimetype').encode()
# set encoding
if self.config.has_option('server', 'encoding'):
self.encoding = self.config.get('server', 'encoding')
# set domainquerytype
if self.config.has_option('server', 'domainquerytype'):
self.domainquerytype = self.config.get('server', 'domainquerytype')
# set XML pretty print
if (self.config.has_option('server', 'pretty_print') and
self.config.get('server', 'pretty_print') == 'true'):
self.pretty_print = 1
# set Spatial Ranking option
if (self.config.has_option('server', 'spatial_ranking') and
self.config.get('server', 'spatial_ranking') == 'true'):
util.ranking_enabled = True
# set language default
if self.config.has_option('server', 'language'):
try:
LOGGER.info('Setting language')
lang_code = self.config.get('server', 'language').split('-')[0]
self.language['639_code'] = lang_code
self.language['text'] = self.context.languages[lang_code]
except Exception as err:
LOGGER.exception('Could not set language: %s', err)
pass
LOGGER.debug('Configuration: %s.', self.config)
LOGGER.debug('Model: %s.', self.context.model)
# load user-defined mappings if they exist
if self.config.has_option('repository', 'mappings'):
# override default repository mappings
try:
import imp
module = self.config.get('repository', 'mappings')
if os.sep in module: # filepath
modulename = '%s' % os.path.splitext(module)[0].replace(
os.sep, '.')
mappings = imp.load_source(modulename, module)
else: # dotted name
mappings = __import__(module, fromlist=[''])
LOGGER.info('Loading custom repository mappings '
'from %s', module)
self.context.md_core_model = mappings.MD_CORE_MODEL
self.context.refresh_dc(mappings.MD_CORE_MODEL)
except Exception as err:
LOGGER.exception('Could not load custom mappings: %s', err)
self.response = self.iface.exceptionreport(
'NoApplicableCode', 'service',
'Could not load repository.mappings')
# load outputschemas
LOGGER.info('Loading outputschemas')
for osch in pycsw.plugins.outputschemas.__all__:
output_schema_module = __import__(
'pycsw.plugins.outputschemas.%s' % osch)
mod = getattr(output_schema_module.plugins.outputschemas, osch)
self.outputschemas[mod.NAMESPACE] = mod
LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas)
LOGGER.debug('Namespaces: %s', self.context.namespaces)
def expand_path(self, path):
""" return safe path for WSGI environments """
if 'local.app_root' in self.environ and not os.path.isabs(path):
return os.path.join(self.environ['local.app_root'], path)
else:
return path
def dispatch_wsgi(self):
""" WSGI handler """
if hasattr(self, 'response'):
return self._write_response()
LOGGER.debug('WSGI mode detected')
if self.environ['REQUEST_METHOD'] == 'POST':
try:
request_body_size = int(self.environ.get('CONTENT_LENGTH', 0))
except (ValueError):
request_body_size = 0
self.requesttype = 'POST'
self.request = self.environ['wsgi.input'].read(request_body_size)
LOGGER.debug('Request type: POST. Request:\n%s\n', self.request)
else: # it's a GET request
self.requesttype = 'GET'
self.request = wsgiref.util.request_uri(self.environ)
try:
query_part = splitquery(self.request)[-1]
self.kvp = dict(parse_qsl(query_part, keep_blank_values=True))
except AttributeError as err:
LOGGER.exception('Could not parse query string')
self.kvp = {}
LOGGER.debug('Request type: GET. Request:\n%s\n', self.request)
return self.dispatch()
def opensearch(self):
""" enable OpenSearch """
if not self.opensearchobj:
self.opensearchobj = opensearch.OpenSearch(self.context)
return self.opensearchobj
def sru(self):
""" enable SRU """
if not self.sruobj:
self.sruobj = sru.Sru(self.context)
return self.sruobj
def oaipmh(self):
""" enable OAI-PMH """
if not self.oaipmhobj:
self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config)
return self.oaipmhobj
def dispatch(self, writer=sys.stdout, write_headers=True):
""" Handle incoming HTTP request """
error = 0
if self.requesttype == 'GET':
self.kvp = self.normalize_kvp(self.kvp)
version_202 = ('version' in self.kvp and
self.kvp['version'] == '2.0.2')
accept_version_202 = ('acceptversions' in self.kvp and
'2.0.2' in self.kvp['acceptversions'])
if version_202 or accept_version_202:
self.request_version = '2.0.2'
elif self.requesttype == 'POST':
if self.request.find(b'cat/csw/2.0.2') != -1:
self.request_version = '2.0.2'
elif self.request.find(b'cat/csw/3.0') != -1:
self.request_version = '3.0.0'
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'sru'):
self.mode = 'sru'
self.request_version = '2.0.2'
LOGGER.info('SRU mode detected; processing request')
self.kvp = self.sru().request_sru2csw(self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'oaipmh'):
self.mode = 'oaipmh'
self.request_version = '2.0.2'
LOGGER.info('OAI-PMH mode detected; processing request.')
self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k)
self.kvp = self.oaipmh().request(self.kvp)
if self.request_version == '2.0.2':
self.iface = csw2.Csw2(server_csw=self)
self.context.set_model('csw')
# configure transaction support, if specified in config
self._gen_manager()
namespaces = self.context.namespaces
ops = self.context.model['operations']
constraints = self.context.model['constraints']
# generate domain model
# NOTE: We should probably avoid this sort of mutable state for WSGI
if 'GetDomain' not in ops:
ops['GetDomain'] = self.context.gen_domains()
# generate distributed search model, if specified in config
if self.config.has_option('server', 'federatedcatalogues'):
LOGGER.info('Configuring distributed search')
constraints['FederatedCatalogues'] = {'values': []}
for fedcat in self.config.get('server',
'federatedcatalogues').split(','):
LOGGER.debug('federated catalogue: %s', fedcat)
constraints['FederatedCatalogues']['values'].append(fedcat)
for key, value in self.outputschemas.items():
get_records_params = ops['GetRecords']['parameters']
get_records_params['outputSchema']['values'].append(
value.NAMESPACE)
get_records_by_id_params = ops['GetRecordById']['parameters']
get_records_by_id_params['outputSchema']['values'].append(
value.NAMESPACE)
if 'Harvest' in ops:
harvest_params = ops['Harvest']['parameters']
harvest_params['ResourceType']['values'].append(
value.NAMESPACE)
LOGGER.info('Setting MaxRecordDefault')
if self.config.has_option('server', 'maxrecords'):
constraints['MaxRecordDefault']['values'] = [
self.config.get('server', 'maxrecords')]
# load profiles
if self.config.has_option('server', 'profiles'):
self.profiles = pprofile.load_profiles(
os.path.join('pycsw', 'plugins', 'profiles'),
pprofile.Profile,
self.config.get('server', 'profiles')
)
for prof in self.profiles['plugins'].keys():
tmp = self.profiles['plugins'][prof](self.context.model,
namespaces,
self.context)
key = tmp.outputschema # to ref by outputschema
self.profiles['loaded'][key] = tmp
self.profiles['loaded'][key].extend_core(self.context.model,
namespaces,
self.config)
LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys()))
# init repository
# look for tablename, set 'records' as default
if not self.config.has_option('repository', 'table'):
self.config.set('repository', 'table', 'records')
repo_filter = None
if self.config.has_option('repository', 'filter'):
repo_filter = self.config.get('repository', 'filter')
if self.config.has_option('repository', 'source'): # load custom repository
rs = self.config.get('repository', 'source')
rs_modname, rs_clsname = rs.rsplit('.', 1)
rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname])
rs_cls = getattr(rs_mod, rs_clsname)
try:
self.repository = rs_cls(self.context, repo_filter)
LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype)
except Exception as err:
msg = 'Could not load custom repository %s: %s' % (rs, err)
LOGGER.exception(msg)
error = 1
code = 'NoApplicableCode'
locator = 'service'
text = 'Could not initialize repository. Check server logs'
else: # load default repository
self.orm = 'sqlalchemy'
from pycsw.core import repository
try:
LOGGER.info('Loading default repository')
self.repository = repository.Repository(
self.config.get('repository', 'database'),
self.context,
self.environ.get('local.app_root', None),
self.config.get('repository', 'table'),
repo_filter
)
LOGGER.debug(
'Repository loaded (local): %s.' % self.repository.dbtype)
except Exception as err:
msg = 'Could not load repository (local): %s' % err
LOGGER.exception(msg)
error = 1
code = 'NoApplicableCode'
locator = 'service'
text = 'Could not initialize repository. Check server logs'
if self.requesttype == 'POST':
LOGGER.debug('HTTP POST request')
LOGGER.debug('CSW version: %s', self.iface.version)
self.kvp = self.iface.parse_postdata(self.request)
if isinstance(self.kvp, str): # it's an exception
error = 1
locator = 'service'
text = self.kvp
if (self.kvp.find('the document is not valid') != -1 or
self.kvp.find('document not well-formed') != -1):
code = 'NoApplicableCode'
else:
code = 'InvalidParameterValue'
LOGGER.debug('HTTP Headers:\n%s.', self.environ)
LOGGER.debug('Parsed request parameters: %s', self.kvp)
if (not isinstance(self.kvp, str) and 'mode' in self.kvp and
self.kvp['mode'] == 'opensearch'):
self.mode = 'opensearch'
LOGGER.info('OpenSearch mode detected; processing request.')
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or
(len(self.kvp) == 1 and 'config' in self.kvp)):
LOGGER.info('Turning on default csw30:Capabilities for base URL')
self.kvp = {
'service': 'CSW',
'acceptversions': '3.0.0',
'request': 'GetCapabilities'
}
http_accept = self.environ.get('HTTP_ACCEPT', '')
if 'application/opensearchdescription+xml' in http_accept:
self.mode = 'opensearch'
self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom'
if error == 0:
# test for the basic keyword values (service, version, request)
basic_options = ['service', 'request']
request = self.kvp.get('request', '')
own_version_integer = util.get_version_integer(
self.request_version)
if self.request_version == '2.0.2':
basic_options.append('version')
if self.request_version == '3.0.0' and 'version' not in self.kvp and self.requesttype == 'POST':
if 'service' not in self.kvp:
self.kvp['service'] = 'CSW'
basic_options.append('service')
self.kvp['version'] = self.request_version
basic_options.append('version')
for k in basic_options:
if k not in self.kvp:
if (k in ['version', 'acceptversions'] and
request == 'GetCapabilities'):
pass
else:
error = 1
locator = k
code = 'MissingParameterValue'
text = 'Missing keyword: %s' % k
break
# test each of the basic keyword values
if error == 0:
# test service
if self.kvp['service'] != 'CSW':
error = 1
locator = 'service'
code = 'InvalidParameterValue'
text = 'Invalid value for service: %s.\
Value MUST be CSW' % self.kvp['service']
# test version
kvp_version = self.kvp.get('version', '')
try:
kvp_version_integer = util.get_version_integer(kvp_version)
except Exception as err:
kvp_version_integer = 'invalid_value'
if (request != 'GetCapabilities' and
kvp_version_integer != own_version_integer):
error = 1
locator = 'version'
code = 'InvalidParameterValue'
text = ('Invalid value for version: %s. Value MUST be '
'2.0.2 or 3.0.0' % kvp_version)
# check for GetCapabilities acceptversions
if 'acceptversions' in self.kvp:
for vers in self.kvp['acceptversions'].split(','):
vers_integer = util.get_version_integer(vers)
if vers_integer == own_version_integer:
break
else:
error = 1
locator = 'acceptversions'
code = 'VersionNegotiationFailed'
text = ('Invalid parameter value in '
'acceptversions: %s. Value MUST be '
'2.0.2 or 3.0.0' %
self.kvp['acceptversions'])
# test request
if self.kvp['request'] not in \
self.context.model['operations']:
error = 1
locator = 'request'
if request in ['Transaction', 'Harvest']:
code = 'OperationNotSupported'
text = '%s operations are not supported' % request
else:
code = 'InvalidParameterValue'
text = 'Invalid value for request: %s' % request
if error == 1: # return an ExceptionReport
LOGGER.error('basic service options error: %s, %s, %s', code, locator, text)
self.response = self.iface.exceptionreport(code, locator, text)
else: # process per the request value
if 'responsehandler' in self.kvp:
# set flag to process asynchronously
import threading
self.asynchronous = True
request_id = self.kvp.get('requestid', None)
if request_id is None:
import uuid
self.kvp['requestid'] = str(uuid.uuid4())
if self.kvp['request'] == 'GetCapabilities':
self.response = self.iface.getcapabilities()
elif self.kvp['request'] == 'DescribeRecord':
self.response = self.iface.describerecord()
elif self.kvp['request'] == 'GetDomain':
self.response = self.iface.getdomain()
elif self.kvp['request'] == 'GetRecords':
if self.asynchronous: # process asynchronously
threading.Thread(target=self.iface.getrecords).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.getrecords()
elif self.kvp['request'] == 'GetRecordById':
self.response = self.iface.getrecordbyid()
elif self.kvp['request'] == 'GetRepositoryItem':
self.response = self.iface.getrepositoryitem()
elif self.kvp['request'] == 'Transaction':
self.response = self.iface.transaction()
elif self.kvp['request'] == 'Harvest':
if self.asynchronous: # process asynchronously
threading.Thread(target=self.iface.harvest).start()
self.response = self.iface._write_acknowledgement()
else:
self.response = self.iface.harvest()
else:
self.response = self.iface.exceptionreport(
'InvalidParameterValue', 'request',
'Invalid request parameter: %s' % self.kvp['request']
)
LOGGER.info('Request processed')
if self.mode == 'sru':
LOGGER.info('SRU mode detected; processing response.')
self.response = self.sru().response_csw2sru(self.response,
self.environ)
elif self.mode == 'opensearch':
LOGGER.info('OpenSearch mode detected; processing response.')
self.response = self.opensearch().response_csw2opensearch(
self.response, self.config)
elif self.mode == 'oaipmh':
LOGGER.info('OAI-PMH mode detected; processing response.')
self.response = self.oaipmh().response(
self.response, self.oaiargs, self.repository,
self.config.get('server', 'url')
)
return self._write_response()
def getcapabilities(self):
""" Handle GetCapabilities request """
return self.iface.getcapabilities()
def describerecord(self):
""" Handle DescribeRecord request """
return self.iface.describerecord()
def getdomain(self):
""" Handle GetDomain request """
return self.iface.getdomain()
def getrecords(self):
""" Handle GetRecords request """
return self.iface.getrecords()
def getrecordbyid(self, raw=False):
""" Handle GetRecordById request """
return self.iface.getrecordbyid(raw)
def getrepositoryitem(self):
""" Handle GetRepositoryItem request """
return self.iface.getrepositoryitem()
def transaction(self):
""" Handle Transaction request """
return self.iface.transaction()
def harvest(self):
""" Handle Harvest request """
return self.iface.harvest()
def _write_response(self):
""" Generate response """
# set HTTP response headers and XML declaration
xmldecl = ''
appinfo = ''
LOGGER.info('Writing response.')
if hasattr(self, 'soap') and self.soap:
self._gen_soap_wrapper()
if etree.__version__ >= '3.5.0': # remove superfluous namespaces
etree.cleanup_namespaces(self.response,
keep_ns_prefixes=self.context.keep_ns_prefixes)
response = etree.tostring(self.response,
pretty_print=self.pretty_print,
encoding='unicode')
if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and
self.kvp['outputformat'] == 'application/json'):
self.contenttype = self.kvp['outputformat']
from pycsw.core.formats import fmt_json
response = fmt_json.xml2json(response,
self.context.namespaces,
self.pretty_print)
else: # it's XML
if 'outputformat' in self.kvp:
self.contenttype = self.kvp['outputformat']
else:
self.contenttype = self.mimetype
xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>'
'\n' % self.encoding)
appinfo = '<!-- pycsw %s -->\n' % self.context.version
if isinstance(self.contenttype, bytes):
self.contenttype = self.contenttype.decode()
s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding)
LOGGER.debug('Response code: %s',
self.context.response_codes[self.status])
LOGGER.debug('Response:\n%s', s)
return [self.context.response_codes[self.status], s]
def _gen_soap_wrapper(self):
""" Generate SOAP wrapper """
LOGGER.info('Writing SOAP wrapper.')
node = etree.Element(
util.nspath_eval('soapenv:Envelope', self.context.namespaces),
nsmap=self.context.namespaces
)
schema_location_ns = util.nspath_eval('xsi:schemaLocation',
self.context.namespaces)
node.attrib[schema_location_ns] = '%s %s' % (
self.context.namespaces['soapenv'],
self.context.namespaces['soapenv']
)
node2 = etree.SubElement(
node, util.nspath_eval('soapenv:Body', self.context.namespaces))
if self.exception:
node3 = etree.SubElement(
node2,
util.nspath_eval('soapenv:Fault', self.context.namespaces)
)
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Code', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Value', self.context.namespaces)
).text = 'soap:Server'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Reason', self.context.namespaces)
)
etree.SubElement(
node4,
util.nspath_eval('soapenv:Text', self.context.namespaces)
).text = 'A server exception was encountered.'
node4 = etree.SubElement(
node3,
util.nspath_eval('soapenv:Detail', self.context.namespaces)
)
node4.append(self.response)
else:
node2.append(self.response)
self.response = node
def _gen_manager(self):
""" Update self.context.model with CSW-T advertising """
if (self.config.has_option('manager', 'transactions') and
self.config.get('manager', 'transactions') == 'true'):
self.manager = True
self.context.model['operations_order'].append('Transaction')
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {}
}
schema_values = [
'http://www.opengis.net/cat/csw/2.0.2',
'http://www.opengis.net/cat/csw/3.0',
'http://www.opengis.net/wms',
'http://www.opengis.net/wmts/1.0',
'http://www.opengis.net/wfs',
'http://www.opengis.net/wfs/2.0',
'http://www.opengis.net/wcs',
'http://www.opengis.net/wps/1.0.0',
'http://www.opengis.net/sos/1.0',
'http://www.opengis.net/sos/2.0',
'http://www.isotc211.org/2005/gmi',
'urn:geoss:waf',
]
self.context.model['operations_order'].append('Harvest')
self.context.model['operations']['Harvest'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'ResourceType': {'values': schema_values}
}
}
self.context.model['operations']['Transaction'] = {
'methods': {'get': False, 'post': True},
'parameters': {
'TransactionSchemas': {'values': sorted(schema_values)}
}
}
self.csw_harvest_pagesize = 10
if self.config.has_option('manager', 'csw_harvest_pagesize'):
self.csw_harvest_pagesize = int(
self.config.get('manager', 'csw_harvest_pagesize'))
def _test_manager(self):
""" Verify that transactions are allowed """
if self.config.get('manager', 'transactions') != 'true':
raise RuntimeError('CSW-T interface is disabled')
""" get the client first forwarded ip """
if 'HTTP_X_FORWARDED_FOR' in self.environ:
ipaddress = self.environ['HTTP_X_FORWARDED_FOR'].split(',')[0].strip()
else:
ipaddress = self.environ['REMOTE_ADDR']
if not self.config.has_option('manager', 'allowed_ips') or \
(self.config.has_option('manager', 'allowed_ips') and not
util.ipaddress_in_whitelist(ipaddress,
self.config.get('manager', 'allowed_ips').split(','))):
raise RuntimeError(
'CSW-T operations not allowed for this IP address: %s' % ipaddress)
def _cql_update_queryables_mappings(self, cql, mappings):
""" Transform CQL query's properties to underlying DB columns """
LOGGER.debug('Raw CQL text = %s', cql)
LOGGER.debug(str(list(mappings.keys())))
if cql is not None:
for key in mappings.keys():
try:
cql = cql.replace(key, mappings[key]['dbcol'])
except:
cql = cql.replace(key, mappings[key])
LOGGER.debug('Interpolated CQL text = %s.', cql)
return cql
def _process_responsehandler(self, xml):
""" Process response handler """
if self.kvp['responsehandler'] is not None:
LOGGER.info('Processing responsehandler %s' %
self.kvp['responsehandler'])
uprh = urlparse(self.kvp['responsehandler'])
if uprh.scheme == 'mailto': # email
import smtplib
LOGGER.debug('Email detected')
smtp_host = 'localhost'
if self.config.has_option('server', 'smtp_host'):
smtp_host = self.config.get('server', 'smtp_host')
body = ('Subject: pycsw %s results\n\n%s' %
(self.kvp['request'], xml))
try:
LOGGER.info('Sending email')
msg = smtplib.SMTP(smtp_host)
msg.sendmail(
self.config.get('metadata:main', 'contact_email'),
uprh.path, body
)
msg.quit()
LOGGER.debug('Email sent successfully.')
except Exception as err:
LOGGER.exception('Error processing email')
elif uprh.scheme == 'ftp':
import ftplib
LOGGER.debug('FTP detected.')
try:
LOGGER.info('Sending to FTP server.')
ftp = ftplib.FTP(uprh.hostname)
if uprh.username is not None:
ftp.login(uprh.username, uprh.password)
ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml))
ftp.quit()
LOGGER.debug('FTP sent successfully.')
except Exception as err:
LOGGER.exception('Error processing FTP')
@staticmethod
def normalize_kvp(kvp):
"""Normalize Key Value Pairs.
This method will transform all keys to lowercase and leave values
unchanged, as specified in the CSW standard (see for example note
C on Table 62 - KVP Encoding for DescribeRecord operation request
of the CSW standard version 2.0.2)
:arg kvp: a mapping with Key Value Pairs
:type kvp: dict
:returns: A new dictionary with normalized parameters
"""
result = dict()
for name, value in kvp.items():
result[name.lower()] = value
return result
def create_config():
"""
Create a Trident configuration file using interaction with the user.
This function is called by :class:`~trident.parse_config` if it appears
that the configuration has not yet been set up for the user. It will
attempt to create a configuration file and download an ion table
datafile from the web. It does this using user interaction from the
python prompt.
"""
default_dir = os.path.expanduser('~/.trident')
trident()
print(
"It appears that this is your first time using Trident. To finalize your"
)
print("Trident installation, you must:")
print(" * create a `~/.trident` directory")
print(" * create a config.tri file in your `~/.trident` directory")
print(" * download an ion table file for calculating ionization fractions")
print("")
print(
"You can do this manually by following the installation docs, or we can"
)
print("do it automatically now if you have web access.")
print("")
print("Would you like to do this automatically? ([y]/n)")
value = input().rstrip()
if not value == '' and not value == 'y':
sys.exit(
'Instructions at http://trident.readthedocs.org/en/latest/installation.html'
)
print("")
print("Where would you like Trident to store the ion table file?")
print("[%s]" % default_dir)
# First assure that the .trident directory is created for storing
# the config file.
ensure_directory(default_dir)
datadir = input().rstrip()
if datadir == '':
datadir = default_dir
datadir = os.path.expanduser(datadir)
# Try to create data directory if it doesn't exist
try:
ensure_directory(datadir)
print("Using %s" % datadir)
print("")
except BaseException:
print("Cannot create directory %s" % datadir)
raise
# Get the datafile from the web
datafile = get_datafiles(datadir=datadir)
# Create the config file and make it to the datadir and datafiles chosen
config = SafeConfigParser()
config.add_section('Trident')
config.set('Trident', 'ion_table_dir', datadir)
config.set('Trident', 'ion_table_file', datafile)
config_filename = os.path.expanduser('~/.trident/config.tri')
with open(config_filename, 'w') as configfile:
config.write(configfile)
print("")
print("Installation complete. I recommend verifying your installation")
print("to assure that everything is working. Try: trident.verify()")
# Return the config file path so we can load it and get parameters.
return config_filename
class Config(object):
"""
Manages the configuration file
"""
def __init__(self):
"""
DEFAULT VALUES
"""
self._basescript = None
self.recentvaults = []
self.pwlength = 10
self.search_notes = False
self.search_passwd = False
self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_"
self.avoid_bigrams = "cl mn nm nn rn vv VV"
self._fname = self.get_config_filename()
self._parser = SafeConfigParser()
if os.path.exists(self._fname):
self._parser.read(self._fname)
if not self._parser.has_section("base"):
self._parser.add_section("base")
for num in range(10):
if (not self._parser.has_option("base", "recentvaults" + str(num))):
break
self.recentvaults.append(self._parser.get("base", "recentvaults" + str(num)))
if self._parser.has_option("base", "pwlength"):
self.pwlength = int(self._parser.get("base", "pwlength"))
if self._parser.has_option("base", "search_notes"):
if self._parser.get("base", "search_notes") == "True":
self.search_notes = True
if self._parser.has_option("base", "search_passwd"):
if self._parser.get("base", "search_passwd") == "True":
self.search_passwd = True
if self._parser.has_option("base", "alphabet"):
self.alphabet = self._parser.get("base", "alphabet")
if self._parser.has_option("base", "avoid_bigrams"):
self.avoid_bigrams = self._parser.get("base", "avoid_bigrams")
if not os.path.exists(self._fname):
self.save()
def set_basescript(self, basescript):
self._basescript = basescript
def get_basescript(self):
return self._basescript
def save(self):
if (not os.path.exists(os.path.dirname(self._fname))):
os.mkdir(os.path.dirname(self._fname))
# remove duplicates and trim to 10 items
_saved_recentvaults = []
for item in self.recentvaults:
if item in _saved_recentvaults:
continue
self._parser.set("base", "recentvaults" + str(len(_saved_recentvaults)), item)
_saved_recentvaults.append(item)
if (len(_saved_recentvaults) >= 10):
break
self._parser.set("base", "pwlength", str(self.pwlength))
self._parser.set("base", "search_notes", str(self.search_notes))
self._parser.set("base", "search_passwd", str(self.search_passwd))
self._parser.set("base", "alphabet", str(self.alphabet))
self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams))
filehandle = open(self._fname, 'w')
self._parser.write(filehandle)
filehandle.close()
@staticmethod
def get_config_filename():
"""
Returns the full filename of the config file
"""
base_fname = "loxodo"
# On Mac OS X, config files go to ~/Library/Application Support/foo/
if platform.system() == "Darwin":
base_path = os.path.join(os.path.expanduser("~"), "Library", "Application Support")
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# On Microsoft Windows, config files go to $APPDATA/foo/
if platform.system() in ("Windows", "Microsoft"):
if ("APPDATA" in os.environ):
base_path = os.environ["APPDATA"]
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# Allow config directory override as per freedesktop.org XDG Base Directory Specification
if ("XDG_CONFIG_HOME" in os.environ):
base_path = os.environ["XDG_CONFIG_HOME"]
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
# Default configuration path is ~/.config/foo/
base_path = os.path.join(os.path.expanduser("~"), ".config")
if os.path.isdir(base_path):
return os.path.join(base_path, base_fname, base_fname + ".ini")
else:
return os.path.join(os.path.expanduser("~"),"."+ base_fname + ".ini")
class PackageConfigHandler(object):
"""
Manager class for packages files for tracking installation of modules
"""
def __init__(self):
# noinspection PyUnresolvedReferences
from six.moves.configparser import SafeConfigParser
self.package_cfg = os.path.expanduser('~/Documents/site-packages/.pypi_packages')
if not os.path.isfile(self.package_cfg):
print('Creating package file')
with open(self.package_cfg, 'w') as outs:
outs.close()
self.parser = SafeConfigParser()
self.parser.read(self.package_cfg)
def save(self):
with open(self.package_cfg, 'w') as outs:
self.parser.write(outs)
def add_module(self, pkg_info):
"""
:param pkg_info: A dict that has name, url, version, summary
:return:
"""
if not self.parser.has_section(pkg_info['name']):
self.parser.add_section(pkg_info['name'])
self.parser.set(pkg_info['name'], 'url', pkg_info['url'])
self.parser.set(pkg_info['name'], 'version', pkg_info['version'])
self.parser.set(pkg_info['name'], 'summary', pkg_info['summary'])
self.parser.set(pkg_info['name'], 'files', pkg_info['files'])
self.parser.set(pkg_info['name'], 'dependency', pkg_info['dependency'])
self.save()
def list_modules(self):
return [module for module in self.parser.sections()]
def module_exists(self, name):
return self.parser.has_section(name)
def get_info(self, name):
if self.parser.has_section(name):
tbl = {}
for opt, value in self.parser.items(name):
tbl[opt] = value
return tbl
def remove_module(self, name):
self.parser.remove_section(name)
self.save()
def get_files_installed(self, section_name):
if self.parser.has_option(section_name, 'files'):
files = self.parser.get(section_name, 'files').strip()
return files.split(',')
else:
return None
def get_dependencies(self, section_name):
if self.parser.has_option(section_name, 'dependency'):
dependencies = self.parser.get(section_name, 'dependency').strip()
return set(dependencies.split(',')) if dependencies != '' else set()
else:
return None
def get_all_dependencies(self, exclude_module=()):
all_dependencies = set()
for section_name in self.parser.sections():
if section_name not in exclude_module and self.parser.has_option(section_name, 'dependency'):
dependencies = self.parser.get(section_name, 'dependency').strip()
if dependencies != '':
for dep in dependencies.split(','):
all_dependencies.add(dep)
return all_dependencies
