def config(args): logger.debug("config: " + str(args)) conf_path = os.path.expanduser(config_path) with open(conf_path, 'w+') as f: cp = SafeConfigParser() cp.add_section("common") cp.set('common', 'secret_id', args.secret_id) cp.set('common', 'secret_key', args.secret_key) if args.token != "": cp.set('common', 'token', args.token) cp.set('common', 'bucket', args.bucket) if args.endpoint: cp.set('common', 'endpoint', args.endpoint) else: cp.set('common', 'region', args.region) cp.set('common', 'max_thread', str(args.max_thread)) cp.set('common', 'part_size', str(args.part_size)) cp.set('common', 'retry', str(args.retry)) cp.set('common', 'timeout', str(args.timeout)) if args.appid != "": cp.set('common', 'appid', args.appid) if args.use_http: cp.set('common', 'schema', 'http') else: cp.set('common', 'schema', 'https') cp.set('common', 'verify', args.verify) if args.anonymous: cp.set('common', 'anonymous', 'True') else: cp.set('common', 'anonymous', 'False') cp.write(f) logger.info("Created configuration file in {path}".format(path=to_printable_str(conf_path)))
def from_sections(cls, sections): config = SafeConfigParser() for section, keyvalues in sections.items(): config.add_section(section) for key, value in keyvalues.items(): config.set(section, key, value) return cls(config)
def create_header(self, data_only): ''' Create the backup file header that contains the meta data about this particular backup. ''' config = SafeConfigParser() config.add_section("ipa") if data_only: config.set('ipa', 'type', 'DATA') else: config.set('ipa', 'type', 'FULL') config.set('ipa', 'time', time.strftime(ISO8601_DATETIME_FMT, time.gmtime())) config.set('ipa', 'host', api.env.host) config.set('ipa', 'ipa_version', str(version.VERSION)) config.set('ipa', 'version', '1') dn = DN(('cn', api.env.host), ('cn', 'masters'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn) services_cns = [] try: conn = self.get_connection() services = conn.get_entries(dn, conn.SCOPE_ONELEVEL) except errors.NetworkError: self.log.critical( "Unable to obtain list of master services, continuing anyway") except Exception as e: self.log.error("Failed to read services from '%s': %s" % (conn.host, e)) else: services_cns = [s.single_value['cn'] for s in services] config.set('ipa', 'services', ','.join(services_cns)) with open(self.header, 'w') as fd: config.write(fd)
class Config(object): def __init__(self): self.config = SafeConfigParser() self.name = '' def parse(self, fname, override): if override: override = [x for x in csv.reader( ' '.join(override).split(','), delimiter='.')] logger.info('Reading configuration file: {}'.format(fname)) if not os.path.isfile(fname): logger.interrupt('File doesn\'t exist: {}'.format(fname)) self.config.optionxform = str self.config.read(fname) for section, option, value in override: if not self.config.has_section(section): self.config.add_section(section) self.config.set(section, option, value) basename = os.path.basename(fname) self.name = os.path.splitext(basename)[0] @safe def _get_options_as_dict(self, section): if section in self.config.sections(): return {p: v for p, v in self.config.items(section)} else: return {}
class Config(object): """A ConfigParser wrapper to support defaults when calling instance methods, and also tied to a single section""" SECTION = 'scrapyd' def __init__(self, values=None, extra_sources=()): if values is None: sources = self._getsources() default_config = get_data(__package__, 'default_scrapyd.conf').decode('utf8') self.cp = SafeConfigParser() self.cp.readfp(io.StringIO(default_config)) sources.extend(extra_sources) for fname in sources: try: with io.open(fname) as fp: self.cp.readfp(fp) except (IOError, OSError): pass else: self.cp = SafeConfigParser(values) self.cp.add_section(self.SECTION) def _getsources(self): sources = ['/etc/scrapyd/scrapyd.conf', r'c:\scrapyd\scrapyd.conf'] sources += sorted(glob.glob('/etc/scrapyd/conf.d/*')) sources += ['scrapyd.conf'] sources += [expanduser('~/.scrapyd.conf')] scrapy_cfg = closest_scrapy_cfg() if scrapy_cfg: sources.append(scrapy_cfg) return sources def _getany(self, method, option, default): try: return method(self.SECTION, option) except (NoSectionError, NoOptionError): if default is not None: return default raise def get(self, option, default=None): return self._getany(self.cp.get, option, default) def getint(self, option, default=None): return self._getany(self.cp.getint, option, default) def getfloat(self, option, default=None): return self._getany(self.cp.getfloat, option, default) def getboolean(self, option, default=None): return self._getany(self.cp.getboolean, option, default) def items(self, section, default=None): try: return self.cp.items(section) except (NoSectionError, NoOptionError): if default is not None: return default raise
def save(self): """Save the modules to @_path. If @modules is an empty dict, then @_path should be removed. """ root_logger.debug("Saving StateFile to '%s'", self._path) for module in list(self.modules.keys()): if len(self.modules[module]) == 0: del self.modules[module] if len(self.modules) == 0: root_logger.debug(" -> no modules, removing file") if os.path.exists(self._path): os.remove(self._path) return p = SafeConfigParser() p.optionxform = str for module in self.modules.keys(): p.add_section(module) for (key, value) in self.modules[module].items(): p.set(module, key, str(value)) with open(self._path, "w") as f: p.write(f)
class Config(object): """A ConfigParser wrapper to support defaults when calling instance methods, and also tied to a single section""" SECTION = 'silkyy' def __init__(self, values=None, extra_sources=()): if values is None: sources = self._getsources() self.cp = ConfigParser() if __package__: default_config = ensure_str(get_data(__package__, 'default.conf')) self._load_config_file(StringIO(default_config)) for source in sources: if os.path.exists(source): self._load_config_file(open(source)) else: self.cp = SafeConfigParser(values) self.cp.add_section(self.SECTION) def _load_config_file(self, fp): config = StringIO() config.write('[' + self.SECTION + ']' + os.linesep) config.write(fp.read()) config.seek(0, os.SEEK_SET) self.cp.readfp(config) def _getsources(self): sources = ['conf/silkyy.conf'] return sources def get(self, option, default=None): env_key = 'SILKYY_' + option.replace('.', '_').upper() try: return os.getenv(env_key) or self.cp.get(self.SECTION, option) except (NoSectionError, NoOptionError): if default is not None: return default raise def _get(self, option, conv, default=None): return conv(self.get(option, default)) def getint(self, option, default=None): return self._get(option, int, default) def getboolean(self, option, default=None): return self._get(option, str2bool, default) def getfloat(self, option, default=None): return self._get(option, float, default) def items(self, section, default=None): try: return self.cp.items(section) except (NoSectionError, NoOptionError): if default is not None: return default raise
def create_header(self, data_only): ''' Create the backup file header that contains the meta data about this particular backup. ''' config = SafeConfigParser() config.add_section("ipa") if data_only: config.set('ipa', 'type', 'DATA') else: config.set('ipa', 'type', 'FULL') config.set('ipa', 'time', time.strftime(ISO8601_DATETIME_FMT, time.gmtime())) config.set('ipa', 'host', api.env.host) config.set('ipa', 'ipa_version', str(version.VERSION)) config.set('ipa', 'version', '1') dn = DN(('cn', api.env.host), ('cn', 'masters'), ('cn', 'ipa'), ('cn', 'etc'), api.env.basedn) services_cns = [] try: conn = self.get_connection() services = conn.get_entries(dn, conn.SCOPE_ONELEVEL) except errors.NetworkError: self.log.critical( "Unable to obtain list of master services, continuing anyway") except Exception as e: self.log.error("Failed to read services from '%s': %s" % (conn.host, e)) else: services_cns = [s.single_value['cn'] for s in services] config.set('ipa', 'services', ','.join(services_cns)) with open(self.header, 'w') as fd: config.write(fd)
def save(self): """Save the modules to @_path. If @modules is an empty dict, then @_path should be removed. """ root_logger.debug("Saving StateFile to '%s'", self._path) for module in list(self.modules): if len(self.modules[module]) == 0: del self.modules[module] if len(self.modules) == 0: root_logger.debug(" -> no modules, removing file") if os.path.exists(self._path): os.remove(self._path) return p = SafeConfigParser() p.optionxform = str for module in self.modules: p.add_section(module) for (key, value) in self.modules[module].items(): p.set(module, key, str(value)) with open(self._path, "w") as f: p.write(f)
def from_config(cls, path, **kwargs): """ Create a LastFM instance from a config file in the following format: [lastfm] api_key = myapikey api_secret = myapisecret username = thesquelched password = plaintext_password # Can be 'password' or 'hashed_password' auth_method = password You can also override config values with keyword arguments. """ config = SafeConfigParser() config.add_section('lastfm') config.read(os.path.expanduser(os.path.expandvars(path))) return LastFM( cls._getoption(config, kwargs, 'api_key'), cls._getoption(config, kwargs, 'api_secret'), username=cls._getoption(config, kwargs, 'username', None), password=cls._getoption(config, kwargs, 'password', None), auth_method=cls._getoption(config, kwargs, 'auth_method', None), session_key=cls._getoption(config, kwargs, 'session_key', None), url=cls._getoption(config, kwargs, 'url', None), )
def set_namespace_options(namespace, options, remove=None): """ Set options in the local namespace configuration file. Can have nasty effects, be careful, only use in test code. :param namespace: the namespace to work with :param options: a dictionary with options to set :param remove: an iterable of options to remove :returns: a dictionary with all options of the namespace """ parser = SafeConfigParser({}) potential_confs = list() actual_confs = list() for p, _ in _config_paths(): potential_confs.append(p) fdone = parser.read((p,)) actual_confs.extend(fdone) if not actual_confs: raise ValueError( "Could not read configuration from any of %s" % potential_confs) if not parser.has_section(namespace): print('Namespace %s was not found in %s' % (namespace, actual_confs)) parser.add_section(namespace) for key, val in options.items(): parser.set(namespace, key, str(val)) if remove: for key in remove: parser.remove_option(namespace, key) with open(actual_confs[-1], 'w') as outfile: parser.write(outfile) return dict(parser.items(namespace))
class Config(object): """A ConfigParser wrapper to support defaults when calling instance methods, and also tied to a single section""" SECTION = 'scrapydartx' def __init__(self, values=None, extra_sources=()): if values is None: sources = self._getsources() default_config = get_data(__package__, 'default_scrapyd.conf').decode('utf8') self.cp = SafeConfigParser() self.cp.readfp(StringIO(default_config)) self.cp.read(sources) for fp in extra_sources: self.cp.readfp(fp) else: self.cp = SafeConfigParser(values) self.cp.add_section(self.SECTION) def _getsources(self): sources = [ '/etc/scrapydartx/scrapydartx.conf', r'c:\scrapyd\scrapyd.conf' ] sources += sorted(glob.glob('/etc/scrapydartx/conf.d/*')) sources += ['scrapydartx.conf'] sources += [expanduser('~/.scrapydartx.conf')] scrapy_cfg = closest_scrapy_cfg() if scrapy_cfg: sources.append(scrapy_cfg) return sources def _getany(self, method, option, default): try: return method(self.SECTION, option) except (NoSectionError, NoOptionError): if default is not None: return default raise def get(self, option, default=None): return self._getany(self.cp.get, option, default) def getint(self, option, default=None): return self._getany(self.cp.getint, option, default) def getfloat(self, option, default=None): return self._getany(self.cp.getfloat, option, default) def getboolean(self, option, default=None): return self._getany(self.cp.getboolean, option, default) def items(self, section, default=None): try: return self.cp.items(section) except (NoSectionError, NoOptionError): if default is not None: return default raise
def load(f): p = SafeConfigParser() p.read(f) if not p.has_section('oauth2'): p.add_section('oauth2') if not p.has_section('oauth2-state'): p.add_section('oauth2-state') return p
def load(f): p = SafeConfigParser() p.read(f) if not p.has_section('oauth2'): p.add_section('oauth2') if not p.has_section('oauth2-state'): p.add_section('oauth2-state') return p
def _init_config_parser(self, sections=None): parser = SafeConfigParser(allow_no_value=True) if sections: for section in sections: parser.add_section(section) for key, value in sections[section].items(): parser.set(section, key, self._value_converter.to_strings(value)) return parser
def _init_config_parser(self, sections=None): parser = SafeConfigParser(allow_no_value=True) if sections: for section in sections: parser.add_section(section) for key, value in sections[section].items(): parser.set(section, key, self._value_converter.to_strings(value)) return parser
def generate_default_engine_config(cls): config = SafeConfigParser() config.add_section("OPENCAFE_ENGINE") config.set("OPENCAFE_ENGINE", "config_directory", EngineDirectoryManager.OPENCAFE_SUB_DIRS.CONFIG_DIR) config.set("OPENCAFE_ENGINE", "data_directory", EngineDirectoryManager.OPENCAFE_SUB_DIRS.DATA_DIR) config.set("OPENCAFE_ENGINE", "log_directory", EngineDirectoryManager.OPENCAFE_SUB_DIRS.LOG_DIR) config.set("OPENCAFE_ENGINE", "temp_directory", EngineDirectoryManager.OPENCAFE_SUB_DIRS.TEMP_DIR) config.set("OPENCAFE_ENGINE", "master_log_file_name", "cafe.master") config.set("OPENCAFE_ENGINE", "logging_verbosity", "STANDARD") config.set("OPENCAFE_ENGINE", "default_test_repo", "cloudroast") return config
def _init_config_parser(self, sections=None): parser = SafeConfigParser(allow_no_value=True) if sections: for section in sections: parser.add_section(section) for key, value in sections[section].items(): str_val = StringConverter( {self._default_value: None}).to_strings(value) parser.set(section, key, str(str_val) if str_val is not None else str_val) return parser
def save_config(self): self.log.info("Finalizing configuration") config = SafeConfigParser() config.add_section("realm") config.set("realm", "realm_name", api.env.realm) config.set("realm", "master_host_name", api.env.host) config.set("realm", "domain_name", api.env.domain) config.set("realm", "destination_host", self.replica_fqdn) config.set("realm", "subject_base", str(self.subject_base)) config.set("realm", "version", str(version.NUM_VERSION)) with open(os.path.join(self.dir, "realm_info"), "w") as fd: config.write(fd)
def save_config(self): self.log.info("Finalizing configuration") config = SafeConfigParser() config.add_section("realm") config.set("realm", "realm_name", api.env.realm) config.set("realm", "master_host_name", api.env.host) config.set("realm", "domain_name", api.env.domain) config.set("realm", "destination_host", self.replica_fqdn) config.set("realm", "subject_base", str(self.subject_base)) config.set("realm", "version", str(version.NUM_VERSION)) with open(os.path.join(self.dir, "realm_info"), "w") as fd: config.write(fd)
def generate_default_engine_config(cls): config = SafeConfigParser() config.add_section('OPENCAFE_ENGINE') config.set('OPENCAFE_ENGINE', 'config_directory', EngineDirectoryManager.OPENCAFE_SUB_DIRS.CONFIG_DIR) config.set('OPENCAFE_ENGINE', 'data_directory', EngineDirectoryManager.OPENCAFE_SUB_DIRS.DATA_DIR) config.set('OPENCAFE_ENGINE', 'log_directory', EngineDirectoryManager.OPENCAFE_SUB_DIRS.LOG_DIR) config.set('OPENCAFE_ENGINE', 'temp_directory', EngineDirectoryManager.OPENCAFE_SUB_DIRS.TEMP_DIR) config.set('OPENCAFE_ENGINE', 'master_log_file_name', 'cafe.master') config.set('OPENCAFE_ENGINE', 'logging_verbosity', 'STANDARD') config.set('OPENCAFE_ENGINE', 'default_test_repo', 'cloudroast') return config
def config(request): path = request.config.getoption("--config") config = SafeConfigParser() for section, values in CONFIG_DEFAULTS.items(): config.add_section(section) for key, value in values.items(): config.set(section, key, value) if path: config.read([path]) with open(path) as f: LOG.debug("Configuration:\n%s", f.read()) return config
def write(cfg_obj, output_file_path): """ Only supports writing out a conflagration object with namespaces that follow the section.key=value pattern that ConfigFile.parse generates """ parser = SafeConfigParser() for k in cfg_obj.__dict__.keys(): parser.add_section(k) try: for sub_k, sub_v in cfg_obj.__dict__[k].__dict__.items(): parser.set(k, sub_k, sub_v) except Exception: raise Exception( "Output to config file not supported for conflagrations" "nested beyond a one dot namespace.") with open(output_file_path, 'w') as f: parser.write(f)
def save_to_config(save_config, config_obj, verbose=False): """ Write all the values in the config object to a given file in the user's home folder :param config_file: Name of config file to store in user's home folder :param config_obj: The config object to export to config file :param verbose: Specify if stdout should display a message :return: """ parser = SafeConfigParser() parser.read(save_config) if not os.path.exists(save_config): parser.add_section('settings') for k, v in viewitems(config_obj._asdict()): if v is not None and isinstance(v, str): parser.set('settings', k, v) with open(save_config, 'w') as f: parser.write(f) if verbose: click.echo("Config file written to {}".format(save_config))
def save(self): """Save the file list to @_index. If @files is an empty dict, then @_index should be removed. """ root_logger.debug("Saving Index File to '%s'", self._index) if len(self.files) == 0: root_logger.debug(" -> no files, removing file") if os.path.exists(self._index): os.remove(self._index) return p = SafeConfigParser() p.optionxform = str p.add_section('files') for (key, value) in self.files.items(): p.set('files', key, str(value)) with open(self._index, "w") as f: p.write(f)
def save(self): """Save the file list to @_index. If @files is an empty dict, then @_index should be removed. """ root_logger.debug("Saving Index File to '%s'", self._index) if len(self.files) == 0: root_logger.debug(" -> no files, removing file") if os.path.exists(self._index): os.remove(self._index) return p = SafeConfigParser() p.optionxform = str p.add_section('files') for (key, value) in self.files.items(): p.set('files', key, str(value)) with open(self._index, "w") as f: p.write(f)
def configure(filename=None): """This function gives to the user application a chance to define where configuration file should live. Subsequent calls to this function will have no effect, unless you call :func:`reconfigure`. :param str filename: Full path to configuration file. """ global retry if getattr(configure, '_configured', False): return filename = filename or DEFAULT_CONFIG_FILENAME _ensure_directory(filename) parser = SafeConfigParser() if os.path.isfile(filename): with open(filename, 'rt') as fp: parser.readfp(fp) if not parser.has_section(RETRY_SECTION): parser.add_section(RETRY_SECTION) parser.set(RETRY_SECTION, 'max_tries', str(constants.BACKOFF_DEFAULT_MAXTRIES)) parser.set(RETRY_SECTION, 'delay', str(constants.BACKOFF_DEFAULT_DELAY)) parser.set(RETRY_SECTION, 'factor', str(constants.BACKOFF_DEFAULT_FACTOR)) with open(filename, 'wt') as fp: parser.write(fp) retry = RetrySettings(max_tries=parser.getint(RETRY_SECTION, 'max_tries'), delay=parser.getint(RETRY_SECTION, 'delay'), factor=parser.getint(RETRY_SECTION, 'factor')) setattr(configure, '_configured', True) setattr(configure, '_configured_filename', filename)
def generate_default_engine_config(cls): config = SafeConfigParser() config.add_section('OPENCAFE_ENGINE') config.set( 'OPENCAFE_ENGINE', 'config_directory', EngineDirectoryManager.OPENCAFE_SUB_DIRS.CONFIG_DIR) config.set( 'OPENCAFE_ENGINE', 'data_directory', EngineDirectoryManager.OPENCAFE_SUB_DIRS.DATA_DIR) config.set( 'OPENCAFE_ENGINE', 'log_directory', EngineDirectoryManager.OPENCAFE_SUB_DIRS.LOG_DIR) config.set( 'OPENCAFE_ENGINE', 'temp_directory', EngineDirectoryManager.OPENCAFE_SUB_DIRS.TEMP_DIR) config.set( 'OPENCAFE_ENGINE', 'master_log_file_name', 'cafe.master') config.set( 'OPENCAFE_ENGINE', 'logging_verbosity', 'STANDARD') config.set( 'OPENCAFE_ENGINE', 'default_test_repo', 'cloudroast') return config
def run_register(): key = input('Please input node key:') key_secret = input('Please input node key_secret:') config = AgentConfig() server_base = config.get('server') if urlparse(server_base).scheme == '': if config.getint('server_https_port'): server_base = 'https://%s:%d' % ( server_base, config.getint('server_https_port')) else: server_base = 'http://%s:%d' % (server_base, config.getint('server_port')) register_url = urljoin(server_base, '/nodes/register') post_data = {'node_key': key} request = authenticated_request(url=register_url, method="POST", app_key=key, app_secret=key_secret, body=urlencode(post_data)) client = HTTPClient() try: response = client.fetch(request) response_data = json.loads(response.body) if not os.path.exists('conf'): os.makedirs('conf') with open('conf/node.conf', 'w') as f: cp = SafeConfigParser() cp.add_section('agent') cp.set('agent', 'node_id', str(response_data['id'])) cp.set('agent', 'node_key', key) cp.set('agent', 'secret_key', key_secret) cp.write(f) print('Register succeed!') except HTTPError as e: print("Error when registering.") print(e.message)
def ConfiguratePaser(configuration_item_name, userid, ip_address_or_domain, userpw=None, installpath=None, sshport=None, config_parser=None): """ Create or update the configuration file based on the input secifications. Parameters ---------- configuration_item_name: str the name of the configuration item userid: str user name on the server side ip_address_or_domain: str the ip address or the domain of the server userpw: str user password to login the server using ssh. If you want to use "id_rsa.pub", just leave userpw to None To copy the public key on the server use ssh-copy-id -i name@server. installpath: str soma workflow source path on server sshport: int (optional) the ssh port config_parser: SafeConfigParser (optional) default None, an empty config parser is created otherwise, the config parser is updatd with the new item Returns ------- config_parser: SafeConfigParser the configuration item """ # Create config generator if not config_parser: config_parser = SafeConfigParser() # Get server info hostname = ip_address_or_domain info_queue = GetQueueNamesOnPBSTORQUE(userid, ip_address_or_domain, userpw) # Add section if configuration_item_name not in config_parser.sections(): config_parser.add_section(configuration_item_name) # Fill section config_parser.set(configuration_item_name, configuration.CFG_CLUSTER_ADDRESS, hostname) config_parser.set(configuration_item_name, configuration.CFG_SUBMITTING_MACHINES, hostname) config_parser.set(configuration_item_name, configuration.OCFG_QUEUES, " ".join(info_queue)) config_parser.set(configuration_item_name, configuration.OCFG_LOGIN, userid) config_parser.set(configuration_item_name, configuration.OCFG_SSHPort, str(sshport or 22)) if installpath: config_parser.set(configuration_item_name, configuration.OCFG_INSTALLPATH, installpath) config_parser.set(configuration_item_name, configuration.OCFG_ALLOWED_PYTHON_VERSIONS, str(sys.version_info[0])) return config_parser
class Csw(object): """ Base CSW server """ def __init__(self, rtconfig=None, env=None, version='3.0.0'): """ Initialize CSW """ if not env: self.environ = os.environ else: self.environ = env self.context = config.StaticContext() # Lazy load this when needed # (it will permanently update global cfg namespaces) self.sruobj = None self.opensearchobj = None self.oaipmhobj = None # init kvp self.kvp = {} self.mode = 'csw' self.async = False self.soap = False self.request = None self.exception = False self.status = 'OK' self.profiles = None self.manager = False self.outputschemas = {} self.mimetype = 'application/xml; charset=UTF-8' self.encoding = 'UTF-8' self.pretty_print = 0 self.domainquerytype = 'list' self.orm = 'django' self.language = {'639_code': 'en', 'text': 'english'} self.process_time_start = time() # define CSW implementation object (default CSW3) self.iface = csw3.Csw3(server_csw=self) self.request_version = version if self.request_version == '2.0.2': self.iface = csw2.Csw2(server_csw=self) self.context.set_model('csw') # load user configuration try: LOGGER.info('Loading user configuration') if isinstance(rtconfig, SafeConfigParser): # serialized already self.config = rtconfig else: self.config = SafeConfigParser() if isinstance(rtconfig, dict): # dictionary for section, options in rtconfig.items(): self.config.add_section(section) for k, v in options.items(): self.config.set(section, k, v) else: # configuration file import codecs with codecs.open(rtconfig, encoding='utf-8') as scp: self.config.readfp(scp) except Exception as err: LOGGER.exception('Could not load user configuration: %s', err) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', 'Error opening configuration %s' % rtconfig ) return # set server.home safely # TODO: make this more abstract self.config.set( 'server', 'home', os.path.dirname(os.path.join(os.path.dirname(__file__), '..')) ) self.context.pycsw_home = self.config.get('server', 'home') self.context.url = self.config.get('server', 'url') log.setup_logger(self.config) LOGGER.info('running configuration %s', rtconfig) LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING']) # set OGC schemas location if not self.config.has_option('server', 'ogc_schemas_base'): self.config.set('server', 'ogc_schemas_base', self.context.ogc_schemas_base) # set mimetype if self.config.has_option('server', 'mimetype'): self.mimetype = self.config.get('server', 'mimetype').encode() # set encoding if self.config.has_option('server', 'encoding'): self.encoding = self.config.get('server', 'encoding') # set domainquerytype if self.config.has_option('server', 'domainquerytype'): self.domainquerytype = self.config.get('server', 'domainquerytype') # set XML pretty print if (self.config.has_option('server', 'pretty_print') and self.config.get('server', 'pretty_print') == 'true'): self.pretty_print = 1 # set Spatial Ranking option if (self.config.has_option('server', 'spatial_ranking') and self.config.get('server', 'spatial_ranking') == 'true'): util.ranking_enabled = True # set language default if self.config.has_option('server', 'language'): try: LOGGER.info('Setting language') lang_code = self.config.get('server', 'language').split('-')[0] self.language['639_code'] = lang_code self.language['text'] = self.context.languages[lang_code] except Exception as err: LOGGER.exception('Could not set language: %s', err) pass LOGGER.debug('Configuration: %s.', self.config) LOGGER.debug('Model: %s.', self.context.model) # load user-defined mappings if they exist if self.config.has_option('repository', 'mappings'): # override default repository mappings try: import imp module = self.config.get('repository', 'mappings') if '/' in module: # filepath modulename = '%s' % os.path.splitext(module)[0].replace( os.sep, '.') mappings = imp.load_source(modulename, module) else: # dotted name mappings = __import__(module, fromlist=['']) LOGGER.info('Loading custom repository mappings ' 'from %s', module) self.context.md_core_model = mappings.MD_CORE_MODEL self.context.refresh_dc(mappings.MD_CORE_MODEL) except Exception as err: LOGGER.exception('Could not load custom mappings: %s', err) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', 'Could not load repository.mappings') # load outputschemas LOGGER.info('Loading outputschemas') for osch in pycsw.plugins.outputschemas.__all__: output_schema_module = __import__( 'pycsw.plugins.outputschemas.%s' % osch) mod = getattr(output_schema_module.plugins.outputschemas, osch) self.outputschemas[mod.NAMESPACE] = mod LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas) LOGGER.debug('Namespaces: %s', self.context.namespaces) def expand_path(self, path): """ return safe path for WSGI environments """ if 'local.app_root' in self.environ and not os.path.isabs(path): return os.path.join(self.environ['local.app_root'], path) else: return path def dispatch_wsgi(self): """ WSGI handler """ if hasattr(self, 'response'): return self._write_response() LOGGER.debug('WSGI mode detected') if self.environ['REQUEST_METHOD'] == 'POST': try: request_body_size = int(self.environ.get('CONTENT_LENGTH', 0)) except (ValueError): request_body_size = 0 self.requesttype = 'POST' self.request = self.environ['wsgi.input'].read(request_body_size) LOGGER.debug('Request type: POST. Request:\n%s\n', self.request) else: # it's a GET request self.requesttype = 'GET' self.request = wsgiref.util.request_uri(self.environ) try: query_part = splitquery(self.request)[-1] self.kvp = dict(parse_qsl(query_part, keep_blank_values=True)) except AttributeError as err: LOGGER.exception('Could not parse query string') self.kvp = {} LOGGER.debug('Request type: GET. Request:\n%s\n', self.request) return self.dispatch() def opensearch(self): """ enable OpenSearch """ if not self.opensearchobj: self.opensearchobj = opensearch.OpenSearch(self.context) return self.opensearchobj def sru(self): """ enable SRU """ if not self.sruobj: self.sruobj = sru.Sru(self.context) return self.sruobj def oaipmh(self): """ enable OAI-PMH """ if not self.oaipmhobj: self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config) return self.oaipmhobj def dispatch(self, writer=sys.stdout, write_headers=True): """ Handle incoming HTTP request """ if self.requesttype == 'GET': self.kvp = self.normalize_kvp(self.kvp) version_202 = ('version' in self.kvp and self.kvp['version'] == '2.0.2') accept_version_202 = ('acceptversions' in self.kvp and '2.0.2' in self.kvp['acceptversions']) if version_202 or accept_version_202: self.request_version = '2.0.2' elif self.requesttype == 'POST': if self.request.find(b'2.0.2') != -1: self.request_version = '2.0.2' if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'sru'): self.mode = 'sru' self.request_version = '2.0.2' LOGGER.info('SRU mode detected; processing request') self.kvp = self.sru().request_sru2csw(self.kvp) if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'oaipmh'): self.mode = 'oaipmh' self.request_version = '2.0.2' LOGGER.info('OAI-PMH mode detected; processing request.') self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k) self.kvp = self.oaipmh().request(self.kvp) if self.request_version == '2.0.2': self.iface = csw2.Csw2(server_csw=self) self.context.set_model('csw') # configure transaction support, if specified in config self._gen_manager() namespaces = self.context.namespaces ops = self.context.model['operations'] constraints = self.context.model['constraints'] # generate domain model # NOTE: We should probably avoid this sort of mutable state for WSGI if 'GetDomain' not in ops: ops['GetDomain'] = self.context.gen_domains() # generate distributed search model, if specified in config if self.config.has_option('server', 'federatedcatalogues'): LOGGER.info('Configuring distributed search') constraints['FederatedCatalogues'] = {'values': []} for fedcat in self.config.get('server', 'federatedcatalogues').split(','): LOGGER.debug('federated catalogue: %s', fedcat) constraints['FederatedCatalogues']['values'].append(fedcat) for key, value in self.outputschemas.items(): get_records_params = ops['GetRecords']['parameters'] get_records_params['outputSchema']['values'].append( value.NAMESPACE) get_records_by_id_params = ops['GetRecordById']['parameters'] get_records_by_id_params['outputSchema']['values'].append( value.NAMESPACE) if 'Harvest' in ops: harvest_params = ops['Harvest']['parameters'] harvest_params['ResourceType']['values'].append( value.NAMESPACE) LOGGER.info('Setting MaxRecordDefault') if self.config.has_option('server', 'maxrecords'): constraints['MaxRecordDefault']['values'] = [ self.config.get('server', 'maxrecords')] # load profiles if self.config.has_option('server', 'profiles'): self.profiles = pprofile.load_profiles( os.path.join('pycsw', 'plugins', 'profiles'), pprofile.Profile, self.config.get('server', 'profiles') ) for prof in self.profiles['plugins'].keys(): tmp = self.profiles['plugins'][prof](self.context.model, namespaces, self.context) key = tmp.outputschema # to ref by outputschema self.profiles['loaded'][key] = tmp self.profiles['loaded'][key].extend_core(self.context.model, namespaces, self.config) LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys())) # init repository # look for tablename, set 'records' as default if not self.config.has_option('repository', 'table'): self.config.set('repository', 'table', 'records') repo_filter = None if self.config.has_option('repository', 'filter'): repo_filter = self.config.get('repository', 'filter') if self.config.has_option('repository', 'source'): # load custom repository rs = self.config.get('repository', 'source') rs_modname, rs_clsname = rs.rsplit('.', 1) rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname]) rs_cls = getattr(rs_mod, rs_clsname) try: self.repository = rs_cls(self.context, repo_filter) LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype) except Exception as err: msg = 'Could not load custom repository' LOGGER.exception(msg) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', msg) else: # load default repository self.orm = 'sqlalchemy' from pycsw.core import repository try: LOGGER.info('Loading default repository') self.repository = repository.Repository( self.config.get('repository', 'database'), self.context, self.environ.get('local.app_root', None), self.config.get('repository', 'table'), repo_filter ) LOGGER.debug( 'Repository loaded (local): %s.' % self.repository.dbtype) except Exception as err: msg = 'Could not load repository (local)' LOGGER.exception(msg) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', msg) if self.requesttype == 'POST': LOGGER.debug('HTTP POST request') LOGGER.debug('CSW version: %s', self.iface.version) self.kvp = self.iface.parse_postdata(self.request) error = 0 if isinstance(self.kvp, str): # it's an exception error = 1 locator = 'service' text = self.kvp if (self.kvp.find('the document is not valid') != -1 or self.kvp.find('document not well-formed') != -1): code = 'NoApplicableCode' else: code = 'InvalidParameterValue' LOGGER.debug('HTTP Headers:\n%s.', self.environ) LOGGER.debug('Parsed request parameters: %s', self.kvp) if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'opensearch'): self.mode = 'opensearch' LOGGER.info('OpenSearch mode detected; processing request.') self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom' if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or (len(self.kvp) == 1 and 'config' in self.kvp)): LOGGER.info('Turning on default csw30:Capabilities for base URL') self.kvp = { 'service': 'CSW', 'acceptversions': '3.0.0', 'request': 'GetCapabilities' } http_accept = self.environ.get('HTTP_ACCEPT', '') if 'application/opensearchdescription+xml' in http_accept: self.mode = 'opensearch' self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom' if error == 0: # test for the basic keyword values (service, version, request) basic_options = ['service', 'request'] request = self.kvp.get('request', '') own_version_integer = util.get_version_integer( self.request_version) if self.request_version == '2.0.2': basic_options.append('version') for k in basic_options: if k not in self.kvp: if (k in ['version', 'acceptversions'] and request == 'GetCapabilities'): pass else: error = 1 locator = k code = 'MissingParameterValue' text = 'Missing keyword: %s' % k break # test each of the basic keyword values if error == 0: # test service if self.kvp['service'] != 'CSW': error = 1 locator = 'service' code = 'InvalidParameterValue' text = 'Invalid value for service: %s.\ Value MUST be CSW' % self.kvp['service'] # test version kvp_version = self.kvp.get('version', '') try: kvp_version_integer = util.get_version_integer(kvp_version) except Exception as err: kvp_version_integer = 'invalid_value' if (request != 'GetCapabilities' and kvp_version_integer != own_version_integer): error = 1 locator = 'version' code = 'InvalidParameterValue' text = ('Invalid value for version: %s. Value MUST be ' '2.0.2 or 3.0.0' % kvp_version) # check for GetCapabilities acceptversions if 'acceptversions' in self.kvp: for vers in self.kvp['acceptversions'].split(','): vers_integer = util.get_version_integer(vers) if vers_integer == own_version_integer: break else: error = 1 locator = 'acceptversions' code = 'VersionNegotiationFailed' text = ('Invalid parameter value in ' 'acceptversions: %s. Value MUST be ' '2.0.2 or 3.0.0' % self.kvp['acceptversions']) # test request if self.kvp['request'] not in \ self.context.model['operations']: error = 1 locator = 'request' if request in ['Transaction', 'Harvest']: code = 'OperationNotSupported' text = '%s operations are not supported' % request else: code = 'InvalidParameterValue' text = 'Invalid value for request: %s' % request if error == 1: # return an ExceptionReport LOGGER.error('basic service options error: %s, %s, %s', code, locator, text) self.response = self.iface.exceptionreport(code, locator, text) else: # process per the request value if 'responsehandler' in self.kvp: # set flag to process asynchronously import threading self.async = True request_id = self.kvp.get('requestid', None) if request_id is None: import uuid self.kvp['requestid'] = str(uuid.uuid4()) if self.kvp['request'] == 'GetCapabilities': self.response = self.iface.getcapabilities() elif self.kvp['request'] == 'DescribeRecord': self.response = self.iface.describerecord() elif self.kvp['request'] == 'GetDomain': self.response = self.iface.getdomain() elif self.kvp['request'] == 'GetRecords': if self.async: # process asynchronously threading.Thread(target=self.iface.getrecords).start() self.response = self.iface._write_acknowledgement() else: self.response = self.iface.getrecords() elif self.kvp['request'] == 'GetRecordById': self.response = self.iface.getrecordbyid() elif self.kvp['request'] == 'GetRepositoryItem': self.response = self.iface.getrepositoryitem() elif self.kvp['request'] == 'Transaction': self.response = self.iface.transaction() elif self.kvp['request'] == 'Harvest': if self.async: # process asynchronously threading.Thread(target=self.iface.harvest).start() self.response = self.iface._write_acknowledgement() else: self.response = self.iface.harvest() else: self.response = self.iface.exceptionreport( 'InvalidParameterValue', 'request', 'Invalid request parameter: %s' % self.kvp['request'] ) LOGGER.info('Request processed') if self.mode == 'sru': LOGGER.info('SRU mode detected; processing response.') self.response = self.sru().response_csw2sru(self.response, self.environ) elif self.mode == 'opensearch': LOGGER.info('OpenSearch mode detected; processing response.') self.response = self.opensearch().response_csw2opensearch( self.response, self.config) elif self.mode == 'oaipmh': LOGGER.info('OAI-PMH mode detected; processing response.') self.response = self.oaipmh().response( self.response, self.oaiargs, self.repository, self.config.get('server', 'url') ) return self._write_response() def getcapabilities(self): """ Handle GetCapabilities request """ return self.iface.getcapabilities() def describerecord(self): """ Handle DescribeRecord request """ return self.iface.describerecord() def getdomain(self): """ Handle GetDomain request """ return self.iface.getdomain() def getrecords(self): """ Handle GetRecords request """ return self.iface.getrecords() def getrecordbyid(self, raw=False): """ Handle GetRecordById request """ return self.iface.getrecordbyid(raw) def getrepositoryitem(self): """ Handle GetRepositoryItem request """ return self.iface.getrepositoryitem() def transaction(self): """ Handle Transaction request """ return self.iface.transaction() def harvest(self): """ Handle Harvest request """ return self.iface.harvest() def _write_response(self): """ Generate response """ # set HTTP response headers and XML declaration xmldecl = '' appinfo = '' LOGGER.info('Writing response.') if hasattr(self, 'soap') and self.soap: self._gen_soap_wrapper() if etree.__version__ >= '3.5.0': # remove superfluous namespaces etree.cleanup_namespaces(self.response, keep_ns_prefixes=self.context.keep_ns_prefixes) response = etree.tostring(self.response, pretty_print=self.pretty_print, encoding='unicode') if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and self.kvp['outputformat'] == 'application/json'): self.contenttype = self.kvp['outputformat'] from pycsw.core.formats import fmt_json response = fmt_json.xml2json(response, self.context.namespaces, self.pretty_print) else: # it's XML if 'outputformat' in self.kvp: self.contenttype = self.kvp['outputformat'] else: self.contenttype = self.mimetype xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>' '\n' % self.encoding) appinfo = '<!-- pycsw %s -->\n' % self.context.version if isinstance(self.contenttype, bytes): self.contenttype = self.contenttype.decode() s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding) LOGGER.debug('Response code: %s', self.context.response_codes[self.status]) LOGGER.debug('Response:\n%s', s) return [self.context.response_codes[self.status], s] def _gen_soap_wrapper(self): """ Generate SOAP wrapper """ LOGGER.info('Writing SOAP wrapper.') node = etree.Element( util.nspath_eval('soapenv:Envelope', self.context.namespaces), nsmap=self.context.namespaces ) schema_location_ns = util.nspath_eval('xsi:schemaLocation', self.context.namespaces) node.attrib[schema_location_ns] = '%s %s' % ( self.context.namespaces['soapenv'], self.context.namespaces['soapenv'] ) node2 = etree.SubElement( node, util.nspath_eval('soapenv:Body', self.context.namespaces)) if self.exception: node3 = etree.SubElement( node2, util.nspath_eval('soapenv:Fault', self.context.namespaces) ) node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Code', self.context.namespaces) ) etree.SubElement( node4, util.nspath_eval('soapenv:Value', self.context.namespaces) ).text = 'soap:Server' node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Reason', self.context.namespaces) ) etree.SubElement( node4, util.nspath_eval('soapenv:Text', self.context.namespaces) ).text = 'A server exception was encountered.' node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Detail', self.context.namespaces) ) node4.append(self.response) else: node2.append(self.response) self.response = node def _gen_manager(self): """ Update self.context.model with CSW-T advertising """ if (self.config.has_option('manager', 'transactions') and self.config.get('manager', 'transactions') == 'true'): self.manager = True self.context.model['operations_order'].append('Transaction') self.context.model['operations']['Transaction'] = { 'methods': {'get': False, 'post': True}, 'parameters': {} } schema_values = [ 'http://www.opengis.net/cat/csw/2.0.2', 'http://www.opengis.net/cat/csw/3.0', 'http://www.opengis.net/wms', 'http://www.opengis.net/wmts/1.0', 'http://www.opengis.net/wfs', 'http://www.opengis.net/wfs/2.0', 'http://www.opengis.net/wcs', 'http://www.opengis.net/wps/1.0.0', 'http://www.opengis.net/sos/1.0', 'http://www.opengis.net/sos/2.0', 'http://www.isotc211.org/2005/gmi', 'urn:geoss:waf', ] self.context.model['operations_order'].append('Harvest') self.context.model['operations']['Harvest'] = { 'methods': {'get': False, 'post': True}, 'parameters': { 'ResourceType': {'values': schema_values} } } self.context.model['operations']['Transaction'] = { 'methods': {'get': False, 'post': True}, 'parameters': { 'TransactionSchemas': {'values': sorted(schema_values)} } } self.csw_harvest_pagesize = 10 if self.config.has_option('manager', 'csw_harvest_pagesize'): self.csw_harvest_pagesize = int( self.config.get('manager', 'csw_harvest_pagesize')) def _test_manager(self): """ Verify that transactions are allowed """ if self.config.get('manager', 'transactions') != 'true': raise RuntimeError('CSW-T interface is disabled') ipaddress = self.environ['REMOTE_ADDR'] if not self.config.has_option('manager', 'allowed_ips') or \ (self.config.has_option('manager', 'allowed_ips') and not util.ipaddress_in_whitelist(ipaddress, self.config.get('manager', 'allowed_ips').split(','))): raise RuntimeError( 'CSW-T operations not allowed for this IP address: %s' % ipaddress) def _cql_update_queryables_mappings(self, cql, mappings): """ Transform CQL query's properties to underlying DB columns """ LOGGER.debug('Raw CQL text = %s', cql) LOGGER.debug(str(list(mappings.keys()))) if cql is not None: for key in mappings.keys(): try: cql = cql.replace(key, mappings[key]['dbcol']) except: cql = cql.replace(key, mappings[key]) LOGGER.debug('Interpolated CQL text = %s.', cql) return cql def _process_responsehandler(self, xml): """ Process response handler """ if self.kvp['responsehandler'] is not None: LOGGER.info('Processing responsehandler %s' % self.kvp['responsehandler']) uprh = urlparse(self.kvp['responsehandler']) if uprh.scheme == 'mailto': # email import smtplib LOGGER.debug('Email detected') smtp_host = 'localhost' if self.config.has_option('server', 'smtp_host'): smtp_host = self.config.get('server', 'smtp_host') body = ('Subject: pycsw %s results\n\n%s' % (self.kvp['request'], xml)) try: LOGGER.info('Sending email') msg = smtplib.SMTP(smtp_host) msg.sendmail( self.config.get('metadata:main', 'contact_email'), uprh.path, body ) msg.quit() LOGGER.debug('Email sent successfully.') except Exception as err: LOGGER.exception('Error processing email') elif uprh.scheme == 'ftp': import ftplib LOGGER.debug('FTP detected.') try: LOGGER.info('Sending to FTP server.') ftp = ftplib.FTP(uprh.hostname) if uprh.username is not None: ftp.login(uprh.username, uprh.password) ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml)) ftp.quit() LOGGER.debug('FTP sent successfully.') except Exception as err: LOGGER.exception('Error processing FTP') @staticmethod def normalize_kvp(kvp): """Normalize Key Value Pairs. This method will transform all keys to lowercase and leave values unchanged, as specified in the CSW standard (see for example note C on Table 62 - KVP Encoding for DescribeRecord operation request of the CSW standard version 2.0.2) :arg kvp: a mapping with Key Value Pairs :type kvp: dict :returns: A new dictionary with normalized parameters """ result = dict() for name, value in kvp.items(): result[name.lower()] = value return result
class Config(MutableMapping): """Read and write to config file. A config object is essentially a string key-value store that can be treated like a dictionary:: c = Config() c['foo'] = 'bar' print c['foo'] The file location may be specified:: c = Config('~/matt/anotherconfig.cfg') c['where'] = 'in a different file' If no location is specified, the environment variable CHEMDATAEXTRACTOR_CONFIG is checked and used if available. Otherwise, a standard config location is used, which varies depending on the operating system. You can check the location using the ``path`` property. For more information see https://github.com/ActiveState/appdirs It is possible to edit the file by hand with a text editor. Warning: multiple instances of Config() pointing to the same file will not see each others' changes, and will overwrite the entire file when any key is changed. """ #: These values will be present in a config object unless they are explicitly defined otherwise in the config file default_values = {} def __init__(self, path=None): """ :param string path: (Optional) Path to config file location. """ self._path = path # Use CHEMDATAEXTRACTOR_CONFIG environment variable if set if not self._path: self._path = os.environ.get('CHEMDATAEXTRACTOR_CONFIG') # Use OS-dependent config directory given by appdirs if not self._path: self._path = os.path.join( appdirs.user_config_dir('ChemDataExtractor'), 'config.cfg') self._parser = SafeConfigParser() if os.path.isfile(self.path): with open(self.path) as f: self._parser.readfp(f) if not self._parser.has_section('cde'): self._parser.add_section('cde') for k, v in six.iteritems(self.default_values): if not self._parser.has_option('cde', k.encode('utf-8')): self._parser.set('cde', k.encode('utf-8'), v.encode('utf-8')) @property def path(self): """The path to the config file.""" return self._path def _flush(self): """Save the contents of data to the file on disk. You should not need to call this manually.""" d = os.path.dirname(self.path) if not os.path.isdir(d): os.makedirs(d) with open(self.path, 'w') as f: self._parser.write(f) def __contains__(self, k): return self._parser.has_option('cde', k.encode('utf-8')) def __getitem__(self, k): try: return self._parser.get('cde', k.encode('utf-8')) except NoOptionError: raise KeyError(k) def __setitem__(self, k, v): self._parser.set('cde', k.encode('utf-8'), v.encode('utf-8')) self._flush() def __delitem__(self, k): try: self._parser.remove_option('cde', k.encode('utf-8')) self._flush() except NoOptionError: raise KeyError(k) def __iter__(self): return (k for k, v in self._parser.items('cde')) def __len__(self): return len(self._parser.items('cde')) def __repr__(self): return '<Config: %s>' % self.path def clear(self): """Clear all values from config.""" self._parser.remove_section('cde') self._parser.add_section('cde') self._flush()
class PackageConfigHandler(object): """ Manager class for packages files for tracking installation of modules """ def __init__(self): # noinspection PyUnresolvedReferences from six.moves.configparser import SafeConfigParser self.package_cfg = os.path.expanduser('~/Documents/site-packages/.pypi_packages') if not os.path.isfile(self.package_cfg): print('Creating package file') with open(self.package_cfg, 'w') as outs: outs.close() self.parser = SafeConfigParser() self.parser.read(self.package_cfg) def save(self): with open(self.package_cfg, 'w') as outs: self.parser.write(outs) def add_module(self, pkg_info): """ :param pkg_info: A dict that has name, url, version, summary :return: """ if not self.parser.has_section(pkg_info['name']): self.parser.add_section(pkg_info['name']) self.parser.set(pkg_info['name'], 'url', pkg_info['url']) self.parser.set(pkg_info['name'], 'version', pkg_info['version']) self.parser.set(pkg_info['name'], 'summary', pkg_info['summary']) self.parser.set(pkg_info['name'], 'files', pkg_info['files']) self.parser.set(pkg_info['name'], 'dependency', pkg_info['dependency']) self.save() def list_modules(self): return [module for module in self.parser.sections()] def module_exists(self, name): return self.parser.has_section(name) def get_info(self, name): if self.parser.has_section(name): tbl = {} for opt, value in self.parser.items(name): tbl[opt] = value return tbl def remove_module(self, name): self.parser.remove_section(name) self.save() def get_files_installed(self, section_name): if self.parser.has_option(section_name, 'files'): files = self.parser.get(section_name, 'files').strip() return files.split(',') else: return None def get_dependencies(self, section_name): if self.parser.has_option(section_name, 'dependency'): dependencies = self.parser.get(section_name, 'dependency').strip() return set(dependencies.split(',')) if dependencies != '' else set() else: return None def get_all_dependencies(self, exclude_module=()): all_dependencies = set() for section_name in self.parser.sections(): if section_name not in exclude_module and self.parser.has_option(section_name, 'dependency'): dependencies = self.parser.get(section_name, 'dependency').strip() if dependencies != '': for dep in dependencies.split(','): all_dependencies.add(dep) return all_dependencies
def createThemeFromTemplate(title, description, baseOn='template'): """Create a new theme from the given title and description based on another theme resource directory """ source = queryResourceDirectory(THEME_RESOURCE_NAME, baseOn) if source is None: raise KeyError("Theme {0:s} not found".format(baseOn)) themeName = getUtility(IURLNormalizer).normalize(title) if isinstance(themeName, six.text_type): themeName = themeName.encode('utf-8') resources = getOrCreatePersistentResourceDirectory() resources.makeDirectory(themeName) target = resources[themeName] cloneResourceDirectory(source, target) manifest = SafeConfigParser() if MANIFEST_FILENAME in target: fp = target.openFile(MANIFEST_FILENAME) try: manifest.readfp(fp) finally: fp.close() if not manifest.has_section('theme'): manifest.add_section('theme') if not isinstance(title, str): title = title.encode('utf-8') if not isinstance(description, str): description = description.encode('utf-8') manifest.set('theme', 'title', title) manifest.set('theme', 'description', description) if manifest.has_option('theme', 'prefix'): prefix = u"/++%s++%s" % (THEME_RESOURCE_NAME, themeName) manifest.set('theme', 'prefix', prefix) if manifest.has_option('theme', 'rules'): rule = manifest.get('theme', 'rules') rule_file_name = rule.split('/')[-1] # extract real rules file name rules = u"/++%s++%s/%s" % (THEME_RESOURCE_NAME, themeName, rule_file_name) manifest.set('theme', 'rules', rules) paths_to_fix = [ 'development-css', 'production-css', 'tinymce-content-css', 'development-js', 'production-js' ] for var_path in paths_to_fix: if not manifest.has_option('theme', var_path): continue val = manifest.get('theme', var_path) if not val: continue template_prefix = '++%s++%s/' % (THEME_RESOURCE_NAME, baseOn) if template_prefix in val: # okay, fix val = val.replace(template_prefix, '++%s++%s/' % (THEME_RESOURCE_NAME, themeName)) manifest.set('theme', var_path, val) manifestContents = StringIO() manifest.write(manifestContents) target.writeFile(MANIFEST_FILENAME, manifestContents) return themeName
class Config(object): """ Manages the configuration file """ def __init__(self): """ DEFAULT VALUES """ self._basescript = None self.recentvaults = [] self.pwlength = 10 self.search_notes = False self.search_passwd = False self.use_pwgen = False # self.alphabet = "abcdefghijklmnopqrstuvwxyz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ_" self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_" self.avoid_bigrams = "cl mn nm nn rn vv VV" self._fname = self.get_config_filename() print("Using config %s" % self._fname) self._parser = SafeConfigParser() if os.path.exists(self._fname): self._parser.read(self._fname) if not self._parser.has_section("base"): self._parser.add_section("base") for num in range(10): if (not self._parser.has_option("base", "recentvaults" + str(num))): break self.recentvaults.append( self._parser.get("base", "recentvaults" + str(num))) if self._parser.has_option("base", "pwlength"): self.pwlength = int(self._parser.get("base", "pwlength")) if self._parser.has_option("base", "search_notes"): if self._parser.get("base", "search_notes") == "True": self.search_notes = True if self._parser.has_option("base", "search_passwd"): if self._parser.get("base", "search_passwd") == "True": self.search_passwd = True if self._parser.has_option("base", "use_pwgen"): if self._parser.get("base", "use_pwgen") == "True": self.use_pwgen = True if self._parser.has_option("base", "alphabet"): self.alphabet = self._parser.get("base", "alphabet") if self._parser.has_option("base", "avoid_bigrams"): self.avoid_bigrams = self._parser.get("base", "avoid_bigrams") if not os.path.exists(self._fname): self.save() def set_basescript(self, basescript): self._basescript = basescript def get_basescript(self): return self._basescript def save(self): if (not os.path.exists(os.path.dirname(self._fname))): os.mkdir(os.path.dirname(self._fname)) # remove duplicates and trim to 10 items _saved_recentvaults = [] for item in self.recentvaults: if item in _saved_recentvaults: continue self._parser.set("base", "recentvaults" + str(len(_saved_recentvaults)), item) _saved_recentvaults.append(item) if (len(_saved_recentvaults) >= 10): break self._parser.set("base", "pwlength", str(self.pwlength)) self._parser.set("base", "search_notes", str(self.search_notes)) self._parser.set("base", "search_passwd", str(self.search_passwd)) self._parser.set("base", "use_pwgen", str(self.use_pwgen)) self._parser.set("base", "alphabet", str(self.alphabet)) self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams)) filehandle = open(self._fname, 'w') self._parser.write(filehandle) filehandle.close() @staticmethod def get_config_filename(): """ Returns the full filename of the config file """ base_fname = "loxodo" # On Mac OS X, config files go to ~/Library/Application Support/foo/ if platform.system() == "Darwin": base_path = os.path.join(os.path.expanduser("~"), "Library", "Application Support") if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # On Microsoft Windows, config files go to $APPDATA/foo/ if platform.system() in ("Windows", "Microsoft"): if ("APPDATA" in os.environ): base_path = os.environ["APPDATA"] if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # Allow config directory override as per freedesktop.org XDG Base Directory Specification if ("XDG_CONFIG_HOME" in os.environ): base_path = os.environ["XDG_CONFIG_HOME"] if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # Default configuration path is ~/.config/foo/ base_path = os.path.join(os.path.expanduser("~"), ".config") if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") else: return os.path.join(os.path.expanduser("~"), "." + base_fname + ".ini")
def main(): config = SafeConfigParser() config.add_section('main') for k, v in DEFAULTS.items(): config.set('main', k, v) config.read([ '.bisque', os.path.expanduser('~/.bisque'), '/etc/bisque/bisque_config' ]) defaults = dict(config.items('main')) defaults.update(config.items('bqpath')) parser = argparse.ArgumentParser( description=DESCRIPTION, formatter_class=argparse.ArgumentDefaultsHelpFormatter, ) #parser.add_argument('--help', action=_HelpAction, help='help for help if you need some help') # add custom help parser.add_argument('--alias', help="do action on behalf of user specified") parser.add_argument('-d', '--debug', action="store_true", default=False, help="log debugging") parser.add_argument('-H', '--host', default=defaults['bisque_host'], help="bisque host") parser.add_argument('-c', '--credentials', default="%s:%s" % (defaults['bisque_user'], defaults["bisque_pass"]), help="user credentials") parser.add_argument('-C', '--compatible', action="store_true", help="Make compatible with old script") parser.add_argument('-V', '--verbose', action='store_true', help='print stuff') #parser.add_argument('-P', '--permission', default="private", help="Set resource permission (compatibility)") #parser.add_argument('--hidden', default=None, help="Set resource visibility (hidden)") #parser.add_argument('command', help="one of ls, cp, mv, rm, ln" ) #parser.add_argument('paths', nargs='+') sp = parser.add_subparsers() lsp = sp.add_parser('ls') lsp.add_argument('-u', '--unique', default=None, action="store_true", help="return unique codes") lsp.add_argument('paths', nargs='+') lsp.set_defaults(func=bisque_list) lnp = sp.add_parser('ln') lnp.add_argument('-T', '--tag_file', default=None, help="tag document for insert") lnp.add_argument('-P', '--permission', default="private", help="Set resource permission (compatibility)") lnp.add_argument('-R', '--resource', default=None, help='force resource type') lnp.add_argument('--hidden', default=None, help="Set resource visibility (hidden)") lnp.add_argument('paths', nargs='+') lnp.set_defaults(func=bisque_link) cpp = sp.add_parser('cp') cpp.add_argument('paths', nargs='+') cpp.add_argument('-T', '--tag_file', default=None, help="tag document for insert") cpp.add_argument('-R', '--resource', default=None, help='force resource type') cpp.add_argument('-P', '--permission', default="private", help="Set resource permission (compatibility)") cpp.add_argument('--hidden', default=None, help="Set resource visibility (hidden)") cpp.set_defaults(func=bisque_copy) mvp = sp.add_parser('mv') mvp.add_argument('paths', nargs='+') mvp.set_defaults(func=bisque_rename) rmp = sp.add_parser('rm') rmp.add_argument('paths', nargs='+') rmp.set_defaults(func=bisque_delete) logging.basicConfig( filename=config.get('bqpath', 'logfile'), level=logging.INFO, format="%(asctime)s %(levelname)-5.5s [%(name)s] %(message)s") log = logging.getLogger('rods2bq') args = parser.parse_args() if args.debug: logging.getLogger().setLevel(logging.DEBUG) #if args.command not in OPERATIONS: # parser.error("command %s must be one of 'ln', 'ls', 'cp', 'mv', 'rm'" % args.command) if len(args.paths) > 1: args.dstpath = args.paths.pop() args.srcpath = args.paths else: args.srcpath = args.paths if args.compatible: paths = [] irods_host = defaults.get('irods_host') for el in args.srcpath: if not el.startswith('irods://'): paths.append(irods_host + el) else: paths.append(el) args.srcpath = paths if args.dstpath and not args.dstpath.startswith('irods://'): args.dstpath = irods_host + args.dstpath if args.debug: six.print_(args, file=sys.stderr) try: session = requests.Session() requests.packages.urllib3.disable_warnings() session.log = logging.getLogger('rods2bq') #session.verify = False session.auth = tuple(args.credentials.split(':')) #session.headers.update ( {'content-type': 'application/xml'} ) #OPERATIONS[args.command] (session, args) args.func(session, args) except requests.exceptions.HTTPError as e: log.exception("exception occurred %s : %s", e, e.response.text) six.print_("ERROR:", e.response and e.response.status_code) sys.exit(0)
class Csw(object): """ Base CSW server """ def __init__(self, rtconfig=None, env=None, version='3.0.0'): """ Initialize CSW """ if not env: self.environ = os.environ else: self.environ = env self.context = config.StaticContext() # Lazy load this when needed # (it will permanently update global cfg namespaces) self.sruobj = None self.opensearchobj = None self.oaipmhobj = None # init kvp self.kvp = {} self.mode = 'csw' self.asynchronous = False self.soap = False self.request = None self.exception = False self.status = 'OK' self.profiles = None self.manager = False self.outputschemas = {} self.mimetype = 'application/xml; charset=UTF-8' self.encoding = 'UTF-8' self.pretty_print = 0 self.domainquerytype = 'list' self.orm = 'django' self.language = {'639_code': 'en', 'text': 'english'} self.process_time_start = time() # define CSW implementation object (default CSW3) self.iface = csw3.Csw3(server_csw=self) self.request_version = version if self.request_version == '2.0.2': self.iface = csw2.Csw2(server_csw=self) self.context.set_model('csw') # load user configuration try: LOGGER.info('Loading user configuration') if isinstance(rtconfig, SafeConfigParser): # serialized already self.config = rtconfig else: self.config = SafeConfigParser() if isinstance(rtconfig, dict): # dictionary for section, options in rtconfig.items(): self.config.add_section(section) for k, v in options.items(): self.config.set(section, k, v) else: # configuration file import codecs with codecs.open(rtconfig, encoding='utf-8') as scp: self.config.readfp(scp) except Exception as err: msg = 'Could not load configuration' LOGGER.exception('%s %s: %s', msg, rtconfig, err) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', msg) return # set server.home safely # TODO: make this more abstract self.config.set( 'server', 'home', os.path.dirname(os.path.join(os.path.dirname(__file__), '..')) ) self.context.pycsw_home = self.config.get('server', 'home') self.context.url = self.config.get('server', 'url') log.setup_logger(self.config) LOGGER.info('running configuration %s', rtconfig) LOGGER.debug('QUERY_STRING: %s', self.environ['QUERY_STRING']) # set OGC schemas location if not self.config.has_option('server', 'ogc_schemas_base'): self.config.set('server', 'ogc_schemas_base', self.context.ogc_schemas_base) # set mimetype if self.config.has_option('server', 'mimetype'): self.mimetype = self.config.get('server', 'mimetype').encode() # set encoding if self.config.has_option('server', 'encoding'): self.encoding = self.config.get('server', 'encoding') # set domainquerytype if self.config.has_option('server', 'domainquerytype'): self.domainquerytype = self.config.get('server', 'domainquerytype') # set XML pretty print if (self.config.has_option('server', 'pretty_print') and self.config.get('server', 'pretty_print') == 'true'): self.pretty_print = 1 # set Spatial Ranking option if (self.config.has_option('server', 'spatial_ranking') and self.config.get('server', 'spatial_ranking') == 'true'): util.ranking_enabled = True # set language default if self.config.has_option('server', 'language'): try: LOGGER.info('Setting language') lang_code = self.config.get('server', 'language').split('-')[0] self.language['639_code'] = lang_code self.language['text'] = self.context.languages[lang_code] except Exception as err: LOGGER.exception('Could not set language: %s', err) pass LOGGER.debug('Configuration: %s.', self.config) LOGGER.debug('Model: %s.', self.context.model) # load user-defined mappings if they exist if self.config.has_option('repository', 'mappings'): # override default repository mappings try: import imp module = self.config.get('repository', 'mappings') if os.sep in module: # filepath modulename = '%s' % os.path.splitext(module)[0].replace( os.sep, '.') mappings = imp.load_source(modulename, module) else: # dotted name mappings = __import__(module, fromlist=['']) LOGGER.info('Loading custom repository mappings ' 'from %s', module) self.context.md_core_model = mappings.MD_CORE_MODEL self.context.refresh_dc(mappings.MD_CORE_MODEL) except Exception as err: LOGGER.exception('Could not load custom mappings: %s', err) self.response = self.iface.exceptionreport( 'NoApplicableCode', 'service', 'Could not load repository.mappings') # load outputschemas LOGGER.info('Loading outputschemas') for osch in pycsw.plugins.outputschemas.__all__: output_schema_module = __import__( 'pycsw.plugins.outputschemas.%s' % osch) mod = getattr(output_schema_module.plugins.outputschemas, osch) self.outputschemas[mod.NAMESPACE] = mod LOGGER.debug('Outputschemas loaded: %s.', self.outputschemas) LOGGER.debug('Namespaces: %s', self.context.namespaces) def expand_path(self, path): """ return safe path for WSGI environments """ if 'local.app_root' in self.environ and not os.path.isabs(path): return os.path.join(self.environ['local.app_root'], path) else: return path def dispatch_wsgi(self): """ WSGI handler """ if hasattr(self, 'response'): return self._write_response() LOGGER.debug('WSGI mode detected') if self.environ['REQUEST_METHOD'] == 'POST': try: request_body_size = int(self.environ.get('CONTENT_LENGTH', 0)) except (ValueError): request_body_size = 0 self.requesttype = 'POST' self.request = self.environ['wsgi.input'].read(request_body_size) LOGGER.debug('Request type: POST. Request:\n%s\n', self.request) else: # it's a GET request self.requesttype = 'GET' self.request = wsgiref.util.request_uri(self.environ) try: query_part = splitquery(self.request)[-1] self.kvp = dict(parse_qsl(query_part, keep_blank_values=True)) except AttributeError as err: LOGGER.exception('Could not parse query string') self.kvp = {} LOGGER.debug('Request type: GET. Request:\n%s\n', self.request) return self.dispatch() def opensearch(self): """ enable OpenSearch """ if not self.opensearchobj: self.opensearchobj = opensearch.OpenSearch(self.context) return self.opensearchobj def sru(self): """ enable SRU """ if not self.sruobj: self.sruobj = sru.Sru(self.context) return self.sruobj def oaipmh(self): """ enable OAI-PMH """ if not self.oaipmhobj: self.oaipmhobj = oaipmh.OAIPMH(self.context, self.config) return self.oaipmhobj def dispatch(self, writer=sys.stdout, write_headers=True): """ Handle incoming HTTP request """ error = 0 if self.requesttype == 'GET': self.kvp = self.normalize_kvp(self.kvp) version_202 = ('version' in self.kvp and self.kvp['version'] == '2.0.2') accept_version_202 = ('acceptversions' in self.kvp and '2.0.2' in self.kvp['acceptversions']) if version_202 or accept_version_202: self.request_version = '2.0.2' elif self.requesttype == 'POST': if self.request.find(b'cat/csw/2.0.2') != -1: self.request_version = '2.0.2' elif self.request.find(b'cat/csw/3.0') != -1: self.request_version = '3.0.0' if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'sru'): self.mode = 'sru' self.request_version = '2.0.2' LOGGER.info('SRU mode detected; processing request') self.kvp = self.sru().request_sru2csw(self.kvp) if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'oaipmh'): self.mode = 'oaipmh' self.request_version = '2.0.2' LOGGER.info('OAI-PMH mode detected; processing request.') self.oaiargs = dict((k, v) for k, v in self.kvp.items() if k) self.kvp = self.oaipmh().request(self.kvp) if self.request_version == '2.0.2': self.iface = csw2.Csw2(server_csw=self) self.context.set_model('csw') # configure transaction support, if specified in config self._gen_manager() namespaces = self.context.namespaces ops = self.context.model['operations'] constraints = self.context.model['constraints'] # generate domain model # NOTE: We should probably avoid this sort of mutable state for WSGI if 'GetDomain' not in ops: ops['GetDomain'] = self.context.gen_domains() # generate distributed search model, if specified in config if self.config.has_option('server', 'federatedcatalogues'): LOGGER.info('Configuring distributed search') constraints['FederatedCatalogues'] = {'values': []} for fedcat in self.config.get('server', 'federatedcatalogues').split(','): LOGGER.debug('federated catalogue: %s', fedcat) constraints['FederatedCatalogues']['values'].append(fedcat) for key, value in self.outputschemas.items(): get_records_params = ops['GetRecords']['parameters'] get_records_params['outputSchema']['values'].append( value.NAMESPACE) get_records_by_id_params = ops['GetRecordById']['parameters'] get_records_by_id_params['outputSchema']['values'].append( value.NAMESPACE) if 'Harvest' in ops: harvest_params = ops['Harvest']['parameters'] harvest_params['ResourceType']['values'].append( value.NAMESPACE) LOGGER.info('Setting MaxRecordDefault') if self.config.has_option('server', 'maxrecords'): constraints['MaxRecordDefault']['values'] = [ self.config.get('server', 'maxrecords')] # load profiles if self.config.has_option('server', 'profiles'): self.profiles = pprofile.load_profiles( os.path.join('pycsw', 'plugins', 'profiles'), pprofile.Profile, self.config.get('server', 'profiles') ) for prof in self.profiles['plugins'].keys(): tmp = self.profiles['plugins'][prof](self.context.model, namespaces, self.context) key = tmp.outputschema # to ref by outputschema self.profiles['loaded'][key] = tmp self.profiles['loaded'][key].extend_core(self.context.model, namespaces, self.config) LOGGER.debug('Profiles loaded: %s' % list(self.profiles['loaded'].keys())) # init repository # look for tablename, set 'records' as default if not self.config.has_option('repository', 'table'): self.config.set('repository', 'table', 'records') repo_filter = None if self.config.has_option('repository', 'filter'): repo_filter = self.config.get('repository', 'filter') if self.config.has_option('repository', 'source'): # load custom repository rs = self.config.get('repository', 'source') rs_modname, rs_clsname = rs.rsplit('.', 1) rs_mod = __import__(rs_modname, globals(), locals(), [rs_clsname]) rs_cls = getattr(rs_mod, rs_clsname) try: self.repository = rs_cls(self.context, repo_filter) LOGGER.debug('Custom repository %s loaded (%s)', rs, self.repository.dbtype) except Exception as err: msg = 'Could not load custom repository %s: %s' % (rs, err) LOGGER.exception(msg) error = 1 code = 'NoApplicableCode' locator = 'service' text = 'Could not initialize repository. Check server logs' else: # load default repository self.orm = 'sqlalchemy' from pycsw.core import repository try: LOGGER.info('Loading default repository') self.repository = repository.Repository( self.config.get('repository', 'database'), self.context, self.environ.get('local.app_root', None), self.config.get('repository', 'table'), repo_filter ) LOGGER.debug( 'Repository loaded (local): %s.' % self.repository.dbtype) except Exception as err: msg = 'Could not load repository (local): %s' % err LOGGER.exception(msg) error = 1 code = 'NoApplicableCode' locator = 'service' text = 'Could not initialize repository. Check server logs' if self.requesttype == 'POST': LOGGER.debug('HTTP POST request') LOGGER.debug('CSW version: %s', self.iface.version) self.kvp = self.iface.parse_postdata(self.request) if isinstance(self.kvp, str): # it's an exception error = 1 locator = 'service' text = self.kvp if (self.kvp.find('the document is not valid') != -1 or self.kvp.find('document not well-formed') != -1): code = 'NoApplicableCode' else: code = 'InvalidParameterValue' LOGGER.debug('HTTP Headers:\n%s.', self.environ) LOGGER.debug('Parsed request parameters: %s', self.kvp) if (not isinstance(self.kvp, str) and 'mode' in self.kvp and self.kvp['mode'] == 'opensearch'): self.mode = 'opensearch' LOGGER.info('OpenSearch mode detected; processing request.') self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom' if ((len(self.kvp) == 0 and self.request_version == '3.0.0') or (len(self.kvp) == 1 and 'config' in self.kvp)): LOGGER.info('Turning on default csw30:Capabilities for base URL') self.kvp = { 'service': 'CSW', 'acceptversions': '3.0.0', 'request': 'GetCapabilities' } http_accept = self.environ.get('HTTP_ACCEPT', '') if 'application/opensearchdescription+xml' in http_accept: self.mode = 'opensearch' self.kvp['outputschema'] = 'http://www.w3.org/2005/Atom' if error == 0: # test for the basic keyword values (service, version, request) basic_options = ['service', 'request'] request = self.kvp.get('request', '') own_version_integer = util.get_version_integer( self.request_version) if self.request_version == '2.0.2': basic_options.append('version') if self.request_version == '3.0.0' and 'version' not in self.kvp and self.requesttype == 'POST': if 'service' not in self.kvp: self.kvp['service'] = 'CSW' basic_options.append('service') self.kvp['version'] = self.request_version basic_options.append('version') for k in basic_options: if k not in self.kvp: if (k in ['version', 'acceptversions'] and request == 'GetCapabilities'): pass else: error = 1 locator = k code = 'MissingParameterValue' text = 'Missing keyword: %s' % k break # test each of the basic keyword values if error == 0: # test service if self.kvp['service'] != 'CSW': error = 1 locator = 'service' code = 'InvalidParameterValue' text = 'Invalid value for service: %s.\ Value MUST be CSW' % self.kvp['service'] # test version kvp_version = self.kvp.get('version', '') try: kvp_version_integer = util.get_version_integer(kvp_version) except Exception as err: kvp_version_integer = 'invalid_value' if (request != 'GetCapabilities' and kvp_version_integer != own_version_integer): error = 1 locator = 'version' code = 'InvalidParameterValue' text = ('Invalid value for version: %s. Value MUST be ' '2.0.2 or 3.0.0' % kvp_version) # check for GetCapabilities acceptversions if 'acceptversions' in self.kvp: for vers in self.kvp['acceptversions'].split(','): vers_integer = util.get_version_integer(vers) if vers_integer == own_version_integer: break else: error = 1 locator = 'acceptversions' code = 'VersionNegotiationFailed' text = ('Invalid parameter value in ' 'acceptversions: %s. Value MUST be ' '2.0.2 or 3.0.0' % self.kvp['acceptversions']) # test request if self.kvp['request'] not in \ self.context.model['operations']: error = 1 locator = 'request' if request in ['Transaction', 'Harvest']: code = 'OperationNotSupported' text = '%s operations are not supported' % request else: code = 'InvalidParameterValue' text = 'Invalid value for request: %s' % request if error == 1: # return an ExceptionReport LOGGER.error('basic service options error: %s, %s, %s', code, locator, text) self.response = self.iface.exceptionreport(code, locator, text) else: # process per the request value if 'responsehandler' in self.kvp: # set flag to process asynchronously import threading self.asynchronous = True request_id = self.kvp.get('requestid', None) if request_id is None: import uuid self.kvp['requestid'] = str(uuid.uuid4()) if self.kvp['request'] == 'GetCapabilities': self.response = self.iface.getcapabilities() elif self.kvp['request'] == 'DescribeRecord': self.response = self.iface.describerecord() elif self.kvp['request'] == 'GetDomain': self.response = self.iface.getdomain() elif self.kvp['request'] == 'GetRecords': if self.asynchronous: # process asynchronously threading.Thread(target=self.iface.getrecords).start() self.response = self.iface._write_acknowledgement() else: self.response = self.iface.getrecords() elif self.kvp['request'] == 'GetRecordById': self.response = self.iface.getrecordbyid() elif self.kvp['request'] == 'GetRepositoryItem': self.response = self.iface.getrepositoryitem() elif self.kvp['request'] == 'Transaction': self.response = self.iface.transaction() elif self.kvp['request'] == 'Harvest': if self.asynchronous: # process asynchronously threading.Thread(target=self.iface.harvest).start() self.response = self.iface._write_acknowledgement() else: self.response = self.iface.harvest() else: self.response = self.iface.exceptionreport( 'InvalidParameterValue', 'request', 'Invalid request parameter: %s' % self.kvp['request'] ) LOGGER.info('Request processed') if self.mode == 'sru': LOGGER.info('SRU mode detected; processing response.') self.response = self.sru().response_csw2sru(self.response, self.environ) elif self.mode == 'opensearch': LOGGER.info('OpenSearch mode detected; processing response.') self.response = self.opensearch().response_csw2opensearch( self.response, self.config) elif self.mode == 'oaipmh': LOGGER.info('OAI-PMH mode detected; processing response.') self.response = self.oaipmh().response( self.response, self.oaiargs, self.repository, self.config.get('server', 'url') ) return self._write_response() def getcapabilities(self): """ Handle GetCapabilities request """ return self.iface.getcapabilities() def describerecord(self): """ Handle DescribeRecord request """ return self.iface.describerecord() def getdomain(self): """ Handle GetDomain request """ return self.iface.getdomain() def getrecords(self): """ Handle GetRecords request """ return self.iface.getrecords() def getrecordbyid(self, raw=False): """ Handle GetRecordById request """ return self.iface.getrecordbyid(raw) def getrepositoryitem(self): """ Handle GetRepositoryItem request """ return self.iface.getrepositoryitem() def transaction(self): """ Handle Transaction request """ return self.iface.transaction() def harvest(self): """ Handle Harvest request """ return self.iface.harvest() def _write_response(self): """ Generate response """ # set HTTP response headers and XML declaration xmldecl = '' appinfo = '' LOGGER.info('Writing response.') if hasattr(self, 'soap') and self.soap: self._gen_soap_wrapper() if etree.__version__ >= '3.5.0': # remove superfluous namespaces etree.cleanup_namespaces(self.response, keep_ns_prefixes=self.context.keep_ns_prefixes) response = etree.tostring(self.response, pretty_print=self.pretty_print, encoding='unicode') if (isinstance(self.kvp, dict) and 'outputformat' in self.kvp and self.kvp['outputformat'] == 'application/json'): self.contenttype = self.kvp['outputformat'] from pycsw.core.formats import fmt_json response = fmt_json.xml2json(response, self.context.namespaces, self.pretty_print) else: # it's XML if 'outputformat' in self.kvp: self.contenttype = self.kvp['outputformat'] else: self.contenttype = self.mimetype xmldecl = ('<?xml version="1.0" encoding="%s" standalone="no"?>' '\n' % self.encoding) appinfo = '<!-- pycsw %s -->\n' % self.context.version if isinstance(self.contenttype, bytes): self.contenttype = self.contenttype.decode() s = (u'%s%s%s' % (xmldecl, appinfo, response)).encode(self.encoding) LOGGER.debug('Response code: %s', self.context.response_codes[self.status]) LOGGER.debug('Response:\n%s', s) return [self.context.response_codes[self.status], s] def _gen_soap_wrapper(self): """ Generate SOAP wrapper """ LOGGER.info('Writing SOAP wrapper.') node = etree.Element( util.nspath_eval('soapenv:Envelope', self.context.namespaces), nsmap=self.context.namespaces ) schema_location_ns = util.nspath_eval('xsi:schemaLocation', self.context.namespaces) node.attrib[schema_location_ns] = '%s %s' % ( self.context.namespaces['soapenv'], self.context.namespaces['soapenv'] ) node2 = etree.SubElement( node, util.nspath_eval('soapenv:Body', self.context.namespaces)) if self.exception: node3 = etree.SubElement( node2, util.nspath_eval('soapenv:Fault', self.context.namespaces) ) node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Code', self.context.namespaces) ) etree.SubElement( node4, util.nspath_eval('soapenv:Value', self.context.namespaces) ).text = 'soap:Server' node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Reason', self.context.namespaces) ) etree.SubElement( node4, util.nspath_eval('soapenv:Text', self.context.namespaces) ).text = 'A server exception was encountered.' node4 = etree.SubElement( node3, util.nspath_eval('soapenv:Detail', self.context.namespaces) ) node4.append(self.response) else: node2.append(self.response) self.response = node def _gen_manager(self): """ Update self.context.model with CSW-T advertising """ if (self.config.has_option('manager', 'transactions') and self.config.get('manager', 'transactions') == 'true'): self.manager = True self.context.model['operations_order'].append('Transaction') self.context.model['operations']['Transaction'] = { 'methods': {'get': False, 'post': True}, 'parameters': {} } schema_values = [ 'http://www.opengis.net/cat/csw/2.0.2', 'http://www.opengis.net/cat/csw/3.0', 'http://www.opengis.net/wms', 'http://www.opengis.net/wmts/1.0', 'http://www.opengis.net/wfs', 'http://www.opengis.net/wfs/2.0', 'http://www.opengis.net/wcs', 'http://www.opengis.net/wps/1.0.0', 'http://www.opengis.net/sos/1.0', 'http://www.opengis.net/sos/2.0', 'http://www.isotc211.org/2005/gmi', 'urn:geoss:waf', ] self.context.model['operations_order'].append('Harvest') self.context.model['operations']['Harvest'] = { 'methods': {'get': False, 'post': True}, 'parameters': { 'ResourceType': {'values': schema_values} } } self.context.model['operations']['Transaction'] = { 'methods': {'get': False, 'post': True}, 'parameters': { 'TransactionSchemas': {'values': sorted(schema_values)} } } self.csw_harvest_pagesize = 10 if self.config.has_option('manager', 'csw_harvest_pagesize'): self.csw_harvest_pagesize = int( self.config.get('manager', 'csw_harvest_pagesize')) def _test_manager(self): """ Verify that transactions are allowed """ if self.config.get('manager', 'transactions') != 'true': raise RuntimeError('CSW-T interface is disabled') """ get the client first forwarded ip """ if 'HTTP_X_FORWARDED_FOR' in self.environ: ipaddress = self.environ['HTTP_X_FORWARDED_FOR'].split(',')[0].strip() else: ipaddress = self.environ['REMOTE_ADDR'] if not self.config.has_option('manager', 'allowed_ips') or \ (self.config.has_option('manager', 'allowed_ips') and not util.ipaddress_in_whitelist(ipaddress, self.config.get('manager', 'allowed_ips').split(','))): raise RuntimeError( 'CSW-T operations not allowed for this IP address: %s' % ipaddress) def _cql_update_queryables_mappings(self, cql, mappings): """ Transform CQL query's properties to underlying DB columns """ LOGGER.debug('Raw CQL text = %s', cql) LOGGER.debug(str(list(mappings.keys()))) if cql is not None: for key in mappings.keys(): try: cql = cql.replace(key, mappings[key]['dbcol']) except: cql = cql.replace(key, mappings[key]) LOGGER.debug('Interpolated CQL text = %s.', cql) return cql def _process_responsehandler(self, xml): """ Process response handler """ if self.kvp['responsehandler'] is not None: LOGGER.info('Processing responsehandler %s' % self.kvp['responsehandler']) uprh = urlparse(self.kvp['responsehandler']) if uprh.scheme == 'mailto': # email import smtplib LOGGER.debug('Email detected') smtp_host = 'localhost' if self.config.has_option('server', 'smtp_host'): smtp_host = self.config.get('server', 'smtp_host') body = ('Subject: pycsw %s results\n\n%s' % (self.kvp['request'], xml)) try: LOGGER.info('Sending email') msg = smtplib.SMTP(smtp_host) msg.sendmail( self.config.get('metadata:main', 'contact_email'), uprh.path, body ) msg.quit() LOGGER.debug('Email sent successfully.') except Exception as err: LOGGER.exception('Error processing email') elif uprh.scheme == 'ftp': import ftplib LOGGER.debug('FTP detected.') try: LOGGER.info('Sending to FTP server.') ftp = ftplib.FTP(uprh.hostname) if uprh.username is not None: ftp.login(uprh.username, uprh.password) ftp.storbinary('STOR %s' % uprh.path[1:], StringIO(xml)) ftp.quit() LOGGER.debug('FTP sent successfully.') except Exception as err: LOGGER.exception('Error processing FTP') @staticmethod def normalize_kvp(kvp): """Normalize Key Value Pairs. This method will transform all keys to lowercase and leave values unchanged, as specified in the CSW standard (see for example note C on Table 62 - KVP Encoding for DescribeRecord operation request of the CSW standard version 2.0.2) :arg kvp: a mapping with Key Value Pairs :type kvp: dict :returns: A new dictionary with normalized parameters """ result = dict() for name, value in kvp.items(): result[name.lower()] = value return result
def create_config(): """ Create a Trident configuration file using interaction with the user. This function is called by :class:`~trident.parse_config` if it appears that the configuration has not yet been set up for the user. It will attempt to create a configuration file and download an ion table datafile from the web. It does this using user interaction from the python prompt. """ default_dir = os.path.expanduser('~/.trident') trident() print( "It appears that this is your first time using Trident. To finalize your" ) print("Trident installation, you must:") print(" * create a `~/.trident` directory") print(" * create a config.tri file in your `~/.trident` directory") print(" * download an ion table file for calculating ionization fractions") print("") print( "You can do this manually by following the installation docs, or we can" ) print("do it automatically now if you have web access.") print("") print("Would you like to do this automatically? ([y]/n)") value = input().rstrip() if not value == '' and not value == 'y': sys.exit( 'Instructions at http://trident.readthedocs.org/en/latest/installation.html' ) print("") print("Where would you like Trident to store the ion table file?") print("[%s]" % default_dir) # First assure that the .trident directory is created for storing # the config file. ensure_directory(default_dir) datadir = input().rstrip() if datadir == '': datadir = default_dir datadir = os.path.expanduser(datadir) # Try to create data directory if it doesn't exist try: ensure_directory(datadir) print("Using %s" % datadir) print("") except BaseException: print("Cannot create directory %s" % datadir) raise # Get the datafile from the web datafile = get_datafiles(datadir=datadir) # Create the config file and make it to the datadir and datafiles chosen config = SafeConfigParser() config.add_section('Trident') config.set('Trident', 'ion_table_dir', datadir) config.set('Trident', 'ion_table_file', datafile) config_filename = os.path.expanduser('~/.trident/config.tri') with open(config_filename, 'w') as configfile: config.write(configfile) print("") print("Installation complete. I recommend verifying your installation") print("to assure that everything is working. Try: trident.verify()") # Return the config file path so we can load it and get parameters. return config_filename
class Config(object): """ Manages the configuration file """ def __init__(self): """ DEFAULT VALUES """ self._basescript = None self.recentvaults = [] self.pwlength = 10 self.search_notes = False self.search_passwd = False self.alphabet = "abcdefghikmnopqrstuvwxyz23456789ABCDEFGHJKLMNPQRSTUVWXYZ_" self.avoid_bigrams = "cl mn nm nn rn vv VV" self._fname = self.get_config_filename() self._parser = SafeConfigParser() if os.path.exists(self._fname): self._parser.read(self._fname) if not self._parser.has_section("base"): self._parser.add_section("base") for num in range(10): if (not self._parser.has_option("base", "recentvaults" + str(num))): break self.recentvaults.append(self._parser.get("base", "recentvaults" + str(num))) if self._parser.has_option("base", "pwlength"): self.pwlength = int(self._parser.get("base", "pwlength")) if self._parser.has_option("base", "search_notes"): if self._parser.get("base", "search_notes") == "True": self.search_notes = True if self._parser.has_option("base", "search_passwd"): if self._parser.get("base", "search_passwd") == "True": self.search_passwd = True if self._parser.has_option("base", "alphabet"): self.alphabet = self._parser.get("base", "alphabet") if self._parser.has_option("base", "avoid_bigrams"): self.avoid_bigrams = self._parser.get("base", "avoid_bigrams") if not os.path.exists(self._fname): self.save() def set_basescript(self, basescript): self._basescript = basescript def get_basescript(self): return self._basescript def save(self): if (not os.path.exists(os.path.dirname(self._fname))): os.mkdir(os.path.dirname(self._fname)) # remove duplicates and trim to 10 items _saved_recentvaults = [] for item in self.recentvaults: if item in _saved_recentvaults: continue self._parser.set("base", "recentvaults" + str(len(_saved_recentvaults)), item) _saved_recentvaults.append(item) if (len(_saved_recentvaults) >= 10): break self._parser.set("base", "pwlength", str(self.pwlength)) self._parser.set("base", "search_notes", str(self.search_notes)) self._parser.set("base", "search_passwd", str(self.search_passwd)) self._parser.set("base", "alphabet", str(self.alphabet)) self._parser.set("base", "avoid_bigrams", str(self.avoid_bigrams)) filehandle = open(self._fname, 'w') self._parser.write(filehandle) filehandle.close() @staticmethod def get_config_filename(): """ Returns the full filename of the config file """ base_fname = "loxodo" # On Mac OS X, config files go to ~/Library/Application Support/foo/ if platform.system() == "Darwin": base_path = os.path.join(os.path.expanduser("~"), "Library", "Application Support") if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # On Microsoft Windows, config files go to $APPDATA/foo/ if platform.system() in ("Windows", "Microsoft"): if ("APPDATA" in os.environ): base_path = os.environ["APPDATA"] if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # Allow config directory override as per freedesktop.org XDG Base Directory Specification if ("XDG_CONFIG_HOME" in os.environ): base_path = os.environ["XDG_CONFIG_HOME"] if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") # Default configuration path is ~/.config/foo/ base_path = os.path.join(os.path.expanduser("~"), ".config") if os.path.isdir(base_path): return os.path.join(base_path, base_fname, base_fname + ".ini") else: return os.path.join(os.path.expanduser("~"),"."+ base_fname + ".ini")
class PackageConfigHandler(object): """ Manager class for packages files for tracking installation of modules """ def __init__(self): # noinspection PyUnresolvedReferences from six.moves.configparser import SafeConfigParser self.package_cfg = os.path.expanduser('~/Documents/site-packages/.pypi_packages') if not os.path.isfile(self.package_cfg): print('Creating package file') with open(self.package_cfg, 'w') as outs: outs.close() self.parser = SafeConfigParser() self.parser.read(self.package_cfg) def save(self): with open(self.package_cfg, 'w') as outs: self.parser.write(outs) def add_module(self, pkg_info): """ :param pkg_info: A dict that has name, url, version, summary :return: """ if not self.parser.has_section(pkg_info['name']): self.parser.add_section(pkg_info['name']) self.parser.set(pkg_info['name'], 'url', pkg_info['url']) self.parser.set(pkg_info['name'], 'version', pkg_info['version']) self.parser.set(pkg_info['name'], 'summary', pkg_info['summary']) self.parser.set(pkg_info['name'], 'files', pkg_info['files']) self.parser.set(pkg_info['name'], 'dependency', pkg_info['dependency']) self.save() def list_modules(self): return [module for module in self.parser.sections()] def module_exists(self, name): return self.parser.has_section(name) def get_info(self, name): if self.parser.has_section(name): tbl = {} for opt, value in self.parser.items(name): tbl[opt] = value return tbl def remove_module(self, name): self.parser.remove_section(name) self.save() def get_files_installed(self, section_name): if self.parser.has_option(section_name, 'files'): files = self.parser.get(section_name, 'files').strip() return files.split(',') else: return None def get_dependencies(self, section_name): if self.parser.has_option(section_name, 'dependency'): dependencies = self.parser.get(section_name, 'dependency').strip() return set(dependencies.split(',')) if dependencies != '' else set() else: return None def get_all_dependencies(self, exclude_module=()): all_dependencies = set() for section_name in self.parser.sections(): if section_name not in exclude_module and self.parser.has_option(section_name, 'dependency'): dependencies = self.parser.get(section_name, 'dependency').strip() if dependencies != '': for dep in dependencies.split(','): all_dependencies.add(dep) return all_dependencies