コード例 #1
0
 def _read_config(self):
     """Reads the config file with the Rebrandly API key"""
     import os
     file_path = os.path.join(os.path.dirname(__file__), './config.ini')
     config = ConfigParser()
     config.read(file_path)
     self._apikey = config.get('Rebrandly', 'apikey')
コード例 #2
0
    def __init__(self, default_config=None, *args, **kwargs):
        super(AirflowConfigParser, self).__init__(*args, **kwargs)

        self.airflow_defaults = ConfigParser(*args, **kwargs)
        if default_config is not None:
            self.airflow_defaults.read_string(default_config)

        self.is_validated = False
コード例 #3
0
ファイル: configuration.py プロジェクト: P79N6A/xTool
    def __init__(self, default_config=None, *args, **kwargs):
        super(XToolConfigParser, self).__init__(*args, **kwargs)
        self.defaults = ConfigParser(*args, **kwargs)
        # 读取配置字符串
        if default_config is not None:
            self.defaults.read_string(default_config)

        self.is_validated = False
コード例 #4
0
    def __init__(self, default_config=None, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.excalibur_defaults = ConfigParser(*args, **kwargs)
        if default_config is not None:
            self.excalibur_defaults.read_string(default_config)

        self.is_validated = False
コード例 #5
0
ファイル: config.py プロジェクト: amolk4games/ichnaea
 def __init__(self, filename):
     ConfigParser.__init__(self)
     # let's read the file
     if isinstance(filename, string_types):
         self.filename = filename
         self.read(filename)
     else:  # pragma: no cover
         self.filename = None
         self.read_file(filename)
コード例 #6
0
ファイル: config.py プロジェクト: amolk4games/ichnaea
 def __init__(self, filename):
     ConfigParser.__init__(self)
     # let's read the file
     if isinstance(filename, string_types):
         self.filename = filename
         self.read(filename)
     else:  # pragma: no cover
         self.filename = None
         self.read_file(filename)
コード例 #7
0
    def __init__(self, lcfname):
        '''
        Constructor
        '''
        super(LayerFileReader, self).__init__(lcfname)

        self.cp = ConfigParser()
        self.lcfilename = LU.standardiseLayerConfigName(self.lcfname)

        self._readConfigFile(self.lcfilename)
コード例 #8
0
ファイル: bootstrap.py プロジェクト: QuittyMR/etlas-collector
    def _populate_config(self) -> ConfigParser:
        config = ConfigParser()
        config_path = './resources/config.ini'

        if 'resources' not in listdir('.'):
            config_path = '.' + config_path

        with open(config_path) as file_pointer:
            config.read_file(file_pointer)

        return config
コード例 #9
0
ファイル: config.py プロジェクト: mikicz/flexipy
 def __init__(self, config_name="flexipy/flexipy.conf"):
     self.conf = ConfigParser()
     #use resource management api to find flexipy.conf, see docs
     filename = resource_filename(Requirement.parse("flexipy"), config_name)
     # Open the file with the correct encoding
     try:
         with codecs.open(filename, 'r', encoding='utf-8') as f:
             self.conf.readfp(f)
     except IOError:
         raise ValueError('Konfiguracni soubor ' + config_name +
                          ' neexistuje nebo jste uvedli spatnou cestu.')
コード例 #10
0
    def __init__(self):
        '''
        Constructor
        '''
        self.dvalue = None
        self.dselect = 'dest'
        #v:x111|MYGROUP, myconf.conf, 2193, 2013-01-01, 2013-01-02
        self.plist = ('lgvalue', 'uconf', 'epsg', 'fd', 'td')

        self.cp = ConfigParser()
        self.fn = os.path.join(os.path.dirname(__file__), self.GUI_PREFS)
        with codecs.open(self.fn, 'r', 'utf-8') as cf:
            self.cp.readfp(cf)
コード例 #11
0
    def setUpClass(cls):
        cls.config = ConfigParser()
        cls.config.read([
            r'pam_mysql_manager-test.conf',
            r'/etc/pam_mysql_manager-test.conf',
            os.path.expanduser('~/.pam_mysql_manager-test.conf')
        ])

        mysql_user = os.getenv(
            "PAMMYSQL_TEST_MYSQL_USER",
            cls.config.get('database', 'user', fallback='root'))
        mysql_pass = os.getenv(
            "PAMMYSQL_TEST_MYSQL_PASS",
            cls.config.get('database', 'password', fallback=''))
        mysql_host = os.getenv(
            "PAMMYSQL_TEST_MYSQL_HOST",
            cls.config.get('database', 'host', fallback='localhost'))
        mysql_port = int(
            os.getenv("PAMMYSQL_TEST_MYSQL_PORT",
                      cls.config.get('database', 'port', fallback="3306")))
        mysql_db = os.getenv(
            "PAMMYSQL_TEST_MYSQL_DB",
            cls.config.get('database', 'database', fallback='auth_test'))

        cls.dbs = pymysql.connect(host=mysql_host,
                                  user=mysql_user,
                                  password=mysql_pass,
                                  db=mysql_db,
                                  port=mysql_port)

        with open(
                os.path.join(os.path.dirname(os.path.realpath(__file__)),
                             "testdb.sql"), "r") as f:
            cls.sql = f.read()
コード例 #12
0
ファイル: configuration.py プロジェクト: jgao54/airflow
    def __init__(self, default_config=None, *args, **kwargs):
        super(AirflowConfigParser, self).__init__(*args, **kwargs)

        self.airflow_defaults = ConfigParser(*args, **kwargs)
        if default_config is not None:
            self.airflow_defaults.read_string(default_config)

        self.is_validated = False
コード例 #13
0
ファイル: ReadConfig.py プロジェクト: josephramsay/LDS
 def __init__(self,lcfname):
     '''
     Constructor
     '''
     super(LayerFileReader,self).__init__(lcfname)
     
     self.cp = ConfigParser()
     self.lcfilename = LU.standardiseLayerConfigName(self.lcfname)
         
     self._readConfigFile(self.lcfilename)
コード例 #14
0
ファイル: config.py プロジェクト: amolk4games/ichnaea
 def get(self, section, option, default=None):
     """
     A get method which returns the default argument when the option
     cannot be found instead of raising an exception.
     """
     try:
         value = ConfigParser.get(self, section, option)
     except (NoOptionError, NoSectionError):  # pragma: no cover
         value = default
     return value
コード例 #15
0
ファイル: config.py プロジェクト: amolk4games/ichnaea
 def get(self, section, option, default=None):
     """
     A get method which returns the default argument when the option
     cannot be found instead of raising an exception.
     """
     try:
         value = ConfigParser.get(self, section, option)
     except (NoOptionError, NoSectionError):  # pragma: no cover
         value = default
     return value
コード例 #16
0
def loadTemplates(injectSection = None):
    global TEMPLATES
    TEMPLATES = {}
    parser = ConfigParser()
    parser.read('templates.conf')
    for sectionName in parser.sections():
        TEMPLATES[sectionName] = Configlet(sectionName, dict(parser.items(sectionName)), injectSection)
コード例 #17
0
def get_value_from_ini_config(overcloud_node,
                              config_path,
                              check_section,
                              check_value,
                              multi_key_values=False):
    """Get value from INI configuration file

    :param overcloud_node:   The node that config should be pulled from
    :param config_path:      The path of the configuration file
    :param check_section:    Section within the config
    :param check_value:      Value that should be checked within the config
                             The variable could hold multiple values separated
                             by comma.
    :param multi_key_values: Flag on request to hold multiple values for
                             single key from ini file
    :return return_value
    """
    class M(OrderedDict):
        def __setitem__(self, key, value):
            v_val = self.get(key)
            if v_val is not None and type(value) == list:
                v_val.append(value[0])
            else:
                v_val = value
            # still using python2.7 super, for backport portability
            super(M, self).__setitem__(key, v_val)

    ini_config = get_overcloud_config(overcloud_node, config_path)
    config_parser_args = {'allow_no_value': True}
    if multi_key_values:
        config_parser_args['dict_type'] = M
    config_parser_args['strict'] = False
    get_value = ConfigParser(**config_parser_args)
    get_value.read_file(StringIO(ini_config))
    value_data = []
    for value in check_value.split(','):
        value_data.append(get_value.get(check_section, value))

    return ','.join(value_data)
コード例 #18
0
class ExcaliburConfigParser(ConfigParser):
    def __init__(self, default_config=None, *args, **kwargs):
        super(ExcaliburConfigParser, self).__init__(*args, **kwargs)

        self.excalibur_defaults = ConfigParser(*args, **kwargs)
        if default_config is not None:
            self.excalibur_defaults.read_string(default_config)

        self.is_validated = False

    def _validate(self):
        if self.get(
                "core",
                "executor") != "SequentialExecutor" and "sqlite" in self.get(
                    "core", "sql_alchemy_conn"):
            raise ValueError("Cannot use sqlite with the {}".format(
                self.get("core", "executor")))

        self.is_validated = True

    def get(self, section, key, **kwargs):
        section = str(section).lower()
        key = str(key).lower()

        if super(ExcaliburConfigParser, self).has_option(section, key):
            return expand_env_var(
                super(ExcaliburConfigParser, self).get(section, key, **kwargs))

        if self.excalibur_defaults.has_option(section, key):
            return expand_env_var(
                self.excalibur_defaults.get(section, key, **kwargs))

        else:
            raise ValueError("section/key [{section}/{key}] not found in"
                             " config".format(**locals()))

    def read(self, filename):
        super(ExcaliburConfigParser, self).read(filename)
        self._validate()
コード例 #19
0
ファイル: ReadConfig.py プロジェクト: josephramsay/LDS
 def __init__(self):
     '''
     Constructor
     '''
     self.dvalue = None
     self.dselect = 'dest'
     #v:x111|MYGROUP, myconf.conf, 2193, 2013-01-01, 2013-01-02
     self.plist = ('lgvalue','uconf','epsg','fd','td')
     
     self.cp = ConfigParser()
     self.fn = os.path.join(os.path.dirname(__file__),self.GUI_PREFS)
     with codecs.open(self.fn,'r','utf-8') as cf:
         self.cp.readfp(cf)
コード例 #20
0
 def _read_config(self):
     """Reads the config file with the API keys which should be placed where aliexpress_api.py is"""
     import os
     file_path = os.path.join(os.path.dirname(__file__), './config.ini')
     config = ConfigParser()
     config.read(file_path)
     self._appkey = config.get('AliExpress', 'appkey')
     self._tracking_id = config.get('AliExpress', 'trackingId')
コード例 #21
0
def main():
    # Import api key
    parser = ConfigParser()
    parser.read('config.ini')
    bot_token = parser.get('Telegram', 'token')

    # Create the Updater and pass it your bot's token.
    updater = Updater(bot_token)

    # todo: build a menu and conversation handler
    updater.dispatcher.add_handler(CommandHandler('promo', promo))
    updater.dispatcher.add_handler(
        CommandHandler('search', search, pass_args=True))
    updater.dispatcher.add_handler(CommandHandler('link', link,
                                                  pass_args=True))
    updater.dispatcher.add_error_handler(error)

    # Start the Bot
    updater.start_polling()

    # Run the bot until the user presses Ctrl-C or the process receives SIGINT,
    # SIGTERM or SIGABRT
    updater.idle()
コード例 #22
0
ファイル: config_readers.py プロジェクト: turbaszek/dbnd
def read_from_config_stream(config_fp, source="<stream>"):
    """
    Read config from config file (.ini, .cfg)
    """
    from backports.configparser import ConfigParser

    parser = ConfigParser()
    parser._read(config_fp, source)

    source = "config[{file_name}]".format(file_name=os.path.basename(str(source)))
    new_config = _ConfigStore()
    for section in parser.sections():
        for option in parser.options(section):
            value = parser.get(section, option)
            new_config.set_config_value(
                section, option, ConfigValue(value, source, require_parse=True)
            )

    return new_config
コード例 #23
0
def config(filename='database.ini', section='postgresql'):
    # create a parser
    parser = ConfigParser()
    # read config file
    parser.read(filename)

    # get section, default to postgresql
    db = {}
    if parser.has_section(section):
        params = parser.items(section)
        for param in params:
            db[param[0]] = param[1]
    else:
        raise Exception('Section {0} not found in the {1} file'.format(
            section, filename))

    return db
コード例 #24
0
ファイル: plugin_wizard.py プロジェクト: Adman/errbot
def new_plugin_wizard(directory=None):
    """
    Start the wizard to create a new plugin in the current working directory.
    """
    if directory is None:
        print("This wizard will create a new plugin for you in the current directory.")
        directory = os.getcwd()
    else:
        print("This wizard will create a new plugin for you in '%s'." % directory)

    if os.path.exists(directory) and not os.path.isdir(directory):
        print("Error: The path '%s' exists but it isn't a directory" % directory)
        sys.exit(1)

    name = ask(
        "What should the name of your new plugin be?",
        validation_regex=r'^[a-zA-Z][a-zA-Z0-9 _-]*$'
    ).strip()
    module_name = name.lower().replace(' ', '_')
    directory_name = name.lower().replace(' ', '-')
    class_name = "".join([s.capitalize() for s in name.lower().split(' ')])

    description = ask(
        "What may I use as a short (one-line) description of your plugin?"
    )
    if PY2:
        default_python_version = "2+"
    else:
        default_python_version = "3"
    python_version = ask(
        "Which python version will your plugin work with? 2, 2+ or 3? I will default to "
        "{version} if you leave this blank.".format(version=default_python_version),
        valid_responses=['2', '2+', '3', '']
    )
    if python_version.strip() == "":
        python_version = default_python_version
    errbot_min_version = ask(
        "Which minimum version of errbot will your plugin work with? "
        "Leave blank to support any version or input CURRENT to select the "
        "current version (%s)" % VERSION
    ).strip()
    if errbot_min_version.upper() == "CURRENT":
        errbot_min_version = VERSION
    errbot_max_version = ask(
        "Which maximum version of errbot will your plugin work with? "
        "Leave blank to support any version or input CURRENT to select the "
        "current version (%s)" % VERSION
    ).strip()
    if errbot_max_version.upper() == "CURRENT":
        errbot_max_version = VERSION

    plug = ConfigParser()
    plug["Core"] = {
        "Name": name,
        "Module": module_name,
    }
    plug["Documentation"] = {
        "Description": description,
    }
    plug["Python"] = {
        "Version": python_version,
    }
    plug["Errbot"] = {}
    if errbot_min_version != "":
        plug["Errbot"]["Min"] = errbot_min_version
    if errbot_max_version != "":
        plug["Errbot"]["Max"] = errbot_max_version

    plugin_path = directory
    plugfile_path = os.path.join(plugin_path, module_name+".plug")
    pyfile_path = os.path.join(plugin_path, module_name+".py")

    try:
        if PY3 or (PY2 and not os.path.isdir(plugin_path)):
            os.makedirs(plugin_path, mode=0o700)
    except IOError as e:
        if e.errno != errno.EEXIST:
            raise

    if os.path.exists(plugfile_path) or os.path.exists(pyfile_path):
        ask(
            "Warning: A plugin with this name was already found at {path}\n"
            "If you continue, these will be overwritten.\n"
            "Press Ctrl+C to abort now or type in 'overwrite' to confirm overwriting of these files."
            "".format(
                path=os.path.join(directory, module_name+".{py,plug}")
            ),
            valid_responses=["overwrite"],
        )

    with open(plugfile_path, 'w') as f:
        plug.write(f)

    with open(pyfile_path, 'w') as f:
        f.write(render_plugin(locals()))

    print("Success! You'll find your new plugin at '%s'" % plugfile_path)
    print("(Don't forget to include a LICENSE file if you are going to publish your plugin)")
コード例 #25
0
ファイル: ReadConfig.py プロジェクト: josephramsay/LDS
 def _readConfigFile(self,fn):
     '''Reads named config file'''
     #Split off so you can override the config file on the same reader object if needed
     self.cp = ConfigParser()
     with codecs.open(fn,'r','utf-8') as cf:
         self.cp.readfp(cf)
コード例 #26
0
ファイル: config.py プロジェクト: mikicz/flexipy
class Config(object):
    """
	Base config class definuje zakladni metody pro praci s konfiguracnim souborem.	
	"""
    def __init__(self, config_name="flexipy/flexipy.conf"):
        self.conf = ConfigParser()
        #use resource management api to find flexipy.conf, see docs
        filename = resource_filename(Requirement.parse("flexipy"), config_name)
        # Open the file with the correct encoding
        try:
            with codecs.open(filename, 'r', encoding='utf-8') as f:
                self.conf.readfp(f)
        except IOError:
            raise ValueError('Konfiguracni soubor ' + config_name +
                             ' neexistuje nebo jste uvedli spatnou cestu.')

    def get_section_list(self, section_name):
        """
		Tato privatni metoda spracuje vsechny sekce v config filu 
		na zaklade jmena sekce a vrati list obsahujici vsechny polozky.
		"""
        result_list = []
        try:
            section_content = self.conf.items(section_name)
            for key, val in section_content:
                result_list.append(val)
        except NoSectionError:
            raise ValueError("Config file neobsahuje sekci " + section_name)
        return result_list

    def get_server_config(self):
        """
		Tato metoda vrati dict obsahujici vsechna nastaveni tykajici se serveru. 		
		"""
        result = {}
        try:
            section_content = self.conf.items("server")
            for key, val in section_content:
                result[key] = val
        except NoSectionError:
            raise ValueError("Config file neobsahuje sekci server")
        return result

    def get_evidence_list(self):
        return self.get_section_list('evidence')

    def get_typy_faktury_prijate(self):
        return self.get_section_list('typ_faktury_prijate')

    def get_typy_faktury_vydane(self):
        return self.get_section_list('typ_faktury_vydane')

    def get_typ_bank_dokladu(self):
        return self.get_section_list('typ_bank_dokladu')

    def get_typ_pohybu(self):
        return self.get_section_list('typ_pohybu')

    def get_bankovni_ucty(self):
        return self.get_section_list('bankovni_ucty')

    def get_typ_polozky_vydane(self):
        return self.get_section_list('typ_polozky_vydane')

    def get_typ_ucetni_operace(self):
        return self.get_section_list('typ_ucetni_operace')

    def get_typ_pokladni_pohyb(self):
        return self.get_section_list('typ_pokladni_pohyb')

    def get_typ_pokladna(self):
        return self.get_section_list('typ_pokladna')
コード例 #27
0
class AirflowConfigParser(ConfigParser):

    # These configuration elements can be fetched as the stdout of commands
    # following the "{section}__{name}__cmd" pattern, the idea behind this
    # is to not store password on boxes in text files.
    # These configs can also be fetched from Secrets backend
    # following the "{section}__{name}__secret" pattern
    sensitive_config_values = {
        ('core', 'sql_alchemy_conn'),
        ('core', 'fernet_key'),
        ('celery', 'broker_url'),
        ('celery', 'flower_basic_auth'),
        ('celery', 'result_backend'),
        # Todo: remove this in Airflow 1.11
        ('celery', 'celery_result_backend'),
        ('atlas', 'password'),
        ('smtp', 'smtp_password'),
        ('ldap', 'bind_password'),
        ('kubernetes', 'git_password'),
    }

    # A two-level mapping of (section -> new_name -> old_name). When reading
    # new_name, the old_name will be checked to see if it exists. If it does a
    # DeprecationWarning will be issued and the old name will be used instead
    deprecated_options = {
        'celery': {
            # Remove these keys in Airflow 1.11
            'worker_concurrency': 'celeryd_concurrency',
            'result_backend': 'celery_result_backend',
            'broker_url': 'celery_broker_url',
            'ssl_active': 'celery_ssl_active',
            'ssl_cert': 'celery_ssl_cert',
            'ssl_key': 'celery_ssl_key',
        },
        'elasticsearch': {
            'host': 'elasticsearch_host',
            'log_id_template': 'elasticsearch_log_id_template',
            'end_of_log_mark': 'elasticsearch_end_of_log_mark',
            'frontend': 'elasticsearch_frontend',
            'write_stdout': 'elasticsearch_write_stdout',
            'json_format': 'elasticsearch_json_format',
            'json_fields': 'elasticsearch_json_fields'

        }
    }

    # A mapping of old default values that we want to change and warn the user
    # about. Mapping of section -> setting -> { old, replace, by_version }
    deprecated_values = {
        'core': {
            'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'),
        },
    }

    # This method transforms option names on every read, get, or set operation.
    # This changes from the default behaviour of ConfigParser from lowercasing
    # to instead be case-preserving
    def optionxform(self, optionstr):
        return optionstr

    def __init__(self, default_config=None, *args, **kwargs):
        super(AirflowConfigParser, self).__init__(*args, **kwargs)

        self.airflow_defaults = ConfigParser(*args, **kwargs)
        if default_config is not None:
            self.airflow_defaults.read_string(default_config)

        self.is_validated = False

    def _validate(self):
        self._validate_config_dependencies()
        for section, replacement in self.deprecated_values.items():
            for name, info in replacement.items():
                old, new, version = info
                if self.get(section, name, fallback=None) == old:
                    # Make sure the env var option is removed, otherwise it
                    # would be read and used instead of the value we set
                    env_var = self._env_var_name(section, name)
                    os.environ.pop(env_var, None)

                    self.set(section, name, new)
                    warnings.warn(
                        'The {name} setting in [{section}] has the old default value '
                        'of {old!r}. This value has been changed to {new!r} in the '
                        'running config, but please update your config before Apache '
                        'Airflow {version}.'.format(
                            name=name, section=section, old=old, new=new, version=version
                        ),
                        FutureWarning
                    )

        self.is_validated = True

    def _validate_config_dependencies(self):
        """
        Validate that config values aren't invalid given other config values
        or system-level limitations and requirements.
        """

        if (
                self.get("core", "executor") not in ('DebugExecutor', 'SequentialExecutor') and
                "sqlite" in self.get('core', 'sql_alchemy_conn')):
            raise AirflowConfigException(
                "error: cannot use sqlite with the {}".format(
                    self.get('core', 'executor')))

        elif (
            self.getboolean("webserver", "authenticate") and
            self.get("webserver", "owner_mode") not in ['user', 'ldapgroup']
        ):
            raise AirflowConfigException(
                "error: owner_mode option should be either "
                "'user' or 'ldapgroup' when filtering by owner is set")

        elif (
            self.getboolean("webserver", "authenticate") and
            self.get("webserver", "owner_mode").lower() == 'ldapgroup' and
            self.get("webserver", "auth_backend") != (
                'airflow.contrib.auth.backends.ldap_auth')
        ):
            raise AirflowConfigException(
                "error: attempt at using ldapgroup "
                "filtering without using the Ldap backend")

        if self.has_option('core', 'mp_start_method'):
            mp_start_method = self.get('core', 'mp_start_method')
            start_method_options = multiprocessing.get_all_start_methods()

            if mp_start_method not in start_method_options:
                raise AirflowConfigException(
                    "mp_start_method should not be " + mp_start_method +
                    ". Possible values are " + ", ".join(start_method_options))

    @staticmethod
    def _env_var_name(section, key):
        return 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())

    def _get_env_var_option(self, section, key):
        # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
        env_var = self._env_var_name(section, key)
        if env_var in os.environ:
            return expand_env_var(os.environ[env_var])
        # alternatively AIRFLOW__{SECTION}__{KEY}_CMD (for a command)
        env_var_cmd = env_var + '_CMD'
        if env_var_cmd in os.environ:
            # if this is a valid command key...
            if (section, key) in self.sensitive_config_values:
                return run_command(os.environ[env_var_cmd])
        # alternatively AIRFLOW__{SECTION}__{KEY}_SECRET (to get from Secrets Backend)
        env_var_secret_path = env_var + '_SECRET'
        if env_var_secret_path in os.environ:
            # if this is a valid secret path...
            if (section, key) in self.sensitive_config_values:
                return _get_config_value_from_secret_backend(os.environ[env_var_secret_path])

    def _get_cmd_option(self, section, key):
        fallback_key = key + '_cmd'
        # if this is a valid command key...
        if (section, key) in self.sensitive_config_values:
            if super(AirflowConfigParser, self).has_option(section, fallback_key):
                command = super(AirflowConfigParser, self).get(section, fallback_key)
                return run_command(command)

    def _get_secret_option(self, section, key):
        """Get Config option values from Secret Backend"""
        fallback_key = key + '_secret'
        # if this is a valid secret key...
        if (section, key) in self.sensitive_config_values:
            if super(AirflowConfigParser, self).has_option(section, fallback_key):
                secrets_path = super(AirflowConfigParser, self).get(section, fallback_key)
                return _get_config_value_from_secret_backend(secrets_path)

    def get(self, section, key, **kwargs):
        section = str(section).lower()
        key = str(key).lower()

        deprecated_name = self.deprecated_options.get(section, {}).get(key, None)

        # first check environment variables
        option = self._get_env_var_option(section, key)
        if option is not None:
            return option
        if deprecated_name:
            option = self._get_env_var_option(section, deprecated_name)
            if option is not None:
                self._warn_deprecate(section, key, deprecated_name)
                return option

        # ...then the config file
        if super(AirflowConfigParser, self).has_option(section, key):
            # Use the parent's methods to get the actual config here to be able to
            # separate the config from default config.
            return expand_env_var(
                super(AirflowConfigParser, self).get(section, key, **kwargs))
        if deprecated_name:
            if super(AirflowConfigParser, self).has_option(section, deprecated_name):
                self._warn_deprecate(section, key, deprecated_name)
                return expand_env_var(super(AirflowConfigParser, self).get(
                    section,
                    deprecated_name,
                    **kwargs
                ))

        # ...then commands
        option = self._get_cmd_option(section, key)
        if option:
            return option
        if deprecated_name:
            option = self._get_cmd_option(section, deprecated_name)
            if option:
                self._warn_deprecate(section, key, deprecated_name)
                return option

        # ...then from secret backends
        option = self._get_secret_option(section, key)
        if option:
            return option
        if deprecated_name:
            option = self._get_secret_option(section, deprecated_name)
            if option:
                self._warn_deprecate(section, key, deprecated_name)
                return option

        # ...then the default config
        if self.airflow_defaults.has_option(section, key) or 'fallback' in kwargs:
            return expand_env_var(
                self.airflow_defaults.get(section, key, **kwargs))

        else:
            log.warning(
                "section/key [%s/%s] not found in config", section, key
            )

            raise AirflowConfigException(
                "section/key [{section}/{key}] not found "
                "in config".format(section=section, key=key))

    def getimport(self, section, key, **kwargs):
        """
        Reads options, imports the full qualified name, and returns the object.
        In case of failure, it throws an exception a clear message with the key aad the section names
        :return: The object or None, if the option is empty
        """
        full_qualified_path = conf.get(section=section, key=key, **kwargs)
        if not full_qualified_path:
            return None

        try:
            return import_string(full_qualified_path)
        except ImportError as e:
            log.error(e)
            raise AirflowConfigException(
                'The object could not be loaded. Please check "{key}" key in "{section}" section. '
                'Current value: "{full_qualified_path}".'.format(
                    key=key, section=section, full_qualified_path=full_qualified_path)
            )

    def getboolean(self, section, key, **kwargs):
        val = str(self.get(section, key, **kwargs)).lower().strip()
        if '#' in val:
            val = val.split('#')[0].strip()
        if val in ('t', 'true', '1'):
            return True
        elif val in ('f', 'false', '0'):
            return False
        else:
            raise ValueError(
                'The value for configuration option "{}:{}" is not a '
                'boolean (received "{}").'.format(section, key, val))

    def getint(self, section, key, **kwargs):
        return int(self.get(section, key, **kwargs))

    def getfloat(self, section, key, **kwargs):
        return float(self.get(section, key, **kwargs))

    def read(self, filenames, **kwargs):
        super(AirflowConfigParser, self).read(filenames, **kwargs)
        self._validate()

    def read_dict(self, *args, **kwargs):
        super(AirflowConfigParser, self).read_dict(*args, **kwargs)
        self._validate()

    def has_option(self, section, option):
        try:
            # Using self.get() to avoid reimplementing the priority order
            # of config variables (env, config, cmd, defaults)
            # UNSET to avoid logging a warning about missing values
            self.get(section, option, fallback=_UNSET)
            return True
        except (NoOptionError, NoSectionError):
            return False

    def remove_option(self, section, option, remove_default=True):
        """
        Remove an option if it exists in config from a file or
        default config. If both of config have the same option, this removes
        the option in both configs unless remove_default=False.
        """
        if super(AirflowConfigParser, self).has_option(section, option):
            super(AirflowConfigParser, self).remove_option(section, option)

        if self.airflow_defaults.has_option(section, option) and remove_default:
            self.airflow_defaults.remove_option(section, option)

    def getsection(self, section):
        """
        Returns the section as a dict. Values are converted to int, float, bool
        as required.

        :param section: section from the config
        :rtype: dict
        """
        if (section not in self._sections and
                section not in self.airflow_defaults._sections):
            return None

        _section = copy.deepcopy(self.airflow_defaults._sections[section])

        if section in self._sections:
            _section.update(copy.deepcopy(self._sections[section]))

        section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper())
        for env_var in sorted(os.environ.keys()):
            if env_var.startswith(section_prefix):
                key = env_var.replace(section_prefix, '')
                if key.endswith("_CMD"):
                    key = key[:-4]
                key = key.lower()
                _section[key] = self._get_env_var_option(section, key)

        for key, val in iteritems(_section):
            try:
                val = int(val)
            except ValueError:
                try:
                    val = float(val)
                except ValueError:
                    if val.lower() in ('t', 'true'):
                        val = True
                    elif val.lower() in ('f', 'false'):
                        val = False
            _section[key] = val
        return _section

    def write(self, fp, space_around_delimiters=True):
        # This is based on the configparser.RawConfigParser.write method code to add support for
        # reading options from environment variables.
        if space_around_delimiters:
            d = " {} ".format(self._delimiters[0])  # type: ignore
        else:
            d = self._delimiters[0]  # type: ignore
        if self._defaults:
            self._write_section(fp, self.default_section, self._defaults.items(), d)  # type: ignore
        for section in self._sections:
            self._write_section(fp, section, self.getsection(section).items(), d)  # type: ignore

    def as_dict(
            self, display_source=False, display_sensitive=False, raw=False,
            include_env=True, include_cmds=True, include_secret=True
    ):
        """
        Returns the current configuration as an OrderedDict of OrderedDicts.

        :param display_source: If False, the option value is returned. If True,
            a tuple of (option_value, source) is returned. Source is either
            'airflow.cfg', 'default', 'env var', or 'cmd'.
        :type display_source: bool
        :param display_sensitive: If True, the values of options set by env
            vars and bash commands will be displayed. If False, those options
            are shown as '< hidden >'
        :type display_sensitive: bool
        :param raw: Should the values be output as interpolated values, or the
            "raw" form that can be fed back in to ConfigParser
        :type raw: bool
        :param include_env: Should the value of configuration from AIRFLOW__
            environment variables be included or not
        :type include_env: bool
        :param include_cmds: Should the result of calling any *_cmd config be
            set (True, default), or should the _cmd options be left as the
            command to run (False)
        :type include_cmds: bool
        :param include_secret: Should the result of calling any *_secret config be
            set (True, default), or should the _secret options be left as the
            path to get the secret from (False)
        :type include_secret: bool
        :return: Dictionary, where the key is the name of the section and the content is
            the dictionary with the name of the parameter and its value.
        """
        cfg = {}
        configs = [
            ('default', self.airflow_defaults),
            ('airflow.cfg', self),
        ]

        for (source_name, config) in configs:
            for section in config.sections():
                sect = cfg.setdefault(section, OrderedDict())
                for (k, val) in config.items(section=section, raw=raw):
                    if display_source:
                        val = (val, source_name)
                    sect[k] = val

        # add env vars and overwrite because they have priority
        if include_env:
            for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
                try:
                    _, section, key = ev.split('__', 2)
                    opt = self._get_env_var_option(section, key)
                except ValueError:
                    continue
                if not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE':
                    opt = '< hidden >'
                elif raw:
                    opt = opt.replace('%', '%%')
                if display_source:
                    opt = (opt, 'env var')

                section = section.lower()
                # if we lower key for kubernetes_environment_variables section,
                # then we won't be able to set any Airflow environment
                # variables. Airflow only parse environment variables starts
                # with AIRFLOW_. Therefore, we need to make it a special case.
                if section != 'kubernetes_environment_variables':
                    key = key.lower()
                cfg.setdefault(section, OrderedDict()).update({key: opt})

        # add bash commands
        if include_cmds:
            for (section, key) in self.sensitive_config_values:
                opt = self._get_cmd_option(section, key)
                if opt:
                    if not display_sensitive:
                        opt = '< hidden >'
                    if display_source:
                        opt = (opt, 'cmd')
                    elif raw:
                        opt = opt.replace('%', '%%')
                    cfg.setdefault(section, OrderedDict()).update({key: opt})
                    del cfg[section][key + '_cmd']

        # add config from secret backends
        if include_secret:
            for (section, key) in self.sensitive_config_values:
                opt = self._get_secret_option(section, key)
                if opt:
                    if not display_sensitive:
                        opt = '< hidden >'
                    if display_source:
                        opt = (opt, 'secret')
                    elif raw:
                        opt = opt.replace('%', '%%')
                    cfg.setdefault(section, OrderedDict()).update({key: opt})
                    del cfg[section][key + '_secret']

        return cfg

    def load_test_config(self):
        """
        Load the unit test configuration.

        Note: this is not reversible.
        """
        # override any custom settings with defaults
        log.info("Overriding settings with defaults from %s", DEFAULT_CONFIG_FILE_PATH)
        self.read_string(parameterized_config(DEFAULT_CONFIG))
        # then read test config
        log.info("Reading default test configuration from %s", TEST_CONFIG_FILE_PATH)
        self.read_string(parameterized_config(TEST_CONFIG))
        # then read any "custom" test settings
        log.info("Reading test configuration from %s", TEST_CONFIG_FILE)
        self.read(TEST_CONFIG_FILE)

    def _warn_deprecate(self, section, key, deprecated_name):
        warnings.warn(
            'The {old} option in [{section}] has been renamed to {new} - the old '
            'setting has been used, but please update your config.'.format(
                old=deprecated_name,
                new=key,
                section=section,
            ),
            DeprecationWarning,
            stacklevel=3,
        )
コード例 #28
0
def new_plugin_wizard(directory=None):
    """
    Start the wizard to create a new plugin in the current working directory.
    """
    if directory is None:
        print(
            "This wizard will create a new plugin for you in the current directory."
        )
        directory = os.getcwd()
    else:
        print("This wizard will create a new plugin for you in '%s'." %
              directory)

    if os.path.exists(directory) and not os.path.isdir(directory):
        print("Error: The path '%s' exists but it isn't a directory" %
              directory)
        sys.exit(1)

    name = ask("What should the name of your new plugin be?",
               validation_regex=r'^[a-zA-Z][a-zA-Z0-9 _-]*$').strip()
    module_name = name.lower().replace(' ', '_')
    directory_name = name.lower().replace(' ', '-')
    class_name = "".join([s.capitalize() for s in name.lower().split(' ')])

    description = ask(
        "What may I use as a short (one-line) description of your plugin?")
    if PY2:
        default_python_version = "2+"
    else:
        default_python_version = "3"
    python_version = ask(
        "Which python version will your plugin work with? 2, 2+ or 3? I will default to "
        "{version} if you leave this blank.".format(
            version=default_python_version),
        valid_responses=['2', '2+', '3', ''])
    if python_version.strip() == "":
        python_version = default_python_version
    errbot_min_version = ask(
        "Which minimum version of errbot will your plugin work with? "
        "Leave blank to support any version or input CURRENT to select the "
        "current version (%s)" % VERSION).strip()
    if errbot_min_version.upper() == "CURRENT":
        errbot_min_version = VERSION
    errbot_max_version = ask(
        "Which maximum version of errbot will your plugin work with? "
        "Leave blank to support any version or input CURRENT to select the "
        "current version (%s)" % VERSION).strip()
    if errbot_max_version.upper() == "CURRENT":
        errbot_max_version = VERSION

    plug = ConfigParser()
    plug["Core"] = {
        "Name": name,
        "Module": module_name,
    }
    plug["Documentation"] = {
        "Description": description,
    }
    plug["Python"] = {
        "Version": python_version,
    }
    plug["Errbot"] = {}
    if errbot_min_version != "":
        plug["Errbot"]["Min"] = errbot_min_version
    if errbot_max_version != "":
        plug["Errbot"]["Max"] = errbot_max_version

    plugin_path = directory
    plugfile_path = os.path.join(plugin_path, module_name + ".plug")
    pyfile_path = os.path.join(plugin_path, module_name + ".py")

    try:
        if PY3 or (PY2 and not os.path.isdir(plugin_path)):
            os.makedirs(plugin_path, mode=0o700)
    except IOError as e:
        if e.errno != errno.EEXIST:
            raise

    if os.path.exists(plugfile_path) or os.path.exists(pyfile_path):
        ask(
            "Warning: A plugin with this name was already found at {path}\n"
            "If you continue, these will be overwritten.\n"
            "Press Ctrl+C to abort now or type in 'overwrite' to confirm overwriting of these files."
            "".format(path=os.path.join(directory, module_name +
                                        ".{py,plug}")),
            valid_responses=["overwrite"],
        )

    with open(plugfile_path, 'w') as f:
        plug.write(f)

    with open(pyfile_path, 'w') as f:
        f.write(render_plugin(locals()))

    print("Success! You'll find your new plugin at '%s'" % plugfile_path)
    print(
        "(Don't forget to include a LICENSE file if you are going to publish your plugin)"
    )
コード例 #29
0
class GUIPrefsReader(object):
    '''
    Reader for GUI prefs. To save re inputting every time 
    '''

    PREFS_SEC = 'prefs'
    GUI_PREFS = '../conf/gui.prefs'

    def __init__(self):
        '''
        Constructor
        '''
        self.dvalue = None
        self.dselect = 'dest'
        #v:x111|MYGROUP, myconf.conf, 2193, 2013-01-01, 2013-01-02
        self.plist = ('lgvalue', 'uconf', 'epsg', 'fd', 'td')

        self.cp = ConfigParser()
        self.fn = os.path.join(os.path.dirname(__file__), self.GUI_PREFS)
        with codecs.open(self.fn, 'r', 'utf-8') as cf:
            self.cp.readfp(cf)

    def read(self):
        '''Read stored DS value and return this and its matching params'''
        try:
            with codecs.open(self.fn, 'r', 'utf-8') as cf:
                self.cp.readfp(cf)
            self.dvalue = self.cp.get(self.PREFS_SEC, self.dselect)
            if LU.assessNone(self.dvalue) is None:
                return (None, ) * (len(self.plist) + 1)
        except NoSectionError as nse:
            #if no sec init sec and opt and ret nones
            ldslog.warn('Error getting GUI prefs section :: ' + str(nse))
            if not self._initSection(self.PREFS_SEC):
                raise
            self._initOption(self.PREFS_SEC, self.dselect)
            return (None, ) * (len(self.plist) + 1)
        except NoOptionError as noe:
            #if no opt init opt and ret nones
            ldslog.warn('Error getting GUI prefs :: ' + str(noe))
            if not self._initOption(self.PREFS_SEC, self.dselect):
                raise
            return (None, ) * (len(self.plist) + 1)
        #if dval is okay ret it and res of a read of that sec
        return (self.dvalue, ) + self.readsec(self.dvalue)

    def readall(self):
        '''Reads entire grp into dict'''
        gpra = {}
        secs = self.cp.sections()
        secs.remove(self.PREFS_SEC)
        for sec in secs:
            gpra[sec] = self.readsec(sec)
        return gpra

    def getDestinations(self):
        return self.cp.sections()

    def readsec(self, section):
        #options per DS type
        rlist = ()

        for p in self.plist:
            try:
                rlist += (self.cp.get(section, p), )
            except NoSectionError as nse:
                #if not ds sec init sec then init opt
                ldslog.warn('Error getting GUI ' + section + ' :: ' + str(nse))
                if not self._initSection(section):
                    raise
                self._initOption(section, p)
                rlist += (None, )
            except NoOptionError as noe:
                #if no opt init the opt
                ldslog.warn('Error getting GUI ' + section + ' pref, ' + p +
                            ' :: ' + str(noe))
                if not self._initOption(section, p):
                    raise
                rlist += (None, )
        return rlist

    def writeline(self, field, value):
        #not the best solution since depends on a current gpr and a recent read/write.
        if self.dvalue:
            self.writesecline(self.dvalue, field, value)

    def writesecline(self, section, field, value):
        try:
            self.cp.set(section, field, value if LU.assessNone(value) else '')
            with codecs.open(self.fn, 'w', 'utf-8') as configfile:
                self.cp.write(configfile)
            ldslog.debug(str(section) + ':' + str(field) + '=' + str(value))
        except Exception as e:
            ldslog.warn('Problem writing GUI prefs. {} - sfv={}'.format(
                e, (section, field, value)))

    def write(self, rlist):
        self.dvalue = rlist[0]

        if self.cp.has_section(self.PREFS_SEC):
            self.cp.set(self.PREFS_SEC, self.dselect, self.dvalue)
        else:
            self.cp.add_section(self.PREFS_SEC)
            self.cp.set(self.PREFS_SEC, self.dselect, self.dvalue)

        for pr in zip(self.plist, rlist[1:]):
            if not self.cp.has_section(self.dvalue):
                self.cp.add_section(self.dvalue)
            try:
                if LU.assessNone(pr[1]):
                    self.cp.set(self.dvalue, pr[0], pr[1])
                    ldslog.debug(self.dvalue + ':' + pr[0] + '=' + pr[1])
            except Exception as e:
                ldslog.warn('Problem writing GUI prefs. ' + str(e))
        with codecs.open(self.fn, 'w', 'utf-8') as configfile:
            self.cp.write(configfile)

    def _initSection(self, section):
        checksec = LU.standardiseDriverNames(section)
        if checksec:
            self.cp.add_section(checksec)
            return True
        elif section == self.PREFS_SEC:
            self.cp.add_section(section)
            return True
        return False

    def _initOption(self, section, option):
        if option in self.plist + (self.dselect, ):
            self.writesecline(section, option, None)
            return True
        return False

    @classmethod
    def validate():
        '''Make sure a guipref file is valid, check pref points to alt least one valid DST'''
        filename = os.path.join(os.path.dirname(__file__),
                                GUIPrefsReader.GUI_PREFS)
        gp = GUIPrefsReader(filename)
        #validate UC check it has a valid dest named and configured
        p = gp.cp.get('prefs', 'dest')
        return gp.cp.has_section(p)
コード例 #30
0
ファイル: config.py プロジェクト: javyxu/space-time-gis
class SpacetimeGISConfigParser(ConfigParser):
    def __init__(self, default_config=None, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.spacetimegis_defaults = ConfigParser(*args, **kwargs)
        if not default_config:
            self.spacetimegis_defaults.read_string(default_config)

    @staticmethod
    def _env_var_name(section, key):
        return 'SPACETIMEGIS__{S}__{K}'.format(S=section.upper(),
                                               K=key.upper())

    def _get_env_var_option(self, section, key):
        # must have format SPACETIMEGIS__{SECTION}__{KEY} (note double underscore)
        env_var = self._env_var_name(section, key)
        if env_var in os.environ:
            return expand_env_var(os.environ[env_var])

    def get(self, section, key, **kwargs):
        section = str(section).lower()
        key = str(key).lower()

        # first check environment variables
        option = self._get_env_var_option(section, key)
        if option is not None:
            return option

        # ...then the config file
        if super().has_option(section, key):
            # Use the parent's methods to get the actual config here to be able to
            # separate the config from default config.
            return expand_env_var(super().get(section, key, **kwargs))

        # ...then the default config
        if self.spacetimegis_defaults.has_option(section,
                                                 key) or 'fallback' in kwargs:
            return expand_env_var(
                self.spacetimegis_defaults.get(section, key, **kwargs))

    def getboolean(self, section, key, **kwargs):
        val = str(self.get(section, key, **kwargs)).lower().strip()
        if '#' in val:
            val = val.split('#')[0].strip()
        if val in ('t', 'true', '1'):
            return True
        elif val in ('f', 'false', '0'):
            return False
        else:
            # logger.writelog(LogLevel.error,
            #     'The value for configuration option "{}:{}" is not a '
            #     'boolean (received "{}").'.format(section, key, val))
            raise ValueError(
                'The value for configuration option "{}:{}" is not a '
                'boolean (received "{}").'.format(section, key, val))

    def getint(self, section, key, **kwargs):
        return int(self.get(section, key, **kwargs))

    def getfloat(self, section, key, **kwargs):
        return float(self.get(section, key, **kwargs))

    def read(self, filenames, **kwargs):
        super().read(filenames, **kwargs)

    def read_dict(self, *args, **kwargs):
        super().read_dict(*args, **kwargs)

    def has_option(self, section, option):
        try:
            # Using self.get() to avoid reimplementing the priority order
            # of config variables (env, config, cmd, defaults)
            # UNSET to avoid logging a warning about missing values
            self.get(section, option, fallback=_UNSET)
            return True
        except NoOptionError:
            return False

    def remove_option(self, section, option, remove_default=True):
        """
        Remove an option if it exists in config from a file or
        default config. If both of config have the same option, this removes
        the option in both configs unless remove_default=False.
        """
        if super().has_option(section, option):
            super().remove_option(section, option)

        if self.spacetimegis_defaults.has_option(section,
                                                 option) and remove_default:
            self.spacetimegis_defaults.remove_option(section, option)

    def getsection(self, section):
        """
        Returns the section as a dict. Values are converted to int, float, bool
        as required.
        :param section: section from the config
        :rtype: dict
        """
        if (section not in self._sections
                and section not in self.spacetimegis_defaults._sections):
            return None

        _section = copy.deepcopy(self.spacetimegis_defaults._sections[section])

        if section in self._sections:
            _section.update(copy.deepcopy(self._sections[section]))

        section_prefix = 'SPACETIMEGIS__{S}__'.format(S=section.upper())
        for env_var in sorted(os.environ.keys()):
            if env_var.startswith(section_prefix):
                key = env_var.replace(section_prefix, '').lower()
                _section[key] = self._get_env_var_option(section, key)

        for key, val in iteritems(_section):
            try:
                val = int(val)
            except ValueError:
                try:
                    val = float(val)
                except ValueError:
                    if val.lower() in ('t', 'true'):
                        val = True
                    elif val.lower() in ('f', 'false'):
                        val = False
            _section[key] = val
        return _section

    def as_dict(self,
                display_source=False,
                display_sensitive=False,
                raw=False):
        """
        Returns the current configuration as an OrderedDict of OrderedDicts.
        :param display_source: If False, the option value is returned. If True,
            a tuple of (option_value, source) is returned. Source is either
            'airflow.cfg', 'default', 'env var', or 'cmd'.
        :type display_source: bool
        :param display_sensitive: If True, the values of options set by env
            vars and bash commands will be displayed. If False, those options
            are shown as '< hidden >'
        :type display_sensitive: bool
        :param raw: Should the values be output as interpolated values, or the
            "raw" form that can be fed back in to ConfigParser
        :type raw: bool
        """
        cfg = {}
        configs = [
            ('default', self.spacetimegis_defaults),
            ('spacetimegis.cfg', self),
        ]

        for (source_name, config) in configs:
            for section in config.sections():
                sect = cfg.setdefault(section, OrderedDict())
                for (k, val) in config.items(section=section, raw=raw):
                    if display_source:
                        val = (val, source_name)
                    sect[k.upper()] = val

        # add env vars and overwrite because they have priority
        for ev in [ev for ev in os.environ if ev.startswith('SPACETIMEGIS__')]:
            try:
                _, section, key = ev.split('__')
                opt = self._get_env_var_option(section, key)
            except ValueError:
                continue
            if not display_sensitive and ev != 'SPACETIMEGIS__CORE__UNIT_TEST_MODE':
                opt = '< hidden >'
            elif raw:
                opt = opt.replace('%', '%%')
            if display_source:
                opt = (opt, 'env var')
            cfg.setdefault(section.lower(),
                           OrderedDict()).update({key.lower(): opt})

        return cfg

    def as_all_dict(self,
                    display_source=False,
                    display_sensitive=False,
                    raw=False):
        tmp = self.as_dict(display_source, display_sensitive, raw)
        all_cfg = {}
        for val in tmp.values():
            if not all_cfg:
                all_cfg = val.copy()
            else:
                all_cfg.update(val)
        return all_cfg
コード例 #31
0
class AirflowConfigParser(ConfigParser):

    # These configuration elements can be fetched as the stdout of commands
    # following the "{section}__{name}__cmd" pattern, the idea behind this
    # is to not store password on boxes in text files.
    as_command_stdout = {('core', 'sql_alchemy_conn'), ('core', 'fernet_key'),
                         ('celery', 'broker_url'),
                         ('celery', 'result_backend')}

    def __init__(self, default_config=None, *args, **kwargs):
        super(AirflowConfigParser, self).__init__(*args, **kwargs)

        self.defaults = ConfigParser(*args, **kwargs)
        if default_config is not None:
            self.defaults.read_string(default_config)

        self.is_validated = False

    def _validate(self):
        if (self.get("core", "executor") != 'SequentialExecutor'
                and "sqlite" in self.get('core', 'sql_alchemy_conn')):
            raise AirflowConfigException(
                "error: cannot use sqlite with the {}".format(
                    self.get('core', 'executor')))

        elif (self.getboolean("webserver", "authenticate") and self.get(
                "webserver", "owner_mode") not in ['user', 'ldapgroup']):
            raise AirflowConfigException(
                "error: owner_mode option should be either "
                "'user' or 'ldapgroup' when filtering by owner is set")

        elif (self.getboolean("webserver", "authenticate")
              and self.get("webserver", "owner_mode").lower() == 'ldapgroup'
              and self.get("webserver", "auth_backend") !=
              ('airflow.contrib.auth.backends.ldap_auth')):
            raise AirflowConfigException(
                "error: attempt at using ldapgroup "
                "filtering without using the Ldap backend")

        self.is_validated = True

    def _get_env_var_option(self, section, key):
        # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
        env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())
        if env_var in os.environ:
            return expand_env_var(os.environ[env_var])

    def _get_cmd_option(self, section, key):
        fallback_key = key + '_cmd'
        # if this is a valid command key...
        if (section, key) in self.as_command_stdout:
            if super(AirflowConfigParser, self) \
                    .has_option(section, fallback_key):
                command = super(AirflowConfigParser, self) \
                    .get(section, fallback_key)
                return run_command(command)

    def get(self, section, key, **kwargs):
        section = str(section).lower()
        key = str(key).lower()

        # first check environment variables
        option = self._get_env_var_option(section, key)
        if option is not None:
            return option

        # ...then the config file
        if super(AirflowConfigParser, self).has_option(section, key):
            # Use the parent's methods to get the actual config here to be able to
            # separate the config from default config.
            return expand_env_var(
                super(AirflowConfigParser, self).get(section, key, **kwargs))

        # ...then commands
        option = self._get_cmd_option(section, key)
        if option:
            return option

        # ...then the default config
        if self.defaults.has_option(section, key):
            return expand_env_var(self.defaults.get(section, key, **kwargs))

        else:
            log.warning(
                "section/key [{section}/{key}] not found in config".format(
                    **locals()))

            raise AirflowConfigException(
                "section/key [{section}/{key}] not found "
                "in config".format(**locals()))

    def getboolean(self, section, key):
        val = str(self.get(section, key)).lower().strip()
        if '#' in val:
            val = val.split('#')[0].strip()
        if val.lower() in ('t', 'true', '1'):
            return True
        elif val.lower() in ('f', 'false', '0'):
            return False
        else:
            raise AirflowConfigException(
                'The value for configuration option "{}:{}" is not a '
                'boolean (received "{}").'.format(section, key, val))

    def getint(self, section, key):
        return int(self.get(section, key))

    def getfloat(self, section, key):
        return float(self.get(section, key))

    def read(self, filenames):
        super(AirflowConfigParser, self).read(filenames)
        self._validate()

    def has_option(self, section, option):
        try:
            # Using self.get() to avoid reimplementing the priority order
            # of config variables (env, config, cmd, defaults)
            self.get(section, option)
            return True
        except AirflowConfigException:
            return False

    def remove_option(self, section, option, remove_default=True):
        """
        Remove an option if it exists in config from a file or
        default config. If both of config have the same option, this removes
        the option in both configs unless remove_default=False.
        """
        if super(AirflowConfigParser, self).has_option(section, option):
            super(AirflowConfigParser, self).remove_option(section, option)

        if self.defaults.has_option(section, option) and remove_default:
            self.defaults.remove_option(section, option)

    def getsection(self, section):
        """
        Returns the section as a dict. Values are converted to int, float, bool
        as required.
        :param section: section from the config
        :return: dict
        """
        if section not in self._sections and section not in self.defaults._sections:
            return None

        _section = copy.deepcopy(self.defaults._sections[section])

        if section in self._sections:
            _section.update(copy.deepcopy(self._sections[section]))

        for key, val in iteritems(_section):
            try:
                val = int(val)
            except ValueError:
                try:
                    val = float(val)
                except ValueError:
                    if val.lower() in ('t', 'true'):
                        val = True
                    elif val.lower() in ('f', 'false'):
                        val = False
            _section[key] = val
        return _section

    def as_dict(self, display_source=False, display_sensitive=False):
        """
        Returns the current configuration as an OrderedDict of OrderedDicts.
        :param display_source: If False, the option value is returned. If True,
            a tuple of (option_value, source) is returned. Source is either
            'airflow.cfg' or 'default'.
        :type display_source: bool
        :param display_sensitive: If True, the values of options set by env
            vars and bash commands will be displayed. If False, those options
            are shown as '< hidden >'
        :type display_sensitive: bool
        """
        cfg = copy.deepcopy(self.defaults._sections)
        cfg.update(copy.deepcopy(self._sections))

        # remove __name__ (affects Python 2 only)
        for options in cfg.values():
            options.pop('__name__', None)

        # add source
        if display_source:
            for section in cfg:
                for k, v in cfg[section].items():
                    cfg[section][k] = (v, 'airflow config')

        # add env vars and overwrite because they have priority
        for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
            try:
                _, section, key = ev.split('__')
                opt = self._get_env_var_option(section, key)
            except ValueError:
                opt = None
            if opt:
                if (not display_sensitive
                        and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'):
                    opt = '< hidden >'
                if display_source:
                    opt = (opt, 'env var')
                cfg.setdefault(section.lower(),
                               OrderedDict()).update({key.lower(): opt})

        # add bash commands
        for (section, key) in self.as_command_stdout:
            opt = self._get_cmd_option(section, key)
            if opt:
                if not display_sensitive:
                    opt = '< hidden >'
                if display_source:
                    opt = (opt, 'bash cmd')
                cfg.setdefault(section, OrderedDict()).update({key: opt})

        return cfg

    def load_test_config(self):
        """
        Load the unit test configuration.

        Note: this is not reversible.
        """
        # override any custom settings with defaults
        self.defaults.read_string(parameterized_config(DEFAULT_CONFIG))
        # then read test config
        self.read_string(parameterized_config(TEST_CONFIG))
        # then read any "custom" test settings
        self.read(TEST_CONFIG_FILE)
コード例 #32
0
ファイル: ReadConfig.py プロジェクト: josephramsay/LDS
class GUIPrefsReader(object):
    '''
    Reader for GUI prefs. To save re inputting every time 
    '''

    PREFS_SEC = 'prefs'
    GUI_PREFS = '../conf/gui.prefs'
    
    def __init__(self):
        '''
        Constructor
        '''
        self.dvalue = None
        self.dselect = 'dest'
        #v:x111|MYGROUP, myconf.conf, 2193, 2013-01-01, 2013-01-02
        self.plist = ('lgvalue','uconf','epsg','fd','td')
        
        self.cp = ConfigParser()
        self.fn = os.path.join(os.path.dirname(__file__),self.GUI_PREFS)
        with codecs.open(self.fn,'r','utf-8') as cf:
            self.cp.readfp(cf)
        
    def read(self):
        '''Read stored DS value and return this and its matching params'''
        try:
            with codecs.open(self.fn, 'r','utf-8') as cf:
                self.cp.readfp(cf)
            self.dvalue = self.cp.get(self.PREFS_SEC, self.dselect) 
            if LU.assessNone(self.dvalue) is None:
                return (None,)*(len(self.plist)+1)
        except NoSectionError as nse:
            #if no sec init sec and opt and ret nones
            ldslog.warn('Error getting GUI prefs section :: '+str(nse))
            if not self._initSection(self.PREFS_SEC):
                raise
            self._initOption(self.PREFS_SEC,self.dselect)
            return (None,)*(len(self.plist)+1)
        except NoOptionError as noe:
            #if no opt init opt and ret nones
            ldslog.warn('Error getting GUI prefs :: '+str(noe))
            if not self._initOption(self.PREFS_SEC,self.dselect):
                raise
            return (None,)*(len(self.plist)+1)
        #if dval is okay ret it and res of a read of that sec
        return (self.dvalue,)+self.readsec(self.dvalue)
    
    
    def readall(self):
        '''Reads entire grp into dict'''
        gpra = {}
        secs = self.cp.sections()
        secs.remove(self.PREFS_SEC)
        for sec in secs:
            gpra[sec] = self.readsec(sec)
        return gpra
        
    def getDestinations(self):
        return self.cp.sections()
    
    def readsec(self,section):
        #options per DS type
        rlist = ()
        
        for p in self.plist:
            try:
                rlist += (self.cp.get(section, p),)
            except NoSectionError as nse:
                #if not ds sec init sec then init opt
                ldslog.warn('Error getting GUI '+section+' :: '+str(nse))
                if not self._initSection(section):
                    raise
                self._initOption(section, p)
                rlist += (None,)
            except NoOptionError as noe:
                #if no opt init the opt
                ldslog.warn('Error getting GUI '+section+' pref, '+p+' :: '+str(noe))
                if not self._initOption(section,p):
                    raise
                rlist += (None,)
        return rlist
    
    def writeline(self,field,value):
        #not the best solution since depends on a current gpr and a recent read/write. 
        if self.dvalue:
            self.writesecline(self.dvalue,field,value)
        
    def writesecline(self,section,field,value):
        try:            
            self.cp.set(section,field,value if LU.assessNone(value) else '')
            with codecs.open(self.fn, 'w','utf-8') as configfile:
                self.cp.write(configfile)
            ldslog.debug(str(section)+':'+str(field)+'='+str(value))                                                                                        
        except Exception as e:
            ldslog.warn('Problem writing GUI prefs. {} - sfv={}'.format(e,(section,field,value)))
            
            
    def write(self,rlist):
        self.dvalue = rlist[0]
        
        if self.cp.has_section(self.PREFS_SEC):
            self.cp.set(self.PREFS_SEC,self.dselect,self.dvalue)
        else:
            self.cp.add_section(self.PREFS_SEC)
            self.cp.set(self.PREFS_SEC,self.dselect,self.dvalue)
            
        for pr in zip(self.plist,rlist[1:]):
            if not self.cp.has_section(self.dvalue):
                self.cp.add_section(self.dvalue)
            try:
                if LU.assessNone(pr[1]):         
                    self.cp.set(self.dvalue,pr[0],pr[1])
                    ldslog.debug(self.dvalue+':'+pr[0]+'='+pr[1])                                                                                        
            except Exception as e:
                ldslog.warn('Problem writing GUI prefs. '+str(e))
        with codecs.open(self.fn, 'w','utf-8') as configfile:
            self.cp.write(configfile)
                      
    
    def _initSection(self,section):
        checksec = LU.standardiseDriverNames(section)
        if checksec:
            self.cp.add_section(checksec)
            return True
        elif section == self.PREFS_SEC:
            self.cp.add_section(section)
            return True
        return False
            
    def _initOption(self,section,option):
        if option in self.plist+(self.dselect,):
            self.writesecline(section,option,None)
            return True
        return False
    
    @classmethod
    def validate():
        '''Make sure a guipref file is valid, check pref points to alt least one valid DST'''
        filename = os.path.join(os.path.dirname(__file__),GUIPrefsReader.GUI_PREFS)
        gp = GUIPrefsReader(filename)
        #validate UC check it has a valid dest named and configured
        p = gp.cp.get('prefs', 'dest')
        return gp.cp.has_section(p)
コード例 #33
0
ファイル: ReadConfig.py プロジェクト: josephramsay/LDS
class MainFileReader(object):
    '''
    Config file reader/writer
    '''
    LDSN = 'LDS'
    PROXY = 'Proxy'
    MISC = 'Misc'
    
    DEFAULT_MF = 'template.conf'


    def __init__(self,cfpath=None,use_defaults=True):
        '''
        Constructor
        '''
        
        from lds.DataStore import DataStore
        self.driverconfig = {i:() for i in DataStore.DRIVER_NAMES}
            
        self.use_defaults = use_defaults
        #if we dont give the constructor a file path is uses the template file which will fill in any default values missed in the user conf
        #but if cpath is requested and doesn't exist we initialise it, otherwise you end up trying to overwrite the template
        if cfpath is None:
            self.filename = LU.standardiseUserConfigName(self.DEFAULT_MF)
        else:
            #Standardise since there is no guarantee cfpath is going to be in the reqd format to start with (since its a user entry)
            self.filename = LU.standardiseUserConfigName(cfpath)

        
        self.cp = None
        self.initMainFile()
        self.fn = re.search('(.+)\.conf',os.path.basename(self.filename)).group(1)
        
    def __str__(self):
        return self.filename
        
    def initMainFile(self,template=''):
        '''Open and populate a new config file with 'template' else just touch it. Then call the reader'''
        if self.filename.split('/')[-1] != self.DEFAULT_MF:
            mode = 'a' if template is '' else 'w'
            with codecs.open(self.filename, mode, 'utf-8') as f:
                f.write(template)
        self._readConfigFile(self.filename)    
     
    def hasSection(self,secname):
        return secname in self.getSections()
    
    def getSections(self):
        '''List of sections (layernames/datasources)'''
        return self.cp.sections()
        
    def _readConfigFile(self,fn):
        '''Reads named config file'''
        #Split off so you can override the config file on the same reader object if needed
        self.cp = ConfigParser()
        with codecs.open(fn,'r','utf-8') as cf:
            self.cp.readfp(cf)

    
    #database
    def readDriverConfig(self,dname):
        dnd = {'PostgreSQL':self.readPostgreSQLConfig,
               'MSSQLSpatial':self.readMSSQLConfig,
               'FileGDB':self.readFileGDBConfig,
               'SQLite':self.readSpatiaLiteConfig,
               'WFS':self.readWFSConfig,
               'Proxy':self.readProxyConfig,
               'Misc':self.readMiscConfig}
        return dnd[dname]()
        
    def readPostgreSQLConfig(self):
        '''PostgreSQL specific config file reader'''            
        from PostgreSQLDataStore import PostgreSQLDataStore as PG
        
        ref = self.fn+':PG. '
        usr = None
        pwd = None
        over = None
        epsg = None
        cql = None
        schema = None
        
        if self.use_defaults:
            host = "127.0.0.1"
            port = 5432
            dbname = "ldsincr"
            schema = "lds"
            config = "external"
        else:
            host = None
            port = None
            dbname = None
            config = None
        
        
        try:
            host = self.cp.get(PG.DRIVER_NAME, 'host')
        except NoSectionError:
            ldslog.warn(ref+"No PostgreSQL section")
            raise
            #return (None,)*10
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            port = self.cp.get(PG.DRIVER_NAME, 'port')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            dbname = self.cp.get(PG.DRIVER_NAME, 'dbname')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            schema = self.cp.get(PG.DRIVER_NAME, 'schema')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            usr = self.cp.get(PG.DRIVER_NAME, 'user')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            pwd = self.cp.get(PG.DRIVER_NAME, 'pass')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            config = self.cp.get(PG.DRIVER_NAME, 'config')
        except NoOptionError:
            ldslog.warn(ref+"No config preference specified, default to "+str(config))
        
        try:
            over = self.cp.get(PG.DRIVER_NAME, 'overwrite')
        except NoOptionError:
            ldslog.warn(ref+"Overwrite not specified, Setting to True")
            over = True
            
        try:
            epsg = self.cp.get(PG.DRIVER_NAME, 'epsg')
        except NoOptionError:
            ldslog.warn(ref+"EPSG not specified, default to none keeping existing SRS")
            epsg = True
            
        try: 
            cql = self.cp.get(PG.DRIVER_NAME, 'cql')
        except NoOptionError:
            ldslog.warn(ref+"No CQL Filter specified, fetching all results")
            cql = None
        
        return (host,port,dbname,schema,usr,pwd,over,config,epsg,cql)
    
    def readMSSQLConfig(self):
        '''MSSQL specific config file reader'''
        
        from MSSQLSpatialDataStore import MSSQLSpatialDataStore as MS
        
        ref = self.fn+':MS. '
        usr = None
        pwd = None
        epsg = None
        cql = None
        schema = None
        
        if self.use_defaults:
            odbc = "FreeTDS"
            server = "127.0.0.1\SQLExpress"
            dbname = "LDSINCR"
            trust = "yes"
            dsn = "LDSINCR"
            config = "external"
        else:
            odbc = None
            server = None
            dbname = None
            trust = None
            dsn = None
            config = None
             
        try:
            odbc = self.cp.get(MS.DRIVER_NAME, 'odbc')
        except NoSectionError:
            ldslog.warn(ref+"No MSSQLSpatial section")
            raise
            #return (None,)*11
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            server = self.cp.get(MS.DRIVER_NAME, 'server')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            dsn = self.cp.get(MS.DRIVER_NAME, 'dsn')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            trust = self.cp.get(MS.DRIVER_NAME, 'trust')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            dbname = self.cp.get(MS.DRIVER_NAME, 'dbname')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            schema = self.cp.get(MS.DRIVER_NAME, 'schema')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            usr = self.cp.get(MS.DRIVER_NAME, 'user')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
            
        try:
            pwd = self.cp.get(MS.DRIVER_NAME, 'pass')
        except NoOptionError as noe:
            ldslog.warn(ref+str(noe))
        
        try:
            config = self.cp.get(MS.DRIVER_NAME, 'config')
        except NoOptionError:
            ldslog.warn(ref+"No config preference specified, default to "+str(config))
            
        try:
            epsg = self.cp.get(MS.DRIVER_NAME, 'epsg')
        except NoOptionError:
            ldslog.warn(ref+"EPSG not specified, default to None keeping existing SRS")
            
        try: 
            cql = self.cp.get(MS.DRIVER_NAME, 'cql')
        except NoOptionError:
            ldslog.warn(ref+"No CQL Filter specified, fetching all results")
            cql = None
        
        return (odbc,server,dsn,trust,dbname,schema,usr,pwd,config,epsg,cql)
    
    def readSpatiaLiteConfig(self):
        '''SpatiaLite specific config file reader'''
        from SpatiaLiteDataStore import SpatiaLiteDataStore as SL

        ref = self.fn+':SL. '
        epsg = None
        cql = None
        
        if self.use_defaults:
            lcfname = "~/LDSSLITE.sqlite3"
            config = "external"
        else:
            lcfname = None
            config = None
        
        try:
            lcfname = self.cp.get(SL.DRIVER_NAME, 'file')
        except NoSectionError:
            ldslog.warn(ref+"No SpatiaLite section")
            raise
            #return (None,)*4
        except NoOptionError:
            ldslog.warn(ref+"No DB name provided, default to "+str(lcfname))
            
        try:
            config = self.cp.get(SL.DRIVER_NAME, 'config')
        except NoOptionError:
            ldslog.warn(ref+"No config preference specified, default to "+str(config))
            
        try:
            epsg = self.cp.get(SL.DRIVER_NAME, 'epsg')
        except NoOptionError:
            ldslog.warn(ref+"EPSG not specified, default to "+str(epsg)+" keeping existing SRS")
            
        try: 
            cql = self.cp.get(SL.DRIVER_NAME, 'cql')
        except NoOptionError:
            ldslog.warn(ref+"No CQL Filter specified, fetching all results")
        
        return (lcfname,config,epsg,cql)
    
    def readFileGDBConfig(self):
        '''FileGDB specific config file reader'''
        from FileGDBDataStore import FileGDBDataStore as FG
        
        ref = self.fn+':FG. '
        epsg = None
        cql = None
        
        if self.use_defaults:
            lcfname = "~/LDSFGDB.gdb"
            config = "external"
        else:
            lcfname = None
            config = None

            
        try:
            lcfname = self.cp.get(FG.DRIVER_NAME, 'file')
        except NoSectionError:
            ldslog.warn(ref+"No FileGDB section")
            raise
            #return (None,)*4
        except NoOptionError:
            ldslog.warn(ref+"No DB name provided, default to "+str(lcfname))
            
        try:
            config = self.cp.get(FG.DRIVER_NAME, 'config')
        except NoOptionError:
            ldslog.warn(ref+"No config preference specified, default to "+str(config))
            
        try:
            epsg = self.cp.get(FG.DRIVER_NAME, 'epsg')
        except NoOptionError:
            ldslog.warn(ref+"EPSG not specified, default to "+str(epsg)+" none keeping existing SRS")
            
        try: 
            cql = self.cp.get(FG.DRIVER_NAME, 'cql')
        except NoOptionError:
            ldslog.warn(ref+"No CQL Filter specified, fetching all results")
        
        return (lcfname,config,epsg,cql)
        
    def readWFSConfig(self):
        '''Generic WFS config file reader'''
        '''Since this now keys on the driver name, WFS is read before LDS and LDS not at all, So...'''
        
        return self.readLDSConfig()
    
    def readLDSConfig(self):
        '''LDS specific config file reader'''
        #***HACK *** reimport NSE aliased
        from ConfigParser import NoSectionError as NSE
        #use_defaults determines whether we use default values. For a user config this may not be wise
        #since a user config is a custom file relying on the main config for absent values not last-resort defaults
        ref = self.fn+':LDS. '
        cql = None
        key = None
        
        #url = "http://data.linz.govt.nz/"
        #ver = "2.0.0"
        if self.use_defaults:
            url = "http://wfs.data.linz.govt.nz/"
            fmt = "GML2"
            svc = "WFS"
            ver = "1.0.0"
        else:
            url = None
            fmt = None
            svc = None
            ver = None
        
        try:
            url = self.cp.get(self.LDSN, 'url')
        #for some reason we cant use global NoSectionError in this function (even though its fine everywhere else) 
        except NSE:
            ldslog.warn(ref+"LDS: No LDS Section")
            return (None,)*6
        except NoOptionError:
            ldslog.warn(ref+"Default URL assumed ")

        
            
        try:   
            key = self.cp.get(self.LDSN, 'key') 
        except NoOptionError, NoSectionError:
            ldslog.error(ref+"Key required to connect to LDS...")
            raise
            
        try: 
            fmt = self.cp.get(self.LDSN, 'fmt')
        except NoOptionError:
            ldslog.warn(ref+"No output format specified")
        
        try: 
            svc = self.cp.get(self.LDSN, 'svc')
        except NoOptionError:
            ldslog.warn(ref+"No service type specified, default to "+str(svc))
        
        try: 
            ver = self.cp.get(self.LDSN, 'ver')
        except NoOptionError:
            ldslog.warn(ref+"No Version specified, assuming WFS and default to version "+str(ver))        
            
        try: 
            cql = self.cp.get(self.LDSN, 'cql')
        except NoOptionError:
            ldslog.warn(ref+"No CQL Filter specified, fetching all results")
            

        
        return (url,key,svc,ver,fmt,cql)
コード例 #34
0
class MainFileReader(object):
    '''
    Config file reader/writer
    '''
    LDSN = 'LDS'
    PROXY = 'Proxy'
    MISC = 'Misc'

    DEFAULT_MF = 'template.conf'

    def __init__(self, cfpath=None, use_defaults=True):
        '''
        Constructor
        '''

        from lds.DataStore import DataStore
        self.driverconfig = {i: () for i in DataStore.DRIVER_NAMES}

        self.use_defaults = use_defaults
        #if we dont give the constructor a file path is uses the template file which will fill in any default values missed in the user conf
        #but if cpath is requested and doesn't exist we initialise it, otherwise you end up trying to overwrite the template
        if cfpath is None:
            self.filename = LU.standardiseUserConfigName(self.DEFAULT_MF)
        else:
            #Standardise since there is no guarantee cfpath is going to be in the reqd format to start with (since its a user entry)
            self.filename = LU.standardiseUserConfigName(cfpath)

        self.cp = None
        self.initMainFile()
        self.fn = re.search('(.+)\.conf',
                            os.path.basename(self.filename)).group(1)

    def __str__(self):
        return self.filename

    def initMainFile(self, template=''):
        '''Open and populate a new config file with 'template' else just touch it. Then call the reader'''
        if self.filename.split('/')[-1] != self.DEFAULT_MF:
            mode = 'a' if template is '' else 'w'
            with codecs.open(self.filename, mode, 'utf-8') as f:
                f.write(template)
        self._readConfigFile(self.filename)

    def hasSection(self, secname):
        return secname in self.getSections()

    def getSections(self):
        '''List of sections (layernames/datasources)'''
        return self.cp.sections()

    def _readConfigFile(self, fn):
        '''Reads named config file'''
        #Split off so you can override the config file on the same reader object if needed
        self.cp = ConfigParser()
        with codecs.open(fn, 'r', 'utf-8') as cf:
            self.cp.readfp(cf)

    #database
    def readDriverConfig(self, dname):
        dnd = {
            'PostgreSQL': self.readPostgreSQLConfig,
            'MSSQLSpatial': self.readMSSQLConfig,
            'FileGDB': self.readFileGDBConfig,
            'SQLite': self.readSpatiaLiteConfig,
            'WFS': self.readWFSConfig,
            'Proxy': self.readProxyConfig,
            'Misc': self.readMiscConfig
        }
        return dnd[dname]()

    def readPostgreSQLConfig(self):
        '''PostgreSQL specific config file reader'''
        from PostgreSQLDataStore import PostgreSQLDataStore as PG

        ref = self.fn + ':PG. '
        usr = None
        pwd = None
        over = None
        epsg = None
        cql = None
        schema = None

        if self.use_defaults:
            host = "127.0.0.1"
            port = 5432
            dbname = "ldsincr"
            schema = "lds"
            config = "external"
        else:
            host = None
            port = None
            dbname = None
            config = None

        try:
            host = self.cp.get(PG.DRIVER_NAME, 'host')
        except NoSectionError:
            ldslog.warn(ref + "No PostgreSQL section")
            raise
            #return (None,)*10
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            port = self.cp.get(PG.DRIVER_NAME, 'port')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            dbname = self.cp.get(PG.DRIVER_NAME, 'dbname')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            schema = self.cp.get(PG.DRIVER_NAME, 'schema')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            usr = self.cp.get(PG.DRIVER_NAME, 'user')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            pwd = self.cp.get(PG.DRIVER_NAME, 'pass')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            config = self.cp.get(PG.DRIVER_NAME, 'config')
        except NoOptionError:
            ldslog.warn(ref + "No config preference specified, default to " +
                        str(config))

        try:
            over = self.cp.get(PG.DRIVER_NAME, 'overwrite')
        except NoOptionError:
            ldslog.warn(ref + "Overwrite not specified, Setting to True")
            over = True

        try:
            epsg = self.cp.get(PG.DRIVER_NAME, 'epsg')
        except NoOptionError:
            ldslog.warn(
                ref +
                "EPSG not specified, default to none keeping existing SRS")
            epsg = True

        try:
            cql = self.cp.get(PG.DRIVER_NAME, 'cql')
        except NoOptionError:
            ldslog.warn(ref + "No CQL Filter specified, fetching all results")
            cql = None

        return (host, port, dbname, schema, usr, pwd, over, config, epsg, cql)

    def readMSSQLConfig(self):
        '''MSSQL specific config file reader'''

        from MSSQLSpatialDataStore import MSSQLSpatialDataStore as MS

        ref = self.fn + ':MS. '
        usr = None
        pwd = None
        epsg = None
        cql = None
        schema = None

        if self.use_defaults:
            odbc = "FreeTDS"
            server = "127.0.0.1\SQLExpress"
            dbname = "LDSINCR"
            trust = "yes"
            dsn = "LDSINCR"
            config = "external"
        else:
            odbc = None
            server = None
            dbname = None
            trust = None
            dsn = None
            config = None

        try:
            odbc = self.cp.get(MS.DRIVER_NAME, 'odbc')
        except NoSectionError:
            ldslog.warn(ref + "No MSSQLSpatial section")
            raise
            #return (None,)*11
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            server = self.cp.get(MS.DRIVER_NAME, 'server')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            dsn = self.cp.get(MS.DRIVER_NAME, 'dsn')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            trust = self.cp.get(MS.DRIVER_NAME, 'trust')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            dbname = self.cp.get(MS.DRIVER_NAME, 'dbname')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            schema = self.cp.get(MS.DRIVER_NAME, 'schema')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            usr = self.cp.get(MS.DRIVER_NAME, 'user')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            pwd = self.cp.get(MS.DRIVER_NAME, 'pass')
        except NoOptionError as noe:
            ldslog.warn(ref + str(noe))

        try:
            config = self.cp.get(MS.DRIVER_NAME, 'config')
        except NoOptionError:
            ldslog.warn(ref + "No config preference specified, default to " +
                        str(config))

        try:
            epsg = self.cp.get(MS.DRIVER_NAME, 'epsg')
        except NoOptionError:
            ldslog.warn(
                ref +
                "EPSG not specified, default to None keeping existing SRS")

        try:
            cql = self.cp.get(MS.DRIVER_NAME, 'cql')
        except NoOptionError:
            ldslog.warn(ref + "No CQL Filter specified, fetching all results")
            cql = None

        return (odbc, server, dsn, trust, dbname, schema, usr, pwd, config,
                epsg, cql)

    def readSpatiaLiteConfig(self):
        '''SpatiaLite specific config file reader'''
        from SpatiaLiteDataStore import SpatiaLiteDataStore as SL

        ref = self.fn + ':SL. '
        epsg = None
        cql = None

        if self.use_defaults:
            lcfname = "~/LDSSLITE.sqlite3"
            config = "external"
        else:
            lcfname = None
            config = None

        try:
            lcfname = self.cp.get(SL.DRIVER_NAME, 'file')
        except NoSectionError:
            ldslog.warn(ref + "No SpatiaLite section")
            raise
            #return (None,)*4
        except NoOptionError:
            ldslog.warn(ref + "No DB name provided, default to " +
                        str(lcfname))

        try:
            config = self.cp.get(SL.DRIVER_NAME, 'config')
        except NoOptionError:
            ldslog.warn(ref + "No config preference specified, default to " +
                        str(config))

        try:
            epsg = self.cp.get(SL.DRIVER_NAME, 'epsg')
        except NoOptionError:
            ldslog.warn(ref + "EPSG not specified, default to " + str(epsg) +
                        " keeping existing SRS")

        try:
            cql = self.cp.get(SL.DRIVER_NAME, 'cql')
        except NoOptionError:
            ldslog.warn(ref + "No CQL Filter specified, fetching all results")

        return (lcfname, config, epsg, cql)

    def readFileGDBConfig(self):
        '''FileGDB specific config file reader'''
        from FileGDBDataStore import FileGDBDataStore as FG

        ref = self.fn + ':FG. '
        epsg = None
        cql = None

        if self.use_defaults:
            lcfname = "~/LDSFGDB.gdb"
            config = "external"
        else:
            lcfname = None
            config = None

        try:
            lcfname = self.cp.get(FG.DRIVER_NAME, 'file')
        except NoSectionError:
            ldslog.warn(ref + "No FileGDB section")
            raise
            #return (None,)*4
        except NoOptionError:
            ldslog.warn(ref + "No DB name provided, default to " +
                        str(lcfname))

        try:
            config = self.cp.get(FG.DRIVER_NAME, 'config')
        except NoOptionError:
            ldslog.warn(ref + "No config preference specified, default to " +
                        str(config))

        try:
            epsg = self.cp.get(FG.DRIVER_NAME, 'epsg')
        except NoOptionError:
            ldslog.warn(ref + "EPSG not specified, default to " + str(epsg) +
                        " none keeping existing SRS")

        try:
            cql = self.cp.get(FG.DRIVER_NAME, 'cql')
        except NoOptionError:
            ldslog.warn(ref + "No CQL Filter specified, fetching all results")

        return (lcfname, config, epsg, cql)

    def readWFSConfig(self):
        '''Generic WFS config file reader'''
        '''Since this now keys on the driver name, WFS is read before LDS and LDS not at all, So...'''

        return self.readLDSConfig()

    def readLDSConfig(self):
        '''LDS specific config file reader'''
        #***HACK *** reimport NSE aliased
        from ConfigParser import NoSectionError as NSE
        #use_defaults determines whether we use default values. For a user config this may not be wise
        #since a user config is a custom file relying on the main config for absent values not last-resort defaults
        ref = self.fn + ':LDS. '
        cql = None
        key = None

        #url = "http://data.linz.govt.nz/"
        #ver = "2.0.0"
        if self.use_defaults:
            url = "http://wfs.data.linz.govt.nz/"
            fmt = "GML2"
            svc = "WFS"
            ver = "1.0.0"
        else:
            url = None
            fmt = None
            svc = None
            ver = None

        try:
            url = self.cp.get(self.LDSN, 'url')
        #for some reason we cant use global NoSectionError in this function (even though its fine everywhere else)
        except NSE:
            ldslog.warn(ref + "LDS: No LDS Section")
            return (None, ) * 6
        except NoOptionError:
            ldslog.warn(ref + "Default URL assumed ")

        try:
            key = self.cp.get(self.LDSN, 'key')
        except NoOptionError, NoSectionError:
            ldslog.error(ref + "Key required to connect to LDS...")
            raise

        try:
            fmt = self.cp.get(self.LDSN, 'fmt')
        except NoOptionError:
            ldslog.warn(ref + "No output format specified")

        try:
            svc = self.cp.get(self.LDSN, 'svc')
        except NoOptionError:
            ldslog.warn(ref + "No service type specified, default to " +
                        str(svc))

        try:
            ver = self.cp.get(self.LDSN, 'ver')
        except NoOptionError:
            ldslog.warn(
                ref +
                "No Version specified, assuming WFS and default to version " +
                str(ver))

        try:
            cql = self.cp.get(self.LDSN, 'cql')
        except NoOptionError:
            ldslog.warn(ref + "No CQL Filter specified, fetching all results")

        return (url, key, svc, ver, fmt, cql)
コード例 #35
0
ファイル: configuration.py プロジェクト: jgao54/airflow
class AirflowConfigParser(ConfigParser):

    # These configuration elements can be fetched as the stdout of commands
    # following the "{section}__{name}__cmd" pattern, the idea behind this
    # is to not store password on boxes in text files.
    as_command_stdout = {
        ('core', 'sql_alchemy_conn'),
        ('core', 'fernet_key'),
        ('celery', 'broker_url'),
        ('celery', 'result_backend'),
        # Todo: remove this in Airflow 1.11
        ('celery', 'celery_result_backend'),
        ('atlas', 'password'),
        ('smtp', 'smtp_password'),
        ('ldap', 'bind_password'),
        ('kubernetes', 'git_password'),
    }

    # A two-level mapping of (section -> new_name -> old_name). When reading
    # new_name, the old_name will be checked to see if it exists. If it does a
    # DeprecationWarning will be issued and the old name will be used instead
    deprecated_options = {
        'celery': {
            # Remove these keys in Airflow 1.11
            'worker_concurrency': 'celeryd_concurrency',
            'result_backend': 'celery_result_backend',
            'broker_url': 'celery_broker_url',
            'ssl_active': 'celery_ssl_active',
            'ssl_cert': 'celery_ssl_cert',
            'ssl_key': 'celery_ssl_key',
        }
    }
    deprecation_format_string = (
        'The {old} option in [{section}] has been renamed to {new} - the old '
        'setting has been used, but please update your config.'
    )

    def __init__(self, default_config=None, *args, **kwargs):
        super(AirflowConfigParser, self).__init__(*args, **kwargs)

        self.airflow_defaults = ConfigParser(*args, **kwargs)
        if default_config is not None:
            self.airflow_defaults.read_string(default_config)

        self.is_validated = False

    def _validate(self):
        if (
                self.get("core", "executor") != 'SequentialExecutor' and
                "sqlite" in self.get('core', 'sql_alchemy_conn')):
            raise AirflowConfigException(
                "error: cannot use sqlite with the {}".format(
                    self.get('core', 'executor')))

        elif (
            self.getboolean("webserver", "authenticate") and
            self.get("webserver", "owner_mode") not in ['user', 'ldapgroup']
        ):
            raise AirflowConfigException(
                "error: owner_mode option should be either "
                "'user' or 'ldapgroup' when filtering by owner is set")

        elif (
            self.getboolean("webserver", "authenticate") and
            self.get("webserver", "owner_mode").lower() == 'ldapgroup' and
            self.get("webserver", "auth_backend") != (
                'airflow.contrib.auth.backends.ldap_auth')
        ):
            raise AirflowConfigException(
                "error: attempt at using ldapgroup "
                "filtering without using the Ldap backend")

        self.is_validated = True

    def _get_env_var_option(self, section, key):
        # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
        env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())
        if env_var in os.environ:
            return expand_env_var(os.environ[env_var])

    def _get_cmd_option(self, section, key):
        fallback_key = key + '_cmd'
        # if this is a valid command key...
        if (section, key) in self.as_command_stdout:
            if super(AirflowConfigParser, self) \
                    .has_option(section, fallback_key):
                command = super(AirflowConfigParser, self) \
                    .get(section, fallback_key)
                return run_command(command)

    def get(self, section, key, **kwargs):
        section = str(section).lower()
        key = str(key).lower()

        deprecated_name = self.deprecated_options.get(section, {}).get(key, None)

        # first check environment variables
        option = self._get_env_var_option(section, key)
        if option is not None:
            return option
        if deprecated_name:
            option = self._get_env_var_option(section, deprecated_name)
            if option is not None:
                self._warn_deprecate(section, key, deprecated_name)
                return option

        # ...then the config file
        if super(AirflowConfigParser, self).has_option(section, key):
            # Use the parent's methods to get the actual config here to be able to
            # separate the config from default config.
            return expand_env_var(
                super(AirflowConfigParser, self).get(section, key, **kwargs))
        if deprecated_name:
            if super(AirflowConfigParser, self).has_option(section, deprecated_name):
                self._warn_deprecate(section, key, deprecated_name)
                return expand_env_var(super(AirflowConfigParser, self).get(
                    section,
                    deprecated_name,
                    **kwargs
                ))

        # ...then commands
        option = self._get_cmd_option(section, key)
        if option:
            return option
        if deprecated_name:
            option = self._get_cmd_option(section, deprecated_name)
            if option:
                self._warn_deprecate(section, key, deprecated_name)
                return option

        # ...then the default config
        if self.airflow_defaults.has_option(section, key):
            return expand_env_var(
                self.airflow_defaults.get(section, key, **kwargs))

        else:
            log.warning(
                "section/key [{section}/{key}] not found in config".format(**locals())
            )

            raise AirflowConfigException(
                "section/key [{section}/{key}] not found "
                "in config".format(**locals()))

    def getboolean(self, section, key):
        val = str(self.get(section, key)).lower().strip()
        if '#' in val:
            val = val.split('#')[0].strip()
        if val.lower() in ('t', 'true', '1'):
            return True
        elif val.lower() in ('f', 'false', '0'):
            return False
        else:
            raise AirflowConfigException(
                'The value for configuration option "{}:{}" is not a '
                'boolean (received "{}").'.format(section, key, val))

    def getint(self, section, key):
        return int(self.get(section, key))

    def getfloat(self, section, key):
        return float(self.get(section, key))

    def read(self, filenames):
        super(AirflowConfigParser, self).read(filenames)
        self._validate()

    def read_dict(self, *args, **kwargs):
        super(AirflowConfigParser, self).read_dict(*args, **kwargs)
        self._validate()

    def has_option(self, section, option):
        try:
            # Using self.get() to avoid reimplementing the priority order
            # of config variables (env, config, cmd, defaults)
            self.get(section, option)
            return True
        except AirflowConfigException:
            return False

    def remove_option(self, section, option, remove_default=True):
        """
        Remove an option if it exists in config from a file or
        default config. If both of config have the same option, this removes
        the option in both configs unless remove_default=False.
        """
        if super(AirflowConfigParser, self).has_option(section, option):
            super(AirflowConfigParser, self).remove_option(section, option)

        if self.airflow_defaults.has_option(section, option) and remove_default:
            self.airflow_defaults.remove_option(section, option)

    def getsection(self, section):
        """
        Returns the section as a dict. Values are converted to int, float, bool
        as required.
        :param section: section from the config
        :return: dict
        """
        if (section not in self._sections and
                section not in self.airflow_defaults._sections):
            return None

        _section = copy.deepcopy(self.airflow_defaults._sections[section])

        if section in self._sections:
            _section.update(copy.deepcopy(self._sections[section]))

        section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper())
        for env_var in sorted(os.environ.keys()):
            if env_var.startswith(section_prefix):
                key = env_var.replace(section_prefix, '').lower()
                _section[key] = self._get_env_var_option(section, key)

        for key, val in iteritems(_section):
            try:
                val = int(val)
            except ValueError:
                try:
                    val = float(val)
                except ValueError:
                    if val.lower() in ('t', 'true'):
                        val = True
                    elif val.lower() in ('f', 'false'):
                        val = False
            _section[key] = val
        return _section

    def as_dict(
            self, display_source=False, display_sensitive=False, raw=False):
        """
        Returns the current configuration as an OrderedDict of OrderedDicts.
        :param display_source: If False, the option value is returned. If True,
            a tuple of (option_value, source) is returned. Source is either
            'airflow.cfg', 'default', 'env var', or 'cmd'.
        :type display_source: bool
        :param display_sensitive: If True, the values of options set by env
            vars and bash commands will be displayed. If False, those options
            are shown as '< hidden >'
        :type display_sensitive: bool
        :param raw: Should the values be output as interpolated values, or the
            "raw" form that can be fed back in to ConfigParser
        :type raw: bool
        """
        cfg = {}
        configs = [
            ('default', self.airflow_defaults),
            ('airflow.cfg', self),
        ]

        for (source_name, config) in configs:
            for section in config.sections():
                sect = cfg.setdefault(section, OrderedDict())
                for (k, val) in config.items(section=section, raw=raw):
                    if display_source:
                        val = (val, source_name)
                    sect[k] = val

        # add env vars and overwrite because they have priority
        for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
            try:
                _, section, key = ev.split('__')
                opt = self._get_env_var_option(section, key)
            except ValueError:
                continue
            if (not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'):
                opt = '< hidden >'
            elif raw:
                opt = opt.replace('%', '%%')
            if display_source:
                opt = (opt, 'env var')
            cfg.setdefault(section.lower(), OrderedDict()).update(
                {key.lower(): opt})

        # add bash commands
        for (section, key) in self.as_command_stdout:
            opt = self._get_cmd_option(section, key)
            if opt:
                if not display_sensitive:
                    opt = '< hidden >'
                if display_source:
                    opt = (opt, 'cmd')
                elif raw:
                    opt = opt.replace('%', '%%')
                cfg.setdefault(section, OrderedDict()).update({key: opt})
                del cfg[section][key + '_cmd']

        return cfg

    def load_test_config(self):
        """
        Load the unit test configuration.

        Note: this is not reversible.
        """
        # override any custom settings with defaults
        self.read_string(parameterized_config(DEFAULT_CONFIG))
        # then read test config
        self.read_string(parameterized_config(TEST_CONFIG))
        # then read any "custom" test settings
        self.read(TEST_CONFIG_FILE)

    def _warn_deprecate(self, section, key, deprecated_name):
        warnings.warn(
            self.deprecation_format_string.format(
                old=deprecated_name,
                new=key,
                section=section,
            ),
            DeprecationWarning,
            stacklevel=3,
        )
コード例 #36
0
def loadConfig():
    global CONFIG
    CONFIG = {} 
    CONFIG = ConfigParser()
    CONFIG.read('global.conf')
コード例 #37
0
class LayerFileReader(LayerReader):
    def __init__(self, lcfname):
        '''
        Constructor
        '''
        super(LayerFileReader, self).__init__(lcfname)

        self.cp = ConfigParser()
        self.lcfilename = LU.standardiseLayerConfigName(self.lcfname)

        self._readConfigFile(self.lcfilename)

    def isCurrent(self):
        '''TF test to decide whether to init'''
        return self.lcfilename and self._fileexists() and len(
            self.cp.sections()) > 0

    def close(self):
        self.cp = None
        self.lcfilename = None
        self.lcfname = None

    def _fileexists(self):
        return os.path.exists(self.lcfilename)

    def buildConfigLayer(self, res):
        '''Just write a file in conf with the name <driver>.layer.properties'''
        ###This should eb a UTF8 write, but only works as ASCII
        #open(self.filename,'w').write(str(res))
        with codecs.open(self.lcfilename, 'w', 'utf-8') as lconf:
            lconf.write(res)
        self._readConfigFile(self.lcfilename)

    def _readConfigFile(self, fn):
        '''Reads named config file'''
        #Split off so you can override the config file on the same reader object if needed
        try:
            with codecs.open(fn, 'r', 'utf-8') as cf:
                self.cp.readfp(cf)
        except ParsingError as pe:
            ldslog.error(
                '{0} file corrupt. Please correct the error; {1} OR delete and rebuild'
                .format(fn, str(pe)))
            raise

    @override(LayerReader)
    def findLayerIdByName(self, name):
        '''Reverse lookup of section by associated name, finds first occurance only'''
        lid = filter(lambda n: name == self.cp.get(n, 'name'),
                     self.cp.sections())
        return lid[0] if len(lid) > 0 else None

    @override(LayerReader)
    def getLayerNames(self, refresh=False):
        '''Returns sections from properties file'''
        lcnames = []
        for sec in self.cp.sections():
            lcn = self.cp.get(sec, 'name')
            lcc = self.cp.get(sec, 'category')
            lcnames += [
                (sec if type(sec) == unicode else unicode(sec, 'utf8'),
                 lcn if type(lcn) == unicode else unicode(lcn, 'utf8'),
                 (lcc if type(lcc) == unicode else unicode(
                     lcc, 'utf8')).split(',')),
            ]
        return lcnames

    @override(LayerReader)
    def readLayerProperty(self, layer, key):
        try:
            if isinstance(layer, tuple) or isinstance(layer, list):
                value = ()
                for l in layer:
                    value += (LU.assessNone(self.cp.get(l, key)), )
            else:
                value = LU.assessNone(self.cp.get(layer, key))
        except:
            '''return a default value otherwise none which would also be a default for some keys'''
            #the logic here may be a bit suss, if the property is blank return none but if there is an error assume a default is needed?
            return {'pkey': 'ID', 'name': layer, 'geocolumn': 'SHAPE'}.get(key)
        return value

    def _readSingleLayerProperty(self, layer, key):
        return LU.assessNone(self.cp.get(layer, key))

    @override(LayerReader)
    def writeLayerProperty(self, layer, field, value):
        '''Write changes to layer config table'''
        #if value: value = value.strip()
        try:
            if (isinstance(layer, tuple) or isinstance(layer, list)) and (
                    isinstance(value, tuple) or isinstance(value, list)):
                for l, v in zip(layer, value):
                    self.cp.set(l, field, v.strip() if v else '')
            else:
                self.cp.set(layer, field, value.strip() if value else '')
            with codecs.open(self.lcfilename, 'w', 'utf-8') as configfile:
                self.cp.write(configfile)
            #ldslog.debug("Check "+str(field)+" for layer "+str(layer)+" is set to "+str(value)+" : GetField="+self.cp.get(layer, field))
        except Exception as e:
            ldslog.warn('Problem writing LM date to layer config file. ' +
                        str(e))

    @override(LayerReader)
    def readLayerParameters(self, id):
        #def readLayerSchemaConfig(self,layer):
        '''Full Layer config reader. Returns the config values for the whole layer or makes sensible guesses for defaults'''
        from LDSUtilities import LayerConfEntry
        #
        #        try:
        #            defn = self.cp.get(layer, 'sql')
        #            #if the user has gone to the trouble of defining their own schema in SQL just return that
        #            return (defn,None,None,None,None,None,None,None,None)
        #        except:
        #            pass
        '''optional but one way to record the type and name of a column is to save a string tuple (name,type) and parse this at build time'''
        try:
            pkey = self.cp.get(id, 'pkey')
        except NoOptionError:
            ldslog.warn(
                "LayerSchema: No Primary Key Column defined, default to 'ID'")
            pkey = 'ID'
        '''names are/can-be stored so we can reverse search by layer name'''
        try:
            name = self.cp.get(id, 'name')
        except NoOptionError:
            ldslog.warn(
                "LayerSchema: No Name saved in config for this layer, returning ID"
            )
            name = id

        if name is None:
            name = id
        '''names are/can-be stored so we can reverse search by layer name'''
        try:
            group = LU.assessNone(self.cp.get(id, 'category'))
        except NoOptionError:
            ldslog.warn("Group List: No Groups defined for this layer")
            group = None

        if not group:
            pass

        try:
            gcol = self.cp.get(id, 'geocolumn')
        except NoOptionError:
            ldslog.warn(
                "LayerSchema: No Geo Column defined, default to 'SHAPE'")
            gcol = 'SHAPE'

        #i dont think we need this anymore using the new logic, if gcol->spatial, if pkey->unique


#        try:
#            index = self.cp.get(layer, 'index')
#        except NoOptionError:
#            ldslog.warn("LayerSchema: No Index Column/Specification defined, default to None")
#            index = None

        try:
            epsg = self.cp.get(id, 'epsg')
        except NoOptionError:
            #print "No Projection Transformation defined"#don't really need to state the default occurance
            epsg = None

        try:
            lmod = self.cp.get(id, 'lastmodified')
        except NoOptionError:
            ldslog.warn(
                "LayerSchema: No Last-Modified date recorded, successful update will write current time here"
            )
            lmod = None

        try:
            disc = self.cp.get(id, 'discard')
        except NoOptionError:
            disc = None

        try:
            cql = self.cp.get(id, 'cql')
        except NoOptionError:
            cql = None

        return LayerConfEntry(id, pkey, name, group, gcol, epsg, lmod, disc,
                              cql)

    @override(LayerReader)
    def readAllLayerParameters(self):
        '''Gets all LC entries as a list of LCEs using readLayerParameters'''
        lcel = []
        for ln in self.getLayerNames():
            lcel += [
                self.readLayerParameters(ln),
            ]
        return lcel
コード例 #38
0
class AirflowConfigParser(ConfigParser):

    # These configuration elements can be fetched as the stdout of commands
    # following the "{section}__{name}__cmd" pattern, the idea behind this
    # is to not store password on boxes in text files.
    as_command_stdout = {
        ('core', 'sql_alchemy_conn'),
        ('core', 'fernet_key'),
        ('celery', 'broker_url'),
        ('celery', 'result_backend'),
        # Todo: remove this in Airflow 1.11
        ('celery', 'celery_result_backend'),
        ('atlas', 'password'),
        ('smtp', 'smtp_password'),
        ('ldap', 'bind_password'),
        ('kubernetes', 'git_password'),
    }

    # A two-level mapping of (section -> new_name -> old_name). When reading
    # new_name, the old_name will be checked to see if it exists. If it does a
    # DeprecationWarning will be issued and the old name will be used instead
    deprecated_options = {
        'celery': {
            # Remove these keys in Airflow 1.11
            'worker_concurrency': 'celeryd_concurrency',
            'result_backend': 'celery_result_backend',
            'broker_url': 'celery_broker_url',
            'ssl_active': 'celery_ssl_active',
            'ssl_cert': 'celery_ssl_cert',
            'ssl_key': 'celery_ssl_key',
        }
    }

    # A mapping of old default values that we want to change and warn the user
    # about. Mapping of section -> setting -> { old, replace, by_version }
    deprecated_values = {
        'core': {
            'task_runner': ('BashTaskRunner', 'StandardTaskRunner', '2.0'),
        },
    }

    def __init__(self, default_config=None, *args, **kwargs):
        super(AirflowConfigParser, self).__init__(*args, **kwargs)

        self.airflow_defaults = ConfigParser(*args, **kwargs)
        if default_config is not None:
            self.airflow_defaults.read_string(default_config)

        self.is_validated = False

    def _validate(self):
        if (self.get("core", "executor") != 'SequentialExecutor'
                and "sqlite" in self.get('core', 'sql_alchemy_conn')):
            raise AirflowConfigException(
                "error: cannot use sqlite with the {}".format(
                    self.get('core', 'executor')))

        elif (self.getboolean("webserver", "authenticate") and self.get(
                "webserver", "owner_mode") not in ['user', 'ldapgroup']):
            raise AirflowConfigException(
                "error: owner_mode option should be either "
                "'user' or 'ldapgroup' when filtering by owner is set")

        elif (self.getboolean("webserver", "authenticate")
              and self.get("webserver", "owner_mode").lower() == 'ldapgroup'
              and self.get("webserver", "auth_backend") !=
              ('airflow.contrib.auth.backends.ldap_auth')):
            raise AirflowConfigException(
                "error: attempt at using ldapgroup "
                "filtering without using the Ldap backend")

        for section, replacement in self.deprecated_values.items():
            for name, info in replacement.items():
                old, new, version = info
                if self.get(section, name, fallback=None) == old:
                    # Make sure the env var option is removed, otherwise it
                    # would be read and used instead of the value we set
                    env_var = self._env_var_name(section, name)
                    os.environ.pop(env_var, None)

                    self.set(section, name, new)
                    warnings.warn(
                        'The {name} setting in [{section}] has the old default value '
                        'of {old!r}. This value has been changed to {new!r} in the '
                        'running config, but please update your config before Apache '
                        'Airflow {version}.'.format(name=name,
                                                    section=section,
                                                    old=old,
                                                    new=new,
                                                    version=version),
                        FutureWarning)

        self.is_validated = True

    @staticmethod
    def _env_var_name(section, key):
        return 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())

    def _get_env_var_option(self, section, key):
        # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
        env_var = self._env_var_name(section, key)
        if env_var in os.environ:
            return expand_env_var(os.environ[env_var])

    def _get_cmd_option(self, section, key):
        fallback_key = key + '_cmd'
        # if this is a valid command key...
        if (section, key) in self.as_command_stdout:
            if super(AirflowConfigParser, self) \
                    .has_option(section, fallback_key):
                command = super(AirflowConfigParser, self) \
                    .get(section, fallback_key)
                return run_command(command)

    def get(self, section, key, **kwargs):
        section = str(section).lower()
        key = str(key).lower()

        deprecated_name = self.deprecated_options.get(section,
                                                      {}).get(key, None)

        # first check environment variables
        option = self._get_env_var_option(section, key)
        if option is not None:
            return option
        if deprecated_name:
            option = self._get_env_var_option(section, deprecated_name)
            if option is not None:
                self._warn_deprecate(section, key, deprecated_name)
                return option

        # ...then the config file
        if super(AirflowConfigParser, self).has_option(section, key):
            # Use the parent's methods to get the actual config here to be able to
            # separate the config from default config.
            return expand_env_var(
                super(AirflowConfigParser, self).get(section, key, **kwargs))
        if deprecated_name:
            if super(AirflowConfigParser,
                     self).has_option(section, deprecated_name):
                self._warn_deprecate(section, key, deprecated_name)
                return expand_env_var(
                    super(AirflowConfigParser,
                          self).get(section, deprecated_name, **kwargs))

        # ...then commands
        option = self._get_cmd_option(section, key)
        if option:
            return option
        if deprecated_name:
            option = self._get_cmd_option(section, deprecated_name)
            if option:
                self._warn_deprecate(section, key, deprecated_name)
                return option

        # ...then the default config
        if self.airflow_defaults.has_option(section,
                                            key) or 'fallback' in kwargs:
            return expand_env_var(
                self.airflow_defaults.get(section, key, **kwargs))

        else:
            log.warning("section/key [%s/%s] not found in config", section,
                        key)

            raise AirflowConfigException(
                "section/key [{section}/{key}] not found "
                "in config".format(section=section, key=key))

    def getboolean(self, section, key, **kwargs):
        val = str(self.get(section, key, **kwargs)).lower().strip()
        if '#' in val:
            val = val.split('#')[0].strip()
        if val in ('t', 'true', '1'):
            return True
        elif val in ('f', 'false', '0'):
            return False
        else:
            raise AirflowConfigException(
                'The value for configuration option "{}:{}" is not a '
                'boolean (received "{}").'.format(section, key, val))

    def getint(self, section, key, **kwargs):
        return int(self.get(section, key, **kwargs))

    def getfloat(self, section, key, **kwargs):
        return float(self.get(section, key, **kwargs))

    def read(self, filenames, **kwargs):
        super(AirflowConfigParser, self).read(filenames, **kwargs)
        self._validate()

    def read_dict(self, *args, **kwargs):
        super(AirflowConfigParser, self).read_dict(*args, **kwargs)
        self._validate()

    def has_option(self, section, option):
        try:
            # Using self.get() to avoid reimplementing the priority order
            # of config variables (env, config, cmd, defaults)
            # UNSET to avoid logging a warning about missing values
            self.get(section, option, fallback=_UNSET)
            return True
        except NoOptionError:
            return False

    def remove_option(self, section, option, remove_default=True):
        """
        Remove an option if it exists in config from a file or
        default config. If both of config have the same option, this removes
        the option in both configs unless remove_default=False.
        """
        if super(AirflowConfigParser, self).has_option(section, option):
            super(AirflowConfigParser, self).remove_option(section, option)

        if self.airflow_defaults.has_option(section,
                                            option) and remove_default:
            self.airflow_defaults.remove_option(section, option)

    def getsection(self, section):
        """
        Returns the section as a dict. Values are converted to int, float, bool
        as required.

        :param section: section from the config
        :rtype: dict
        """
        if (section not in self._sections
                and section not in self.airflow_defaults._sections):
            return None

        _section = copy.deepcopy(self.airflow_defaults._sections[section])

        if section in self._sections:
            _section.update(copy.deepcopy(self._sections[section]))

        section_prefix = 'AIRFLOW__{S}__'.format(S=section.upper())
        for env_var in sorted(os.environ.keys()):
            if env_var.startswith(section_prefix):
                key = env_var.replace(section_prefix, '').lower()
                _section[key] = self._get_env_var_option(section, key)

        for key, val in iteritems(_section):
            try:
                val = int(val)
            except ValueError:
                try:
                    val = float(val)
                except ValueError:
                    if val.lower() in ('t', 'true'):
                        val = True
                    elif val.lower() in ('f', 'false'):
                        val = False
            _section[key] = val
        return _section

    def as_dict(self,
                display_source=False,
                display_sensitive=False,
                raw=False):
        """
        Returns the current configuration as an OrderedDict of OrderedDicts.
        :param display_source: If False, the option value is returned. If True,
            a tuple of (option_value, source) is returned. Source is either
            'airflow.cfg', 'default', 'env var', or 'cmd'.
        :type display_source: bool
        :param display_sensitive: If True, the values of options set by env
            vars and bash commands will be displayed. If False, those options
            are shown as '< hidden >'
        :type display_sensitive: bool
        :param raw: Should the values be output as interpolated values, or the
            "raw" form that can be fed back in to ConfigParser
        :type raw: bool
        """
        cfg = {}
        configs = [
            ('default', self.airflow_defaults),
            ('airflow.cfg', self),
        ]

        for (source_name, config) in configs:
            for section in config.sections():
                sect = cfg.setdefault(section, OrderedDict())
                for (k, val) in config.items(section=section, raw=raw):
                    if display_source:
                        val = (val, source_name)
                    sect[k] = val

        # add env vars and overwrite because they have priority
        for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
            try:
                _, section, key = ev.split('__')
                opt = self._get_env_var_option(section, key)
            except ValueError:
                continue
            if not display_sensitive and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE':
                opt = '< hidden >'
            elif raw:
                opt = opt.replace('%', '%%')
            if display_source:
                opt = (opt, 'env var')
            cfg.setdefault(section.lower(),
                           OrderedDict()).update({key.lower(): opt})

        # add bash commands
        for (section, key) in self.as_command_stdout:
            opt = self._get_cmd_option(section, key)
            if opt:
                if not display_sensitive:
                    opt = '< hidden >'
                if display_source:
                    opt = (opt, 'cmd')
                elif raw:
                    opt = opt.replace('%', '%%')
                cfg.setdefault(section, OrderedDict()).update({key: opt})
                del cfg[section][key + '_cmd']

        return cfg

    def load_test_config(self):
        """
        Load the unit test configuration.

        Note: this is not reversible.
        """
        # override any custom settings with defaults
        self.read_string(parameterized_config(DEFAULT_CONFIG))
        # then read test config
        self.read_string(parameterized_config(TEST_CONFIG))
        # then read any "custom" test settings
        self.read(TEST_CONFIG_FILE)

    def _warn_deprecate(self, section, key, deprecated_name):
        warnings.warn(
            'The {old} option in [{section}] has been renamed to {new} - the old '
            'setting has been used, but please update your config.'.format(
                old=deprecated_name,
                new=key,
                section=section,
            ),
            DeprecationWarning,
            stacklevel=3,
        )
コード例 #39
0
class AirflowConfigParser(ConfigParser):

    # These configuration elements can be fetched as the stdout of commands
    # following the "{section}__{name}__cmd" pattern, the idea behind this
    # is to not store password on boxes in text files.
    as_command_stdout = {
        ('core', 'sql_alchemy_conn'),
        ('core', 'fernet_key'),
        ('celery', 'broker_url'),
        ('celery', 'result_backend')
    }

    def __init__(self, default_config=None, *args, **kwargs):
        super(AirflowConfigParser, self).__init__(*args, **kwargs)

        self.defaults = ConfigParser(*args, **kwargs)
        if default_config is not None:
            self.defaults.read_string(default_config)

        self.is_validated = False

    def _validate(self):
        if (
                self.get("core", "executor") != 'SequentialExecutor' and
                "sqlite" in self.get('core', 'sql_alchemy_conn')):
            raise AirflowConfigException(
                "error: cannot use sqlite with the {}".format(
                    self.get('core', 'executor')))

        elif (
            self.getboolean("webserver", "authenticate") and
            self.get("webserver", "owner_mode") not in ['user', 'ldapgroup']
        ):
            raise AirflowConfigException(
                "error: owner_mode option should be either "
                "'user' or 'ldapgroup' when filtering by owner is set")

        elif (
            self.getboolean("webserver", "authenticate") and
            self.get("webserver", "owner_mode").lower() == 'ldapgroup' and
            self.get("webserver", "auth_backend") != (
                'airflow.contrib.auth.backends.ldap_auth')
        ):
            raise AirflowConfigException(
                "error: attempt at using ldapgroup "
                "filtering without using the Ldap backend")

        self.is_validated = True

    def _get_env_var_option(self, section, key):
        # must have format AIRFLOW__{SECTION}__{KEY} (note double underscore)
        env_var = 'AIRFLOW__{S}__{K}'.format(S=section.upper(), K=key.upper())
        if env_var in os.environ:
            return expand_env_var(os.environ[env_var])

    def _get_cmd_option(self, section, key):
        fallback_key = key + '_cmd'
        # if this is a valid command key...
        if (section, key) in self.as_command_stdout:
            if super(AirflowConfigParser, self) \
                    .has_option(section, fallback_key):
                command = super(AirflowConfigParser, self) \
                    .get(section, fallback_key)
                return run_command(command)

    def get(self, section, key, **kwargs):
        section = str(section).lower()
        key = str(key).lower()

        # first check environment variables
        option = self._get_env_var_option(section, key)
        if option is not None:
            return option

        # ...then the config file
        if super(AirflowConfigParser, self).has_option(section, key):
            # Use the parent's methods to get the actual config here to be able to
            # separate the config from default config.
            return expand_env_var(
                super(AirflowConfigParser, self).get(section, key, **kwargs))

        # ...then commands
        option = self._get_cmd_option(section, key)
        if option:
            return option

        # ...then the default config
        if self.defaults.has_option(section, key):
            return expand_env_var(
                self.defaults.get(section, key, **kwargs))

        else:
            log.warning(
                "section/key [{section}/{key}] not found in config".format(**locals())
            )

            raise AirflowConfigException(
                "section/key [{section}/{key}] not found "
                "in config".format(**locals()))

    def getboolean(self, section, key):
        val = str(self.get(section, key)).lower().strip()
        if '#' in val:
            val = val.split('#')[0].strip()
        if val.lower() in ('t', 'true', '1'):
            return True
        elif val.lower() in ('f', 'false', '0'):
            return False
        else:
            raise AirflowConfigException(
                'The value for configuration option "{}:{}" is not a '
                'boolean (received "{}").'.format(section, key, val))

    def getint(self, section, key):
        return int(self.get(section, key))

    def getfloat(self, section, key):
        return float(self.get(section, key))

    def read(self, filenames):
        super(AirflowConfigParser, self).read(filenames)
        self._validate()

    def has_option(self, section, option):
        try:
            # Using self.get() to avoid reimplementing the priority order
            # of config variables (env, config, cmd, defaults)
            self.get(section, option)
            return True
        except AirflowConfigException:
            return False

    def remove_option(self, section, option, remove_default=True):
        """
        Remove an option if it exists in config from a file or
        default config. If both of config have the same option, this removes
        the option in both configs unless remove_default=False.
        """
        if super(AirflowConfigParser, self).has_option(section, option):
            super(AirflowConfigParser, self).remove_option(section, option)

        if self.defaults.has_option(section, option) and remove_default:
            self.defaults.remove_option(section, option)

    def getsection(self, section):
        """
        Returns the section as a dict. Values are converted to int, float, bool
        as required.
        :param section: section from the config
        :return: dict
        """
        if section not in self._sections and section not in self.defaults._sections:
            return None

        _section = copy.deepcopy(self.defaults._sections[section])

        if section in self._sections:
            _section.update(copy.deepcopy(self._sections[section]))

        for key, val in iteritems(_section):
            try:
                val = int(val)
            except ValueError:
                try:
                    val = float(val)
                except ValueError:
                    if val.lower() in ('t', 'true'):
                        val = True
                    elif val.lower() in ('f', 'false'):
                        val = False
            _section[key] = val
        return _section

    def as_dict(self, display_source=False, display_sensitive=False):
        """
        Returns the current configuration as an OrderedDict of OrderedDicts.
        :param display_source: If False, the option value is returned. If True,
            a tuple of (option_value, source) is returned. Source is either
            'airflow.cfg' or 'default'.
        :type display_source: bool
        :param display_sensitive: If True, the values of options set by env
            vars and bash commands will be displayed. If False, those options
            are shown as '< hidden >'
        :type display_sensitive: bool
        """
        cfg = copy.deepcopy(self.defaults._sections)
        cfg.update(copy.deepcopy(self._sections))

        # remove __name__ (affects Python 2 only)
        for options in cfg.values():
            options.pop('__name__', None)

        # add source
        if display_source:
            for section in cfg:
                for k, v in cfg[section].items():
                    cfg[section][k] = (v, 'airflow config')

        # add env vars and overwrite because they have priority
        for ev in [ev for ev in os.environ if ev.startswith('AIRFLOW__')]:
            try:
                _, section, key = ev.split('__')
                opt = self._get_env_var_option(section, key)
            except ValueError:
                opt = None
            if opt:
                if (
                        not display_sensitive
                        and ev != 'AIRFLOW__CORE__UNIT_TEST_MODE'):
                    opt = '< hidden >'
                if display_source:
                    opt = (opt, 'env var')
                cfg.setdefault(section.lower(), OrderedDict()).update(
                    {key.lower(): opt})

        # add bash commands
        for (section, key) in self.as_command_stdout:
            opt = self._get_cmd_option(section, key)
            if opt:
                if not display_sensitive:
                    opt = '< hidden >'
                if display_source:
                    opt = (opt, 'bash cmd')
                cfg.setdefault(section, OrderedDict()).update({key: opt})

        return cfg

    def load_test_config(self):
        """
        Load the unit test configuration.

        Note: this is not reversible.
        """
        # override any custom settings with defaults
        self.defaults.read_string(parameterized_config(DEFAULT_CONFIG))
        # then read test config
        self.read_string(parameterized_config(TEST_CONFIG))
        # then read any "custom" test settings
        self.read(TEST_CONFIG_FILE)
コード例 #40
0
ファイル: ReadConfig.py プロジェクト: josephramsay/LDS
class LayerFileReader(LayerReader):
    
    def __init__(self,lcfname):
        '''
        Constructor
        '''
        super(LayerFileReader,self).__init__(lcfname)
        
        self.cp = ConfigParser()
        self.lcfilename = LU.standardiseLayerConfigName(self.lcfname)
            
        self._readConfigFile(self.lcfilename)
        
    def isCurrent(self):
        '''TF test to decide whether to init'''
        return self.lcfilename and self._fileexists() and len(self.cp.sections())>0
    
    def close(self):
        self.cp = None
        self.lcfilename = None
        self.lcfname = None
        
    def _fileexists(self):
        return os.path.exists(self.lcfilename)
    
    def buildConfigLayer(self,res):
        '''Just write a file in conf with the name <driver>.layer.properties'''
        ###This should eb a UTF8 write, but only works as ASCII
        #open(self.filename,'w').write(str(res))
        with codecs.open(self.lcfilename,'w','utf-8') as lconf: 
            lconf.write(res)
        self._readConfigFile(self.lcfilename)
        
    def _readConfigFile(self,fn):
        '''Reads named config file'''
        #Split off so you can override the config file on the same reader object if needed
        try:
            with codecs.open(fn,'r','utf-8') as cf:
                self.cp.readfp(cf)
        except ParsingError as pe:
            ldslog.error('{0} file corrupt. Please correct the error; {1} OR delete and rebuild'.format(fn,str(pe)))
            raise
    
    @override(LayerReader)
    def findLayerIdByName(self,name):
        '''Reverse lookup of section by associated name, finds first occurance only'''
        lid = filter(lambda n: name==self.cp.get(n,'name'),self.cp.sections())
        return lid[0] if len(lid)>0 else None
    
    @override(LayerReader)
    def getLayerNames(self,refresh=False):
        '''Returns sections from properties file'''
        lcnames = []
        for sec in self.cp.sections():
            lcn = self.cp.get(sec, 'name')
            lcc = self.cp.get(sec,'category')
            lcnames += [(sec if type(sec)==unicode else unicode(sec,'utf8'),
                         lcn if type(lcn)==unicode else unicode(lcn,'utf8'),
                        (lcc if type(lcc)==unicode else unicode(lcc,'utf8')).split(',')),]
        return lcnames
    
    @override(LayerReader)
    def readLayerProperty(self,layer,key):
        try:
            if isinstance(layer,tuple) or isinstance(layer,list):
                value = () 
                for l in layer:
                    value += (LU.assessNone(self.cp.get(l, key)),)
            else:
                value = LU.assessNone(self.cp.get(layer, key))
        except:
            '''return a default value otherwise none which would also be a default for some keys'''
            #the logic here may be a bit suss, if the property is blank return none but if there is an error assume a default is needed?
            return {'pkey':'ID','name':layer,'geocolumn':'SHAPE'}.get(key)
        return value
    
    def _readSingleLayerProperty(self,layer,key):
        return LU.assessNone(self.cp.get(layer, key))
        
    
    @override(LayerReader)
    def writeLayerProperty(self,layer,field,value):
        '''Write changes to layer config table'''
        #if value: value = value.strip()
        try:    
            if (isinstance(layer,tuple) or isinstance(layer,list)) and (isinstance(value,tuple) or isinstance(value,list)): 
                for l,v in zip(layer,value):
                    self.cp.set(l,field,v.strip() if v else '')
            else:
                self.cp.set(layer,field,value.strip() if value else '')
            with codecs.open(self.lcfilename, 'w','utf-8') as configfile:
                self.cp.write(configfile)
            #ldslog.debug("Check "+str(field)+" for layer "+str(layer)+" is set to "+str(value)+" : GetField="+self.cp.get(layer, field))                                                                                        
        except Exception as e:
            ldslog.warn('Problem writing LM date to layer config file. '+str(e))
            
    @override(LayerReader)
    def readLayerParameters(self,id):
    #def readLayerSchemaConfig(self,layer):
        '''Full Layer config reader. Returns the config values for the whole layer or makes sensible guesses for defaults'''
        from LDSUtilities import LayerConfEntry
#        
#        try:
#            defn = self.cp.get(layer, 'sql')
#            #if the user has gone to the trouble of defining their own schema in SQL just return that
#            return (defn,None,None,None,None,None,None,None,None)
#        except:
#            pass
            
        '''optional but one way to record the type and name of a column is to save a string tuple (name,type) and parse this at build time'''
        try:
            pkey = self.cp.get(id, 'pkey')
        except NoOptionError:
            ldslog.warn("LayerSchema: No Primary Key Column defined, default to 'ID'")
            pkey = 'ID'
            
        '''names are/can-be stored so we can reverse search by layer name'''
        try:
            name = self.cp.get(id, 'name')
        except NoOptionError:
            ldslog.warn("LayerSchema: No Name saved in config for this layer, returning ID")
            name = id
            
        if name is None:
            name = id
            
        '''names are/can-be stored so we can reverse search by layer name'''
        try:
            group = LU.assessNone(self.cp.get(id, 'category'))
        except NoOptionError:
            ldslog.warn("Group List: No Groups defined for this layer")
            group = None
        
        if not group:
            pass     
            
        try:
            gcol = self.cp.get(id, 'geocolumn')
        except NoOptionError:
            ldslog.warn("LayerSchema: No Geo Column defined, default to 'SHAPE'")
            gcol = 'SHAPE'
        
        #i dont think we need this anymore using the new logic, if gcol->spatial, if pkey->unique    
#        try:
#            index = self.cp.get(layer, 'index')
#        except NoOptionError:
#            ldslog.warn("LayerSchema: No Index Column/Specification defined, default to None")
#            index = None
            
        try:
            epsg = self.cp.get(id, 'epsg')
        except NoOptionError:
            #print "No Projection Transformation defined"#don't really need to state the default occurance
            epsg = None
            
        try:
            lmod = self.cp.get(id, 'lastmodified')
        except NoOptionError:
            ldslog.warn("LayerSchema: No Last-Modified date recorded, successful update will write current time here")
            lmod = None
            
        try:
            disc = self.cp.get(id, 'discard')
        except NoOptionError:
            disc = None 
            
        try:
            cql = self.cp.get(id, 'cql')
        except NoOptionError:
            cql = None
            
        return LayerConfEntry(id,pkey,name,group,gcol,epsg,lmod,disc,cql)
    
    @override(LayerReader)
    def readAllLayerParameters(self):
        '''Gets all LC entries as a list of LCEs using readLayerParameters'''
        lcel = []
        for ln in self.getLayerNames():
            lcel += [self.readLayerParameters(ln),]
        return lcel
コード例 #41
0
 def _readConfigFile(self, fn):
     '''Reads named config file'''
     #Split off so you can override the config file on the same reader object if needed
     self.cp = ConfigParser()
     with codecs.open(fn, 'r', 'utf-8') as cf:
         self.cp.readfp(cf)
コード例 #42
0
ファイル: config.py プロジェクト: javyxu/space-time-gis
    def __init__(self, default_config=None, *args, **kwargs):
        super().__init__(*args, **kwargs)

        self.spacetimegis_defaults = ConfigParser(*args, **kwargs)
        if not default_config:
            self.spacetimegis_defaults.read_string(default_config)