def test_none(self): colony.conf_s("AGE", None) result = colony.conf("AGE", cast=int) self.assertEqual(result, None) result = colony.conf("HEIGHT", cast=int) self.assertEqual(result, None)
def connect(self, connection, parameters = {}): db_prefix = parameters.get("db_prefix", "") db_suffix = parameters.get("db_suffix", "default") db_prefix = colony.conf("DB_PREFIX", db_prefix) db_suffix = colony.conf("DB_SUFFIX", db_suffix) db_default = db_prefix + db_suffix if db_prefix else self.entity_manager.id host = parameters.get("host", "localhost") user = parameters.get("user", "root") password = parameters.get("password", "root") database = parameters.get("database", db_default) isolation = parameters.get("isolation", ISOLATION_LEVEL) host = colony.conf("DB_HOST", host) user = colony.conf("DB_USER", user) password = colony.conf("DB_PASSWORD", password) database = colony.conf("DB_NAME", database) isolation = colony.conf("DB_ISOLATION", isolation) show_sql = colony.conf("SHOW_SQL", False) connection._connection = MysqlConnection( host = host, user = user, password = password, database = database, isolation = isolation ) connection._transaction_level = 0 connection._user = user connection._host = host connection._database = database connection._isolation = isolation connection._show_sql = show_sql connection.open()
def connect(self, connection, parameters = {}): file_path = parameters.get("file_path", None) cache_size = parameters.get("cache_size", 200000) synchronous = parameters.get("synchronous", 2) show_sql = colony.conf("SHOW_SQL", False) file_path = colony.conf("DB_FILE", file_path) file_path = file_path or self._get_temporary() connection._connection = SqliteConnection( file_path, cache_size = cache_size, synchronous = synchronous ) connection._show_sql = show_sql connection.open()
def test_basic(self): colony.conf_s("NAME", "name") result = colony.conf("NAME") self.assertEqual(result, "name") result = colony.conf("NAME", cast=str) self.assertEqual(result, "name") self.assertEqual(type(result), str) result = colony.conf("NAME", cast="str") self.assertEqual(result, "name") self.assertEqual(type(result), str) colony.conf_s("AGE", "10") result = colony.conf("AGE", cast=int) self.assertEqual(result, 10) self.assertEqual(type(result), int) result = colony.conf("AGE", cast="int") self.assertEqual(result, 10) self.assertEqual(type(result), int) result = colony.conf("AGE", cast=str) self.assertEqual(result, "10") self.assertEqual(type(result), str) result = colony.conf("HEIGHT") self.assertEqual(result, None)
def _get_email_attributes(self): # retrieves the resources manager plugin resources_manager_plugin = self.plugin.resources_manager_plugin # retrieves the various smtp resource values that should # have been set in the current configuration smtp_server = resources_manager_plugin.get_resource("system.mail.smtp_server") smtp_port = resources_manager_plugin.get_resource("system.mail.smtp_port") smtp_use_tls = resources_manager_plugin.get_resource("system.mail.smtp_use_tls") smtp_user = resources_manager_plugin.get_resource("system.mail.smtp_user") smtp_password = resources_manager_plugin.get_resource("system.mail.smtp_password") # verifies that the smtp server configuration is correctly defined # otherwise raises an exception indicating the problem if not smtp_server: raise hive_site.MissingConfiguration("smtp server configuration") # retrieves the data from the smtp resources, defaulting # to the unset value in case the server is not valid smtp_server_data = smtp_server.data smtp_port_data = smtp_port and smtp_port.data or 25 smtp_use_tls_data = smtp_use_tls and smtp_use_tls.data or False smtp_user_data = smtp_user and smtp_user.data or None smtp_password_data = smtp_password and smtp_password.data or None # uses the base configuration values to try to retrieve the # smtp attributes, defaulting to the the pre-defined resource # based values in case the values are not found smtp_server_data = colony.conf("SMTP_HOST", smtp_server_data) smtp_port_data = colony.conf("SMTP_PORT", smtp_port_data, cast = int) smtp_use_tls_data = colony.conf("SMTP_STARTTLS", smtp_use_tls_data, cast = bool) smtp_user_data = colony.conf("SMTP_USER", smtp_user_data) smtp_password_data = colony.conf("SMTP_PASSWORD", smtp_password_data) # returns the smtp attributes as a set of tuple values to # be unpacked in the calling method return ( smtp_server_data, smtp_port_data, smtp_use_tls_data, smtp_user_data, smtp_password_data )
def main(): kwargs = dict() server = colony.conf("SERVER", "legacy") host = colony.conf("HOST", "127.0.0.1") port = colony.conf("PORT", "8080") ssl = colony.conf("SSL", False, cast=bool) key_file = colony.conf("KEY_FILE", None) cer_file = colony.conf("CER_FILE", None) for name, value in os.environ.items(): if not name.startswith("SERVER_"): continue name_s = name.lower()[7:] if name_s in EXCLUDED_NAMES: continue kwargs[name_s] = value host = str(host) port = str(port) hosts = [value.strip() for value in host.split(",")] ports = [int(value.strip()) for value in port.split(",")] serve_multiple(server=server, hosts=hosts, ports=ports, ssl=ssl, key_file=key_file, cer_file=cer_file, kwargs=kwargs) def handler(signum=None, frame=None): raise SystemExit() signal.signal(signal.SIGTERM, handler) run = True while run: try: time.sleep(86400) except (KeyboardInterrupt, SystemExit): run = False
def main(): kwargs = dict() server = colony.conf("SERVER", "legacy") host = colony.conf("HOST", "127.0.0.1") port = colony.conf("PORT", "8080") ssl = colony.conf("SSL", False, cast = bool) key_file = colony.conf("KEY_FILE", None) cer_file = colony.conf("CER_FILE", None) for name, value in os.environ.items(): if not name.startswith("SERVER_"): continue name_s = name.lower()[7:] if name_s in EXCLUDED_NAMES: continue kwargs[name_s] = value host = str(host) port = str(port) hosts = [value.strip() for value in host.split(",")] ports = [int(value.strip()) for value in port.split(",")] serve_multiple( server = server, hosts = hosts, ports = ports, ssl = ssl, key_file = key_file, cer_file = cer_file, kwargs = kwargs ) def handler(signum = None, frame = None): raise SystemExit() signal.signal(signal.SIGTERM, handler) run = True while run: try: time.sleep(86400) except (KeyboardInterrupt, SystemExit): run = False
def parse_configuration(cwd, mode, config_file_path, level, layout_mode, run_mode, daemon_file_path, logger_path, library_path, meta_path, plugin_path, manager_path): """ Parses the configuration using the given values as default values. The configuration file used is given as a parameter to the function. :type cwd: String :param cwd: The (original) current working directory to be used in the resolution of the relative import values. :type mode: String :param mode: The mode that is going to be used for non standard execution of the plugin manager (eg: testing). This value is not set by default (for default execution). :type config_file_path: Sting :param config_file_path: The path to the configuration file. :type level: String :param level: The logging level value described as a string that is going to be used in the plugin system. :type layout_mode: String :param layout_mode: The layout mode to be used by the plugin system. :type run_mode: String :param run_mode: The run mode to be used by the plugin system. :type daemon_file_path: String :param daemon_file_path: The file path to the daemon file, for information control. :type logger_path: String :param logger_path: The path to the logger. :type library_path: String :param library_path: The set of paths to the various library locations separated by a semi-column. :type meta_path: String :param meta_path: The set of paths to the various meta locations separated by a semi-column. :type plugin_path: String :param plugin_path: The set of paths to the various plugin locations separated by a semi-column. :type manager_path: String :param manager_path: The path to the plugin system. :rtype: Tuple :return: The tuple with the values parsed value. """ # retrieves the configuration directory from the configuration # file path (the directory is going to be used to include the module) config_dir = os.path.dirname(config_file_path) # in case the configuration directory path is not an absolute path if not os.path.isabs(config_dir): # creates the (complete) configuration directory path # prepending the manager path config_dir = os.path.normpath(manager_path + "/" + config_dir) # in case the configuration directory path is valid inserts it into the system path config_dir and sys.path.insert(0, config_dir) # retrieves the configuration file base path from the configuration file path config_file_base_path = os.path.basename(config_file_path) # retrieves the configuration module name and the configuration # module extension by splitting the configuration base path into # base name and extension and then imports the referring module config_module, _configuration_module_extension = os.path.splitext( config_file_base_path) try: config = __import__(config_module) except ImportError: import colony.config.base as module config = module # retrieves the contents of the configuration file that has just # been loaded, this is the default operation to be performed names = dir(config) # sets the proper configuration attributes taking into account the # presence or not of such attributes in the loaded file if "level" in names: level = level or config.level if "layout_mode" in names: layout_mode = layout_mode or config.layout_mode if "run_mode" in names: run_mode = run_mode or config.run_mode if "prefix_paths" in names: prefix_paths = config.prefix_paths if "stop_on_cycle_error" in names: stop_on_cycle_error = config.stop_on_cycle_error if "daemon_file_path" in names: daemon_file_path = config.daemon_file_path if "logger_path" in names: logger_path = config.logger_path # tries to load some of the environment "loadable" properties for the # starting of the colony infra-structure, defaulting to the provided # values in case they are present (as expected), then in case there's # still no valid values for such variables default values are used mode = colony.conf("MODE", mode) level = colony.conf("LEVEL", level) layout_mode = colony.conf("LAYOUT_MODE", layout_mode) run_mode = colony.conf("RUN_MODE", run_mode) layout_mode = layout_mode or "default" run_mode = run_mode or "default" # retrieves the complete set of configuration variables associated # with the various paths to be used by colony and then adds the # proper static file based configuration values to them, so that # these list are properly started with the initial values library_path_list = colony.conf("LIBRARY_PATH", [], cast=list) meta_path_list = colony.conf("META_PATH", [], cast=list) plugin_path_list = colony.conf("PLUGIN_PATH", [], cast=list) library_path_list = library_path_list + config.library_path_list meta_path_list = meta_path_list + config.meta_path_list plugin_path_list = plugin_path_list + config.plugin_path_list # in case the library path is defined, must appends a # separator to the library path to mark the initial separation # otherwise creates a new library path string initializing the # value to an empty string so that it can be extended if library_path: library_path += ";" else: library_path = "" # in case the meta path is defined, must appends a # separator to the meta path to mark the initial separation # otherwise creates a new meta path string initializing the # value to an empty string so that it can be extended if meta_path: meta_path += ";" else: meta_path = "" # in case the plugin path is defined, must appends a # separator to the plugin path to mark the initial separation # otherwise creates a new plugin path string initializing the # value to an empty string so that it can be extended if plugin_path: plugin_path += ";" else: plugin_path = "" # retrieves the current prefix paths current_prefix_paths = prefix_paths[layout_mode] # retrieves the extra library path as the dereferenced values # from the colony configuration library path list and adds the # extra library path to the library path extra_library_path = convert_reference_path_list(cwd, manager_path, current_prefix_paths, library_path_list) library_path += extra_library_path # retrieves the extra meta path as the dereferenced values # from the colony configuration meta path list and adds the # extra meta path to the meta path extra_meta_path = convert_reference_path_list(cwd, manager_path, current_prefix_paths, meta_path_list) meta_path += extra_meta_path # loads the plugin paths file path and adds the plugin paths # file path to the plugin path plugin_paths_file_path = load_plugin_paths_file(manager_path) plugin_path += plugin_paths_file_path # retrieves the extra plugin path as the dereferenced values # from the colony configuration plugin path list and adds the # extra plugin path to the plugin path extra_plugin_path = convert_reference_path_list(cwd, manager_path, current_prefix_paths, plugin_path_list) plugin_path += extra_plugin_path return (mode, level, layout_mode, run_mode, stop_on_cycle_error, current_prefix_paths, daemon_file_path, logger_path, library_path, meta_path, plugin_path)
def setUp(self): colony.ColonyTestCase.setUp(self) self.httpbin = colony.conf("HTTPBIN", "httpbin.org")
def _install(name=None, id=None, version=None, upgrade=False): import appier # verifies if the provided version string is wildcard based and # for such situations invalidated the version value (sets to invalid) if version == "x.x.x": version = None # constructs the proper description string taking into account # if the name or the id has been provided and then prints a # message about the installation operation that is going to start description = name or id output("Installing package %s" % description) # creates the map containing the various parameters that are # going to be sent as part of the filtering process for the # remote request of package retrieval params = dict() if name: params["name"] = name if id: params["identifier"] = id # retrieves the proper repository url that is currently defined # then enforces the value to be a valid sequence, so that the # logic is defined as cycle of url based package detection repo_url = colony.conf("REPO_URL", REPO_URL) if not type(repo_url) in (list, tuple): repo_url = (("colony", repo_url), ) # starts the variable that will hold the found package at invalid # so that the value is set only when a repo contains a package # matching the defined criteria package = None # iterates over the complete set of repositories defined in the # repository url value trying to find the proper package, note # that the package is found when at least one result is returned # matching the provided criteria (as defined in specification) for name, _repo_url in repo_url: url = _repo_url + "packages" result = appier.get(url, params=params) package = result[0] if result else dict() if not package: continue repo_url = _repo_url break # in case no package has been found for any of the defined repos # an exception must be raised indicating the problem to the user if not package: raise RuntimeError("Package not found") # constructs the proper url for package information retrieval and # runs it so that the complete set of information (including dependencies) # is gathered providing the system with the complete set of options url = repo_url + "packages/%s/info" % package["name"] info = appier.get(url, params=dict(version=version)) # verifies if the package is already installed under the current # system and if that's the case returns immediately as there's # nothing remaining to be done for such situation if _exists(info, upgrade=upgrade): output("Package %s is already installed, skipping" % description) return # runs the dependencies operation for the current package information # this operation should be able to install all the requirements for # the current package in transit (avoid package corruption) try: indent() _dependencies(info, upgrade=upgrade) finally: unindent() # prints information about the starting of the package download, this # is required for the user to be notified about such action output("Downloading %s" % description) # creates the proper package retrieval url and runs the remote get request # to try to retrieve the package contents of so that they are installed url = repo_url + "packages/%s" % info["short_name"] data = appier.get(url, params=dict(version=info["version"])) # creates a new temporary directory for the new bundle file that is going # to be created and stores it under such directory (for deployment) temp_path = tempfile.mkdtemp() target_path = os.path.join(temp_path, "%s.cbx" % info["short_name"]) file = open(target_path, "wb") try: file.write(data) finally: file.close() # runs the deployment process for the package bundle that has been retrieved # and then removes the temporary directory path, as it's no longer required _deploy(target_path, timestamp=info["timestamp"]) shutil.rmtree(temp_path) # prints a message about the end of the installation process for the current # package, this will allow the user to be aware of the end of operation output("Finished installing %s" % description)
def _upload(path, repo="colony", generate=True, delete=True): import json import appier # tries to runs the expansion of the glob for the provided path and # in case it expands to multiple values the same upload operation is # run on each of the items that compromise the expansion expansion = glob.glob(path) is_multiple = len(expansion) > 1 if is_multiple: for item in expansion: _upload(item, repo=repo, generate=generate, delete=delete) return if not expansion: raise RuntimeError("No path found for '%s'" % path) path = expansion[0] # in case the generate flag is active the package file is generated # for the path and then the descriptor information dictionary is read # so that it's possible to properly upload the file to the repository if generate: path = _generate(path) descriptor = _read(path) # opens the path of the package file that is going to be uploaded # in order to be able to read the full contents of it, these are # going to be the contents to be uploaded to the repository file = open(path, "rb") try: contents = file.read() finally: file.close() # tries to retrieve the currently targeted repository info taking # into account both the environment and the static values repo_url = colony.conf("REPO_URL", REPO_URL) repo_username = colony.conf("REPO_USERNAME", "root") repo_password = colony.conf("REPO_PASSWORD", "root") # enforces the repository url to be defined as a sequence for better # handling of the logic and then retrieves the requested target # repository base url value, raising an exception in case it's not found if not type(repo_url) in (list, tuple): repo_url = (("colony", repo_url), ) repo_url = dict(repo_url) repo_url = repo_url.get(repo) repo_url = colony.conf("REPO_URL_" + repo.upper(), repo_url) if not repo_url: raise RuntimeError("URL for repository '%s' not found" % repo) # prints a message about the upload operation that is going to occur # so that the end user knows where the upload is going output("Uploading %s into %s repo (%s)" % (descriptor["short_name"], repo, repo_url)) # creates the url format, taking into account the defined url and the # current descriptor and then runs the upload, using a post operation url = repo_url + "packages" login_url = repo_url + "api/admin/login" auth = appier.post(login_url, params=dict(username=repo_username, password=repo_password)) appier.post(url, data_m=dict(sid=auth["sid"], identifier=descriptor["id"], name=descriptor["short_name"], version=descriptor["version"], type=descriptor["type"], info=json.dumps(descriptor), contents=("contents", contents))) # in case the delete flag is active the package file is delete after # a proper upload operation is performed (house keeping) if delete: os.remove(path)
import colony # runs the resolution process in order to be able to retrieve # the "real" a "best" match for the manager path, this method # should decide between the personal and the master versions # then ensures that the proper directory tree is created manager_path = colony.resolve_manager(base_path) colony.ensure_tree(manager_path) # registers the ignore flag in the deprecation warnings so that # no message with this kind of warning is printed (clean console) warnings.filterwarnings("ignore", category=DeprecationWarning) # retrieves the layout mode that is going to be used for the # resolution of resources in the colony infra-structure layout_mode = colony.conf("LAYOUT_MODE", "default") # tries to retrieve the run mode from the currently set # environment variables, in case of failure defaults to # the default value (as expected by the specification) run_mode = colony.conf("RUN_MODE", "development") # tries to retrieve the prefix to be used to shorten the path # resolution in the request handling prefix = colony.conf("PREFIX", None) # gathers the path to the alias file that contains JSON information # about the mapping prefixes for the HTTP server, the file should # contain a set of prefix to resolution prefix values alias_path = colony.conf("ALIAS_PATH", None) alias_path = alias_path and os.path.expanduser(alias_path)
def parse_configuration( cwd, mode, config_file_path, level, layout_mode, run_mode, daemon_file_path, logger_path, library_path, meta_path, plugin_path, manager_path ): """ Parses the configuration using the given values as default values. The configuration file used is given as a parameter to the function. @type cwd: String @param cwd: The (original) current working directory to be used in the resolution of the relative import values. @type mode: String @param mode: The mode that is going to be used for non standard execution of the plugin manager (eg: testing). This value is not set by default (for default execution). @type config_file_path: Sting @param config_file_path: The path to the configuration file. @type level: String @param level: The logging level value described as a string that is going to be used in the plugin system. @type layout_mode: String @param layout_mode: The layout mode to be used by the plugin system. @type run_mode: String @param run_mode: The run mode to be used by the plugin system. @type daemon_file_path: String @param daemon_file_path: The file path to the daemon file, for information control. @type logger_path: String @param logger_path: The path to the logger. @type library_path: String @param library_path: The set of paths to the various library locations separated by a semi-column. @type meta_path: String @param meta_path: The set of paths to the various meta locations separated by a semi-column. @type plugin_path: String @param plugin_path: The set of paths to the various plugin locations separated by a semi-column. @type manager_path: String @param manager_path: The path to the plugin system. @rtype: Tuple @return: The tuple with the values parsed value. """ # retrieves the configuration directory from the configuration # file path (the directory is going to be used to include the module) config_dir = os.path.dirname(config_file_path) # in case the configuration directory path is not an absolute path if not os.path.isabs(config_dir): # creates the (complete) configuration directory path # prepending the manager path config_dir = os.path.normpath(manager_path + "/" + config_dir) # in case the configuration directory path is valid inserts it into the system path config_dir and sys.path.insert(0, config_dir) # retrieves the configuration file base path from the configuration file path config_file_base_path = os.path.basename(config_file_path) # retrieves the configuration module name and the configuration # module extension by splitting the configuration base path into # base name and extension and then imports the referring module config_module, _configuration_module_extension = os.path.splitext(config_file_base_path) try: config = __import__(config_module) except ImportError: import colony.config.base as module; config = module # retrieves the contents of the configuration file that has just # been loaded, this is the default operation to be performed names = dir(config) # sets the proper configuration attributes taking into account the # presence or not of such attributes in the loaded file if "level" in names: level = level or config.level if "layout_mode" in names: layout_mode = layout_mode or config.layout_mode if "run_mode" in names: run_mode = run_mode or config.run_mode if "prefix_paths" in names: prefix_paths = config.prefix_paths if "stop_on_cycle_error" in names: stop_on_cycle_error = config.stop_on_cycle_error if "daemon_file_path" in names: daemon_file_path = config.daemon_file_path if "logger_path" in names: logger_path = config.logger_path # tries to load some of the environment "loadable" properties for the # starting of the colony infra-structure, defaulting to the provided # values in case they are present (as expected), then in case there's # still no valid values for such variables default values are used mode = colony.conf("MODE", mode) level = colony.conf("LEVEL", level) layout_mode = colony.conf("LAYOUT_MODE", layout_mode) run_mode = colony.conf("RUN_MODE", run_mode) layout_mode = layout_mode or "default" run_mode = run_mode or "default" # retrieves the complete set of configuration variables associated # with the various paths to be used by colony and then adds the # proper static file based configuration values to them, so that # these list are properly started with the initial values library_path_list = colony.conf("LIBRARY_PATH", [], cast = list) meta_path_list = colony.conf("META_PATH", [], cast = list) plugin_path_list = colony.conf("PLUGIN_PATH", [], cast = list) library_path_list = library_path_list + config.library_path_list meta_path_list = meta_path_list + config.meta_path_list plugin_path_list = plugin_path_list + config.plugin_path_list # in case the library path is defined, must appends a # separator to the library path to mark the initial separation # otherwise creates a new library path string initializing the # value to an empty string so that it can be extended if library_path: library_path += ";" else: library_path = "" # in case the meta path is defined, must appends a # separator to the meta path to mark the initial separation # otherwise creates a new meta path string initializing the # value to an empty string so that it can be extended if meta_path: meta_path += ";" else: meta_path = "" # in case the plugin path is defined, must appends a # separator to the plugin path to mark the initial separation # otherwise creates a new plugin path string initializing the # value to an empty string so that it can be extended if plugin_path: plugin_path += ";" else: plugin_path = "" # retrieves the current prefix paths current_prefix_paths = prefix_paths[layout_mode] # retrieves the extra library path as the dereferenced values # from the colony configuration library path list and adds the # extra library path to the library path extra_library_path = convert_reference_path_list( cwd, manager_path, current_prefix_paths, library_path_list ) library_path += extra_library_path # retrieves the extra meta path as the dereferenced values # from the colony configuration meta path list and adds the # extra meta path to the meta path extra_meta_path = convert_reference_path_list( cwd, manager_path, current_prefix_paths, meta_path_list ) meta_path += extra_meta_path # loads the plugin paths file path and adds the plugin paths # file path to the plugin path plugin_paths_file_path = load_plugin_paths_file(manager_path) plugin_path += plugin_paths_file_path # retrieves the extra plugin path as the dereferenced values # from the colony configuration plugin path list and adds the # extra plugin path to the plugin path extra_plugin_path = convert_reference_path_list( cwd, manager_path, current_prefix_paths, plugin_path_list ) plugin_path += extra_plugin_path return ( mode, level, layout_mode, run_mode, stop_on_cycle_error, current_prefix_paths, daemon_file_path, logger_path, library_path, meta_path, plugin_path )
def __init__(self, plugin): colony.System.__init__(self, plugin) self.data = dict() self.state = dict() self.diagnostics = colony.conf("DIAGNOSTICS", False, cast = bool)
import colony # runs the resolution process in order to be able to retrieve # the "real" a "best" match for the manager path, this method # should decide between the personal and the master versions # then ensures that the proper directory tree is created manager_path = colony.resolve_manager(base_path) colony.ensure_tree(manager_path) # registers the ignore flag in the deprecation warnings so that # no message with this kind of warning is printed (clean console) warnings.filterwarnings("ignore", category = DeprecationWarning) # retrieves the layout mode that is going to be used for the # resolution of resources in the colony infra-structure layout_mode = colony.conf("LAYOUT_MODE", "default") # tries to retrieve the run mode from the currently set # environment variables, in case of failure defaults to # the default value (as expected by the specification) run_mode = colony.conf("RUN_MODE", "development") # tries to retrieve the prefix to be used to shorten the path # resolution in the request handling prefix = colony.conf("PREFIX", None) # gathers the path to the alias file that contains json information # about the mapping prefixes for the http server, the file should # contain a set of prefix to resolution prefix values alias_path = colony.conf("ALIAS_PATH", None) alias_path = alias_path and os.path.expanduser(alias_path)