def get_config_template():
return {
'enabled': confuse.TypeTemplate(bool),
'friends': confuse.StrSeq(),
'tag_keywords': confuse.StrSeq(),
'friend_keywords': confuse.StrSeq(),
'number_keywords': NumberKeywordsTemplate()
}
def handle(self):
from trakt_scrobbler import config
import confuse
whitelist = config["fileinfo"]["whitelist"].get(confuse.StrSeq(default=[]))
if not whitelist:
self.line("Whitelist empty!")
return
whitelist = list(sorted(whitelist))
choices = self.choice(
"Select the paths to be removed from whitelist", whitelist, multiple=True
)
paths = ', '.join(fmt(c) for c in choices)
if not self.confirm(
f"This will remove {paths} from whitelist. Continue?",
default=True,
):
self.line("Aborted", "error")
return
for choice in choices:
whitelist.remove(choice)
config["fileinfo"]["whitelist"] = whitelist
ConfigCommand.save_config(config)
self.call_sub("whitelist show")
async def main(
config: confuse.Configuration,
*,
from_rev: Optional[str],
from_last_tag: bool,
to_rev: str,
reverse: bool,
) -> AsyncIterable[git.Commit]:
if from_last_tag:
if from_rev is not None:
logger.warning(
"--from-last-tag is ignored when combined with --from")
else:
excluded = config["tags"]["exclude"].get(
confuse.StrSeq(split=False))
try:
tag_filter = config["tags"]["filter"].get(str)
except confuse.NotFoundError:
tag_filter = None
tags = await git.get_tags(pattern=tag_filter,
sort="creatordate",
reverse=True)
from_rev = next(tag["name"] for tag in tags
if tag["name"] not in excluded)
async for commit in git.get_commits(start=from_rev,
end=to_rev,
reverse=reverse):
yield commit
def main():
register_exception_handler()
assert get_access_token()
scrobble_queue = Queue()
backlog_cleaner = BacklogCleaner()
scrobbler = Scrobbler(scrobble_queue, backlog_cleaner)
scrobbler.start()
allowed_monitors = config['players']['monitored'].get(
confuse.StrSeq(default=[]))
all_monitors = collect_monitors()
unknown = set(allowed_monitors).difference(Mon.name
for Mon in all_monitors)
if unknown:
logger.warning(f"Unknown player(s): {', '.join(unknown)}")
for Mon in all_monitors:
if Mon.name not in allowed_monitors:
continue
mon = Mon(scrobble_queue)
if not mon or not mon._initialized:
logger.warning(f"Could not start monitor for {Mon.name}")
continue
mon.start()
def get_config_template():
return {
'enabled': confuse.TypeTemplate(bool),
'keywords': confuse.StrSeq(),
'max_following': confuse.Integer(),
'multiple': confuse.TypeTemplate(bool)
}
def __init__(self, config: confuse.ConfigView) -> None:
subject_regex = self._get_subject_regex(
config["types"].get(confuse.StrSeq(split=False))
)
self._parsers: ParserCollection = {
"subject": lambda text: subject_regex.match(text),
"body": lambda text: self._body_regex.match(text),
"footer": lambda text: self._footer_regex.finditer(text),
}
def get_config_template():
# Fixme: minor hack because StrSeq doesnt support default
strseq = confuse.StrSeq()
strseq.default = []
return {
'enabled': confuse.TypeTemplate(bool),
'keywords': strseq,
'users': strseq,
}
def handle(self):
from trakt_scrobbler import config
import confuse
whitelist = config["fileinfo"]["whitelist"].get(confuse.StrSeq(default=[]))
if not whitelist:
self.line("Whitelist empty!")
return
self.info("Whitelist:")
for path in whitelist:
self.line(path)
self.line("Don't forget to restart the service for any changes to take effect.")
def handle(self):
from trakt_scrobbler import config
import confuse
whitelist = config["fileinfo"]["whitelist"].get(confuse.StrSeq(default=[]))
if not whitelist:
self.line("Whitelist empty!")
return
whitelist = list(sorted(whitelist))
self.info("Whitelisted paths:")
for path in whitelist:
self.line(path)
async def _yield_commits() -> AsyncIterable[Change]:
from .list_commits import main as list_commits
from .parse_commit import main as parse_commit
def _yield_commit_range(from_rev: Optional[str],
to_rev: str) -> AsyncIterable[Change]:
return parse_commit(
config,
input=list_commits(
config,
from_rev=from_rev,
from_last_tag=False,
to_rev=to_rev,
reverse=True,
),
include_unparsed=True,
)
excluded = config["tags"]["exclude"].get(confuse.StrSeq(split=False))
try:
tag_filter = config["tags"]["filter"].get(str)
except confuse.NotFoundError:
tag_filter = None
from_rev = None # type: Optional[str]
for tag in await git.get_tags(pattern=tag_filter, sort="creatordate"):
if tag["name"] in excluded:
continue
to_rev = tag["name"]
if from_rev is None:
logger.debug(f"Retrieving commits up until, {to_rev}")
else:
logger.debug(
f"Retrieving commits from, {from_rev}, to, {to_rev}")
async for commit in _yield_commit_range(from_rev, to_rev):
yield commit
from_rev = to_rev
logger.debug(f"Retrieving remaining commits since {from_rev}")
to_rev = "HEAD"
async for commit in _yield_commit_range(from_rev, "HEAD"):
yield commit
def convert(self, value, view):
if isinstance(value, dict):
return value
else:
self.fail(u"must be a dict", view, True)
_metric_template = {
"name": confuse.String(),
"type": confuse.Choice(choices=["Enum", "Gauge"]),
"desc": confuse.String(default=""),
"kwargs": Dict(default={}),
"make_label": confuse.Choice(choices=[True, False], default=False),
"uve_type": confuse.String(),
"uve_module": confuse.String(),
"uve_instances": confuse.StrSeq(default=[]),
"json_path": confuse.String(),
"append_field_name": confuse.Choice(choices=[True, False], default=True),
"labels_from_path": Dict(default={}),
}
_global_template = {
"analytics": {
"host": confuse.String(pattern=r"^https?://"),
"base_url": confuse.String(pattern=r"^/", default="/analytics/uves"),
},
"prometheus": {
"port": confuse.Integer(default=8080),
"metric_name_prefix": confuse.String(default="tungsten"),
},
"scraper": {
from lib import github_authenticate
from lib.dates import set_start_date
from lib.gh import create_data_for_spreadsheet
from lib.gh import get_filtered_repos
from lib.gh import get_users_by_ids
from lib.gh import get_all_users_issues
from lib.xlsx import create_worksheet
from lib.xlsx import get_workbook_and_formats
from lib.xlsx import write_gh_data_to_worksheet
import confuse
import argparse
template = {
'users_ids': confuse.StrSeq(),
'repos': confuse.StrSeq(),
'milestone': confuse.StrSeq()
}
def main(users_ids, repos, milestone):
github_authenticate(args.token)
set_start_date(args.date)
repos = get_filtered_repos(repos)
users = get_users_by_ids(users_ids)
all_issues = get_all_users_issues(repos, users, milestone)
worksheet_data = create_data_for_spreadsheet(all_issues, users)
workbook, workbook_formats = get_workbook_and_formats()
worksheet = create_worksheet(workbook, workbook_formats)
write_gh_data_to_worksheet(worksheet, worksheet_data, workbook_formats)
workbook.close()
async def main(
config: confuse.Configuration,
*,
input: AsyncIterable[Change],
include_unparsed: bool,
unreleased_version: Optional[str],
) -> jinja2.environment.TemplateStream:
def _is_unreleased(tag: Optional[git.Tag]) -> bool:
return tag is None or tag["name"] == unreleased_version
def _read_config(view: confuse.ConfigView,
default: Any = DEFAULT,
typ: Type = str) -> Any:
try:
return view.get(typ)
except confuse.NotFoundError:
if default is DEFAULT:
raise
return default
excluded = config["tags"]["exclude"].get(confuse.StrSeq(split=False))
try:
tag_filter = config["tags"]["filter"].get(str)
except confuse.NotFoundError:
tag_filter = None
def _ignore_tag(tag: str) -> bool:
if tag_filter and not fnmatch.fnmatch(tag, tag_filter):
return True
if tag in excluded:
return True
return False
versions: List[VersionTuple] = []
version = Version()
async for change in input:
if change["data"] is not None or include_unparsed:
data = change["data"] or {}
typ = data.get("subject", {}).get("type")
if typ not in version:
version[typ] = []
version[typ].append(change)
if change["source"]["tags"]:
tags = [
tag for tag in change["source"]["tags"]
if not _ignore_tag(tag["name"])
]
if tags:
tag = sorted(tags, key=lambda tag: tag["name"])[0]
logger.debug(f"Found new version tag, {tag['name']} "
f"({len(version.get_commits())} commit(s))")
versions.append((tag, version))
version = Version()
if version.has_commits():
logger.debug(
f"Found {len(version.get_commits())} unreleased commit(s)")
unreleased_tag: Optional[git.Tag] = None
if unreleased_version is not None:
logger.debug(
f"Using {unreleased_version} as the version for unreleased commit(s)"
)
unreleased_tag = {
"name": unreleased_version,
"object_name": "",
"subject": "",
"body": "",
}
versions.append((unreleased_tag, version))
logger.debug(f"{len(versions)} versions found")
if not any(version.has_commits() for _, version in versions):
raise exceptions.NoCommitsError()
# Reverse versions list so that it is in reverse chronological order
# (ie. most recent release first)
versions.reverse()
loaders: List[jinja2.BaseLoader] = []
for package in config["template"]["package"].get(
confuse.StrSeq(split=False)):
logger.debug(f"Creating package loader for: {package}")
loaders.append(jinja2.PackageLoader(package))
for directory in config["template"]["directory"].get(
confuse.Sequence(Filename())):
logger.debug(f"Creating file-system loader for: {directory}")
loaders.append(jinja2.FileSystemLoader(directory))
environment = jinja2.Environment(
loader=jinja2.ChoiceLoader(loaders),
extensions=["jinja2.ext.loopcontrols"],
)
environment.filters["read_config"] = _read_config
environment.tests["unreleased"] = _is_unreleased
template = environment.get_template(config["template"]["name"].get(str))
template_config = config["template"]["config"]
return template.stream(versions=versions,
config=template_config,
confuse=confuse)
"mongo": {
"url": str,
"name": str,
"username": optional(str),
"password": optional(str),
},
"interface": {
"port":
confuse.OneOf([confuse.String(pattern="COM\\d+$"),
confuse.Filename()]),
"baudrate":
int,
"encoding":
optional(str, "utf-8"),
},
"manufacturer_code": confuse.StrSeq(),
"secret_key": optional(str),
"max_session_age": confuse.OneOf([int, None]),
"https": optional(bool, False),
"sentry_dsn": optional(str),
"sentry_csp_url": optional(str),
"log_level": optional(str),
}
config = confuse.LazyConfig("cherrydoor", __name__)
def add_args(parser):
"""Add arguments to the arparse parser.
Parameters
def test_invalid_sequence_type(self):
config = _root({'foo': ['bar', 2126]})
with self.assertRaises(confuse.ConfigTypeError):
config['foo'].get(confuse.StrSeq())
from functools import lru_cache
from typing import Union, List
from urllib.parse import unquote, urlsplit, urlunsplit
import confuse
import guessit
from trakt_scrobbler import config, logger
from trakt_scrobbler.utils import RegexPat, cleanup_encoding, is_url
from urlmatch import BadMatchPattern, urlmatch
cfg = config["fileinfo"]
whitelist = cfg["whitelist"].get(confuse.StrSeq())
regexes: dict = cfg['include_regexes'].get({
"movie":
confuse.Sequence(RegexPat()),
"episode":
confuse.Sequence(RegexPat()),
})
use_regex = any(regexes.values())
exclude_patterns: list = cfg["exclude_patterns"].get(
confuse.Sequence(RegexPat()))
def split_whitelist(whitelist: List[str]):
"""Split whitelist into local and remote urls"""
local, remote = [], []
for path in whitelist:
try:
urlmatch(path,
"<dummy_path>",
path_required=False,
def test_whitespace_separated_string(self):
config = _root({'foo': 'bar baz'})
valid = config['foo'].get(confuse.StrSeq())
self.assertEqual(valid, ['bar', 'baz'])
def test_string_tuple(self):
config = _root({'foo': ('bar', 'baz')})
valid = config['foo'].get(confuse.StrSeq())
self.assertEqual(valid, ['bar', 'baz'])
class Config:
template = {
'twitter': {
'consumer_key': confuse.String(),
'consumer_secret': confuse.String(),
'access_token_key': confuse.String(),
'access_token_secret': confuse.String(),
'min_ratelimit_percent': confuse.Integer(),
'search': {
'queries': confuse.TypeTemplate(list),
'max_queue': confuse.Integer(),
'max_quote_depth': confuse.Integer(),
'min_quote_similarity': confuse.Number(),
'skip_retweeted': confuse.TypeTemplate(bool),
'filter': {
'min_retweets': {
'enabled': confuse.TypeTemplate(bool),
'number': confuse.Integer()
}
},
'sort': {
'by_keywords': {
'enabled': confuse.TypeTemplate(bool),
'keywords': confuse.StrSeq()
},
'by_age': {
'enabled': confuse.TypeTemplate(bool),
},
'by_retweets_count': {
'enabled': confuse.TypeTemplate(bool),
}
}
},
'actions': {
'follow': {
'enabled': confuse.TypeTemplate(bool),
'keywords': confuse.StrSeq(),
'max_following': confuse.Integer(),
'multiple': confuse.TypeTemplate(bool)
},
'favorite': {
'enabled': confuse.TypeTemplate(bool),
'keywords': confuse.StrSeq()
},
'tag_friend': {
'enabled': confuse.TypeTemplate(bool),
'friends': confuse.StrSeq(),
}
},
'scheduler': {
'search_interval': confuse.Integer(),
'retweet_interval': confuse.Integer(),
'retweet_random_margin': confuse.Integer(),
'blocked_users_update_interval': confuse.Integer(),
'clear_queue_interval': confuse.Integer(),
'rate_limit_update_interval': confuse.Integer(),
'check_mentions_interval': confuse.Integer(),
},
},
'notifiers': {
'mail': {
'enabled': confuse.TypeTemplate(bool),
'host': confuse.String(),
'port': confuse.Integer(),
'tls': confuse.TypeTemplate(bool),
'username': confuse.String(),
'password': confuse.String(),
'recipient': confuse.String()
},
'pushbullet': {
'enabled': confuse.TypeTemplate(bool),
'token': confuse.String()
}
}
}
_valid = None
@staticmethod
def get():
"""
Gets the static config object
:return:
"""
if Config._valid is None:
raise ValueError("Configuration not loaded")
return Config._valid
@staticmethod
def load(filename=None):
"""
Loads a file and imports the settings
:param filename: the file to import
"""
config = confuse.LazyConfig('Yatcobot', __name__)
# Add default config when in egg (using this way because egg is breaking the default way)
if len(config.sources) == 0:
default_config_text = pkg_resources.resource_string("yatcobot", "config_default.yaml")
default_config = confuse.ConfigSource(yaml.load(default_config_text, Loader=confuse.Loader),
'pkg/config_default.yaml',
True)
config.add(default_config)
# Add user specified config
if filename is not None and os.path.isfile(filename):
config.set_file(filename)
logger.info('Loading config files (From highest priority to lowest):')
for i, config_source in enumerate(config.sources):
logger.info('{}: Path: {}'.format(i, config_source.filename))
Config._valid = config.get(Config.template)
"""An example application using Confuse for configuration."""
from __future__ import division, absolute_import, print_function
import confuse
import argparse
template = {
'library': confuse.Filename(),
'import_write': confuse.OneOf([bool, 'ask', 'skip']),
'ignore': confuse.StrSeq(),
'plugins': list,
'paths': {
'directory': confuse.Filename(),
'default': confuse.Filename(relative_to='directory'),
}
}
config = confuse.LazyConfig('ConfuseExample', __name__)
def main():
parser = argparse.ArgumentParser(description='example Confuse program')
parser.add_argument('--library', '-l', dest='library', metavar='LIBPATH',
help='library database file')
parser.add_argument('--directory', '-d', dest='directory',
metavar='DIRECTORY',
help='destination music directory')
parser.add_argument('--verbose', '-v', dest='verbose', action='store_true',
help='print debugging messages')
def get_config_template():
template = {
'enabled': confuse.TypeTemplate(bool),
'keywords': confuse.StrSeq()
}
return template
def test_invalid_type(self):
config = _root({'foo': 9})
with self.assertRaises(confuse.ConfigTypeError):
config['foo'].get(confuse.StrSeq())
"""An example application using Confuse for configuration."""
from __future__ import division, absolute_import, print_function
import confuse
import argparse
template = {
'library': confuse.Filename(),
'import_write': confuse.OneOf([bool, 'ask', 'skip']),
'ignore': confuse.StrSeq(),
'plugins': list,
'paths': {
'directory': confuse.Filename(),
'default': confuse.Filename(relative_to='directory'),
},
'servers': confuse.Sequence({
'hostname': str,
'options': confuse.StrSeq(),
})
}
config = confuse.LazyConfig('ConfuseExample', __name__)
def main():
parser = argparse.ArgumentParser(description='example Confuse program')
parser.add_argument('--library',
'-l',
dest='library',
metavar='LIBPATH',
help='library database file')
parser.add_argument('--directory',
from IPython import get_ipython, InteractiveShell
from azmeta.access.config import direct
from azmeta.access.context import AzmetaResourceContext, default_resource_context
import confuse
_billing_scope_template = {'accounts': confuse.StrSeq(split=False)}
def resource_context() -> AzmetaResourceContext:
return default_resource_context()
def connect_kqlmagic() -> None:
ipython: InteractiveShell = get_ipython()
c = direct()
cluster = c['azmeta_kusto']['cluster'].as_str()
database = c['azmeta_kusto']['database'].as_str()
try:
default_billing_scope = c['azmeta']['default_billing_scope'].get(
_billing_scope_template)
except confuse.NotFoundError:
default_billing_scope = None
pass
ipython.run_line_magic("config", "Kqlmagic.auto_popup_schema=False")
ipython.run_line_magic("load_ext", "Kqlmagic")
ipython.run_line_magic(
"kql",
f"azuredataexplorer://code;cluster='{cluster}';database='{database}';alias='azmeta' -try_azcli_login"
)