def test_secret_entry(self):
config_entry = StringConfigEntry(key_path=["string"], required=False)
self.assertFalse(config_entry.secret)
config_entry = StringConfigEntry(key_path=["string"],
required=False,
secret=False)
self.assertFalse(config_entry.secret)
config_entry = StringConfigEntry(key_path=["string"],
required=False,
secret=True)
self.assertTrue(config_entry.secret)
def test_ignore_case(self):
str_entry1 = StringConfigEntry(
key_path=["tHiS", "Is", "CaseInsensitive"], default="value")
str_entry2 = StringConfigEntry(
key_path=["This", "is", "caseinsensitive"], default="value")
str_entry3 = StringConfigEntry(key_path=["this", "or", "iSit"],
default="value")
source = YamlSource("case_insensitive", ["./", "./tests"],
ignore_case_in_keys=True)
source.load()
self.assertTrue(source.has(str_entry1))
self.assertEqual(source.get(str_entry1), "value")
self.assertTrue(source.has(str_entry2))
self.assertEqual(source.get(str_entry2), "value")
self.assertTrue(source.has(str_entry3))
self.assertEqual(source.get(str_entry3), "?")
def test_env(self):
str_entry = StringConfigEntry(
key_path=["test-ing", "key1"],
default="value"
)
int_entry = IntConfigEntry(
key_path=["testing", "key2"],
default=2
)
source = EnvSource()
original_key = EnvSource.env_key(str_entry)
expected = "expected"
with mock.patch.dict(os.environ, {original_key: expected}, clear=True):
source.load()
self.assertTrue(source.has(str_entry))
self.assertEqual(source.get(str_entry), expected)
self.assertFalse(source.has(int_entry))
normalized_env_key = original_key.replace('-', '_')
self.assertNotEqual(original_key, normalized_env_key)
with mock.patch.dict(os.environ, {normalized_env_key: expected + '2'}, clear=True):
source.load()
self.assertTrue(source.has(str_entry))
self.assertEqual(source.get(str_entry), expected + '2')
self.assertFalse(source.has(int_entry))
def test_string_entry(self):
config_entry = StringConfigEntry(key_path=["string"], required=False)
input_output = [("5", "5"), ("hello", "hello"),
("$stuff=)(&/%$§", "$stuff=)(&/%$§"), ("", ""),
("None", "None"), (None, None)]
self.assert_input_output(config_entry, input_output)
def test_json(self):
str_entry = StringConfigEntry(key_path=["testing", "key1"],
default="value")
int_entry = IntConfigEntry(key_path=["testing", "key2"], default=2)
source = JsonSource("test")
source.load()
self.assertTrue(source.has(str_entry))
self.assertEqual(source.get(str_entry), "value")
self.assertTrue(source.has(int_entry))
self.assertEqual(source.get(int_entry), 2)
class Config(ConfigBase):
def __new__(cls, *args, **kwargs):
if "data_sources" not in kwargs.keys():
yaml_source = YamlSource("n26")
toml_source = TomlSource("n26")
data_sources = [EnvSource(), yaml_source, toml_source]
kwargs["data_sources"] = data_sources
return super(Config, cls).__new__(cls, *args, **kwargs)
USERNAME = StringConfigEntry(description="N26 account username",
example="*****@*****.**",
key_path=[NODE_ROOT, "username"],
required=True)
PASSWORD = StringConfigEntry(description="N26 account password",
example="$upersecret",
key_path=[NODE_ROOT, "password"],
required=True,
secret=True)
DEVICE_TOKEN = StringConfigEntry(
description="N26 device token",
example="00000000-0000-0000-0000-000000000000",
key_path=[NODE_ROOT, "device_token"],
required=True,
regex=
"[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}",
)
LOGIN_DATA_STORE_PATH = FileConfigEntry(
description="File path to store login data",
example="~/.config/n26/token_data",
key_path=[NODE_ROOT, "login_data_store_path"],
required=False,
default=None)
MFA_TYPE = StringConfigEntry(
description="Multi-Factor-Authentication type to use",
example=MFA_TYPE_APP,
key_path=[NODE_ROOT, "mfa_type"],
regex="^({})$".format("|".join([MFA_TYPE_APP, MFA_TYPE_SMS])),
default=MFA_TYPE_APP)
class TestConfigBase(ConfigBase):
BOOL = BoolConfigEntry(key_path=["test", "bool"], default=False)
STRING = StringConfigEntry(key_path=["test", "string"],
default="default value")
REGEX = RegexConfigEntry(key_path=["test", "regex"],
default="^[a-zA-Z0-9]$")
INT = IntConfigEntry(key_path=["test", "int"], default=100)
FLOAT = FloatConfigEntry(key_path=["test", "float"], default=1.23)
DATE = DateConfigEntry(
key_path=["test", "this", "date", "is", "nested", "deep"],
default=datetime.now())
TIMEDELTA = TimeDeltaConfigEntry(
key_path=["test", "this", "timediff", "is", "in", "this", "branch"],
default=timedelta(seconds=10))
FILE = FileConfigEntry(key_path=["test", "file"], )
DIRECTORY = DirectoryConfigEntry(key_path=["test", "directory"], )
RANGE = RangeConfigEntry(key_path=["test", "this", "is", "a", "range"],
default=Range(0, 100))
DICT = DictConfigEntry(key_path=["dict"], schema=Schema({str: str}))
DICT_LIST = ListConfigEntry(
item_type=DictConfigEntry,
item_args={"schema": Schema({str: str})},
key_path=[
"dict_list",
],
)
STRING_LIST = ListConfigEntry(item_type=StringConfigEntry,
key_path=["test", "this", "is", "a", "list"],
example=["these", "are", "test", "values"],
secret=False)
NONE_INT = IntConfigEntry(
key_path=["none", "int"],
default=None,
)
NONE_DATE = DateConfigEntry(
key_path=["none", "date"],
default=None,
)
SECRET_BOOL = BoolConfigEntry(key_path=["secret", "bool"],
default=False,
secret=True)
SECRET_INT = IntConfigEntry(key_path=["secret", "int"],
default=None,
secret=True)
SECRET_REGEX = RegexConfigEntry(key_path=["secret", "regex"],
default=None,
secret=True)
SECRET_LIST = ListConfigEntry(item_type=RegexConfigEntry,
key_path=["secret", "list"],
default=["[a-zA-Z]*"],
secret=True)
class Config(ConfigBase):
def __new__(cls, *args, **kwargs):
data_sources = [
EnvSource(),
YamlSource(FILE_NAME),
TomlSource(FILE_NAME)
]
kwargs["data_sources"] = data_sources
return super(Config, cls).__new__(cls, *args, **kwargs)
TELEGRAM_BOT_TOKEN = StringConfigEntry(
key_path=[KEY_ROOT, KEY_TELEGRAM, "bot_token"],
description="ID of the telegram chat to send messages to.",
required=True,
secret=True,
example="123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11")
TELEGRAM_CHAT_ID = StringConfigEntry(
key_path=[KEY_ROOT, KEY_TELEGRAM, "chat_id"],
description="ID of the telegram chat to send messages to.",
required=True,
secret=True,
example="-123456789")
RETRY_ENABLED = BoolConfigEntry(
key_path=[KEY_ROOT, KEY_RETRY, "enabled"],
description="Whether to retry sending messages or not.",
default=True,
)
RETRY_TIMEOUT = TimeDeltaConfigEntry(
key_path=[KEY_ROOT, KEY_RETRY, "timeout"],
description="Timeout between tries.",
default="10s",
)
RETRY_GIVE_UP_AFTER = TimeDeltaConfigEntry(
key_path=[KEY_ROOT, KEY_RETRY, "give_up_after"],
description="Time interval after which the retry should be cancelled.",
default="1h",
)
def test_string_entry_regex(self):
config_entry = StringConfigEntry(key_path=["string"],
required=True,
regex="^[0-9]*$")
input_output = [
("5", "5"),
("hello", ValueError),
("$stuff=)(&/%$§", ValueError),
]
self.assert_input_output(config_entry, input_output)
class AppConfig(ConfigBase):
def __new__(cls, *args, **kwargs):
yaml_source = YamlSource("deinemudda")
data_sources = [EnvSource(), yaml_source]
return super(AppConfig, cls).__new__(cls, data_sources=data_sources)
TELEGRAM_BOT_TOKEN = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_TELEGRAM, "bot_token"],
required=True,
secret=True,
example="123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11")
TELEGRAM_ADMIN_USERNAMES = ListConfigEntry(
item_type=StringConfigEntry,
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_TELEGRAM, "admin_usernames"],
default=[],
example=["myadminuser", "myotheradminuser"])
SQL_PERSISTENCE_URL = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_PERSISTENCE, "url"],
default=DEFAULT_SQL_PERSISTENCE_URL,
secret=True)
WORD_COUNT_RANGE = RangeConfigEntry(key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_BEHAVIOUR, CONFIG_NODE_WORD_COUNT_RANGE
],
default="[1..10]")
CHAR_COUNT_RANGE = RangeConfigEntry(key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_BEHAVIOUR, CONFIG_NODE_CHAR_COUNT_RANGE
],
default="[3..255]")
STATS_PORT = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_STATS, CONFIG_NODE_PORT],
default=8000)
class AppConfig(ConfigBase):
"""
Main InfiniteWisdom bot configuration
"""
LOGGER = logging.getLogger(__name__)
def __new__(cls, *args, **kwargs):
yaml_source = YamlSource(CONFIG_FILE_NAME)
data_sources = [EnvSource(), yaml_source]
return super(AppConfig, cls).__new__(cls, data_sources=data_sources)
LOG_LEVEL = StringConfigEntry(
description="Log level",
key_path=[CONFIG_NODE_ROOT, "log_level"],
regex=re.compile(f" {'|'.join(logging._nameToLevel.keys())}",
flags=re.IGNORECASE),
default="DEBUG",
)
TELEGRAM_BOT_TOKEN = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_TELEGRAM, "bot_token"],
example="123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11",
secret=True)
TELEGRAM_INLINE_BADGE_SIZE = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_TELEGRAM, "inline_badge_size"],
default=16)
TELEGRAM_GREETING_MESSAGE = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_TELEGRAM, "greeting_message"],
required=False,
default=
'Send /inspire for more inspiration :blush: Or use @InfiniteWisdomBot in a group chat and select one of the suggestions.'
)
TELEGRAM_CAPTION_IMAGES_WITH_TEXT = BoolConfigEntry(key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_TELEGRAM, "caption_images_with_text"
],
default=False)
TELEGRAM_ADMIN_USERNAMES = ListConfigEntry(
item_type=StringConfigEntry,
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_TELEGRAM, "admin_usernames"],
default=[],
example=["myadminuser", "myotheradminuser"])
UPLOADER_INTERVAL = FloatConfigEntry(key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_UPLOADER, CONFIG_NODE_INTERVAL
],
default=3.0)
UPLOADER_CHAT_ID = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_UPLOADER, "chat_id"],
default=None,
example=12345678)
CRAWLER_INTERVAL = FloatConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_CRAWLER, CONFIG_NODE_INTERVAL],
default=1.0)
SQL_PERSISTENCE_URL = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_PERSISTENCE, "url"],
default=DEFAULT_SQL_PERSISTENCE_URL,
secret=True)
FILE_PERSISTENCE_BASE_PATH = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_PERSISTENCE, "file_base_path"],
default=DEFAULT_FILE_PERSISTENCE_BASE_PATH)
IMAGE_ANALYSIS_INTERVAL = FloatConfigEntry(key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_IMAGE_ANALYSIS, CONFIG_NODE_INTERVAL
],
default=1.0)
IMAGE_ANALYSIS_TESSERACT_ENABLED = BoolConfigEntry(key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_IMAGE_ANALYSIS, CONFIG_NODE_TESSERACT,
CONFIG_NODE_ENABLED
],
default=False)
IMAGE_ANALYSIS_GOOGLE_VISION_ENABLED = BoolConfigEntry(key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_IMAGE_ANALYSIS,
CONFIG_NODE_GOOGLE_VISION, CONFIG_NODE_ENABLED
],
default=False)
IMAGE_ANALYSIS_GOOGLE_VISION_AUTH_FILE = StringConfigEntry(
key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_IMAGE_ANALYSIS,
CONFIG_NODE_GOOGLE_VISION, "auth_file"
],
default=None,
example="./InfiniteWisdom-1522618e7d39.json")
IMAGE_ANALYSIS_GOOGLE_VISION_CAPACITY = IntConfigEntry(key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_IMAGE_ANALYSIS,
CONFIG_NODE_GOOGLE_VISION, CONFIG_NODE_CAPACITY_PER_MONTH
],
default=None,
example=1000)
IMAGE_ANALYSIS_MICROSOFT_AZURE_ENABLED = BoolConfigEntry(key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_IMAGE_ANALYSIS,
CONFIG_NODE_MICROSOFT_AZURE, CONFIG_NODE_ENABLED
],
default=False)
IMAGE_ANALYSIS_MICROSOFT_AZURE_SUBSCRIPTION_KEY = StringConfigEntry(
key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_IMAGE_ANALYSIS,
CONFIG_NODE_MICROSOFT_AZURE, "subscription_key"
],
default=None,
example="1234567890684c3baa5a0605712345ab",
secret=True)
IMAGE_ANALYSIS_MICROSOFT_AZURE_REGION = StringConfigEntry(
key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_IMAGE_ANALYSIS,
CONFIG_NODE_MICROSOFT_AZURE, "region"
],
default="francecentral")
IMAGE_ANALYSIS_MICROSOFT_AZURE_CAPACITY = IntConfigEntry(key_path=[
CONFIG_NODE_ROOT, CONFIG_NODE_IMAGE_ANALYSIS,
CONFIG_NODE_MICROSOFT_AZURE, CONFIG_NODE_CAPACITY_PER_MONTH
],
default=5000)
STATS_PORT = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_STATS, CONFIG_NODE_PORT],
default=8000)
def _validate(self):
"""
Validates the current configuration and throws an exception if something is wrong
"""
if len(self.TELEGRAM_BOT_TOKEN.value) <= 0:
raise AssertionError("Bot token is missing!")
if self.CRAWLER_INTERVAL.value < 0:
raise AssertionError("Image polling interval must be >= 0!")
if self.IMAGE_ANALYSIS_GOOGLE_VISION_ENABLED.value:
if self.IMAGE_ANALYSIS_GOOGLE_VISION_AUTH_FILE.value is None:
raise AssertionError(
"Google Vision authentication file is required")
if not os.path.isfile(
self.IMAGE_ANALYSIS_GOOGLE_VISION_AUTH_FILE.value):
raise IsADirectoryError(
"Google Vision Auth file path is not a file: {}".format(
self.IMAGE_ANALYSIS_GOOGLE_VISION_AUTH_FILE.value))
if self.IMAGE_ANALYSIS_MICROSOFT_AZURE_ENABLED.value:
if self.IMAGE_ANALYSIS_MICROSOFT_AZURE_SUBSCRIPTION_KEY.value is None:
raise AssertionError(
"Microsoft Azure subscription key is required")
if self.IMAGE_ANALYSIS_MICROSOFT_AZURE_REGION.value is None:
raise AssertionError("Microsoft Azure region is required")
class Config(ConfigBase):
def __new__(cls, *args, **kwargs):
yaml_source = YamlSource(NODE_MAIN)
toml_source = TomlSource(NODE_MAIN)
data_sources = [
EnvSource(),
yaml_source,
toml_source
]
return super(Config, cls).__new__(cls, data_sources=data_sources)
LOG_LEVEL = StringConfigEntry(
description="Log level",
key_path=[
NODE_MAIN,
"log_level"
],
regex=re.compile(f"{'|'.join(logging._nameToLevel.keys())}", flags=re.IGNORECASE),
default="WARNING",
)
TELEGRAM_BOT_TOKEN = StringConfigEntry(
description="The telegram bot token to use",
key_path=[
NODE_MAIN,
NODE_TELEGRAM,
"bot_token"
],
example="123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11",
secret=True,
required=True
)
TELEGRAM_ADMIN_USERNAMES = ListConfigEntry(
item_type=StringConfigEntry,
key_path=[
NODE_MAIN,
NODE_TELEGRAM,
"admin_usernames"
],
required=True,
example=[
"myadminuser",
"myotheradminuser"
]
)
TELEGRAM_CHAT_IDS = ListConfigEntry(
item_type=StringConfigEntry,
key_path=[
NODE_MAIN,
NODE_TELEGRAM,
"chat_ids"
],
required=True,
example=[
12345678,
87654321
]
)
KEEL_HOST = StringConfigEntry(
description="Hostname of the keel HTTP endpoint",
key_path=[
NODE_MAIN,
NODE_KEEL,
NODE_HOST
],
default="localhost",
required=True
)
KEEL_PORT = IntConfigEntry(
description="Port of the keel HTTP endpoint",
key_path=[
NODE_MAIN,
NODE_KEEL,
NODE_PORT
],
default=9300,
required=True
)
KEEL_SSL = BoolConfigEntry(
description="Whether to use HTTPS or not",
key_path=[
NODE_MAIN,
NODE_KEEL,
"ssl"
],
default=True
)
KEEL_USER = StringConfigEntry(
description="Keel basic auth username",
key_path=[
NODE_MAIN,
NODE_KEEL,
"username"
],
required=True
)
KEEL_PASSWORD = StringConfigEntry(
description="Keel basic auth password",
key_path=[
NODE_MAIN,
NODE_KEEL,
"password"
],
required=True,
secret=True
)
MONITOR_INTERVAL = TimeDeltaConfigEntry(
description="Interval to check for new pending approvals",
key_path=[
NODE_MAIN,
"monitor"
"interval"
],
default="1m",
required=True,
)
STATS_ENABLED = BoolConfigEntry(
description="Whether to enable prometheus statistics or not.",
key_path=[
NODE_MAIN,
NODE_STATS,
NODE_ENABLED
],
default=True
)
STATS_PORT = IntConfigEntry(
description="The port to expose statistics on.",
key_path=[
NODE_MAIN,
NODE_STATS,
NODE_PORT
],
default=8000
)
class Config(ConfigBase):
def __new__(cls, *args, **kwargs):
yaml_source = YamlSource(NODE_MAIN)
toml_source = TomlSource(NODE_MAIN)
data_sources = [EnvSource(), yaml_source, toml_source]
return super(Config, cls).__new__(cls, data_sources=data_sources)
LOG_LEVEL = StringConfigEntry(
description="Log level",
key_path=[NODE_MAIN, "log_level"],
regex=re.compile(f"{'|'.join(logging._nameToLevel.keys())}",
flags=re.IGNORECASE),
default="WARNING",
)
LOCALE = StringConfigEntry(
description="Bot Locale",
key_path=[NODE_MAIN, "locale"],
default="en",
)
TELEGRAM_BOT_TOKEN = StringConfigEntry(
description="The telegram bot token to use",
key_path=[NODE_MAIN, NODE_TELEGRAM, "bot_token"],
example="123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11",
secret=True)
TELEGRAM_ADMIN_USERNAMES = ListConfigEntry(
item_type=StringConfigEntry,
key_path=[NODE_MAIN, NODE_TELEGRAM, "admin_usernames"],
required=True,
example=["myadminuser", "myotheradminuser"])
GROCY_CACHE_DURATION = TimeDeltaConfigEntry(
description="Duration to cache Grocy REST api call responses",
key_path=[NODE_MAIN, NODE_GROCY, "cache_duration"],
required=True,
default="60s",
)
NOTIFICATION_CHAT_IDS = ListConfigEntry(
item_type=StringConfigEntry,
key_path=[NODE_MAIN, NODE_NOTIFICATION, "chat_ids"],
default=[])
GROCY_HOST = StringConfigEntry(
description="Hostname of the Grocy instance",
key_path=[NODE_MAIN, NODE_GROCY, NODE_HOST],
required=True,
default="127.0.0.1")
GROCY_PORT = IntConfigEntry(description="Port of the Grocy REST api",
key_path=[NODE_MAIN, NODE_GROCY, NODE_PORT],
range=Range(1, 65535),
default=80)
GROCY_API_KEY = StringConfigEntry(
description="Grocy API Key used for REST authentication",
key_path=[NODE_MAIN, NODE_GROCY, NODE_API_KEY],
required=True,
example="abcdefgh12345678",
secret=True)
STATS_ENABLED = BoolConfigEntry(
description="Whether to enable prometheus statistics or not.",
key_path=[NODE_MAIN, NODE_STATS, NODE_ENABLED],
default=True)
STATS_PORT = IntConfigEntry(
description="The port to expose statistics on.",
key_path=[NODE_MAIN, NODE_STATS, NODE_PORT],
default=8000)
class DeduplicatorConfig(ConfigBase):
def __new__(cls, *args, **kwargs):
yaml_source = YamlSource("py_image_dedup")
data_sources = [EnvSource(), yaml_source]
return super(DeduplicatorConfig,
cls).__new__(cls, data_sources=data_sources)
DRY_RUN = BoolConfigEntry(
description="If enabled no source file will be touched",
key_path=[NODE_MAIN, NODE_DRY_RUN],
default=True)
ELASTICSEARCH_HOST = StringConfigEntry(
description="Hostname of the elasticsearch backend instance to use",
key_path=[NODE_MAIN, NODE_ELASTICSEARCH, NODE_HOST],
default="127.0.0.1")
ELASTICSEARCH_PORT = IntConfigEntry(
description="Hostname of the elasticsearch backend instance to use",
key_path=[NODE_MAIN, NODE_ELASTICSEARCH, NODE_PORT],
range=Range(1, 65535),
default=9200)
ELASTICSEARCH_MAX_DISTANCE = FloatConfigEntry(
description=
"Maximum signature distance [0..1] to query from elasticsearch backend.",
key_path=[NODE_MAIN, NODE_ELASTICSEARCH, NODE_MAX_DISTANCE],
default=0.10)
ELASTICSEARCH_AUTO_CREATE_INDEX = BoolConfigEntry(
description=
"Whether to automatically create an index in the target database.",
key_path=[NODE_MAIN, NODE_ELASTICSEARCH, NODE_AUTO_CREATE_INDEX],
default=True)
ELASTICSEARCH_INDEX = StringConfigEntry(
description=
"The index name to use for storing and querying image analysis data.",
key_path=[NODE_MAIN, NODE_ELASTICSEARCH, NODE_INDEX],
default="images")
ANALYSIS_USE_EXIF_DATA = BoolConfigEntry(
description="Whether to scan for EXIF data or not.",
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_USE_EXIF_DATA],
default=True)
SOURCE_DIRECTORIES = ListConfigEntry(
description=
"Comma separated list of source paths to analyse and deduplicate.",
item_type=DirectoryConfigEntry,
item_args={"check_existence": True},
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_SOURCE_DIRECTORIES],
required=True,
example=["/home/myuser/pictures/"])
RECURSIVE = BoolConfigEntry(
description="When set all directories will be recursively analyzed.",
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_RECURSIVE],
default=True)
SEARCH_ACROSS_ROOT_DIRS = BoolConfigEntry(
description=
"When set duplicates will be found even if they are located in different root directories.",
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_SEARCH_ACROSS_ROOT_DIRS],
default=False)
FILE_EXTENSION_FILTER = ListConfigEntry(
description="Comma separated list of file extensions.",
item_type=StringConfigEntry,
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_FILE_EXTENSIONS],
required=True,
default=[".png", ".jpg", ".jpeg"])
ANALYSIS_THREADS = IntConfigEntry(
description="Number of threads to use for image analysis phase.",
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_THREADS],
default=1)
MAX_FILE_MODIFICATION_TIME_DELTA = TimeDeltaConfigEntry(
description="Maximum file modification date difference between multiple "
"duplicates to be considered the same image",
key_path=[
NODE_MAIN, NODE_DEDUPLICATION, NODE_MAX_FILE_MODIFICATION_TIME_DIFF
],
default=None,
example=timedelta(minutes=5))
REMOVE_EMPTY_FOLDERS = BoolConfigEntry(
description="Whether to remove empty folders or not.",
key_path=[NODE_MAIN, NODE_REMOVE_EMPTY_FOLDERS],
default=False)
DEDUPLICATOR_DUPLICATES_TARGET_DIRECTORY = DirectoryConfigEntry(
description=
"Directory path to move duplicates to instead of deleting them.",
key_path=[
NODE_MAIN, NODE_DEDUPLICATION, NODE_DUPLICATES_TARGET_DIRECTORY
],
check_existence=True,
default=None,
example="/home/myuser/pictures/duplicates/")
DAEMON_PROCESSING_TIMEOUT = TimeDeltaConfigEntry(
description=
"Time to wait for filesystems changes to settle before analysing.",
key_path=[NODE_MAIN, NODE_DAEMON, NODE_PROCESSING_TIMEOUT],
default="30s")
DAEMON_FILE_OBSERVER_TYPE = StringConfigEntry(
description="Type of file observer to use.",
key_path=[NODE_MAIN, NODE_DAEMON, NODE_FILE_OBSERVER_TYPE],
regex="|".join(
[FILE_OBSERVER_TYPE_POLLING, FILE_OBSERVER_TYPE_INOTIFY]),
default=FILE_OBSERVER_TYPE_POLLING,
required=True)
STATS_ENABLED = BoolConfigEntry(
description="Whether to enable prometheus statistics or not.",
key_path=[NODE_MAIN, NODE_STATS, NODE_ENABLED],
default=True)
STATS_PORT = IntConfigEntry(
description="The port to expose statistics on.",
key_path=[NODE_MAIN, NODE_STATS, NODE_PORT],
default=8000)
class AppConfig(ConfigBase):
def __new__(cls, *args, **kwargs):
yaml_source = YamlSource(CONFIG_NODE_ROOT)
toml_source = TomlSource(CONFIG_NODE_ROOT)
data_sources = [
EnvSource(),
yaml_source,
toml_source,
]
return super(AppConfig, cls).__new__(cls, data_sources=data_sources)
LOG_LEVEL = StringConfigEntry(
description="Log level",
key_path=[CONFIG_NODE_ROOT, "log_level"],
regex=re.compile(f" {'|'.join(logging._nameToLevel.keys())}",
flags=re.IGNORECASE),
default="INFO",
)
SERVER_HOST = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_SERVER, "host"],
default=DEFAULT_SERVER_HOST,
secret=True)
SERVER_PORT = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_SERVER, CONFIG_NODE_PORT],
range=Range(1, 65534),
default=9465)
SERVER_API_TOKEN = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_SERVER, "api_token"],
default=None,
secret=True)
DROP_EVENT_QUEUE_AFTER = TimeDeltaConfigEntry(
key_path=[CONFIG_NODE_ROOT, "drop_event_queue_after"],
default="2h",
)
RETRY_INTERVAL = TimeDeltaConfigEntry(
key_path=[CONFIG_NODE_ROOT, "retry_interval"],
default="2s",
)
HTTP_METHOD = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_HTTP, "method"],
required=True,
default="POST",
regex="GET|POST|PUT|PATCH")
HTTP_URL = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_HTTP, "url"], required=False)
HTTP_HEADERS = ListConfigEntry(
item_type=StringConfigEntry,
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_HTTP, "headers"],
default=[])
MQTT_HOST = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "host"], required=False)
MQTT_PORT = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "port"],
required=True,
default=1883,
range=Range(1, 65534),
)
MQTT_CLIENT_ID = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "client_id"],
default="barcode-server")
MQTT_USER = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "user"])
MQTT_PASSWORD = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "password"], secret=True)
MQTT_TOPIC = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "topic"],
default="barcode-server/barcode",
required=True)
MQTT_QOS = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "qos"],
default=2,
required=True)
MQTT_RETAIN = BoolConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "retain"],
default=False,
required=True)
DEVICE_PATTERNS = ListConfigEntry(item_type=RegexConfigEntry,
item_args={"flags": re.IGNORECASE},
key_path=[CONFIG_NODE_ROOT, "devices"],
default=[])
DEVICE_PATHS = ListConfigEntry(item_type=FileConfigEntry,
key_path=[CONFIG_NODE_ROOT, "device_paths"],
default=[])
STATS_PORT = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_STATS, CONFIG_NODE_PORT],
default=8000,
required=False)
def validate(self):
super(AppConfig, self).validate()
if len(self.DEVICE_PATHS.value) == len(
self.DEVICE_PATTERNS.value) == 0:
raise AssertionError(
"You must provide at least one device pattern or device_path!")
class TestConfigBaseClashing(ConfigBase):
clashing_key_path = ["test", "bool"]
BOOL = BoolConfigEntry(key_path=clashing_key_path, default=False)
STRING = StringConfigEntry(key_path=clashing_key_path, default="test")