def test_contains_int_range_inverted(self):
int_range = Range(2, 0)
self.assertNotIn(-1, int_range)
self.assertIn(0, int_range)
self.assertIn(1, int_range)
self.assertIn(2, int_range)
self.assertNotIn(3, int_range)
def test_contains_float_range_inverted(self):
float_range = Range(2, 0)
self.assertNotIn(-1, float_range)
self.assertIn(0, float_range)
self.assertIn(1, float_range)
self.assertIn(2, float_range)
self.assertNotIn(3, float_range)
def test_contains_int_range_over_zero(self):
int_range = Range(-5, 5)
self.assertNotIn(-6, int_range)
self.assertIn(-5, int_range)
self.assertIn(0, int_range)
self.assertIn(5, int_range)
self.assertNotIn(6, int_range)
def test_contains_float_range_over_zero(self):
float_range = Range(-5, 5)
self.assertNotIn(-6, float_range)
self.assertIn(-5, float_range)
self.assertIn(0, float_range)
self.assertIn(5, float_range)
self.assertNotIn(6, float_range)
def test_lt_int_range(self):
int_range = Range(0, 5)
self.assertTrue(-1 < int_range)
self.assertFalse(0 < int_range)
self.assertFalse(2 < int_range)
self.assertFalse(5 < int_range)
self.assertFalse(6 < int_range)
def test_gt_float_range(self):
float_range = Range(0, 5.2)
self.assertFalse(-1 > float_range)
self.assertFalse(0 > float_range)
self.assertFalse(2 > float_range)
self.assertFalse(5.2 > float_range)
self.assertTrue(6 > float_range)
def test_gt_int_range(self):
int_range = Range(0, 5)
self.assertFalse(-1 > int_range)
self.assertFalse(0 > int_range)
self.assertFalse(2 > int_range)
self.assertFalse(5 > int_range)
self.assertTrue(6 > int_range)
def test_lt_float_range(self):
float_range = Range(0, 5.2)
self.assertTrue(-1 < float_range)
self.assertFalse(0 < float_range)
self.assertFalse(2 < float_range)
self.assertFalse(5.2 < float_range)
self.assertFalse(6 < float_range)
def test_range_entry(self):
config_entry = RangeConfigEntry(key_path=["range"])
input_output = [
("[-5..5]", Range(-5, 5)),
]
self.assert_input_output(config_entry, input_output)
def test_float_entry(self):
config_entry = FloatConfigEntry(key_path=["float"],
range=Range(-10.0, 10))
input_output = [("5", 5.0), (5, 5.0), ("-3.0", -3.0), (-3.0, -3.0),
(1.2, 1.2), ("1.2", 1.2), ("3%", 0.03),
(-20.0, ValueError)]
self.assert_input_output(config_entry, input_output)
def test_contains_int_range(self):
int_range = Range(0, 5)
self.assertNotIn(-1, int_range)
self.assertIn(0, int_range)
self.assertIn(3, int_range)
self.assertNotIn(3.3, int_range)
self.assertIn(3.0, int_range)
self.assertIn(5, int_range)
self.assertNotIn(6, int_range)
def test_contains_float_infinity(self):
float_range = Range(-math.inf, 5)
self.assertIn(-math.inf, float_range)
self.assertIn(-10000, float_range)
self.assertIn(-6, float_range)
self.assertIn(-5, float_range)
self.assertIn(0, float_range)
self.assertIn(5, float_range)
self.assertNotIn(6, float_range)
class TestConfigBase(ConfigBase):
BOOL = BoolConfigEntry(key_path=["test", "bool"], default=False)
STRING = StringConfigEntry(key_path=["test", "string"],
default="default value")
REGEX = RegexConfigEntry(key_path=["test", "regex"],
default="^[a-zA-Z0-9]$")
INT = IntConfigEntry(key_path=["test", "int"], default=100)
FLOAT = FloatConfigEntry(key_path=["test", "float"], default=1.23)
DATE = DateConfigEntry(
key_path=["test", "this", "date", "is", "nested", "deep"],
default=datetime.now())
TIMEDELTA = TimeDeltaConfigEntry(
key_path=["test", "this", "timediff", "is", "in", "this", "branch"],
default=timedelta(seconds=10))
FILE = FileConfigEntry(key_path=["test", "file"], )
DIRECTORY = DirectoryConfigEntry(key_path=["test", "directory"], )
RANGE = RangeConfigEntry(key_path=["test", "this", "is", "a", "range"],
default=Range(0, 100))
DICT = DictConfigEntry(key_path=["dict"], schema=Schema({str: str}))
DICT_LIST = ListConfigEntry(
item_type=DictConfigEntry,
item_args={"schema": Schema({str: str})},
key_path=[
"dict_list",
],
)
STRING_LIST = ListConfigEntry(item_type=StringConfigEntry,
key_path=["test", "this", "is", "a", "list"],
example=["these", "are", "test", "values"],
secret=False)
NONE_INT = IntConfigEntry(
key_path=["none", "int"],
default=None,
)
NONE_DATE = DateConfigEntry(
key_path=["none", "date"],
default=None,
)
SECRET_BOOL = BoolConfigEntry(key_path=["secret", "bool"],
default=False,
secret=True)
SECRET_INT = IntConfigEntry(key_path=["secret", "int"],
default=None,
secret=True)
SECRET_REGEX = RegexConfigEntry(key_path=["secret", "regex"],
default=None,
secret=True)
SECRET_LIST = ListConfigEntry(item_type=RegexConfigEntry,
key_path=["secret", "list"],
default=["[a-zA-Z]*"],
secret=True)
def test_contains_float_range(self):
float_range = Range(0.0, 5.2)
self.assertNotIn(-1.0, float_range)
self.assertIn(-0.0, float_range)
self.assertIn(0.0, float_range)
self.assertIn(0, float_range)
self.assertIn(3, float_range)
self.assertIn(3.0, float_range)
self.assertIn(3.3, float_range)
self.assertIn(5, float_range)
self.assertIn(5.2, float_range)
self.assertNotIn(5.3, float_range)
def test_contains_int_range_exclusive(self):
int_range = Range(0, 2, False, False)
self.assertNotIn(0, int_range)
self.assertIn(1, int_range)
self.assertNotIn(2, int_range)
from py_range_parse import parse_range, Range
# parse a string
range = parse_range("[0..5]")
# or create one directly
range = Range(0, 5)
class Config(ConfigBase):
def __new__(cls, *args, **kwargs):
yaml_source = YamlSource(NODE_MAIN)
toml_source = TomlSource(NODE_MAIN)
data_sources = [EnvSource(), yaml_source, toml_source]
return super(Config, cls).__new__(cls, data_sources=data_sources)
LOG_LEVEL = StringConfigEntry(
description="Log level",
key_path=[NODE_MAIN, "log_level"],
regex=re.compile(f"{'|'.join(logging._nameToLevel.keys())}",
flags=re.IGNORECASE),
default="WARNING",
)
LOCALE = StringConfigEntry(
description="Bot Locale",
key_path=[NODE_MAIN, "locale"],
default="en",
)
TELEGRAM_BOT_TOKEN = StringConfigEntry(
description="The telegram bot token to use",
key_path=[NODE_MAIN, NODE_TELEGRAM, "bot_token"],
example="123456:ABC-DEF1234ghIkl-zyx57W2v1u123ew11",
secret=True)
TELEGRAM_ADMIN_USERNAMES = ListConfigEntry(
item_type=StringConfigEntry,
key_path=[NODE_MAIN, NODE_TELEGRAM, "admin_usernames"],
required=True,
example=["myadminuser", "myotheradminuser"])
GROCY_CACHE_DURATION = TimeDeltaConfigEntry(
description="Duration to cache Grocy REST api call responses",
key_path=[NODE_MAIN, NODE_GROCY, "cache_duration"],
required=True,
default="60s",
)
NOTIFICATION_CHAT_IDS = ListConfigEntry(
item_type=StringConfigEntry,
key_path=[NODE_MAIN, NODE_NOTIFICATION, "chat_ids"],
default=[])
GROCY_HOST = StringConfigEntry(
description="Hostname of the Grocy instance",
key_path=[NODE_MAIN, NODE_GROCY, NODE_HOST],
required=True,
default="127.0.0.1")
GROCY_PORT = IntConfigEntry(description="Port of the Grocy REST api",
key_path=[NODE_MAIN, NODE_GROCY, NODE_PORT],
range=Range(1, 65535),
default=80)
GROCY_API_KEY = StringConfigEntry(
description="Grocy API Key used for REST authentication",
key_path=[NODE_MAIN, NODE_GROCY, NODE_API_KEY],
required=True,
example="abcdefgh12345678",
secret=True)
STATS_ENABLED = BoolConfigEntry(
description="Whether to enable prometheus statistics or not.",
key_path=[NODE_MAIN, NODE_STATS, NODE_ENABLED],
default=True)
STATS_PORT = IntConfigEntry(
description="The port to expose statistics on.",
key_path=[NODE_MAIN, NODE_STATS, NODE_PORT],
default=8000)
def test_str_float(self):
int_range = Range(-4.2123, 123.4324)
self.assertEqual(str(int_range), "[-4.2123..123.4324]")
def test_contains_float_range_negative(self):
float_range = Range(-5, -2)
self.assertNotIn(-6, float_range)
self.assertIn(-3, float_range)
self.assertIn(-2, float_range)
self.assertNotIn(-1, float_range)
def test_str_int(self):
int_range = Range(-4, 123)
self.assertEqual(str(int_range), "[-4..123]")
def test_contains_float_range_exclusive(self):
float_range = Range(0.0, 2, False, False)
self.assertNotIn(0, float_range)
self.assertIn(0.1, float_range)
self.assertIn(1, float_range)
self.assertNotIn(2, float_range)
def test_str_exclusive(self):
int_range = Range(123, -4, False, False)
self.assertEqual(str(int_range), "]-4..123[")
def test_contains_int_range_negative(self):
int_range = Range(-5, -2)
self.assertNotIn(-6, int_range)
self.assertIn(-3, int_range)
self.assertIn(-2, int_range)
self.assertNotIn(-1, int_range)
def test_str_inf(self):
int_range = Range(-math.inf, math.inf)
self.assertEqual(str(int_range), "[-inf..inf]")
class DeduplicatorConfig(ConfigBase):
def __new__(cls, *args, **kwargs):
yaml_source = YamlSource("py_image_dedup")
data_sources = [EnvSource(), yaml_source]
return super(DeduplicatorConfig,
cls).__new__(cls, data_sources=data_sources)
DRY_RUN = BoolConfigEntry(
description="If enabled no source file will be touched",
key_path=[NODE_MAIN, NODE_DRY_RUN],
default=True)
ELASTICSEARCH_HOST = StringConfigEntry(
description="Hostname of the elasticsearch backend instance to use",
key_path=[NODE_MAIN, NODE_ELASTICSEARCH, NODE_HOST],
default="127.0.0.1")
ELASTICSEARCH_PORT = IntConfigEntry(
description="Hostname of the elasticsearch backend instance to use",
key_path=[NODE_MAIN, NODE_ELASTICSEARCH, NODE_PORT],
range=Range(1, 65535),
default=9200)
ELASTICSEARCH_MAX_DISTANCE = FloatConfigEntry(
description=
"Maximum signature distance [0..1] to query from elasticsearch backend.",
key_path=[NODE_MAIN, NODE_ELASTICSEARCH, NODE_MAX_DISTANCE],
default=0.10)
ELASTICSEARCH_AUTO_CREATE_INDEX = BoolConfigEntry(
description=
"Whether to automatically create an index in the target database.",
key_path=[NODE_MAIN, NODE_ELASTICSEARCH, NODE_AUTO_CREATE_INDEX],
default=True)
ELASTICSEARCH_INDEX = StringConfigEntry(
description=
"The index name to use for storing and querying image analysis data.",
key_path=[NODE_MAIN, NODE_ELASTICSEARCH, NODE_INDEX],
default="images")
ANALYSIS_USE_EXIF_DATA = BoolConfigEntry(
description="Whether to scan for EXIF data or not.",
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_USE_EXIF_DATA],
default=True)
SOURCE_DIRECTORIES = ListConfigEntry(
description=
"Comma separated list of source paths to analyse and deduplicate.",
item_type=DirectoryConfigEntry,
item_args={"check_existence": True},
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_SOURCE_DIRECTORIES],
required=True,
example=["/home/myuser/pictures/"])
RECURSIVE = BoolConfigEntry(
description="When set all directories will be recursively analyzed.",
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_RECURSIVE],
default=True)
SEARCH_ACROSS_ROOT_DIRS = BoolConfigEntry(
description=
"When set duplicates will be found even if they are located in different root directories.",
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_SEARCH_ACROSS_ROOT_DIRS],
default=False)
FILE_EXTENSION_FILTER = ListConfigEntry(
description="Comma separated list of file extensions.",
item_type=StringConfigEntry,
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_FILE_EXTENSIONS],
required=True,
default=[".png", ".jpg", ".jpeg"])
ANALYSIS_THREADS = IntConfigEntry(
description="Number of threads to use for image analysis phase.",
key_path=[NODE_MAIN, NODE_ANALYSIS, NODE_THREADS],
default=1)
MAX_FILE_MODIFICATION_TIME_DELTA = TimeDeltaConfigEntry(
description="Maximum file modification date difference between multiple "
"duplicates to be considered the same image",
key_path=[
NODE_MAIN, NODE_DEDUPLICATION, NODE_MAX_FILE_MODIFICATION_TIME_DIFF
],
default=None,
example=timedelta(minutes=5))
REMOVE_EMPTY_FOLDERS = BoolConfigEntry(
description="Whether to remove empty folders or not.",
key_path=[NODE_MAIN, NODE_REMOVE_EMPTY_FOLDERS],
default=False)
DEDUPLICATOR_DUPLICATES_TARGET_DIRECTORY = DirectoryConfigEntry(
description=
"Directory path to move duplicates to instead of deleting them.",
key_path=[
NODE_MAIN, NODE_DEDUPLICATION, NODE_DUPLICATES_TARGET_DIRECTORY
],
check_existence=True,
default=None,
example="/home/myuser/pictures/duplicates/")
DAEMON_PROCESSING_TIMEOUT = TimeDeltaConfigEntry(
description=
"Time to wait for filesystems changes to settle before analysing.",
key_path=[NODE_MAIN, NODE_DAEMON, NODE_PROCESSING_TIMEOUT],
default="30s")
DAEMON_FILE_OBSERVER_TYPE = StringConfigEntry(
description="Type of file observer to use.",
key_path=[NODE_MAIN, NODE_DAEMON, NODE_FILE_OBSERVER_TYPE],
regex="|".join(
[FILE_OBSERVER_TYPE_POLLING, FILE_OBSERVER_TYPE_INOTIFY]),
default=FILE_OBSERVER_TYPE_POLLING,
required=True)
STATS_ENABLED = BoolConfigEntry(
description="Whether to enable prometheus statistics or not.",
key_path=[NODE_MAIN, NODE_STATS, NODE_ENABLED],
default=True)
STATS_PORT = IntConfigEntry(
description="The port to expose statistics on.",
key_path=[NODE_MAIN, NODE_STATS, NODE_PORT],
default=8000)
def test_int_entry(self):
config_entry = IntConfigEntry(key_path=["int"], range=Range(-5, 5))
input_output = [("5", 5), (5, 5), ("-3", -3), (-3, -3),
(-6, ValueError)]
self.assert_input_output(config_entry, input_output)
class AppConfig(ConfigBase):
def __new__(cls, *args, **kwargs):
yaml_source = YamlSource(CONFIG_NODE_ROOT)
toml_source = TomlSource(CONFIG_NODE_ROOT)
data_sources = [
EnvSource(),
yaml_source,
toml_source,
]
return super(AppConfig, cls).__new__(cls, data_sources=data_sources)
LOG_LEVEL = StringConfigEntry(
description="Log level",
key_path=[CONFIG_NODE_ROOT, "log_level"],
regex=re.compile(f" {'|'.join(logging._nameToLevel.keys())}",
flags=re.IGNORECASE),
default="INFO",
)
SERVER_HOST = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_SERVER, "host"],
default=DEFAULT_SERVER_HOST,
secret=True)
SERVER_PORT = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_SERVER, CONFIG_NODE_PORT],
range=Range(1, 65534),
default=9465)
SERVER_API_TOKEN = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_SERVER, "api_token"],
default=None,
secret=True)
DROP_EVENT_QUEUE_AFTER = TimeDeltaConfigEntry(
key_path=[CONFIG_NODE_ROOT, "drop_event_queue_after"],
default="2h",
)
RETRY_INTERVAL = TimeDeltaConfigEntry(
key_path=[CONFIG_NODE_ROOT, "retry_interval"],
default="2s",
)
HTTP_METHOD = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_HTTP, "method"],
required=True,
default="POST",
regex="GET|POST|PUT|PATCH")
HTTP_URL = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_HTTP, "url"], required=False)
HTTP_HEADERS = ListConfigEntry(
item_type=StringConfigEntry,
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_HTTP, "headers"],
default=[])
MQTT_HOST = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "host"], required=False)
MQTT_PORT = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "port"],
required=True,
default=1883,
range=Range(1, 65534),
)
MQTT_CLIENT_ID = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "client_id"],
default="barcode-server")
MQTT_USER = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "user"])
MQTT_PASSWORD = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "password"], secret=True)
MQTT_TOPIC = StringConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "topic"],
default="barcode-server/barcode",
required=True)
MQTT_QOS = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "qos"],
default=2,
required=True)
MQTT_RETAIN = BoolConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_MQTT, "retain"],
default=False,
required=True)
DEVICE_PATTERNS = ListConfigEntry(item_type=RegexConfigEntry,
item_args={"flags": re.IGNORECASE},
key_path=[CONFIG_NODE_ROOT, "devices"],
default=[])
DEVICE_PATHS = ListConfigEntry(item_type=FileConfigEntry,
key_path=[CONFIG_NODE_ROOT, "device_paths"],
default=[])
STATS_PORT = IntConfigEntry(
key_path=[CONFIG_NODE_ROOT, CONFIG_NODE_STATS, CONFIG_NODE_PORT],
default=8000,
required=False)
def validate(self):
super(AppConfig, self).validate()
if len(self.DEVICE_PATHS.value) == len(
self.DEVICE_PATTERNS.value) == 0:
raise AssertionError(
"You must provide at least one device pattern or device_path!")