def __init__(self,
config: SectionProxy,
method: str,
query_string: bytes,
response_start: Callable[..., None],
response_body: Callable[..., None],
response_done: Callable[..., None],
error_log: Callable[[str], int] = sys.stderr.write) -> None:
self.config = config # type: SectionProxy
self.charset_bytes = self.config['charset'].encode('ascii')
self.method = method # Request method to the UX; bytes
self.response_start = response_start
self.response_body = response_body
self._response_done = response_done
self.error_log = error_log # function to log errors to
self.test_uri = None # type: str
self.req_hdrs = None # type: StrHeaderListType
self.format = None # type: str
self.test_id = None # type: str
self.check_name = None # type: str
self.descend = None # type: bool
self.save = None # type: bool
self.timeout = None # type: Any
self.referer_spam_domains = [] # type: List[str]
if config.get("limit_origin_tests", ""):
if self._origin_period == None:
self._origin_period = config.getfloat("limit_origin_period",
fallback=1) * 3600
thor.schedule(self._origin_period, self.ratelimit_cleanup)
if config.get("referer_spam_domains", ""):
self.referer_spam_domains = [i.strip() for i in \
config["referer_spam_domains"].split()]
self.run(query_string)
def __init__(self, aws_config: SectionProxy, dwh_config: SectionProxy):
KEY = aws_config.get('KEY')
SECRET = aws_config.get('SECRET')
REGION = dwh_config.get('DWH_REGION')
self.__iam = boto3.client(
'iam',
region_name=REGION,
aws_access_key_id=KEY,
aws_secret_access_key=SECRET
)
self.__redshift = boto3.client(
'redshift',
region_name=REGION,
aws_access_key_id=KEY,
aws_secret_access_key=SECRET
)
self.__ec2 = boto3.resource(
'ec2',
region_name=REGION,
aws_access_key_id=KEY,
aws_secret_access_key=SECRET
)
self.dwh_config = dwh_config
self.__redshift_cluster_creation_timeout = 600
self.__redshift_cluster_creation_waiting_period = 60
self.__redshift_cluster_failed_statuses = ['failed', 'unavailable', 'deleting', 'hardware-failure']
self.__redshift_cluster_acceptable_statuses = ['available']
def _parse_option(
self,
option_key: str,
option_value: str,
template: LinterOptions,
section: SectionProxy,
) -> Any:
template_key = option_key.replace('-', '_')
option_type: Any = option_types.get(template_key)
processed_value: Any
if not option_type:
dv = getattr(template, template_key, None)
if dv is None:
if self.report_unrecognized_options:
print(
f'Unrecognized option: {option_key} = {option_value}',
file=sys.stderr,
)
return None
option_type = type(dv)
try:
if option_type is bool:
processed_value = section.getboolean(template_key)
elif callable(option_type):
processed_value = option_type(section.get(option_key))
else:
print(
f'Cannot determine what type {option_key} should have',
file=sys.stderr,
)
return None
except (ValueError, argparse.ArgumentTypeError) as error:
print(f'{option_key}: {error}', file=sys.stderr)
return None
return processed_value
def __get_option_with_default_value(section: configparser.SectionProxy,
option,
default_value,
data_type: str = ""):
if section is not None and option in section:
try:
if data_type == "bool":
temp = section.getboolean(option)
elif data_type == "int":
temp = section.getint(option)
elif data_type == "float":
temp = section.getfloat(option)
else:
temp = section.get(option)
if option is None:
log.warning(
f"Setting '{option}' is present but has no value. Please refer to 'config.default' for an example config."
)
return default_value
else:
return temp
except ValueError:
return default_value
log.error(
f"Setting '{option}' is missing from config. Please refer to 'config.default' for an example config."
)
return default_value
def build_scheduler(cfg_section: SectionProxy, optimizer):
name = cfg_section['Name']
args = [optimizer] + loads(cfg_section['Args']) if cfg_section.get('Args') is not None else []
kwargs = loads(cfg_section['Kwargs']) if cfg_section.get('Kwargs') is not None else {}
optim_constr = getattr(lr_scheduler, name)
return optim_constr(*args, **kwargs)
def create(
cls,
name: str,
config: configparser.SectionProxy,
) -> "NetworkBandwidth":
try:
interfaces = config["interfaces"].split(",")
interfaces = [i.strip() for i in interfaces if i.strip()]
if not interfaces:
raise ConfigurationError("No interfaces configured")
host_interfaces = psutil.net_if_addrs().keys()
for interface in interfaces:
if interface not in host_interfaces:
raise ConfigurationError(
"Network interface {} does not exist".format(
interface))
threshold_send = config.getfloat("threshold_send", fallback=100)
threshold_receive = config.getfloat("threshold_receive",
fallback=100)
return cls(name, interfaces, threshold_send, threshold_receive)
except KeyError as error:
raise ConfigurationError(
"Missing configuration key: {}".format(error)) from error
except ValueError as error:
raise ConfigurationError(
"Threshold in wrong format: {}".format(error)) from error
def setup_global_begin(config: SectionProxy) -> None:
"""
Sets up the tables globally for ipv4 and ipv6
:param config: the configuration used
"""
# noinspection PyUnresolvedReferences
def setup(handler: Iptables, _config: SectionProxy) -> None:
"""
Sets up the tables to accept new rules : resets all rules, set defaults and allow global traffic
:param handler: the Iptables instance on which to operate
:param _config: the configuration used
"""
handler.reset()
for chain in _config.getlist("closed_chains", []):
handler.set_default(chain, "DROP")
if _config.getboolean("allow_established_traffic", False):
handler.allow_existing_traffic()
for interface in _config.getlist("allow_traffic_on_interface", []):
handler.allow_traffic_on_interface(interface)
if _config.getboolean("drop_invalid_traffic", False):
handler.drop_invalid_traffic()
if config.getboolean("ipv4", False):
setup(ipv4_handler, config)
if config.getboolean("ipv6", False):
setup(ipv6_handler, config)
def _load_config(config_section: configparser.SectionProxy):
supplied_keys = set(config_section.keys())
missing_keys = _REQUIRED_KEYS - supplied_keys
if missing_keys:
raise _errors.StreamConfigMissingKeysError(missing_keys)
unexpected_keys = supplied_keys - (_REQUIRED_KEYS | _OPTIONAL_KEYS)
if unexpected_keys:
raise _errors.StreamConfigUnexpectedKeysError(unexpected_keys)
# Make sure to_emails and to_sms_emails are valid json
to_emails = config_section[_TO_EMAILS_KEY]
try:
json.loads(to_emails)
except json.decoder.JSONDecodeError as e:
raise _errors.StreamConfigToEmailsFormatError(to_emails) from e
to_emails = config_section.get(_TO_SMS_EMAILS_KEY)
if to_emails:
try:
json.loads(to_emails)
except json.decoder.JSONDecodeError as e:
raise _errors.StreamConfigToSmsEmailsFormatError(to_emails) from e
return config_section
def get_sensor_panel(sensor_mac: str, sensor_name: str, images: Icons, fonts: Fonts, config: SectionProxy, draw_title:bool = True) -> Image.Image:
logger = logging.getLogger(__name__)
logger.info('Generating sensor panel')
x_size = 550
y_size = 330
image = Image.new('L', (x_size, y_size), 0xff)
draw = ImageDraw.Draw(image)
if (draw_title):
utils.draw_title(draw, fonts['font_sm'], 'SENSOR', sensor_name, fonts['font_xs'])
try:
if (not config.getboolean('USE_FAKE_SENSOR_DATA')):
timeout = config.getint('SENSOR_POLL_TIMEOUT')
logger.info(f'Fetching sensor data (timeout: {timeout})')
ruuvi_reactive = RuuviTagReactive([sensor_mac])
sensor_data = ruuvi_reactive\
.get_subject()\
.map(lambda x: {x[0]: x[1]})\
.merge(rx.Observable.timer(timeout).map(lambda x: {}))\
.to_blocking()\
.first()
ruuvi_reactive.stop()
logger.info('Received data: %s', repr(sensor_data))
else:
sensor_data = {sensor_mac: {"temperature": random.uniform(18, 30), "humidity": random.uniform(20, 80), "battery": random.uniform(2000, 3000), "rssi": random.uniform(-120, -10)}}
logger.info('Using fake data: %s', repr(sensor_data))
except Exception as e:
logger.error('get_data_for_sensors() failed: %s', repr(e))
sensor_data = {}
if (sensor_mac in sensor_data):
data_y_base = 100 if (draw_title) else 0
state_in = sensor_data[sensor_mac]
utils.draw_quantity(draw, (x_size//2 + 160, data_y_base + 120), str(round(state_in['temperature'], 1)), '°C', fonts, 'font_lg', 'font_sm')
utils.draw_quantity(draw, (x_size//2 + 160, data_y_base + 210), str(round(state_in['humidity'])), '%', fonts)
# Battery level
battery_icon = icons.get_scaled_image(get_battery_icon(state_in['battery'], images), 120)
image.paste(battery_icon, (30, data_y_base - 30), battery_icon)
utils.draw_quantity(draw, (130, data_y_base + 125), str(round(state_in['battery']/1000, 2)), 'V', fonts, 'font_xs', 'font_xxs')
# RSSI - not yet part of ruuvitag-sensor
# Adding is trivial by editing ruuvitag-sensor package's decoder.py
# See: https://github.com/ttu/ruuvitag-sensor/issues/52
if ('rssi' in state_in):
utils.draw_quantity(draw, (130, data_y_base + 190), str(round(state_in['rssi'])), 'dBm', fonts, 'font_xs', 'font_xxs')
else:
logger.info(f'Could not find mac {sensor_mac} in sensor data')
no_wifi_image = icons.get_scaled_image(images['misc']['no_wifi'], 200)
image.paste(no_wifi_image, (x_size//2 - no_wifi_image.width//2, y_size//2 - no_wifi_image.height//2), no_wifi_image)
# Borders
if (config.getboolean('DRAW_PANEL_BORDERS')):
draw.polygon([(0, 0), (x_size-1, 0), (x_size-1, y_size-1), (0, y_size-1), (0, 0)])
return image
def extract_included_region(config: configparser.SectionProxy) -> tuple:
"""
This functions extracts the region for which primer shall be generated
from the config file.
:param config: contains the values.
:return: extracted values formatted as int
"""
if config is not None:
seq_begin = config.getint('SEQUENCE_INCLUDED_BEGIN', -1)
seq_end = config.getint('SEQUENCE_INCLUDED_END', -1)
else:
seq_begin = 0
seq_end = 0
# is one of the values invalid?
if seq_begin < 0 or seq_end < 0:
logging.error(
('Found negative or invalid values for the region'
'for which the primer will be generated or have been generated if '
'custom primers are used via --keep-primer.'
'(SEQUENCE_INCLUDED_BEGIN,SEQUENCE_INCLUDED_END)'
' = {}. If you want to generate primers for the '
'whole sequence set both values to 0. Aborting').format(
(config['SEQUENCE_INCLUDED_BEGIN'],
config['SEQUENCE_INCLUDED_END']))
)
sys.exit(1)
return seq_begin, seq_end
def process_config(self):
config = super(Application, self).process_config()
if 'ETHEREUM_NODE_URL' in os.environ:
config['ethereum'] = {'url': os.environ['ETHEREUM_NODE_URL']}
if 'PUSH_URL' in os.environ:
config.setdefault('pushserver', SectionProxy(
config, 'pushserver'))['url'] = os.environ['PUSH_URL']
if 'PUSH_PASSWORD' in os.environ:
config.setdefault('pushserver', SectionProxy(
config,
'pushserver'))['password'] = os.environ['PUSH_PASSWORD']
if 'PUSH_USERNAME' in os.environ:
config.setdefault('pushserver', SectionProxy(
config,
'pushserver'))['username'] = os.environ['PUSH_USERNAME']
if 'GCM_SERVER_KEY' in os.environ:
config.setdefault('gcm', SectionProxy(
config, 'gcm'))['server_key'] = os.environ['GCM_SERVER_KEY']
configure_logger(services_log)
configure_logger(monitor_log)
return config
def build_optimizer(cfg_section: SectionProxy, net_params):
name = cfg_section['Name']
args = [net_params] + loads(cfg_section['Args']) if cfg_section.get('Args') is not None else []
kwargs = loads(cfg_section['Kwargs']) if cfg_section.get('Kwargs') is not None else {}
# dynamically get optimizer constructor
optim_constr = getattr(optim, name)
return optim_constr(*args, **kwargs)
def loadFont(self, section: SectionProxy, name: str, fontNameDef: str = "Helvetica", fontSizeDef: int = 32, isBoldDef: bool = False, isItalicDef: bool = False) -> Tuple[str, int, bool, bool]:
if not isinstance(section, SectionProxy):
raise TypeError("section")
fontName = section.get(name + "Name", fontNameDef)
fontSize = section.getint(name + "Size", fontSizeDef)
isBold = section.getboolean(name + "Bold", isBoldDef)
isItalic = section.getboolean(name + "Italic", isItalicDef)
return (fontName, fontSize, isBold, isItalic)
def __init__(self) -> None:
parser = ConfigParser()
path = Path.home() / Path(".plot-cli/config")
if path.exists():
parser.read(path)
self.settings = SectionProxy(parser, "settings")
def main_loop(panel_size: tuple[int, int], fonts: Fonts, images: Icons, config: configparser.SectionProxy, epd_so: Optional[ctypes.CDLL]) -> None:
logger = logging.getLogger(__name__)
logger.info("main_loop() started")
wakeup_time = datetime.datetime.now()
if((wakeup_time.minute - 5) % config.getint('REFRESH_FULL_INTERVAL') == 0):
refresh.refresh(panel_size, fonts, images, config, epd_so, True)
elif(wakeup_time.minute % config.getint('REFRESH_PARTIAL_INTERVAL') == 0):
refresh.refresh(panel_size, fonts, images, config, epd_so, False)
def from_config_section(cls, section: SectionProxy) -> Config:
"""Creates a credentials tuple from
the respective config section.
"""
host = section['host']
port = section.getint('port')
passwd = section.get('passwd')
return cls(host, port, passwd)
def __init__(self, config: SectionProxy, experiment):
super().__init__(
filename=config.get("name"),
directory=config.get("path"),
activation_level=config.getint("level"),
experiment=experiment,
name=config.get("name"),
encrypt=config.getboolean("encrypt", fallback=False),
)
def setup_global_end(config: SectionProxy) -> None:
"""
Sets up the last things : logging, drops and ssh knocking
:param config: the config to use
"""
def setup(handler: Iptables, _config: SectionProxy, version) -> None:
"""
Ties up the settings : logging, drops and ssh knocking
:param handler: the Iptables instance on which to operate
:param _config: the configuration used
:param version: the version of ip protocol used (4 or 6)
"""
if _config.parser.has_section("logging"):
for entry in _config.parser.items("logging"):
if not entry[0].startswith("ignore_"):
continue
chain = entry[0].replace("ignore_", "").upper()
values = [
item for item in re.split(r";\s*", entry[1]) if item != ""
]
for value in values:
data = [
item if item != "" else None
for item in re.split(r",\s*", value.strip())
]
address1, address2 = data[4:6]
if address1 is not None:
address1 = get_ip_address(address1)
if address2 is not None:
address2 = get_ip_address(address2)
if (address1 is not None and address1.version != version
) or (address2 is not None
and address2.version != version):
continue
handler.no_log(chain, *data)
if _config.getboolean("ssh_knocking"):
handler.enable_ssh_knocking(_config.parser["ssh_knocking"])
if _config.parser.has_section("logging"):
section = _config.parser["logging"]
for chain in section.getlist("log"):
handler.log(chain, section.get("prefix"),
section.get("rate", None),
section.getint("level", 4))
if config.getboolean("ipv4", False):
setup(ipv4_handler, config, version=4)
if config.getboolean("ipv6", False):
setup(ipv6_handler, config, version=6)
def get_epd_data(
config: SectionProxy) -> tuple[Optional[ctypes.CDLL], tuple[int, int]]:
if (is_supported_epd(config.get('EPD_MODEL'))):
if (config.getboolean('FILE_OUTPUT')):
return (None, (1872, 1404))
else:
return (ctypes.CDLL("lib/epd78.so"), (1872, 1404))
else:
raise Exception(f'Unsupported model: {config.get("EPD_MODEL")}')
def create(
cls,
name: str,
config: configparser.SectionProxy,
) -> "LogindSessionsIdle":
types = config.get("types", fallback="tty,x11,wayland").split(",")
types = [t.strip() for t in types]
states = config.get("states", fallback="active,online").split(",")
states = [t.strip() for t in states]
return cls(name, types, states)
def get_encryption(section: configparser.SectionProxy) -> Encryption:
imaps = section.getboolean("imaps", fallback=False)
starttls = section.getboolean("starttls", fallback=not imaps)
if starttls and imaps:
raise ImapError(f"You cannot enable both starttls and imaps")
if imaps:
return Encryption.IMAPS
if starttls:
return Encryption.STARTTLS
return Encryption.NONE
def load(section: SectionProxy) -> CacheConfig:
"""Load overridden variables from a section within a config file."""
config = CacheConfig()
config.path = os.path.expanduser(section.get("path", fallback=config.path))
config.max_entries = section.getint("max_entries", fallback=config.max_entries)
config.max_size = section.getint("max_size", fallback=config.max_size)
return config
def create(cls, name: str, config: configparser.SectionProxy) -> "Mpd":
try:
host = config.get("host", fallback="localhost")
port = config.getint("port", fallback=6600)
timeout = config.getint("timeout", fallback=5)
return cls(name, host, port, timeout)
except ValueError as error:
raise ConfigurationError(
"Host port or timeout configuration wrong: {}".format(
error)) from error
def allow_ssh_temporarily(_config: SectionProxy) -> None:
"""
Allows ssh temporarily
:param _config: the config to use
"""
number_of_required_chains = len(_config.getlist("ports"))
for i in range(1, number_of_required_chains):
self.execute("-N SSH-KNOCKING-{}".format(i))
command = "-A INPUT -m state --state NEW -m tcp -p tcp"
if _config.get("interface", None):
command += " -i " + _config.get("interface")
command += " --dport " + _config.get("ssh_port", "22")
command += " -m recent --rcheck --seconds " + _config.get(
"timeout", 30)
command += " --name SSH{}".format(
len(_config.getlist("ports")) - 1)
command += \
' -m comment --comment "Allow port {} for ssh for {} if the connecting ip is in the list SSH{}"'.format(
_config.get("ssh_port", "22"), _config.get("timeout", 30), len(_config.getlist("ports")) - 1
)
self.execute(command)
def allow_ssh_temporarily(_config: SectionProxy) -> None:
"""
Allows ssh temporarily
:param _config: the config to use
"""
number_of_required_chains = len(_config.getlist("ports"))
for i in range(1, number_of_required_chains):
self.execute("-N SSH-KNOCKING-{}".format(i))
command = "-A INPUT -m state --state NEW -m tcp -p tcp"
if _config.get("interface", None):
command += " -i " + _config.get("interface")
command += " --dport " + _config.get("ssh_port", "22")
command += " -m recent --rcheck --seconds " + _config.get("timeout", 30)
command += " --name SSH{}".format(len(_config.getlist("ports")) - 1)
command += \
' -m comment --comment "Allow port {} for ssh for {} if the connecting ip is in the list SSH{}"'.format(
_config.get("ssh_port", "22"), _config.get("timeout", 30), len(_config.getlist("ports")) - 1
)
self.execute(command)
def translate_bootstrap_servers(
section: configparser.SectionProxy) -> List[str]:
server_tpl = "{}"
if section.get("bootstrap_domain"):
server_tpl += "." + section["bootstrap_domain"]
if section.get("bootstrap_port"):
server_tpl += ":" + section["bootstrap_port"]
return [
server_tpl.format(server.strip())
for server in section["bootstrap_hosts"].split(",")
]
def read_authorization_settings(
self, authorization_section: configparser.SectionProxy):
"""
read settings from given sections, if no value found, no value would change
:param authorization_section: the authorization in config
:return: None
"""
self.domain = authorization_section.get('domain', self.domain)
self.auth_token = authorization_section.get('auth_token',
self.auth_token)
self.upload_repo_id = authorization_section.get(
'upload_repo_id', self.upload_repo_id)
def setup(self, config: SectionProxy) -> None:
"""Set up the counters for config."""
client_limit = config.getint("limit_client_tests", fallback=0)
if client_limit:
client_period = config.getfloat("limit_client_period", fallback=1) * 3600
self._setup("client_id", client_limit, client_period)
origin_limit = config.getint("limit_origin_tests", fallback=0)
if origin_limit:
origin_period = config.getfloat("limit_origin_period", fallback=1) * 3600
self._setup("origin", origin_limit, origin_period)
self.running = True
def __createInstanceFromSettings(
base_class: Type[T],
section: configparser.SectionProxy) -> Optional[T]:
instance = Storage.__getInstance(base_class,
section.get("type", fallback="UNSET"))
if instance is None:
return None
if hasattr(instance, "name"):
instance.name = section.get("name", fallback=instance.name)
for setting in instance:
if setting.type.key in section:
setting.value = section[setting.type.key]
return instance
def create(cls, name: str, config: configparser.SectionProxy) -> "Users":
with warnings.catch_warnings():
warnings.simplefilter("ignore", FutureWarning)
try:
user_regex = re.compile(config.get("name", fallback=r".*"))
terminal_regex = re.compile(
config.get("terminal", fallback=r".*"))
host_regex = re.compile(config.get("host", fallback=r".*"))
return cls(name, user_regex, terminal_regex, host_regex)
except re.error as error:
raise ConfigurationError(
"Regular expression is invalid: {}".format(error),
) from error
def handle_service(config: SectionProxy) -> None:
"""
Sets a rule or a service
:param config: the configuration for the rule
"""
for src in config.getlist("source", [None]):
for dst in config.getlist("destination", [None]):
source = None
destination = None
if src is not None:
source = get_ip_address(src)
if source is None:
print(
"[ERROR] Could not determine ip address for {} : skipping"
.format(src))
continue
if dst is not None:
destination = get_ip_address(dst)
if destination is None:
print(
"[ERROR] Could not determine ip address for {} : skipping"
.format(dst))
continue
rule = IptablesRule(name=config.name,
interface=config.get("interface"),
chain=config.get("chain"),
protocol=config.get("protocol"),
action=config.get("action"),
source=source,
destination=destination,
sport=config.get("sport"),
dport=config.get("dport"),
remote=config.get("remote", None))
if config.getboolean("ipv4", False) and (rule.source is None or rule.source.version == 4) and \
(rule.destination is None or rule.source.version == 4):
ipv4_handler.add_rule(rule)
if config.getboolean("ipv6") and (rule.source is None or rule.source.version == 6) and \
(rule.destination is None or rule.source.version == 6):
ipv6_handler.add_rule(rule)
if (rule.source is not None and rule.destination is not None) and \
rule.destination.version != rule.source.version:
print(
"[ERROR] Could not add rule with ip versions no matching: {} and {}"
.format(str(rule.source, rule.destination)))
def extract_config(section: SectionProxy,
config_filename: str) -> Dict[str, str]:
AlertRouter.validate_config_fields_existence(section, config_filename)
if "twilio" in config_filename:
return {
'id':
section.get('id'),
'parent_ids':
[x for x in section.get('parent_ids').split(",") if x.strip()],
'info':
False,
'warning':
False,
'error':
False,
'critical':
True,
}
return {
'id':
section.get('id'),
'parent_ids':
[x for x in section.get('parent_ids').split(",") if x.strip()],
'info':
section.getboolean('info'),
'warning':
section.getboolean('warning'),
'error':
section.getboolean('error'),
'critical':
section.getboolean('critical'),
}
def show_temperatur_warning_icon(temperature: float, time: Datetime,
config: SectionProxy) -> bool:
if temperature >= config.getint('HIGH_TEMPERATURE_WARNING_THRESHOLD'):
return True
if temperature <= config.getint('LOW_TEMPERATURE_WARNING_THRESHOLD'):
return True
if temperature >= config.getint(
'TROPICAL_NIGHT_TEMPERATURE_WARNING_THRESHOLD') and (
time.hour > 21 or time.hour < 8):
return True
return False
def standalone_main(config: SectionProxy) -> None:
"""Run REDbot as a standalone Web server."""
# load static files
static_types = {
b'.js': b'text/javascript',
b'.css': b'text/css',
b'.png': b'image/png',
}
static_files = {}
for root, dirs, files in os.walk(config['static_dir']):
for name in files:
try:
path = os.path.join(root, name)
uri = os.path.relpath(path, config['static_dir'])
static_files[b"/static/%s" % uri.encode('utf-8')] = open(path, 'rb').read()
except IOError:
sys.stderr.write("* Problem loading %s\n" % path)
def red_handler(x: EventEmitter) -> None:
@thor.events.on(x)
def request_start(method: bytes, uri: bytes, req_hdrs: RawHeaderListType) -> None:
p_uri = urlsplit(uri)
if p_uri.path in static_files:
headers = []
file_ext = os.path.splitext(p_uri.path)[1].lower()
content_encoding = static_types.get(file_ext, b'application/octet-stream')
headers.append((b'Content-Encoding', content_encoding))
headers.append((b'Cache-Control', b'max-age=300'))
x.response_start(b"200", b"OK", headers)
x.response_body(static_files[p_uri.path])
x.response_done([])
elif p_uri.path == b"/":
try:
RedWebUi(config, method.decode(config['charset']), p_uri.query,
x.response_start, x.response_body, x.response_done)
except Exception:
sys.stderr.write("""
*** FATAL ERROR
REDbot has encountered a fatal error which it really, really can't recover from
in standalone server mode. Details follow.
""")
import traceback
traceback.print_exc()
thor.stop()
sys.exit(1)
else:
x.response_start(b"404", b"Not Found", [])
x.response_done([])
server = thor.http.HttpServer(config.get('host', ''), int(config['port']))
server.on('exchange', red_handler)
try:
thor.run()
except KeyboardInterrupt:
sys.stderr.write("Stopping...\n")
thor.stop()
def from_config(cls, section: configparser.SectionProxy, option_handler_id: str = None) -> OptionHandler:
"""
Create a handler of this class based on the configuration in the config section.
:param section: The configuration section
:param option_handler_id: Optional extra identifier
:return: A handler object
:rtype: OptionHandler
"""
sub_options = []
for name, value in section.items():
# Strip numbers from the end, this can be used to supply the same option multiple times
name = name.rstrip('0123456789-')
if '-' in name or '_' in name:
suboption_name = name.replace('_', '-').lower()
else:
suboption_name = camelcase_to_dash(name)
suboption = name_registry.get(suboption_name)
if not suboption:
raise configparser.ParsingError("Unknown suboption: {}".format(suboption_name))
for suboption_value in re.split('[,\t ]+', value):
if not suboption_value:
raise configparser.ParsingError("{} option has no value".format(name))
sub_options.append(suboption.from_string(suboption_value))
return cls(sub_options)
def __init__(self, config: SectionProxy, method: str, query_string: bytes,
response_start: Callable[..., None],
response_body: Callable[..., None],
response_done: Callable[..., None],
error_log: Callable[[str], int] = sys.stderr.write) -> None:
self.config = config # type: SectionProxy
self.charset_bytes = self.config['charset'].encode('ascii')
self.method = method # Request method to the UX; bytes
self.response_start = response_start
self.response_body = response_body
self._response_done = response_done
self.error_log = error_log # function to log errors to
self.test_uri = None # type: str
self.req_hdrs = None # type: StrHeaderListType
self.format = None # type: str
self.test_id = None # type: str
self.check_name = None # type: str
self.descend = None # type: bool
self.save = None # type: bool
self.timeout = None # type: Any
self.referer_spam_domains = [] # type: List[str]
if config.get("referer_spam_domains", ""):
self.referer_spam_domains = [i.strip() for i in \
config["referer_spam_domains"].split()]
self.run(query_string)
def __init__(self,
config: SectionProxy,
method: str,
query_string: bytes,
req_headers: RawHeaderListType,
response_start: Callable[..., None],
response_body: Callable[..., None],
response_done: Callable[..., None],
error_log: Callable[[str], int] = sys.stderr.write) -> None:
self.config = config # type: SectionProxy
self.charset_bytes = self.config['charset'].encode('ascii')
self.method = method
self.req_headers = req_headers
self.response_start = response_start
self.response_body = response_body
self._response_done = response_done
self.error_log = error_log # function to log errors to
self.test_uri = None # type: str
self.test_id = None # type: str
self.robot_time = None # type: str
self.robot_hmac = None # type: str
self.req_hdrs = None # type: StrHeaderListType
self.format = None # type: str
self.check_name = None # type: str
self.descend = None # type: bool
self.save = None # type: bool
self.save_path = None # type: str
self.timeout = None # type: Any
self.referer_spam_domains = [] # type: List[str]
if config.get("limit_client_tests", fallback=""):
if self._client_period is None:
self._client_period = config.getfloat("limit_client_period", fallback=1) * 3600
thor.schedule(self._client_period, self.client_ratelimit_cleanup)
if config.get("limit_origin_tests", fallback=""):
if self._origin_period is None:
self._origin_period = config.getfloat("limit_origin_period", fallback=1) * 3600
thor.schedule(self._origin_period, self.origin_ratelimit_cleanup)
if config.get("referer_spam_domains", fallback=""):
self.referer_spam_domains = [i.strip() for i in \
config["referer_spam_domains"].split()]
self.run(query_string)
def from_config_section(cls, section: configparser.SectionProxy):
domain_names = section.get('domain-names')
if domain_names is None:
raise configparser.NoOptionError('domain-names', section.name)
domain_names = re.split('[,\t ]+', domain_names)
option = cls(search_list=domain_names)
option.validate()
return option
def __init__(self, config: SectionProxy) -> None:
self.config = config
self.static_files = {} # type: Dict[bytes, bytes]
# Load static files
self.walk_files(config['asset_dir'], b"static/")
if config.get('extra_base_dir'):
self.walk_files(config['extra_base_dir'])
# Set up the watchdog
if os.environ.get("SYSTEMD_WATCHDOG"):
thor.schedule(self.watchdog_freq, self.watchdog_ping)
# Set up the server
server = thor.http.HttpServer(config.get('host', ''), int(config['port']))
server.on('exchange', self.red_handler)
try:
thor.run()
except KeyboardInterrupt:
sys.stderr.write("Stopping...\n")
thor.stop()
def from_config(cls, section: configparser.SectionProxy, option_handler_id: str = None) -> OptionHandler:
"""
Create a handler of this class based on the configuration in the config section.
:param section: The configuration section
:param option_handler_id: Optional extra identifier
:return: A handler object
:rtype: OptionHandler
"""
authoritative = section.getboolean('authoritative', False)
return cls(authoritative)
def factory(conf: SectionProxy, force_installation: bool):
"""
Parses the configuration and returns the correct Installer
:param conf: the configuration of the program to install
:param force_installation: if reinstalling is enabled or not
:return: an Installer instance
"""
if "git_repo" in conf:
return GitInstaller(conf, force_installation)
elif "svn_repo" in conf:
return SVNInstaller(conf, force_installation)
elif conf.get("system_package", False):
return DependenciesInstaller.factory([conf.get("name")])
elif conf.getboolean("license", False):
return LicensedSourceInstaller(conf, force_installation)
elif conf.get("url", None):
return DownloadableSourceInstaller(conf, force_installation)
elif conf.get("source", None):
return SourceInstaller(conf, force_installation)
else:
raise RuntimeError("There was an error while configuring the Installer to use")
def from_config(cls, section: configparser.SectionProxy, option_handler_id: str = None) -> OptionHandler:
"""
Create a handler of this class based on the configuration in the config section.
:param section: The configuration section
:param option_handler_id: Optional extra identifier
:return: A handler object
:rtype: OptionHandler
"""
preference = section.getint('preference')
if preference is None:
raise configparser.NoOptionError('preference', section.name)
return cls(preference)
def from_config(cls, section: configparser.SectionProxy, option_handler_id: str = None) -> OptionHandler:
"""
Create a handler of this class based on the configuration in the config section.
:param section: The configuration section
:param option_handler_id: Optional extra identifier
:return: A handler object
:rtype: OptionHandler
"""
address = section.get('server-address')
if address is None:
raise configparser.NoOptionError('server-address', section.name)
address = IPv6Address(address)
return cls(address)
def from_config(cls, section: configparser.SectionProxy, option_handler_id: str = None) -> OptionHandler:
"""
Create a handler of this class based on the configuration in the config section.
:param section: The configuration section
:param option_handler_id: Optional extra identifier
:return: A handler object
:rtype: OptionHandler
"""
domain_names = []
for name, value in section.items():
# Strip numbers from the end, this can be used to supply the same option multiple times
name = name.rstrip("0123456789-")
if name != "domain-name":
continue
domain_names.append(value)
return cls(domain_names)
def from_config(cls, section: configparser.SectionProxy, option_handler_id: str = None) -> OptionHandler:
"""
Create a handler of this class based on the configuration in the config section.
:param section: The configuration section
:param option_handler_id: Optional extra identifier
:return: A handler object
:rtype: OptionHandler
"""
# The option handler ID is our primary link prefix
responsible_for_links = []
try:
prefix = IPv6Network(option_handler_id)
responsible_for_links.append(prefix)
except ValueError:
raise configparser.ParsingError("The ID of [{}] must be the primary link prefix".format(section.name))
# Add any extra prefixes
additional_prefixes = section.get("additional-prefixes", "").split(" ")
for additional_prefix in additional_prefixes:
if not additional_prefix:
continue
try:
prefix = IPv6Network(additional_prefix)
responsible_for_links.append(prefix)
except ValueError:
raise configparser.ParsingError("'{}' is not a valid IPv6 prefix".format(additional_prefix))
# Get the lifetimes
address_preferred_lifetime = section.getint("address-preferred-lifetime", 3600)
address_valid_lifetime = section.getint("address-valid-lifetime", 7200)
prefix_preferred_lifetime = section.getint("prefix-preferred-lifetime", 43200)
prefix_valid_lifetime = section.getint("prefix-valid-lifetime", 86400)
sqlite_filename = section.get("assignments-file")
return cls(
sqlite_filename,
responsible_for_links,
address_preferred_lifetime,
address_valid_lifetime,
prefix_preferred_lifetime,
prefix_valid_lifetime,
)
def from_config(cls, section: configparser.SectionProxy, option_handler_id: str = None) -> OptionHandler:
"""
Create a handler of this class based on the configuration in the config section.
:param section: The configuration section
:param option_handler_id: Optional extra identifier
:return: A handler object
:rtype: OptionHandler
"""
min_t1 = cls.str_to_time(section.get('min-t1', '0'))
max_t1 = cls.str_to_time(section.get('max-t1', 'INFINITY'))
factor_t1 = cls.str_to_factor(section.get('factor-t1', '0.5'))
min_t2 = cls.str_to_time(section.get('min-t2', '0'))
max_t2 = cls.str_to_time(section.get('max-t2', 'INFINITY'))
factor_t2 = cls.str_to_factor(section.get('factor-t2', '0.8'))
return cls(min_t1, max_t1, factor_t1, min_t2, max_t2, factor_t2)
def enable_ssh_knocking(self, config: SectionProxy) -> None:
"""
enables iptables-only ssh knocking
:param config: the configuration to use for the knocking
"""
def allow_ssh_temporarily(_config: SectionProxy) -> None:
"""
Allows ssh temporarily
:param _config: the config to use
"""
number_of_required_chains = len(_config.getlist("ports"))
for i in range(1, number_of_required_chains):
self.execute("-N SSH-KNOCKING-{}".format(i))
command = "-A INPUT -m state --state NEW -m tcp -p tcp"
if _config.get("interface", None):
command += " -i " + _config.get("interface")
command += " --dport " + _config.get("ssh_port", "22")
command += " -m recent --rcheck --seconds " + _config.get("timeout", 30)
command += " --name SSH{}".format(len(_config.getlist("ports")) - 1)
command += \
' -m comment --comment "Allow port {} for ssh for {} if the connecting ip is in the list SSH{}"'.format(
_config.get("ssh_port", "22"), _config.get("timeout", 30), len(_config.getlist("ports")) - 1
)
self.execute(command)
def remove_from_list(entry_number: int) -> None:
"""
Removes the ip from the list given by the number
:param entry_number: the number for which to remove the list
"""
command = "-A INPUT -m state --state NEW -m tcp -p tcp -m recent --name SSH{} --remove".format(entry_number)
command += ' -j DROP -m comment --comment "Remove connecting ip from the SSH{} list"'.format(entry_number)
self.execute(command)
def enable_jump(_port: int, entry_number: int) -> None:
"""
Enables jumping to the given chain
:param _port: the port on which to enable the jump
:param entry_number: the number of the entry to which to jump
"""
command = "-A INPUT -m state --state NEW -m tcp -p tcp --dport {} -m recent --rcheck --name SSH{}".format(
_port, entry_number - 1
)
command += ' -j SSH-KNOCKING-{} -m comment --comment "Checks for the sequence and jumps if correct"'.format(
entry_number
)
self.execute(command)
def initiate_knocking(_port: int) -> None:
"""
Sequence initiation for the port knocking
:param _port: the port on which to knock
"""
command = "-A INPUT -m state --state NEW -m tcp -p tcp --dport {} -m recent --name SSH0 --set".format(_port)
command += ' -j DROP -m comment --comment "Sequence initiation for port knocking"'
self.execute(command)
def hide_port(number: int) -> None:
"""
Hides the port given by the number by dropping it
:param number: the number of the chain on which to drop
"""
command = "-A INPUT -m recent --name SSH{} --set -j DROP -m comment".format(number)
command += '--comment "Disguise successful knock as a closed port for obfuscation"'
self.execute(command)
# noinspection PyTypeChecker
allow_ssh_temporarily(config)
# noinspection PyUnresolvedReferences
ports = config.getlist("ports")
for port in reversed(ports):
remove_from_list(ports.index(port))
if ports.index(port) != 0:
enable_jump(port, ports.index(port))
else:
initiate_knocking(port)
for entry in range(len(ports) - 1):
hide_port(entry)