def cli(log_level, slack_webhook, slack_username, slack_format):
logger.remove()
logger.add(stderr, level=log_level)
if slack_webhook:
params = {"username": slack_username, "webhook_url": slack_webhook}
slack = NotificationHandler("slack", defaults=params)
logger.add(slack, format=slack_format, level="SUCCESS")
def add_gmail_sender(logger):
params = {
"username": notify_vars.get('mail_login'),
"password": notify_vars.get('mail_pass'),
"to": notify_vars.get('mail_to'),
"subject": strings.Report.mail_subject
}
handler = NotificationHandler("gmail", defaults=params)
logger.add(handler, level="ERROR")
return logger
def _setup_telegram_notifier():
t_settings = [x for x in _read_params() if x[0].lower() == "telegram"]
if not t_settings:
raise ValueError("To use the telegram notifier you need to edit the file ./notifier_params.tsv and specify "
"'telegram' followed by your API_TOKEN and the target chat_id")
for config in t_settings:
notification_handler = NotificationHandler(
provider="telegram", defaults={
"token": config[1],
"chat_id": config[2]
}
)
notification_handler_wrapper = NotificationHandlerWrapper(notification_handler)
logger.add(sink=notification_handler_wrapper,
level="ERROR",
format="<green>{time:YYYY-MM-DD HH:mm:ss Z}</green> | "
"<level>{level: <8}</level> | "
"<cyan>{name}</cyan>:<cyan>{function}</cyan>:<cyan>{line}</cyan> - <level>{message}</level>",
backtrace=False, diagnose=False) # disables sending exception stacktrace
def test():
# logger.debug("That's it,beautiful and simple logging")
params = {
"username": "******",
"password": "******",
"to": "*****@*****.**"
}
# Send a single notification
notifier = notifiers.get_notifier("gmail")
print(notifier.required)
notifier.notify(message='this is test message',
to='*****@*****.**')
notifier.notify(message="The application is running!", **params)
# Be alerted on each error message
from notifiers.logging import NotificationHandler
handler = NotificationHandler("gmail", defaults=params)
logger.add(handler, level="ERROR")
def enableTelegram(self, chatID, token):
self.telegram = namedtuple('Telegram', 'notifier chatID token')(
get_notifier('telegram'),
chatID,
token,
)
defaults = {
'chat_id': chatID,
'token': token,
}
handler = NotificationHandler('telegram', defaults=defaults)
handler.setLevel(logging.ERROR)
handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s'))
log = logging.getLogger()
log.addHandler(handler)
self.notify('Telegram enabled for logging')
def enablePushover(self, userkey, APIkey):
self.pushover = namedtuple('Pushover', 'notifier APIkey userkey')(
get_notifier('pushover'),
APIkey,
userkey,
)
defaults = {
'user': userkey,
'token': APIkey,
}
handler = NotificationHandler('pushover', defaults=defaults)
handler.setLevel(logging.ERROR)
handler.setFormatter(logging.Formatter('%(asctime)s - %(message)s'))
log = logging.getLogger()
log.addHandler(handler)
self.notify('Pushover enabled for logging')
def main(source, target, recycle, exclude, dry_run, sleep_time, log_level,
slack_webhook, slack_username, slack_format):
logger.remove()
logger.add(stderr, level=log_level)
if slack_webhook:
params = {"username": slack_username, "webhook_url": slack_webhook}
slack = NotificationHandler("slack", defaults=params)
logger.add(slack, format=slack_format, level="SUCCESS")
logger.success('{} Started with {} source(s)', basename(argv[0]),
len(source))
logger.info(' --source "{}"', ':'.join(source))
logger.info(' --target "{}"', target)
logger.info(' --recycle "{}"', recycle)
if dry_run:
logger.info(' --dry-run')
logger.info(' --sleep-time {}', sleep_time)
logger.info(' --log-level "{}"', log_level)
logger.info(' --slack-webhook "{}"', slack_webhook)
logger.info(' --slack-username "{}"', slack_username)
logger.info(' --slack-format "{}"', slack_format)
try:
makedirs(target, exist_ok=True)
makedirs(recycle, exist_ok=True)
h = Harvestr(target,
recycle,
source,
exclude=[exclude] if exclude else None,
dry_run=dry_run)
while True:
logger.debug(f'Sleeping {sleep_time} seconds')
sleep(sleep_time)
h.main()
finally:
lwt()
BALANCE_WARNING_THRESHOLD,
BALANCE_ERROR_THRESHOLD,
APPLY_GAS_PRICE_STRATEGY,
MAX_TX_WAIT_SECONDS,
LOG_LEVEL,
PROCESS_INTERVAL,
)
from src.utils import (
get_web3_client,
configure_default_account,
InterruptHandler,
check_default_account_balance,
)
# Send notification to admins on error
handler = NotificationHandler("telegram")
logger.remove(0)
logger.add(
sink=sys.stderr,
format="<green>{time:YYYY-MM-DD HH:mm:ss.SSS}</green>"
" <level>{level}</level> <level>{message}</level>",
level=LOG_LEVEL,
)
logger.add(handler, level="ERROR", backtrace=False, diagnose=False)
@logger.catch
def main() -> None:
# setup Web3 client
web3_client = get_web3_client(
http_endpoint=WEB3_HTTP_ENDPOINT,
def __init__(self, config_file):
# Load config.
try:
logger.debug(f"Reading config from '{config_file}'")
self.config = config.Config(config_file)
except (OSError, threatingestor.exceptions.IngestorError):
# Error loading config.
logger.exception("Couldn't read config")
sys.exit(1)
# Configure logging with optional notifiers.
logger.configure(**self.config.logging())
try:
logger.level("NOTIFY", no=35, color="<yellow>", icon="\U0001F514")
except TypeError:
# logger raises TypeError if NOTIFY is already defined
pass
if notifiers:
notifier_config = self.config.notifiers()
notifier = notifiers.get_notifier(notifier_config.get('provider'))
if notifier:
logger.debug(
f"Adding notification handler '{notifier_config.get('provider')}'"
)
# Notifier 'provider_name' is set and valid.
handler = NotificationHandler(**notifier_config)
logger.add(handler, level="NOTIFY")
logger.debug("Log handler reconfigured")
# Configure statsd.
try:
self.statsd = statsd.StatsClient(**self.config.statsd())
self.statsd.incr('start')
except TypeError:
logger.exception("Couldn't initialize statsd client; bad config?")
sys.exit(1)
# Load state DB.
try:
logger.debug(
f"Opening state database '{self.config.state_path()}'")
self.statedb = threatingestor.state.State(self.config.state_path())
except (OSError, IOError, threatingestor.exceptions.IngestorError):
# Error loading state DB.
logger.exception("Error reading state database")
sys.exit(1)
# Instantiate plugins.
try:
logger.debug("Initializing sources")
self.sources = {
name: source(**kwargs)
for name, source, kwargs in self.config.sources()
}
logger.debug("Initializing operators")
self.operators = {
name: operator(**kwargs)
for name, operator, kwargs in self.config.operators()
}
logger.debug("Initializing whitelists")
self.whitelist = threatingestor.whitelist.Whitelist(
self.config.whitelists())
except (TypeError, ConnectionError,
threatingestor.exceptions.PluginError):
logger.warning(
"Twitter config format has recently changed. See https://github.com/InQuest/ThreatIngestor/releases/tag/v1.0.0b5"
)
logger.exception("Error initializing plugins")
sys.exit(1)
# from utils import secrets
from loguru import logger
from notifiers.logging import NotificationHandler
import os
if os.getenv("IS_PROD") == 'True':
handler = NotificationHandler(
"slack",
defaults=dict(webhook_url=os.getenv("SLACK_WEBHOOK")),
)
logger.add(handler, level="ERROR", diagnose=True)
level = record.levelno
# Find caller from where originated the logged message
frame, depth = logging.currentframe(), 2
while frame.f_code.co_filename == logging.__file__:
frame = frame.f_back
depth += 1
logger.opt(depth=depth, exception=record.exc_info).log(level, record.getMessage())
if not LOG_IS_SET:
handler = NotificationHandler(
"telegram",
defaults={
"token": NOTIFY_BOT_TOKEN,
"chat_id": NOTIFY_CHAT_ID
}
)
logger.add(handler, level=logging.WARNING)
bot_logger.setLevel(logging.WARNING)
bot_logger.removeHandler(bot_logger.handlers[0])
bot_logger.addHandler(InterceptHandler())
logger.add('./logs/{time:YYYY-MM-DD_HH-mm-ss}.log',
encoding='UTF-8',
backtrace=True,
diagnose=True,
rotation='10 MB',
compression='zip',
'server': SETTINGS['jira'],
'basic_auth': (os.getenv('JIRA_USERNAME'), os.getenv('JIRA_PASSWORD')),
'options': {'verify': False},
}
COMMVAULT = {
'webconsole_hostname': SETTINGS['commvault']['api'],
'commcell_username': os.getenv('COMMVAULT_USERNAME'),
'commcell_password': os.getenv('COMMVAULT_PASSWORD'),
}
SMTP_PARAMS = {
'from': SETTINGS['smtp']['from'],
'to': SETTINGS['smtp']['to'],
'subject': 'Commvault | Automation scripts',
'host': SETTINGS['smtp']['host'],
'port': SETTINGS['smtp']['port'],
'tls': SETTINGS['smtp']['tls'],
'username': os.getenv('SMTP_USERNAME'),
'password': os.getenv('SMTP_PASSWORD'),
'html': SETTINGS['smtp']['html'],
}
email = get_notifier('email')
email.notify = partial(email.notify, **SMTP_PARAMS)
logger.remove()
logger.add(sink=NotificationHandler('email', defaults=SMTP_PARAMS),
format=SETTINGS['logging']['format'],
level='ERROR')
class SitRep:
"""
SitRep is an automated data comparison utility that reports its
findings via Discord.
https://github.com/EthanC/SitRep
"""
def Initialize(self: Any) -> None:
"""Initialize SitRep and begin primary functionality."""
logger.info("SitRep")
logger.info("https://github.com/EthanC/SitRep")
self.config: Dict[str, Any] = SitRep.LoadConfig(self)
SitRep.SetupLogging(self)
self.git: Github = Utility.GitLogin(self)
for source in self.config["dataSources"]:
SitRep.ProcessDataSource(self, source)
logger.success("Finished processing data sources")
def LoadConfig(self: Any) -> Dict[str, Any]:
"""Load the configuration values specified in config.json"""
try:
with open("config.json", "r") as file:
config: Dict[str, Any] = json.loads(file.read())
except Exception as e:
logger.critical(f"Failed to load configuration, {e}")
exit(1)
logger.success("Loaded configuration")
return config
def SetupLogging(self: Any) -> None:
"""Setup the logger using the configured values."""
settings: Dict[str, Any] = self.config["logging"]
if (level := settings["severity"].upper()) != "DEBUG":
try:
logger.remove()
logger.add(stderr, level=level)
logger.success(f"Set logger severity to {level}")
except Exception as e:
# Fallback to default logger settings
logger.add(stderr, level="DEBUG")
logger.error(f"Failed to set logger severity to {level}, {e}")
if settings["discord"]["enable"] is True:
level: str = settings["discord"]["severity"].upper()
url: str = settings["discord"]["webhookUrl"]
try:
# Notifiers library does not natively support Discord at
# this time. However, Discord will accept payloads which
# are compatible with Slack by appending to the url.
# https://github.com/liiight/notifiers/issues/400
handler: NotificationHandler = NotificationHandler(
"slack", defaults={"webhook_url": f"{url}/slack"}
)
logger.add(
handler,
level=level,
format="```\n{time:YYYY-MM-DD HH:mm:ss.SSS} | {level:<8} | {name}:{function}:{line} - {message}\n```",
)
logger.success(f"Enabled logging to Discord with severity {level}")
except Exception as e:
logger.error(f"Failed to enable logging to Discord, {e}")
def wrapper_function(*args, **kwargs):
args[1].msg += head
NotificationHandler.emit(*args, **kwargs)
import os
from notifiers.logging import NotificationHandler
TELEGRAM_NOTIFY_TOKEN = os.environ.get("TELEGRAM_NOTIFY_TOKEN")
TELEGRAM_NOTIFY_CHAT_ID = int(os.environ.get("TELEGRAM_NOTIFY_CHAT_ID"))
params = {'token': TELEGRAM_NOTIFY_TOKEN, 'chat_id': TELEGRAM_NOTIFY_CHAT_ID}
notify_handler = NotificationHandler("telegram", defaults=params)
def TraefikFlare(
log_level,
slack_webhook,
slack_username,
slack_format,
sleep_time,
traefik_url,
ipify_url,
cloudflare_email,
cloudflare_api_key,
):
logger.remove()
logger.add(stderr, level=log_level)
if slack_webhook:
params = {"username": slack_username, "webhook_url": slack_webhook}
slack = NotificationHandler("slack", defaults=params)
logger.add(slack, format=slack_format, level="SUCCESS")
logger.success(f"{basename(argv[0])} Started")
logger.info(f' --log-level "{log_level}"')
logger.info(f' --slack-webhook "{slack_webhook}"')
logger.info(f' --slack-username "{slack_username}"')
logger.info(f' --slack-format "{slack_format}"')
logger.info(f" --sleep-time {sleep_time}")
logger.info(f' --traefik-url "{traefik_url}"')
logger.info(f' --ipify-url "{ipify_url}"')
logger.info(f' --cloudflare-email "{cloudflare_email}"')
logger.info(f' --cloudflare-api-key "{cloudflare_api_key}"')
traefik = Traefik(url=traefik_url)
ipify = IPIFY(url=ipify_url)
cloudflare = CloudFlare(email=cloudflare_email, token=cloudflare_api_key)
zones = None
while True:
try:
zones = {
zone["name"]: zone["id"]
for zone in cloudflare.zones.get(params={"per_page": 100})
}
logger.info(f"Cloudflare connection OK, {len(zones)} zones found:")
for zone in zones:
logger.info(f" {zones[zone]}: {zone}")
except CloudFlareAPIError as e:
logger.error(
f"CloudFlareAPIError getting zones for {cloudflare_email}: {e}"
)
continue
except Exception as e:
logger.error(
f"Exception getting zones for {cloudflare_email}: {e}")
continue
if zones is not None:
break
logger.info(f"Sleeping {sleep_time} seconds")
sleep(sleep_time)
previous = None
current = None
while True:
try:
hosts = traefik.routes.hosts
logger.info(f"Traefik connection OK, {len(hosts)} hosts found:")
for host in hosts:
logger.info(f" {host}")
except Exception as e:
logger.error(f"Exception getting hosts from {traefik_url}: {e}")
previous = None
logger.info(f"Sleeping {sleep_time} seconds")
sleep(sleep_time)
continue
try:
ip_address = ipify.ip_address
except Exception as e:
logger.error(f"Exception getting IP address from {ipify_url}: {e}")
previous = None
logger.info(f"Sleeping {sleep_time} seconds")
sleep(sleep_time)
continue
current = {
host: str(ip_address)
for host in hosts if host.endswith(tuple(zones))
}
if current != previous:
logger.info(
f"Hosts changed, {len(current)} hosts found ({len(hosts) - len(current)} filtered):"
)
previous = current
for host in current:
zone_name = [zone for zone in zones if host.endswith(zone)][0]
zone_id = zones[zone_name]
logger.debug(
f"Starting {host} {current[host]} in zone {zone_id} {zone_name}"
)
try:
dns_records = cloudflare.zones.dns_records.get(zone_id,
params={
"name":
host,
"match":
"all",
"type":
"A"
})
except CloudFlareAPIError as e:
logger.error(
f"CloudFlareAPIError getting dns records for {zone_id}: {host}: {e}"
)
previous = None
continue
except Exception as e:
logger.error(
f"Exception getting dns records for {zone_id}: {host}: {e}"
)
previous = None
continue
if len(dns_records) > 1:
logger.error(
f"Received {len(dns_records)} dns records for {zone_id}: {host}, expected 1"
)
previous = None
continue
if len(dns_records) == 0:
# Create a new record
try:
cloudflare.zones.dns_records.post(
zone_id,
data={
"name": host,
"type": "A",
"content": current[host],
"proxied": False,
},
)
except CloudFlareAPIError as e:
logger.error(
f"CloudFlareAPIError creating dns record for {zone_id}: {host}: {e}"
)
previous = None
continue
except Exception as e:
logger.error(
f"Exception creating dns record for {zone_id}: {host}: {e}"
)
previous = None
continue
logger.success(
f"{host} created and set to {current[host]}.")
if len(dns_records) == 1:
# update the record - unless it's already correct
dns_record = dns_records[0]
old_ip_address = dns_record["content"]
if current[host] == old_ip_address:
logger.info(
f"{host} already set to {current[host]}, no change required."
)
continue
dns_record_id = dns_record["id"]
try:
cloudflare.zones.dns_records.put(
zone_id,
dns_record_id,
data={
"name": host,
"type": "A",
"content": current[host],
"proxied": False,
},
)
except CloudFlareAPIError as e:
logger.error(
f"CloudFlareAPIError updating dns record for {zone_id}: {host}: {e}"
)
previous = None
continue
except Exception as e:
logger.error(
f"Exception updating dns record for {zone_id}: {host}: {e}"
)
previous = None
continue
logger.success(
f"{host} updated from {old_ip_address} to {current[host]}."
)
if sleep_time == 0:
break
logger.info(f"Sleeping {sleep_time} seconds")
sleep(sleep_time)
logger.enable("my_library")
logger.info("This message however is propagated to the sinks")
import notifiers
params = {
"username": "******",
"password": "******",
"to": "*****@*****.**"
}
# Send a single notification
notifier = notifiers.get_notifier("gmail")
notifier.notify(message="The application is running!", **params)
# Be alerted on each error message
from notifiers.logging import NotificationHandler
handler = NotificationHandler("gmail", defaults=params)
logger.add(handler, level="ERROR")
logger.info("some error happened")## Define System Environement and Parse the log# Linux / OSX
export LOGURU_FORMAT="{time} | <lvl>{message}</lvl>"
# Windows
setx LOGURU_DEBUG_COLOR "<green>"
pattern = r"(?P<time>.*) - (?P<level>[0-9]+) - (?P<message>.*)" # Regex with named groups
caster_dict = dict(time=dateutil.parser.parse, level=int) # Transform matching groups
for groups in logger.parse("file.log", pattern, cast=caster_dict):
print("Parsed:", groups)
# {"level": 30, "message": "Log example", "time": datetime(2018, 12, 09, 11, 23, 55)}
def return_handler(provider_name, logging_level, data=None, **kwargs):
caplog.set_level(logging.INFO)
hdlr = NotificationHandler(provider_name, data, **kwargs)
hdlr.setLevel(logging_level)
return hdlr
logger.info("Start")
logger.info("Read config file")
CONFIG_INI = 'hire_config.ini'
config = ConfigParser()
config.read(CONFIG_INI)
API_HOST = config['default']['api_host']
SERVER_NAME = config['default']['server_name']
print(API_HOST)
EXCLUDE_PORTS = config['default']['exclude_ports'].split()
telegram = NotificationHandler('telegram',
defaults={
'token': config['default']['telegram_api'],
'chat_id': config['default']['telegram_id']
})
logger.add(telegram, level=logging.ERROR)
sg.ChangeLookAndFeel('Reddit')
window = sg.Window("SMS Receiver")
table = sg.Table(
[[' ' * 15, ' ' * 18, ' ' * 12, ' ' * 16, ' ' * 12, ' ' * 36]],
size=(200, 33),
max_col_width=100,
headings=['Port', 'IMSI', 'Network', 'Phone', 'Signal', 'Status'],
justification='right',
key='thread_table')
window.Layout([[
sg.Column([[table]]),
