def __init__(self, start_year: float = 2017, parent_log: LogWithInflux = None):
self.logg = LogWithInflux(parent_log, child_name=self.__class__.__name__)
# Set temp dir for downloading the edgar filings
self.tmp_dir = os.path.join(tempfile.gettempdir(), 'edgar')
# Get ticker to CIK mapping
self.logg.debug('Downloading ticket to CIK mapping...')
self.t2cik_df = self.get_ticker_to_cik_map()
# Download EDGAR indexes and retrieve the filepaths associated with them
self.logg.debug('Downloading indexes (this may take ~2 mins)...')
self.edgar_fpaths = self._download_indexes(start_year)
def __init__(self,
driver_path: str = '/usr/bin/chromedriver',
timeout: float = 60,
options: List[str] = None,
headless: bool = True,
parent_log: 'Log' = None):
self.driver = ChromeDriver(driver_path, timeout, options, headless)
self.pid = self.driver.service.process.pid
self.port = self.driver.service.port
self.log = LogWithInflux(parent_log,
child_name=self.__class__.__name__)
self.elem_by_xpath = self.driver.find_element_by_xpath
self.elems_by_xpath = self.driver.find_elements_by_xpath
self.log.debug(
f'Chromedriver started up with pid: {self.pid} receiving on port: {self.port}'
)
def __init__(self, bot: str = 'sasha', parent_log: LogWithInflux = None):
creds = read_props()
credstore = SecretStore('secretprops.kdbx',
creds['secretprops_secret'])
self.log = LogWithInflux(parent_log,
child_name=self.__class__.__name__)
self.st = SlackTools(credstore=credstore,
parent_log=self.log,
slack_cred_name=bot)
self.bkb = BlockKitBuilder()
if bot == 'sasha':
self.hoiatuste_kanal = 'hoiatused'
self.ilma_kanal = 'ilm'
self.kodu_kanal = 'kodu'
self.kaamerate_kanal = 'kaamerad'
self.meemide_kanal = 'meemid'
self.test_kanal = 'test'
self.koduv6rgu_kanal = 'koduvõrk'
self.user_me = 'U015WMFQ0DV'
self.user_marelle = 'U016N5RJZ9C'
elif bot == 'viktor':
self.user_me = 'UM35HE6R5'
self.user_marelle = 'UM3E3G72S'
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import time
from datetime import datetime as dtt
from kavalkilu import LogWithInflux, Keys, NetTools, Hosts, HOME_SERVER_HOSTNAME
from servertools import BrowserAction, SlackComm
ip = NetTools().get_ip()
debug = Hosts().get_ip_from_host(HOME_SERVER_HOSTNAME) != ip
logg = LogWithInflux('vpulse_auto')
# TODO
# build out a table of when monthly, weekly things were last done.
# Handle weekly tasks either based on DOW or recorded table with date_last_done and freq columns
# Get credentials
creds = Keys().get_key('vpulse-creds')
def message_channel_and_log(msg):
slack_comm.send_message(notify_channel, msg)
if debug:
logg.debug(msg)
def get_vpoints():
"""Collects amount of points currently available"""
points_script = "return document.getElementById('progress-bar-menu-points-total-value')"
points = ba.driver.execute_script(points_script).get_attribute(
'textContent').strip()
def setUpClass(cls) -> None:
cls.logg = LogWithInflux('cam-test', log_to_file=False)
cam_ip = Hosts().get_ip_from_host('ac-garaaz')
cls.cam = Amcrest(cam_ip, parent_log=cls.logg)
"""Collect forecast data"""
from datetime import datetime, timedelta
from kavalkilu import LogWithInflux, InfluxDBLocal, InfluxDBHomeAuto
from servertools import OpenWeather, OWMLocation, NWSForecast, NWSForecastZone, \
YrNoWeather, YRNOLocation
# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('forecast', log_dir='weather')
influx = InfluxDBLocal(InfluxDBHomeAuto.WEATHER)
# Number of hours we're looking forward
period_h = 24
# Start & end of the lookahead
p_start = (datetime.now() + timedelta(hours=1))
p_end = (p_start + timedelta(hours=period_h))
owm_fc = OpenWeather(OWMLocation.ATX).hourly_forecast()
nws_fc = NWSForecast(NWSForecastZone.ATX).get_hourly_forecast()
yrno_fc = YrNoWeather(YRNOLocation.ATX).hourly_summary()
# Push all weather data into influx
for svc, df in zip(['own', 'nws', 'yrno'], [owm_fc, nws_fc, yrno_fc]):
log.debug(f'Collecting data from {svc.upper()}...')
cols = ['date', 'humidity', 'temp-avg', 'feels-temp-avg']
if svc == 'nws':
# Replace relative-humidity with humidity
df['humidity'] = df['relative-humidity']
elif svc == 'yrno':
# No humidity, feelslike included in this one
_ = [cols.pop(cols.index(x)) for x in ['feels-temp-avg', 'humidity']]
df['date'] = df['from']
df = df[cols]
import re
from typing import List
from slacktools import BlockKitBuilder as bkb
from servertools import XPathExtractor, SlackComm
from kavalkilu import LogWithInflux
logg = LogWithInflux('wotd')
wotd_url = 'https://www.dictionary.com/e/word-of-the-day/'
sotd_url = 'https://www.thesaurus.com/e/synonym-of-the-day/'
def extract_otd(url: str, is_wotd: bool = False) -> List[dict]:
"""Extract Synonym/Word of the day"""
xtool = XPathExtractor(url)
# Get the most recent WOTD
class_prefix = 'wotd' if is_wotd else 'sotd'
title_section = 'word' if is_wotd else 'synonym'
otd = xtool.xpath(f'//div[contains(@class, "{class_prefix}-items")]/div',
single=True)
sotd_xpath = f'.//div[contains(@class, "{class_prefix}-item__title")]'
wotd_xpath = f'.//div[contains(@class, "otd-item-headword__word")]'
word = xtool.xpath(wotd_xpath if is_wotd else sotd_xpath,
obj=otd,
get_text=True).strip()
# Pronunciation
pronunc = xtool.xpath(
'.//div[contains(@class, "otd-item-headword__pronunciation")]',
obj=otd,
single=True,
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Read temperature and humidity from living room"""
from kavalkilu import LogWithInflux
from pitools import Sensor
from pitools.peripherals import PiElutuba
logg = LogWithInflux('elutuba_temp', log_dir='weather')
# Set the pin (BCM)
sensor = Sensor('DHT22', data_pin=PiElutuba.dht.pin)
# Take readings & log to db
sensor.log_to_db()
logg.debug('Temp logging successfully completed.')
logg.close()
"""Collect current weather data"""
from kavalkilu import LogWithInflux, InfluxDBLocal, InfluxDBHomeAuto
from servertools import OpenWeather, OWMLocation
# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('local', log_dir='weather')
influx = InfluxDBLocal(InfluxDBHomeAuto.WEATHER)
current = OpenWeather(OWMLocation.ATX).current_weather()
# Push all weather data into influx
current = current.drop('date', axis=1)
current['loc'] = 'austin'
influx.write_df_to_table(current, 'loc', current.columns.tolist()[:-1])
influx.close()
log.debug('Temp logging successfully completed.')
log.close()
class BrowserAction:
"""
Performs action to Selenium-class webdriver
Args for __init__:
driver: Selenium-type driver class
"""
# Predefined wait ranges, in seconds
_slow_wait = [15, 30]
_medium_wait = [5, 15]
_fast_wait = [1, 8]
REST_S = 2 # Standard seconds to rest between attempts
STD_ATTEMPTS = 3 # Standard attempts to make before failing
def __init__(self,
driver_path: str = '/usr/bin/chromedriver',
timeout: float = 60,
options: List[str] = None,
headless: bool = True,
parent_log: 'Log' = None):
self.driver = ChromeDriver(driver_path, timeout, options, headless)
self.pid = self.driver.service.process.pid
self.port = self.driver.service.port
self.log = LogWithInflux(parent_log,
child_name=self.__class__.__name__)
self.elem_by_xpath = self.driver.find_element_by_xpath
self.elems_by_xpath = self.driver.find_elements_by_xpath
self.log.debug(
f'Chromedriver started up with pid: {self.pid} receiving on port: {self.port}'
)
def _do_attempts(self,
func: Callable,
*args,
sub_method: str = None,
sub_method_input: str = None,
attempts: int = 3,
rest_s: float = 2) -> Optional[Any]:
"""Attempts a certain method for n times before gracefully failing"""
for i in range(0, attempts):
if i > 0:
# Sleep between attempts, but let a one-off attempt through without much delay
time.sleep(rest_s)
try:
thing = func(*args)
if sub_method is not None:
if sub_method_input is None:
getattr(thing, sub_method)()
break
else:
getattr(thing, sub_method)(sub_method_input)
break
else:
return thing
except Exception as e:
self.log.error(f'Attempt {i + 1}: fail -- {e}')
def tear_down(self):
"""Make sure the browser is closed on cleanup"""
self.log.info('Shutting down browser.')
self.driver.quit()
def get(self, url: str):
"""
Navigates browser to url
Args:
url: str, url to navigate to
"""
self.log.debug(f'Loading url: {url}')
self.driver.get(url)
def click(self,
xpath: str,
attempts: int = STD_ATTEMPTS,
rest_s: float = REST_S):
"""
Clicks HTML element
Args:
xpath: str, xpath to element to click
attempts: int, number of attempts to make before failing
rest_s: int, number of seconds to rest between attempts
"""
self._do_attempts(self.elem_by_xpath,
xpath,
sub_method='click',
attempts=attempts,
rest_s=rest_s)
def clear(self,
xpath: str,
attempts: int = STD_ATTEMPTS,
rest_s: float = REST_S):
"""
Clears form element of text
Args:
xpath: str, xpath to form element
attempts: int, number of attempts to make before failing
rest_s: int, number of seconds to rest between attempts
"""
self._do_attempts(self.elem_by_xpath,
xpath,
sub_method='clear',
attempts=attempts,
rest_s=rest_s)
def enter(self,
xpath: str,
entry_text: str,
attempts: int = STD_ATTEMPTS,
rest_s: float = REST_S):
"""
Enters text into form element
Args:
xpath: str, xpath to form
entry_text: str, text to enter into form
attempts: int, number of attempts to make before failing
rest_s: int, number of seconds to rest between attempts
"""
self._do_attempts(self.elem_by_xpath,
xpath,
sub_method='send_keys',
sub_method_input=entry_text,
attempts=attempts,
rest_s=rest_s)
def elem_exists(self,
xpath: str,
attempts: int = 1,
rest_s: float = REST_S) -> bool:
"""
Determines if particular element exists
Args:
xpath: str, xpath to HTML element
attempts: int, number of attempts to make before failing
rest_s: int, number of seconds to rest between attempts
Returns: True if exists
"""
elem = self._do_attempts(self.elem_by_xpath,
xpath,
attempts=attempts,
rest_s=rest_s)
if elem is not None:
return True
return False
def get_elem(self, xpath: str, single: bool = True, attempts: int = STD_ATTEMPTS, rest_s: float = REST_S) ->\
Union[WebElement, List[WebElement]]:
"""
Returns HTML elements as selenium objects
Args:
xpath: str, xpath of element to return
single: boolean, True if returning only one element. default: True
attempts: int, number of attempts to make before failing
rest_s: int, number of seconds to rest between attempts
Returns: HTML element(s) matching xpath text
"""
if single:
return self._do_attempts(self.elem_by_xpath,
xpath,
attempts=attempts,
rest_s=rest_s)
else:
return self._do_attempts(self.elems_by_xpath,
xpath,
attempts=attempts,
rest_s=rest_s)
def get_text(self, xpath: str, single: bool = True, attempts: int = STD_ATTEMPTS, rest_s: float = REST_S) ->\
Union[str, List[str]]:
"""
Returns text in element(s)
Args:
xpath: str, xpath to element
single: boolean, Whether to extract from single element or multiple. default = True
attempts: int, number of attempts to make before failing
rest_s: int, number of seconds to rest between attempts
Returns: Text from inside element(s)
"""
if single:
elem = self._do_attempts(self.elem_by_xpath,
xpath,
attempts=attempts,
rest_s=rest_s)
if elem is not None:
return elem.text
else:
elems = self._do_attempts(self.elems_by_xpath,
xpath,
attempts=attempts,
rest_s=rest_s)
text_list = []
for e in elems:
if e is not None:
text_list.append(e.text)
return text_list
def remove(self,
xpath: str,
single: bool = True,
attempts: int = STD_ATTEMPTS,
rest_s: float = REST_S):
"""
Uses JavaScript commands to remove desired element
Args:
xpath: str, xpath to element
single: boolean whether to apply to single element or multiple. default = True
attempts: int, number of attempts to make before failing
rest_s: int, number of seconds to rest between attempts
"""
script = """
var element = arguments[0];
element.remove();
"""
if single:
elem = self._do_attempts(self.elem_by_xpath,
xpath,
attempts=attempts,
rest_s=rest_s)
if elem is not None:
self.driver.execute_script(script, elem)
else:
elems = self._do_attempts(self.elems_by_xpath,
xpath,
attempts=attempts,
rest_s=rest_s)
for e in elems:
if e is not None:
self.driver.execute_script(script, e)
def add_style_to_elem(self, elem: WebElement, css_str: str):
"""Injects CSS into elem style"""
js = f"arguments[0].setAttribute('style', '{css_str}');"
self.driver.execute_script(js, elem)
def click_by_id(self, elem_id: float):
js = f'document.getElementById("{elem_id}").click();'
self.driver.execute_script(js)
def rand_wait(self, sleep_range_secs: List[int]):
"""
Determines sleep time as random number between upper and lower limit,
then sleeps for that given time. After sleep, moves randomly vertically and horizontally on page
for up to four times
Args:
sleep_range_secs: list, min and max number of seconds to sleep
"""
sleep_range_secs = sorted(sleep_range_secs)
if len(sleep_range_secs) == 2:
sleep_secs_lower, sleep_secs_higher = sleep_range_secs
elif len(sleep_range_secs) > 2:
sleep_secs_lower, sleep_secs_higher = sleep_range_secs[
0], sleep_range_secs[-1]
else:
raise ValueError(
'Input for sleep range must be at least two items')
sleeptime = randint(sleep_secs_lower, sleep_secs_higher)
self.log.debug(f'Waiting {sleeptime}s')
time.sleep(sleeptime)
# after wait period, scroll through window randomly
for i in range(4):
r_x = randint(-20, 20)
r_y = randint(150, 300)
self.scroll_absolute(direction=f'{r_x},{r_y}')
def fast_wait(self):
self.rand_wait(self._fast_wait)
def medium_wait(self):
self.rand_wait(self._medium_wait)
def slow_wait(self):
self.rand_wait(self._slow_wait)
def scroll_to_element(self,
elem: WebElement,
use_selenium_method: bool = True):
"""
Scrolls to get element in view
Args:
elem: Selenium-class element
use_selenium_method: bool, if True, uses built-in Selenium method of scrolling an element to view
otherwise, uses Javascript (scrollIntoView)
"""
if use_selenium_method:
actions = ActionChains(self.driver)
actions.move_to_element(elem).perform()
else:
scroll_center_script = """
var viewPortHeight = Math.max(document.documentElement.clientHeight, window.innerHeight || 0);
var elementTop = arguments[0].getBoundingClientRect().top;
window.scrollBy(0, elementTop-(viewPortHeight/2));
"""
self.driver.execute_script(scroll_center_script, elem)
def scroll_absolute(self, direction: str = 'up'):
"""Scrolls all the way up/down or to specific x,y coordinates"""
if direction == 'up':
coords = '0, 0'
elif direction == 'down':
coords = '0, document.body.scrollHeight'
else:
if ',' in direction:
coords = direction
else:
raise ValueError(
'Invalid parameters entered. Must be an x,y coordinate, or up/down command.'
)
self.driver.execute_script(f'window.scrollTo({coords});')
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from flask import Flask
from kavalkilu import LogWithInflux
from servertools import get_secret_file
from api.hosts.hosts import hosts
from api.hosts.keys import keys
app = Flask(__name__)
for blueprint in [keys, hosts]:
app.register_blueprint(blueprint)
@app.route('/')
def main_page():
return 'Main page!'
@app.errorhandler(500)
def handle_errors(error):
return 'Not found!'
if __name__ == '__main__':
secret_key = get_secret_file('FLASK_SECRET')
app.secret_key = secret_key
app.run(host='0.0.0.0', port='5002')
# Instantiate log here, as the hosts API is requested to comunicate with influx
log = LogWithInflux('hosts_api')
import time
from kavalkilu import LogWithInflux, InfluxDBLocal, InfluxDBHomeAuto
from servertools import HueBulb
INTERVAL_MINS = 30
WAIT_S = 290
end_time = time.time() + INTERVAL_MINS * 60
logg = LogWithInflux('mushroom-grow-toggle')
influx = InfluxDBLocal(InfluxDBHomeAuto.TEMPS)
h = HueBulb('mushroom-plug')
# TODO: Use HASS instead of Influx to get current values
rounds = 0
while end_time > time.time():
if rounds % 2 == 0:
# Turn on during even rounds
h.turn_on()
else:
# Turn off for off rounds
h.turn_off()
rounds += 1
logg.debug(f'Waiting {WAIT_S / 60:.0f} mins...')
time.sleep(WAIT_S)
logg.close()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Detects whether the garage door is up or down"""
from kavalkilu import LogWithInflux, HAHelper
from pitools import DistanceSensor
from pitools.peripherals import PiGarage
logg = LogWithInflux('garage_door', log_dir='gdoor')
TRIGGER_PIN = PiGarage.ultrasonic.trigger
ECHO_PIN = PiGarage.ultrasonic.echo
logg.debug('Initializing sensor...')
ds = DistanceSensor(TRIGGER_PIN, ECHO_PIN)
# Take an average of 10 readings
readings = []
logg.debug('Taking readings...')
for i in range(10):
readings.append(ds.measure())
avg = sum(readings) / len(readings)
# Instantiate HASS
ha = HAHelper()
# Collect last reading
last_status = ha.get_state(PiGarage.ha_garage_door_sensor).get('state')
# Typically, reading is ca. 259cm when door is closed. ca. 50cm when open
if avg < 6000:
status = 'open'
# TODO: Depth when car is in
def setUpClass(cls) -> None:
cls.log = LogWithInflux('invest-test')
cls.ed = EdgarCollector(start_year=2017, parent_log=cls.log)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from datetime import datetime
import pandas as pd
from kavalkilu import LogWithInflux
from servertools import SlackWeatherNotification, NWSForecast, NWSForecastZone
# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('frost_warn', log_dir='weather')
now = datetime.now()
weather = NWSForecast(NWSForecastZone.ATX)
hours_df = weather.get_hourly_forecast()
hours_df['date'] = pd.to_datetime(hours_df['date'])
# Filter by column & get only the next 12 hours of forecasted weather
cols = ['date', 'temp-avg', 'feels-temp-avg', 'dewpoint', 'wind-speed']
hours_df = hours_df.loc[hours_df.date < (now + pd.Timedelta(hours=12)), cols]
logic_dict = {
'freeze': (hours_df['temp-avg'] < 0) & ((hours_df['dewpoint'] < -8) | (hours_df['wind-speed'] > 5)),
'frost': (hours_df['temp-avg'] < 2) & ((hours_df['dewpoint'] < -6) | (hours_df['wind-speed'] >= 5)),
'light frost': (hours_df['temp-avg'] < 2) & ((hours_df['dewpoint'] < -6) | (hours_df['wind-speed'] < 5)),
}
warning = None
for name, cond in logic_dict.items():
if any(cond.tolist()):
# We want the warnings to move from severe to relatively mild &
# break on the first one that matches the condition
warning = name
ETL for RTL_433 json objects via syslog -> processed Dataframe -> influx
Note: depends on `rf_stream` already being running and feeding data to port 1433
via `rtl_433 -F syslog::1433`
"""
import json
from json import JSONDecodeError
import socket
from datetime import datetime
import pandas as pd
from kavalkilu import InfluxDBLocal, InfluxDBHomeAuto, LogWithInflux, \
GracefulKiller, Hosts, HOME_SERVER_HOSTNAME, HAHelper
from servertools import SlackComm
logg = LogWithInflux('rf_temp')
sc = SlackComm(parent_log=logg)
UDP_IP = Hosts().get_ip_from_host(HOME_SERVER_HOSTNAME)
UDP_PORT = 1433
# device id to device-specific data mapping
mappings = {
210: {'name': 'neighbor-porch'},
3092: {'name': 'ylemine-r6du'},
5252: {'name': 'elutuba'},
6853: {'name': 'kontor-wc'},
8416: {'name': 'alumine-r6du'},
9459: {'name': 'freezer'},
9533: {'name': 'kontor'},
10246: {'name': 'v2lisuks'},
12476: {'name': 'suur-wc'},
import re
import pandas as pd
from servertools import BrowserAction
from kavalkilu import LogWithInflux, InfluxDBLocal, InfluxDBTracker
logg = LogWithInflux('apt-prices', log_to_file=True)
influx = InfluxDBLocal(InfluxDBTracker.APT_PRICES)
ba = BrowserAction(headless=True, parent_log=logg)
url = 'https://www.maac.com/available-apartments/?propertyId=611831&Bedroom=2%20Bed'
ba.get(url)
ba.medium_wait()
listings = ba.get_elem(
'//div[contains(@class, "apartment-listing")]/div[contains(@class, "apartment")]',
single=False)
logg.debug(f'Returned {len(listings)} initial listings...')
apt_list = []
for apt in listings:
apt_dict = {}
# Get unit
unit = apt.find_element_by_xpath(
'.//div[contains(@class, "apartment__unit-number")]').text
# Clean unit of non-number chars
unit = re.search(r'\d+', unit).group()
# Get sqft, beds, baths
desc = apt.find_element_by_xpath(
'.//div[contains(@class, "apartment__unit-description")]').text.split(
'\n')
for d in desc:
for item in ['bed', 'bath', 'sq. ft.']:
"""Message links to memes for Viktor to post periodically"""
import os
import re
import time
import numpy as np
from typing import List
from datetime import datetime, timedelta
from slacktools import SlackTools
from kavalkilu import Keys, LogWithInflux
logg = LogWithInflux('memeraker')
vcreds = Keys().get_key('viktor')
st = SlackTools(**vcreds)
user_me = 'UM35HE6R5'
# Wine review text & previous stop timestamp
ddir = os.path.join(os.path.expanduser('~'), 'data')
fpath = os.path.join(ddir, 'mkov_wines.txt')
ts_path = os.path.join(ddir, 'last_memeraker_ts')
def post_memes(reviews: List[str], memes: List[str], wait_min: int = 5, wait_max: int = 60):
# Begin the upload process, include a review
for meme in memes:
review = np.random.choice(reviews, 1)[0]
st.upload_file('memes-n-shitposts', meme, 'image', is_url=True, txt=review)
# Wait some seconds before posting again
wait_s = np.random.choice(range(wait_min, wait_max), 1)[0]
logg.debug(f'Waiting {wait_s}s.')
time.sleep(wait_s)
"""Checks slackmojis daily for new additions"""
import json
from kavalkilu import Path, LogWithInflux
from servertools import SlackComm, XPathExtractor
logg = LogWithInflux('emoji-scraper')
scom = SlackComm(bot='viktor', parent_log=logg)
p = Path()
fpath = p.easy_joiner(p.data_dir, 'slackmojis.json')
chan = 'emoji_suggestions'
url = 'https://slackmojis.com/emojis/recent'
xpath_extractor = XPathExtractor(url)
emoji_list = xpath_extractor.xpath('//ul[@class="emojis"]', single=True)
emojis = emoji_list.getchildren()
# Read in the previous emoji id list
if not p.exists(fpath):
prev_emojis = {}
else:
with open(fpath) as f:
prev_emojis = json.loads(f.read())
new_emojis = {}
for emoji in emojis:
emo_id = emoji.getchildren()[0].get('data-emoji-id-name')
emo_name = emoji.getchildren()[0].getchildren()[1].text.strip()
if emo_id not in prev_emojis.keys():
# Get link and add to the id list
emo_link = emoji.findall('.//img')[0].get('src')
import tempfile
from datetime import datetime as dt, timedelta
from kavalkilu import Hosts, LogWithInflux
from easylogger import ArgParse
from servertools import SlackComm, Amcrest, VidTools
logg = LogWithInflux('motion_alerts')
sc = SlackComm(parent_log=logg)
args = [{
'names': ['-c', '--camera'],
'other': {
'action': 'store',
'default': 'ac-allr6du'
}
}, {
'names': ['-i', '--interval'],
'other': {
'action': 'store',
'default': '60'
}
}]
ap = ArgParse(args, parse_all=False)
CAMERA = ap.arg_dict.get('camera')
INTERVAL_MINS = int(ap.arg_dict.get('interval'))
start_dt = (dt.now() - timedelta(minutes=INTERVAL_MINS)).replace(second=0,
microsecond=0)
end_dt = (start_dt + timedelta(minutes=INTERVAL_MINS))
cam_ip = Hosts().get_ip_from_host(CAMERA)
cam = Amcrest(cam_ip)
class EdgarCollector:
"""Handles the entire process of collecting SEC filing data and processing that into financial ratios"""
base_url = 'https://www.sec.gov'
def __init__(self, start_year: float = 2017, parent_log: LogWithInflux = None):
self.logg = LogWithInflux(parent_log, child_name=self.__class__.__name__)
# Set temp dir for downloading the edgar filings
self.tmp_dir = os.path.join(tempfile.gettempdir(), 'edgar')
# Get ticker to CIK mapping
self.logg.debug('Downloading ticket to CIK mapping...')
self.t2cik_df = self.get_ticker_to_cik_map()
# Download EDGAR indexes and retrieve the filepaths associated with them
self.logg.debug('Downloading indexes (this may take ~2 mins)...')
self.edgar_fpaths = self._download_indexes(start_year)
@staticmethod
def get_ticker_to_cik_map() -> pd.DataFrame:
"""Collects the mapping of stock ticker to CIK"""
t2cik_df = pd.read_csv('https://www.sec.gov/include/ticker.txt', sep='\t', header=None)
t2cik_df.columns = ['tick', 'cik']
return t2cik_df
def _download_indexes(self, since_year: float) -> List[str]:
"""Downloads company indexes from Edgar into temporary directory"""
# Begin download
edgar.download_index(self.tmp_dir, since_year=since_year)
# Retrieve the file paths downloaded
fpaths = []
for a, b, fnames in os.walk(self.tmp_dir):
fpaths = [os.path.join(self.tmp_dir, f) for f in fnames]
break
self.logg.debug(f'Collected {len(fpaths)} files...')
return sorted(fpaths)
def get_data_for_stock_tickers(self, tickers: Union[str, List[str]]) -> pd.DataFrame:
"""Collects the quarterly financial ratios for a given stock ticker"""
if isinstance(tickers, str):
tickers = [tickers]
stocks_df = pd.DataFrame({'tick': list(map(str.lower, tickers))})
# Merge stocks on cik
stocks_df = stocks_df.merge(self.t2cik_df, how='left', on='tick')
ratio_df = pd.DataFrame()
# Capture filing locations for each quarter for each stock
for i, fpath in enumerate(sorted(self.edgar_fpaths)):
self.logg.debug(f'Working on file {i + 1} of {len(self.edgar_fpaths)}')
# Parse the quarter
year, qtr = os.path.split(fpath)[1].split('.')[0].split('-')
self.logg.debug(f'Parsed: {qtr}-{year}')
df = pd.read_csv(fpath, sep='|', header=None)
df.columns = ['cik', 'company_name', 'form', 'filing_date', 'txt_file', 'html_file']
# Filter by CIK and on either 10-K (annual) or 10-Q (quarterly) reports
df = df.loc[df.cik.isin(stocks_df.cik) & df.form.isin(['10-Q', '10-K'])]
self.logg.debug(f'Matched {df.shape[0]} rows of data')
for idx, row in df.iterrows():
# Grab the HTML file - this is the file list
url = f'{self.base_url}/Archives/{row["html_file"]}'
xpe = XPathExtractor(url)
# Grab the link to the 10-* file
form_type = row['form']
form_row = xpe.xpath(f'//table[@class="tableFile"]/tr[td[text()="{form_type}"]]', single=True)
form_url = xpe.xpath('./td/a', obj=form_row, single=True).get('href')
# Grab the 10-* file data
complete_url = f'{self.base_url}{form_url}'
# Work on SOps (this sets a point to look 'after')
stmt_ops = EdgarFinStatement(complete_url, 'STATEMENTS? OF OPERATIONS')
net_sales = stmt_ops.get_line_item('net sales')
cogs = stmt_ops.get_line_item('cost of sales')
op_income = stmt_ops.get_line_item('operating income')
net_income = stmt_ops.get_line_item('net income')
eps = stmt_ops.get_line_item('earnings per share', line_after=True)
shares = stmt_ops.get_line_item('shares used in computing earnings', line_after=True)
# Work on Balance Sheet
stmt_bs = EdgarFinStatement(complete_url, 'CONSOLIDATED BALANCE SHEETS')
current_assets = stmt_bs.get_line_item('total current assets')
intangible_assets = stmt_bs.get_line_item('intangible assets')
total_assets = stmt_bs.get_line_item('total assets')
current_liabilities = stmt_bs.get_line_item('total current liabilities')
total_liabilities = stmt_bs.get_line_item('total liabilities')
common_stock_eq = stmt_bs.get_line_item('common stock')
total_shareholders_equity = stmt_bs.get_line_item('total shareholders')
net_tangible_assets = total_assets - intangible_assets - total_liabilities
# Work on Cash Flow (SCF)
stmt_cf = EdgarFinStatement(complete_url, 'STATEMENTS OF CASH FLOWS')
dep_and_amort = stmt_cf.get_line_item('depreciation and amortization')
ebitda = op_income + dep_and_amort
# Ratios
asset_turnover = net_sales / net_tangible_assets
profit_margin = net_income / net_sales
debt_to_ebitda = total_liabilities / ebitda
debt_to_equity = total_liabilities / total_shareholders_equity
roa = net_income / total_assets
roe = net_income / total_shareholders_equity
bvps = total_shareholders_equity / shares
# Apply ratios to dataframe
ratio_df = ratio_df.append({
'year': year,
'quarter': qtr,
'cik': row['cik'],
'source': row['form'],
'ato': asset_turnover,
'pm': profit_margin,
'd2ebitda': debt_to_ebitda,
'd2e': debt_to_equity,
'roa': roa,
'roe': roe,
'bvps': bvps
}, ignore_index=True)
# Merge the financial info back in with the stocks
stocks_df = stocks_df.merge(ratio_df, how='left', on='cik')
return stocks_df
import subprocess
from kavalkilu import LogWithInflux, HOME_SERVER_HOSTNAME, Hosts
logg = LogWithInflux('rf_stream', log_dir='rf')
serv_ip = Hosts().get_ip_from_host(HOME_SERVER_HOSTNAME)
cmd = ['/usr/local/bin/rtl_433', '-F', f'syslog:{serv_ip}:1433']
logg.info(f'Sending command: {" ".join(cmd)}')
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
process_output, _ = process.communicate()
logg.debug(f'Process output: {process_output}')
import os
from datetime import datetime as dt, timedelta
from kavalkilu import LogWithInflux, DateTools, InfluxDBLocal, InfluxDBPiHole, SQLLiteLocal, Hosts
from servertools import SlackComm
logg = LogWithInflux('pihole_etl', log_to_db=True)
sc = SlackComm(parent_log=logg)
hosts = {x['ip']: x['name'] for x in Hosts().get_all_hosts()}
datetools = DateTools()
FTL_DB_PATH = os.path.join('/etc', *['pihole', 'pihole-FTL.db'])
sqll = SQLLiteLocal(FTL_DB_PATH)
INTERVAL_MINS = 60
end = dt.now().astimezone().replace(second=0, microsecond=0)
start = (end - timedelta(minutes=INTERVAL_MINS))
unix_start = datetools.dt_to_unix(start, from_tz='US/Central')
unix_end = datetools.dt_to_unix(end, from_tz='US/Central')
query = f"""
SELECT
client
, domain
, CASE
WHEN status = 0 THEN 'UNKNOWN'
WHEN status = 1 OR status > 3 THEN 'BLOCKED'
WHEN status = 2 OR status = 3 THEN 'ALLOWED'
ELSE 'UNKNOWN'
END AS query_status
, CASE
WHEN status = 0 THEN 'NO ANSWER'
"""For joining timelapse shots"""
import os
from moviepy.editor import ImageClip, concatenate_videoclips
from kavalkilu import Path, LogWithInflux
from servertools import SlackComm
log = LogWithInflux('timelapse_combi')
p = Path()
tl_dir = p.easy_joiner(p.data_dir, 'timelapse')
fnames = {}
for dirpath, _, filenames in os.walk(tl_dir):
dirname = os.path.basename(dirpath)
if dirname != 'timelapse':
fnames[os.path.basename(dirpath)] = filenames
files = []
# Begin combining shots
for k, v in fnames.items():
if not any([k.startswith(x) for x in ['ac-', 're-']]):
continue
log.debug(f'Working on {k}. {len(v)} files.')
full_paths = sorted([os.path.join(tl_dir, *[k, x]) for x in v])
clips = []
for fpath in full_paths:
try:
clips.append(ImageClip(fpath).set_duration(1))
except ValueError:
log.debug(f'Error with this path: {fpath}')
continue
clip = concatenate_videoclips(clips)
clip = clip.set_fps(30).speedx(30)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Activates nighttime mode on cameras"""
from servertools import Amcrest
from kavalkilu import LogWithInflux, Hosts
# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('cam_night')
hosts = Hosts()
# Get only cameras without numbers in the name
cam_info_list = hosts.get_hosts_and_ips(r'^ac-(ga|el)')
for cam_dict in cam_info_list:
# Instantiate cam & arm
cam = Amcrest(cam_dict['ip'])
if cam.camera_type != 'doorbell':
cam.arm_camera(True)
# publish.single(f'sensors/cameras/{cam.name}/status', 'ARMED', hostname='tinyserv.local')
log.close()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Log memory, cpu use and temp of each machine"""
from kavalkilu import LogWithInflux, InfluxDBHomeAuto
from pitools import Sensor
logg = LogWithInflux('machine_data')
# Set the pin (BCM)
for sensor_name in ['CPU', 'MEM', 'CPUTEMP', 'DISK']:
logg.debug(f'Logging {sensor_name}...')
sensor = Sensor(sensor_name)
# Take readings & log to db
sensor.measure_and_log_to_db(
tbl=InfluxDBHomeAuto().__getattribute__(sensor_name), n_times=2)
logg.debug('Temp logging successfully completed.')
logg.close()
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Determines if mobile is connected to local network. If not, will arm the cameras"""
from servertools import Amcrest, OpenWRT
from kavalkilu import LogWithInflux, Hosts
# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('cam_active')
ow = OpenWRT()
hosts = Hosts()
# Get only cameras without numbers in the name
cam_info_list = hosts.get_hosts_and_ips(r'(?!^ac-.*(\d.*|doorbell)$)^ac-.+$')
res_list = []
currently_active_ips = ow.get_active_connections()
# Check if mobile(s) are connected to LAN
for ip in [i['ip'] for i in hosts.get_hosts_and_ips('an-[bm]a.*')]:
res_list.append(ip in currently_active_ips.keys())
# If anyone home, don't arm, otherwise arm
arm_cameras = not any(res_list)
arm_status = 'ARMED' if arm_cameras else 'UNARMED'
if not arm_cameras:
log.debug(
'One of the devices are currently in the network. Disabling motion detection.'
)
else:
log.debug(
'None of the devices are currently in the network. Enabling motion detection.'
)
for cam_dict in cam_info_list:
"""Sends latest temperature readings to HASS"""
from kavalkilu import LogWithInflux, InfluxDBHomeAuto, InfluxDBLocal, HAHelper
log = LogWithInflux('ha-temps', log_dir='weather')
influx = InfluxDBLocal(InfluxDBHomeAuto.TEMPS)
query = '''
SELECT
last("temp") AS temp,
last("humidity") AS humidity
FROM "temps"
WHERE
location =~ /mushroom|r6du|elutuba|wc|v2lis|freezer|fridge|kontor/
AND time > now() - 30m
GROUP BY
"location"
fill(null)
ORDER BY ASC
'''
df = influx.read_query(query, time_col='time')
log.debug(f'Collected {df.shape[0]} rows of data')
log.debug('Beginning to send updates to HASS')
ha = HAHelper()
for i, row in df.iterrows():
loc_name = row['location'].replace('-', '_')
for sensor_type in ['temp', 'humidity']:
dev_name = f'sensor.{loc_name}_{sensor_type}'
log.debug(f'Updating {dev_name}...')
ha.set_state(dev_name, data={'state': row[sensor_type]}, data_class=sensor_type)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from datetime import datetime, timedelta
import pandas as pd
from kavalkilu import LogWithInflux
from servertools import NWSForecast, NWSForecastZone, SlackWeatherNotification
# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('significant_change', log_dir='weather')
swno = SlackWeatherNotification(parent_log=log)
now = datetime.now()
tomorrow = (now + timedelta(days=1))
weather = NWSForecast(NWSForecastZone.ATX)
hours_df = weather.get_hourly_forecast()
hours_df['date'] = pd.to_datetime(hours_df['date'])
# focus on just the following day's measurements
nextday = hours_df[hours_df['date'].dt.day == tomorrow.day].copy()
nextday['day'] = nextday['date'].dt.day
nextday['hour'] = nextday['date'].dt.hour
temp_dict = {}
metrics_to_collect = dict(
zip(
['temp-avg', 'feels-temp-avg', 'precip-intensity', 'precip-prob'],
['temp', 'apptemp', 'precip_int', 'precip_prob'],
))
# Determine which hours are important to examine (e.g., for commuting / outside work)
important_hours = [0, 6, 12, 15, 18]
for hour in important_hours:
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import time
import requests
from grafana_api.grafana_face import GrafanaFace
from kavalkilu import Keys, LogWithInflux
from servertools import SlackComm
log = LogWithInflux('grafana_snapper')
creds = Keys().get_key('grafana')
scom = SlackComm()
def get_pic_and_upload(url, name):
"""Captures dashboard panel at URL and uploads to #notifications slack channel"""
resp = requests.get(url, headers={'Authorization': f'Bearer {creds["token"]}'})
temp_file = os.path.abspath('/tmp/dash_snap.png')
with open(temp_file, 'wb') as f:
for chunk in resp:
f.write(chunk)
scom.st.upload_file(scom.notify_channel, temp_file, name)
# The URL template to use
base_url = 'http://{host}/render/{name}?orgId=1&from={from}&to={to}&panelId={pid}&width=1000&height=500'
# time range to snap
