示例#1
0
 def __init__(self, start_year: float = 2017, parent_log: LogWithInflux = None):
     self.logg = LogWithInflux(parent_log, child_name=self.__class__.__name__)
     # Set temp dir for downloading the edgar filings
     self.tmp_dir = os.path.join(tempfile.gettempdir(), 'edgar')
     # Get ticker to CIK mapping
     self.logg.debug('Downloading ticket to CIK mapping...')
     self.t2cik_df = self.get_ticker_to_cik_map()
     # Download EDGAR indexes and retrieve the filepaths associated with them
     self.logg.debug('Downloading indexes (this may take ~2 mins)...')
     self.edgar_fpaths = self._download_indexes(start_year)
示例#2
0
 def __init__(self,
              driver_path: str = '/usr/bin/chromedriver',
              timeout: float = 60,
              options: List[str] = None,
              headless: bool = True,
              parent_log: 'Log' = None):
     self.driver = ChromeDriver(driver_path, timeout, options, headless)
     self.pid = self.driver.service.process.pid
     self.port = self.driver.service.port
     self.log = LogWithInflux(parent_log,
                              child_name=self.__class__.__name__)
     self.elem_by_xpath = self.driver.find_element_by_xpath
     self.elems_by_xpath = self.driver.find_elements_by_xpath
     self.log.debug(
         f'Chromedriver started up with pid: {self.pid} receiving on port: {self.port}'
     )
 def __init__(self, bot: str = 'sasha', parent_log: LogWithInflux = None):
     creds = read_props()
     credstore = SecretStore('secretprops.kdbx',
                             creds['secretprops_secret'])
     self.log = LogWithInflux(parent_log,
                              child_name=self.__class__.__name__)
     self.st = SlackTools(credstore=credstore,
                          parent_log=self.log,
                          slack_cred_name=bot)
     self.bkb = BlockKitBuilder()
     if bot == 'sasha':
         self.hoiatuste_kanal = 'hoiatused'
         self.ilma_kanal = 'ilm'
         self.kodu_kanal = 'kodu'
         self.kaamerate_kanal = 'kaamerad'
         self.meemide_kanal = 'meemid'
         self.test_kanal = 'test'
         self.koduv6rgu_kanal = 'koduvõrk'
         self.user_me = 'U015WMFQ0DV'
         self.user_marelle = 'U016N5RJZ9C'
     elif bot == 'viktor':
         self.user_me = 'UM35HE6R5'
         self.user_marelle = 'UM3E3G72S'
示例#4
0
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import time
from datetime import datetime as dtt
from kavalkilu import LogWithInflux, Keys, NetTools, Hosts, HOME_SERVER_HOSTNAME
from servertools import BrowserAction, SlackComm

ip = NetTools().get_ip()
debug = Hosts().get_ip_from_host(HOME_SERVER_HOSTNAME) != ip
logg = LogWithInflux('vpulse_auto')

# TODO
# build out a table of when monthly, weekly things were last done.
# Handle weekly tasks either based on DOW or recorded table with date_last_done and freq columns

# Get credentials
creds = Keys().get_key('vpulse-creds')


def message_channel_and_log(msg):
    slack_comm.send_message(notify_channel, msg)
    if debug:
        logg.debug(msg)


def get_vpoints():
    """Collects amount of points currently available"""
    points_script = "return document.getElementById('progress-bar-menu-points-total-value')"
    points = ba.driver.execute_script(points_script).get_attribute(
        'textContent').strip()
示例#5
0
 def setUpClass(cls) -> None:
     cls.logg = LogWithInflux('cam-test', log_to_file=False)
     cam_ip = Hosts().get_ip_from_host('ac-garaaz')
     cls.cam = Amcrest(cam_ip, parent_log=cls.logg)
示例#6
0
"""Collect forecast data"""
from datetime import datetime, timedelta
from kavalkilu import LogWithInflux, InfluxDBLocal, InfluxDBHomeAuto
from servertools import OpenWeather, OWMLocation, NWSForecast, NWSForecastZone, \
    YrNoWeather, YRNOLocation

# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('forecast', log_dir='weather')
influx = InfluxDBLocal(InfluxDBHomeAuto.WEATHER)
# Number of hours we're looking forward
period_h = 24
# Start & end of the lookahead
p_start = (datetime.now() + timedelta(hours=1))
p_end = (p_start + timedelta(hours=period_h))

owm_fc = OpenWeather(OWMLocation.ATX).hourly_forecast()
nws_fc = NWSForecast(NWSForecastZone.ATX).get_hourly_forecast()
yrno_fc = YrNoWeather(YRNOLocation.ATX).hourly_summary()

# Push all weather data into influx
for svc, df in zip(['own', 'nws', 'yrno'], [owm_fc, nws_fc, yrno_fc]):
    log.debug(f'Collecting data from {svc.upper()}...')
    cols = ['date', 'humidity', 'temp-avg', 'feels-temp-avg']
    if svc == 'nws':
        # Replace relative-humidity with humidity
        df['humidity'] = df['relative-humidity']
    elif svc == 'yrno':
        # No humidity, feelslike included in this one
        _ = [cols.pop(cols.index(x)) for x in ['feels-temp-avg', 'humidity']]
        df['date'] = df['from']
    df = df[cols]
import re
from typing import List
from slacktools import BlockKitBuilder as bkb
from servertools import XPathExtractor, SlackComm
from kavalkilu import LogWithInflux

logg = LogWithInflux('wotd')

wotd_url = 'https://www.dictionary.com/e/word-of-the-day/'
sotd_url = 'https://www.thesaurus.com/e/synonym-of-the-day/'


def extract_otd(url: str, is_wotd: bool = False) -> List[dict]:
    """Extract Synonym/Word of the day"""
    xtool = XPathExtractor(url)
    # Get the most recent WOTD
    class_prefix = 'wotd' if is_wotd else 'sotd'
    title_section = 'word' if is_wotd else 'synonym'
    otd = xtool.xpath(f'//div[contains(@class, "{class_prefix}-items")]/div',
                      single=True)

    sotd_xpath = f'.//div[contains(@class, "{class_prefix}-item__title")]'
    wotd_xpath = f'.//div[contains(@class, "otd-item-headword__word")]'
    word = xtool.xpath(wotd_xpath if is_wotd else sotd_xpath,
                       obj=otd,
                       get_text=True).strip()
    # Pronunciation
    pronunc = xtool.xpath(
        './/div[contains(@class, "otd-item-headword__pronunciation")]',
        obj=otd,
        single=True,
示例#8
0
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Read temperature and humidity from living room"""
from kavalkilu import LogWithInflux
from pitools import Sensor
from pitools.peripherals import PiElutuba


logg = LogWithInflux('elutuba_temp', log_dir='weather')

# Set the pin (BCM)
sensor = Sensor('DHT22', data_pin=PiElutuba.dht.pin)
# Take readings & log to db
sensor.log_to_db()

logg.debug('Temp logging successfully completed.')

logg.close()
示例#9
0
"""Collect current weather data"""
from kavalkilu import LogWithInflux, InfluxDBLocal, InfluxDBHomeAuto
from servertools import OpenWeather, OWMLocation

# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('local', log_dir='weather')
influx = InfluxDBLocal(InfluxDBHomeAuto.WEATHER)

current = OpenWeather(OWMLocation.ATX).current_weather()

# Push all weather data into influx
current = current.drop('date', axis=1)
current['loc'] = 'austin'
influx.write_df_to_table(current, 'loc', current.columns.tolist()[:-1])
influx.close()

log.debug('Temp logging successfully completed.')

log.close()
示例#10
0
class BrowserAction:
    """
    Performs action to Selenium-class webdriver
    Args for __init__:
        driver: Selenium-type driver class
    """
    # Predefined wait ranges, in seconds
    _slow_wait = [15, 30]
    _medium_wait = [5, 15]
    _fast_wait = [1, 8]
    REST_S = 2  # Standard seconds to rest between attempts
    STD_ATTEMPTS = 3  # Standard attempts to make before failing

    def __init__(self,
                 driver_path: str = '/usr/bin/chromedriver',
                 timeout: float = 60,
                 options: List[str] = None,
                 headless: bool = True,
                 parent_log: 'Log' = None):
        self.driver = ChromeDriver(driver_path, timeout, options, headless)
        self.pid = self.driver.service.process.pid
        self.port = self.driver.service.port
        self.log = LogWithInflux(parent_log,
                                 child_name=self.__class__.__name__)
        self.elem_by_xpath = self.driver.find_element_by_xpath
        self.elems_by_xpath = self.driver.find_elements_by_xpath
        self.log.debug(
            f'Chromedriver started up with pid: {self.pid} receiving on port: {self.port}'
        )

    def _do_attempts(self,
                     func: Callable,
                     *args,
                     sub_method: str = None,
                     sub_method_input: str = None,
                     attempts: int = 3,
                     rest_s: float = 2) -> Optional[Any]:
        """Attempts a certain method for n times before gracefully failing"""

        for i in range(0, attempts):
            if i > 0:
                # Sleep between attempts, but let a one-off attempt through without much delay
                time.sleep(rest_s)
            try:
                thing = func(*args)
                if sub_method is not None:
                    if sub_method_input is None:
                        getattr(thing, sub_method)()
                        break
                    else:
                        getattr(thing, sub_method)(sub_method_input)
                        break
                else:
                    return thing
            except Exception as e:
                self.log.error(f'Attempt {i + 1}: fail -- {e}')

    def tear_down(self):
        """Make sure the browser is closed on cleanup"""
        self.log.info('Shutting down browser.')
        self.driver.quit()

    def get(self, url: str):
        """
        Navigates browser to url
        Args:
            url: str, url to navigate to
        """
        self.log.debug(f'Loading url: {url}')
        self.driver.get(url)

    def click(self,
              xpath: str,
              attempts: int = STD_ATTEMPTS,
              rest_s: float = REST_S):
        """
        Clicks HTML element
        Args:
            xpath: str, xpath to element to click
            attempts: int, number of attempts to make before failing
            rest_s: int, number of seconds to rest between attempts
        """
        self._do_attempts(self.elem_by_xpath,
                          xpath,
                          sub_method='click',
                          attempts=attempts,
                          rest_s=rest_s)

    def clear(self,
              xpath: str,
              attempts: int = STD_ATTEMPTS,
              rest_s: float = REST_S):
        """
        Clears form element of text
        Args:
            xpath: str, xpath to form element
            attempts: int, number of attempts to make before failing
            rest_s: int, number of seconds to rest between attempts
        """
        self._do_attempts(self.elem_by_xpath,
                          xpath,
                          sub_method='clear',
                          attempts=attempts,
                          rest_s=rest_s)

    def enter(self,
              xpath: str,
              entry_text: str,
              attempts: int = STD_ATTEMPTS,
              rest_s: float = REST_S):
        """
        Enters text into form element
        Args:
            xpath: str, xpath to form
            entry_text: str, text to enter into form
            attempts: int, number of attempts to make before failing
            rest_s: int, number of seconds to rest between attempts
        """
        self._do_attempts(self.elem_by_xpath,
                          xpath,
                          sub_method='send_keys',
                          sub_method_input=entry_text,
                          attempts=attempts,
                          rest_s=rest_s)

    def elem_exists(self,
                    xpath: str,
                    attempts: int = 1,
                    rest_s: float = REST_S) -> bool:
        """
        Determines if particular element exists
        Args:
            xpath: str, xpath to HTML element
            attempts: int, number of attempts to make before failing
            rest_s: int, number of seconds to rest between attempts
        Returns: True if exists
        """
        elem = self._do_attempts(self.elem_by_xpath,
                                 xpath,
                                 attempts=attempts,
                                 rest_s=rest_s)
        if elem is not None:
            return True
        return False

    def get_elem(self, xpath: str, single: bool = True, attempts: int = STD_ATTEMPTS, rest_s: float = REST_S) ->\
            Union[WebElement, List[WebElement]]:
        """
        Returns HTML elements as selenium objects
        Args:
            xpath: str, xpath of element to return
            single: boolean, True if returning only one element. default: True
            attempts: int, number of attempts to make before failing
            rest_s: int, number of seconds to rest between attempts
        Returns: HTML element(s) matching xpath text
        """
        if single:
            return self._do_attempts(self.elem_by_xpath,
                                     xpath,
                                     attempts=attempts,
                                     rest_s=rest_s)
        else:
            return self._do_attempts(self.elems_by_xpath,
                                     xpath,
                                     attempts=attempts,
                                     rest_s=rest_s)

    def get_text(self, xpath: str, single: bool = True, attempts: int = STD_ATTEMPTS, rest_s: float = REST_S) ->\
            Union[str, List[str]]:
        """
        Returns text in element(s)
        Args:
            xpath: str, xpath to element
            single: boolean, Whether to extract from single element or multiple. default = True
            attempts: int, number of attempts to make before failing
            rest_s: int, number of seconds to rest between attempts
        Returns: Text from inside element(s)
        """
        if single:
            elem = self._do_attempts(self.elem_by_xpath,
                                     xpath,
                                     attempts=attempts,
                                     rest_s=rest_s)
            if elem is not None:
                return elem.text
        else:
            elems = self._do_attempts(self.elems_by_xpath,
                                      xpath,
                                      attempts=attempts,
                                      rest_s=rest_s)
            text_list = []
            for e in elems:
                if e is not None:
                    text_list.append(e.text)
            return text_list

    def remove(self,
               xpath: str,
               single: bool = True,
               attempts: int = STD_ATTEMPTS,
               rest_s: float = REST_S):
        """
        Uses JavaScript commands to remove desired element
        Args:
            xpath: str, xpath to element
            single: boolean whether to apply to single element or multiple. default = True
            attempts: int, number of attempts to make before failing
            rest_s: int, number of seconds to rest between attempts
        """
        script = """
        var element = arguments[0];
        element.remove();
        """
        if single:
            elem = self._do_attempts(self.elem_by_xpath,
                                     xpath,
                                     attempts=attempts,
                                     rest_s=rest_s)
            if elem is not None:
                self.driver.execute_script(script, elem)
        else:
            elems = self._do_attempts(self.elems_by_xpath,
                                      xpath,
                                      attempts=attempts,
                                      rest_s=rest_s)
            for e in elems:
                if e is not None:
                    self.driver.execute_script(script, e)

    def add_style_to_elem(self, elem: WebElement, css_str: str):
        """Injects CSS into elem style"""
        js = f"arguments[0].setAttribute('style', '{css_str}');"
        self.driver.execute_script(js, elem)

    def click_by_id(self, elem_id: float):
        js = f'document.getElementById("{elem_id}").click();'
        self.driver.execute_script(js)

    def rand_wait(self, sleep_range_secs: List[int]):
        """
        Determines sleep time as random number between upper and lower limit,
            then sleeps for that given time. After sleep, moves randomly vertically and horizontally on page
            for up to four times
        Args:
            sleep_range_secs: list, min and max number of seconds to sleep
        """
        sleep_range_secs = sorted(sleep_range_secs)
        if len(sleep_range_secs) == 2:
            sleep_secs_lower, sleep_secs_higher = sleep_range_secs
        elif len(sleep_range_secs) > 2:
            sleep_secs_lower, sleep_secs_higher = sleep_range_secs[
                0], sleep_range_secs[-1]
        else:
            raise ValueError(
                'Input for sleep range must be at least two items')
        sleeptime = randint(sleep_secs_lower, sleep_secs_higher)
        self.log.debug(f'Waiting {sleeptime}s')
        time.sleep(sleeptime)
        # after wait period, scroll through window randomly
        for i in range(4):
            r_x = randint(-20, 20)
            r_y = randint(150, 300)
            self.scroll_absolute(direction=f'{r_x},{r_y}')

    def fast_wait(self):
        self.rand_wait(self._fast_wait)

    def medium_wait(self):
        self.rand_wait(self._medium_wait)

    def slow_wait(self):
        self.rand_wait(self._slow_wait)

    def scroll_to_element(self,
                          elem: WebElement,
                          use_selenium_method: bool = True):
        """
        Scrolls to get element in view
        Args:
            elem: Selenium-class element
            use_selenium_method: bool, if True, uses built-in Selenium method of scrolling an element to view
                otherwise, uses Javascript (scrollIntoView)
        """
        if use_selenium_method:
            actions = ActionChains(self.driver)
            actions.move_to_element(elem).perform()
        else:
            scroll_center_script = """
                var viewPortHeight = Math.max(document.documentElement.clientHeight, window.innerHeight || 0);
                var elementTop = arguments[0].getBoundingClientRect().top;
                window.scrollBy(0, elementTop-(viewPortHeight/2));
            """
            self.driver.execute_script(scroll_center_script, elem)

    def scroll_absolute(self, direction: str = 'up'):
        """Scrolls all the way up/down or to specific x,y coordinates"""
        if direction == 'up':
            coords = '0, 0'
        elif direction == 'down':
            coords = '0, document.body.scrollHeight'
        else:
            if ',' in direction:
                coords = direction
            else:
                raise ValueError(
                    'Invalid parameters entered. Must be an x,y coordinate, or up/down command.'
                )

        self.driver.execute_script(f'window.scrollTo({coords});')
示例#11
0
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from flask import Flask
from kavalkilu import LogWithInflux
from servertools import get_secret_file
from api.hosts.hosts import hosts
from api.hosts.keys import keys

app = Flask(__name__)
for blueprint in [keys, hosts]:
    app.register_blueprint(blueprint)


@app.route('/')
def main_page():
    return 'Main page!'


@app.errorhandler(500)
def handle_errors(error):
    return 'Not found!'


if __name__ == '__main__':
    secret_key = get_secret_file('FLASK_SECRET')
    app.secret_key = secret_key
    app.run(host='0.0.0.0', port='5002')
    # Instantiate log here, as the hosts API is requested to comunicate with influx
    log = LogWithInflux('hosts_api')
示例#12
0
import time
from kavalkilu import LogWithInflux, InfluxDBLocal, InfluxDBHomeAuto
from servertools import HueBulb

INTERVAL_MINS = 30
WAIT_S = 290
end_time = time.time() + INTERVAL_MINS * 60
logg = LogWithInflux('mushroom-grow-toggle')
influx = InfluxDBLocal(InfluxDBHomeAuto.TEMPS)
h = HueBulb('mushroom-plug')
# TODO: Use HASS instead of Influx to get current values

rounds = 0
while end_time > time.time():
    if rounds % 2 == 0:
        # Turn on during even rounds
        h.turn_on()
    else:
        # Turn off for off rounds
        h.turn_off()
    rounds += 1
    logg.debug(f'Waiting {WAIT_S / 60:.0f} mins...')
    time.sleep(WAIT_S)

logg.close()
示例#13
0
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Detects whether the garage door is up or down"""
from kavalkilu import LogWithInflux, HAHelper
from pitools import DistanceSensor
from pitools.peripherals import PiGarage

logg = LogWithInflux('garage_door', log_dir='gdoor')

TRIGGER_PIN = PiGarage.ultrasonic.trigger
ECHO_PIN = PiGarage.ultrasonic.echo
logg.debug('Initializing sensor...')
ds = DistanceSensor(TRIGGER_PIN, ECHO_PIN)

# Take an average of 10 readings
readings = []
logg.debug('Taking readings...')
for i in range(10):
    readings.append(ds.measure())

avg = sum(readings) / len(readings)

# Instantiate HASS
ha = HAHelper()
# Collect last reading
last_status = ha.get_state(PiGarage.ha_garage_door_sensor).get('state')

# Typically, reading is ca. 259cm when door is closed. ca. 50cm when open
if avg < 6000:
    status = 'open'
    # TODO: Depth when car is in
示例#14
0
 def setUpClass(cls) -> None:
     cls.log = LogWithInflux('invest-test')
     cls.ed = EdgarCollector(start_year=2017, parent_log=cls.log)
示例#15
0
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from datetime import datetime
import pandas as pd
from kavalkilu import LogWithInflux
from servertools import SlackWeatherNotification, NWSForecast, NWSForecastZone

# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('frost_warn', log_dir='weather')

now = datetime.now()
weather = NWSForecast(NWSForecastZone.ATX)
hours_df = weather.get_hourly_forecast()
hours_df['date'] = pd.to_datetime(hours_df['date'])

# Filter by column & get only the next 12 hours of forecasted weather
cols = ['date', 'temp-avg', 'feels-temp-avg', 'dewpoint', 'wind-speed']
hours_df = hours_df.loc[hours_df.date < (now + pd.Timedelta(hours=12)), cols]

logic_dict = {
    'freeze': (hours_df['temp-avg'] < 0) & ((hours_df['dewpoint'] < -8) | (hours_df['wind-speed'] > 5)),
    'frost': (hours_df['temp-avg'] < 2) & ((hours_df['dewpoint'] < -6) | (hours_df['wind-speed'] >= 5)),
    'light frost': (hours_df['temp-avg'] < 2) & ((hours_df['dewpoint'] < -6) | (hours_df['wind-speed'] < 5)),
}

warning = None
for name, cond in logic_dict.items():
    if any(cond.tolist()):
        # We want the warnings to move from severe to relatively mild &
        # break on the first one that matches the condition
        warning = name
示例#16
0
ETL for RTL_433 json objects via syslog -> processed Dataframe -> influx

Note: depends on `rf_stream` already being running and feeding data to port 1433
    via `rtl_433 -F syslog::1433`
"""
import json
from json import JSONDecodeError
import socket
from datetime import datetime
import pandas as pd
from kavalkilu import InfluxDBLocal, InfluxDBHomeAuto, LogWithInflux, \
    GracefulKiller, Hosts, HOME_SERVER_HOSTNAME, HAHelper
from servertools import SlackComm


logg = LogWithInflux('rf_temp')
sc = SlackComm(parent_log=logg)
UDP_IP = Hosts().get_ip_from_host(HOME_SERVER_HOSTNAME)
UDP_PORT = 1433

# device id to device-specific data mapping
mappings = {
    210: {'name': 'neighbor-porch'},
    3092: {'name': 'ylemine-r6du'},
    5252: {'name': 'elutuba'},
    6853: {'name': 'kontor-wc'},
    8416: {'name': 'alumine-r6du'},
    9459: {'name': 'freezer'},
    9533: {'name': 'kontor'},
    10246: {'name': 'v2lisuks'},
    12476: {'name': 'suur-wc'},
示例#17
0
import re
import pandas as pd
from servertools import BrowserAction
from kavalkilu import LogWithInflux, InfluxDBLocal, InfluxDBTracker

logg = LogWithInflux('apt-prices', log_to_file=True)
influx = InfluxDBLocal(InfluxDBTracker.APT_PRICES)
ba = BrowserAction(headless=True, parent_log=logg)
url = 'https://www.maac.com/available-apartments/?propertyId=611831&Bedroom=2%20Bed'

ba.get(url)

ba.medium_wait()
listings = ba.get_elem(
    '//div[contains(@class, "apartment-listing")]/div[contains(@class, "apartment")]',
    single=False)
logg.debug(f'Returned {len(listings)} initial listings...')
apt_list = []
for apt in listings:
    apt_dict = {}
    # Get unit
    unit = apt.find_element_by_xpath(
        './/div[contains(@class, "apartment__unit-number")]').text
    # Clean unit of non-number chars
    unit = re.search(r'\d+', unit).group()
    # Get sqft, beds, baths
    desc = apt.find_element_by_xpath(
        './/div[contains(@class, "apartment__unit-description")]').text.split(
            '\n')
    for d in desc:
        for item in ['bed', 'bath', 'sq. ft.']:
示例#18
0
"""Message links to memes for Viktor to post periodically"""
import os
import re
import time
import numpy as np
from typing import List
from datetime import datetime, timedelta
from slacktools import SlackTools
from kavalkilu import Keys, LogWithInflux


logg = LogWithInflux('memeraker')
vcreds = Keys().get_key('viktor')
st = SlackTools(**vcreds)
user_me = 'UM35HE6R5'

# Wine review text & previous stop timestamp
ddir = os.path.join(os.path.expanduser('~'), 'data')
fpath = os.path.join(ddir, 'mkov_wines.txt')
ts_path = os.path.join(ddir, 'last_memeraker_ts')


def post_memes(reviews: List[str], memes: List[str], wait_min: int = 5, wait_max: int = 60):
    # Begin the upload process, include a review
    for meme in memes:
        review = np.random.choice(reviews, 1)[0]
        st.upload_file('memes-n-shitposts', meme, 'image', is_url=True, txt=review)
        # Wait some seconds before posting again
        wait_s = np.random.choice(range(wait_min, wait_max), 1)[0]
        logg.debug(f'Waiting {wait_s}s.')
        time.sleep(wait_s)
示例#19
0
"""Checks slackmojis daily for new additions"""
import json
from kavalkilu import Path, LogWithInflux
from servertools import SlackComm, XPathExtractor

logg = LogWithInflux('emoji-scraper')
scom = SlackComm(bot='viktor', parent_log=logg)

p = Path()
fpath = p.easy_joiner(p.data_dir, 'slackmojis.json')
chan = 'emoji_suggestions'
url = 'https://slackmojis.com/emojis/recent'
xpath_extractor = XPathExtractor(url)

emoji_list = xpath_extractor.xpath('//ul[@class="emojis"]', single=True)
emojis = emoji_list.getchildren()

# Read in the previous emoji id list
if not p.exists(fpath):
    prev_emojis = {}
else:
    with open(fpath) as f:
        prev_emojis = json.loads(f.read())

new_emojis = {}
for emoji in emojis:
    emo_id = emoji.getchildren()[0].get('data-emoji-id-name')
    emo_name = emoji.getchildren()[0].getchildren()[1].text.strip()
    if emo_id not in prev_emojis.keys():
        # Get link and add to the id list
        emo_link = emoji.findall('.//img')[0].get('src')
import tempfile
from datetime import datetime as dt, timedelta
from kavalkilu import Hosts, LogWithInflux
from easylogger import ArgParse
from servertools import SlackComm, Amcrest, VidTools

logg = LogWithInflux('motion_alerts')
sc = SlackComm(parent_log=logg)

args = [{
    'names': ['-c', '--camera'],
    'other': {
        'action': 'store',
        'default': 'ac-allr6du'
    }
}, {
    'names': ['-i', '--interval'],
    'other': {
        'action': 'store',
        'default': '60'
    }
}]
ap = ArgParse(args, parse_all=False)
CAMERA = ap.arg_dict.get('camera')
INTERVAL_MINS = int(ap.arg_dict.get('interval'))
start_dt = (dt.now() - timedelta(minutes=INTERVAL_MINS)).replace(second=0,
                                                                 microsecond=0)
end_dt = (start_dt + timedelta(minutes=INTERVAL_MINS))

cam_ip = Hosts().get_ip_from_host(CAMERA)
cam = Amcrest(cam_ip)
示例#21
0
class EdgarCollector:
    """Handles the entire process of collecting SEC filing data and processing that into financial ratios"""
    base_url = 'https://www.sec.gov'

    def __init__(self, start_year: float = 2017, parent_log: LogWithInflux = None):
        self.logg = LogWithInflux(parent_log, child_name=self.__class__.__name__)
        # Set temp dir for downloading the edgar filings
        self.tmp_dir = os.path.join(tempfile.gettempdir(), 'edgar')
        # Get ticker to CIK mapping
        self.logg.debug('Downloading ticket to CIK mapping...')
        self.t2cik_df = self.get_ticker_to_cik_map()
        # Download EDGAR indexes and retrieve the filepaths associated with them
        self.logg.debug('Downloading indexes (this may take ~2 mins)...')
        self.edgar_fpaths = self._download_indexes(start_year)

    @staticmethod
    def get_ticker_to_cik_map() -> pd.DataFrame:
        """Collects the mapping of stock ticker to CIK"""
        t2cik_df = pd.read_csv('https://www.sec.gov/include/ticker.txt', sep='\t', header=None)
        t2cik_df.columns = ['tick', 'cik']
        return t2cik_df

    def _download_indexes(self, since_year: float) -> List[str]:
        """Downloads company indexes from Edgar into temporary directory"""
        # Begin download
        edgar.download_index(self.tmp_dir, since_year=since_year)
        # Retrieve the file paths downloaded
        fpaths = []
        for a, b, fnames in os.walk(self.tmp_dir):
            fpaths = [os.path.join(self.tmp_dir, f) for f in fnames]
            break
        self.logg.debug(f'Collected {len(fpaths)} files...')
        return sorted(fpaths)

    def get_data_for_stock_tickers(self, tickers: Union[str, List[str]]) -> pd.DataFrame:
        """Collects the quarterly financial ratios for a given stock ticker"""
        if isinstance(tickers, str):
            tickers = [tickers]

        stocks_df = pd.DataFrame({'tick': list(map(str.lower, tickers))})
        # Merge stocks on cik
        stocks_df = stocks_df.merge(self.t2cik_df, how='left', on='tick')

        ratio_df = pd.DataFrame()
        # Capture filing locations for each quarter for each stock
        for i, fpath in enumerate(sorted(self.edgar_fpaths)):
            self.logg.debug(f'Working on file {i + 1} of {len(self.edgar_fpaths)}')
            # Parse the quarter
            year, qtr = os.path.split(fpath)[1].split('.')[0].split('-')
            self.logg.debug(f'Parsed: {qtr}-{year}')
            df = pd.read_csv(fpath, sep='|', header=None)
            df.columns = ['cik', 'company_name', 'form', 'filing_date', 'txt_file', 'html_file']
            # Filter by CIK and on either 10-K (annual) or 10-Q (quarterly) reports
            df = df.loc[df.cik.isin(stocks_df.cik) & df.form.isin(['10-Q', '10-K'])]
            self.logg.debug(f'Matched {df.shape[0]} rows of data')
            for idx, row in df.iterrows():
                # Grab the HTML file - this is the file list
                url = f'{self.base_url}/Archives/{row["html_file"]}'
                xpe = XPathExtractor(url)
                # Grab the link to the 10-* file
                form_type = row['form']
                form_row = xpe.xpath(f'//table[@class="tableFile"]/tr[td[text()="{form_type}"]]', single=True)
                form_url = xpe.xpath('./td/a', obj=form_row, single=True).get('href')

                # Grab the 10-* file data
                complete_url = f'{self.base_url}{form_url}'

                # Work on SOps (this sets a point to look 'after')
                stmt_ops = EdgarFinStatement(complete_url, 'STATEMENTS? OF OPERATIONS')
                net_sales = stmt_ops.get_line_item('net sales')
                cogs = stmt_ops.get_line_item('cost of sales')
                op_income = stmt_ops.get_line_item('operating income')
                net_income = stmt_ops.get_line_item('net income')
                eps = stmt_ops.get_line_item('earnings per share', line_after=True)
                shares = stmt_ops.get_line_item('shares used in computing earnings', line_after=True)

                # Work on Balance Sheet
                stmt_bs = EdgarFinStatement(complete_url, 'CONSOLIDATED BALANCE SHEETS')
                current_assets = stmt_bs.get_line_item('total current assets')
                intangible_assets = stmt_bs.get_line_item('intangible assets')
                total_assets = stmt_bs.get_line_item('total assets')
                current_liabilities = stmt_bs.get_line_item('total current liabilities')
                total_liabilities = stmt_bs.get_line_item('total liabilities')
                common_stock_eq = stmt_bs.get_line_item('common stock')
                total_shareholders_equity = stmt_bs.get_line_item('total shareholders')
                net_tangible_assets = total_assets - intangible_assets - total_liabilities

                # Work on Cash Flow (SCF)
                stmt_cf = EdgarFinStatement(complete_url, 'STATEMENTS OF CASH FLOWS')
                dep_and_amort = stmt_cf.get_line_item('depreciation and amortization')
                ebitda = op_income + dep_and_amort

                # Ratios
                asset_turnover = net_sales / net_tangible_assets
                profit_margin = net_income / net_sales
                debt_to_ebitda = total_liabilities / ebitda
                debt_to_equity = total_liabilities / total_shareholders_equity
                roa = net_income / total_assets
                roe = net_income / total_shareholders_equity
                bvps = total_shareholders_equity / shares

                # Apply ratios to dataframe
                ratio_df = ratio_df.append({
                    'year': year,
                    'quarter': qtr,
                    'cik': row['cik'],
                    'source': row['form'],
                    'ato': asset_turnover,
                    'pm': profit_margin,
                    'd2ebitda': debt_to_ebitda,
                    'd2e': debt_to_equity,
                    'roa': roa,
                    'roe': roe,
                    'bvps': bvps
                }, ignore_index=True)

        # Merge the financial info back in with the stocks
        stocks_df = stocks_df.merge(ratio_df, how='left', on='cik')
        return stocks_df
示例#22
0
import subprocess
from kavalkilu import LogWithInflux, HOME_SERVER_HOSTNAME, Hosts

logg = LogWithInflux('rf_stream', log_dir='rf')
serv_ip = Hosts().get_ip_from_host(HOME_SERVER_HOSTNAME)
cmd = ['/usr/local/bin/rtl_433', '-F', f'syslog:{serv_ip}:1433']

logg.info(f'Sending command: {" ".join(cmd)}')
process = subprocess.Popen(cmd,
                           stdout=subprocess.PIPE,
                           stderr=subprocess.STDOUT)
process_output, _ = process.communicate()
logg.debug(f'Process output: {process_output}')
示例#23
0
import os
from datetime import datetime as dt, timedelta
from kavalkilu import LogWithInflux, DateTools, InfluxDBLocal, InfluxDBPiHole, SQLLiteLocal, Hosts
from servertools import SlackComm

logg = LogWithInflux('pihole_etl', log_to_db=True)
sc = SlackComm(parent_log=logg)
hosts = {x['ip']: x['name'] for x in Hosts().get_all_hosts()}
datetools = DateTools()

FTL_DB_PATH = os.path.join('/etc', *['pihole', 'pihole-FTL.db'])
sqll = SQLLiteLocal(FTL_DB_PATH)

INTERVAL_MINS = 60
end = dt.now().astimezone().replace(second=0, microsecond=0)
start = (end - timedelta(minutes=INTERVAL_MINS))
unix_start = datetools.dt_to_unix(start, from_tz='US/Central')
unix_end = datetools.dt_to_unix(end, from_tz='US/Central')

query = f"""
    SELECT
        client
        , domain
        , CASE
            WHEN status = 0 THEN 'UNKNOWN'
            WHEN status = 1 OR status > 3 THEN 'BLOCKED'
            WHEN status = 2 OR status = 3 THEN 'ALLOWED' 
            ELSE 'UNKNOWN'
        END AS query_status
        , CASE
            WHEN status = 0 THEN 'NO ANSWER'
"""For joining timelapse shots"""
import os
from moviepy.editor import ImageClip, concatenate_videoclips
from kavalkilu import Path, LogWithInflux
from servertools import SlackComm

log = LogWithInflux('timelapse_combi')
p = Path()
tl_dir = p.easy_joiner(p.data_dir, 'timelapse')
fnames = {}
for dirpath, _, filenames in os.walk(tl_dir):
    dirname = os.path.basename(dirpath)
    if dirname != 'timelapse':
        fnames[os.path.basename(dirpath)] = filenames

files = []
# Begin combining shots
for k, v in fnames.items():
    if not any([k.startswith(x) for x in ['ac-', 're-']]):
        continue
    log.debug(f'Working on {k}. {len(v)} files.')
    full_paths = sorted([os.path.join(tl_dir, *[k, x]) for x in v])
    clips = []
    for fpath in full_paths:
        try:
            clips.append(ImageClip(fpath).set_duration(1))
        except ValueError:
            log.debug(f'Error with this path: {fpath}')
            continue
    clip = concatenate_videoclips(clips)
    clip = clip.set_fps(30).speedx(30)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Activates nighttime mode on cameras"""
from servertools import Amcrest
from kavalkilu import LogWithInflux, Hosts

# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('cam_night')
hosts = Hosts()
# Get only cameras without numbers in the name
cam_info_list = hosts.get_hosts_and_ips(r'^ac-(ga|el)')

for cam_dict in cam_info_list:
    # Instantiate cam & arm
    cam = Amcrest(cam_dict['ip'])
    if cam.camera_type != 'doorbell':
        cam.arm_camera(True)
        # publish.single(f'sensors/cameras/{cam.name}/status', 'ARMED', hostname='tinyserv.local')

log.close()
示例#26
0
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Log memory, cpu use and temp of each machine"""
from kavalkilu import LogWithInflux, InfluxDBHomeAuto
from pitools import Sensor

logg = LogWithInflux('machine_data')
# Set the pin (BCM)
for sensor_name in ['CPU', 'MEM', 'CPUTEMP', 'DISK']:
    logg.debug(f'Logging {sensor_name}...')
    sensor = Sensor(sensor_name)
    # Take readings & log to db
    sensor.measure_and_log_to_db(
        tbl=InfluxDBHomeAuto().__getattribute__(sensor_name), n_times=2)

logg.debug('Temp logging successfully completed.')

logg.close()
示例#27
0
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""Determines if mobile is connected to local network. If not, will arm the cameras"""
from servertools import Amcrest, OpenWRT
from kavalkilu import LogWithInflux, Hosts

# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('cam_active')
ow = OpenWRT()
hosts = Hosts()
# Get only cameras without numbers in the name
cam_info_list = hosts.get_hosts_and_ips(r'(?!^ac-.*(\d.*|doorbell)$)^ac-.+$')

res_list = []
currently_active_ips = ow.get_active_connections()
# Check if mobile(s) are connected to LAN
for ip in [i['ip'] for i in hosts.get_hosts_and_ips('an-[bm]a.*')]:
    res_list.append(ip in currently_active_ips.keys())

# If anyone home, don't arm, otherwise arm
arm_cameras = not any(res_list)
arm_status = 'ARMED' if arm_cameras else 'UNARMED'
if not arm_cameras:
    log.debug(
        'One of the devices are currently in the network. Disabling motion detection.'
    )
else:
    log.debug(
        'None of the devices are currently in the network. Enabling motion detection.'
    )
for cam_dict in cam_info_list:
示例#28
0
"""Sends latest temperature readings to HASS"""
from kavalkilu import LogWithInflux, InfluxDBHomeAuto, InfluxDBLocal, HAHelper


log = LogWithInflux('ha-temps', log_dir='weather')
influx = InfluxDBLocal(InfluxDBHomeAuto.TEMPS)

query = '''
    SELECT 
        last("temp") AS temp,
        last("humidity") AS humidity
    FROM "temps"
    WHERE 
        location =~ /mushroom|r6du|elutuba|wc|v2lis|freezer|fridge|kontor/
        AND time > now() - 30m
    GROUP BY 
        "location" 
    fill(null)
    ORDER BY ASC
'''
df = influx.read_query(query, time_col='time')
log.debug(f'Collected {df.shape[0]} rows of data')

log.debug('Beginning to send updates to HASS')
ha = HAHelper()
for i, row in df.iterrows():
    loc_name = row['location'].replace('-', '_')
    for sensor_type in ['temp', 'humidity']:
        dev_name = f'sensor.{loc_name}_{sensor_type}'
        log.debug(f'Updating {dev_name}...')
        ha.set_state(dev_name, data={'state': row[sensor_type]}, data_class=sensor_type)
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
from datetime import datetime, timedelta
import pandas as pd
from kavalkilu import LogWithInflux
from servertools import NWSForecast, NWSForecastZone, SlackWeatherNotification

# Initiate Log, including a suffix to the log name to denote which instance of log is running
log = LogWithInflux('significant_change', log_dir='weather')

swno = SlackWeatherNotification(parent_log=log)

now = datetime.now()
tomorrow = (now + timedelta(days=1))
weather = NWSForecast(NWSForecastZone.ATX)
hours_df = weather.get_hourly_forecast()
hours_df['date'] = pd.to_datetime(hours_df['date'])

# focus on just the following day's measurements
nextday = hours_df[hours_df['date'].dt.day == tomorrow.day].copy()
nextday['day'] = nextday['date'].dt.day
nextday['hour'] = nextday['date'].dt.hour
temp_dict = {}
metrics_to_collect = dict(
    zip(
        ['temp-avg', 'feels-temp-avg', 'precip-intensity', 'precip-prob'],
        ['temp', 'apptemp', 'precip_int', 'precip_prob'],
    ))
# Determine which hours are important to examine (e.g., for commuting / outside work)
important_hours = [0, 6, 12, 15, 18]
for hour in important_hours:
示例#30
0
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import os
import time
import requests
from grafana_api.grafana_face import GrafanaFace
from kavalkilu import Keys, LogWithInflux
from servertools import SlackComm


log = LogWithInflux('grafana_snapper')
creds = Keys().get_key('grafana')
scom = SlackComm()


def get_pic_and_upload(url, name):
    """Captures dashboard panel at URL and uploads to #notifications slack channel"""

    resp = requests.get(url, headers={'Authorization': f'Bearer {creds["token"]}'})

    temp_file = os.path.abspath('/tmp/dash_snap.png')
    with open(temp_file, 'wb') as f:
        for chunk in resp:
            f.write(chunk)
    scom.st.upload_file(scom.notify_channel, temp_file, name)


# The URL template to use
base_url = 'http://{host}/render/{name}?orgId=1&from={from}&to={to}&panelId={pid}&width=1000&height=500'

# time range to snap