class RSSCache:
def __init__(self, cache_max_size, cache_ttl_seconds):
self.cache = TTLCache(ttl=cache_ttl_seconds, maxsize=cache_max_size)
def update_cache(self, key, value):
self.cache.update({key: value})
def get_from_cache(self, key):
return self.cache.get(key)
class Bot:
"""The twitter bot."""
# Consts
CALL_TEXT = "{} second encrypted call at {}"
HASHTAGS = "#SeattleEncryptedComms"
TWEET_PADDING = 20
BASE_URL = "https://api.openmhz.com/kcers1b/calls/newer?time={}&filter-type=talkgroup&filter-code=44912,45040,45112,45072,45136"
# DEBUG URL TO GET A LOT OF API RESPONSES
# BASE_URL = "https://api.openmhz.com/kcers1b/calls/newer?time={}"
def __init__(self) -> None:
"""Initializes the class."""
self.callThreshold = int(os.getenv("CALL_THRESHOLD", 1))
self.debug = os.getenv("DEBUG", "true").lower() == "true"
self.reportLatency = os.getenv("REPORT_LATENCY",
"false").lower() == "true"
self.window_minutes = int(os.getenv("WINDOW_M", 5))
self.timezone = pytz.timezone(os.getenv("TIMEZONE", "US/Pacific"))
# The actual look back is the length of this lookback + lag compensation. For example: 300+45=345 seconds
self.lookback = os.getenv("LOOKBACK_S", 300)
self.cachedTweet: int = None
self.cachedTime: datetime = None
self.cache = TTLCache(maxsize=100, ttl=self.lookback)
self.scraper = Scraper.Instance(self.BASE_URL, self.lookback)
self.latency = [timedelta(seconds=0)]
if not self.debug:
# Does not need to be saved for later.
# If the keys aren't in env this will still run.
auth = tweepy.OAuthHandler(os.getenv("CONSUMER_KEY", ""),
os.getenv("CONSUMER_SECRET", ""))
auth.set_access_token(os.getenv("ACCESS_TOKEN_KEY", ""),
os.getenv("ACCESS_TOKEN_SECRET", ""))
self.api = tweepy.API(auth)
# Test the authentication. This will gracefully fail if the keys aren't present.
try:
self.api.rate_limit_status()
except TweepError as e:
if e.api_code == 215:
log.error("No keys or bad keys")
else:
log.error("Other API error: {}".format(e))
exit(1)
self.interval = Set.Interval(30, self._check)
def _kill(self) -> None:
"""This kills the c̶r̶a̶b̶ bot."""
self.interval.cancel()
exit(0)
def _getUniqueCalls(self, calls: dict) -> list:
"""Filters the return from the scraper to only tweet unique calls.
Works by checking if the cache already has that call ID.
Args:
calls (list): The complete list of calls scraped.
Returns:
list: A filtered list of calls.
"""
res: List[dict] = []
for call in calls:
# If the call is already in the cache skip.
if call["_id"] in self.cache.keys():
continue
# If it isn't, cache it and return it.
else:
# Might want to actually store somthing? Who knows.
self.cache.update({call["_id"]: 0})
res.append(call)
return res
def _check(self) -> None:
"""Checks the API and sends a tweet if needed."""
try:
log.info(f"Checking!: {datetime.now()}")
try:
json = self.scraper.getJSON()
calls = self._getUniqueCalls(json["calls"])
log.info(f"Found {len(calls)} calls.")
if len(calls) > 0:
self._postTweet(calls)
except TypeError as e:
if json == None:
# We already have an error message from the scraper
return
log.exception(e)
return
except KeyboardInterrupt as e:
# Literally impossible to hit which might be an issue? Catching keyboard interrupt could happen in its own thread or something but that sounds complicated 👉👈
self._kill()
if self.reportLatency:
sum = sum(self.latency).total_seconds()
avg = round(sum / len(self.latency), 3)
log.info(f"Average latency for the last 100 calls: {avg} seconds")
def _postTweet(self, calls: list) -> None:
"""Posts a tweet.
Args:
calls (list): The call objects to post about.
"""
# Filter to make sure that calls are actually recent. There can be a weird behavior of the API returning multiple hours old calls all at once. Also filters for calls under the length threshold.
filteredCalls: List[dict] = []
for call in calls:
diff = datetime.now(pytz.utc) - datetime.strptime(
call["time"], "%Y-%m-%dT%H:%M:%S.000%z")
if not abs(diff.total_seconds()) >= 1.8e3:
if call["len"] < self.callThreshold:
log.debug(
f"Call of size {call['len']} below threshold ({self.callThreshold})"
)
continue
filteredCalls.append(call)
if self.reportLatency:
# Store latency
self.latency.append(diff)
if len(self.latency) > 100:
self.latency.pop(0)
if len(filteredCalls) == 0:
# If there's nothing to post, simply leave
return
msgs = self._generateTweets(filteredCalls)
if self.debug:
msg = " | ".join(msgs)
log.debug(f"Would have posted: {msg}")
return
# Check for a cached tweet, then check if the last tweet was less than the window ago. If the window has expired dereference the cached tweet.
if (self.cachedTime != None
and self.cachedTime + timedelta(minutes=self.window_minutes) <=
datetime.now()):
self.cachedTweet = None
try:
if self.cachedTweet != None:
for msg in msgs:
# Every time it posts the new ID gets stored so this works
self.cachedTweet = self.api.update_status(
msg, self.cachedTweet).id
else:
for index, msg in enumerate(msgs):
if index == 0:
# Since there isn't a cached tweet yet we have to send a non-reply first
self.cachedTweet = self.api.update_status(msg).id
else:
self.cachedTweet = self.api.update_status(
msg, self.cachedTweet).id
self.cachedTime = datetime.now()
except tweepy.TweepError as e:
log.exception(e)
def _timeString(self, call: dict) -> str:
"""Generates a time code string for a call.
Args:
call (dict): The call to get time from.
Returns:
str: A timestamp string in I:M:S am/pm format.
"""
# Get time from the call.
date = datetime.strptime(call["time"], "%Y-%m-%dT%H:%M:%S.000%z")
# F**k I hate how computer time works
localized = date.replace(tzinfo=pytz.utc).astimezone(self.timezone)
normalized = self.timezone.normalize(localized)
return normalized.strftime("%#I:%M:%S %p")
def _chunk(self, callStrings: list) -> list:
"""Chunks tweets into an acceptable length.
Chunking. Shamelessly stolen from `SeattleDSA/signal_scanner_bot/twitter.py` :)
Args:
call_strings (list): List of strings derived from calls.
Returns:
list: A list of tweet strings to post
"""
tweetList: List[str] = []
baseIndex = 0
# Instead of spliting on words I want to split along call lines.
subTweet: str = ""
for index in range(len(callStrings)):
if len(tweetList) == 0:
subTweet = (", ".join(callStrings[baseIndex:index]) + " ... " +
self.HASHTAGS)
elif index < len(callStrings):
subTweet = ", ".join(callStrings[baseIndex:index]) + " ..."
elif index == len(callStrings):
subTweet = ", ".join(callStrings[baseIndex:index])
if len(subTweet) > 280 - self.TWEET_PADDING:
lastIndex = index - 1
tweetList.append(", ".join(callStrings[baseIndex:lastIndex]) +
" ...")
baseIndex = lastIndex
tweetList.append(", ".join(callStrings[baseIndex:]))
listLength = len(tweetList)
for index in range(len(tweetList)):
if index == 0:
tweetList[
index] += f" {self.HASHTAGS} {index + 1}/{listLength}"
else:
tweetList[index] += f" {index + 1}/{listLength}"
return tweetList
def _generateTweets(self, calls: list) -> list:
"""Generates tweet messages.
Args:
call (list): The calls to tweet about.
Returns:
list: The tweet messages, hopefully right around the character limit.
"""
callStrings: List[str] = []
# First, take all of the calls and turn them into strings.
for call in calls:
callStrings.append(
self.CALL_TEXT.format(
call["len"],
self._timeString(call),
))
tweet = ", ".join(callStrings) + " " + self.HASHTAGS
# If we don't have to chunk we can just leave.
if len(tweet) <= 280:
return [tweet]
else:
tweetList = self._chunk(callStrings)
return tweetList
class KodiClient(object):
def __init__(self, config):
self._cache = TTLCache(maxsize=2048, ttl=3600)
if 'user' in config and 'password' in config:
self.auth = (config['user'], config['password'])
else:
self.auth = None
self.host = config['host']
self.port = config['port']
self.chunk_size = 750
self._api = Server(
url='http://{host}:{port}/jsonrpc'.format(**config),
auth=self.auth)
def _make_generator(self, method, data_key, **params):
logger.debug("Fetching first chunk of {}".format(data_key))
params.update({'limits': {'start': 0, 'end': self.chunk_size}})
resp = method(**params)
for d in resp[data_key]:
yield d
num_total = resp['limits']['total']
cur_start = self.chunk_size
while cur_start < num_total:
params['limits']['start'] = cur_start
params['limits']['end'] = cur_start + self.chunk_size
logger.debug("Fetching next chunk from #{}".format(cur_start))
resp = method(**params)
for d in resp[data_key]:
yield d
cur_start += self.chunk_size
@cached()
def get_artists(self):
artists = list(self._make_generator(
self._api.AudioLibrary.GetArtists, 'artists',
properties=PROPERTIES['artist']))
self._cache.update({'artist.{}'.format(a['artistid']): a
for a in artists})
return artists
def get_artist(self, artist_id):
artist_id = int(artist_id)
cached = self._cache.get('artist.{}'.format(artist_id))
if cached is None:
try:
artist = self._api.AudioLibrary.GetArtistDetails(
artistid=artist_id,
properties=PROPERTIES['artist'])['artistdetails']
self._cache['artist.{}'.format(artist_id)] = artist
return artist
except Exception as e:
return None
else:
return cached
@cached()
def get_albums(self, artist_id=None, recently_added=False):
if recently_added:
return self._api.AudioLibrary.GetRecentlyAddedAlbums(
properties=PROPERTIES['album'])['albums']
if artist_id is not None:
artist_id = int(artist_id)
params = {'properties': PROPERTIES['album'],
'data_key': 'albums'}
if artist_id:
params['filter'] = {'artistid': artist_id}
albums = list(self._make_generator(
self._api.AudioLibrary.GetAlbums, **params))
self._cache.update({'album.{}'.format(a['albumid']): a
for a in albums})
return albums
def get_album(self, album_id):
album_id = int(album_id)
cached = self._cache.get('album.{}'.format(album_id))
if cached is None:
try:
album = self._api.AudioLibrary.GetAlbumDetails(
albumid=album_id,
properties=PROPERTIES['album'])['albumdetails']
self._cache['album.{}'.format(album_id)] = album
return album
except Exception as e:
self._cache['album.{}'.format(album_id)] = None
return None
else:
return cached
@cached() # First-level cache for accessing all tracks
def get_songs(self, album_id=None):
if album_id is not None:
album_id = int(album_id)
params = {'properties': PROPERTIES['song'],
'data_key': 'songs'}
if album_id:
params['filter'] = {'albumid': album_id}
songs = list(self._make_generator(
self._api.AudioLibrary.GetSongs, **params))
# Second level cache so that get_song doesn't have to make an API call
self._cache.update({'song.{}'.format(s['songid']): s for s in songs})
return songs
def get_song(self, song_id):
song_id = int(song_id)
cached = self._cache.get('song.{}'.format(song_id))
if cached is None:
try:
song = self._api.AudioLibrary.GetSongDetails(
songid=song_id,
properties=PROPERTIES['song'])['songdetails']
self._cache['song.{}'.format(song_id)] = song
return song
except Exception as e:
self._cache['song.{}'.format(song_id)] = None
return None
else:
return cached
@cached()
def get_url(self, filepath):
path = self._api.Files.PrepareDownload(filepath)
url = "http://{}{}:{}/{}".format(
"{}:{}@".format(*self.auth) if self.auth else '',
self.host, self.port, path['details']['path'])
self._cache['trackurl.{}'.format(url)] = filepath
return url
class MicrosoftGraphHelper:
def __init__(self, tenant_id, client_id, client_secret):
self.__cache = TTLCache(
maxsize=1, ttl=55 *
60) # Set to expire after 55 minutes so it is always fresh
self.__tenant_id = tenant_id
self.__client_id = client_id,
self.__client_secret = client_secret
self.__get_cache('microsoft_security_graph_access_token')
def __set_cache(self, d):
try:
for k, v in d.items():
self.__cache.update([(k, v)])
except AttributeError:
for k, v in d.items():
self.__cache.update([(k, v)])
def __get_cache(self, key):
if key not in self.__cache:
self.__set_cache({key: self.__refresh_access_token()})
return self.__cache.get(key)
def __refresh_access_token(self):
token_url = 'https://login.microsoftonline.com/{}/oauth2/v2.0/token'.format(
self.__tenant_id)
post_data = {
"client_id": self.__client_id,
"scope": ["https://graph.microsoft.com/.default"],
"client_secret": self.__client_secret,
"grant_type": "client_credentials"
}
r = requests.post(token_url, data=post_data)
log.debug(r.content)
json = r.json()
return json.get("access_token")
def check_status_code(self, response):
if 200 <= response.status_code <= 299:
return True
# Access token has expired, request a new one
elif response.status_code == 401:
log.debug(response.content)
access_token = self.__refresh_access_token()
self.__set_cache(
{"microsoft_security_graph_access_token": access_token})
return False
else:
raise ValueError("Invalid response from Microsoft Security Graph")
def microsoft_graph_request(self, method, url, headers, json=None):
r = None
for i in list(range(2)):
if method == "GET":
r = requests.get(url, headers=headers)
elif method == "PATCH":
r = requests.patch(url, headers=headers, json=json)
else:
raise ValueError("{} not implemented.".format(method))
if self.check_status_code(r):
break
# If it fails a second time, something more serious is wrong, ie: creds, query, etc.
elif i == 1:
log.info(r.content)
return False
return r
def get_access_token(self):
return self.__get_cache("microsoft_security_graph_access_token")
def clear_cache(self):
self.__cache.clear()
class AutoTTLCache(MutableMapping):
def __init__(self,
items=None,
*,
maxsize,
ttl,
timer=time.monotonic,
getsizeof=None):
self._cache_lock = threading.Lock()
self._cache = TTLCache(maxsize, ttl, timer=timer, getsizeof=getsizeof)
if items is not None:
self._cache.update(items)
self._monitor = CacheMonitor(self)
@property
def ttl(self):
with self._cache_lock:
return self._cache.ttl
@property
def maxsize(self):
with self._cache_lock:
return self._cache.maxsize
@property
def timer(self):
with self._cache_lock:
return self._cache.timer
def expire(self):
with self._cache_lock:
self._cache.expire()
def __contains__(self, key):
with self._cache_lock:
return key in self._cache
def __setitem__(self, k, v):
with self._cache_lock:
self._cache[k] = v
def __delitem__(self, k):
with self._cache_lock:
del self._cache[k]
def __getitem__(self, k):
with self._cache_lock:
return self._cache[k]
def __len__(self) -> int:
with self._cache_lock:
return len(self._cache)
def __iter__(self):
with self._cache_lock:
keys = list(self._cache)
yield from keys
# TODO: __reduce__ and __setstate__
def __repr__(self):
return f"{type(self).__name__}(max_size={self.maxsize}, ttl={self.ttl})"
def clear(self):
with self._cache_lock:
self._cache.clear()
def get(self, *args, **kwargs):
with self._cache_lock:
self._cache.get(*args, **kwargs)
def pop(self, *args, **kwargs):
with self._cache_lock:
self._cache.pop(*args, **kwargs)
def setdefault(self, *args, **kwargs):
with self._cache_lock:
self._cache.setdefault(*args, **kwargs)
def popitem(self):
with self._cache_lock:
self._cache.popitem()
