def test_filter_short():
# TODO: This was used in dev and does not work.
awapi = ActivityWatchClient("cleaner", testing=True)
events = awapi.get_events(BUCKET_WEB, limit=-1)
filter_short(events, threshold=1)
events = awapi.get_events(BUCKET_WINDOW, limit=-1)
filter_short(events, threshold=1)
events = awapi.get_events(BUCKET_AFK, limit=-1)
filter_short(events, threshold=30)
def test_filter_short():
# TODO: This was used in dev and does not work.
awapi = ActivityWatchClient("cleaner", testing=True)
events = awapi.get_events("aw-watcher-web-test", limit=-1)
filter_short(events, threshold=1)
events = awapi.get_events("aw-watcher-window-testing_erb-main2-arch",
limit=-1)
filter_short(events, threshold=1)
events = awapi.get_events("aw-watcher-afk-testing_erb-main2-arch",
limit=-1)
filter_short(events, threshold=30)
class Detector:
def __init__(self) -> None:
self.client = ActivityWatchClient("status-checker")
# TODO: Move to aw-client?
# TODO: Doesn't care if the event was old (as can happen if you have a dead watcher)
def _get_last_event(self, bucket_id: str) -> Event:
last_events = self.client.get_events(bucket_id, limit=1)
if last_events:
return last_events[0]
else:
raise Exception("no event found")
def get_bucket_id(self, type: str) -> str:
# TODO: Doesn't care about hostname
# TODO (maybe): Create a better way to query buckets
buckets = self.client.get_buckets()
# print(buckets)
window_bucket = find(
lambda bucket: bucket["type"] == type and "testing" not in bucket[
"id"], buckets.values())
if window_bucket is None:
raise Exception("Bucket not found")
return window_bucket["id"]
def detect(self, bucket_id: str, filter_str: str) -> Optional[Event]:
last_event = self._get_last_event(bucket_id)
return last_event if find(lambda label: filter_str in label.lower(),
last_event.labels) else None
def main(dryrun=True):
# TODO: Use wordlist instead of argument
sensitive_words = ["sensitive"]
aw = ActivityWatchClient(testing=True)
re_word = r'\b{}\b'
buckets = aw.get_buckets()
for bid in buckets.keys():
if "window" not in bid:
continue
print("Checking bucket: {}".format(bid))
events = aw.get_events(bid, limit=-1)
old_matches = set()
for event in events:
for key, val in event.data.items():
if isinstance(val, str):
matches = [re.findall(re_word.format(word), val.lower())
for word in sensitive_words]
matches = set(sum(matches, []))
if matches:
event.data[key] = "REDACTED"
if val not in old_matches:
print(f"{'(DRYRUN) ' if dryrun else ''} Matches: {matches}, redacting: {val}")
old_matches.add(val)
if not dryrun:
aw.insert_event(bid, event)
def all_active_webactivity():
"""Returns activity during non-afk events or when tab is audible"""
awapi = ActivityWatchClient("test", testing=True)
start = datetime.now() - timedelta(days=7)
tabevents = awapi.get_events("aw-watcher-web-chrome", start=start)
afkevents = awapi.get_events("aw-watcher-afk_erb-laptop2-arch",
start=start)
afkevents_notafk = list(
filter(lambda e: e.data["status"] == "not-afk", afkevents))
tabevents_audible = list(
filter(lambda e: "audible" in e.data and e.data["audible"], tabevents))
# TODO: Implement merge
# activeevents = merge(afkevents_notafk, tabevents_audible)
# This isn't perfect, buggy when a notafk/audible events is contained by another
activeevents = afkevents_notafk + tabevents_audible
return filter_period_intersect(tabevents, activeevents)
def _get_window_events(n=1000):
client = ActivityWatchClient("aw-analyser", testing=True)
buckets = client.get_buckets()
bucket_id = None
for _bid in buckets.keys():
if "window" in _bid and "testing" not in _bid:
bucket_id = _bid
if bucket_id:
return client.get_events(bucket_id, limit=n)
else:
print("Did not find bucket")
return []
class CLIWatcher:
def __init__(self, bucket, etype, testing=False):
self.client = ActivityWatchClient("aw-watcher-cli", testing=testing)
self.bucketname = bucket
self.etype = etype
self.client.create_bucket(self.bucketname, event_type=self.etype)
def run(self):
logger.info(
"aw-watcher-cli started for bucket '%s' containing events of type '%s'"
% (self.bucketname, self.etype))
def updateLastEvent(self):
logger.info("Updating previous event duration.")
try:
last = self.client.get_events(bucket_id=self.bucketname,
limit=1)[0]
except IndexError:
logger.info("No previous event found.")
return
duration = abs((datetime.now(timezone.utc) - last.timestamp).seconds)
logger.info(
"Previous event '%s' occurred at %s. Duration since: %ss" %
(last.data, last.timestamp.strftime("%Y-%m-%d %T%z"), duration))
last.duration = duration
self.client.insert_event(self.bucketname, last)
def addStringEvent(self, eventString, duration):
logger.info("Adding event %s (duration: %s) to bucket %s" %
(eventString, duration, self.bucketname))
data = {"label": eventString}
event = Event(timestamp=datetime.now(timezone.utc),
data=data,
duration=duration)
inserted_event = self.client.insert_event(self.bucketname, event)
assert inserted_event.id is not None
def test_failqueue():
client_name = "aw-test-client-" + str(randint(0, 10000))
bucket_id = "test-bucket-" + str(randint(0, 10000))
bucket_etype = "test"
input("Make sure aw-server isn't running, then press enter > ")
client1 = ActivityWatchClient(client_name, testing=True)
client1.create_bucket(bucket_id, bucket_etype, queued=True)
print("Creating events")
events = [create_unique_event() for _ in range(3)]
for i, e in enumerate(events):
e.timestamp += timedelta(seconds=i)
client1.heartbeat(bucket_id, e, pulsetime=1, queued=True)
print("Trying to send events (should fail)")
with client1:
time.sleep(1)
input("Start aw-server with --testing, then press enter > ")
client2 = ActivityWatchClient(client_name, testing=True)
client2.create_bucket(bucket_id, bucket_etype, queued=True)
# Here the previously queued events should be sent
with client2:
time.sleep(1)
print("Getting events")
recv_events = client2.get_events(bucket_id)
print("Asserting latest event")
pprint(recv_events)
pprint(recv_events[0].data['label'])
pprint(events[2].data['label'])
assert recv_events[0].data['label'] == events[2].data['label']
def test_failqueue():
client_name = "aw-test-client-" + str(randint(0, 10000))
bucket_id = "test-bucket-" + str(randint(0, 10000))
bucket_etype = "test"
input("Make sure aw-server isn't running, then press enter > ")
client1 = ActivityWatchClient(client_name, testing=True)
client1.create_bucket(bucket_id, bucket_etype, queued=True)
print("Creating events")
events = [create_unique_event() for _ in range(3)]
for i, e in enumerate(events):
e.timestamp += timedelta(seconds=i)
client1.heartbeat(bucket_id, e, pulsetime=1, queued=True)
print("Trying to send events (should fail)")
with client1:
time.sleep(1)
input("Start aw-server with --testing, then press enter > ")
client2 = ActivityWatchClient(client_name, testing=True)
client2.create_bucket(bucket_id, bucket_etype, queued=True)
# Here the previously queued events should be sent
with client2:
time.sleep(1)
print("Getting events")
recv_events = client2.get_events(bucket_id)
print("Asserting latest event")
pprint(recv_events)
pprint(recv_events[0].data['label'])
pprint(events[2].data['label'])
assert recv_events[0].data['label'] == events[2].data['label']
# Usage:
# python3 test_continous_events.py aw-watcher-afk-testing_{hostname}
#
# Example:
# python3 test_continous_events.py aw-watcher-afk-testing_erb-laptop-ubuntu
import sys
from datetime import timedelta
from aw_client import ActivityWatchClient
client = ActivityWatchClient("aw-watcher-afk-test", testing=True)
print(client.get_buckets())
bucket_id = sys.argv[1]
events = client.get_events(
bucket_id) # For example "aw-watcher-afk-testing_erb-laptop-ubuntu"
print("\n\n")
last_event = None
wrong_events = 0
for event in sorted(events, key=lambda e: e.timestamp):
if last_event:
# The diff is the gap between the two events, should be zero
# In reality it is currently sometimes negative and almost always larger than 1s
diff = (event.timestamp - last_event.timestamp) - last_event.duration
print("{} at {}".format(event.label, event.timestamp))
print("Duration: {}".format(event.duration))
if not timedelta(seconds=1) > abs(diff):
class EventExtractor:
def __init__(self):
self.aw_client = ActivityWatchClient()
self.all_buckets = {}
self.bucket = self._get_main_bucket()
self.aw_watcher_window_id = self.bucket['id']
self._all_buckets_events = {}
def _get_main_bucket(self):
if not self.all_buckets:
self.all_buckets = self.aw_client.get_buckets()
return [
bucket for (name, bucket) in self.all_buckets.items()
if 'window' in name
][0]
def _get_events_for_all_buckets(
self,
start_date: datetime.date,
end_date: Optional[datetime.date] = None,
duration: int = 86400,
limit: int = 9999) -> Dict[str, List[Event]]:
"""Gets all events for each bucket (host) within specified time window [start_date, end_date].
If `end_date` is not defined then it returns events based on duration, i.e. [start_date, start_date+duration].
Duration is in seconds so the default value is a day.
"""
if end_date is None:
end_date = start_date + datetime.timedelta(seconds=duration)
# Wipe previous data
self._all_buckets_events = {}
for bucket_name in self.all_buckets:
self._all_buckets_events[bucket_name] = self.aw_client.get_events(
bucket_name, limit=limit, start=start_date, end=end_date)
return self._all_buckets_events
@staticmethod
def _aggregate_events(all_buckets_events) -> List[Event]:
return list(merge_events(*all_buckets_events.values()))
@staticmethod
def _find_overlapping_event(bucket_events, ref_event):
beg_time = ref_event['timestamp']
end_time = beg_time + ref_event['duration']
for event in bucket_events:
if event['timestamp'] >= beg_time and event[
'timestamp'] <= end_time:
return event
return None
@staticmethod
def is_afk(event: Event):
return "data" in event and "status" in event["data"] and event["data"][
"status"] == 'afk'
@staticmethod
def _clean_data(event: Event):
data = event['data']
for key, value in data.items():
data[key] = re.sub(u"\u2013", "-", value)
return event
def get_events_between_dates(self,
start_date: datetime.date,
end_date: datetime.date,
limit: int = 99999) -> List[Dict]:
"""Gets all ActivityWatch events that happened between start_date and end_date.
The number of events is limitted to `limit` to avoid to large overflows.
"""
self._all_buckets_events = self._get_events_for_all_buckets(
start_date=start_date, end_date=end_date, limit=limit)
all_hosts = [
bucket_name.rsplit('_', 1)[1]
for bucket_name in self._all_buckets_events.keys()
if '_' in bucket_name
]
unique_hosts = list(set(all_hosts))
agg_host_events = {}
for host in unique_hosts:
# Aggregate events per host
host_events = {
k: v
for (k, v) in self._all_buckets_events.items() if host in k
}
_agg_host_events = self._aggregate_events(host_events)
# Filter out afk events
not_afk_events = [
event for event in _agg_host_events if not self.is_afk(event)
]
clean_events = [
self._clean_data(event) for event in not_afk_events
]
agg_host_events[host] = clean_events
events = self._aggregate_events(agg_host_events)
return events
def get_events_for_day(self,
start_date: datetime.date,
end_date: Optional[datetime.date] = None,
limit: int = 99999):
self._all_buckets_events = self._get_events_for_all_buckets(
start_date=start_date, end_date=end_date, limit=limit)
hosts = list(
set(_[1] for _ in (key.split("_")
for key in self._all_buckets_events.keys()
if "_" in key)))
agg_host_events = {}
for host in hosts:
# Aggregate events per host
host_events = {
k: v
for (k, v) in self._all_buckets_events.items() if host in k
}
_agg_host_events = self._aggregate_events(host_events)
# Filter out afk events
not_afk_events = [
event for event in _agg_host_events if not self.is_afk(event)
]
clean_events = [
self._clean_data(event) for event in not_afk_events
]
agg_host_events[host] = clean_events
events = self._aggregate_events(agg_host_events)
return events
def get_main_bucket_id(self):
return self.aw_watcher_window_id
def get_all_buckets_events(self):
return self._all_buckets_events
# Usage:
# python3 test_continous_events.py aw-watcher-afk-testing_{hostname}
#
# Example:
# python3 test_continous_events.py aw-watcher-afk-testing_erb-laptop-ubuntu
import sys
from datetime import timedelta
from aw_client import ActivityWatchClient
client = ActivityWatchClient("aw-watcher-afk-test", testing=True)
print(client.get_buckets())
bucket_id = sys.argv[1]
events = client.get_events(bucket_id) # For example "aw-watcher-afk-testing_erb-laptop-ubuntu"
print("\n\n")
last_event = None
wrong_events = 0
for event in sorted(events, key=lambda e: e.timestamp):
if last_event:
# The diff is the gap between the two events, should be zero
# In reality it is currently sometimes negative and almost always larger than 1s
diff = (event.timestamp - last_event.timestamp) - last_event.duration
print("{} at {}".format(event.label, event.timestamp))
print("Duration: {}".format(event.duration))
if not timedelta(seconds=1) > abs(diff):
# Update timestamp for next heartbeat
heartbeat_event.timestamp = datetime.now(timezone.utc)
# Give the dispatcher thread some time to complete sending the last events.
# If we don't do this the events might possibly queue up and be sent the
# next time the client starts instead.
sleep(1)
# Synchronous example, insert an event
event_data = {"label": "non-heartbeat event"}
now = datetime.now(timezone.utc)
event = Event(timestamp=now, data=event_data)
inserted_event = client.insert_event(bucket_id, event)
# The event returned from insert_event has been assigned an id by aw-server
assert inserted_event.id is not None
# Fetch last 10 events from bucket
# Should be two events in order of newest to oldest
# - "shutdown" event with a duration of 0
# - "heartbeat" event with a duration of 5*sleeptime
events = client.get_events(bucket_id=bucket_id, limit=10)
print(events)
# Now lets clean up after us.
# You probably don't want this in your watchers though!
client.delete_bucket(bucket_id)
# If something doesn't work, run aw-server with --verbose to see why some request doesn't go through
# Good luck with writing your own watchers :-)
def test_filter_data() -> None:
awapi = ActivityWatchClient("cleaner", testing=True)
events = awapi.get_events("aw-watcher-web-test", limit=-1)
events = filter_datafields(events, ["title"])
assert "title" not in events[0].data
def copy_bucket_to_influxdb(aw: ActivityWatchClient, bucket_id: str):
print(f"Copying bucket {bucket_id} to InfluxDB")
print("Getting events...")
events = aw.get_events(bucket_id)
print("Sending events...")
send_events_to_influxdb(events, bucket_id)
client.connect()
bucket_name = "test-bucket"
bucket_etype = "test"
# Delete bucket before creating it, and handle error if it doesn't already exist
try:
client.delete_bucket(bucket_name)
except HTTPError as e:
pass
client.create_bucket(bucket_name, bucket_etype)
e1 = create_unique_event()
client.insert_event(bucket_name, e1)
print("Getting events")
events = client.get_events(bucket_name)
print("Asserting events")
assert events[0]['data']['label'] == e1['data']['label']
print("Getting eventcount")
eventcount = client.get_eventcount(bucket_name)
assert eventcount == 1
print("Getting bucket")
buckets = client.get_buckets()
print("Asserting bucket")
assert bucket_name in buckets
assert bucket_name == buckets[bucket_name]['id']
assert bucket_etype == buckets[bucket_name]['type']