def interpolate(self, resolution, missing=False, limit=None):
self.logger.debug('Interpolating some data')
result = self.raw.copy()
first, last = math.ceil(
min(self.raw['timestamp']) / resolution) * resolution, math.floor(
max(self.raw['timestamp']) / resolution) * resolution
self.logger.debug("from %s to %s" %
tuple(utils.datetime_from_timestamp([first, last])))
result['timestamp'] = np.arange(first, last, resolution)
result['value'] = np.interp(result['timestamp'], self.raw['timestamp'],
self.raw['value'])
result['datetime'] = utils.datetime_from_timestamp(result['timestamp'])
if missing:
if limit == None: limit = resolution * 2
mymissing = np.array([False] * len(result['timestamp']),
dtype=bool)
mygaps = self.gaps(self.raw['timestamp'], limit)
for g in mygaps:
self.logger.debug("Flagging gap %s as missing" % g)
a = (result['timestamp'] >= g['from']) & (result['timestamp']
<= g['to'])
mymissing[a] = True
result['missing'] = mymissing
result['gaps'] = mygaps
result['interpolated'] = {
'resolution': resolution,
'missing': missing,
'limit': limit
}
return result
def process_comment(comment):
body = comment['body'].lower().strip()
single_trigger_found = False
single_string_found = False
single_date_found = False
if trigger_in_text(body, trigger_single):
single_trigger_found = True
if single_trigger_found:
time_string, target_date = parse_comment(
comment['body'], trigger_single,
utils.datetime_from_timestamp(comment['created_utc']))
if time_string is not None:
single_string_found = True
if target_date is not None:
single_date_found = True
split_trigger_found = False
split_string_found = False
split_date_found = False
if trigger_start_of_line(body, trigger_split):
split_trigger_found = True
if split_trigger_found:
time_string, target_date = parse_comment(
comment['body'], trigger_split,
utils.datetime_from_timestamp(comment['created_utc']))
if time_string is not None:
split_string_found = True
if target_date is not None:
split_date_found = True
return single_trigger_found, single_string_found, single_date_found, split_trigger_found, split_string_found, split_date_found
def user_stats(connection=None, cursor=None):
if connection is None or cursor is None:
connection, cursor = utils.connect_to_db(
"study", cursor_type=psycopg2.extras.DictCursor)
query_string = sql.SQL("""
SELECT COUNT(p.place_id) as nb_places, s.ip_address, s.session_start, s.session_end
FROM places p JOIN sessions s ON s.session_id = p.session_id
GROUP BY s.session_id, s.ip_address, s.session_start, s.session_end;""")
cursor.execute(query_string)
res = []
for record in cursor:
res.append({
'nb_places':
record['nb_places'],
'ip_address':
record['ip_address'],
'start':
'N/A' if record['session_start'] is None else
utils.datetime_from_timestamp(
record['session_start']).strftime("%Y-%m-%d %H:%M:%S"),
'end':
'N/A' if record['session_end'] is None else
utils.datetime_from_timestamp(
record['session_end']).strftime("%Y-%m-%d %H:%M:%S")
})
return res
def process_cakeday_message(message, reddit, database):
log.info("Processing cakeday")
if database.user_has_cakeday_reminder(message.author.name):
log.info("Cakeday already exists")
return ["It looks like you already have a cakeday reminder set."
], False
next_anniversary = utils.get_next_anniversary(message.author.created_utc)
reminder = Reminder(source=utils.message_link(message.id),
message=static.CAKEDAY_MESSAGE,
user=database.get_or_add_user(message.author.name),
requested_date=utils.datetime_from_timestamp(
message.created_utc),
target_date=next_anniversary,
recurrence="1 year",
defaulted=False)
database.add_reminder(reminder)
database.commit()
log.info(
f"Cakeday reminder created: {reminder.id} : {utils.get_datetime_string(reminder.target_date)}"
)
bldr = reminder.render_message_confirmation(
None, pushshift_minutes=reddit.get_effective_pushshift_lag())
return [''.join(bldr)], True
def process_remind_me(message, database):
log.info("Processing RemindMe message")
time = utils.find_reminder_time(message.body)
message_text = utils.find_reminder_message(message.body)
reminder = Reminder(
source=utils.message_link(message.id),
message=message_text,
user=message.author.name,
requested_date=utils.datetime_from_timestamp(message.created_utc),
time_string=time,
timezone=database.get_settings(message.author.name).timezone)
if not reminder.valid:
log.debug("Reminder not valid, returning")
return [reminder.result_message]
if not database.save_reminder(reminder):
log.info("Something went wrong saving the reminder")
return ["Something went wrong saving the reminder"]
log.info(
f"Reminder created: {reminder.db_id} : {utils.get_datetime_string(reminder.target_date)}"
)
return reminder.render_message_confirmation()
def parse_comment(comment, database, count_string):
if comment['author'] == static.ACCOUNT_NAME:
log.debug("Comment is from remindmebot")
return None
log.info(
f"{count_string}: Processing comment {comment['id']} from u/{comment['author']}"
)
body = comment['body'].lower()
if f"{static.TRIGGER_LOWER}!" not in body and f"!{static.TRIGGER_LOWER}" not in body:
log.debug("Command not in comment")
return None
time = utils.find_reminder_time(comment['body'])
message_text = utils.find_reminder_message(comment['body'])
reminder = Reminder(
source=utils.reddit_link(comment['permalink']),
message=message_text,
user=comment['author'],
requested_date=utils.datetime_from_timestamp(comment['created_utc']),
time_string=time,
timezone=database.get_settings(comment['author']).timezone)
if not reminder.valid:
return None
if not database.save_reminder(reminder):
reminder.result_message = "Something went wrong saving the reminder"
reminder.valid = False
log.warning(reminder.result_message)
return reminder
def process_remind_me(message, reddit, database, recurring):
log.info("Processing RemindMe message")
trigger = static.TRIGGER_RECURRING_LOWER if recurring else static.TRIGGER_LOWER
time = utils.find_reminder_time(message.body, trigger)
message_text = utils.find_reminder_message(message.body, trigger)
reminder, result_message = Reminder.build_reminder(
source=utils.message_link(message.id),
message=message_text,
user=database.get_or_add_user(message.author.name),
requested_date=utils.datetime_from_timestamp(message.created_utc),
time_string=time,
recurring=recurring)
if reminder is None:
log.debug("Reminder not valid, returning")
return [result_message], False
database.add_reminder(reminder)
database.commit()
log.info(
f"Reminder created: {reminder.id} : {utils.get_datetime_string(reminder.target_date)}"
)
bldr = reminder.render_message_confirmation(
result_message, pushshift_minutes=reddit.get_effective_pushshift_lag())
return [''.join(bldr)], True
def checkUnreadMessages(workers=10):
lastSeenKey = "messages"
lastSeen = utils.get_last_seen(lastSeenKey)
lastMessageAt = utils.get_last_seen(lastSeenKey, True);
inbox = config.reddit.inbox
pool = ThreadPoolExecutor(max_workers=workers)
messages = list(inbox.unread(limit=None))
for item in messages:
isComment = isinstance(item, Comment)
createdAt = item.created_utc
if lastSeen >= utils.datetime_from_timestamp(createdAt):
break;
if createdAt > lastMessageAt:
logging.info(f"Found new last message: {utils.datetime_from_timestamp(createdAt)}")
lastMessageAt = createdAt;
if item.new:
logging.info(f"Sending unread item to be processed: itemCreatedAt={createdAt}")
pool.submit(actions.processUnreadItem, (item))
if messages:
utils.set_last_seen(lastSeenKey, lastMessageAt)
inbox.mark_read(messages)
return pool
def get_keyword_comments(self, keyword, last_seen):
if not len(self.processed_comments.list):
last_seen = last_seen + timedelta(seconds=1)
log.debug(
f"Fetching comments for keyword: {keyword} : {utils.get_datetime_string(last_seen)}"
)
url = f"https://api.pushshift.io/reddit/comment/search?q={keyword}&limit=100&sort=desc"
try:
json = requests.get(url,
headers={'User-Agent': static.USER_AGENT},
timeout=10)
if json.status_code != 200:
log.warning(
f"Could not parse data for search term: {keyword} status: {str(json.status_code)}"
)
return []
comments = json.json()['data']
if self.timeout_warn_threshold > 1:
log.warning(
f"Recovered from timeouts after {self.consecutive_timeouts} attempts"
)
self.consecutive_timeouts = 0
self.timeout_warn_threshold = 1
except requests.exceptions.ReadTimeout:
self.consecutive_timeouts += 1
if self.consecutive_timeouts >= pow(self.timeout_warn_threshold,
2) * 5:
log.warning(
f"{self.consecutive_timeouts} consecutive timeouts for search term: {keyword}"
)
self.timeout_warn_threshold += 1
return []
except Exception as err:
log.warning(f"Could not parse data for search term: {keyword}")
log.warning(traceback.format_exc())
return []
if not len(comments):
log.warning(f"No comments found for search term: {keyword}")
return []
result_comments = []
for comment in comments:
date_time = utils.datetime_from_timestamp(comment['created_utc'])
if last_seen > date_time:
break
if not self.processed_comments.contains(comment['id']):
result_comments.append(comment)
log.debug(f"Found comments: {len(result_comments)}")
return result_comments
def process_comments(reddit, database):
comments = reddit.get_keyword_comments(
static.TRIGGER_COMBINED,
database_get_seen(database).replace(tzinfo=None))
counters.pushshift_delay.labels(client="prod").set(
reddit.pushshift_prod_client.lag_minutes())
counters.pushshift_delay.labels(client="beta").set(
reddit.pushshift_beta_client.lag_minutes())
counters.pushshift_delay.labels(client="auto").set(
reddit.get_effective_pushshift_lag())
if reddit.recent_pushshift_client == PushshiftType.PROD:
counters.pushshift_client.labels(client="prod").set(1)
counters.pushshift_client.labels(client="beta").set(0)
elif reddit.recent_pushshift_client == PushshiftType.BETA:
counters.pushshift_client.labels(client="prod").set(0)
counters.pushshift_client.labels(client="beta").set(1)
else:
counters.pushshift_client.labels(client="prod").set(0)
counters.pushshift_client.labels(client="beta").set(0)
counters.pushshift_failed.labels(
client="prod").set(1 if reddit.pushshift_prod_client.failed() else 0)
counters.pushshift_failed.labels(
client="beta").set(1 if reddit.pushshift_beta_client.failed() else 0)
counters.pushshift_seconds.labels("prod").observe(
reddit.pushshift_prod_client.request_seconds)
counters.pushshift_seconds.labels("beta").observe(
reddit.pushshift_beta_client.request_seconds)
if len(comments):
log.debug(f"Processing {len(comments)} comments")
i = 0
for comment in comments[::-1]:
i += 1
mark_read = True
try:
process_comment(comment, reddit, database, f"{i}/{len(comments)}")
except Exception as err:
mark_read = not utils.process_error(
f"Error processing comment: {comment['id']} : {comment['author']}",
err, traceback.format_exc())
if mark_read:
reddit.mark_keyword_comment_processed(comment['id'])
database_set_seen(
database,
utils.datetime_from_timestamp(comment['created_utc']))
else:
return i
return len(comments)
def to_dictionary_slow(self):
dictionary = vars(self).copy()
# remove trailing underscores
tmp = {(k[1:] if k.startswith("_") else k): dictionary[k] for k in dictionary.keys()}
dictionary = tmp
del dictionary["orders"]
if "signals" in dictionary:
del dictionary["signals"]
dictionary["strategy"] = dictionary["strategy"].get_short_summary()
dictionary["utilized_signals"] = ", ".join(get_distinct_signal_types(self.order_signals))
dictionary["start_time"] = datetime_from_timestamp(dictionary["start_time"])
dictionary["end_time"] = datetime_from_timestamp(dictionary["end_time"])
dictionary["mean_buy_sell_pair_return"] = self.mean_buy_sell_pair_return
dictionary["transaction_currency"] = self._end_crypto_currency
if "horizon" not in vars(self._strategy):
dictionary["horizon"] = "N/A"
else:
dictionary["horizon"] = self._strategy.horizon.name
if self.end_price == None:
dictionary["profit"] = "N/A"
dictionary["profit_percent"] = "N/A"
dictionary["profit_USDT"] = "N/A"
dictionary["profit_percent_USDT"] = "N/A"
else:
try:
dictionary["profit"] = self.profit
dictionary["profit_percent"] = self.profit_percent
dictionary["profit_USDT"] = self.profit_usdt
dictionary["profit_percent_USDT"] = self.profit_percent_usdt
except NoPriceDataException:
logging.error("No price data!")
dictionary["profit"] = "N/A"
dictionary["profit_percent"] = "N/A"
dictionary["profit_USDT"] = "N/A"
dictionary["profit_percent_USDT"] = "N/A"
return dictionary
def __str__(self):
delta_currency, delta_cash = self.execute()
return "{0} \t {1: <16} \t delta_cash -> {2:13.2f} {3} \t " \
"delta_currency -> {4:13.6f} {5} \t (1 {6} = {7:.8f} {8} {9}), slippage = {10:.2f}". format(
datetime_from_timestamp(self.timestamp),
self.order_type,
delta_cash,
self.counter_currency,
delta_currency,
self.transaction_currency,
self.transaction_currency,
self.unit_price,
self.counter_currency,
" -> delayed trading with delay = {} seconds, original price = {}".
format(self.time_delay, self.original_price) if self.time_delay != 0 else "",
self.slippage
)
def check_curriculum_updated(uid):
"""Checks if the curriculum has updated within `CURR_UPDATE_TIME_LIMIT`\n
Return: `bool`"""
res = db.session.execute(
text('SELECT * FROM Course.curriculum_check WHERE uid=:uid'),
{'uid': uid})
if res.rowcount:
if res.rowcount > 1:
raise RuntimeError('More than one uid existed in curriculum_check')
p = res.fetchone() # person
dt = utils.datetime_from_timestamp(p['time'])
now = utils.datetime_now()
if DISABLE_CURR_UPDATE_TIME_LIMIT:
return False
return now - dt < CURR_UPDATE_TIME_LIMIT
else:
return False
def load_schedule_data_if_not_expired(uid):
"""Load the schedule data in dict if not expired"""
# Get time and time check
user = models.get_user(uid)
# if the schedule needed to update (course insert/delete)
if user['schedule_data_changed']:
db.session.execute(text('UPDATE user SET schedule_data_changed=0 WHERE uid=:uid'), {'uid': uid})
db.session.commit()
return None
# time expired?
last = user['schedule_data_time']
if not last:
return None
last = utils.datetime_from_timestamp(last)
if not Auto_course_insert.DISABLE_TIME_LIMIT and utils.time_limit_check(last, Auto_course_insert.TIME_LIMIT):
d = Auto_course_insert.load_schedule_data(uid)
if d:
return json.loads(d)
return None
def clean(self, data, sd_limit):
"""
The ConsumptionCleaner class filters extreme data based on variations in the rate of consumption
>>> from pyEMIS.DataAccess import adapters
>>> from pyEMIS.DataCleaning import cleaners
>>> import numpy as np
>>> test = adapters.Test()
>>> cleaner = cleaners.ConsumptionCleaner()
>>> cleaner #doctest: +ELLIPSIS
<pyEMIS.DataCleaning.cleaners.ConsumptionCleaner object at 0x...>
>>> valid = test.timeseries('valid')
>>> clean = cleaner.clean(valid, 30)
>>> clean['datetime'] == valid['datetime']
True
>>> np.testing.assert_array_almost_equal(clean['value'][1:], valid['value'][1:])
"""
self.logger.debug('cleaning process started')
clean = data.copy()
ts = utils.timestamp_from_datetime(clean['datetime'])
if ts.size == 0:
raise NoDataError("Can't clean an empty dataset")
value = clean['value']
ts, value = self._remove_invalid_dates(ts, value)
ts, value = self._trimmed_ends(ts, value)
if not clean['integ']:
self.logger.debug('converting to integ')
value = utils.integ_from_movement(value)
ts, value = self._remove_extremes(ts, value, sd_limit)
if not clean['integ']:
self.logger.debug('converting back to movement')
value = utils.movement_from_integ(value)
clean['timestamp'] = ts
clean['datetime'] = utils.datetime_from_timestamp(ts)
clean['value'] = value
clean['cleaned'] = {'sd_limit': sd_limit}
return clean
def process_comments(reddit, database):
comments = reddit.get_keyword_comments(static.TRIGGER_COMBINED,
database_get_seen(database))
if len(comments):
log.debug(f"Processing {len(comments)} comments")
i = 0
for comment in comments[::-1]:
i += 1
try:
process_comment(comment, reddit, database, f"{i}/{len(comments)}")
except Exception:
log.warning(
f"Error processing comment: {comment['id']} : {comment['author']}"
)
log.warning(traceback.format_exc())
reddit.mark_keyword_comment_processed(comment['id'])
database_set_seen(
database, utils.datetime_from_timestamp(comment['created_utc']))
return len(comments)
def process_cakeday_message(message, database):
log.info("Processing cakeday")
if database.get_cakeday(message.author.name) is not None:
log.info("Cakeday already exists")
return ["It looks like you already have a cakeday reminder set."]
account_created = utils.datetime_from_timestamp(message.author.created_utc)
next_anniversary = utils.add_years(
account_created,
utils.datetime_now().year - account_created.year)
if next_anniversary < utils.datetime_now():
next_anniversary = utils.add_years(next_anniversary, 1)
log.debug(
f"u/{message.author.name} created {utils.get_datetime_string(account_created)}, "
f"anniversary {utils.get_datetime_string(next_anniversary)}")
cakeday = Cakeday(message.author.name, next_anniversary)
database.add_cakeday(cakeday)
return cakeday.render_confirmation(
database.get_settings(message.author.name).timezone)
def authored_by(self):
return (
self.transactions[0]['authorPHID'],
datetime_from_timestamp(self.transactions[0]['dateCreated'])
)
def __str__(self):
return ("{} {}-{} strength={} trend={} horizon={} timestamp={} rsi_value={}".format(self.signal_signature, self.transaction_currency, self.counter_currency, self.strength_value,
self.trend, self.horizon, datetime_from_timestamp(self.timestamp), self.rsi_value))
def parse_comment(comment, database, count_string, reddit):
if comment['author'] == static.ACCOUNT_NAME:
log.debug("Comment is from remindmebot")
return None, None
if comment['author'] in static.BLACKLISTED_ACCOUNTS:
log.debug("Comment is from a blacklisted account")
return None, None
log.info(
f"{count_string}: Processing comment {comment['id']} from u/{comment['author']}"
)
body = comment['body'].lower().strip()
recurring = False
cakeday = False
allow_default = True
if trigger_in_text(body, static.TRIGGER_RECURRING_LOWER):
log.debug("Recurring reminder comment")
recurring = True
trigger = static.TRIGGER_RECURRING_LOWER
elif trigger_in_text(body, static.TRIGGER_LOWER):
log.debug("Regular comment")
trigger = static.TRIGGER_LOWER
elif trigger_start_of_line(body, static.TRIGGER_CAKEDAY_LOWER):
log.debug("Cakeday comment")
cakeday = True
recurring = True
trigger = static.TRIGGER_CAKEDAY_LOWER
elif trigger_start_of_line(body, static.TRIGGER_SPLIT_LOWER):
log.debug("Regular split comment")
trigger = static.TRIGGER_SPLIT_LOWER
allow_default = False
else:
log.debug("Command not in comment")
return None, None
target_date = None
if cakeday:
if database.user_has_cakeday_reminder(comment['author']):
log.info("Cakeday already exists")
return None, None
target_date = utils.get_next_anniversary(
reddit.get_user_creation_date(comment['author']))
message_text = static.CAKEDAY_MESSAGE
time = "1 year"
else:
time = utils.find_reminder_time(comment['body'], trigger)
message_text = utils.find_reminder_message(comment['body'], trigger)
reminder, result_message = Reminder.build_reminder(
source=utils.reddit_link(comment['permalink']),
message=message_text,
user=database.get_or_add_user(comment['author']),
requested_date=utils.datetime_from_timestamp(comment['created_utc']),
time_string=time,
recurring=recurring,
target_date=target_date,
allow_default=allow_default)
if reminder is None:
return None, None
if cakeday:
counters.replies.labels(source='comment', type='cake').inc()
elif recurring:
counters.replies.labels(source='comment', type='repeat').inc()
elif not allow_default:
counters.replies.labels(source='comment', type='split').inc()
else:
counters.replies.labels(source='comment', type='single').inc()
database.add_reminder(reminder)
reminder.user.recurring_sent = 0
return reminder, result_message
def get_shifted_forward(self, seconds, name=None):
start_time_str = datetime_from_timestamp(self.start_time + seconds)
end_time_str = datetime_from_timestamp(self.end_time + seconds)
return Period(start_time_str, end_time_str)
objects = json.json()['data']
if len(objects) == 0:
break
for comment in objects:
previousEpoch = comment['created_utc'] - 1
if comment['author'] not in static.BLACKLISTED_ACCOUNTS and comment[
'subreddit'] != "RemindMeBot":
body = comment['body'].lower().strip()
trigger = None
if trigger_in_text(body, trigger):
#log.debug(f"Trigger: https://reddit.com{comment['permalink']}")
trigger_count += 1
elif trigger_start_of_line(body, trigger_split):
time_string, target_date = parse_comment(
comment['body'], trigger_split,
utils.datetime_from_timestamp(comment['created_utc']))
trigger_split_start_count += 1
if time_string is not None:
trigger_split_start_string_count += 1
if target_date is not None:
trigger_split_start_date_count += 1
if time_string is None:
log.debug(
f"Start no string: https://reddit.com{comment['permalink']}"
)
elif target_date is None:
log.debug(
f"Start no date : https://reddit.com{comment['permalink']}"
)
elif trigger_in_text(body, trigger_split):
time_string, target_date = parse_comment(
def schedule_notifications():
import os
import math
import datetime
import calendar
import logging
os.chdir("/home/ucfabb0/code/semantica-docker/src/")
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
datefmt='%m/%d/%Y %I:%M:%S %p',
level=logging.INFO)
def is_visit_confirmed(visit):
return visit['visited'] is not None or visit['deleted']
def is_aggregated_personal_information_reviewed(pi):
return pi['rpi'] > 0 and pi['rpriv'] > 0
def is_personal_information_reviewed(pi):
return pi['r'] > 0
try:
users = user_traces_db.get_all_users_push_ids()
utc_dt = datetime.datetime.utcnow()
utc_time = calendar.timegm(utc_dt.utctimetuple())
number_of_days = 2
days = set()
day = utils.today_string()
today = day
for i in range(number_of_days):
days.add(day)
day = utils.previous_day(day)
count_users = 0
# user_filter = {'cab40ba2-244b-4394-aca1-4b1d87d969df', 'f5eb6ac8-adbc-4d1c-acf3-627d6fa95664',
# '4b2c57ae-5060-464d-9a9a-38a92268910f', '02cd63b6-a58d-4209-a9b4-b733a7fff2ae'}
for user in users:
user_id = user['user_id']
# if user_id not in user_filter:
# continue
has_visits_to_confirm = False
has_places_to_review = False
has_personal_information_to_review = False
last_update_within_48_hours = False
has_join_for_more_than_24_hours = False
# get the last update
last_user_update = user_traces_db.get_last_user_update(user_id)
if not last_user_update:
logging.warning("[%s] no traces available for user" % user_id)
continue
# get the user join date
user_join_date = user_traces_db.get_user_join_date(user_id)
if not user_join_date:
logging.warning("[%s] no join date available" % user_id)
last_update_within_48_hours = math.fabs(last_user_update -
utc_time) <= 96 * 3600
has_join_for_more_than_24_hours = math.fabs(user_join_date -
utc_time) > 24 * 3600
if not last_update_within_48_hours:
logging.warning("[%s] not updated since 48 hours" % user_id)
continue
if not has_join_for_more_than_24_hours:
logging.warning("[%s] not joined since 24 hours" % user_id)
continue
# get the latest utc offset
utc_offset = user_traces_db.get_last_user_utc_offset(user_id)
local_time = utc_time - utc_offset
logging.info("[%s] utc offset: %s, local time: %s" %
(user_id, utc_offset, utils.day_hm(local_time)))
local_dt = utils.datetime_from_timestamp(local_time)
hour = local_dt.hour
if hour != 20:
logging.warning("[%s] not 20:00 local time yet (%s, %s)" %
(user_id, utils.day_hm(local_time), hour))
continue
# visits
visits = user_traces_db.load_user_all_visits(user_id)
if len(visits) > 0:
# if this is around the time to send a notification,
# check if we can send a notification
recent_visits_to_confirm = [
v['visit_id'] for v in visits
if not is_visit_confirmed(v) and v['day'] in days
]
total_number_of_recent_visits_to_confirm = len(
recent_visits_to_confirm)
if total_number_of_recent_visits_to_confirm > 0:
has_visits_to_confirm = True
# personal information per places
personal_information = user_traces_db.load_all_user_personal_information(
user_id)
if len(personal_information) > 0:
personal_information_per_place = {}
for pi in personal_information:
pid = pi['pid']
if pid not in personal_information_per_place:
personal_information_per_place[pid] = []
personal_information_per_place[pid].append(pi)
total_number_of_personal_information = 0
total_number_of_personal_information_to_review = 0
total_number_of_personal_information_reviewed = 0
for pid, pis in personal_information_per_place.items():
number_of_personal_information = len(pis)
personal_information_to_review = [
pi['piid'] for pi in pis
if not is_personal_information_reviewed(pi)
]
personal_information_reviewed = [
pi['piid'] for pi in pis
if is_personal_information_reviewed(pi)
]
total_number_of_personal_information += number_of_personal_information
total_number_of_personal_information_to_review += 1 if len(
personal_information_to_review) > 0 else 0
total_number_of_personal_information_reviewed += 1 if len(
personal_information_reviewed) > 0 else 0
if total_number_of_personal_information_to_review > 0:
has_places_to_review = True
# aggregated personal information
aggregated_personal_information = user_traces_db.load_user_aggregated_personal_information(
user_id)
if len(aggregated_personal_information) > 0:
aggregated_personal_information_to_review = [
pi['piid'] for pi in aggregated_personal_information
if not is_aggregated_personal_information_reviewed(pi)
]
number_of_aggregated_personal_information_to_review = len(
aggregated_personal_information_to_review)
if number_of_aggregated_personal_information_to_review > 0:
has_personal_information_to_review = True
count_users += 1
# prepare the message to send via push notification
logging.info(
"[%s] visits %s / places %s / personal information %s" %
(user_id, has_visits_to_confirm, has_places_to_review,
has_personal_information_to_review))
if has_visits_to_confirm:
logging.info("[%s] Send notification - visits" % user_id)
message = "✅ We have detected new visits! Tap to review them"
type = "timeline"
send_push_notification(user_id,
type,
message,
today,
use_sandbox=True)
send_push_notification(user_id,
type,
message,
today,
use_sandbox=False)
elif has_places_to_review or has_personal_information_to_review:
logging.info("[%s] Send notification - reviews" % user_id)
message = "✅ We have detected new personal information about you! Tap to review them"
type = "review"
send_push_notification(user_id,
type,
message,
use_sandbox=True)
send_push_notification(user_id,
type,
message,
use_sandbox=False)
except:
client.captureException()
logging.info("[Done] processed %s users out of %s total users" %
(count_users, len(users)))
def get(self):
last_seen = utils.get_last_seen(self.keyword)
log.debug(
f"Fetching comments for keyword: {self.keyword} : {last_seen}")
url = f"https://api.pushshift.io/reddit/comment/search?q={self.keyword}&limit=100&sort=desc&fields=created_utc,id"
lag_url = "https://api.pushshift.io/reddit/comment/search?limit=1&sort=desc"
try:
json = requests.get(url,
headers={'User-Agent': config.USER_AGENT},
timeout=10)
if json.status_code != 200:
log.warning(
f"Could not parse data for search term: {self.keyword} status: {str(json.status_code)}"
)
return []
comments = json.json()['data']
if self.pushshift_lag_checked is None or \
utils.datetime_now() - timedelta(minutes=10) > self.pushshift_lag_checked:
log.debug("Updating pushshift comment lag")
json = requests.get(lag_url,
headers={'User-Agent': config.USER_AGENT},
timeout=10)
if json.status_code == 200:
comment_created = utils.datetime_from_timestamp(
json.json()['data'][0]['created_utc'])
self.pushshift_lag = round(
(utils.datetime_now() - comment_created).seconds / 60,
0)
self.pushshift_lag_checked = utils.datetime_now()
if self.timeout_warn_threshold > 1:
log.warning(
f"Recovered from timeouts after {self.consecutive_timeouts} attempts"
)
self.consecutive_timeouts = 0
self.timeout_warn_threshold = 1
except requests.exceptions.ReadTimeout:
self.consecutive_timeouts += 1
if self.consecutive_timeouts >= pow(self.timeout_warn_threshold,
2) * 5:
log.warning(
f"{self.consecutive_timeouts} consecutive timeouts for search term: {self.keyword}"
)
self.timeout_warn_threshold += 1
return []
except Exception as err:
log.warning(
f"Could not parse data for search term: {self.keyword}")
log.warning(traceback.format_exc())
return []
if not len(comments):
log.warning(f"No comments found for search term: {self.keyword}")
return []
result_comments = []
for comment in comments:
date_time = utils.datetime_from_timestamp(comment['created_utc'])
if last_seen > date_time:
break
if not self.processed_comments.contains(comment['id']):
result_comments.append(comment)
return result_comments
log.warning(f"Hit 10 exceptions, giving up")
return None
end_time = utils.parse_datetime_string("2021-01-01 00:00:00")
start_time = utils.parse_datetime_string("2021-04-01 00:00:00")
log.info(
f"Counting comments from {utils.get_datetime_string(start_time, False)} to {utils.get_datetime_string(end_time, False)}, {int((start_time - end_time).total_seconds())} seconds"
)
current_time = start_time
current_count = 0
while current_time > end_time:
current_comments = get_comments(current_time)
if current_comments is None:
break
ingest_delay_seconds = int(
(utils.datetime_from_timestamp(current_comments[0]['retrieved_on']) -
utils.datetime_from_timestamp(
current_comments[0]['created_utc'])).total_seconds())
for comment in current_comments:
comment_time = utils.datetime_from_timestamp(comment['created_utc'])
if comment_time != current_time:
log.info(
f"{utils.get_datetime_string(current_time)} {current_count} {ingest_delay_seconds}"
)
current_count = 0
current_time = current_time - timedelta(minutes=15)
break
current_count += 1
def to_dictionary(self):
dictionary = {
'strategy': self._strategy,
'transaction_currency': self._transaction_currency,
'counter_currency': self._counter_currency,
'start_cash': self._start_cash,
'start_crypto': self._start_crypto,
'start_time': self._start_time,
'end_time': self._end_time,
'source': self._source,
'resample_period': self._resample_period,
'evaluate_profit_on_last_order': self._evaluate_profit_on_last_order,
'transaction_cost_percent': self._transaction_cost_percent,
'benchmark_backtest': self._benchmark_backtest,
'time_delay': self._time_delay,
'slippage': self._slippage,
'order_generator_type': self._order_generator_type,
'cash': self._cash,
'crypto': self._crypto,
'num_trades': self._num_trades,
'num_buys': self._num_buys,
'num_sells': self._num_sells,
'order_signals': self.order_signals,
'trading_df': self.trading_df,
'end_cash': self._end_cash,
'end_crypto': self._end_crypto,
'max_drawdown': self._max_drawdown,
'max_drawdown_duration': self._max_drawdown_duration,
'sharpe_ratio': self._sharpe_ratio,
'orders_df': self.orders_df,
'buy_sell_pair_returns': self._buy_sell_pair_returns,
'buy_sell_pair_gains': self._buy_sell_pair_gains,
'buy_sell_pair_losses': self._buy_sell_pair_losses,
'num_gains': self._num_gains,
'num_losses': self._num_losses,
'alpha': self._alpha,
'beta': self._beta,
}
dictionary["strategy"] = dictionary["strategy"].get_short_summary()
dictionary["utilized_signals"] = ", ".join(get_distinct_signal_types(self.order_signals))
dictionary["start_time"] = datetime_from_timestamp(dictionary["start_time"])
dictionary["end_time"] = datetime_from_timestamp(dictionary["end_time"])
dictionary["mean_buy_sell_pair_return"] = self.mean_buy_sell_pair_return
if self.end_price == None:
dictionary["profit"] = "N/A"
dictionary["profit_percent"] = "N/A"
dictionary["profit_USDT"] = "N/A"
dictionary["profit_percent_USDT"] = "N/A"
else:
try:
dictionary["profit"] = self.profit
dictionary["profit_percent"] = self.profit_percent
dictionary["profit_USDT"] = self.profit_usdt
dictionary["profit_percent_USDT"] = self.profit_percent_usdt
except NoPriceDataException:
logging.error("No price data!")
dictionary["profit"] = "N/A"
dictionary["profit_percent"] = "N/A"
dictionary["profit_USDT"] = "N/A"
dictionary["profit_percent_USDT"] = "N/A"
return dictionary
def get_report(self, include_order_signals=True):
output = []
output.append(str(self._strategy))
# output.append(self.strategy.get_signal_report())
output.append("--")
output.append("\n* Order execution log *\n")
output.append("Start balance: cash = {} {}, crypto = {} {}".format(self._start_cash, self._counter_currency,
self._start_crypto, self._transaction_currency
if self._start_crypto != 0 else ""))
output.append("Start time: {}\n--".format(datetime_from_timestamp(self._start_time)))
output.append("--")
'''
for i, order in enumerate(self.orders):
output.append(str(order))
if include_order_signals and len(self.order_signals) == len(self.orders): # for buy & hold we don't have signals
output.append(" signal: {}".format(self.order_signals[i]))
'''
for i, row in self.orders_df.iterrows():
order = row.order_obj
signal = row.signal_obj
output.append(str(order))
if include_order_signals and signal is not None: # for buy & hold we don't have signals
output.append(" signal: {}".format(signal))
output.append(f' cash: {row.cash} crypto: {row.crypto}')
output.append("End time: {}".format(datetime_from_timestamp(self._end_time)))
output.append("\nSummary")
output.append("--")
output.append("Number of trades: {}".format(self._num_trades))
output.append("End cash: {0:.2f} {1}".format(self.end_cash, self._counter_currency))
output.append("End crypto: {0:.6f} {1}".format(self.end_crypto, self._transaction_currency))
output.append("End price: {}".format(self.end_price))
sign = "+" if self.profit != None and self.profit >= 0 else ""
output.append("Total value invested: {} {}".format(self._format_price_dependent_value(self.start_value),
self._counter_currency))
output.append(
"Total value after investment: {0:.2f} {1} ({2}{3:.2f}%)".format(self._format_price_dependent_value(self.end_value),
self._counter_currency,
sign,
self._format_price_dependent_value(self.profit_percent)))
output.append("Profit: {0:.2f} {1}".format(self._format_price_dependent_value(self.profit), self._counter_currency))
if self._counter_currency != "USDT":
sign = "+" if self.profit_usdt is not None and self.profit_usdt >= 0 else ""
output.append("Total value invested: {:.2f} {} (conversion on {})".format(
self._format_price_dependent_value(self.start_value_usdt),
"USDT",
datetime_from_timestamp(self._start_time)))
output.append(
"Total value after investment: {0:.2f} {1} ({2}{3:.2f}%) (conversion on {4})".format(
self._format_price_dependent_value(self.end_value_usdt), "USDT", sign,
self._format_price_dependent_value(self.profit_percent_usdt),
datetime_from_timestamp(self._end_time)))
output.append("Profit: {0:.2f} {1}".format(self._format_price_dependent_value(self.profit_usdt),
"USDT"))
output.append("\nAdditional stats:")
output.append(" Max drawdown: {}".format(self.max_drawdown))
output.append(" Max drawdown duration: {}".format(self.max_drawdown_duration))
output.append(" Sharpe ratio: {}".format(self.sharpe_ratio))
output.append(" Alpha: {}".format(self.alpha))
output.append(" Beta: {}".format(self.beta))
output.append(" Buy-sell pair gains - overall stats")
output.append(" min = {}, max = {}, mean = {}, stdev = {}".format(
self.min_buy_sell_pair_gain,
self.max_buy_sell_pair_gain,
self.mean_buy_sell_pair_gain,
self.std_buy_sell_pair_gain
))
output.append(" Buy-sell pair losses - overall stats")
output.append(" min = {}, max = {}, mean = {}, stdev = {}".format(
self.min_buy_sell_pair_loss,
self.max_buy_sell_pair_loss,
self.mean_buy_sell_pair_loss,
self.std_buy_sell_pair_loss
))
output.append(" Buy-sell pair returns - overall stats")
output.append(" min = {}, max = {}, mean = {}, stdev = {}".format(
self.min_buy_sell_pair_return,
self.max_buy_sell_pair_return,
self.mean_buy_sell_pair_return,
self.std_buy_sell_pair_return
))
output.append(" Total buy-sell pairs: {}".format(self.num_buy_sell_pairs))
output.append(" Total profitable trades: {}".format(self.num_profitable_trades))
output.append(" Percent profitable trades: {}".format(self.percent_profitable_trades))
output.append(" Percent unprofitable trades: {}".format(self.percent_unprofitable_trades))
return "\n".join(output)
import discord_logging
from datetime import timedelta
import requests
import time
log = discord_logging.init_logging()
import utils
USER_AGENT = "Pushshift tester by u/Watchful1"
BETA_START = utils.datetime_from_timestamp(1622071192)
START_TIME = utils.datetime_now()
PROD_START = None
while True:
url = "https://api.pushshift.io/reddit/comment/search"
comments = requests.get(url,
headers={
'User-Agent': USER_AGENT
},
timeout=10).json()['data']
comment_time = utils.datetime_from_timestamp(comments[0]['created_utc'])
if PROD_START is None:
PROD_START = comment_time
change_from_prod_start = comment_time - PROD_START
seconds_since_start = utils.datetime_now() - START_TIME
ratio = (change_from_prod_start).seconds / (seconds_since_start).seconds
if ratio > 0:
catchup_seconds = (BETA_START - comment_time).seconds / ratio
def committed_by(self):
return [(t['authorPHID'], datetime_from_timestamp(t['dateCreated']))
for t in self.transactions if t['action'] == 'commit']
