def main():
version = time.time()
with LazyIntegrations(logger_iden=LOGGER_IDEN) as itgs:
itgs.logger.print(Level.DEBUG, 'Successfully booted up')
with LazyIntegrations(logger_iden=LOGGER_IDEN) as itgs:
listen_event(itgs, 'loans.request', partial(handle_loan_request, version))
def main():
version = time.time()
with LazyIntegrations(
logger_iden='runners/trust_loan_delays.py#main') as itgs:
itgs.logger.print(Level.DEBUG, 'Successfully booted up')
with LazyIntegrations(
logger_iden='runners/trust_loan_delays.py#main') as itgs:
# Keeps as few connections alive as possible when not working
listen_event(itgs, 'loans.paid', partial(handle_loan_paid, version))
def main():
"""Spawn all of the subprocesses as daemons and then works jobs until one
one them dies or a signal to shutdown is received."""
retry_helper.handle()
subprocs = []
with LazyIntegrations(logger_iden='main.py#main') as itgs:
itgs.logger.print(Level.DEBUG, 'Booting up..')
for modnm in SUBPROCESSES:
itgs.logger.print(Level.TRACE, 'Spawning subprocess {}', modnm)
proc = Process(target=subprocess_runner, name=modnm, args=(modnm,), daemon=True)
proc.start()
subprocs.append(proc)
shutting_down = False
def onexit(*args, **kwargs):
nonlocal shutting_down
if shutting_down:
return
shutting_down = True
try:
with LazyIntegrations(logger_iden='main.py#main#onexit') as itgs:
itgs.logger.print(Level.INFO, 'Shutting down')
finally:
for proc in subprocs:
if proc.is_alive():
proc.terminate()
for proc in subprocs:
proc.join()
atexit.register(onexit)
signal.signal(signal.SIGINT, onexit)
signal.signal(signal.SIGTERM, onexit)
running = True
while running and not shutting_down:
for proc in subprocs:
if not proc.is_alive():
with LazyIntegrations(logger_iden='main.py#main') as itgs:
itgs.logger.print(Level.ERROR, 'A child process has died ({})! Terminating...', proc.name)
running = False
break
if not running:
break
for _ in range(20):
time.sleep(0.5)
if shutting_down:
break
def test_existing(self):
resps = Table('responses')
with LazyIntegrations() as itgs:
itgs.write_cursor.execute(
Query.into(resps).columns(
resps.name, resps.response_body, resps.description).insert(
*[Parameter('%s')
for _ in range(3)]).returning(resps.id).get_sql(),
('my_response', 'I like to {foo} the {bar}', 'Testing desc'))
(respid, ) = itgs.write_cursor.fetchone()
try:
itgs.write_conn.commit()
res: str = responses.get_response(itgs,
'my_response',
foo='open',
bar='door')
self.assertEqual(res, 'I like to open the door')
res: str = responses.get_response(itgs,
'my_response',
foo='eat',
buzz='bear')
self.assertIsInstance(res, str)
self.assertTrue(res.startswith('I like to eat the '), res)
# it's not important how we choose to format the error, but it
# needs the missing key or debugging will be a pain
self.assertIn('bar', res)
finally:
itgs.write_conn.rollback()
itgs.write_cursor.execute(
Query.from_(resps).delete().where(
resps.id == Parameter('%s')).get_sql(), (respid, ))
itgs.write_conn.commit()
def main():
with LazyIntegrations(logger_iden=LOGGER_IDEN) as itgs:
itgs.logger.print(Level.DEBUG, 'Successfully booted up')
while True:
# 8AM UTC = 1AM PST = 4AM EST = presumably off-peak hours
sleep_until_hour_and_minute(8, 0)
update_stats()
def test_cache(self):
with LazyIntegrations() as itgs:
key = 'test_integrations'
val = secrets.token_urlsafe(16).encode('utf-8')
itgs.cache.set(key, val, expire=1)
self.assertEqual(itgs.cache.get(key), val)
itgs.cache.delete(key)
self.assertIsNone(itgs.cache.get(key))
def main():
"""Periodically scans for new links in relevant subreddits."""
version = time.time()
with LazyIntegrations(logger_iden=LOGGER_IDEN) as itgs:
itgs.logger.print(Level.DEBUG, 'Successfully booted up')
while True:
with LazyIntegrations(no_read_only=True,
logger_iden=LOGGER_IDEN) as itgs:
try:
scan_for_links(itgs, version)
except: # noqa
itgs.write_conn.rollback()
itgs.logger.exception(
Level.ERROR, 'Unhandled exception while handling links')
traceback.print_exc()
time.sleep(120)
def main():
version = time.time()
with LazyIntegrations(logger_iden=LOGGER_IDEN) as itgs:
itgs.logger.print(Level.DEBUG, 'Successfully booted up')
while True:
# 1PM UTC = 6AM PST = 9AM EST; at half to avoid conflict
# with deprecated_alerts
sleep_until_hour_and_minute(13, 30)
send_messages(version)
def handle_loan_paid(version, body):
"""Called when we detect a loan was repaid. If there are no more loans unpaid
by the borrower, and the borrower is banned, we unban them.
"""
with LazyIntegrations(logger_iden=LOGGER_IDEN) as itgs:
borrower_username = body['borrower']['username']
borrower_id = body['borrower']['id']
was_unpaid = body['was_unpaid']
itgs.logger.print(Level.TRACE, 'Detected /u/{} repaid a loan',
borrower_username)
if not was_unpaid:
itgs.logger.print(
Level.TRACE,
'Nothing to do about /u/{} repaying a loan - was not unpaid',
borrower_username)
return
info = perms.manager.fetch_info(itgs, borrower_username, RPIDEN,
version)
if not info['borrow_banned']:
itgs.logger.print(
Level.TRACE,
'Nothing to do about /u/{} repaying a loan - not banned',
borrower_username)
return
loans = Table('loans')
itgs.read_cursor.execute(
Query.from_(loans).select(Count(Star())).where(
loans.deleted_at.isnull()).where(
loans.unpaid_at.notnull()).where(
loans.borrower_id == Parameter('%s')).get_sql(),
(borrower_id, ))
(cnt, ) = itgs.read_cursor.fetchone()
if cnt > 0:
itgs.logger.print(
Level.TRACE,
'Nothing to do about /u/{} repaying a loan - still has {} unpaid loans',
borrower_username, cnt)
return
itgs.logger.print(Level.DEBUG,
'Unbanning /u/{} (no more loans unpaid)',
borrower_username)
utils.reddit_proxy.send_request(itgs, RPIDEN, version, 'unban_user', {
'subreddit': 'borrow',
'username': borrower_username
})
perms.manager.flush_cache(itgs, borrower_username.lower())
itgs.logger.print(
Level.INFO, 'Unbanned /u/{} - repaid all outstanding unpaid loans',
borrower_username)
def main():
with LazyIntegrations(
logger_iden='runners/deprecated_alerts.py#main') as itgs:
itgs.logger.print(Level.DEBUG, 'Successfully booted up')
version = time.time()
while True:
# 1pm utc = 6am pst = 9am est -> a good time for sending reddit messages
# we want people to read and process
sleep_until_hour_and_minute(13, 0)
send_messages(version)
def main():
"""Periodically scans for expired temporary bans."""
version = time.time()
logger_iden = 'runners/temp_ban_expired_cache_flush.py#main'
with LazyIntegrations(logger_iden=logger_iden) as itgs:
itgs.logger.print(Level.DEBUG, 'Successfully booted up')
while True:
with LazyIntegrations(no_read_only=True,
logger_iden=logger_iden) as itgs:
try:
scan_for_expired_temp_bans(itgs, version)
except: # noqa
itgs.write_conn.rollback()
itgs.logger.exception(
Level.ERROR,
'Unhandled exception while handling expired temporary bans'
)
traceback.print_exc()
time.sleep(600)
def test_amqp(self):
with LazyIntegrations() as itgs:
itgs.channel.queue_declare('test_integrations')
pub_body = secrets.token_urlsafe(16).encode('utf-8')
itgs.channel.basic_publish(exchange='',
routing_key='test_integrations',
body=pub_body)
for mf, props, body in itgs.channel.consume('test_integrations',
inactivity_timeout=5):
self.assertIsNotNone(mf)
itgs.channel.basic_ack(mf.delivery_tag)
itgs.channel.cancel()
self.assertEqual(body, pub_body)
break
def onexit(*args, **kwargs):
nonlocal shutting_down
if shutting_down:
return
shutting_down = True
try:
with LazyIntegrations(logger_iden='main.py#main#onexit') as itgs:
itgs.logger.print(Level.INFO, 'Shutting down')
finally:
for proc in subprocs:
if proc.is_alive():
proc.terminate()
for proc in subprocs:
proc.join()
def test_kv(self):
with LazyIntegrations() as itgs:
key = 'test_lazy_integrations'
val = secrets.token_urlsafe()
self.assertIsNotNone(itgs.kvs_db)
db = itgs.kvs_conn.database(key)
self.assertTrue(db.create_if_not_exists())
coll = db.collection(key)
self.assertTrue(coll.create_if_not_exists())
self.assertIsNone(coll.create_or_overwrite_doc(key, val))
self.assertEqual(coll.read_doc(key), val)
self.assertTrue(coll.force_delete_doc(key))
self.assertTrue(coll.force_delete())
self.assertTrue(db.force_delete())
def subprocess_runner(name):
"""Runs the given submodule
Arguments:
- `name (str)`: The name of the module to run
"""
mod = importlib.import_module(name)
try:
mod.main()
except: # noqa
with LazyIntegrations(logger_iden='main.py#subprocess_runner') as itgs:
itgs.logger.exception(
Level.WARN,
'Child process {} failed with an unhandled exception',
name
)
def send_messages(version):
alert_executors = (
(execute_get_missing_initial_alerts, 'initial_pm'),
(execute_get_missing_alerts_by_calendar_month, 'reminder'),
(execute_get_missing_alerts_by_urgent, 'reminder'),
)
with LazyIntegrations(no_read_only=True) as itgs:
for alert_executor, alert_type in alert_executors:
alert_executor(itgs)
alerts_grouped_by_user_id = group_alerts_by_user_id(itgs)
unique_endpoint_ids = get_unique_endpoint_ids(
alerts_grouped_by_user_id)
endpoint_info_by_id = get_endpoint_info_by_id(
itgs, tuple(unique_endpoint_ids))
title_message_format, body_message_format = get_letter_message_format(
itgs, alert_type)
send_grouped_alerts(itgs, alerts_grouped_by_user_id,
endpoint_info_by_id, title_message_format,
body_message_format, alert_type, version)
def listen_event_with_itgs(itgs, event_name, handler, keepalive=10):
"""Listen to events on the `"events"` topic exchange which match the given
event name. When they come in, sends them to the `handler` function. Hence
this operates very similarly to `listen_event`, except this also forwards
a `LazyIntegrations` object to `handler`.
It does _not_ forward the itgs object that it receives as an argument.
Rather, it starts up a new `LazyIntegrations` if it does not have one open
when receiving an event and forwards that integrations object to
`handler`. This avoids keeping a database connection open while we are
listening for events but not getting any, which would be wasteful.
The integrations object that we open is reused until `keepalive` seconds pass
without an event, and then we close it. This means that `handler` should be
careful not to depend on implicit rollbacks; it should explicitly rollback.
Arguments:
- `itgs (LazyIntegrations)`: The lazy integrations to use for listening to
the topic exchange. This should be fairly fresh as this function will not
return naturally and hence any connections open in this `itgs` will stay
alive even if they are not needed. Typically this is newly created.
New `LazyIntegration` objects created by this function will copy `itgs`
`logger_iden`.
- `event_name (str)`: The name or pattern for events that should be listened
for. May use a star (*) to substitute for exactly one word and a hash (#)
to substitute for zero or more words.
- `handler (callable)`: A function which we call with `(handler_itgs, event)`
whenever we receive a matching event name. `handler_itgs` is an instance of
`LazyIntegrations` and `event` is the payload of the event interpreted as
utf-8 text and parsed as json.
- `keepalive (int, float)`: If we do not receive an event for `keepalive`
seconds after a previous event we will close the `LazyIntegrations` object
we use for `handler` and will reopen it for the next event.
Returns:
- This function never returns unless an exception occurs, in which case the
exception is logged and this function returns.
"""
itgs.channel.exchange_declare('events', 'topic')
consumer_channel = itgs.amqp.channel()
queue_declare_result = consumer_channel.queue_declare('', exclusive=True)
queue_name = queue_declare_result.method.queue
consumer_channel.queue_bind(queue_name, 'events', event_name)
while True:
consumer = consumer_channel.consume(queue_name,
inactivity_timeout=None)
handler_itgs = None
for method_frame, props, body_bytes in consumer:
break
with LazyIntegrations(logger_iden=itgs.logger_iden) as handler_itgs:
def handle_event():
body_str = body_bytes.decode('utf-8')
body = json.loads(body_str)
try:
handler(handler_itgs, body)
except: # noqa
handler_itgs.logger.exception(Level.ERROR)
consumer_channel.basic_nack(method_frame.delivery_tag,
requeue=False)
return False
consumer_channel.basic_ack(method_frame.delivery_tag)
return True
cont = handle_event()
consumer_channel.cancel()
if not cont:
break
consumer = consumer_channel.consume(queue_name,
inactivity_timeout=keepalive)
for method_frame, props, body_bytes in consumer:
if method_frame is None:
break
cont = handle_event()
if not cont:
break
consumer_channel.cancel()
if not cont:
break
def test_logger(self):
with LazyIntegrations() as itgs:
itgs.logger.print(Level.DEBUG, 'Hello world!')
def main():
"""Listens for requests to recheck comments."""
version = time.time()
with LazyIntegrations(no_read_only=True, logger_iden='runners/rechecks.py#main') as itgs:
itgs.logger.print(Level.DEBUG, 'Successfully booted up')
read_channel = itgs.amqp.channel()
read_channel.queue_declare(QUEUE_NAME)
while True:
for row in read_channel.consume(QUEUE_NAME, inactivity_timeout=3000):
(method_frame, properties, body_bytes) = row
if method_frame is None:
itgs.logger.print(Level.TRACE, 'No rechecks in last 30 minutes; still alive')
continue
body_str = body_bytes.decode('utf-8')
try:
body = json.loads(body_str)
except json.JSONDecodeError as exc:
itgs.logger.exception(
Level.WARN,
(
'Received non-json packet! Error info: '
'doc={}, msg={}, pos={}, lineno={}, colno={}'
),
exc.doc, exc.msg, exc.pos, exc.lineno, exc.colno
)
read_channel.basic_nack(method_frame.delivery_tag, requeue=False)
continue
errors = get_packet_errors(body)
if errors:
itgs.logger.print(
Level.WARN,
'Received packet {} which had {} errors:\n- {}',
body,
len(errors),
'\n- '.join(errors)
)
read_channel.basic_nack(method_frame.delivery_tag, requeue=False)
continue
rp_body = utils.reddit_proxy.send_request(
itgs, 'rechecks', version, 'lookup_comment', {
'link_fullname': body['link_fullname'],
'comment_fullname': body['comment_fullname']
}
)
if rp_body['type'] != 'copy':
itgs.logger.print(
Level.INFO,
'Got unexpected response type {} for comment lookup request '
'during recheck; recheck suppressed (comment might not exist)',
rp_body['type']
)
read_channel.basic_nack(method_frame.delivery_tag, requeue=False)
continue
comment = rp_body['info']
handle_comment(itgs, comment, 'rechecks', version)
read_channel.basic_ack(method_frame.delivery_tag)
def test_database(self):
with LazyIntegrations() as itgs:
itgs.read_cursor.execute('SELECT NOW()')
row = itgs.read_cursor.fetchone()
self.assertIsNotNone(row)
self.assertEqual(len(row), 1)
def handle_loan_paid(version, body):
"""Called when we detect that a loan was repaid. Checks for any loan
delays and, if there are any, checks if they are triggered. If they
are triggered this removes the loan delay and adds the lender to the
trust queue.
Arguments:
- `version (float)`: Our version string when using the reddit proxy.
- `body (dict)`: The event body. Has the following keys:
- `loan_id (int)`: The id of the loan which was just repaid
- `lender (dict)`: The lender for the loan. Has the following keys:
- `id (int)`: The id of the user who lent the money
- `username (str)`: The username of the user who lent the money
- `borrower (dict)`: The borrower for the loan. Has the following keys:
- `id (int)`: The id of the user who borrowed the money
- `username (str)`: The username of the user who borrowed the money
- `amount (dict)`: The total principal of the loan which is now repaid.
Essentially a serialized Money object.
- `was_unpaid (bool)`: True if the loan was unpaid, false if it was not.
"""
with LazyIntegrations(
logger_iden='runners/trust_loan_delays.py#handle_loan_paid',
no_read_only=True) as itgs:
money = Money(**body['amount'])
itgs.logger.print(Level.TRACE,
'Detected a {} loan from /u/{} to /u/{} was repaid',
money, body['lender']['username'],
body['borrower']['username'])
loan_delays = Table('trust_loan_delays')
itgs.read_cursor.execute(
Query.from_(loan_delays).select(
loan_delays.id, loan_delays.loans_completed_as_lender,
loan_delays.min_review_at).where(
loan_delays.user_id == Parameter('%s')).get_sql(),
(body['lender']['id'], ))
row = itgs.read_cursor.fetchone()
if row is None:
itgs.logger.print(Level.TRACE,
'/u/{} has no loan delay - finished',
body['lender']['username'])
return
(loan_delay_id, loan_delay_loans_completed_as_lender,
loan_delay_min_review_at) = row
loans = Table('loans')
itgs.read_cursor.execute(
Query.from_(loans).select(Count(
Star())).where(loans.lender_id == Parameter('%s')).where(
loans.repaid_at.notnull()).get_sql(),
(body['lender']['id'], ))
(num_completed_as_lender, ) = itgs.read_cursor.fetchone()
if num_completed_as_lender < loan_delay_loans_completed_as_lender:
itgs.logger.print(
Level.TRACE,
'/u/{} has a loan delay for {} loans. They are now at {} ' +
'loans; nothing to do.', body['lender']['username'],
loan_delay_loans_completed_as_lender, num_completed_as_lender)
return
usrs = Table('users')
itgs.read_cursor.execute(
Query.from_(usrs).select(
usrs.id).where(usrs.username == Parameter('%s')).get_sql(),
('loansbot', ))
row = itgs.read_cursor.fetchone()
if row is None:
itgs.write_cursor.execute(
Query.into(usrs).columns(usrs.username).insert(
Parameter('%s')).returning(usrs.id).get_sql(),
('loansbot', ))
row = itgs.write_cursor.fetchone()
(loansbot_user_id, ) = row
trust_comments = Table('trust_comments')
itgs.write_cursor.execute(
Query.into(trust_comments).columns(
trust_comments.author_id, trust_comments.target_id,
trust_comments.comment).insert(
*[Parameter('%s') for _ in range(3)]).get_sql(),
(loansbot_user_id, body['lender']['id'],
('/u/{} has reached {}/{} of the loans completed as lender for ' +
'review and has been added back to the trust queue.').format(
body['lender']['username'], num_completed_as_lender,
loan_delay_loans_completed_as_lender)))
itgs.write_cursor.execute(
Query.from_(loan_delays).delete().where(
loan_delays.id == Parameter('%s')).get_sql(),
(loan_delay_id, ))
delqueue.store_event(itgs,
delqueue.QUEUE_TYPES['trust'],
max(datetime.now(), loan_delay_min_review_at),
{'username': body['lender']['username'].lower()},
commit=True)
itgs.logger.print(
Level.INFO,
'/u/{} reached {}/{} of the loan delay loans completed as lender '
+ 'and has been added to the trust queue',
body['lender']['username'], num_completed_as_lender,
loan_delay_loans_completed_as_lender)
def handle_loan_created(version, body):
"""Called whenever we detect that a loan was just created.
Arguments:
- `version (float)`: The version for communicating with the reddit-proxy
- `body (dict)`: The body of the loans.create event
"""
with LazyIntegrations(logger_iden=LOGGER_IDEN) as itgs:
lender_username = body['lender']['username']
borrower_username = body['borrower']['username']
borrower_id = body['borrower']['id']
itgs.logger.print(Level.TRACE,
'Detected that /u/{} received a loan from /u/{}',
borrower_username, lender_username)
loans = Table('loans')
itgs.read_cursor.execute(
Query.from_(loans).select(Count(Star())).where(
loans.deleted_at.isnull()).where(
loans.lender_id == Parameter('%s')).get_sql(),
(borrower_id, ))
(num_as_lender, ) = itgs.read_cursor.fetchone()
if num_as_lender == 0:
itgs.logger.print(Level.TRACE,
'Nothing to do - /u/{} has no loans as lender',
borrower_username)
return
substitutions = {
'lender_username':
lender_username,
'borrower_username':
borrower_username,
'loan_id':
body['loan_id'],
'loans_table':
loan_format_helper.get_and_format_all_or_summary(
itgs, borrower_username)
}
info = perms.manager.fetch_info(itgs, borrower_username, RPIDEN,
version)
if info['borrow_moderator']:
itgs.logger.print(
Level.DEBUG,
'Ignoring that moderator /u/{} received a loan as lender',
borrower_username)
return
if info['borrow_approved_submitter']:
itgs.logger.print(
Level.DEBUG,
'/u/{} - who previously acted as lender - received a loan, '
'but they are on the approved submitter list. Sending a pm but '
'not taking any other action.', borrower_username)
utils.reddit_proxy.send_request(
itgs, RPIDEN, version, 'compose', {
'recipient':
'/r/borrow',
'subject':
get_response(
itgs, 'approved_lender_received_loan_modmail_pm_title',
**substitutions),
'body':
get_response(
itgs, 'approved_lender_received_loan_modmail_pm_body',
**substitutions)
})
return
itgs.logger.print(
Level.DEBUG,
'/u/{} - who has previously acted as a lender - received a loan. '
'Messaging moderators and ensuring they are not in /r/lenderscamp',
borrower_username)
utils.reddit_proxy.send_request(
itgs, RPIDEN, version, 'compose', {
'recipient':
'/r/borrow',
'subject':
get_response(itgs, 'lender_received_loan_modmail_pm_title', **
substitutions),
'body':
get_response(itgs, 'lender_received_loan_modmail_pm_body', **
substitutions)
})
is_approved = utils.reddit_proxy.send_request(
itgs, RPIDEN, version, 'user_is_approved', {
'subreddit': 'lenderscamp',
'username': borrower_username
})
is_moderator = utils.reddit_proxy.send_request(
itgs, RPIDEN, version, 'user_is_moderator', {
'subreddit': 'lenderscamp',
'username': borrower_username
})
if is_moderator:
itgs.logger.print(
Level.DEBUG,
'Removing /u/{} as contributor on /r/lenderscamp suppressed - they are a mod there',
borrower_username)
return
if is_approved:
utils.reddit_proxy.send_request(itgs, RPIDEN, version,
'disapprove_user', {
'subreddit': 'lenderscamp',
'username': borrower_username
})
itgs.logger.print(
Level.INFO,
'Finished alerting about lender-gone-borrower /u/{} and removing from lenderscamp',
borrower_username)
else:
itgs.logger.print(
Level.INFO,
'Alerted /r/borrow about /u/{} receiving a loan. They were '
'already not a contributor to /r/lenderscamp.',
borrower_username)
def handle_loan_request(version, event):
"""Handle a loan request event from the events queue.
Arguments:
version (any): The version to pass to the reddit proxy
event (dict): Describes the request
post (dict):
A self post from reddit-proxy "subreddit_links" (Documented
at reddit-proxy/src/handlers/links.py)
request (dict):
A dictified utils.req_post_interpreter.LoanRequest
"""
post = event['post']
with LazyIntegrations(logger_iden='runners/borrower_request.py#handle_loan_request') as itgs:
itgs.logger.print(
Level.TRACE,
'Detected loan request from /u/{}',
post['author']
)
users = Table('users')
itgs.read_cursor.execute(
users.select(users.id)
.where(users.username == Parameter('%s'))
.get_sql(),
(post['author'].lower(),)
)
row = itgs.read_cursor.fetchone()
if row is None:
itgs.logger.print(
Level.TRACE,
'Ignoring loan request from /u/{} - they do not have any '
+ 'outstanding loans (no history)',
post['author']
)
return
(author_user_id,) = row
loans = Table('loans')
itgs.read_cursor.execute(
loan_format_helper.create_loans_query()
.select(loans.lender_id)
.where(loans.borrower_id == Parameter('%s'))
.where(loans.repaid_at.isnull())
.where(loans.unpaid_at.isnull())
.get_sql(),
(author_user_id,)
)
row = itgs.read_cursor.fetchone()
outstanding_borrowed_loans = []
while row is not None:
outstanding_borrowed_loans.append({
'pretty': loan_format_helper.fetch_loan(row[:-1]),
'lender_id': row[-1]
})
row = itgs.read_cursor.fetchone()
if not outstanding_borrowed_loans:
itgs.logger.print(
Level.TRACE,
'Ignoring loan request from /u/{} - no outstanding loans',
post['author']
)
return
unique_lenders = frozenset(loan['lender_id'] for loan in outstanding_borrowed_loans)
itgs.logger.print(
Level.INFO,
'/u/{} made a loan request while they have {} open loans from '
+ '{} unique lenders: {}. Going to inform each lender which has not '
+ 'opted out of borrower request pms.',
post['author'], len(outstanding_borrowed_loans), len(unique_lenders),
unique_lenders
)
for lender_id in unique_lenders:
lender_settings = get_settings(itgs, lender_id)
if lender_settings.borrower_req_pm_opt_out:
itgs.logger.print(
Level.TRACE,
'Not sending an alert to user {} - opted out',
lender_id
)
continue
pretty_loans = [
loan['pretty']
for loan in outstanding_borrowed_loans
if loan['lender_id'] == lender_id
]
formatted_body = get_response(
itgs,
'borrower_request',
lender_username=pretty_loans[0].lender,
borrower_username=post['author'],
thread='https://www.reddit.com/r/{}/comments/{}/redditloans'.format(
post['subreddit'], post['fullname'][3:]
),
loans=loan_format_helper.format_loan_table(pretty_loans, include_id=True)
)
utils.reddit_proxy.send_request(
itgs, 'borrower_request', version, 'compose',
{
'recipient': pretty_loans[0].lender,
'subject': '/u/{} has made a request thread'.format(post['author']),
'body': formatted_body
}
)
def handle_loan_create(version, event):
"""Handle a loan create event from the events queue.
Arguments:
version (any): The version to pass to the reddit proxy
event (dict): Describes the loan
loan_id (int): The id of the loan that was generated
comment (dict): The comment that generated the loan.
link_fullname (str): The fullname of the link the comment is in
fullname (str): The fullname of the comment
lender (dict): The lender
id (int): The id of the user in our database
username (str): The username for the lender
borrower (dict): The borrower
id (int): The id of the user in our database
username (str): The username for the borrower
amount (dict): The amount of money transfered. Has the same keys as
the Money object has attributes.
minor (int)
currency (int)
exp (int)
symbol (str, None)
symbol_on_left (bool)
permalink (str): A permanent link to the loan.
"""
with LazyIntegrations(
logger_iden='runners/new_lender.py#handle_loan_create') as itgs:
itgs.logger.print(Level.TRACE, 'Detected loan from /u/{} to /u/{}',
event['lender']['username'],
event['borrower']['username'])
amount = Money(**event['amount'])
loans = Table('loans')
itgs.read_cursor.execute(
Query.from_(loans).select(
Count('*')).where(loans.lender_id == Parameter('%s')).where(
loans.id < Parameter('%s')).get_sql(),
(event['lender']['id'], event['loan_id']))
(num_previous_loans, ) = itgs.read_cursor.fetchone()
if num_previous_loans > 0:
itgs.logger.print(
Level.TRACE,
('Ignoring the loan by /u/{} to /u/{} - /u/{} has {} ' +
'previous loans, so they are not new'),
event['lender']['username'], event['borrower']['username'],
event['lender']['username'], num_previous_loans)
return
itgs.logger.print(
Level.INFO,
'/u/{} just made his first loan as lender. Messaging the mods.',
event['lender']['username'])
formatted_body = get_response(
itgs,
'new_lender',
lender_username=event['lender']['username'],
borrower_username=event['borrower']['username'],
amount=amount,
permalink=event['permalink'])
utils.reddit_proxy.send_request(
itgs, 'new_lender', version, 'compose', {
'recipient': '/r/borrow',
'subject': 'New Lender: /u/{}'.format(
event['lender']['username']),
'body': formatted_body
})
def handle_loan_unpaid(version, body):
time.sleep(5) # give the transaction some time to complete
loan_unpaid_event_id = body['loan_unpaid_event_id']
with LazyIntegrations(logger_iden=LOGGER_IDEN) as itgs:
itgs.logger.print(Level.DEBUG, 'Detected loan unpaid event: {}', loan_unpaid_event_id)
loan_unpaid_events = Table('loan_unpaid_events')
loans = Table('loans')
usrs = Table('users')
borrowers = usrs.as_('borrowers')
lenders = usrs.as_('lenders')
itgs.read_cursor.execute(
Query.from_(loan_unpaid_events)
.join(loans).on(loans.id == loan_unpaid_events.loan_id)
.join(borrowers).on(borrowers.id == loans.borrower_id)
.join(lenders).on(lenders.id == loans.lender_id)
.select(borrowers.username, lenders.username)
.where(loan_unpaid_events.id == Parameter('%s'))
.get_sql(),
(loan_unpaid_event_id,)
)
row = itgs.read_cursor.fetchone()
if row is None:
itgs.logger.print(
Level.WARN, 'Loan unpaid event {} did not exist!', loan_unpaid_event_id)
return
(username, lender_username) = row
itgs.logger.print(
Level.TRACE,
'Ensuring /u/{} is moderator or banned from unpaid event {}',
username, loan_unpaid_event_id
)
info = perms.manager.fetch_info(itgs, username, RPIDEN, version)
if info is None:
itgs.logger.print(
Level.INFO,
'/u/{} defaulted on a loan then deleted their account.',
username
)
return
if info['borrow_banned']:
itgs.logger.print(
Level.DEBUG,
'/u/{} defaulted on a loan but they are already banned.',
username
)
return
if info['borrow_moderator']:
itgs.logger.print(
Level.INFO,
'/u/{} defaulted on a loan but is a moderator - no ban',
username
)
return
if info['borrow_approved_submitter']:
itgs.logger.print(
Level.INFO,
'/u/{} defaulted on a loan but is an approved submitter - no ban',
username
)
# easy to forget about approved submitters
utils.reddit_proxy.send_request(
itgs, RPIDEN, version, 'compose',
{
'recipient': '/r/borrow',
'subject': 'Approved Submitter Unpaid Loan',
'body': (
'/u/{} defaulted on a loan but did not get banned since they are '
+ 'an approved submitter.'
).format(username)
}
)
return
itgs.logger.print(
Level.TRACE,
'Banning /u/{} because they defaulted on a loan',
username
)
substitutions = {
'borrower_username': username,
'lender_username': lender_username
}
utils.reddit_proxy.send_request(
itgs, RPIDEN, version, 'ban_user',
{
'subreddit': 'borrow',
'username': username,
'message': get_response(itgs, 'unpaid_ban_message', **substitutions),
'note': get_response(itgs, 'unpaid_ban_note', **substitutions)
}
)
itgs.logger.print(
Level.INFO,
'Banned /u/{} on /r/borrow - failed to repay loan with /u/{}',
username,
lender_username
)
perms.manager.flush_cache(itgs, username)
def update_stats():
the_time = time.time()
with LazyIntegrations(logger_iden=LOGGER_IDEN) as itgs:
plots = {}
for unit in ('count', 'usd'):
plots[unit] = {}
frequency = 'monthly'
frequency_unit = 'month'
plots[unit][frequency] = {
'title': f'{frequency} {unit}'.title(),
'x_axis': frequency_unit.title(),
'y_axis': unit.title(),
'generated_at': the_time,
'data': {
# Categories will be added later
'series': {} # Will be listified later
}
}
for style in ('lent', 'repaid', 'unpaid'):
plots[unit][frequency]['data']['series'][style] = {}
loans = Table('loans')
moneys = Table('moneys')
principals = moneys.as_('principals')
time_parts = {
'month': DatePart('month', loans.created_at),
'year': DatePart('year', loans.created_at)
}
query = (
Query.from_(loans)
.join(principals).on(principals.id == loans.principal_id)
.select(
time_parts['year'], # Which month are we counting?
time_parts['month'], # Which year are we counting?
Count(Star()), # Total # of Loans Lent In Interval
Sum(principals.amount_usd_cents) # Total USD of Loans Lent In Interval
)
.groupby(time_parts['year'], time_parts['month'])
.where(loans.deleted_at.isnull())
)
sql = query.get_sql()
itgs.logger.print(Level.TRACE, sql)
count_series = plots['count']['monthly']['data']['series']['lent']
usd_series = plots['usd']['monthly']['data']['series']['lent']
itgs.read_cursor.execute(sql)
row = itgs.read_cursor.fetchone()
while row is not None:
count_series[(row[0], row[1])] = row[2]
usd_series[(row[0], row[1])] = row[3] / 100
row = itgs.read_cursor.fetchone()
time_parts = {
'month': DatePart('month', loans.repaid_at),
'year': DatePart('year', loans.repaid_at)
}
query = (
Query.from_(loans)
.join(principals).on(principals.id == loans.principal_id)
.select(
time_parts['year'],
time_parts['month'],
Count(Star()),
Sum(principals.amount_usd_cents)
)
.groupby(time_parts['year'], time_parts['month'])
.where(loans.deleted_at.isnull())
.where(loans.repaid_at.notnull())
)
sql = query.get_sql()
itgs.logger.print(Level.TRACE, sql)
count_series = plots['count']['monthly']['data']['series']['repaid']
usd_series = plots['usd']['monthly']['data']['series']['repaid']
itgs.read_cursor.execute(sql)
row = itgs.read_cursor.fetchone()
while row is not None:
count_series[(row[0], row[1])] = row[2]
usd_series[(row[0], row[1])] = row[3] / 100
row = itgs.read_cursor.fetchone()
time_parts = {
'month': DatePart('month', loans.unpaid_at),
'year': DatePart('year', loans.unpaid_at)
}
query = (
Query.from_(loans)
.join(principals).on(principals.id == loans.principal_id)
.select(
time_parts['year'],
time_parts['month'],
Count(Star()),
Sum(principals.amount_usd_cents)
)
.groupby(time_parts['year'], time_parts['month'])
.where(loans.deleted_at.isnull())
.where(loans.unpaid_at.notnull())
)
sql = query.get_sql()
itgs.logger.print(Level.TRACE, sql)
count_series = plots['count']['monthly']['data']['series']['unpaid']
usd_series = plots['usd']['monthly']['data']['series']['unpaid']
itgs.read_cursor.execute(sql)
row = itgs.read_cursor.fetchone()
while row is not None:
count_series[(row[0], row[1])] = row[2]
usd_series[(row[0], row[1])] = row[3] / 100
row = itgs.read_cursor.fetchone()
# We've now fleshed out all the monthly plots. We first standardize the
# series to a categories list and series list, rather than a series dict.
# So series[k]: {"foo": 3, "bar": 2} -> "categories": ["foo", "bar"],
# series[k]: [3, 2]. This introduces time-based ordering
all_keys = set()
for unit_dict in plots.values():
for plot in unit_dict.values():
for series in plot['data']['series'].values():
for key in series.keys():
all_keys.add(key)
categories = sorted(all_keys)
categories_pretty = [f'{int(year)}-{int(month)}' for (year, month) in categories]
for unit_dict in plots.values():
for plot in unit_dict.values():
plot['data']['categories'] = categories_pretty
for key in tuple(plot['data']['series'].keys()):
dict_fmted = plot['data']['series'][key]
plot['data']['series'][key] = [
dict_fmted.get(cat, 0) for cat in categories
]
# We now map series from a dict to a list, moving the key into name
for unit_dict in plots.values():
for plot in unit_dict.values():
plot['data']['series'] = [
{
'name': key.title(),
'data': val
}
for (key, val) in plot['data']['series'].items()
]
# We can now augment monthly to quarterly. 1-3 -> q1, 4-6 -> q2, etc.
def map_month_to_quarter(month):
return int((month - 1) / 3) + 1
quarterly_categories = []
for (year, month) in categories:
quarter = map_month_to_quarter(month)
pretty_quarter = f'{int(year)}Q{quarter}'
if not quarterly_categories or quarterly_categories[-1] != pretty_quarter:
quarterly_categories.append(pretty_quarter)
for unit, unit_dict in plots.items():
monthly_plot = unit_dict['monthly']
quarterly_plot = {
'title': f'Quarterly {unit}'.title(),
'x_axis': 'Quarter',
'y_axis': unit.title(),
'generated_at': the_time,
'data': {
'categories': quarterly_categories,
'series': []
}
}
unit_dict['quarterly'] = quarterly_plot
for series in monthly_plot['data']['series']:
quarterly_series = []
quarterly_plot['data']['series'].append({
'name': series['name'],
'data': quarterly_series
})
last_year_and_quarter = None
for idx, (year, month) in enumerate(categories):
quarter = map_month_to_quarter(month)
year_and_quarter = (year, quarter)
if year_and_quarter == last_year_and_quarter:
quarterly_series[-1] += series['data'][idx]
else:
last_year_and_quarter = year_and_quarter
quarterly_series.append(series['data'][idx])
# And finally we fill caches
for unit, unit_dict in plots.items():
for frequency, plot in unit_dict.items():
cache_key = f'stats/loans/{unit}/{frequency}'
jsonified = json.dumps(plot)
itgs.logger.print(Level.TRACE, '{} -> {}', cache_key, jsonified)
encoded = jsonified.encode('utf-8')
itgs.cache.set(cache_key, encoded)
itgs.logger.print(Level.INFO, 'Successfully updated loans statistics')
def send_messages(version):
with LazyIntegrations(logger_iden=LOGGER_IDEN) as itgs:
itgs.logger.print(Level.TRACE,
'Sending moderator onboarding messages...')
mod_onboarding_messages = Table('mod_onboarding_messages')
itgs.read_cursor.execute(
Query.from_(mod_onboarding_messages).select(
Max(mod_onboarding_messages.msg_order)).get_sql())
(max_msg_order, ) = itgs.read_cursor.fetchone()
if max_msg_order is None:
itgs.logger.print(Level.DEBUG,
'There are no moderator onboarding messages.')
return
mod_onboarding_progress = Table('mod_onboarding_progress')
moderators = Table('moderators')
users = Table('users')
itgs.read_cursor.execute(
Query.from_(moderators).join(users).on(
users.id == moderators.user_id).
left_join(mod_onboarding_progress).on(
mod_onboarding_progress.moderator_id == moderators.id).select(
users.id, moderators.id, users.username,
mod_onboarding_progress.msg_order).where(
mod_onboarding_progress.msg_order.isnull()
| (mod_onboarding_progress.msg_order < Parameter('%s'))
).get_sql(), (max_msg_order, ))
rows = itgs.read_cursor.fetchall()
responses = Table('responses')
titles = responses.as_('titles')
bodies = responses.as_('bodies')
for (user_id, mod_id, username, cur_msg_order) in rows:
itgs.read_cursor.execute(
Query.from_(mod_onboarding_messages).join(titles).on(
titles.id == mod_onboarding_messages.title_id).join(bodies)
.on(bodies.id == mod_onboarding_messages.body_id).select(
mod_onboarding_messages.msg_order, titles.id, titles.name,
bodies.id, bodies.name).where(
Parameter('%s').isnull()
| (mod_onboarding_messages.msg_order > Parameter('%s'))
).orderby(mod_onboarding_messages.msg_order,
order=Order.asc).limit(1).get_sql(), (
cur_msg_order,
cur_msg_order,
))
(new_msg_order, title_id, title_name, body_id,
body_name) = itgs.read_cursor.fetchone()
title_formatted = get_response(itgs, title_name, username=username)
body_formatted = get_response(itgs, body_name, username=username)
utils.reddit_proxy.send_request(
itgs, 'mod_onboarding_messages', version, 'compose', {
'recipient': username,
'subject': title_formatted,
'body': body_formatted
})
utils.mod_onboarding_utils.store_letter_message_with_id_and_names(
itgs, user_id, title_id, title_name, body_id, body_name)
if cur_msg_order is None:
itgs.write_cursor.execute(
Query.into(mod_onboarding_progress).columns(
mod_onboarding_progress.moderator_id,
mod_onboarding_progress.msg_order).insert(
*(Parameter('%s') for _ in range(2))).get_sql(),
(mod_id, new_msg_order))
else:
itgs.write_cursor.execute(
Query.update(mod_onboarding_progress).set(
mod_onboarding_progress.msg_order,
Parameter('%s')).set(
mod_onboarding_progress.updated_at,
CurTimestamp()).where(
mod_onboarding_progress.moderator_id ==
Parameter('%s')).get_sql(),
(new_msg_order, mod_id))
itgs.write_conn.commit()
itgs.logger.print(
Level.INFO,
'Successfully sent moderator onboarding message (msg_order={}) to /u/{}',
new_msg_order, username)
def test_missing(self):
with LazyIntegrations() as itgs:
res = responses.get_response(itgs, 'my_missing_key')
self.assertIsInstance(res, str)
self.assertIn('my_missing_key', res)
