def _update_agg_unfollowers(self, mysql: MySQLHelper, cursor: Cursor,
unfollowers: Set[InstaUser], src_user: str,
scrape_ts: datetime, follow_side: str):
"""Update agg table with unfollowers"""
self.logger.info("Updating unfollowers in agg table (found %d)",
len(unfollowers))
ids = ', '.join(
["'{0}'".format(dst_user.user_id) for dst_user in unfollowers])
params = [scrape_ts, src_user]
sql = """
UPDATE {table}
SET {side}_unfollows_latest_timestamp = ?, {side}_follows = 0
WHERE src_user_name = ? and {side}_user_id in ({ids})
""".format(table=self.FOLLOWS_TABLE,
ids=ids,
side=follow_side)
mysql.execute(sql, params, cursor)
self.logger.info("Done updating agg table records")
def main():
with open('/opt/InstaProfiler/logs/user-follows.log') as fp:
txt = fp.read()
matches = LOG_RECORD_REGEX.findall(txt)
all_queries = []
unfollow_users_distinct = set()
for match in matches:
if 'UPDATE follows' in match[5]:
unfollow_params = UNFOLLOW_PARAMS_REGEX.search(match[5])
ts_params = [
unfollow_params.group('year'),
unfollow_params.group('month'),
unfollow_params.group('day'),
unfollow_params.group('hour'),
unfollow_params.group('minute'),
unfollow_params.group('second'),
unfollow_params.group('frac')
]
unfollow_ts = datetime(*[int(x) for x in ts_params])
src_user = unfollow_params.group('src_user')
unfollow_users = UNFOLLOW_USERS_REGEX.search(
match[5]).group('users').split(', ')
if len(unfollow_users) < 8:
for u in unfollow_users:
unfollow_users_distinct.add(u.strip("'"))
query = "UPDATE follows set dst_follows=0, dst_unfollows_latest_timestamp=? where src_user_name=? and dst_user_id in ({users})".format(
users=','.join(unfollow_users))
params = (unfollow_ts, src_user)
print(query)
all_queries.append((query, params))
else:
print("Too much users", len(unfollow_users))
print(','.join(unfollow_users_distinct))
odbc_helper = MySQLHelper('mysql-insta-local')
cursor = odbc_helper.get_cursor()
for query in all_queries:
odbc_helper.execute(query[0], query[1], cursor)
odbc_helper.commit()
odbc_helper.close()