def run(table, check_column, update_column, id_column, app):
p = subprocess.Popen([app], stdin=subprocess.PIPE, stdout=subprocess.PIPE)
while True:
cur.execute(
"select * from %s where %s is distinct from 't' for update skip locked limit 1"
% (table, check_column))
r = cur.fetchone()
if not r:
break
print("Processing row %s..." % r[id_column], end="")
sys.stdout.flush()
p.stdin.write((json.dumps(r, default=handler) + "\n").encode())
p.stdin.flush()
res = p.stdout.readline()
try:
res = json.loads(res)
except:
print(" Fail.")
print(res)
raise
if not update_column in res or not id_column in res:
print(" Fail.")
print(res)
raise Exception("Missing %s key, broken app!" %
(update_column, id_column))
cur.execute(
"update %s set %s = 't', %s = %s where %s = %s" %
(table, check_column, update_column, '%s', id_column, '%s'),
(res[update_column], res[id_column]))
print(" Done.")
conn.commit()
def prepare(table, check_column):
cur.execute(
"SELECT column_name FROM information_schema.columns WHERE table_name=%s and column_name=%s",
(table, check_column))
if not cur.fetchone():
cur.execute("alter table %s add column %s boolean" %
(table, check_column))
conn.commit()
def handle_tweets_response(resp):
if isinstance(resp["result"], list):
for tweet in resp["result"]:
insert_tweet(tweet)
elif "code" in resp["result"] and resp["result"]["code"] == 34: # page does not exist
pass
elif not resp["result"]:
pass
else:
raise Exception("Unknown format.")
conn.commit()
return True
def handle_users_response(response):
for u in response["result"]:
cur.execute("""
insert into users (
uid,
name,
profile_image_url,
location,
created_at,
favourites_count,
utc_offset,
profile_use_background_image,
lang,
followers_count,
protected,
geo_enabled,
description,
verified,
notifications,
time_zone,
statuses_count,
friends_count,
screen_name
) values (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s) on conflict (uid) do nothing
""", (
s(u["id_str"]),
s(u["name"]),
s(u["profile_image_url"]),
u["location"],
u["created_at"],
u["favourites_count"],
u["utc_offset"],
u["profile_use_background_image"],
u["lang"],
u["followers_count"],
u["protected"],
u["geo_enabled"],
s(u["description"]),
u["verified"],
u["notifications"],
u["time_zone"],
u["statuses_count"],
u["friends_count"],
s(u["screen_name"])
))
conn.commit()
return True
def on_message(self, message):
# print ("on message:", message, self.client_id)
log.debug("====message====[%s]===" % (str(message)))
data = json.loads(message)
self.post_msg(self.client_id,
msg_type=data["msgtype"],
message=data["payload"])
### 持久化
sql = "insert into message(roomid, username, msg, msg_type, created_time) \
values('%s', '%s', '%s', '%s', datetime('now'))" % (str(
self.room), data["username"], data["payload"], data["msgtype"])
try:
conn.execute(sql)
conn.commit()
except Exception as e:
print(e)
log.error(e)
def handle_followers_response(res):
if "next_cursor" in res["result"] and res["result"]["next_cursor"] != 0:
command(
"get", "followers/ids", {
"user_id": res["metadata"]["user_id"],
"stringify_ids": True,
"cursor": res["result"]["next_cursor"]
}, "followers", res["metadata"])
if "ids" not in res["result"]:
return True
for follower in res["result"]["ids"]:
nest_level = res["metadata"]["nest_level"] + 1
process_user(follower, res["metadata"]["user_id"],
res["metadata"]["top_level_followee"], nest_level,
nest_level >= int(sys.argv[1]))
conn.commit()
return True
"insert into followers (follower_uid, folowee_uid) values (%s, %s) on conflict (follower_uid, folowee_uid) do nothing",
(uid, follower_of))
if aggregate("users", uid) >= 100:
users = get_aggregate("users", 100)
command("post", "users/lookup", {"user_id": ",".join(users)}, "users")
if not no_followers:
command("get", "followers/ids", {
"user_id": uid,
"stringify_ids": True
}, "followers", {
"user_id": uid,
"top_level_followee": top_level_followee,
"nest_level": nest_level
})
command(
"get", "statuses/user_timeline", {
"user_id": uid,
"trim_user": True,
"count": 200,
"include_rts": True,
"exclude_replies": False
}, "tweets", {"user_id": uid})
if __name__ == "__main__":
if len(sys.argv) < 2:
print("Provide uid!")
sys.exit(2)
process_user(sys.argv[1])
conn.commit()