def get_vector(user, pipe=None):
"""
Given a username, fetch all of the data needed to build a behavior vector
from the database.
:param user: The GitHub username.
:param pipe: (optional) if provided, simply add the requests to the
existing redis pipeline and don't execute the request.
"""
no_pipe = False
if pipe is None:
pipe = get_pipeline()
no_pipe = True
user = user.lower()
pipe.zscore(format_key("user"), user)
pipe.hgetall(format_key("user:{0}:day".format(user)))
pipe.zrevrange(format_key("user:{0}:event".format(user)), 0, -1,
withscores=True)
pipe.zcard(format_key("user:{0}:contribution".format(user)))
pipe.zcard(format_key("user:{0}:connection".format(user)))
pipe.zcard(format_key("user:{0}:repo".format(user)))
pipe.zcard(format_key("user:{0}:lang".format(user)))
pipe.zrevrange(format_key("user:{0}:lang".format(user)), 0, -1,
withscores=True)
if no_pipe:
return pipe.execute()
def rebuild_index():
"""
Rebuild the K-nearest neighbors index based on 50000 of the most active
users (ignoring the top 500 most active).
"""
pipe = get_pipeline()
usernames = pipe.zrevrange(format_key("user"), 500, 50500).execute()[0]
for user in usernames:
get_vector(user, pipe=pipe)
results = pipe.execute()
points = np.zeros([len(usernames), nvector])
for i in range(len(usernames)):
points[i, :] = parse_vector(results[8 * i:8 * (i + 1)])
flann = pyflann.FLANN()
flann.build_index(points)
# Save the index.
fn1 = _h5_filename(index_filename)
tmp1 = fn1 + ".tmp"
flann.save_index(tmp1)
# Save the index coordinates.
fn2 = _h5_filename(points_filename)
tmp2 = fn2 + ".tmp"
with h5py.File(tmp2, "w") as f:
f["points"] = points
f["names"] = usernames
# Atomically move the index files into place.
shutil.move(tmp1, fn1)
shutil.move(tmp2, fn2)
def get_social_stats(username, max_connected=5, max_users=50):
if _is_robot():
return {
"connected_users": [],
"similar_users": [],
"repositories": [],
}
r = get_connection()
pipe = r.pipeline()
user = username.lower()
# Find the connected users.
connection_key = format_key("social:connection:{0}".format(user))
pipe.exists(connection_key)
pipe.zrevrange(connection_key, 0, max_connected-1)
pipe.zrevrange(format_key("social:user:{0}".format(user)), 0, -1,
withscores=True)
flag, connected_users, repos = pipe.execute()
if not _is_robot() and not flag:
[pipe.zrevrange(format_key("social:repo:{0}".format(repo)), 0,
max_users)
for repo, count in repos]
users = pipe.execute()
[pipe.zincrby(connection_key, u, 1) for l in users for u in l
if u != user]
pipe.expire(connection_key, 172800)
pipe.zrevrange(connection_key, 0, max_connected-1)
connected_users = pipe.execute()[-1]
# Get the nearest neighbors in behavior space.
similar_users = get_neighbors(user)
[pipe.get(format_key("user:{0}:name".format(u)))
for u in connected_users + similar_users]
names = pipe.execute()
# Parse all the users.
users = [{"username": u, "name": n.decode("utf-8") if n is not None else u}
for u, n in zip(connected_users+similar_users, names)]
nc = len(connected_users)
return {
"connected_users": users[:nc],
"similar_users": users[nc:],
"repositories": [{"repo": repo, "count": int(count)}
for repo, count in repos[:5] if int(count) > 5],
}
def get_repo_info(username, reponame, maxusers=5, max_recommend=5):
if _is_robot():
return None
# Normalize the repository name.
repo = "{0}/{1}".format(username, reponame)
rkey = format_key("social:repo:{0}".format(repo))
recommend_key = format_key("social:recommend:{0}".format(repo))
# Get the list of users.
pipe = get_pipeline()
pipe.exists(format_key("user:{0}:optout".format(username.lower())))
pipe.exists(rkey)
pipe.exists(recommend_key)
pipe.zrevrange(recommend_key, 0, max_recommend-1)
pipe.zrevrange(rkey, 0, maxusers-1, withscores=True)
flag0, flag1, flag2, recommendations, users = pipe.execute()
if flag0 or not flag1:
return None
if not flag2:
# Compute the repository similarities.
[pipe.zrevrange(format_key("social:user:{0}".format(u)), 0, -1)
for u, count in users]
repos = pipe.execute()
[pipe.zincrby(recommend_key, r, 1) for l in repos for r in l
if r != repo]
pipe.expire(recommend_key, 172800)
pipe.zrevrange(recommend_key, 0, max_recommend-1)
recommendations = pipe.execute()[-1]
# Get the contributor names.
users = [(u, c) for u, c in users if int(c) > 1]
[pipe.get(format_key("user:{0}:name".format(u))) for u, count in users]
names = pipe.execute()
return {
"repository": repo,
"recommendations": recommendations,
"contributors": [{"username": u, "name": n.decode("utf-8")
if n is not None else u,
"count": int(count)}
for (u, count), n in zip(users, names)]
}
def del_connections():
pipe = get_pipeline()
# Get the list of all keys.
keys = pipe.keys(format_key("social:connection:*")).execute()[0]
n = float(len(keys))
print("Found {0:.0f} keys".format(n))
# Loop over the keys and deal with each one.
for i, key in enumerate(keys):
pipe.delete(key)
pipe.execute()
def del_connections():
pipe = get_pipeline()
# Get the list of all keys.
keys = pipe.keys(format_key("social:connection:*")).execute()[0]
n = float(len(keys))
print("Found {0:.0f} keys".format(n))
# Loop over the keys and deal with each one.
for i, key in enumerate(keys):
pipe.delete(key)
pipe.execute()
def opt_out_callback(username):
state1 = flask.session.get("state")
state2 = flask.request.args.get("state")
code = flask.request.args.get("code")
if state1 is None or state2 is None or code is None or state1 != state2:
flask.flash("Couldn't authorize access.")
return flask.redirect(flask.url_for(".opt_out_error",
username=username))
# Get an access token.
params = dict(
client_id=flask.current_app.config["GITHUB_ID"],
client_secret=flask.current_app.config["GITHUB_SECRET"],
code=code,
)
r = requests.post("https://github.com/login/oauth/access_token",
data=params, headers={"Accept": "application/json"})
if r.status_code != requests.codes.ok:
flask.flash("Couldn't acquire an access token from GitHub.")
return flask.redirect(flask.url_for(".opt_out_error",
username=username))
data = r.json()
access = data.get("access_token", None)
if access is None:
flask.flash("No access token returned.")
return flask.redirect(flask.url_for(".opt_out_error",
username=username))
# Check the username.
r = requests.get("https://api.github.com/user",
params={"access_token": access})
if r.status_code != requests.codes.ok:
flask.flash("Couldn't get user information.")
return flask.redirect(flask.url_for(".opt_out_error",
username=username))
data = r.json()
login = data.get("login", None)
if login is None or login.lower() != username.lower():
flask.flash("You have to log in as '{0}' in order to opt-out."
.format(username))
return flask.redirect(flask.url_for(".opt_out_error",
username=username))
# Save the opt-out to the database.
user = username.lower()
get_connection().set(format_key("user:{0}:optout".format(user)), True)
return flask.redirect(flask.url_for(".opt_out_success", username=username))
def _google_geocode(location):
# Check for quota limits.
pipe = get_pipeline()
usage_key = format_key("google_usage_limit")
usage = pipe.get(usage_key).execute()[0]
if usage is not None:
logging.warn("Skipping Google geocode request for usage limits.")
return None
# Submit the request.
params = dict(
address=location,
sensor="false",
key=flask.current_app.config["GOOGLE_KEY"],
)
r = requests.get(goapi_url, params=params)
if r.status_code != requests.codes.ok:
logging.error(r.content)
return None
data = r.json()
# Try not to go over usage limits.
status = data.get("status", None)
if status == "OVER_QUERY_LIMIT":
pipe.set(usage_key, 1).expire(usage_key, 60*60)
pipe.execute()
return None
# Parse the results.
results = data.get("results", [])
if not len(results):
return None
# Find the coordinates.
loc = results[0].get("geometry", {}).get("location", None)
return loc
def get_comparison(user1, user2):
# Normalize the usernames.
user1, user2 = user1.lower(), user2.lower()
# Grab the stats from the database.
pipe = get_pipeline()
pipe.zscore(format_key("user"), user1)
pipe.zscore(format_key("user"), user2)
pipe.zrevrange(format_key("user:{0}:event".format(user1)), 0, -1,
withscores=True)
pipe.zrevrange(format_key("user:{0}:event".format(user2)), 0, -1,
withscores=True)
pipe.zrevrange(format_key("user:{0}:lang".format(user1)), 0, -1,
withscores=True)
pipe.zrevrange(format_key("user:{0}:lang".format(user2)), 0, -1,
withscores=True)
pipe.hgetall(format_key("user:{0}:day".format(user1)))
pipe.hgetall(format_key("user:{0}:day".format(user2)))
raw = pipe.execute()
# Get the total number of events.
total1 = float(raw[0]) if raw[0] is not None else 0
total2 = float(raw[1]) if raw[1] is not None else 0
if not total1:
return "is more active on GitHub"
elif not total2:
return "is less active on GitHub"
# Load the event types from disk.
with flask.current_app.open_resource("event_types.json") as f:
evttypes = json.load(f)
# Compare the fractional event types.
evts1 = dict(raw[2])
evts2 = dict(raw[3])
diffs = []
for e, desc in evttypes.iteritems():
if e in evts1 and e in evts2:
d = float(evts2[e]) / total2 / float(evts1[e]) * total1
if d != 1:
more = "more" if d > 1 else "less"
if d > 1:
d = 1.0 / d
diffs.append((desc.format(more=more, user=user2), d * d))
# Compare language usage.
langs1 = dict(raw[4])
langs2 = dict(raw[5])
for l in set(langs1.keys()) | set(langs2.keys()):
n = float(langs1.get(l, 0)) / total1
d = float(langs2.get(l, 0)) / total2
if n != d and d > 0:
if n > 0:
d = d / n
else:
d = 1.0 / d
more = "more" if d > 1 else "less"
desc = "is {{more}} of a {0} aficionado".format(l)
if d > 1:
d = 1.0 / d
diffs.append((desc.format(more=more), d * d))
# Number of languages.
nl1, nl2 = len(raw[4]), len(raw[5])
if nl1 and nl2:
desc = "speaks {more} languages"
if nl1 > nl2:
diffs.append((desc.format(more="fewer"),
nl2 * nl2 / nl1 / nl1))
else:
diffs.append((desc.format(user=user2, more="more"),
nl1 * nl1 / nl2 / nl2))
# Compare the average weekly schedules.
week1 = map(lambda v: int(v[1]), raw[6].iteritems())
week2 = map(lambda v: int(v[1]), raw[7].iteritems())
mu1, mu2 = sum(week1) / 7.0, sum(week2) / 7.0
var1 = np.sqrt(sum(map(lambda v: (v - mu1) ** 2, week1)) / 7.0) / mu1
var2 = np.sqrt(sum(map(lambda v: (v - mu2) ** 2, week2)) / 7.0) / mu2
if var1 or var2 and var1 != var2:
if var1 > var2:
diffs.append(("has a more consistent weekly schedule", var2/var1))
else:
diffs.append(("has a less consistent weekly schedule", var1/var2))
# Compute the relative probabilities of the comparisons and normalize.
ps = map(lambda v: v[1], diffs)
norm = sum(ps)
# Choose a random description weighted by the probabilities.
return np.random.choice([d[0] for d in diffs], p=[p / norm for p in ps])
def get_usage_stats(username):
user = username.lower()
pipe = get_pipeline()
# Get the total number of events performed by this user.
pipe.zscore(format_key("user"), user)
# The timezone estimate.
pipe.get(format_key("user:{0}:tz".format(user)))
# Get the top <= 5 most common events.
pipe.zrevrangebyscore(format_key("user:{0}:event".format(user)),
"+inf", 0, 0, 5, withscores=True)
# The average daily and weekly schedules.
pipe.hgetall(format_key("user:{0}:hour".format(user)))
pipe.hgetall(format_key("user:{0}:day".format(user)))
# The language stats.
pipe.zrevrange(format_key("user:{0}:lang".format(user)), 0, -1,
withscores=True)
# Parse the results.
results = pipe.execute()
total_events = int(results[0]) if results[0] is not None else 0
if not total_events:
return None
timezone = results[1]
offset = int(timezone) + 8 if timezone is not None else 0
event_counts = results[2]
daily_histogram = make_histogram(results[3].items(), 24, offset)
weekly_histogram = make_histogram(results[4].items(), 7)
languages = results[5]
# Parse the languages into a nicer form and get quantiles.
[(pipe.zcount(format_key("lang:{0}:user".format(l)), 100, "+inf"),
pipe.zrevrank(format_key("lang:{0}:user".format(l)), user))
for l, c in languages]
quants = pipe.execute()
languages = [{"language": l,
"quantile": (min([100, int(100 * float(pos) / tot) + 1])
if tot > 0 and pos is not None
else 100),
"count": int(c)}
for (l, c), tot, pos in zip(languages, quants[::2],
quants[1::2])]
# Generate some stats for the event specific event types.
[(pipe.hgetall(format_key("user:{0}:event:{1}:day".format(user, e))),
pipe.hgetall(format_key("user:{0}:event:{1}:hour".format(user, e))))
for e, c in event_counts]
results = pipe.execute()
events = [{"type": e[0],
"total": int(e[1]),
"week": map(int, make_histogram(w.items(), 7)),
"day": map(int, make_histogram(d.items(), 24, offset))}
for e, w, d in zip(event_counts, results[::2], results[1::2])]
return {
"total": total_events,
"events": events,
"day": map(int, daily_histogram),
"week": map(int, weekly_histogram),
"languages": languages,
}
def get_user_info(username):
# Normalize the username.
user = username.lower()
# Get the cached information.
pipe = get_pipeline()
pipe.get(format_key("user:{0}:name".format(user)))
pipe.get(format_key("user:{0}:etag".format(user)))
pipe.get(format_key("user:{0}:gravatar".format(user)))
pipe.get(format_key("user:{0}:tz".format(user)))
pipe.exists(format_key("user:{0}:optout".format(user)))
name, etag, gravatar, timezone, optout = pipe.execute()
if optout:
return None, True
if name is not None:
name = name.decode("utf-8")
# Return immediately if it's a robot.
if not _is_robot():
# Work out the authentication headers.
auth = {}
client_id = flask.current_app.config.get("GITHUB_ID", None)
client_secret = flask.current_app.config.get("GITHUB_SECRET", None)
if client_id is not None and client_secret is not None:
auth["client_id"] = client_id
auth["client_secret"] = client_secret
# Perform a conditional fetch on the database.
headers = {}
if etag is not None:
headers = {"If-None-Match": etag}
r = requests.get(ghapi_url.format(username=username), params=auth,
headers=headers)
code = r.status_code
if code != 304 and code == requests.codes.ok:
data = r.json()
name = data.get("name") or data.get("login") or username
etag = r.headers["ETag"]
gravatar = data.get("gravatar_id", "none")
location = data.get("location", None)
if location is not None:
tz = estimate_timezone(location)
if tz is not None:
timezone = tz
# Update the cache.
_redis_execute(pipe, "set", "user:{0}:name".format(user), name)
_redis_execute(pipe, "set", "user:{0}:etag".format(user), etag)
_redis_execute(pipe, "set", "user:{0}:gravatar".format(user),
gravatar)
if timezone is not None:
_redis_execute(pipe, "set", "user:{0}:tz".format(user),
timezone)
pipe.execute()
return {
"username": username,
"name": name if name is not None else username,
"gravatar": gravatar if gravatar is not None else "none",
"timezone": int(timezone) if timezone is not None else None,
}, False
def _redis_execute(pipe, cmd, key, *args, **kwargs):
key = format_key(key)
r = getattr(pipe, cmd)(key, *args, **kwargs)
pipe.expire(key, DEFAULT_TTL)
return r
