def update_world_location():
options.logger.info("start update world_location")
global world_location_map
global world_map
if not world_location_map: # get file into world_location_map
resp = yield GetPage(options.api_url + options.world_location_map_gist)
if resp.code == 200:
resp = escape.json_decode(resp.body)
raw_url = resp["files"]["world_location_map.json"]['raw_url']
resp = yield GetPage(raw_url)
if resp.code == 200:
world_location_map = escape.json_decode(resp.body)
else:
options.logger.error("Fetch raw data error %d, %s" %
(resp.code, resp.message))
else:
options.logger.error("Get gist error %d, %s" %
(resp.code, resp.message))
else: # update world_location_map file every hour
if int(time.time()) % 36000 < 60:
resp = yield update_file(
options.api_url + options.world_location_map_gist,
"world_location_map.json", world_location_map)
if resp.code != 200:
options.logger.error("update gists error %d, %s" %
(resp.code, resp.message))
temp_world_map = {}
for country_code in options.country_code_list:
temp_world_map[country_code] = {"score": 0, "staticInitColor": 6}
for user in github_world:
if not user["location"]:
continue
try:
location = user["location"].strip()
location = ','.join(
filter(lambda d: d, ','.join(location.split()).split(',')))
except Exception, e:
options.logger.error("Error: %s" % e)
continue
if location in world_location_map:
country_code = world_location_map[location]
else:
country_code = yield match_world_geoname(location)
if country_code:
temp_world_map[country_code]["score"] += 1
world_location_map[location] = country_code
else:
options.logger.warning("location: %s can't match" % location)
def update_china_location():
global china_location_map
global china_map
if not china_location_map: # Fetch location_map file
resp = yield GetPage(options.api_url + options.location_map_gist)
if resp.code == 200:
resp = escape.json_decode(resp.body)
raw_url = resp['files']['location_map.json']['raw_url']
resp = yield GetPage(raw_url)
if resp.code == 200:
china_location_map = escape.json_decode(resp.body)
else:
options.logger.error("Fetch raw data error %d, %s" %
(resp.code, resp.message))
else:
options.logger.error("Get gist error %d, %s" %
(resp.code, resp.message))
else: # update location_map file every hour
if int(time.time()) % 36000 < 60:
resp = yield update_file(
options.api_url + options.location_map_gist,
"location_map.json", china_location_map)
if resp.code != 200:
options.logger.error("update gists error %d, %s" %
(resp.code, resp.message))
temp_china_map = {}
for city in options.city_list:
temp_china_map[city] = {"score": 0, "stateInitColor": 6}
for user in github_china:
try:
location = user["location"].lower()
location = ','.join(location.strip().split())
except Exception, e:
options.logger.error("lower location error %s" % e)
continue
# because acording to geoname match china will match to shanghai
if location == "china":
continue
if location in china_location_map:
city = china_location_map[location]
else:
city = yield match_geoname(location)
if city:
temp_china_map[city]["score"] += 1
china_location_map[location] = city
else:
options.logger.warning("location: %s can't match" % location)
def contribute(login):
resp = yield GetPage(options.contribution_url(login))
if resp.code == 200:
resp = escape.json_decode(resp.body)
all_contribute = sum([day[1] for day in resp])
else:
all_contribute = 0
options.logger.error("fetch contribution error %d, %s" %
(resp.code, resp.message))
raise gen.Return(all_contribute)
def search_world(page):
url = options.api_url + "/legacy/user/search/followers:>0?start_page=" + str(
page) + "&sort=followers&order=desc"
options.logger.info("search world %s" % url)
resp = yield GetPage(url)
raise gen.Return(resp)
def search_china(page):
url = options.api_url + "/legacy/user/search/location:china?start_page=" + str(
page) + "&sort=followers&order=desc"
resp = yield GetPage(url)
options.logger.info("search china %s" % url)
raise gen.Return(resp)