def update_world_location(): options.logger.info("start update world_location") global world_location_map global world_map if not world_location_map: # get file into world_location_map resp = yield GetPage(options.api_url + options.world_location_map_gist) if resp.code == 200: resp = escape.json_decode(resp.body) raw_url = resp["files"]["world_location_map.json"]['raw_url'] resp = yield GetPage(raw_url) if resp.code == 200: world_location_map = escape.json_decode(resp.body) else: options.logger.error("Fetch raw data error %d, %s" % (resp.code, resp.message)) else: options.logger.error("Get gist error %d, %s" % (resp.code, resp.message)) else: # update world_location_map file every hour if int(time.time()) % 36000 < 60: resp = yield update_file( options.api_url + options.world_location_map_gist, "world_location_map.json", world_location_map) if resp.code != 200: options.logger.error("update gists error %d, %s" % (resp.code, resp.message)) temp_world_map = {} for country_code in options.country_code_list: temp_world_map[country_code] = {"score": 0, "staticInitColor": 6} for user in github_world: if not user["location"]: continue try: location = user["location"].strip() location = ','.join( filter(lambda d: d, ','.join(location.split()).split(','))) except Exception, e: options.logger.error("Error: %s" % e) continue if location in world_location_map: country_code = world_location_map[location] else: country_code = yield match_world_geoname(location) if country_code: temp_world_map[country_code]["score"] += 1 world_location_map[location] = country_code else: options.logger.warning("location: %s can't match" % location)
def update_china_location(): global china_location_map global china_map if not china_location_map: # Fetch location_map file resp = yield GetPage(options.api_url + options.location_map_gist) if resp.code == 200: resp = escape.json_decode(resp.body) raw_url = resp['files']['location_map.json']['raw_url'] resp = yield GetPage(raw_url) if resp.code == 200: china_location_map = escape.json_decode(resp.body) else: options.logger.error("Fetch raw data error %d, %s" % (resp.code, resp.message)) else: options.logger.error("Get gist error %d, %s" % (resp.code, resp.message)) else: # update location_map file every hour if int(time.time()) % 36000 < 60: resp = yield update_file( options.api_url + options.location_map_gist, "location_map.json", china_location_map) if resp.code != 200: options.logger.error("update gists error %d, %s" % (resp.code, resp.message)) temp_china_map = {} for city in options.city_list: temp_china_map[city] = {"score": 0, "stateInitColor": 6} for user in github_china: try: location = user["location"].lower() location = ','.join(location.strip().split()) except Exception, e: options.logger.error("lower location error %s" % e) continue # because acording to geoname match china will match to shanghai if location == "china": continue if location in china_location_map: city = china_location_map[location] else: city = yield match_geoname(location) if city: temp_china_map[city]["score"] += 1 china_location_map[location] = city else: options.logger.warning("location: %s can't match" % location)
def contribute(login): resp = yield GetPage(options.contribution_url(login)) if resp.code == 200: resp = escape.json_decode(resp.body) all_contribute = sum([day[1] for day in resp]) else: all_contribute = 0 options.logger.error("fetch contribution error %d, %s" % (resp.code, resp.message)) raise gen.Return(all_contribute)
def search_world(page): url = options.api_url + "/legacy/user/search/followers:>0?start_page=" + str( page) + "&sort=followers&order=desc" options.logger.info("search world %s" % url) resp = yield GetPage(url) raise gen.Return(resp)
def search_china(page): url = options.api_url + "/legacy/user/search/location:china?start_page=" + str( page) + "&sort=followers&order=desc" resp = yield GetPage(url) options.logger.info("search china %s" % url) raise gen.Return(resp)