def from_map(map_id, user_id, cache, replay_id): """ Creates a Replay instance from a replay by the given user on the given map. Args: String map_id: The map_id to download the replay from. String user_id: The user id to download the replay of. Also used as the username of the Replay. Returns: The Replay instance created with the given information. """ lzma_bytes = Loader.replay_data(map_id, user_id) parsed_replay = osrparse.parse_replay(lzma_bytes, pure_lzma=True) replay_data = parsed_replay.play_data if (cache): Cacher.cache(map_id, user_id, lzma_bytes, replay_id) return OnlineReplay(replay_data, user_id)
class Extractor(object): def __init__(self): self._cache_enabled = False self._cache_connection_port = None self._cache_connection_host = None self._cacher = None self._arg = {"country_codes" : ["hrv", "usa"], "indicator_codes" : ["SP.POP.TOTL"], "start_date" : 1980, "end_date" : 2010, "pause" : 0} def arg(self): return copy.deepcopy(self._arg) def normalize(self, arg): if arg==None: arg = self.arg() if "interval" in arg: arg["start_date"], arg["end_date"] = arg["interval"] capitalize_list = lambda elems : [el.upper() for el in elems] arg["country_codes"] = capitalize_list(arg["country_codes"]) return arg def grab(self, arg=None): rca_indicators = [] wb_indicators = [] try: import local_storage.rca.api # Check if indicator is in RCA dataset, if not then it must be in WB. # TODO: Make indicator type checking universal. available_rca_indicators = set(local_storage.rca.api.all_indicators()) for indicator_code in arg["indicator_codes"]: if indicator_code in available_rca_indicators: rca_indicators.append(indicator_code) else: wb_indicators.append(indicator_code) # Grab RCA data. rca_arg = copy.deepcopy(arg) rca_arg["indicator_codes"] = rca_indicators rca_countries = self._grab_from_api(rca_arg, local_storage.rca.api) except ImportError: print("Can't import local_storage.rca.api! Will assume there aren't any indicators from there") rca_countries = [] wb_indicators = arg["indicator_codes"] # Grab WB data. wb_arg = copy.deepcopy(arg) wb_arg["indicator_codes"] = wb_indicators wb_countries = self._grab_from_api(wb_arg, wb.api) #TODO: Separate function for merging two or more list of countries. # Not assuming that wb and rca countries are in corresponding order in lists. all_codes = set() wb_dict = {} for wb_country in wb_countries: all_codes.add(wb_country.code) wb_dict[wb_country.code] = wb_country rca_dict = {} for rca_country in rca_countries: all_codes.add(rca_country.code) rca_dict[rca_country.code] = rca_country combined_countries = [] for code in all_codes: current_countries = [] try: current_countries.append(wb_dict[code]) except KeyError: pass try: current_countries.append(rca_dict[code]) except KeyError: pass if len(current_countries)!=0: if len(current_countries)>1: current_countries[0].merge_with_country(current_countries[1]) combined_countries.append(current_countries[0]) return combined_countries # #TODO: Merge RCA and WB data. # # Assumptions is that rca_countries and wb_countries have elements in same order (maybe write unit test for that). # for wb_country,rca_country in zip(wb_countries,rca_countries): # wb_country.merge_with_country(rca_country) # # return wb_countries def _grab_from_api(self, arg=None, api=wb.api): #TODO: think about using kwargs """ possible args are: 1) the default by leaving None, defined in arg() 2) setting your own such as: {"country_codes" : ["hrv", "usa", "chn"], "indicator_codes" : ["SP.POP.TOTL"], "start_date":1980, "end_date":2010} 3) using interval to set the dates more quickly (overrides start/end_date) {"country_codes" : ["hrv", "usa", "chn"], "indicator_codes" : ["SP.POP.TOTL"], "interval":(1980,2010)} """ arg = self.normalize(arg) if self._cache_enabled: countries = self._cacher.retreive(arg) if countries: self.cache_hit = True else: self.cache_hit = False if not self._cache_enabled or not self.cache_hit: countries = api.query_multiple_data( arg["country_codes"], arg["indicator_codes"], arg["start_date"], arg["end_date"]) if arg["pause"]: time.sleep(arg["pause"]) if self._cache_enabled and not self.cache_hit: self._cache(countries) return countries def grab_metadata(self, what="countries", api = wb.api): """ @param what: string of what to get - 'countries' - list of countries - 'indicators' - list of indicators """ if what=="countries": return api.all_countries() elif what=="indicators": return api.all_indicators() def enable_cache(self, host="localhost", port=27017, test=False): """ @param host: the hostname of the server where mongodb is running @param port: port mongodb is listening on @param test: True if you want to work on a db separate than the main cache """ self._cache_connection_host = host self._cache_connection_port = port self._cache_enabled = True self._cacher = Cacher(self._cache_connection_host, self._cache_connection_port, test) def disable_cache(self): self._cache_enabled = False def clear_cache(self): """ clear everything from the cache """ self._cacher.clear() def _cache(self,countries): if self._cache_enabled: self._cacher.cache(countries) else: return def is_cached(self, arg): arg = self.normalize(arg) countries = self._cacher.retreive(arg) if countries: return True else: return False def was_cached(self): """ tells if the data was found in the cache on the last grab """ return self.cache_hit
class Extractor(object): def __init__(self): self._cache_enabled = False self._cache_connection_port = None self._cache_connection_host = None self._cacher = None self._arg = { "country_codes": ["hrv", "usa"], "indicator_codes": ["SP.POP.TOTL"], "start_date": 1980, "end_date": 2010, "pause": 0 } def arg(self): return copy.deepcopy(self._arg) def normalize(self, arg): if arg == None: arg = self.arg() if "interval" in arg: arg["start_date"], arg["end_date"] = arg["interval"] capitalize_list = lambda elems: [el.upper() for el in elems] arg["country_codes"] = capitalize_list(arg["country_codes"]) return arg def grab(self, arg=None): rca_indicators = [] wb_indicators = [] try: import local_storage.rca.api # Check if indicator is in RCA dataset, if not then it must be in WB. # TODO: Make indicator type checking universal. available_rca_indicators = set( local_storage.rca.api.all_indicators()) for indicator_code in arg["indicator_codes"]: if indicator_code in available_rca_indicators: rca_indicators.append(indicator_code) else: wb_indicators.append(indicator_code) # Grab RCA data. rca_arg = copy.deepcopy(arg) rca_arg["indicator_codes"] = rca_indicators rca_countries = self._grab_from_api(rca_arg, local_storage.rca.api) except ImportError: print( "Can't import local_storage.rca.api! Will assume there aren't any indicators from there" ) rca_countries = [] wb_indicators = arg["indicator_codes"] # Grab WB data. wb_arg = copy.deepcopy(arg) wb_arg["indicator_codes"] = wb_indicators wb_countries = self._grab_from_api(wb_arg, wb.api) #TODO: Separate function for merging two or more list of countries. # Not assuming that wb and rca countries are in corresponding order in lists. all_codes = set() wb_dict = {} for wb_country in wb_countries: all_codes.add(wb_country.code) wb_dict[wb_country.code] = wb_country rca_dict = {} for rca_country in rca_countries: all_codes.add(rca_country.code) rca_dict[rca_country.code] = rca_country combined_countries = [] for code in all_codes: current_countries = [] try: current_countries.append(wb_dict[code]) except KeyError: pass try: current_countries.append(rca_dict[code]) except KeyError: pass if len(current_countries) != 0: if len(current_countries) > 1: current_countries[0].merge_with_country( current_countries[1]) combined_countries.append(current_countries[0]) return combined_countries # #TODO: Merge RCA and WB data. # # Assumptions is that rca_countries and wb_countries have elements in same order (maybe write unit test for that). # for wb_country,rca_country in zip(wb_countries,rca_countries): # wb_country.merge_with_country(rca_country) # # return wb_countries def _grab_from_api(self, arg=None, api=wb.api): #TODO: think about using kwargs """ possible args are: 1) the default by leaving None, defined in arg() 2) setting your own such as: {"country_codes" : ["hrv", "usa", "chn"], "indicator_codes" : ["SP.POP.TOTL"], "start_date":1980, "end_date":2010} 3) using interval to set the dates more quickly (overrides start/end_date) {"country_codes" : ["hrv", "usa", "chn"], "indicator_codes" : ["SP.POP.TOTL"], "interval":(1980,2010)} """ arg = self.normalize(arg) if self._cache_enabled: countries = self._cacher.retreive(arg) if countries: self.cache_hit = True else: self.cache_hit = False if not self._cache_enabled or not self.cache_hit: countries = api.query_multiple_data(arg["country_codes"], arg["indicator_codes"], arg["start_date"], arg["end_date"]) if arg["pause"]: time.sleep(arg["pause"]) if self._cache_enabled and not self.cache_hit: self._cache(countries) return countries def grab_metadata(self, what="countries", api=wb.api): """ @param what: string of what to get - 'countries' - list of countries - 'indicators' - list of indicators """ if what == "countries": return api.all_countries() elif what == "indicators": return api.all_indicators() def enable_cache(self, host="localhost", port=27017, test=False): """ @param host: the hostname of the server where mongodb is running @param port: port mongodb is listening on @param test: True if you want to work on a db separate than the main cache """ self._cache_connection_host = host self._cache_connection_port = port self._cache_enabled = True self._cacher = Cacher(self._cache_connection_host, self._cache_connection_port, test) def disable_cache(self): self._cache_enabled = False def clear_cache(self): """ clear everything from the cache """ self._cacher.clear() def _cache(self, countries): if self._cache_enabled: self._cacher.cache(countries) else: return def is_cached(self, arg): arg = self.normalize(arg) countries = self._cacher.retreive(arg) if countries: return True else: return False def was_cached(self): """ tells if the data was found in the cache on the last grab """ return self.cache_hit
def refresh_request( self, req ): cacher = Cacher( self._tab_data ) cacher.cache() success = True return 'refresh.html', { 'success': success }, None