def _doApiRequest(self, type, append=""): type_to_update_intervall = { 'playerData': timedelta(days=7), 'alliances': timedelta(days=1), 'players': timedelta(days=1), 'highscore': timedelta(hours=1), } api_data = self.cache.lookup(self.server+"_"+type+append) need_download = False if not api_data: logger.info("Need download because %s is not cached" % self.server+"_"+type) need_download = True else: try: # exception when response is "player not found" timestamp = int(textextract(api_data, 'timestamp="', '"')) except: timestamp = os.path.getmtime(self.cache.get_path(self.server+"_"+type+append)) timestamp = datetime.fromtimestamp(timestamp) if timestamp + type_to_update_intervall[type] < datetime.now(): logger.info("Need download because %s is too old" % (self.server+"_"+type)) need_download = True if need_download: r = self.requests.get('http://'+self.server+'/api/'+type+'.xml'+append) self.cache.write(self.server+"_"+type+append, r.text) api_data = r.text return (need_download, type, append), api_data
def _doApiRequestAsync(self, type, appendList=[]): # also the newest grequests doesn't work with gevent >= 1.x - but don't know how to check this here try: import grequests except: return type_to_update_intervall = { 'playerData': timedelta(days=7), } urlList = [] for append in appendList: api_data = self.cache.lookup(self.server+"_"+type+append) need_download = False if not api_data: logger.info("Need download because %s is not cached") need_download = True else: try: # exception when response is "player not found" timestamp = int(textextract(api_data, 'timestamp="', '"')) except: timestamp = int(time.time()) timestamp = datetime.fromtimestamp(timestamp) if timestamp + type_to_update_intervall[type] < datetime.now(): logger.info("Need download because %s is more than 12h old" % (self.server+"_"+type)) need_download = True if need_download: urlList.append('http://'+self.server+'/api/'+type+'.xml'+append) if len(urlList): rs = (grequests.get(u, session=self.requests, timeout=3) for u in urlList) responses = grequests.map(rs) for r in responses: if r: append = textextract(r.url, type+".xml", "") self.cache.write(self.server+"_"+type+append, r.text)