def forecast(): logging.info("Getting weather forecast") request = requests.get(url_weather) if request.status_code == 429: period_remaining = ( 60 - datetime.datetime.now(datetime.timezone.utc).minute) * 60 raise RateLimitException( 'API response: {}'.format(request.status_code), period_remaining) forecast = request.json() today_iso = datetime.date.today().isoformat() expression = r"{} (\d\d)".format(today_iso) for item in forecast["result"]["watts"].items(): m = re.match(expression, item[0]) if m: watt_hours[int(m.group(1))] = item[1] watt_battery = 0 for hour in range(datetime.datetime.utcnow().hour, 13 + 1): if watt_hours[hour] >= 4600: watt_battery += watt_hours[hour] - 4600 #watt_day = forecast["result"]["watt_hours_day"][today] watt_day = sum(watt_hours) return watt_hours, watt_day, watt_battery
def requests_get(path: str, params: dict = {}): if 'token' not in params.keys(): if HttpApi.token is None or len(HttpApi.token) == 0: HttpApi.input_user_token() params['token'] = HttpApi.token # print(params['token']) result = requests.get(HttpApi.base_url + path, params) if result.status_code != 200: raise ConnectionError(result.reason) json_data = result.json() if 'status' in json_data: status_message = json_data['message'] if status_message == 'Invalid access token': HttpApi.token = None if status_message == '10 requests per second rate limit exceeded': raise RateLimitException(status_message, 1) return json_data
def request(self, *args, **kwargs): """request""" try: return super().request(*args, **kwargs) except HttpError as error: if RATE_LIMIT_ERROR in str(error): raise RateLimitException("too many calls", RATE_LIMIT_PERIOD) raise error
def request(self, method, auth=None, url=None, ignore_http_status_codes=[], **kwargs): if 'headers' not in kwargs: kwargs['headers'] = {} if auth: if auth == 'user' and \ (self.__user_access_token is None or \ self.__user_expires_at <= datetime.utcnow()): self.refresh_user_access_token() elif auth == 'app' and \ (self.__app_access_token is None or \ self.__app_expires_at <= datetime.utcnow()): self.refresh_app_access_token() if auth == 'user': access_token = self.__user_access_token elif auth == 'app': access_token = self.__app_access_token else: raise Exception('Auth mode "{}" not supported'.format(auth)) kwargs['headers']['Authorization'] = 'Bearer {}'.format(access_token) if 'endpoint' in kwargs: endpoint = kwargs['endpoint'] del kwargs['endpoint'] else: endpoint = None if self.__user_agent: kwargs['headers']['User-Agent'] = self.__user_agent url = url.format(account_id=self.__account_id) with metrics.http_request_timer(endpoint) as timer: response = self.__session.request(method, url, **kwargs) timer.tags[metrics.Tag.http_status_code] = response.status_code if response.status_code in ignore_http_status_codes: return None if response.status_code >= 500: raise Server5xxError() if response.status_code == 429: retry_after = int(response.headers.get('retry-after', 15)) message = 'Rate limit hit - 429 - retrying after {} seconds'.format(retry_after) LOGGER.warn(message) raise RateLimitException(message, retry_after) response.raise_for_status() return response.json()
def request(self, method, auth='user', path=None, url=None, **kwargs): if url is None: if auth == 'user' and \ (self.__user_access_token is None or \ self.__user_expires_at <= datetime.utcnow()): self.refresh_user_access_token() elif auth == 'app' and \ (self.__app_access_token is None or \ self.__app_expires_at <= datetime.utcnow()): self.refresh_app_access_token() if url is None and path: url = '{}{}'.format( self.BASE_URL.format(account_id=self.__account_id), path) if 'endpoint' in kwargs: endpoint = kwargs['endpoint'] del kwargs['endpoint'] else: endpoint = None if 'headers' not in kwargs: kwargs['headers'] = {} if auth == 'user': access_token = self.__user_access_token elif auth == 'app': access_token = self.__app_access_token else: raise Exception('Auth mode "{}" not supported'.format(auth)) kwargs['headers']['Authorization'] = 'Bearer {}'.format(access_token) if self.__user_agent: kwargs['headers']['User-Agent'] = self.__user_agent with metrics.http_request_timer(endpoint) as timer: response = self.__session.request(method, url, **kwargs) timer.tags[metrics.Tag.http_status_code] = response.status_code if response.status_code >= 500: raise Server5xxError() if response.status_code == 429: LOGGER.warn('Rate limit hit - 429') raise RateLimitException() response.raise_for_status() return response.json()
def check_response(self, res): if res.status_code != 200: if res.status_code == 429: # This exception will make the process sleep for 1 second raise RateLimitException('Excessive query behavior detected', 1) else: raise Exception(f'ERROR in api call status_code: {res.status_code}, {res.text}') else: if not isinstance(res.json(), dict): raise Exception('ERROR in api response: not a json')
def request(self, method, path=None, url=None, ignore_zoom_error_codes=[], ignore_http_error_codes=[], **kwargs): if url is None and \ self.__use_jwt == False and \ (self.__access_token is None or \ self.__expires_at <= datetime.utcnow()): self.refresh_access_token() if url is None and path: url = '{}{}'.format(self.BASE_URL, path) if 'endpoint' in kwargs: endpoint = kwargs['endpoint'] del kwargs['endpoint'] else: endpoint = None if 'headers' not in kwargs: kwargs['headers'] = {} kwargs['headers']['Authorization'] = 'Bearer {}'.format( self.__access_token) if self.__user_agent: kwargs['headers']['User-Agent'] = self.__user_agent with metrics.http_request_timer(endpoint) as timer: response = self.__session.request(method, url, **kwargs) metrics_status_code = response.status_code if response.status_code in [400, 404 ] and response.status_code < 500: if response.status_code in ignore_http_error_codes or \ (response.status_code == 400 and response.json().get('code') in ignore_zoom_error_codes): metrics_status_code = 200 return None timer.tags[metrics.Tag.http_status_code] = metrics_status_code if response.status_code >= 500: raise Server5xxError() if response.status_code == 429: LOGGER.warn('Rate limit hit - 429') raise RateLimitException() response.raise_for_status() return response.json()
def getSectionDataByCRN(self, term, crn): res = requests.get( "https://selfservice.mypurdue.purdue.edu/prod/bwckschd.p_disp_detail_sched", headers={ "User-Agent": USER_AGENT }, params={ "term_in": term, "crn_in": crn } ) if ERROR_MESSAGE in res.text: # logger.warn("Too many requests to purdue class api. Retrying..") raise RateLimitException("", 1) bsdoc = BeautifulSoup(res.text, "html.parser") courseTitle = bsdoc.find( "th", {"class": "ddlabel", "scope": "row"}).get_text() availTable = bsdoc.find("table", { "summary": "This layout table is used to present the seating numbers."}).find_all("tr") availSeats = availTable[1].find_all("td") availWaitlistSeats = availTable[2].find_all("td") restrictions = str(bsdoc.find( "td", {"class": "dddefault"}).contents[-1]) si = restrictions.find( '<span class="fieldlabeltext">Restrictions:</span>') si += len('<span class="fieldlabeltext">Restrictions:</span>') fi = restrictions.find('<span', si) restrictions = restrictions[si:fi].replace( "\n", "").replace("<br/>", "\n").strip() course_struct = { "title": courseTitle, "restrictions": restrictions, "crn": crn, "seats_capacity": int(availSeats[0].get_text()), "seats_actual": int(availSeats[1].get_text()), "seats_remaining": int(availSeats[2].get_text()), "waitlistseats_capacity": int(availWaitlistSeats[0].get_text()), "waitlistseats_actual": int(availWaitlistSeats[1].get_text()), "waitlistseats_remaining": int(availWaitlistSeats[2].get_text()) } return course_struct
def response_hook(response, *request_args, **request_kwargs): date = factory_kwargs["date"] __Logger.info('fetch record on <{}>'.format(response.url)) if response.status_code == 200: __Logger.info('response status code: {}'.format( response.status_code)) elif response.status_code == 429: raise RateLimitException("api response: 429", 5) else: _Logger.warning('failed to fetch, response status code: {}'.format( response.status_code)) return if response.text == "": return response.encoding = "utf-8" soup = BeautifulSoup(response.text, "lxml") try: main_text = soup.find(name='div', attrs={'class': 'maintext'}) records_table = main_text.find(name='table', attrs={'class': 'mdctable'}) records_trs = records_table.find_all(name='tr')[1:] records = [] for tr in records_trs: tds = tr.find_all(name='td') record = {} record['Film'] = tds[0].get_text().strip() record['Developer'] = tds[1].get_text().strip() record['Dilution'] = tds[2].get_text().strip() record['ASA/ISO'] = tds[3].get_text().strip() record['35mm'] = tds[4].get_text().strip() record['120'] = tds[5].get_text().strip() record['Sheet'] = tds[6].get_text().strip() record['Temp'] = tds[7].get_text().strip() record['Notes'] = tds[8].get_text().strip() records.append(record) with open( 'tmp/record_{}_{}.json'.format(md5hash(response.url), date), 'w') as fd: fd.write(json.dumps(records, indent=4)) except Exception as e: __Logger.error('failed to parse, err: {}'.format(e)) return None
def request(self, method, path, **kwargs): if 'endpoint' in kwargs: endpoint = kwargs['endpoint'] del kwargs['endpoint'] else: endpoint = None if 'headers' not in kwargs: kwargs['headers'] = {} kwargs['headers']['x-api-key'] = self.__token kwargs['headers']['Accept'] = 'application/json' if self.__user_agent: kwargs['headers']['User-Agent'] = self.__user_agent if 'params' not in kwargs: kwargs['params'] = {} kwargs['params']['token'] = self.__token with metrics.http_request_timer(endpoint) as timer: response = self.__session.request(method, self.BASE_URL + path, **kwargs) timer.tags[metrics.Tag.http_status_code] = response.status_code if response.status_code >= 500: raise Server5xxError() elif response.status_code == 429: raise RateLimitException('Rate limit exceeded', 1) response.raise_for_status() try: return response.json() except: LOGGER.info('response.headers["content-type"]: %s', response.headers['content-type']) # This raise lets the backoff retry to kick in raise