def forecast():
logging.info("Getting weather forecast")
request = requests.get(url_weather)
if request.status_code == 429:
period_remaining = (
60 - datetime.datetime.now(datetime.timezone.utc).minute) * 60
raise RateLimitException(
'API response: {}'.format(request.status_code), period_remaining)
forecast = request.json()
today_iso = datetime.date.today().isoformat()
expression = r"{} (\d\d)".format(today_iso)
for item in forecast["result"]["watts"].items():
m = re.match(expression, item[0])
if m:
watt_hours[int(m.group(1))] = item[1]
watt_battery = 0
for hour in range(datetime.datetime.utcnow().hour, 13 + 1):
if watt_hours[hour] >= 4600:
watt_battery += watt_hours[hour] - 4600
#watt_day = forecast["result"]["watt_hours_day"][today]
watt_day = sum(watt_hours)
return watt_hours, watt_day, watt_battery
def requests_get(path: str, params: dict = {}):
if 'token' not in params.keys():
if HttpApi.token is None or len(HttpApi.token) == 0:
HttpApi.input_user_token()
params['token'] = HttpApi.token
# print(params['token'])
result = requests.get(HttpApi.base_url + path, params)
if result.status_code != 200:
raise ConnectionError(result.reason)
json_data = result.json()
if 'status' in json_data:
status_message = json_data['message']
if status_message == 'Invalid access token':
HttpApi.token = None
if status_message == '10 requests per second rate limit exceeded':
raise RateLimitException(status_message, 1)
return json_data
def request(self, *args, **kwargs):
"""request"""
try:
return super().request(*args, **kwargs)
except HttpError as error:
if RATE_LIMIT_ERROR in str(error):
raise RateLimitException("too many calls", RATE_LIMIT_PERIOD)
raise error
def request(self,
method,
auth=None,
url=None,
ignore_http_status_codes=[],
**kwargs):
if 'headers' not in kwargs:
kwargs['headers'] = {}
if auth:
if auth == 'user' and \
(self.__user_access_token is None or \
self.__user_expires_at <= datetime.utcnow()):
self.refresh_user_access_token()
elif auth == 'app' and \
(self.__app_access_token is None or \
self.__app_expires_at <= datetime.utcnow()):
self.refresh_app_access_token()
if auth == 'user':
access_token = self.__user_access_token
elif auth == 'app':
access_token = self.__app_access_token
else:
raise Exception('Auth mode "{}" not supported'.format(auth))
kwargs['headers']['Authorization'] = 'Bearer {}'.format(access_token)
if 'endpoint' in kwargs:
endpoint = kwargs['endpoint']
del kwargs['endpoint']
else:
endpoint = None
if self.__user_agent:
kwargs['headers']['User-Agent'] = self.__user_agent
url = url.format(account_id=self.__account_id)
with metrics.http_request_timer(endpoint) as timer:
response = self.__session.request(method, url, **kwargs)
timer.tags[metrics.Tag.http_status_code] = response.status_code
if response.status_code in ignore_http_status_codes:
return None
if response.status_code >= 500:
raise Server5xxError()
if response.status_code == 429:
retry_after = int(response.headers.get('retry-after', 15))
message = 'Rate limit hit - 429 - retrying after {} seconds'.format(retry_after)
LOGGER.warn(message)
raise RateLimitException(message, retry_after)
response.raise_for_status()
return response.json()
def request(self,
method,
auth='user',
path=None,
url=None,
**kwargs):
if url is None:
if auth == 'user' and \
(self.__user_access_token is None or \
self.__user_expires_at <= datetime.utcnow()):
self.refresh_user_access_token()
elif auth == 'app' and \
(self.__app_access_token is None or \
self.__app_expires_at <= datetime.utcnow()):
self.refresh_app_access_token()
if url is None and path:
url = '{}{}'.format(
self.BASE_URL.format(account_id=self.__account_id),
path)
if 'endpoint' in kwargs:
endpoint = kwargs['endpoint']
del kwargs['endpoint']
else:
endpoint = None
if 'headers' not in kwargs:
kwargs['headers'] = {}
if auth == 'user':
access_token = self.__user_access_token
elif auth == 'app':
access_token = self.__app_access_token
else:
raise Exception('Auth mode "{}" not supported'.format(auth))
kwargs['headers']['Authorization'] = 'Bearer {}'.format(access_token)
if self.__user_agent:
kwargs['headers']['User-Agent'] = self.__user_agent
with metrics.http_request_timer(endpoint) as timer:
response = self.__session.request(method, url, **kwargs)
timer.tags[metrics.Tag.http_status_code] = response.status_code
if response.status_code >= 500:
raise Server5xxError()
if response.status_code == 429:
LOGGER.warn('Rate limit hit - 429')
raise RateLimitException()
response.raise_for_status()
return response.json()
def check_response(self, res):
if res.status_code != 200:
if res.status_code == 429:
# This exception will make the process sleep for 1 second
raise RateLimitException('Excessive query behavior detected', 1)
else:
raise Exception(f'ERROR in api call status_code: {res.status_code}, {res.text}')
else:
if not isinstance(res.json(), dict):
raise Exception('ERROR in api response: not a json')
def request(self,
method,
path=None,
url=None,
ignore_zoom_error_codes=[],
ignore_http_error_codes=[],
**kwargs):
if url is None and \
self.__use_jwt == False and \
(self.__access_token is None or \
self.__expires_at <= datetime.utcnow()):
self.refresh_access_token()
if url is None and path:
url = '{}{}'.format(self.BASE_URL, path)
if 'endpoint' in kwargs:
endpoint = kwargs['endpoint']
del kwargs['endpoint']
else:
endpoint = None
if 'headers' not in kwargs:
kwargs['headers'] = {}
kwargs['headers']['Authorization'] = 'Bearer {}'.format(
self.__access_token)
if self.__user_agent:
kwargs['headers']['User-Agent'] = self.__user_agent
with metrics.http_request_timer(endpoint) as timer:
response = self.__session.request(method, url, **kwargs)
metrics_status_code = response.status_code
if response.status_code in [400, 404
] and response.status_code < 500:
if response.status_code in ignore_http_error_codes or \
(response.status_code == 400 and response.json().get('code') in ignore_zoom_error_codes):
metrics_status_code = 200
return None
timer.tags[metrics.Tag.http_status_code] = metrics_status_code
if response.status_code >= 500:
raise Server5xxError()
if response.status_code == 429:
LOGGER.warn('Rate limit hit - 429')
raise RateLimitException()
response.raise_for_status()
return response.json()
def getSectionDataByCRN(self, term, crn):
res = requests.get(
"https://selfservice.mypurdue.purdue.edu/prod/bwckschd.p_disp_detail_sched",
headers={
"User-Agent": USER_AGENT
},
params={
"term_in": term,
"crn_in": crn
}
)
if ERROR_MESSAGE in res.text:
# logger.warn("Too many requests to purdue class api. Retrying..")
raise RateLimitException("", 1)
bsdoc = BeautifulSoup(res.text, "html.parser")
courseTitle = bsdoc.find(
"th", {"class": "ddlabel", "scope": "row"}).get_text()
availTable = bsdoc.find("table", {
"summary": "This layout table is used to present the seating numbers."}).find_all("tr")
availSeats = availTable[1].find_all("td")
availWaitlistSeats = availTable[2].find_all("td")
restrictions = str(bsdoc.find(
"td", {"class": "dddefault"}).contents[-1])
si = restrictions.find(
'<span class="fieldlabeltext">Restrictions:</span>')
si += len('<span class="fieldlabeltext">Restrictions:</span>')
fi = restrictions.find('<span', si)
restrictions = restrictions[si:fi].replace(
"\n", "").replace("<br/>", "\n").strip()
course_struct = {
"title": courseTitle,
"restrictions": restrictions,
"crn": crn,
"seats_capacity": int(availSeats[0].get_text()),
"seats_actual": int(availSeats[1].get_text()),
"seats_remaining": int(availSeats[2].get_text()),
"waitlistseats_capacity": int(availWaitlistSeats[0].get_text()),
"waitlistseats_actual": int(availWaitlistSeats[1].get_text()),
"waitlistseats_remaining": int(availWaitlistSeats[2].get_text())
}
return course_struct
def response_hook(response, *request_args, **request_kwargs):
date = factory_kwargs["date"]
__Logger.info('fetch record on <{}>'.format(response.url))
if response.status_code == 200:
__Logger.info('response status code: {}'.format(
response.status_code))
elif response.status_code == 429:
raise RateLimitException("api response: 429", 5)
else:
_Logger.warning('failed to fetch, response status code: {}'.format(
response.status_code))
return
if response.text == "":
return
response.encoding = "utf-8"
soup = BeautifulSoup(response.text, "lxml")
try:
main_text = soup.find(name='div', attrs={'class': 'maintext'})
records_table = main_text.find(name='table',
attrs={'class': 'mdctable'})
records_trs = records_table.find_all(name='tr')[1:]
records = []
for tr in records_trs:
tds = tr.find_all(name='td')
record = {}
record['Film'] = tds[0].get_text().strip()
record['Developer'] = tds[1].get_text().strip()
record['Dilution'] = tds[2].get_text().strip()
record['ASA/ISO'] = tds[3].get_text().strip()
record['35mm'] = tds[4].get_text().strip()
record['120'] = tds[5].get_text().strip()
record['Sheet'] = tds[6].get_text().strip()
record['Temp'] = tds[7].get_text().strip()
record['Notes'] = tds[8].get_text().strip()
records.append(record)
with open(
'tmp/record_{}_{}.json'.format(md5hash(response.url),
date), 'w') as fd:
fd.write(json.dumps(records, indent=4))
except Exception as e:
__Logger.error('failed to parse, err: {}'.format(e))
return None
def request(self, method, path, **kwargs):
if 'endpoint' in kwargs:
endpoint = kwargs['endpoint']
del kwargs['endpoint']
else:
endpoint = None
if 'headers' not in kwargs:
kwargs['headers'] = {}
kwargs['headers']['x-api-key'] = self.__token
kwargs['headers']['Accept'] = 'application/json'
if self.__user_agent:
kwargs['headers']['User-Agent'] = self.__user_agent
if 'params' not in kwargs:
kwargs['params'] = {}
kwargs['params']['token'] = self.__token
with metrics.http_request_timer(endpoint) as timer:
response = self.__session.request(method, self.BASE_URL + path,
**kwargs)
timer.tags[metrics.Tag.http_status_code] = response.status_code
if response.status_code >= 500:
raise Server5xxError()
elif response.status_code == 429:
raise RateLimitException('Rate limit exceeded', 1)
response.raise_for_status()
try:
return response.json()
except:
LOGGER.info('response.headers["content-type"]: %s',
response.headers['content-type'])
# This raise lets the backoff retry to kick in
raise