class HTTPRequest2(object):
"""记录cookies信息给下一次请求使用"""
def __init__(self):
# 创建session对象
self.session = Session()
def request(self, method, url,
params=None, data=None,
headers=None, cookies=None, json=None):
method = method.lower()
if method == "post":
# 判断是否使用json来传参（适用于接口项目有使用json传参）
if json:
logging.info("正在发送请求，请求地址：{}， 请求参数：{}".format(url, json))
return self.session.post(url=url, json=json, headers=headers, cookies=cookies)
else:
logging.info("正在发送请求，请求地址：{}， 请求参数：{}".format(url, data))
return self.session.post(url=url, data=data, headers=headers, cookies=cookies)
elif method == "get":
logging.info("正在发送请求，请求地址：{}， 请求参数：{}".format(url, params))
return self.session.get(url=url, params=params, headers=headers, cookies=cookies)
def close(self):
self.session.close()
def test_getDataList(self, case):
se = Session()
# 准备用例数据
login_url = "http://doctor.yy365.cn/index/login"
login_data = {
"username": conf.get("test_data", "admin_user"),
"password": conf.get("test_data", "admin_pwd")
}
response = se.post(url=login_url, data=login_data)
url1 = conf.get("env", "url") + case["url"]
data1 = {"params": '{"length":10,"search":' ',"start":0,"page":1}'}
response2 = se.post(url=url1, data=data1, verify=False)
res = response2.json()
row = case["case_id"] + 1
expected = eval(case["expected"])
try:
self.assertEqual(expected["draw"], res["draw"])
except AssertionError as e:
# 结果回写excel中
log.error("用例--{}--执行未通过".format(case["title"]))
log.debug("预期结果：{}".format(expected))
log.debug("实际结果：{}".format(res))
log.exception(e)
self.excel.write_data(row=row, column=8, value="未通过")
raise e
else:
# 结果回写excel中
log.info("用例--{}--执行通过".format(case["title"]))
self.excel.write_data(row=row, column=8, value="通过")
def login(username: str, password: str, session: Session) -> None:
login_url = f'{ATCODER_URL}/login'
data = {'username': username, 'password': password}
data['csrf_token'] = _get_csrf(login_url, session)
session.post(login_url, data=data)
class HttpRequest:
def __init__(self):
self.session = Session()
def __del__(self):
self.session.close()
def request(self, method, url, data=None, params=None, json=None, headers=None, cookies=None, timeout=None):
if method.lower() == 'get':
my_log.info(f'Sending {method}:{url} {params}')
res = self.session.get(url=url, params=params, headers=headers, cookies=cookies, timeout=timeout)
elif method.lower() == 'post':
if json:
my_log.info(f'Sending {method}:{url} {json}')
res = self.session.post(url=url, json=json, headers=headers, cookies=cookies, timeout=timeout)
else:
my_log.info(f'Sending {method}:{url} {data}')
res = self.session.post(url=url, data=data, headers=headers, cookies=cookies, timeout=timeout)
else:
res = None
if res.status_code == 404:
my_log.error(f'404 not found!')
raise RuntimeError
return res.text
def close(self):
self.session.close()
def update(title, tags, desc):
s = Session()
URL = 'https://ankiweb.net/account/login'
rsp = s.get(URL)
soup = BeautifulSoup(rsp.text, features="html.parser")
csrf_token = soup.find('input', {'name': 'csrf_token'}).get('value')
s.post(URL,
data={
'submit': 1,
'csrf_token': csrf_token,
'username': username,
'password': password
})
URL = 'https://ankiweb.net/shared/upload'
file = {'v21file': open(f'{MODEL_NAME}.zip', 'rb')}
rsp = s.post(URL,
files=file,
data={
'title': title,
'tags': tags,
'desc': desc,
'id': addon_id,
'submit': 'Update',
'v21file': file,
'v20file': '',
})
if rsp.url == f'https://ankiweb.net/shared/info/{addon_id}':
return True
else:
return False
def submit_custom_test(contest: str, lang: str, src: str,
stdin: str, session: Session) -> None:
custom_test_url = f'{ATCODER_URL}/contests/{contest}/custom_test'
custom_test_submit_api = f'{custom_test_url}/submit/json'
data = {'data.LanguageId': lang, 'sourceCode': src, 'input': stdin}
data['csrf_token'] = _get_csrf(custom_test_url, session)
session.post(custom_test_submit_api, data)
def submit(contest: str, problem: str, lang: str,
src: str, session: Session) -> None:
submit_url = f'{ATCODER_URL}/contests/{contest}/submit'
data = {'data.TaskScreenName': f'{contest}_{problem}',
'data.LanguageId': lang,
'sourceCode': src}
data['csrf_token'] = _get_csrf(submit_url, session)
session.post(submit_url, data)
class HTTPRequest(object):
def __init__(self):
# 创建session对象
self.session = Session()
# 记录cookies信息，给下一次请求使用
def request(self,
method,
url,
params=None,
data=None,
headers=None,
cookies=None,
json=None):
# 将方法名转化为小写
method = method.lower()
if method == "post":
# 判断是否使用json来传参，适用于接口项目使用json传参
if json:
logging.info("正在发送请求，请求地址：{}，请求参数：{}".format(url, json))
response = self.session.post(url=url,
json=json,
headers=headers,
cookies=cookies)
return response
else:
logging.info("正在发送请求，请求地址：{}，请求参数：{}".format(url, data))
response = self.session.post(url=url,
data=data,
headers=headers,
cookies=cookies)
return response
elif method == "get":
logging.info("正在发送请求，请求地址：{}，请求参数：{}".format(url, params))
response = self.session.get(url=url,
params=params,
headers=headers,
cookies=cookies)
return response
elif method == "put":
logging.info("正在发送请求，请求地址：{}，请求参数：{}".format(url, data))
response = self.session.put(url=url,
data=data,
headers=headers,
cookies=cookies)
return response
def close(self):
self.session.close()
class Test_Api(object):
def __init__(self):
self.base_url = row_data[2]
# 'http://127.0.0.1:5000/api/users/'
self.session = Session()
def test_create_user(self):
url = urljoin(self.base_url, url_path)
response = self.session.post(url, data) #.json()
# response = requests.post(url, data)
resp_json = response.json()
print response
print resp_json
print type(resp_json)
try:
assert resp_json == dict(
expected_resp) and response.status_code == 200
except AssertionError:
print traceback.print_exc()
# response = requests.post(url, data)
# print response.json()
# print response.content
# print response.cookies
# print response.headers
# print response.status_code
# print response.text
# print self.session.headers
# print self.session.cookies
# print self.session.params
def get_user(self):
url = urljoin(self.base_url, '002')
response = self.session.get(url)
# print response
print response.text
print response.json()
print response.content
def update_user(self):
url = urljoin(self.base_url, '001')
resp = self.session.post(url)
print resp.json()
def get_users(self):
url = 'http://127.0.0.1:5000/api/users'
response = self.session.get(url)
print response.content
def create_task_from_task_obj(s: Session, todolist_id: str, task: TodoTask, is_update: bool = False) -> str:
res = None
if is_update:
res = s.patch(GRAPH_URL + f'/me/todo/lists/{todolist_id}/tasks' , json=task.to_json())
else:
res = s.post(GRAPH_URL + f'/me/todo/lists/{todolist_id}/tasks' , json=task.to_json())
return res.json()
def fetch_data(election_id):
s = Session()
post_data = {
'__EVENTTARGET':
'ctl00$ContentPlaceHolder1$btnText',
'ctl00$pnlMenu_CollapsiblePanelExtender_ClientState':
'true',
'ctl00$AccordionStateBoardMenu_AccordionExtender_ClientState':
'0',
'ctl00$mtbSearch':
'',
'ctl00$AccordionPaneStateBoardMenu_content$AccordionMainContent_AccordionExtender_ClientState':
'-1',
'hiddenInputToUpdateATBuffer_CommonToolkitScripts':
'1',
'__EVENTARGUMENT':
'',
}
url = 'http://www.elections.state.il.us/ElectionInformation/CandDataFile.aspx?id=%s' % election_id
g = s.get(url)
soup = BeautifulSoup(g.content)
view_state = soup.find('input', attrs={'id': '__VIEWSTATE'}).get('value')
event_val = soup.find('input', attrs={
'id': '__EVENTVALIDATION'
}).get('value')
post_data['__VIEWSTATE'] = view_state
post_data['__EVENTVALIDATION'] = event_val
dl_page = s.post(url, data=post_data)
if dl_page.status_code == 200:
return dl_page.content
else:
return None
def get_instantaneous_demand(s: Session, endpoint: str,
meter_id: str) -> float:
payload = f"""
<Command>
<Name>device_query</Name>
<DeviceDetails>
<HardwareAddress>{meter_id}</HardwareAddress>
</DeviceDetails>
<Components>
<Component>
<Name>Main</Name>
<Variables>
<Variable>
<Name>zigbee:InstantaneousDemand</Name>
</Variable>
</Variables>
</Component>
</Components>
</Command>
"""
res = s.post(endpoint, data=payload)
res.raise_for_status()
xml = bs(res.text, "lxml")
all_variables = xml.find_all("variable")
return float([
variables for variables in all_variables
if variables.find("name").text == "zigbee:InstantaneousDemand"
][0].find("value").text)
class EMDRUploader(Thread):
def __init__(self, statsCollector):
Thread.__init__(self)
self._queue = Queue()
self.setDaemon(True)
self._session = Session()
self._session.headers.update({
"User-Agent": "CRESTMarketTrawler/{0} ([email protected])".format(VERSION)
})
self._pool = Pool(size=10)
self.statsCollector = statsCollector
def notify(self, regionID, typeID, orders):
self._queue.put((timestampString(), regionID, typeID, orders))
self.statsCollector.tally("emdr_send_queued")
queueSize = self._queue.qsize()
self.statsCollector.datapoint("emdr_queue_size", queueSize)
if queueSize > 100:
logger.error("EMDR submit queue is about {0} items long!".format(queueSize))
elif queueSize > 10:
logger.warn("EMDR submit queue is about {0} items long!".format(queueSize))
def run(self):
def submit(generationTime, regionID, typeID, orders):
uudif = json.dumps(EMDROrdersAdapter(generationTime, regionID, typeID, orders))
res = self._session.post("http://upload.eve-emdr.com/upload/", data=uudif)
self.statsCollector.tally("emdr_sent")
if res.status_code != 200:
logger.error("Error {0} submitting to EMDR: {1}".format(res.status_code, res.content))
self.statsCollector.tally("emdr_errored")
while True:
(generationTime, regionID, typeID, orders) = self._queue.get()
self._pool.spawn(submit, generationTime, regionID, typeID, orders)
def login(session: Session) -> Tuple[bool, Response]:
# GET infor page
r = session.get(infor_login_url)
if r.status_code != 200:
return (False, r)
page = BeautifulSoup(r.text, 'html.parser')
cookie = r.headers["Set-Cookie"].split()[0]
# TODO the cookie is set in the request itself
session.headers.update({"Cookie": cookie})
login_form = page.find(id="loginFormBean")
action = login_form["action"]
login_url = infor_base_url + action
# POST login attempt
form_data = {
"username": config.defaults()['username'],
"password": config.defaults()['password']
}
r = session.post(login_url, data=form_data)
if r.status_code != 200:
return (False, r)
# Check if correctly authenticated
page = BeautifulSoup(r.text, 'html.parser')
errors_div = page.find(id="div_erros_preenchimento_formulario")
if errors_div is not None:
error_text = errors_div.div.ul.li.text
logging.info(error_text)
return (False, r)
return (True, r)
def login(self, session: Session, url: str, login_data: dict):
return session.post(f"{url}/php/ajaxSet_Password.php",
headers={
"Content-Type": "application/json",
"csrfNonce": "undefined"
},
data=json.dumps(login_data))
async def markfinish(app, s: Session, group, member, msg: str):
text = msg.split(' ')
if len(text) == 1:
await app.sendGroupMessage(group,
MessageChain.create([Plain('请输入标题！')]))
elif len(text) != 2:
return
event_title = text[1]
url = 'http://canvas.tongji.edu.cn/api/v1/planner/items?per_page=50&start_date=' + \
datetime.now().strftime('%Y-%m-%d')
r = s.get(url)
data = json.loads(r.text.replace('while(1);', ''))
plannable_id = ''
event_id = None
for i in data:
if i['context_type'] == 'User' and i['plannable'][
'title'] == event_title:
plannable_id = i['plannable_id']
if i['planner_override'] != None:
event_id = i['planner_override']['id']
break
else:
await app.sendGroupMessage(
group, MessageChain.create([Plain('查无此事件，请检查标题是否输入正确')]))
data = {
'id':
event_id,
'marked_complete':
True,
'plannable_id':
plannable_id,
'plannable_type':
'calendar_event',
'user_id':
get_id(member.id),
'authenticity_token':
parse.unquote(
requests.utils.dict_from_cookiejar(s.cookies)['_csrf_token'])
}
if event_id == None:
url = 'http://canvas.tongji.edu.cn/api/v1/planner/overrides'
r = s.post(url, data=data)
else:
url = 'http://canvas.tongji.edu.cn/api/v1/planner/overrides/' + \
str(event_id)
r = s.put(url, data=data)
if not is_json(r.text) or 'errors' in r.json():
await app.sendGroupMessage(group,
MessageChain.create([Plain('标记为完成失败！')]))
else:
await app.sendGroupMessage(group,
MessageChain.create([Plain('标记为完成成功！')]))
def grant_session_iDP(response: Response, session: Session) -> Response:
"""Grants the current session an iDP by posting the SAMLResponse to a specific url. Return the response page."""
soup = BeautifulSoup(response.content, "lxml")
# HTML contains a hidden form with a URL to POST to and the SAMLResponse value
url = soup.find("form").get("action")
SAMLResponse = soup.find("input").get("value")
response = session.post(url, {"SAMLResponse": SAMLResponse})
return response
def grant_session_MDL_SSP_AuthToken(response: Response,
session: Session) -> Response:
"""Grants the current session an MDL_SSP_AuthToken cookie by simply posting to a specific URL with our SAMLResponse value."""
soup = BeautifulSoup(response.content, "lxml")
url = soup.find("form").get("action")
SAMLResponse = soup.find("input").get("value")
response = session.post(url, {"SAMLResponse": SAMLResponse})
return response
class BaseRequest(object):
def __init__(self, session=None):
if not isinstance(session, Session):
self.session = Session()
self.session.mount('http://', HTTPAdapter(max_retries=1,
pool_maxsize=50))
self.session.mount('http://', HTTPAdapter(max_retries=1,
pool_maxsize=50))
else:
self.session = session
@set_default
def get(self, url, params, timeout=1, callback=None, **kwargs):
with self.catch_exception():
r = self._get_result(url, params, timeout, **kwargs)
if callable(callback):
callback(r)
return r
@set_default
def post(self, url, data, timeout=1, callback=None, **kwargs):
with self.catch_exception():
r = self._post_result(url, data, timeout, **kwargs)
if callable(callback):
callback(r)
return r
def _get_result(self, url, params, timeout, **kwargs):
r = self.session.get(url, params=params, timeout=timeout, **kwargs)
r.raise_for_status()
return r
def _post_result(self, url, data, timeout, **kwargs):
r = self.session.post(url, data, timeout=timeout, **kwargs)
r.raise_for_status()
return r
@contextmanager
def catch_exception(self):
try:
yield
except(ConnectionError, Timeout) as err:
raise VendorConnectionError(str(err), data=err)
except HTTPError as err:
raise VendorHTTPError(str(err), data=err)
except RequestException as err:
raise VendorRequestError(str(err), data=err)
class DemoApi(object):
def __init__(self, base_url):
self.base_url = base_url
# 创建session实例
self.session = Session()
def login(self, mobile, password):
"""
登录接口
:param username: 用户名
:param password: 密码
"""
url = urljoin(self.base_url, 'account/login')
data = {'mobile': mobile, 'password': password}
headers = {
'content-type':
'application/json',
'X-Token':
'mobile',
'User-Agent':
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.132 Safari/537.36'
}
response = self.session.post(url,
data=json.dumps(data),
headers=headers).json()
print('\n*****************************************')
print(u'\n1、请求url: \n%s' % url)
print(u'\n2、请求头信息:')
pprint(self.session.headers)
print(u'\n3、请求参数:')
pprint(data)
print(u'\n4、响应:')
pprint(response)
return response
def info(self):
"""
详情接口
"""
url = urljoin(self.base_url, 'account/my_assets')
response = self.session.get(url).json()
print('\n*****************************************')
print(u'\n1、请求url: \n%s' % url)
print(u'\n2、请求头信息:')
pprint(self.session.headers)
print(u'\n3、请求cookies:')
pprint(dict(self.session.cookies))
print(u'\n4、响应:')
pprint(response)
return response
def init_session_id(login: str, password: str, cluster_host: str) -> None:
"""
Initialize Session ID value from a Cookie after authentication.
:param login: Login of a Static Dex user
:type login: str
:param password: Password of a Static Dex user
:type password: str
:param cluster_host: Base Host of a cluster
:type cluster_host: str
:return: None
"""
global _session_cookies, _jenkins_credentials
session = Session()
for auth_endpoint_url in AUTH_ENDPOINT_URLS:
response = session.get(auth_endpoint_url.format(cluster_host))
if response.status_code != 200:
raise IOError(
'Authentication endpoint is unavailable, got {} http code'.
format(response.status_code))
if response.url.startswith(AUTHENTICATION_HOSTNAME.format(
cluster_host)): # if auth form is opened
match = re.search(REQUEST_ID_REGEXP, response.text)
if match:
request_id = match.group(1)
else:
raise ValueError('Request ID was not found on page')
url = AUTHENTICATION_PATH.format(cluster_host, request_id)
data = {PARAM_NAME_LOGIN: login, PARAM_NAME_PASSWORD: password}
response = session.post(url, data)
if response.status_code != 200:
raise IOError(
'Unable to authorise, got {} http code from {} for the query to {} with data, response {}'
.format(response.status_code,
auth_endpoint_url.format(cluster_host), url, data,
response.text))
for cookie_name in session.cookies.keys():
if cookie_name.startswith(SESSION_ID_COOKIE_NAMES):
_session_cookies[cookie_name] = session.cookies.get(
cookie_name)
if len(_session_cookies) == 0:
raise ValueError('Cant find any session ID in Cookies')
response = session.get(JENKINS_PROFILE_URL.format(cluster_host, login))
if response.status_code == 200:
regex_output = JENKINS_API_TOKEN_REGEX.search(response.text)
if regex_output:
_jenkins_credentials = (login, regex_output.group(1))
def test_importData(self, case):
se = Session()
# 准备用例数据
login_url = "http://doctor.yy365.cn/index/login"
login_data = {
"username": conf.get("test_data", "admin_user"),
"password": conf.get("test_data", "admin_pwd")
}
response = se.post(url=login_url, data=login_data)
url1 = conf.get("env", "url") + case["url"]
headers = {"Content-Type": "multipart/form-data"}
file = {
'file':
open(
r"C:\Users\Administrator\Desktop\Romens_Api_Test\data\891407.xls",
'rb')
}
response2 = se.post(url=url1,
files=file,
verify=False,
headers=headers)
res = response2.json()
row = case["case_id"] + 1
expected = eval(case["expected"])
try:
self.assertEqual(expected["status_code"], response2.status_code)
except AssertionError as e:
# 结果回写excel中
log.error("用例--{}--执行未通过".format(case["title"]))
log.debug("预期结果：{}".format(expected))
log.debug("实际结果：{}".format(res))
log.exception(e)
self.excel.write_data(row=row, column=8, value="未通过")
raise e
else:
# 结果回写excel中
log.info("用例--{}--执行通过".format(case["title"]))
self.excel.write_data(row=row, column=8, value="通过")
def __queryNear(self, sess: Session) -> bool: # 上次填写记录
sess.headers.update({"referer": reportUrl})
res = sess.post(
url=tempHeader +
"/com.sudytech.work.shgcd.jkxxcj.jkxxcj.queryNear.biz.ext?vpn-12-o2-workflow.sues.edu.cn",
verify=False)
near_list = res.json()["resultData"]
if len(near_list) == 0:
return False
else:
self.__lastData = near_list[0]
lower_json(self.__lastData)
return True
def post_delete_endpoint_with_session_and_url(
session: Session,
url: str,
payload_obj: dict,
) -> typing.Tuple[str, int]:
payload = json.dumps(payload_obj)
response = session.post(url, payload)
summary = parse_run_restli_response(response)
urn = summary.get("urn", "")
rows_affected = summary.get("rows", 0)
return urn, rows_affected
def _do_request(self, short_uri, payload, files=None):
payload.update(self.pre_params())
trace_id = getattr(self.__class__._thread_vars, "trace_id", "")
# need put traceid in url
if self.traceid_in_url:
if len(short_uri.split('?')) == 1:
short_uri += "?trace_id=%s" % trace_id
else:
short_uri += "&trace_id=%s" % trace_id
# put traceid in body
else:
payload["trace_id"] = trace_id
res = None
url = os.path.join(self.url_prefix, short_uri)
try:
ts = time.time()
if self.session_reuse:
# 如果使用长连接，则复用session
c_session = self.get_session(url)
else:
c_session = Session()
params_dict = dict(data=payload, timeout=self.timeout)
res = c_session.post(url, **params_dict)
res.raise_for_status()
tc = time.time() - ts
self.trace_out("[%s] [%f] [%s]", url, tc, trace_id)
except ConnectionError:
# try_count += 1
# time.sleep(1)
self.warn_out("connect failed to %s [%s]", self.url_prefix, trace_id)
except ReadTimeout:
self.error_out("Timeout for url: %s [%s]", url, trace_id)
raise
except HTTPError:
self.error_out("[%s] [%s] [%s] bad response[%s]", url, trace_id, res.status_code, res.text)
if res.status_code == 504:
raise
if res is None:
raise ConnectException("connect upstream error. all backend tried %s" % self.url_prefix)
return res
def post(
self,
url,
data=None,
json=None,
**kwargs
):
return Session.post(
self,
self._get_resource_uri(url),
data, json,
**self._set_default_timeout(
**kwargs
)
)
def create_user(session: Session, user: User) -> Response:
app.logger.debug("UserApi#create_user: {} {}".format(user.last_name, user.first_name))
user_dict = {
'profile': {
'id': user.email,
'firstName': user.first_name,
'lastName': user.last_name,
'email': user.email
},
'credentials': {
'password': '******'
}
}
return session.post(url=URL_ENGINE_API + '/user/create', json=user_dict)
def __Shorten(urlsToShorten):
flipped = {v:k for k,v in urlsToShorten.items()}
shortned = {}
connection = engine.connect()
r = connection.execute(urls.select(None).where(urls.c.url.in_(flipped.keys())))
rows = r.fetchall()
for keyy, url in rows:
shortned[flipped[url]] = 'smirko.cc/%s' % keyy
urlsToShorten.pop(flipped[url])
if urlsToShorten:
s = Session()
r = s.post('http://smirko.cc/index2shorten.php',data={'data':json.dumps(urlsToShorten)})
shortned2 = json.loads(r.text)
shortned2.update(shortned)
return shortned2
return shortned
def fetch_data(dl_type=None, **kwargs):
"""
Fetch Receipts, Expenditures, and Committees.
dl_type is one of those three choices.
kwargs depend on the choice.
Receipts and Expenditures need start_date and end_date for search.
Committees need a name_start kwarg to pass into the search.
Seems like the maximum that you can get is about 250,000 records at a time.
"""
s = Session()
post_data = {
'__EVENTTARGET': 'ctl00$ContentPlaceHolder1$btnText',
'ctl00$pnlMenu_CollapsiblePanelExtender_ClientState': 'true',
'ctl00$AccordionStateBoardMenu_AccordionExtender_ClientState': '0',
'ctl00$mtbSearch': '',
'ctl00$AccordionPaneStateBoardMenu_content$AccordionMainContent_AccordionExtender_ClientState': '-1',
'hiddenInputToUpdateATBuffer_CommonToolkitScripts': '1',
'__EVENTARGUMENT': '',
'__VIEWSTATEGENERATOR': 'E8D1F59A'
}
if dl_type == 'Receipts':
CONT_GET_PARAMS['RcvDate'] = kwargs['start_date']
CONT_GET_PARAMS['RcvDateThru'] = kwargs['end_date']
url = '%s/DownloadList.aspx?%s' % (BASE_URL, urlencode(CONT_GET_PARAMS))
elif dl_type == 'Committees':
COMM_GET_PARAMS['Name'] = kwargs['name_start']
url = '%s/DownloadList.aspx?%s' % (BASE_URL, urlencode(COMM_GET_PARAMS))
elif dl_type == 'Expenditures':
EXP_GET_PARAMS['ExpendedDate'] = kwargs['start_date']
EXP_GET_PARAMS['ExpendedDateThru'] = kwargs['end_date']
url = '%s/DownloadList.aspx?%s' % (BASE_URL, urlencode(EXP_GET_PARAMS))
elif dl_type == 'Candidates':
url = 'http://www.elections.state.il.us/ElectionInformation/CandDataFile.aspx?id=%s' % kwargs['election_id']
g = s.get(url)
if 'Unexpected errors occurred trying to populate page' in g.content:
return None
soup = BeautifulSoup(g.content)
view_state = soup.find('input', attrs={'id': '__VIEWSTATE'}).get('value')
event_val = soup.find('input', attrs={'id': '__EVENTVALIDATION'}).get('value')
post_data['__VIEWSTATE'] = view_state
post_data['__EVENTVALIDATION'] = event_val
dl_page = s.post(url, data=post_data)
if dl_page.status_code == 200:
return dl_page.content
else:
return None
def login(response: Response, session: Session,
credentials: NetnameCredentials) -> Response:
"""Log into the (Concorida) Netname login page with the given credentials. Return the response page."""
# The form POST method's action contains the URL to post to when we have entered our login information
url = BeautifulSoup(response.content, "lxml").select_one(
"form[action^='https://fas.concordia.ca:443']").get("action")
login_info = {
"UserName": credentials.username,
"Password": credentials.password
}
response = session.post(url, login_info)
if "Incorrect netname or password. Type the correct netname and password, and try again." in response.text:
raise Exception(
"The provided Concordia login information is incorrect")
return response
class EMDRUploader(Thread):
def __init__(self, statsCollector):
Thread.__init__(self)
self._queue = Queue(EMDR_QUEUE_SIZE)
self.setDaemon(True)
self._session = Session()
self._session.headers.update({
"User-Agent": USER_AGENT_STRING
})
self._pool = Pool(size=10)
self.statsCollector = statsCollector
def notify(self, regionID, orders):
self._queue.put((timestampString(), regionID, orders))
self.statsCollector.tally("emdr_send_queued")
queueSize = self._queue.qsize()
self.statsCollector.datapoint("emdr_queue_size", queueSize)
if queueSize > EMDR_QUEUE_SIZE / 2:
logger.warn("EMDR submit queue is about {0} items long!".format(queueSize))
def run(self):
def submit(generationTime, regionID, orders):
chunks = chunkOrders(orders)
for idx, orderChunk in enumerate(chunks):
with TemporaryFile() as gzfile:
ujson.dump(
EMDROrdersAdapter(generationTime, regionID, orderChunk),
gzip.GzipFile(fileobj=gzfile, mode="wb")
)
headers = {'Content-Length': str(gzfile.tell()),
'Content-Encoding': 'gzip', # what EMDR wants
# 'Transfer-Encoding': 'gzip' # what is strictly true
}
gzfile.seek(0, 0)
logger.info(
"Submitting to EMDR for region {} (chunk {} of {})".format(regionID, idx + 1, len(chunks)))
res = self._session.post("http://upload.eve-emdr.com/upload/", data=gzfile, headers=headers)
self.statsCollector.tally("emdr_chunks_sent")
if res.status_code != 200:
logger.error("Error {0} submitting to EMDR: {1}".format(res.status_code, res.content))
self.statsCollector.tally("emdr_errored")
while True:
(generationTime, regionID, orders) = self._queue.get()
self.statsCollector.datapoint("emdr_queue_size", self._queue.qsize())
self._pool.spawn(submit, generationTime, regionID, orders)
class DemoApi(object):
def __init__(self, base_url):
self.base_url = base_url
self.session = Session(
) #登录接口的http响应会把session以 cookie的形式set到客户端，之后的接口都会使用此session去请求
def login(self, username, password):
url = urljoin(self.base_url, 'login')
data = {'username': username, 'password': password}
response = self.session.post(url, data=data).json()
return response
def info(self):
url = urljoin(self.base_url, 'info')
response = self.session.get(url).json()
return response
def get_smartmeter_id(s: Session, endpoint: str) -> str:
meter_id = load_meter_id()
if isinstance(meter_id, str):
return meter_id
payload = """
<Command>
<Name>device_list</Name>
</Command>
"""
res = s.post(endpoint, data=payload)
res.raise_for_status()
xml = bs(res.text, "lxml")
meter_id = xml.find("hardwareaddress").text
save_meter_id(meter_id)
return get_smartmeter_id(s, endpoint)
class Api(object):
API_URL = 'http://192.168.33.10' # todo: add to config
API_VERSION = 'v1'
def __init__(self, device_key):
self.base_url = '{0}/{1}/'.format(self.API_URL, self.API_VERSION)
self.session = Session()
self.session.auth = KeyAuth(device_key)
self.session.headers.update({
'Content-Type': 'application/json'
})
def request(self, method, url, **kwargs):
"""Constructs and sends a Request to the Pinaple API."""
full_url = urljoin(self.base_url, url)
if 'data' in kwargs:
kwargs['data'] = self._encode_data(kwargs['data'])
return super(Api, self).request(method, full_url, **kwargs)
def _encode_data(self, data, **kwargs):
"""Returns data encoded as JSON using a custom encoder."""
encoder = JSONEncoder(**kwargs) if kwargs else self._json_encoder
return encoder.encode(data)
def test(self):
url = urljoin(self.base_url, 'functions/test')
response = self.session.get(url)
return response
def login(self):
url = urljoin( self.base_url, 'login' )
response = self.session.post( url )
if response.status_code is not 200:
print('[error] device is not authorized')
exit()
data = response.json()
self.session.auth = SessionAuth(data['session_token'])
