def _get_token(self):
http.request('https://www.baidu.com/')
# Token
ret = http.request(
'https://passport.baidu.com/v2/api/?getapi&tpl=pp' + \
'&apiver=v3&class=login' + \
'&gid=965D5E0-3500-4CE2-B4DC-8AA0D833BF6D&tt=%s&logintype=basicLogin&callback=0' % int(time.time())).replace('\'', '\"')
foo = json.loads(ret)
return foo['data']['token']
def download_resource_file(on_request, resDir):
global toneDir
global on_callback
global on_download
global cb_data
toneDir = resDir
data = {
'url': on_request['url'],
'method': 'GET',
'headers': {
'Accept': '*/*'
},
'timeout': 30000,
'params': ''
}
def cb(data):
global on_callback
global cb_data
print(on_callback)
on_callback = True
cb_data = data
http.request(data, cb)
while True:
if on_callback:
break
else:
time.sleep(1)
response = json.loads(cb_data)
format = response['format']
size = response['size']
format = response['format']
audio = response['audios'][0]
id = audio['id']
path = toneDir + id + '.' + format
print('************ begin to download: ' + path)
d_data = {'url': audio['url'], 'filepath': path}
def d_cb(data):
global on_download
on_download = True
http.download(d_data, d_cb)
while True:
if not on_download:
break
else:
time.sleep(1)
print('download succeed :' + path)
def getFile(self, file_id, filename=None, directory=None):
data = {}
data['file_id'] = file_id
response = http.request(self.baseurl + self.token + '/getFile',
method='POST',
body=json.dumps(data),
headers={'Content-Type': 'application/json'})
if (response[0] != 200):
return False
jsonRespone = json.loads(response[2])
if (jsonRespone['ok']):
filepath = jsonRespone['result']['file_path']
extension = filepath.split('.')[-1]
import uuid, urllib
basename = '%s.%s' % (filename if filename else str(uuid.uuid4()),
extension)
savefile = os.path.sep.join([
directory.rstrip(os.path.sep), basename
]) if directory else basename
downloadlink = 'https://api.telegram.org/file/bot%s/%s' % (
self.token, filepath)
urllib.urlretrieve(downloadlink, savefile)
return True
else:
return False
def get_geodata():
print("Retrieving geo data...")
url = 'https://raw.githubusercontent.com/datasets/geo-boundaries-world-110m/master/countries.geojson'
http = urllib3.PoolManager()
r = http.request('GET',url)
geodata = json.loads(r.data.decode('utf-8'))
geodata_features = geodata['features']
return geodata_features
def sendChatAction(self, chat_id, action):
data = {}
data['chat_id'] = chat_id
data['action'] = action
return http.request(self.baseurl + self.token + '/sendChatAction',
method='POST',
body=json.dumps(data),
headers={'Content-Type': 'application/json'})
def get_geodata():
print("Retrieving geo data...")
url = 'https://raw.githubusercontent.com/datasets/geo-boundaries-world-110m/master/countries.geojson'
http = urllib3.PoolManager()
r = http.request('GET', url)
geodata = json.loads(r.data.decode('utf-8'))
geodata_features = geodata['features']
return geodata_features
def login_check(self,uname):
url = 'http://wappass.baidu.com/wp/api/login/check?'
# check used
CLIENTFROM='clientfrom=native'
gid='gid='+GID
data = CLIENTFROM + '&' + gid + '&tt=' + str(int(time.time()*1000)) + '&username='+uname
data = http.request(url+data)
print data
def login_check(self,uname):
url = 'http://wappass.baidu.com/wp/api/login/check?'
# check used
CLIENTFROM='clientfrom=native'
GID='gid=BE56965-3AC1-4C07-9A86-F5E366F23BF8'
data = CLIENTFROM + '&' + GID + '&tt=' + str(int(time.time()*1000)) + '&username='+uname
data = http.request(url+data)
print data
def test_redirect(self):
redirect_sites = [
"http://www.naver.com/",
"http://browser.engineering/redirect",
]
for site in redirect_sites:
headers, body = request(site)
self.assertGreater(len(body), 0)
self.assertIn("content-type", headers)
def getUpdates(self, offset=None, limit=None, timeout=None):
data = {}
if (offset): data['offset'] = offset
if (limit): data['limit'] = limit
if (timeout): data['timeout'] = timeout
return http.request(self.baseurl + self.token + '/getUpdates',
method='GET',
body=json.dumps(data),
headers={'Content-Type': 'application/json'})
def sendLocation(self, chat_id, latitude, longitude, disable_notification=None, reply_to_message_id=None, reply_markup=None):
data = {}
data['chat_id'] = chat_id
data['latitude'] = latitude
data['longitude'] = longitude
if(disable_notification): data['disable_notification'] = disable_notification
if(reply_to_message_id): data['reply_to_message_id'] = reply_to_message_id
if(reply_markup): data['reply_markup'] = reply_markup
return http.request(self.baseurl + self.token + '/sendLocation', method='POST', body=json.dumps(data), headers={'Content-Type': 'application/json'})
def download_resource_file(self, on_request, resDir):
global on_callback, on_download, cb_data
data = {
'url': on_request['url'],
'method': 'GET',
'headers': {},
'timeout': 30000,
'params': ''
}
def cb(data):
global on_callback, cb_data
on_callback = True
cb_data = data
http.request(data, cb)
while True:
if on_callback:
on_callback = False
break
else:
time.sleep(1)
response = json.loads(cb_data['body'])
audio = response['audios'][0]
format = audio['format']
id = audio['id']
size = audio['size']
path = self.toneDir + id + '.' + format
print('************ begin to download: ' + path)
d_data = {'url': audio['url'], 'filepath': path}
def d_cb(data):
global on_download
on_download = True
http.download(d_data, d_cb)
while True:
if on_download:
on_download = False
break
else:
time.sleep(1)
print('download succeed :' + path)
def test_https_request(self):
https_sites = [
"https://www.google.com/",
"https://www.facebook.com/",
"https://example.com/",
]
for site in https_sites:
headers, body = request(site)
self.assertGreater(len(body), 0)
self.assertIn("content-type", headers)
def getUserProfilePhotos(self, user_id, offset=None, limit=None):
data = {}
data['user_id'] = user_id
if (offset): data.append(('offset', offset))
if (limit): data.append(('limit', limit))
return http.request(self.baseurl + self.token +
'/getUserProfilePhotos',
method='POST',
body=json.dumps(data),
headers={'Content-Type': 'application/json'})
def generate_wayback_uri(url, last_n=-5, format='json', verbose=0):
"""
call the https://github.com/internetarchive/wayback/tree/master/wayback-cdx-server#basic-usage
API to obtain the last snapshots of the wayback machine for a specific URL.
:param str url: a URL
:param int last_n: -5 indicates the 5 latest snapshots and 5 the first 5 snapshots
:param str format: supported: 'json'
:rtype: tuple
:return: (status, URL or None)
"""
http = urllib3.PoolManager()
wb_url = None
params = {'url': url, 'output': format, 'limit': last_n}
encoded_uri = WAYBACK_CDX_SERVER + urlencode(params)
try:
r = http.request('GET', encoded_uri)
except urllib3.exceptions.MaxRetryError:
return 'http request failed', url
if r.status != 200:
status = 'status code not 200'
if verbose >= 4:
print(f'status code: {r.status}')
data_as_string = r.data.decode('utf-8')
try:
snapshots = ast.literal_eval(data_as_string[:-1])
except:
# org.archive.util.io.RuntimeIOException: org.archive.wayback.exception.AdministrativeAccessControlException: Blocked Site Error
snapshots = ['']
for (urlkey, timestamp, original, mimetype, statuscode, digest,
length) in snapshots[1:]:
if statuscode == '-':
continue
if int(statuscode) == 200:
wb_url = f'http://web.archive.org/web/{timestamp}/{original}'
status = 'succes'
if wb_url is None:
status = 'Wayback Machine URL not found'
if status == 'succes':
if verbose >= 3:
print()
print(f'Wayback machine: {wb_url} for url {url}')
print(f'used the following query: {encoded_uri}')
return status, wb_url
def attempt(url, http=http, retry=retry):
r = None
try:
r = http.request('GET', url, retries=retry)
except Exception as e:
retry = r.retries if r else retry
retry = retry.increment('GET', url, error=e)
retry.sleep()
logging.warning("Retrying (%r) after connection broken by '%r': '%s'", retry, e, url)
return attempt(url, retry=retry)
return r
def FuncCurlib3ClientConnection(self, jsonInputData):
encoded_data = json.dumps(jsonInputData).encode('utf-8')
http = urllib3.PoolManager(maxsize=10, block=True)
timeout = 2
socket.setdefaulttimeout(timeout)
r = http.request(
'POST',
'http://' + ModFmptCom.GL_FMPT_CLOUDCON_PAR_IP_ADDR_SET_BY_USER +
':' + ModFmptCom.GL_FMPT_CLOUDCON_HUITP_JSON_PORT + '/',
body=encoded_data,
headers={'Content-Type': 'application/json'})
return json.loads(r.data)
def downloadUrl_2(url, localfile):
import urllib3
http = urllib3.PoolManager()
# url = 'https://raw.githubusercontent.com/abidrahmank/OpenCV2-Python-Tutorials/master/data/left.jpg'
try:
r = http.request('GET', url, timeout=10)
with open(localfile, "wb") as code:
code.write(r.data)
except Exception as e:
return 'err'
else:
return 'ok'
def is_login(self):
if (not self.bduss) or (not self.baiduid):
print 'login fail!'
return False
self.bdauth = 'BAIDUID=' + self.baiduid + '; BDUSS=' + self.bduss
url='http://tieba.baidu.com/dc/common/tbs'
data=http.request(url,cookie=self.bdauth)
k=json.loads(data)
if k["is_login"]==1:
print 'login success!'
return True
else:
print 'login fail!'
return False
def FuncCurlib3ClientConnection(self, jsonInputData):
encoded_data = json.dumps(jsonInputData).encode('utf-8')
http = urllib3.PoolManager(maxsize=10, block=True)
timeout = 3
socket.setdefaulttimeout(timeout)
r = http.request(
'POST',
#'http://' + ModFawsCom.GL_FAWS_CLOUDCON_PAR_IP_ADDR_SET_BY_USER + ':' + ModFawsCom.GL_FAWS_CLOUDCON_HUITP_JSON_PORT + '/',
'http://' + ModFawsCom.GL_FAWS_CLOUDCON_PAR_IP_ADDR_SET_BY_USER +
':' + '80' + '/mfunhcu/l1mainentry/cloud_callback_faam.php',
body=encoded_data,
headers={'Content-Type': 'application/json'})
res = json.loads(r.data)
return res
def request(url, data):
#logger.debug('url:'+url)
#logger.debug('data:'+json.dumps(data))
#headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36'}
# 这里的urlencode用于把一个请求对象用'&'来接来字符串化，接着就是编码成utf-8
#data = urllib.parse.urlencode(data).encode('utf-8')
#request = urllib.request.Request(url, data, headers)
#response = urllib.request.urlopen(request)
#text = response.read().decode('utf-8')
#log_info(text)
text = http.request(url,data)
#log_info(text)
print text
return text
def getRateDetail(itemId,currentPage):
# Url最后的callback字段是用于天猫网站内部回调，和我们没关系，不过这个字段的值关系到返回数据的前缀，我们可以利用这个值截取返回数据
url='https://rate.tmall.com/list_detail_rate.htm?itemId=' + str(itemId) + '&spuId=418357413&sellerId=765844183&order=3¤tPage=' + str(currentPage) + '&append=0......&callback=jsonp462'
r = http.request('GET',url,headers = headers)
# 返回数据时GB18030编码，所以要用这个编码格式进行解码
c = r.data.decode('GB18030')
# 下面的代码返回评论数据转换为JSON格式
c = c.replace('jasonp1278(','')
c = c.replace(')','')
c = c.replace('false','"false"')
c = c.replace('true','"true"')
# 将JSON格式的评论数据转换成字典
tmalljson = json.loads(c)
return tmalljson
def request_through_cache(url, data=None, use_cache=True, cache_after=None, proxy=None, timeout=HTTP_TIMEOUT):
if data is None:
cache_url = url
else:
cache_url = url+'|post:%s'%(data,)#some stupid sites use POST to get data..
if use_cache:
cached_content, cache_time = cache.http_get_t(cache_url)
if cached_content: #cache exists
if (not cache_after) or cache_time > cache_after:
debug_print('cache hit: %s'%(cache_url,))
return HTTPResponse(cached_content)
response = request(url, data=data, proxy=proxy, timeout=timeout)
if response.status_code == 200:
cache.http_set(cache_url, response.content)
return response
def tieba_list(self):
# 获取关注的贴吧列表
url="http://c.tieba.baidu.com/c/f/forum/favolike"
singbase= 'BDUSS='+self.bduss +'&'+ client_id+'&'+ client_type+'&'+ client_version+'&'+ phone_imei+'&'+ net_type+'&'+ pn
signmd5= 'BDUSS='+self.bduss + client_id+ client_type+ client_version+ phone_imei+ net_type+ pn
sign = '&sign=' + hashlib.md5((signmd5+'tiebaclient!!!').encode()).hexdigest()
data=singbase+sign
data=http.request(url,self.__paramparse__(data))
tieba = []
data=json.loads(data)
list=data['forum_list']
for x in list:
tieba.append(x['name'].encode('gbk'))
tbs='tbs='+data['anti']['tbs']
return [tieba,tbs]
def forwardMessage(self,
chat_id,
from_chat_id,
message_id,
disable_notification=None):
data = {
'data_id': data_id,
'from_chat_id': from_chat_id,
'message_id': message_id
}
if (disable_notification):
data['disable_notification'] = disable_notification
return http.request(self.baseurl + self.token + '/forwardMessage',
method='POST',
body=json.dumps(data),
headers={'Content-Type': 'application/json'})
def check_download(http, base_url, encoded_query, fmt=None, sep=None):
url = base_url
if fmt:
url = url + f'?format={fmt}'
if fmt.lower() == 'csv' and sep:
url = url + f'&field_separator={sep}'
response = http.request('POST',
url,
headers={'Content-Type': 'application/json'},
body=encoded_query)
print("URL Called: ", response.geturl())
print("Response HEADERS: ", response.headers)
print("Response INFO: ", response.info())
if fmt.lower() in ['json', 'csv'] or fmt is None:
print(response.data.decode("utf-8"))
def getFile(self, file_id, filename=None, directory=None):
data = {}
data['file_id'] = file_id
response = http.request(self.baseurl + self.token + '/getFile', method='POST', body=json.dumps(data), headers={'Content-Type': 'application/json'})
if(response[0] != 200):
return False
jsonRespone = json.loads(response[2])
if(jsonRespone['ok']):
filepath = jsonRespone['result']['file_path']
extension = filepath.split('.')[-1]
import uuid, urllib
basename = '%s.%s' % (filename if filename else str(uuid.uuid4()), extension)
savefile = os.path.sep.join([directory.rstrip(os.path.sep), basename]) if directory else basename
downloadlink = 'https://api.telegram.org/file/bot%s/%s' % (self.token, filepath)
urllib.urlretrieve(downloadlink, savefile)
return True
else:
return False
def getProductIdList():
url = 'https://list.tmall.com/search_product.htm... ...'
r = http.request('GET', url,headers = headers)
c = r.data.decode('GB18030')
soup = BeautifulSoup(c,'lxml')
linkList = []
idList = []
# 用Beautiful Soup提取商品页面中所有的商品ID
tags = soup.find_all(href=re.compile('detail.tmall.com/item.htm'))
for tag in tags:
linkList.append(tag['href'])
linkList = list(set(linkList))
for link in linkList:
aList = link.split('&')
# //detail.tmall.com/item.htm?id=562173686340
# 将商品ID添加到列表中
idList.append(aList[0].replace('//detail.tmall.com/item.htm?id=',''))
return idList
def sign(self):
tieba = self.tieba_list()
url='http://c.tieba.baidu.com/c/c/forum/sign'
tbs = tieba[1]
for x in tieba[0]:
kw='kw='+x.decode('gbk')
sign='&sign='
signmd5= 'BDUSS='+self.bduss + kw + tbs + 'tiebaclient!!!'
signbase= 'BDUSS='+self.bduss +'&'+ kw + '&' + tbs
sign=sign+hashlib.md5(signmd5.encode('utf-8')).hexdigest()
data=signbase+sign
data=http.request(url,self.__paramparse__(data))
data=json.loads(data)
if data['error_code']=='0':
print x , u'吧 签到成功！'
else:
print x , u'吧 签到失败！ 失败原因:' , data['error_msg'].encode('gbk')
time.sleep(float(SIGN_INTERVAL))
def get(baseURL, data={}):
'''
Gets site contents and attempts to parse them as JSON data.
Optiona data parameter for get requests.
'''
getURL = ''
if len(data.keys()) > 0:
getURL = '?' + urlencode(data)
http = httplib2.Http()
resp, respData = http.request(baseURL + getURL, 'GET')
if not (resp.status in [200]):
raise Exception(resp.status)
else:
return decodeJSON(respData)
def _start_remote(self):
http_request = self.request_builder.http_request
url = http_request.path
parsed_url = urlparse.urlparse(url)
try:
host = Host.get(url=parsed_url.hostname)
if host.ssl:
url = url.replace('http', 'https')
self.cachebrowsed = True
except Host.DoesNotExist:
pass
logging.info("[%s] %s %s" % (http_request.method, url, '<CACHEBROWSED>' if self.cachebrowsed else ''))
request = http_request.get_raw()
# request = re.sub(r'^(GET|POST|PUT|DELETE|HEAD) http[s]?://[^/]+/(.+) (\w+)', r'\1 /\2 \3', request)
response = http.request(url, raw_request=request)
self._connection.start_remote(response)
def download(url):
ssl._create_default_https_context = ssl._create_unverified_context
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
http = urllib3.PoolManager()
# ip = ['121.34.156.197', '175.31.128.78', '124.219.217.120'];
headers = {
'User-Agent': r'Mozilla/5.0 (Windows NT 6.1; WOW64)'
r'Chrome/45.0.2454.85 Safari/537.36 115Browser/6.0.3',
'Referer':
r'https://www.douban.com/doulist/240962/?start=50&sort=seq&sub_type=',
'Connection': 'keep-alive',
# 'X-Forwarded-For': ip[0]
}
r = http.request("GET", url, headers=headers)
#
return r.data.decode('utf-8')
def run_all(config_dict, log_folder, verbose):
dns_logger = create_logger("dns", log_folder, verbose)
name_dict = config_dict.get("dns", {})
for name in name_dict:
expected = name_dict[name].get("expected")
dns_logger.info(';'.join(str(x) for x in resolve(name, expected)))
ping_logger = create_logger("ping", log_folder, verbose)
for host in config_dict.get("ping",[]):
ping_logger.info(';'.join(str(x) for x in ping(host)))
ping6_logger = create_logger("ping6", log_folder, verbose)
for host in config_dict.get("ping6", []):
ping6_logger.info(';'.join(str(x) for x in ping6(host)))
http_logger = create_logger("http", log_folder, verbose)
url_dict = config_dict.get("url", {})
for url in url_dict:
http_logger.info(';'.join(str(x) for x in request(url, url_dict[url])))
def sendLocation(self,
chat_id,
latitude,
longitude,
disable_notification=None,
reply_to_message_id=None,
reply_markup=None):
data = {}
data['chat_id'] = chat_id
data['latitude'] = latitude
data['longitude'] = longitude
if (disable_notification):
data['disable_notification'] = disable_notification
if (reply_to_message_id):
data['reply_to_message_id'] = reply_to_message_id
if (reply_markup): data['reply_markup'] = reply_markup
return http.request(self.baseurl + self.token + '/sendLocation',
method='POST',
body=json.dumps(data),
headers={'Content-Type': 'application/json'})
def action_get(request):
keys = ['url', 'target', 'method', 'scheme', 'port']
kwargs = {k: request[k] for k in keys if k in request}
response = http.request(**kwargs)
if request.get('json', False):
response_message = {
'status': response.status,
'reason': response.reason
}
if request.get('headers', True):
response_message['headers'] = {k: v for (k, v) in response.getheaders()}
if request.get('raw', False):
response_message['raw'] = response.read(raw=True)
elif request.get('body', True):
response_message['body'] = response.read()
return response_message
else:
if request.get('raw', False):
return response.get_raw(), ResponseOptions(send_json=False)
else:
return response.body, ResponseOptions(send_json=False)
def sendMessage(self,
chat_id,
text,
parse_mode='Markdown',
disable_web_page_preview=True,
disable_notification=False,
reply_to_message_id=None,
reply_markup=None):
data = {
'chat_id': chat_id,
'text': text,
'parse_mode': parse_mode,
'disable_web_page_preview': disable_web_page_preview,
'disable_notification': disable_notification
}
if (reply_to_message_id):
data['reply_to_message_id'] = reply_to_message_id
if (reply_markup): data['reply_markup'] = reply_markup
return http.request(self.baseurl + self.token + '/sendMessage',
method='POST',
body=json.dumps(data),
headers={'Content-Type': 'application/json'})
def action_get(self, message, cb):
keys = ['url', 'target', 'method', 'scheme', 'port']
kwargs = {k: message[k] for k in keys if k in message}
response = http.request(**kwargs)
if message.get('json', False):
response_message = {
'status': response.status,
'reason': response.reason
}
if message.get('headers', True):
response_message['headers'] = {k: v for (k, v) in response.getheaders()}
if message.get('raw', False):
response_message['raw'] = response.read(raw=True)
elif message.get('body', True):
response_message['body'] = response.read()
cb(response_message, send_json=True)
else:
if message.get('raw', False):
cb(response.get_raw(), send_json=False)
else:
cb(response.body, send_json=False)
def sendMessage(self, chat_id, text, parse_mode='Markdown', disable_web_page_preview=True, disable_notification=False, reply_to_message_id=None, reply_markup=None):
data = {'chat_id': chat_id, 'text': text, 'parse_mode': parse_mode, 'disable_web_page_preview': disable_web_page_preview, 'disable_notification': disable_notification}
if(reply_to_message_id): data['reply_to_message_id'] = reply_to_message_id
if(reply_markup): data['reply_markup'] = reply_markup
return http.request(self.baseurl + self.token + '/sendMessage', method='POST', body=json.dumps(data), headers={'Content-Type': 'application/json'})
def download_resource_file(on_request, resDir):
global toneDir, on_callback, on_download, cb_data
toneDir = resDir
data = {
'url': on_request['url'],
'method': 'GET',
'headers': {
'content-type': 'application/x-www-form-urlencoded'
},
'timeout': 30000,
'params': ''
}
def cb(data):
global on_callback, cb_data
on_callback = True
cb_data = data
http.request(data, cb)
while True:
if on_callback:
on_callback = False
break
else:
time.sleep(1)
# {
# "audios":
# [
# { "format":"wav",
# "id":"zfbGet",
# "size":40204,
# "type":"custom",
# "url":"http://speech-solution.oss-cn-shanghai.aliyuncs.com/xxxx"
# }
# ],
# "format":"wav",
# "size":40204
# }
print(cb_data)
response = json.loads(cb_data['body'])
audio = response['audios'][0]
format = audio['format']
id = audio['id']
size = audio['size']
path = toneDir + id + '.' + format
print('************ begin to download: ' + path)
d_data = {'url': audio['url'], 'filepath': path}
def d_cb(data):
global on_download
on_download = True
http.download(d_data, d_cb)
while True:
if on_download:
on_download = False
break
else:
time.sleep(1)
print('download succeed :' + path)
def request_the_url_with_method(step, url, method):
lettuce.world.response = http.request(url, method)
lettuce.world.response = http.json_loads_data(lettuce.world.response)
def load(self, url):
headers, body = http.request(url)
text = http.lex(body)
self.display_list = self.layout(text)
self.render()
def getMe(self):
return http.request(self.baseurl + self.token + '/getMe',
method='GET',
headers={'Content-Type': 'application/json'})
def setWebhook(self, hook=None):
self.hook = hook or ''
return http.request(self.baseurl + self.token + '/setWebhook',
method='POST',
body=json.dumps({'url': hook}),
headers={'Content-Type': 'application/json'})
def sendChatAction(self, chat_id, action):
data = {}
data['chat_id'] = chat_id
data['action'] = action
return http.request(self.baseurl + self.token + '/sendChatAction', method='POST', body=json.dumps(data), headers={'Content-Type': 'application/json'})
def getUpdates(self, offset=None, limit=None, timeout=None):
data = {}
if(offset): data['offset'] = offset
if(limit): data['limit'] = limit
if(timeout): data['timeout'] = timeout
return http.request(self.baseurl + self.token + '/getUpdates', method='GET', body=json.dumps(data), headers={'Content-Type': 'application/json'})
def login(self,acc,pwd,vcode='',vcodestr=''):
url = 'https://passport.baidu.com/v2/api/?login'
pubkey , rsakey = self._get_publickey()
#print pubkey
#print rsakey
key = rsa.PublicKey.load_pkcs1_openssl_pem(pubkey)
password_rsaed = base64.b64encode(rsa.encrypt(pwd, key))
data = {}
data[u'staticpage'] = 'https://passport.baidu.com/static/passpc-account/html/v3Jump.html'
data[u'charset'] = 'UTF-8'
data[u'token'] = self.token
data[u'tpl']='tb'
data[u'subpro'] = ''
data[u'apiver'] = 'v3'
data[u'tt']=str(int(time.time()))
data[u'codestring']=vcodestr
data[u'safeflg'] = 0
data[u'u']='https://passport.baidu.com/'
data[u'isPhone'] = ''
data[u'detect']= 1
data[u'gid']='6A3375C-5CE5-4696-A8C2-8BBE62516FAF'
data[u'quick_user']=0
data[u'logintype']='basicLogin'
data[u'logLoginType']= 'pc_loginBasic'
data[u'idc']=''
data[u'loginmerge']='true'
data[u'username']=acc
data[u'password']=password_rsaed
data[u'verifycode'] = vcode
data[u'rsakey']=rsakey
data[u'crypttype']=12
data[u'countrycode']=''
data[u'callback']=0
# start login
data=http.request(url,data)
#data=json.loads(data)
err_no,code_string = self._check_login_error(data)
'''
if data[u'errInfo'][u'no']=='0':
#login success & need save bduss
bduss=data['user']['BDUSS'].encode('utf-8')
bduss = 'BDUSS='+bduss.decode()
self.__writebduss__(bduss)
'''
'''
if data[u'errInfo'][u'no']=='500001':
url = 'http://wappass.baidu.com/cgi-bin/genimage?'+data[u'data'][u'codeString'] + '&v='+str(int(time.time()*1000))
http.download(url,cur_dir()+u'/',u'vcode.jpg')
vc = raw_input('请输入验证码:')
return self.login(baiduUtf(acc),base64.b64encode(pwd.encode('utf-8')).decode(),vc,data[u'data'][u'codeString'])
'''
# 是否需要验证码
if err_no=='257' or err_no=='6' :
url = 'https://passport.baidu.com/cgi-bin/genimage?'+ code_string + '&v='+str(int(time.time()*1000))
http.download(url,cur_dir()+u'/',u'vcode.jpg')
vc = raw_input('请输入验证码:')
self.login(acc,pwd,vc,code_string)
if err_no=='0':
# bduss = None
for item in http.cookies():
if item.name == 'BDUSS':
self.bduss = item.value
if item.name == 'BAIDUID':
self.baiduid = item.value
#self.bdauth = 'BAIDUID=' + self.baiduid + '; BDUSS=' + self.bduss
#print self.bdauth
self.__writebduss__(self.bduss,self.baiduid)
print 'relogin access!'
def _get_publickey(self):
url = 'https://passport.baidu.com/v2/getpublickey?token=' + \
self.token
content = http.request(url)
jdata = json.loads(content.replace('\'','"'))
return (jdata['pubkey'], jdata['key'])
def setWebhook(self, hook=None):
self.hook = hook or ''
return http.request(self.baseurl + self.token + '/setWebhook', method='POST', body=json.dumps({'url': hook}), headers={'Content-Type': 'application/json'})
ftp.login(user='******', passwd='REPLACE')
np.save('./data/array', np.random.standard_normal((100, 100)))
f = open('./data/array.npy', 'r')
ftp.storbinary('STOR array.npy', f)
ftp.retrlines('LIST')
f = open('./data/array_ftp.npy', 'wb').write
ftp.retrbinary('RETR array.npy', f)
ftp.delete('array.npy')
ftp.retrlines('LIST')
ftp.close()
###httplib
import http
import http.client
http = http.client.HTTPConnection('hilpisch.com')
http.request('GET', '/index.htm')
#使用getresponse方法测试请求是否成功
resp = http.getresponse()
resp.status, resp.reason
#读取内容
content = resp.read()
content[:100]
index = content.find(b' E ')
index
content[index:index + 29]
http.close()
###urllib
import urllib.request
url = 'http://ichart.finance.yahoo.com/table.csv?g=d&ignore=.csv'
url += '&s=YHOO&a=01&b=1&c=2014&d=02&e=6&f=2014'
def getMe(self):
return http.request(self.baseurl + self.token + '/getMe', method='GET', headers={'Content-Type': 'application/json'})
from socket import *
import urllib.request
import urllib3
import http
#serverName='localhost'
#serverPort=1200
#clientSocket=socket(AF_INET, SOCK_STREAM)
#clientSocket.connect((serverName, serverPort))
#sentence=('Ingrese texto en minusculas:')
#clientSocket.send(bytes(sentence, encoding='utf8'))
#modifiedSentence=clientSocket.recv(1024)
#print ('Desde el servidor:',modifiedSentence)
#clientSocket.close()
i = 0
while (1):
#contents = urllib.request.urlopen("http://localhost:1200").read()
http = urllib3.PoolManager()
response = http.request('GET', 'http://localhost:1200', headers={
'key1': 'value1',
'key2': 'value2'
})
print (response)
print (i)
i = i+1
def request(self, method, **kwargs):
"""do an http request."""
response = http.request(method, self.url(),
encode=json.dumps, **kwargs)
if response.ok:
return json.loads(response.get_content())
def forwardMessage(self, chat_id, from_chat_id, message_id, disable_notification=None):
data = {'data_id': data_id, 'from_chat_id': from_chat_id, 'message_id': message_id}
if(disable_notification): data['disable_notification'] = disable_notification
return http.request(self.baseurl + self.token + '/forwardMessage', method='POST', body=json.dumps(data), headers={'Content-Type': 'application/json'})
def login(self,acc,pwd,vcode='',vcodestr=''):
#url = 'https://passport.baidu.com/v2/api/?login' # 抄袭不如自己搞一遍
url = 'http://wappass.baidu.com/wp/api/login'
pubkey , rsakey = self._get_publickey()
#print pubkey
#print rsakey
key = rsa.PublicKey.load_pkcs1_openssl_pem(pubkey)
password_rsaed = base64.b64encode(rsa.encrypt(pwd, key))
data = {}
data[u'loginInitType']='0'
data[u'logLoginType']= 'sdk_login'
data[u'regLink'] = '1'
data[u'loginLink'] = '1'
data[u'isPhone'] = '0'
data[u'dialogVcodestr'] = '0'
data[u'code'] = ''
data[u'gid']=GID
data[u'countrycode']=''
data[u'clientfrom']='native'
data[u'regtype']='1'
data[u'password']=password_rsaed
# 可以传空
data[u'verifycode'] = vcode
data[u'client']='ios'
data[u'action']='login'
data[u'adapter']='1'
data[u'dialogVerifyCode']=''
data[u'servertime'] = '4cd9f60139'
data[u'tpl']='tb'
data[u'dialogVcodesign'] = ''
data[u'smsLoginLink'] = '1'
data[u'v'] = str(int(time.time()))
data[u'login_share_strategy'] = 'silent'
data[u'loginmerge']='1'
data[u'username']=acc
# start login
data=http.request(url,data)
data=json.loads(data)
#err_no,code_string = self._check_login_error(data)
err_no = data[u'errInfo'][u'no']
err_msg = data[u'errInfo'][u'msg']
print err_no + err_msg
'''
if data[u'errInfo'][u'no']=='0':
#login success & need save bduss
bduss=data['user']['BDUSS'].encode('utf-8')
bduss = 'BDUSS='+bduss.decode()
self.__writebduss__(bduss)
'''
'''
if data[u'errInfo'][u'no']=='500001':
url = 'http://wappass.baidu.com/cgi-bin/genimage?'+data[u'data'][u'codeString'] + '&v='+str(int(time.time()*1000))
http.download(url,cur_dir()+u'/',u'vcode.jpg')
vc = raw_input('请输入验证码:')
return self.login(baiduUtf(acc),base64.b64encode(pwd.encode('utf-8')).decode(),vc,data[u'data'][u'codeString'])
'''
# 是否需要验证码
'''
if err_no=='257' or err_no=='6' :
url = 'https://passport.baidu.com/cgi-bin/genimage?'+ code_string + '&v='+str(int(time.time()*1000))
http.download(url,cur_dir()+u'/',u'vcode.jpg')
vc = raw_input('请输入验证码:')
self.login(acc,pwd,vc,code_string)
'''
if err_no=='0':
# bduss = None
for item in http.cookies():
if item.name == 'BDUSS':
self.bduss = item.value
if item.name == 'BAIDUID':
self.baiduid = item.value
#self.bdauth = 'BAIDUID=' + self.baiduid + '; BDUSS=' + self.bduss
#print self.bdauth
self.__writebduss__(self.bduss,self.baiduid)
print 'relogin access!'
def make_request(self, url, target=None, *args):
"""
Make a http request using CacheBrowser
"""
response = http.request(url, target=target)
self.send_line(response.read())
def performTaskWithoutBrowser(self, url):
try:
url = str(url)
'check only those urls which starts with http and not traversed earlier and having lenskart domain'
if (url in self.globalTraversedSet):
pass
elif not (self.ifLenskartDomain(url)):
pass
elif not (url.startswith('http')):
pass
else:
try:
'keep a copy so that its not traversed again'
lock = threading.RLock()
lock.acquire(blocking=True)
try:
self.globalTraversedSet.add(url)
finally:
lock.release()
'trying a different library'
# response = requests.get(url)
# pageSource = response.text
# status_code = response.status_code
urllib3.disable_warnings()
http = urllib3.PoolManager()
try:
response = http.request('GET', url)
pageSource = str(response.data)
status_code = response.status
except Exception:
pageSource = 'This page isn’t working'
status_code = int(200)
print(' ^^^^^^^^^^^^^^^^^ status_code ====> ', status_code,
' url ===> ' + url)
if (str(status_code).startswith('4')
| str(status_code).startswith('5')):
print(
'Found a non responsive page ==> ' + url +
" status code ==> ", status_code)
lock = threading.RLock()
lock.acquire(blocking=True)
try:
self.damnPagesMap.update({url: status_code})
finally:
lock.release()
elif (pageSource.__contains__('DAMN!!')):
print('Found a DAMN Page ==> ' + url)
lock = threading.RLock()
lock.acquire(blocking=True)
try:
self.damnPagesMap.update({url: 'DAMN'})
finally:
lock.release()
elif (pageSource.__contains__("This page isn’t working")):
print("This page isn't working ==> " + url)
lock = threading.RLock()
lock.acquire(blocking=True)
try:
self.damnPagesMap.update({url: 'Not_Working_Page'})
finally:
lock.release()
else:
''' apply regex to get urls from response - urls starting with http '''
urlList = re.findall('(?<=href=").*?(?=")', pageSource)
''' update the global url set and list '''
lock = threading.RLock()
lock.acquire(blocking=True)
try:
'convert received urls into set for uniqueness and map and remove non http urls + already browsed urls'
for x in urlList:
if ((x.startswith('http')) &
(self.ifLenskartDomain(x)) &
((self.globalUrlMap.get(x) == None) |
(self.globalUrlMap.get(x) == False))):
self.globalUrlMap.update({x: False})
else:
urlList.remove(x)
print('After Updating traversed ==> ',
len(self.globalTraversedSet),
' global map ==> ', len(self.globalUrlMap),
' global DAMN map ==> ',
len(self.damnPagesMap))
except Exception:
traceback.print_exc(file=sys.stdout)
finally:
lock.release()
except Exception:
print('exception occurred with url ==> ' + url)
traceback.print_exc(file=sys.stdout)
# print('final global traversed set ==> ',len(self.globalTraversedSet), ' global map ==> ', len(self.globalUrlMap), ' global DAMN map ==> ' , len(self.damnPagesMap))
except Exception:
traceback.print_exc(file=sys.stdout)
def run_block(runner, block):
testcase_vars = {}
success = True
if block.__class__.__name__=='TestCase':
if not match_tags(runner.tags, block.tags): return True # tags don't match, won't run
else: block.executed = True
print '\nTest Case \'%s\'' % block.id
for step in block.steps:
runner.global_vars['random'] = str(uuid.uuid4()).replace('-','') # a global random value
if step.__class__.__name__ == 'Var':
if block.__class__.__name__ == 'Block':
runner.global_vars[step.name] = jinja2.Template(step.value).render(runner.global_vars)
step.value = runner.global_vars[step.name]
else:
testcase_vars[step.name] = jinja2.Template(step.value).render(merge_dicts(runner.global_vars, testcase_vars))
step.value = testcase_vars[step.name]
elif step.__class__.__name__ == 'Sleep':
step.value = int(jinja2.Template(step.value).render(merge_dicts(runner.global_vars, testcase_vars)))
print ' * sleeping for %s seconds...' % step.value
sleep(step.value)
elif step.__class__.__name__ == 'Print':
step.value = jinja2.Template(step.value).render(merge_dicts(runner.global_vars, testcase_vars))
print ' * %s' % step.value
elif step.__class__.__name__ == 'ClearCookies':
http.clear_cookies()
print ' * Cookies cleared'
elif step.__class__.__name__ == 'CallFunction':
print ' * Calling function "%s"...' % (step.file),
for k in step.params.keys(): step.params[k] = jinja2.Template(step.params[k]).render(merge_dicts(runner.global_vars, testcase_vars))
step.passed, step.return_value = callfunctions.run(runner, step)
print 'OK' if step.passed else '*FAILED*'
if step.save_global:
runner.global_vars[step.save_global] = step.return_value
testcase_vars[step.save_global] = step.return_value
if step.save_var:
testcase_vars[step.save_var] = step.return_value
if step.save_file:
saved, log = save_file(step.save_file, step.return_value)
print log
if not saved: step.passed = False
elif step.__class__.__name__ == 'Request':
#<Request>
print ' * Running request \'%s\'...' % step.id,
local_vars = {}
for k in step.vars.keys(): local_vars[k] = jinja2.Template(step.vars[k]).render(merge_dicts(runner.global_vars, testcase_vars, local_vars))
step.context_vars = merge_dicts(runner.global_vars, testcase_vars, local_vars)
if step.path: step.path = jinja2.Template(step.path).render(step.context_vars)
if step.method: step.method = jinja2.Template(step.method).render(step.context_vars)
if step.body: step.body = jinja2.Template(step.body).render(step.context_vars)
for k in step.path_fields.keys():
step.path_fields[k] = jinja2.Template(step.path_fields[k]).render(step.context_vars)
for k in step.body_file_headers.keys():
for h in step.body_file_headers[k]:
h = jinja2.Template(h).render(step.context_vars)
for k in step.body_fields.keys():
step.body_fields[k] = jinja2.Template(step.body_fields[k]).render(step.context_vars)
for k in step.body_files.keys():
step.body_files[k] = jinja2.Template(step.body_files[k]).render(step.context_vars)
for k in step.headers.keys():
step.headers[k] = jinja2.Template(step.headers[k]).render(step.context_vars)
#The Request
if step.path_full:
url = step.path
else:
if step.base_url: url = runner.other_base_urls[step.base_url]
else: url = runner.base_url
url = url if url[-1]!='/' else url[:-1]
if step.path: url = '%s%s%s' % (url, '/' if step.path[0]!='/' else '', step.path)
if step.path_fields: # querystring
querystring = ''
for qs in step.path_fields.keys():
if step.path_fields[qs]:
querystring += "&%s=%s" % (qs, step.path_fields[qs])
if '?' not in url: querystring = querystring.replace('&', '?', 1)
url = '%s%s' % (url, querystring)
step.path = url
#TODO: encode url
already_set_cookie = False
for h in step.headers.keys():
if h.lower() == 'cookie': already_set_cookie = True
if not already_set_cookie:
cookie = http.get_cookie(runner.base_url)
if cookie: step.headers['cookie'] = cookie
#TODO: Certificate
response = http.request(url=url, method=step.method, body=step.body, headers=step.headers, fields=step.body_fields, files=step.body_files, file_headers=step.body_file_headers, proxy=None if step.force_no_proxy else runner.proxy)
step.resp_status, step.resp_headers, step.resp_body, step.resp_time = response.status, response.headers, response.body, (response.finished_on-response.started_on)
if ('set-cookie' in step.resp_headers.keys()):
http.save_cookie(runner.base_url, step.resp_headers['set-cookie'])
for a in step.assertions:
if not a.value: a.value = ''
a.value = jinja2.Template(a.value).render(step.context_vars)
if not assertions.run_assertion(a, step):
step.passed = False
success = False
if a.save_global:
runner.global_vars[a.save_global] = a.received
step.context_vars[a.save_global] = a.received #so it can be use on the next assertion
if a.save_var:
testcase_vars[a.save_var] = a.received
step.context_vars[a.save_var] = a.received #so it can be use on the next assertion
if a.save_file:
saved, log = save_file(a.save_file, a.received)
print log
if not saved: a.passed, step.passed = False, False # if I can't save content, assertion fails
print 'OK' if step.passed else '*FAILED*'
if step.desc: step.desc = jinja2.Template(step.desc).render(step.context_vars)
if block.desc: block.desc = jinja2.Template(block.desc).render(step.context_vars)
#</Request>
elif step.__class__.__name__ in ['Block', 'TestCase']:
if not run_block(runner, step): success = False
return success
def getUserProfilePhotos(self, user_id, offset=None, limit=None):
data = {}
data['user_id'] = user_id
if(offset): data.append(('offset', offset))
if(limit): data.append(('limit', limit))
return http.request(self.baseurl + self.token + '/getUserProfilePhotos', method='POST', body=json.dumps(data), headers={'Content-Type': 'application/json'})
file.write(compiled.cpp_file())
# Automaton no. 1: accept everything up to a comma, eg `.*,`.
until_comma = Automaton()
start = until_comma.start()
comma = until_comma.add_state("Comma")
comma.mark_enter()
start.add_transition(',', LabelType.Char, comma)
start.add_transition("All", LabelType.Special, start)
output("until_comma", until_comma)
# Automaton no. 2: http method terminated by a space
method, final = methods()
output("http_method", method)
# Automaton no. 3: HTTP request line
http_req, final = req_line()
output("http_req", http_req)
# A whole request line + headers, ending with a body
want_headers = {
'X-Api-Key': read_header_value('XApiKey'),
'Content-Length': read_header_value('ContentLength'),
'Content-Type': read_boundary(),
'Connection': connection_header(),
}
http, final = request(want_headers)
output("http", http)
