def test_valid_doi_status_200(self):
mock_is_valid = self.mocker.mock()
mock_is_valid.status_code
self.mocker.result(200)
requests = self.mocker.replace("requests.get")
requests("http://dx.doi.org/10.1590/S2179-975X2012005000031", timeout=2.5)
self.mocker.result(mock_is_valid)
self.mocker.replay()
self.assertTrue(utils.is_valid_doi("10.1590/S2179-975X2012005000031"))
def test_valid_doi_status_500(self):
mock_is_valid = self.mocker.mock()
mock_is_valid.status_code
self.mocker.result(500)
requests = self.mocker.replace("requests.get")
requests('http://dx.doi.org/10.1590/S2179-975X2012005XXXX', timeout=2.5)
self.mocker.result(mock_is_valid)
self.mocker.replay()
self.assertFalse(utils.is_valid_doi('10.1590/S2179-975X2012005XXXX'))
def getdata(self):
self.rates = {}
# grab the data set from the the Bank of Canada exchange
try:
date = "Unknown"
fl = requests("http://www.bankofcanada.ca/en/markets/csv/exchange_eng.csv")
fh = fl.read()
# iterate through file
for line in fh:
line = line.rstrip()
if not line or line.startswith(("#", "Closing")):
continue
fields = line.split(",")
if line.startswith("Date "):
date = fields[-1]
else:
try:
value = float(fields[-1])
self.rates[fields[0]] = value
except ValueError:
pass
return "Exchange rates date: " + date
except Exception as e:
return "Failed to download: \n%s" % e
def crawl_chapters():
print 'get update tasks from %s...' % yiwanshu
for book in get_tasks():
bid = book.get('bid')
source_site = book.get('source_site')
latest_chapter = book.get('latest_chapter')
url = book.get('source_url')
print 'update book %s, try to get new chapters from %s' % (bid, url)
try:
chapters = get_all_chapters(url, source_site)
except requests(Timeout, ConnectionError, HTTPError):
print 'get chapters timeout. %s' % url
continue
new_chapters = get_new_chapters(url, bid, chapters, latest_chapter,
crawler_name)
if len(new_chapters) == 0:
print 'no new chapters\n'
continue
print '%d new chapters.' % len(new_chapters)
for title, url in new_chapters:
try:
content = get_content(url, source_site)
update(bid, content, title, crawler_name, 'text')
print 'update book %s, chapter %s' % (bid, title)
except (Timeout, ConnectionError, HTTPError):
print 'get content timeout. %s' % url
continue
print 'book %s update finish.\n\n' % bid
def Parse_Page(self,url):
response=requests(url,headers=random.choice(self.headers))
selector=etree.HTML(response.content)
content_fields=selector.xpath('//div[@class="l_post j_l_post l_post_bright "]')
item={}
for content in content_fields:
data_field=json.loads(content.xpath('@data-field')[0])
author=data_field['author']['user_time']
def getSportLinks(url, handle):
r = requests(url)
match = re.compile('<a href=(.+?) .+? gday=(.+?) class="matchtime">(.+?)</span> (.+?)</a>', re.DOTALL).findall(r.content)
for href,gameday,gametime,name in match:
addLink('[%s - %s] %s'%(gameday,gametime,name), href, 'play',name,name+'_fanart', handle)
match2 = re.compile('<div class=.+?>(.+?)</div><div class=(.+?)>: <a target="_blank" title=(.+?) href=(.+?)>(.+?)</a> </div>', re.DOTALL).findall(r.content)
for defi,cls,title,href2,name2 in match2:
if name == title:
addLink('%s: %s'%(defi,name2), href2, 'play',name2,name2+'_fanart', handle)
def get_tags(token, newerThan=0, url=SandBoxURL):
payload = {}
if newerThan > 0 :
payload['newerThan'] = newerThan
return requests(url++'/v3/markers/tags',
params=payload,
headers=get_auth_headers(token)
).json()
import requests
url = ('https://newsapi.org/v2/top-headlines?'
'country=ghana&'
'apiKey=58cf5cb108cb493eb6e82291ad58c3f6')
requests = requests.get(url)
print (requests())
import requests, json
data_list = []
for i in range(20):
url = 'https://api.themoviedb.org/3/movie/now_playing?api_key={}&language=ko-KR&page={}'.format(
key, i + 1)
response = requests(url).json()['results']
data_list.extend(response)
records = [json.loads(line) for line in ul.urlopen("https://raw.githubusercontent.com/pydata/pydata-book/master/ch02/usagov_bitly_data2012-03-16-1331923249.txt")]
import urllib2
import requests
r = requests.get('https://raw.githubusercontent.com/pydata/pydata-book/master/ch02/usagov_bitly_data2012-03-16-1331923249.txt')
r.status_code
r.headers['content-type']
r.encoding
r.txt
r.text
r.json()
r.text
import json
records = [json.loads(line) for line in r.text]
r.json()
import requests
r = requests("https://raw.githubusercontent.com/yukoga/python-for-data-analysis-review/master/usagov_bitly_data2012-03-16-1331923249.json")
r = requests.get('https://raw.githubusercontent.com/yukoga/python-for-data-analysis-review/master/usagov_bitly_data2012-03-16-1331923249.json')
r.status_code
r.
r.headers["content-type"]
r.json()
r = requests.get('http://stackoverflow.com/questions/21058935/python-json-loads-shows-valueerror-extra-data')
r.text
r.json()
import json
r = requests.get('https://raw.githubusercontent.com/yukoga/python-for-data-analysis-review/master/usagov_bitly_data2012-03-16-1331923249.json')
records = [json.loads(line) for line in r]
records = [json.loads(line) for line in r.text]
json.loads('{key1:1,key2:2,key3:3}')
from igraph import *
g = Graph()
import requests
payload = {'ie': 'utf-8', 'kw': 'red', 'fr': 'search'}
r = requests('http://tieba.baidu.com/f', param=payload)
print(r.url)
elif ruq == 'get':
r = requests.get(url,api)
else:
print("没有设置协议类型，请先设置！")
# print(r.text)
# print("接口请求成功！")
return r
num = 0
for j in (api):
num = num + 1
y1 = ""
y1 = "第"+str(num)+"请求数据："+str(j)+"\n"
# print(y1)
log(y1,"test")
x = requests(j,url,ruq)
# x1 = eval(x.text)
y2 = ""
y2 = "第"+str(num)+"返回参数："+str(x.json())+"\n"
log(y2,"test")
if x.ok:
print("第",num,"次接口请求成功！")
x1 = x.json()
# print(x1['OutputData'])
x2 = x1['OutputData'][0]['Success']
# x3 = x2[0]['Data'][0]['Success']
if x2 == "True":
print("第",num,"接口返回参数正确！")
else:
print("第",num,"接口返回参数错误，请检查日志！")
else:
#!/usr/bin/env python
"""
Requirements:
requests (installation: pip install requests)
"""
import requests
import os
SEARCH_URL = 'http://127.0.0.1:9000/api/search.json'
_ALL_FILES_ = []
for (d, f, filenames) in os.walk(os.path.join(os.getcwd(),
'../test/queries/')):
if filenames == "README.txt":
continue
else:
_ALL_FILES_ = filenames
for single_file in _ALL_FILES_:
path_to_file = os.path.join(os.path.abspath('../test/queries'),
single_file)
with open(path_to_file, 'rb') as f:
queries = f.readlines()
for query in queries:
resp = requests.get(url=SEARCH_URL, params={'source': 'cache',
'q': query.strip()})
data = resp.json()
statuses = data['statuses']
for status in statuses:
import requests
headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"}
req = requests("http://jnhcm.com",headers=headers)
print(req.status_code)
while index < len(data_lode["availableCourses"]):
available_Courses = (data_lode["availableCourses"][index])
coursesName = available_Courses["name"]
coursesId = available_Courses["id"]
courses_Id_list.append(coursesId)
print(index), coursesName, coursesId
index = index + 1
return courses_Id_list
if os.path.exists("courses.json"):
getData = get_Read("courses.json")
get_Av_Courses(getData)
print "kkkkkkkkkkkkkkkkk"
else:
requests(url_link)
getData = get_Read("courses.json")
get_Av_Courses(getData)
# user_ex = int(raw_input("enter a number"))
# userEx = courses_Id_list [user_ex - 1]
# print userEx
# exercise_url = url_link+"/"+str(userEx)+"/exercises"
# def get_exercise(link):
# ex_data = requests.get(exercise_url)
# return (ex_data.json())
# data_e = get_exercise(exercise_url)
# data_exer=data_e["data"]
# def ex_get_Write(ex_data,user_ex):
def book_spider(book_tag):
page_num = 0
book_list = []
try_times = 0
while (1):
# url='http://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book?start=0' # For Test
url = 'http://www.douban.com/tag/' + urllib.quote(
book_tag) + '/book?start=' + str(page_num * 15)
time.sleep(np.random.rand() * 5)
# Last Version
try:
req = requests(url, headers=hds[page_num % len(hds)])
source_code = urllib2.urlopen(req).read()
plain_text = str(source_code)
except (urllib2.HTTPError, urllib2.URLError) as e:
print(e)
continue
##Previous Version, IP is easy to be Forbidden
# source_code = requests.get(url)
# plain_text = source_code.text
soup = BeautifulSoup(plain_text)
list_soup = soup.find('div', {'class': 'mod book-list'})
try_times += 1
if list_soup == None and try_times < 200:
continue
elif list_soup == None or len(list_soup) <= 1:
break # Break when no informatoin got after 200 times requesting
for book_info in list_soup.findAll('dd'):
title = book_info.find('a', {'class': 'title'}).string.strip()
desc = book_info.find('div', {'class': 'desc'}).string.strip()
desc_list = desc.split('/')
book_url = book_info.find('a', {'class': 'title'}).get('href')
try:
author_info = '作者/译者： ' + '/'.join(desc_list[0:-3])
except:
author_info = '作者/译者： 暂无'
try:
pub_info = '出版信息： ' + '/'.join(desc_list[-3:])
except:
pub_info = '出版信息： 暂无'
try:
rating = book_info.find('span', {
'class': 'rating_nums'
}).string.strip()
except:
rating = '0.0'
try:
# people_num = book_info.findAll('span')[2].string.strip()
people_num = get_people_num(book_url)
people_num = people_num.strip('人评价')
except:
people_num = '0'
book_list.append(
[title, rating, people_num, author_info, pub_info])
try_times = 0 # set 0 when got valid information
page_num += 1
print('Downloading Information From Page %d' % page_num)
return book_list
#!/usr/bin/env python
"""
Requirements:
requests (installation: pip install requests)
"""
import requests
import json
import os
SEARCH_URL = 'http://localhost:9000/api/search.json'
_ALL_FILES_ = []
for (d, f, filenames) in os.walk(os.path.join(os.getcwd(),
'../test/queries/')):
if filenames == "README.txt":
continue
else:
_ALL_FILES_ = filenames
for single_file in _ALL_FILES_:
path_to_file = os.path.join(os.path.abspath('../test/queries'),
single_file)
with open(path_to_file, 'rb') as f:
queries = f.readlines()
for query in queries:
resp = requests.get(url=SEARCH_URL,
params={
'source': 'cache',
'q': query.strip()
import requests
r = requests("https://www.schoology.com/")
while True:
print(r.header)
def getWeatherFiveDays(city):
appid = 'b1b15e88fa797225412429c1c50c122a1'
url = ('https://api.openweathermap.org/data/2.5/forecast?q=%s' % (city))
r = requests(url, verify=False)
print(r.text)
def getEstudante(id):
url = ('https://fierce-island-41854.herokuapp.com/api/estudante/%s' % (id))
r = requests(url, verify=False)
print(r.text)
json=data,
**kwargs)
else:
res = self.my_session.request(methmod,
url,
data=data,
**kwargs)
else:
res = None
print("不支持{}的请求方法".format(methmod))
return res
self.my_session.close()
if __name__ == '__main__':
# 注册
url_1 = "http://120.78.128.25:8080/futureloan/mvc/api/member/register"
data1 = {"mobilephone": "13666661661", "pwd": "123456"}
# 登陆
login_url = "http://120.78.128.25:8080/futureloan/mvc/api/member/login"
# 充值
recharge_url = "http://120.78.128.25:8080/futureloan/mvc/api/member/recharge"
recharge_data = {"mobilephone": "13666661661", "amount": 2345}
requests = MyRequests()
res_1 = requests("post", url=url_1, data=data1)
res_2 = requests("post", url=login_url, data=data1)
res_3 = requests("post", url=recharge_url, data=recharge_data)
print(res_1.text)
print(res_2.text)
print(res_3.text)
def requests():
# 总共处理的请求数
print nginxStatus[2].split()[2]
try:
if sys.argv[1]:
pass
except IndexError, e:
print Fore.GREEN + '#####'
print './%s [active|reading|writing|waiting|accepts|requests]' % sys.argv[0]
print Fore.GREEN + '#####'
else:
if sys.argv[1] in 'active':
active()
elif sys.argv[1] in 'reading':
reading()
elif sys.argv[1] in 'writing':
writing()
elif sys.argv[1] in 'waiting':
waiting()
elif sys.argv[1] in 'accepts':
accepts()
elif sys.argv[1] in 'handled':
handled()
elif sys.argv[1] in 'requests':
requests()
else:
print './%s [active|reading|writing|waiting|accepts|requests]' % sys.argv[0]
def download_imgs(self, imgs_url):
img = requests(imgs_url)
with open(file, 'wb') as img_file:
img_file.write(img)
'''import requests
r = requests.get("https://maker.ifttt.com/trigger/Beautiful/with/key/p0jWqMouy0MlwbXYQ6oQeSjaHnAcmIrLJhqzwRXciJ2")
c = r.status_code
print(c)'''
'''import requests
r = requests.get("https://maker.ifttt.com/trigger/HTU/with/key/p0jWqMouy0MlwbXYQ6oQeSjaHnAcmIrLJhqzwRXciJ2")
c = r.status_code
print(c)'''
import requests
name = "Bridget"
message = "Hi, how was lunch? you are splendid."
r = requests.get(
"https://maker.ifttt.com/trigger/message/messenger/with/key/2Kud4E0CUmxZX3JAbrzbl"
)
print(r)
requests()
def handle_ssh_request(request):
return requests(request)
'start_date': '2020-01-01T00:00:00+00:00',
'is_processing': ''
})
response = requests(
method='POST',
url='https://' + ip + '/api/v1/assets/',
verify=False,
data={
"mimetype": "webpage",
"asset_id": "male_ad3",
"end_date": "9999-01-01T00:00:00+00:00",
"name": "male_ad3",
"nocache": "",
"is_enabled": "1",
"is_active": 1,
"uri":
"https://www.telegraph.co.uk/content/dam/men/2018/07/02/TELEMMGLPICT000168190077_trans_NvBQzQNjv4BqpVlberWd9EgFPZtcLiMQfyf2A9a6I9YchsjMeADBa08.jpeg",
"start_date": "2020-01-01T00:00:00+00:00",
"duration": 15,
"skip_asset_check": 1,
"is_processing": ""
},
# data={'asset_id': 'male_ad4', 'mimetype': 'webpage', 'name': 'male_ad4', 'end_date': '9999-01-01T00:00:00+00:00', 'is_enabled': 1, 'nocache': '', 'is_active': 1, 'uri': 'https://www.thetrendspotter.net/wp-content/uploads/2017/01/Best-Clothing-Stores-men.jpg', 'skip_asset_check': 1, 'duration': '15', 'play_order': 1, 'start_date': '2020-01-01T00:00:00+00:00', 'is_processing': '0'},
# headers={
# # "Authorization": "Token {token}",
# "accept": "application/json",
# "content-Type": "application/json"
# }
)
print(response.json())
# response = requests.request(
from bs4 import BeautifulSoup
import requests, re
r = requests().content
soup = BeautifulSoup(r, "xtml")
soup.findAll("a", "href" :re.compile
)
