def test_valid_doi_status_200(self): mock_is_valid = self.mocker.mock() mock_is_valid.status_code self.mocker.result(200) requests = self.mocker.replace("requests.get") requests("http://dx.doi.org/10.1590/S2179-975X2012005000031", timeout=2.5) self.mocker.result(mock_is_valid) self.mocker.replay() self.assertTrue(utils.is_valid_doi("10.1590/S2179-975X2012005000031"))
def test_valid_doi_status_500(self): mock_is_valid = self.mocker.mock() mock_is_valid.status_code self.mocker.result(500) requests = self.mocker.replace("requests.get") requests('http://dx.doi.org/10.1590/S2179-975X2012005XXXX', timeout=2.5) self.mocker.result(mock_is_valid) self.mocker.replay() self.assertFalse(utils.is_valid_doi('10.1590/S2179-975X2012005XXXX'))
def getdata(self): self.rates = {} # grab the data set from the the Bank of Canada exchange try: date = "Unknown" fl = requests("http://www.bankofcanada.ca/en/markets/csv/exchange_eng.csv") fh = fl.read() # iterate through file for line in fh: line = line.rstrip() if not line or line.startswith(("#", "Closing")): continue fields = line.split(",") if line.startswith("Date "): date = fields[-1] else: try: value = float(fields[-1]) self.rates[fields[0]] = value except ValueError: pass return "Exchange rates date: " + date except Exception as e: return "Failed to download: \n%s" % e
def crawl_chapters(): print 'get update tasks from %s...' % yiwanshu for book in get_tasks(): bid = book.get('bid') source_site = book.get('source_site') latest_chapter = book.get('latest_chapter') url = book.get('source_url') print 'update book %s, try to get new chapters from %s' % (bid, url) try: chapters = get_all_chapters(url, source_site) except requests(Timeout, ConnectionError, HTTPError): print 'get chapters timeout. %s' % url continue new_chapters = get_new_chapters(url, bid, chapters, latest_chapter, crawler_name) if len(new_chapters) == 0: print 'no new chapters\n' continue print '%d new chapters.' % len(new_chapters) for title, url in new_chapters: try: content = get_content(url, source_site) update(bid, content, title, crawler_name, 'text') print 'update book %s, chapter %s' % (bid, title) except (Timeout, ConnectionError, HTTPError): print 'get content timeout. %s' % url continue print 'book %s update finish.\n\n' % bid
def Parse_Page(self,url): response=requests(url,headers=random.choice(self.headers)) selector=etree.HTML(response.content) content_fields=selector.xpath('//div[@class="l_post j_l_post l_post_bright "]') item={} for content in content_fields: data_field=json.loads(content.xpath('@data-field')[0]) author=data_field['author']['user_time']
def getSportLinks(url, handle): r = requests(url) match = re.compile('<a href=(.+?) .+? gday=(.+?) class="matchtime">(.+?)</span> (.+?)</a>', re.DOTALL).findall(r.content) for href,gameday,gametime,name in match: addLink('[%s - %s] %s'%(gameday,gametime,name), href, 'play',name,name+'_fanart', handle) match2 = re.compile('<div class=.+?>(.+?)</div><div class=(.+?)>: <a target="_blank" title=(.+?) href=(.+?)>(.+?)</a> </div>', re.DOTALL).findall(r.content) for defi,cls,title,href2,name2 in match2: if name == title: addLink('%s: %s'%(defi,name2), href2, 'play',name2,name2+'_fanart', handle)
def get_tags(token, newerThan=0, url=SandBoxURL): payload = {} if newerThan > 0 : payload['newerThan'] = newerThan return requests(url++'/v3/markers/tags', params=payload, headers=get_auth_headers(token) ).json()
import requests url = ('https://newsapi.org/v2/top-headlines?' 'country=ghana&' 'apiKey=58cf5cb108cb493eb6e82291ad58c3f6') requests = requests.get(url) print (requests())
import requests, json data_list = [] for i in range(20): url = 'https://api.themoviedb.org/3/movie/now_playing?api_key={}&language=ko-KR&page={}'.format( key, i + 1) response = requests(url).json()['results'] data_list.extend(response)
records = [json.loads(line) for line in ul.urlopen("https://raw.githubusercontent.com/pydata/pydata-book/master/ch02/usagov_bitly_data2012-03-16-1331923249.txt")] import urllib2 import requests r = requests.get('https://raw.githubusercontent.com/pydata/pydata-book/master/ch02/usagov_bitly_data2012-03-16-1331923249.txt') r.status_code r.headers['content-type'] r.encoding r.txt r.text r.json() r.text import json records = [json.loads(line) for line in r.text] r.json() import requests r = requests("https://raw.githubusercontent.com/yukoga/python-for-data-analysis-review/master/usagov_bitly_data2012-03-16-1331923249.json") r = requests.get('https://raw.githubusercontent.com/yukoga/python-for-data-analysis-review/master/usagov_bitly_data2012-03-16-1331923249.json') r.status_code r. r.headers["content-type"] r.json() r = requests.get('http://stackoverflow.com/questions/21058935/python-json-loads-shows-valueerror-extra-data') r.text r.json() import json r = requests.get('https://raw.githubusercontent.com/yukoga/python-for-data-analysis-review/master/usagov_bitly_data2012-03-16-1331923249.json') records = [json.loads(line) for line in r] records = [json.loads(line) for line in r.text] json.loads('{key1:1,key2:2,key3:3}') from igraph import * g = Graph()
import requests payload = {'ie': 'utf-8', 'kw': 'red', 'fr': 'search'} r = requests('http://tieba.baidu.com/f', param=payload) print(r.url)
elif ruq == 'get': r = requests.get(url,api) else: print("没有设置协议类型,请先设置!") # print(r.text) # print("接口请求成功!") return r num = 0 for j in (api): num = num + 1 y1 = "" y1 = "第"+str(num)+"请求数据:"+str(j)+"\n" # print(y1) log(y1,"test") x = requests(j,url,ruq) # x1 = eval(x.text) y2 = "" y2 = "第"+str(num)+"返回参数:"+str(x.json())+"\n" log(y2,"test") if x.ok: print("第",num,"次接口请求成功!") x1 = x.json() # print(x1['OutputData']) x2 = x1['OutputData'][0]['Success'] # x3 = x2[0]['Data'][0]['Success'] if x2 == "True": print("第",num,"接口返回参数正确!") else: print("第",num,"接口返回参数错误,请检查日志!") else:
#!/usr/bin/env python """ Requirements: requests (installation: pip install requests) """ import requests import os SEARCH_URL = 'http://127.0.0.1:9000/api/search.json' _ALL_FILES_ = [] for (d, f, filenames) in os.walk(os.path.join(os.getcwd(), '../test/queries/')): if filenames == "README.txt": continue else: _ALL_FILES_ = filenames for single_file in _ALL_FILES_: path_to_file = os.path.join(os.path.abspath('../test/queries'), single_file) with open(path_to_file, 'rb') as f: queries = f.readlines() for query in queries: resp = requests.get(url=SEARCH_URL, params={'source': 'cache', 'q': query.strip()}) data = resp.json() statuses = data['statuses'] for status in statuses:
import requests headers = {"User-Agent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36"} req = requests("http://jnhcm.com",headers=headers) print(req.status_code)
while index < len(data_lode["availableCourses"]): available_Courses = (data_lode["availableCourses"][index]) coursesName = available_Courses["name"] coursesId = available_Courses["id"] courses_Id_list.append(coursesId) print(index), coursesName, coursesId index = index + 1 return courses_Id_list if os.path.exists("courses.json"): getData = get_Read("courses.json") get_Av_Courses(getData) print "kkkkkkkkkkkkkkkkk" else: requests(url_link) getData = get_Read("courses.json") get_Av_Courses(getData) # user_ex = int(raw_input("enter a number")) # userEx = courses_Id_list [user_ex - 1] # print userEx # exercise_url = url_link+"/"+str(userEx)+"/exercises" # def get_exercise(link): # ex_data = requests.get(exercise_url) # return (ex_data.json()) # data_e = get_exercise(exercise_url) # data_exer=data_e["data"] # def ex_get_Write(ex_data,user_ex):
def book_spider(book_tag): page_num = 0 book_list = [] try_times = 0 while (1): # url='http://www.douban.com/tag/%E5%B0%8F%E8%AF%B4/book?start=0' # For Test url = 'http://www.douban.com/tag/' + urllib.quote( book_tag) + '/book?start=' + str(page_num * 15) time.sleep(np.random.rand() * 5) # Last Version try: req = requests(url, headers=hds[page_num % len(hds)]) source_code = urllib2.urlopen(req).read() plain_text = str(source_code) except (urllib2.HTTPError, urllib2.URLError) as e: print(e) continue ##Previous Version, IP is easy to be Forbidden # source_code = requests.get(url) # plain_text = source_code.text soup = BeautifulSoup(plain_text) list_soup = soup.find('div', {'class': 'mod book-list'}) try_times += 1 if list_soup == None and try_times < 200: continue elif list_soup == None or len(list_soup) <= 1: break # Break when no informatoin got after 200 times requesting for book_info in list_soup.findAll('dd'): title = book_info.find('a', {'class': 'title'}).string.strip() desc = book_info.find('div', {'class': 'desc'}).string.strip() desc_list = desc.split('/') book_url = book_info.find('a', {'class': 'title'}).get('href') try: author_info = '作者/译者: ' + '/'.join(desc_list[0:-3]) except: author_info = '作者/译者: 暂无' try: pub_info = '出版信息: ' + '/'.join(desc_list[-3:]) except: pub_info = '出版信息: 暂无' try: rating = book_info.find('span', { 'class': 'rating_nums' }).string.strip() except: rating = '0.0' try: # people_num = book_info.findAll('span')[2].string.strip() people_num = get_people_num(book_url) people_num = people_num.strip('人评价') except: people_num = '0' book_list.append( [title, rating, people_num, author_info, pub_info]) try_times = 0 # set 0 when got valid information page_num += 1 print('Downloading Information From Page %d' % page_num) return book_list
#!/usr/bin/env python """ Requirements: requests (installation: pip install requests) """ import requests import json import os SEARCH_URL = 'http://localhost:9000/api/search.json' _ALL_FILES_ = [] for (d, f, filenames) in os.walk(os.path.join(os.getcwd(), '../test/queries/')): if filenames == "README.txt": continue else: _ALL_FILES_ = filenames for single_file in _ALL_FILES_: path_to_file = os.path.join(os.path.abspath('../test/queries'), single_file) with open(path_to_file, 'rb') as f: queries = f.readlines() for query in queries: resp = requests.get(url=SEARCH_URL, params={ 'source': 'cache', 'q': query.strip()
import requests r = requests("https://www.schoology.com/") while True: print(r.header)
def getWeatherFiveDays(city): appid = 'b1b15e88fa797225412429c1c50c122a1' url = ('https://api.openweathermap.org/data/2.5/forecast?q=%s' % (city)) r = requests(url, verify=False) print(r.text)
def getEstudante(id): url = ('https://fierce-island-41854.herokuapp.com/api/estudante/%s' % (id)) r = requests(url, verify=False) print(r.text)
json=data, **kwargs) else: res = self.my_session.request(methmod, url, data=data, **kwargs) else: res = None print("不支持{}的请求方法".format(methmod)) return res self.my_session.close() if __name__ == '__main__': # 注册 url_1 = "http://120.78.128.25:8080/futureloan/mvc/api/member/register" data1 = {"mobilephone": "13666661661", "pwd": "123456"} # 登陆 login_url = "http://120.78.128.25:8080/futureloan/mvc/api/member/login" # 充值 recharge_url = "http://120.78.128.25:8080/futureloan/mvc/api/member/recharge" recharge_data = {"mobilephone": "13666661661", "amount": 2345} requests = MyRequests() res_1 = requests("post", url=url_1, data=data1) res_2 = requests("post", url=login_url, data=data1) res_3 = requests("post", url=recharge_url, data=recharge_data) print(res_1.text) print(res_2.text) print(res_3.text)
def requests(): # 总共处理的请求数 print nginxStatus[2].split()[2] try: if sys.argv[1]: pass except IndexError, e: print Fore.GREEN + '#####' print './%s [active|reading|writing|waiting|accepts|requests]' % sys.argv[0] print Fore.GREEN + '#####' else: if sys.argv[1] in 'active': active() elif sys.argv[1] in 'reading': reading() elif sys.argv[1] in 'writing': writing() elif sys.argv[1] in 'waiting': waiting() elif sys.argv[1] in 'accepts': accepts() elif sys.argv[1] in 'handled': handled() elif sys.argv[1] in 'requests': requests() else: print './%s [active|reading|writing|waiting|accepts|requests]' % sys.argv[0]
def download_imgs(self, imgs_url): img = requests(imgs_url) with open(file, 'wb') as img_file: img_file.write(img)
'''import requests r = requests.get("https://maker.ifttt.com/trigger/Beautiful/with/key/p0jWqMouy0MlwbXYQ6oQeSjaHnAcmIrLJhqzwRXciJ2") c = r.status_code print(c)''' '''import requests r = requests.get("https://maker.ifttt.com/trigger/HTU/with/key/p0jWqMouy0MlwbXYQ6oQeSjaHnAcmIrLJhqzwRXciJ2") c = r.status_code print(c)''' import requests name = "Bridget" message = "Hi, how was lunch? you are splendid." r = requests.get( "https://maker.ifttt.com/trigger/message/messenger/with/key/2Kud4E0CUmxZX3JAbrzbl" ) print(r) requests()
def handle_ssh_request(request): return requests(request)
'start_date': '2020-01-01T00:00:00+00:00', 'is_processing': '' }) response = requests( method='POST', url='https://' + ip + '/api/v1/assets/', verify=False, data={ "mimetype": "webpage", "asset_id": "male_ad3", "end_date": "9999-01-01T00:00:00+00:00", "name": "male_ad3", "nocache": "", "is_enabled": "1", "is_active": 1, "uri": "https://www.telegraph.co.uk/content/dam/men/2018/07/02/TELEMMGLPICT000168190077_trans_NvBQzQNjv4BqpVlberWd9EgFPZtcLiMQfyf2A9a6I9YchsjMeADBa08.jpeg", "start_date": "2020-01-01T00:00:00+00:00", "duration": 15, "skip_asset_check": 1, "is_processing": "" }, # data={'asset_id': 'male_ad4', 'mimetype': 'webpage', 'name': 'male_ad4', 'end_date': '9999-01-01T00:00:00+00:00', 'is_enabled': 1, 'nocache': '', 'is_active': 1, 'uri': 'https://www.thetrendspotter.net/wp-content/uploads/2017/01/Best-Clothing-Stores-men.jpg', 'skip_asset_check': 1, 'duration': '15', 'play_order': 1, 'start_date': '2020-01-01T00:00:00+00:00', 'is_processing': '0'}, # headers={ # # "Authorization": "Token {token}", # "accept": "application/json", # "content-Type": "application/json" # } ) print(response.json()) # response = requests.request(
from bs4 import BeautifulSoup import requests, re r = requests().content soup = BeautifulSoup(r, "xtml") soup.findAll("a", "href" :re.compile )