def init_data(self): print('Data initialization...') data = self.load_data() for d in data: req = Request('PUT', d) rdm_sockets = random.sample(list(self._sockets.keys()), self._options.k) for i in rdm_sockets: cur_socket = self._sockets[i] # Send request's OPTIONS cur_socket.send(req.get_options()) cur_socket.recv(100) # Send request cur_socket.send(req.get_request()) # Receive response's OPTIONS res_options = pickle.loads(cur_socket.recv(100)) # Send confirmation confirm = Response() cur_socket.send(confirm.get_response()) # Receive response res = pickle.loads(cur_socket.recv(res_options['res_size']))
def retrieve_data(self): try: request = Request() result = request.get(SERVICE_ENDPOINT) if result: return json.loads(result) except ConnectionError as e: msg = str(e) logging.error(msg)
def add_uptime_information_entry(): data = {'source': 'lasvegas'} try: request = Request() result = request.post(AWS_ENDPOINT, data) print(result) except ConnectionError as e: msg = str(e)
def test_pass(self, params_init): """ 用例描述:登陆正向用例 """ self.request = Request() self.params = params_init.get_params_list('login') self.test_assert = Assertions() response = self.request.post_request(self.params['Login'][0]['url'],\ self.params['Login'][0]['data'], \ self.params['Login'][0]['header']) assert self.test_assert.assert_code(response['code'], 200) assert self.test_assert.assert_body(response['body'], 'msg', 'OK')
def _is_root_key_duplicated(self, payload): root_key, temp_result = payload.split(':', 1)[0].strip('"'), {} req = Request('GET', root_key) for ip in self._sockets.keys(): self._stream(req, ip, temp_result) # If there is a record with a given key, it will be removed if 200 in temp_result: if not self._is_all_servers_available(): raise Exception( '[WARNING] Cannot perform PUT operation in order to update a record. One or more servers are unavailable!' ) else: req = Request('DELETE', root_key) for ip in self._sockets.keys(): self._stream(req, ip, {})
def getThreadsInTopic(self, topicUrl): ''' Get all threads in a topic ''' res = [] #html = urllib.urlopen(topicUrl).read() html = Request.get_page_content(topicUrl) soup = BeautifulSoup(html) threads = soup.find('ol', {'id' : 'threads'}) if threads: for thread in threads.findAll('li', {'class' : 'threadbit '}): tLink = thread.find('a', {'class' : 'title'}) if tLink : tUrl = tLink['href'] if tUrl: pos = tUrl.find('?s=') # loai bo chuoi ?s= ..... if pos != -1: tUrl = tUrl[0:pos] # them base URL tUrl = 'http://www.lamchame.com/forum/' + tUrl res.append(tUrl.encode('utf-8')) #print tUrl #print '-----------' return res
def getThreadsInTopic(self, topicUrl): ''' Lay tat ca cac thread goc co trong 1 topic ''' res = [] try: html = Request.get_page_content(topicUrl) soup = BeautifulSoup(html) ulThread = soup.find('ul', {'id' : 'threads'}) #print ulThread if ulThread: threads = ulThread.findAll('li', {'class' : 'threadbit_nam_fix_select'}) if threads: for thread in threads: try: #import pdb #pdb.set_trace() #print thread tLink = thread.find('a', {'class' : 'title'}) if (tLink) : tUrl = tLink['href'] res.append(tUrl) #print tUrl #print '--------------------' except Exception,e: print e.message; tb = traceback.format_exc() print tb except Exception, e: print e.message; tb = traceback.format_exc() print tb pass
def getThreadsInTopic(self, topicUrl): ''' Get all threads in a topic ''' res = [] try: html = Request.get_page_content(topicUrl) soup = BeautifulSoup(html) threads = soup.find('ol', {'id' : 'threads'}) if threads: #find thread for thread in threads.findAll('h3', {'class' : 'threadtitle'}): #print thread tLink = thread.find('a', {'class' : 'title'}) if (tLink) : tUrl = tLink['href'] pos = tUrl.find('?s=') if pos: tUrl = tUrl[0:pos] tUrl = 'http://www.otofun.net/forums/' + tUrl if tUrl not in res: res.append(tUrl) except : print 'Error when get threads in topic' pass return res
def getAllTopics(self): ''' Lay ds cac topic co trong forum ''' res = [] try: baseUrl = 'http://www.webtretho.com/forum/f' url = 'http://www.webtretho.com/forum/search.php?search_type=1&contenttype=vBForum_Post' html = Request.get_page_content(url) soup = BeautifulSoup(html) soupCates = soup.find('select', {'id' : 'forumchoice'}) cates = soupCates.findAll('option') for cate in cates: topicNumber = cate['value'] if topicNumber.isdigit(): topicUrl = baseUrl + topicNumber + '/' res.append(topicUrl) except Exception, e: print e.message; tb = traceback.format_exc() print tb pass
def getTotalPageInTopic(self, topicUrl): ''' Lay tong so page trong 1 topic ''' total = 0 try: html = Request.get_page_content(topicUrl) soup = BeautifulSoup(html) nav = soup.find('div', {'class' : 'threadpagenav'}) if nav: lastPage = nav.find('span', {'class' : 'first_last'}) if lastPage: aLink = lastPage.find('a') if aLink: url = aLink['href'] pos1 = url.find('/page') pos2 = url.find('?s=') if pos1 != -1: if pos2 != -1: # ton tai ?s= page = url[pos1+5:pos2] else : page = url[pos1+5:] total = int(page) except: print 'Error when get total page in topic' pass return total
def getThreadDetail(self, url, fileContent = '', page = 1): res = () html = '' try: html = Request.get_page_content(url); except Exception, e: return None
def _heart_beat(self): req = Request('HEART_BEAT') ip_address = list(self._sockets.keys()) for ip in ip_address: try: # Send request's OPTIONS self._sockets[ip].send(req.get_options()) pickle.loads(self._sockets[ip].recv(100)) # Send request self._sockets[ip].send(req.get_request()) # Receive response pickle.loads(self._sockets[ip].recv(100)) except socket.error as e: print(f'[Error] {e}') except EOFError as e: del self._sockets[ip]
def is_reachable(): status = None success = False try: request = Request() status = retrieve_status(request) success = status['success'] except ConnectionError as e: msg = str(e) return success
def getThreadDetail(self, url): res = {} try : html = Request.get_page_content(url) soup = BeautifulSoup(html) #print soup postContainer = soup.find('ol', {'id' : 'posts'}) title = soup.find('span', {'class' : 'threadtitle'}).get_text().strip() posts = postContainer.findAll('li', {'class' : 'postbit postbitim postcontainer'}) count = 0 comments = [] for post in posts: #print post count += 1 postContent = post.find('blockquote', {'class' : 'postcontent restore'}) postContent = postContent.get_text() #postContent = re.sub('<br/>+', '_NEW_LINE_', postContent) postContent = re.sub('[\t]+', ' ', postContent) postContent = re.sub('[ ]+', ' ', postContent) postContent = re.sub('[\\r\\n]+', '\n', postContent) postContent = postContent.strip() # date infomation dateInfo = post.find('span', {'class' : 'postdate'}).get_text().strip() dateInfo = re.sub('\s+', ' ', dateInfo) # user infomation userInfo = post.find('div', {'class' : 'username_container'}).find('strong').get_text().strip() info = {'user' : userInfo, 'date': dateInfo, 'content' : postContent} if count == 1: # post postInfo = info; postInfo['title'] = title res['post']= postInfo else : # comment comments.append(info) res['comments'] = comments except: #print 'ERROR when crawling URL : ' , url print 'ERROR when get thread detail' pass return res
def getAllTopics(self): res = [] baseUrl = 'http://www.lamchame.com/forum/forumdisplay.php/' searchUrl = 'http://www.lamchame.com/forum/search.php?search_type=1&contenttype=vBForum_Post' html = Request.get_page_content(searchUrl) soup = BeautifulSoup(html) soupCates = soup.find('select', {'id' : 'forumchoice'}) cates = soupCates.findAll('option') for cate in cates: topicNumber = cate['value'] if topicNumber.isdigit(): title = cate.string.strip() topicUrl = baseUrl + topicNumber + '-' + title.replace(' ', '-') res.append(topicUrl.encode('utf-8')) return res
class TestLogin(): @allure.severity('登陆') @allure.story('登陆正向用例') @pytest.mark.skip('不执行登录用例') def test_pass(self, params_init): """ 用例描述:登陆正向用例 """ self.request = Request() self.params = params_init.get_params_list('login') self.test_assert = Assertions() response = self.request.post_request(self.params['Login'][0]['url'],\ self.params['Login'][0]['data'], \ self.params['Login'][0]['header']) assert self.test_assert.assert_code(response['code'], 200) assert self.test_assert.assert_body(response['body'], 'msg', 'OK')
def getTotalPageInTopic(self, topicUrl): ''' Get total page in a topic ''' total = 0 try: html = Request.get_page_content(topicUrl) soup = BeautifulSoup(html) pageNav = soup.find('div', {'class' : 'threadpagenav'}) if pageNav: lastPage = pageNav.find('span', {'class' : 'first_last1'}) if lastPage: #print lastPage.string total = int(lastPage.string) except: print 'Error when get total page' pass return total
def getAllTopics(self): res = [] baseUrl ='http://www.otofun.net/' url = 'http://www.otofun.net/forum.php' html = Request.get_page_content(url) soup = BeautifulSoup(html) links = soup.findAll('a') for link in links: if link['href'].startswith('forums/'): u = link['href'] pos = u.find('?s=') if pos != -1: u = u[0:pos] u = baseUrl + u res.append(u) return res
def getAllTopics(self): res = [] try: baseUrl ='http://www.otofun.net/' url = 'http://www.otofun.net/forum.php' html = Request.get_page_content(url) soup = BeautifulSoup(html) links = soup.findAll('a') for link in links: if link['href'].startswith('forums/'): u = link['href'] pos = u.find('?s=') if pos != -1: u = u[0:pos] u = baseUrl + u res.append(u.encode('utf-8')) except: print 'Error when get all topic' pass return res
def getTotalPageInThread(self, url): ''' Lay tong so page co trong 1 thread ''' total = 1 html = Request.get_page_content(url); soup = BeautifulSoup(html) pageInfo = soup.find('div', {'class' : 'pageRefix'}) if pageInfo: lastPage = pageInfo.find('a', {'class' : 'arrowLstPage'}) if lastPage: #print lastPage link = lastPage['href'] #print link pos = link.index('.html') if pos != False: p = re.compile("index(\d+).html") a = p.search(link) #print link[pos-1:] total = a.group(1) total = int(total) #print total return total
def getThreadDetail(self, url): ''' Lay thong tin chi tiet cua 1 thread ''' #res = { 'post' : {'user' : '', 'title': '', 'date' : '', 'content' : ''}, 'comments' : [{'user' : '', 'date': '', 'content' : ''}] } res = {} try: html = Request.get_page_content(url); soup = BeautifulSoup(html) soupPost = soup.find('ol', {'id' : 'posts'}) title = soup.find('div', {'id' : 'widgetRefix'}).find('h1').get_text().strip() #print title # list posts posts = soupPost.findAll('li', {'class' : 'postbit postbitim postcontainer'}) #print posts #print len(posts) #print posts.size() count = 0 comments = [] for post in posts: #print post count += 1 postContent = post.find('blockquote', {'class' : 'postcontent restore'}) #postContent = postContent.renderContents() postContent = postContent.get_text() #print postContent #print '----------------------' postContent = re.sub('[\t]+', ' ', postContent) postContent = re.sub('[ ]+', ' ', postContent) postContent = re.sub('[\\r\\n]+', '\n', postContent) postContent = postContent.strip() #print soup2.get_text().strip() #print postContent # date infomation dateInfo = post.find('span', {'class' : 'postdate'}).get_text().strip() dateInfo = re.sub('\s+', ' ', dateInfo) #print dateInfo # user infomation userInfo = post.find('div', {'class' : 'username_container'}).get_text().strip() #print userInfo info = {'user' : userInfo, 'date': dateInfo, 'content' : postContent} if count == 1: # post postInfo = info; postInfo['title'] = title res['post']= postInfo else : comments.append(info) #print '--------------------------------' res['comments'] = comments #print res except Exception, e: print e.message; tb = traceback.format_exc() print tb pass
def request_init(): return Request()
if len(sys.argv) < 2: print Colors.FAIL + "Please enter a hero name. Eg. 'Faceless Void'" + Colors.ENDC else: hero_name = sys.argv[1] client = MongoClient('localhost', 27017) print Colors.OKGREEN + "Connected to MongoDB." + Colors.ENDC db = client.assignment_1 heroes_coll = db.heroes benchmarks_coll = db.hero_benchmarks matchups_coll = db.hero_matchups # time.sleep(5) # hack for the mongoDb database to get running scrapper = Request() heroes = scrapper.getHeroes() get_heroes(db) selected_hero = get_selected_hero() if selected_hero != None: print Colors.OKGREEN + "\nDisplaying Hero Data for " + hero_name + ": " + Colors.ENDC print pd.DataFrame(selected_hero) selected_hero_id = selected_hero['id'] get_hero_benchmark() get_hero_matchups()
def start_shell(self): while True: cmd = input('kvStore> ').strip() if not cmd: continue if cmd == '!DISCONNECT': sys.exit('kvBroker has been disconnected! Bye!') else: self._heart_beat() is_replication_valid = self._is_replication_valid() try: req_type, payload = cmd.split(' ', 1) except ValueError as e: print('[Error] Payload is missing!') continue if req_type == 'PUT': if self._is_replication_supported(): print( '[WARNING] Not enough servers to support replication! Please restart the servers!' ) continue # Check if a record with the given root key already exists try: self._is_root_key_duplicated(payload) except Exception as e: print(e) continue # Create request req = Request(req_type, payload) # Select random sockets rdm_sockets = random.sample(list(self._sockets.keys()), self._options.k) # Send request to the selected servers result = {} for ip in rdm_sockets: self._stream(req, ip, result) self.print_result(payload, result) elif req_type == 'GET' or req_type == 'QUERY': result = {} req = Request(req_type, payload) for ip in self._sockets.keys(): self._stream(req, ip, result) if not is_replication_valid: print( f'[WARNING] Data may be inconsistent. {self._options.k} or more servers are unavailable!' ) self.print_result(payload, result) elif req_type == 'DELETE': if not self._is_all_servers_available(): print( '[WARNING] Cannot perform delete operation. One or more servers are unavailable!' ) continue result = {} req = Request(req_type, payload) for ip in self._sockets.keys(): self._stream(req, ip, result) self.print_result(payload, result) else: print('[INFO] Invalid request type!') print('[INFO] Supported types: PUT | GET | QUERY | DELETE')
def getThreadsInTopic(self, topicUrl, page = 1): ''' Get all threads in a topic ''' res = {} html = Request.get_page_content(topicUrl) soup = BeautifulSoup(html) threads = soup.find('ol', {'id' : 'threads'}) if threads: #find thread for thread in threads.findAll('h3', {'class' : 'threadtitle'}): #print thread tLink = thread.find('a', {'class' : 'title'}) if (tLink) : tUrl = tLink['href'] pos = tUrl.find('?s=') if pos: tUrl = tUrl[0:pos] tUrl = 'http://www.otofun.net/forums/' + tUrl if tUrl not in self.listUrl: self.listUrl.append(tUrl) print 'Thread ',tUrl print tLink.string print '===========================' query = u"INSERT INTO url SET url = '%s', domain = '%s', created_date = UNIX_TIMESTAMP(NOW()), modified_date = UNIX_TIMESTAMP(NOW()), done = 0, running = 0" %(tUrl, 'otofun.net') self.dbCursor.execute(query) self.connection.commit() # if self.COUNT_TOPIC % 100 == 0: # import os; # if not os.path.exists(self.dataDir + str(self.COUNT_TOPIC) + '-' + str(self.COUNT_TOPIC + 100)): # os.makedirs(self.dataDir + str(self.COUNT_TOPIC) + '-' + str(self.COUNT_TOPIC + 100)) # self.dataFolder = self.dataDir + str(self.COUNT_TOPIC) + '-' + str(self.COUNT_TOPIC + 100) # # self.getThreadDetail(tUrl) # self.COUNT_TOPIC += 1 # # print 'Topic number: ', self.COUNT_TOPIC # print 'Post number: ', self.COUNT_POST time.sleep(1) print '-----------' #find sub-topic topics = soup.find('ol', {'class' : 'subforumlist'}) if topics: for topic in topics.findAll('li', {'class' : 'subforum'}): tLink = topic.find('a') if (tLink) : tUrl = tLink['href'] if tUrl not in self.listUrl: self.listUrl.append(tUrl) if 'http://' not in tUrl: tUrl = self.baseUrl + tUrl print 'Topic Level- ', page, tUrl print tLink.string self.getThreadsInTopic(tUrl) print '-----------' page = page + 1 if '?' in topicUrl: topicUrl = topicUrl + '&page=' + str(page) else: topicUrl = topicUrl + '?page=' + str(page) self.getThreadsInTopic(tUrl, page) return res
import asyncio from config import TEAM_ID from utils.Request import Request request = Request() class Api: @staticmethod async def getPlayer(nickname): return await request.get('/players?nickname=' + nickname + '&game=csgo') @staticmethod async def getPlayerStats(playerId): return await request.get('/players/' + playerId + '/stats/csgo') @staticmethod async def getTeamMembers(): return await request.get('/teams/' + TEAM_ID) @staticmethod async def getPlayerMatch(playerId, rightBound): return await request.get('/players/' + playerId + '/history?game=csgo&from=' + str(rightBound) + '&offset=0&limit=1') @staticmethod async def getMatchInfo(gameId): return await request.get('/matches/' + gameId) @staticmethod async def getMatchStats(gameId): return await request.get('/matches/' + gameId + '/stats')