def rtmp_play_url(self, stream_id, app_name=None): if app_name is None: app_name = self._app_name url_path = '/' + urljoin(quote_plus(app_name)+'/', quote_plus(stream_id)) if not self._auth_type: return 'rtmp://' + self._play_domain + url_path elif self._auth_type == self.AUTH_TYPE_A: query = {'auth_key': self._calc_auth_token(url_path)} return 'rtmp://' + self._publish_domain + url_path + '?' + urlencode(query) else: raise IVRError('Unknown auth type {0}'.format(self._auth_type))
def get_cloud_space(session, csrf='', login=LOGIN): """ returns available free space in bytes """ assert csrf is not None, 'no CSRF' timestamp = str(int(time.mktime(datetime.datetime.now().timetuple())* 1000)) quoted_login = quote_plus(login) command = ('user/space?api=' + str(API_VER) + '&email=' + quoted_login + '&x-email=' + quoted_login + '&token=' + csrf + '&_=' + timestamp) url = urljoin(CLOUD_URL, command) try: r = session.get(url, verify = VERIFY_SSL) except Exception as e: if LOGGER: LOGGER.error('Get cloud space HTTP request error: {}'.format(e)) return 0 if r.status_code == requests.codes.ok: r_json = r.json() total_bytes = r_json['body']['total'] * 1024 * 1024 used_bytes = r_json['body']['used'] * 1024 * 1024 return total_bytes - used_bytes elif LOGGER: LOGGER.error('Cloud free space request error. Check your connection. \ HTTP code: {}, msg: {}'.format(r.status_code, r.text)) return 0
def safe_encode(*args, pattern=' ', space_char='+'): """default: replace spaces with '+' """ # SPACE_CHAR = '+' # return SPACE_CHAR.join(args).replace(' ', SPACE_CHAR) # args = (quote_plus(str(arg) for arg in args)) args = itertools.chain.from_iterable( (quote_plus(argwd) for argwd in arg.split(pattern)) for arg in args) return re.sub(re.compile(pattern), space_char, space_char.join(args), re.DOTALL)
def new_search(request): search = request.POST.get('search') final_url = BASE_CRAIGLIST_URL.format(quote_plus(search)) response = requests.get(final_url) soup = BeautifulSoup(data, features='html.parser') post_titles = soup.find_all('a', {'class': 'result-title'}) data = response.text stuff_for_frontend = { 'search': search, } return render(request, 'my_app/new_search.html')
def preprocess_media_tags(element): if isinstance(element, html.HtmlElement): if element.tag == 'figcaption': # figcaption may have only text content [e.drop_tag() for e in element.findall('*')] elif element.tag in ['ol', 'ul']: # ignore any spaces between <ul> and <li> element.text = '' elif element.tag == 'li': # ignore spaces after </li> element.tail = '' elif element.tag == 'iframe' and element.get('src'): iframe_src = element.get('src') youtube = youtube_re.match(iframe_src) vimeo = vimeo_re.match(iframe_src) if youtube or vimeo: if youtube: yt_id = urlparse(iframe_src).path.replace('/embed/', '') element.set( 'src', '/embed/youtube?url=' + quote_plus('https://www.youtube.com/watch?v=' + yt_id)) elif vimeo: element.set( 'src', '/embed/vimeo?url=' + quote_plus('https://vimeo.com/' + vimeo.group(2))) element = _wrap_tag(element, 'figure') elif element.tag == 'blockquote' and element.get( 'class') == 'twitter-tweet': twitter_links = element.cssselect('a') for tw_link in twitter_links: if twitter_re.match(tw_link.get('href')): twitter_frame = html.HtmlElement() twitter_frame.tag = 'iframe' twitter_frame.set( 'src', '/embed/twitter?url=' + quote_plus(tw_link.get('href'))) element = _wrap_tag(twitter_frame, 'figure') return element
def new_search(request): search = request.POST.get('search') if search: time = timezone.now() s1=Search.objects.create(search=search,created=time) final_url = BASE_CGSLIST_URL.format(quote_plus(search)) no_res = '' try: response = requests.get(final_url) data = response.text soup =BeautifulSoup(data, features='html.parser') post_listings = soup.find_all('li', {'class': 'result-row'}) if soup.find_all('div', {'class': 'alert alert-sm alert-warning'}): no_result = soup.find_all('div', {'class': 'alert alert-sm alert-warning'}) no_res = "No result found For '{}'. (All words must match)".format(search) print(no_result, ' ', no_res) final_postings = [] for post in post_listings: post_titles = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' if post.find(class_='result-image').get('data-ids'): post_image = post.find(class_='result-image').get('data-ids').split(',')[0].split(':')[1] post_image_url = BASE_IMAGE_URL.format(post_image) else: post_image_url = 'https://previews.123rf.com/images/urfandadashov/urfandadashov1809/urfandadashov180901275/109135379-photo-not-available-vector-icon-isolated-on-transparent-background-photo-not-available-logo-concept.jpg' print(post_image_url) final_postings.append((post_titles, post_url, post_price, post_image_url)) stuff_for_front_end = { 'search' : search, 'final_postings': final_postings, 'no_res' :no_res, } except: stuff_for_front_end = { 'errormessage' : 'Check Your Internet Connection'} if search: s1.save() return render(request, 'my_app/new_search.html', stuff_for_front_end)
def new_search(request): # What we are typing in the search box we are storing database via search Model search = request.POST.get('search') models.Search.objects.create(search=search) #print(quote_plus(search)) final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) #print(final_url) response = requests.get(final_url) data = response.text #print(data) soup = BeautifulSoup(data, features='html.parser') #print(soup) #post_titles=soup.find_all('a',{'class':'result-title'}) #print(post_titles[0].get('href')) post_listings = soup.find_all('li', {'class': 'result-row'}) #post_title=post_listings[0].find(class_='result-title').text #post_url=post_listings[0].find('a').get('href') #post_price=post_listings[0].find(class_='result-price').text #print(post_title) #print(post_url) #print(post_price) final_postings = [] for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' if post.find(class_='result-image').get('data-ids'): post_image_id = post.find(class_='result-image').get( 'data-ids').split(',')[0].split(':')[1] #print(post_image_id) post_image_url = BASE_IMAGE_URL.format(post_image_id) #print(post_image_url) else: post_image_url = 'https://craigslist.org/images/peace.jpg' final_postings.append( (post_title, post_url, post_price, post_image_url)) stuff_front_end = { 'search': search, 'final_postings': final_postings, } #print(search) #Context Dictionary return render(request, 'my_app/new_search.html', stuff_front_end)
def new_search(request): search = request.POST.get('search') models.Search.objects.create( search=search ) #creating search object #this writes to the database based off of user posts # print(quote_plus(search)) final_url = BASE_CRAIGLIST_URL.format(quote_plus(search)) # print(final_url) response = requests.get(final_url) data = response.text soup = BeautifulSoup(data, features='html.parser') post_title = soup.find_all( 'a', {'class': 'result-title' }) # find all links where class is results-title # print(post_title.get('href')) post_lisitngs = soup.find_all('li', {'class': 'result-row'}) # # post_title = post_lisitngs.find('a', {'class' : 'result-title'}) # find all links where class is results-title # # post_url = post_lisitngs.find('a').get('href') # post_price = post_lisitngs.find(class_= 'result-price') # # post_text = new_soup.find(id='postingbody').text final_postings = [] for post in post_lisitngs: post_title = post.find('a', {'class': 'result-title'}).text post_url = post.find('a').get('href') post_price = post.find(class_='result-price') if post.find('a', {'class': 'result-image'}).get('data-ids'): post_image_url = post.find( class_='result-image').get('data-ids').split(',')[0].split( ':')[1] #parsing the images in the URL post_image_url = BASE_IMAGE_URL.format(post_image_url) print(post_image_url) # else: # post_image_url = '' final_postings.append( (post_title, post_url, post_price, post_image_url)) # print(post_title) # print(post_url) # print(post_price) # print(data) # print(search) stuff_for_frontend = { 'search': search, 'final_postings': final_postings, } return render(request, 'myapp/new_search.html', stuff_for_frontend)
def new_search(request): # ver 1 # # wyjmujemy z metody post z playholdera search co zostalo wpisane # search = request.POST.get('search') # # print(search) # # tworze zmmnienna do wysysylania do html # staff_for_frontend = { # 'search': search, # } # # tworze html i wysylam do niego dane do wyswietlenia # return render(request, 'my_app/new_search.html', staff_for_frontend) # wyjmujemy z metody post z playholdera search co zostalo wpisane # ver 2 # dostajemy od html search = request.POST.get('search') # twozymy obiekt w DB models.Search.objects.create(search=search) # robimy dobre query z + zamiast ' ' final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) # dostajemy odpowiedzi z neta response = requests.get(final_url) data = response.text # zaba w web scraping soup = BeautifulSoup(data, features='html.parser') post_listings = soup.find_all('li', {'class': 'result-row'}) final_postings = [] for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' if post.find(class_='result-image').get('data-ids'): post_image_id = post.find(class_='result-image').get( 'data-ids').split(',')[0].split(':')[1] post_image_url = BASE_IMAGE_URL.format(post_image_id) else: post_image_url = 'https://craigslist.org/images/peace.jpg' final_postings.append( (post_title, post_url, post_price, post_image_url)) # tworze zmmnienna do wysysylania do html staff_for_frontend = { 'search': search, 'final_postings': final_postings, } # tworze html i wysylam do niego dane do wyswietlenia return render(request, 'my_app/new_search.html', staff_for_frontend)
def new_search(request): search = request.POST.get('search') models.Search.objects.create(search=search) # print(quote_plus(search)) # quote_plus formats the content of the search with plus signs and the requirements for use as valid part of a url final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) # this concartinates the base url and the quote_plus formatted content of the search response = requests.get(final_url) data = response.text print(final_url) # print(data) soup = BeautifulSoup(data, features='html.parser') # this creates a beautiful soup object o fthe data variable as a html post_listings = soup.find_all('li', {'class': 'result-row'}) final_postings = [] for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: new_response = requests.get(post_url) new_data = new_response.text new_soup = BeautifulSoup(new_data, features='html.parser') post_text = new_soup.find(id='postingbody').text rl = requests.findall(r'\$\w+', post_text) if rl: post_price = rl[0] else: post_price = 'N/A' if post.find(class_='result-image').get('data-ids'): # post_image_id = post.find(class_='result-image').get('data-ids').split(',')[:] post_image_id = post.find(class_='result-image').get( 'data-ids').split(',')[0].split(':')[1] post_image_url = "https://images.craigslist.org/{}_300x300.jpg".format( post_image_id) else: post_image_url = "https://craigslist.org/images/peace.jpg" final_postings.append( (post_title, post_url, post_price, post_image_url)) print(final_postings) stuff_for_frontend = {'search': search, 'final_postings': final_postings} return render(request, 'my_app/new_search.html', stuff_for_frontend)
def new_search(request): search=request.POST.get('search') models.Search.objects.create(search=search) final_url=BASE_CRAIGSLIST_URL.format(quote_plus(search)) response=requests.get(final_url) data=response.text print(data) stuff_for_frontend={ 'search':search } return render(request,'myapp/new_search.html',stuff_for_frontend)
def new_search(request): search = request.POST.get("search") models.Search.objects.create(search=search) final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) response = requests.get(final_url) data = response.text soup = BeautifulSoup(data, features="html.parser") post_title = soup.find_all("a", {"class": "result-title"}) print(post_title) frontend = { "search": search, } return render(request, "pages/new_search.html", frontend)
def new_search(request): """ Handles request for new searches and renders a list of results """ search = request.POST.get('search') models.Search.objects.create(search=search) # quote_plus joins search query keywords with '+' delimiters (e.g "experienced+python+tutor") final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) # gets the response object from the specified '@url' argument. response = requests.get(final_url) # returns the HTML source code of response data = response.text # create a soup object for the source code soup = BeautifulSoup(data, features='html.parser') # returns all list items with class="result-row" post_listings = soup.find_all('li', {'class': 'result-row'}) final_postings = [] for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' if post.find(class_="result-image").get('data-ids'): post_image_id = post.find(class_="result-image").get( 'data-ids').split(',')[0].split(':')[1] post_image_url = BASE_IMAGE_URL.format(post_image_id) else: post_image_url = "https://craigslist.org/images/peace.jpg" final_postings.append( (post_title, post_url, post_price, post_image_url)) stuff_for_frontend = { 'search': search, 'final_postings': final_postings[:12], } return render(request, 'search_app/new_search.html', stuff_for_frontend)
def new_search(request): headers = { "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0", "Accept-Encoding": "gzip, deflate", "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", "DNT": "1", "Connection": "close", "Upgrade-Insecure-Requests": "1" } search = request.POST.get('search') models.Search.objects.create(search=search) final_url = BASE_AMAZON_URL.format(quote_plus(search)) response = requests.get(final_url, headers=headers) data = response.text print(final_url) soup = BeautifulSoup(data, features='html.parser') final_postings = [] for d in soup.findAll( 'div', attrs= { 'class': 'sg-col-4-of-24 sg-col-4-of-12 sg-col-4-of-36 s-result-item s-asin sg-col-4-of-28 sg-col-4-of-16 sg-col sg-col-4-of-20 sg-col-4-of-32' }): link = d.find('a', attrs={ 'class': 'a-link-normal a-text-normal' }).get('href') link = 'https://www.amazon.in/' + link images = d.find('img', {'src': re.compile('.jpg')}) name = d.find( 'span', attrs={'class': 'a-size-base-plus a-color-base a-text-normal'}) pricediv = d.find('div', attrs={'class': 'a-row a-size-base a-color-base'}) price = None if pricediv is not None: price = pricediv.find('span', attrs={'class': 'a-offscreen'}) if name is not None and price is not None: final_postings.append((link, name.text, price.text, images['src'])) elif price is None: final_postings.append( (link, name.text, 'Not Available', images['src'])) else: final_postings.append("unknown-product") context = { 'search': search, 'final_postings': final_postings, } return render(request, 'my_app/new_search.html', context)
def new_search(request): # get inputed search text search = request.POST.get('search') # add search to database for tracking models.Search.objects.create(search=search) # this is the final url with corrected formating using quote_plus # which will fill in any spaces in the search term string final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) # Getting the Webpage, creating a Response object. response = requests.get(final_url) # Extracting the source code of the page. data = response.text # Passing the source code to Beautiful Soup to create a # BeautifulSoup object for it. soup = BeautifulSoup(data, features="html.parser") # Extracting all the <a> tags whose class name is # 'results-title' into a list. post_listings = soup.find_all('li', {'class': 'result-row'}) # post_title = post_listings[0].find(class_='result-title').text # post_url = post_listings[0].find('a').get('href') # post_price = post_listings[0].find(class_='result-price').text final_postings = [] for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' if post.find(class_='result-image').get('data-ids'): post_image_id = post.find(class_='result-image').get( 'data-ids').split(',')[0].split(':')[1] print(post_image_id) post_image_url = BASE_IMAGE_URL.format(post_image_id) print(post_image_url) else: post_image_url = "https://craigslist.org/images/peace.jpg" final_postings.append( (post_title, post_url, post_price, post_image_url)) stuff_for_frontend = { 'search': search, 'final_postings': final_postings, } return render(request, 'my_app/new_search.html', stuff_for_frontend)
def new_search(request): search = request.POST.get('search') models.Search.objects.create(search=search) print(search) final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) #print(final_url) response = requests.get(final_url) data = response.text soup = BeautifulSoup(data, features='html.parser') #post_titles=soup.find_all('a',{'class':'result-title'}) post_listings = soup.find_all('li', {'class': 'result-row'}) post_title = post_listings[0].find(class_='result-title').text post_url = post_listings[0].find('a').get('href') ''' for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') post_price = post.find(class_='result-price').text final_postings.append((post_title, post_url, post_price))#, post_image_url)) #parsed=url_parse.text #print(quote_plus(parsed)) #print(quote_plus(search))''' final_postings = [] for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' if post.find(class_='result-image').get('data-ids'): post_image_id = post.find(class_='result-image').get( 'data-ids').split(',')[0].split(':')[1] post_image_url = BASE_IMAGE_URL.format(post_image_id) print(post_image_url) else: post_image_url = 'https://craigslist.org/images/peace.jpg' final_postings.append( (post_title, post_url, post_price, post_image_url)) disp = { 'search': search, 'final_postings': final_postings, } return render(request, 'my_app/new_search.html', disp)
def new_search(request): search = request.POST.get('search') models.Search.objects.create(search=search) final_url = BASE_CRAIGSLIST_URL.format(quote_plus( search)) #joing the BASE_CRAIGSLIST_URL and the word that you search response = requests.get( final_url ) #using bs4 requests that is imported which stores the final_url in var response data = response.text #converting that url html to text soup = BeautifulSoup(data, features='html.parser' ) #parsing and create bs4 object in a variable soup post_listings = soup.find_all('li', {'class': 'result-row'}) final_postings = [] for post in post_listings: post_title = post.find( class_='result-title' ).text #finds all items with class 'result-title' post_url = post.find('a').get( 'href') #finds all items with link to that item #if a post has a price then print its price else print N/A if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' #if post has an image display it else display a peace sign image #logic to grab image of posts-> bs4 cant grab images through direct img names but through data ids so #data ids have been parsed and splitted and then grabbed if post.find(class_='result-image').get('data-ids'): post_image_id = post.find(class_='result-image').get( 'data-ids').split(',')[0].split(':')[1] post_image_url = BASE_IMAGE_URL.format(post_image_id) print(post_image_url) else: post_image_url = 'https://craigslist.org/images/peace.jpg' #creating a tuple of post's title,url,price,image so that it is visible on frontent by accessing through its indices final_postings.append( (post_title, post_url, post_price, post_image_url)) print(search) stuff_for_frontend = { 'search': search, 'final_postings': final_postings, } return render(request, 'myapp/new_search.html', stuff_for_frontend)
def new_search(request): search=request.POST.get('search') #print(quote_plus(search)) models.Search.objects.create(search=search) final_url=BASE_CRAIGS_LIST_URL.format(quote_plus(search)) print(final_url) #response=requests.get('https://delhi.craigslist.org/search/bbb?query=python&sort=rel') response = requests.get(final_url) data=response.text soup = BeautifulSoup(data,features='html.parser') post_listings=soup.find_all('li',{'class':'result-row'}) #print(len(post_listings)) #post_title=post_listings[0].find(class_='result-title').text #post_url=post_listings[0].find('a').get('href') #post_price=post_listings[0].find(class_='result-price').text final_postings=[] for post in post_listings: post_title=post.find(class_='result-title').text post_url=post.find('a').get('href') if post.find(class_='result-price'): post_price=post.find(class_='result-price').text else: post_price='N/A' print(post_title) if post.find('a').get('data-ids'): #print(post.find('a').get('data-ids').split(',')[0].split(':')[1]) post_image_id=post.find('a').get('data-ids').split(',')[0].split(':')[1] post_image_url=BASE_IMAGE_URL.format(post_image_id) #print(post_image_url) else: post_image_url='https://craigslist.org/images/peace.jpg' final_postings.append((post_title,post_url,post_price,post_image_url)) #print(post_title) #print(post_url) #print(post_price) #post_titles=soup.find_all('a',{'class':'result-title'}) #print(post_titles[0].text) #print(post_titles[0].get('href')) #print(data) #print(search) stuff_for_frontend={ 'search':search, 'final_postings':final_postings, } return render(request,'webscrapping/newsearch.html',stuff_for_frontend)
def new_search(request): search = request.POST.get('search') models.Search.objects.create(search=search) print(quote_plus(search)) final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) print(final_url) response = requests.get(final_url) data = response.text # print(data) soup = BeautifulSoup(data, features='html.parser') post_listings = soup.find_all('li', {'class': 'result-row'}) final_postings = [] for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = "N/A" if post.find(class_='result-image').get('data-ids'): post_img_id = post.find(class_='result-image').get( 'data-ids').split(',')[0].split(':')[1] post_img_url = BASE_IMAGE_URL.format(post_img_id) print(post_img_url) else: post_img_url = 'https://craigslist.org/images/peace.jpg' final_postings.append((post_title, post_url, post_price, post_img_url)) stuff_for_frontend = {'search': search, 'final_postings': final_postings} return render(request, 'Craigslist_WebScraping_app/new_search.html', stuff_for_frontend)
def new_search(request): search = request.POST.get('search') models.Search.objects.create(search=search) final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) response = requests.get(final_url) data = response.text # create a beautifulsoup object that returns an object of the results soup = BeautifulSoup(data, features='html.parser') # extract all the links with class 'result-title' post_listings = soup.find_all('li', {'class':'result-row'}) final_post_listing = [] for p in post_listings: post_url = p.find('a').get('href') post_title = p.find(class_='result-title').text if p.find(class_='result-price'): post_price = p.find(class_ ='result-price').text else: post_price ='N/A' if p.find(class_='result-image').get('data-ids'): post_image_id = p.find(class_='result-image').get('data-ids').split(',')[0].split(':')[1] post_image_url = BASE_IMAGE_URL.format(quote_plus(post_image_id)) print(post_image_url) else: post_image_url = 'https://images.craigslist.org/images/peace.jpg' final_post_listing.append((post_title,post_url,post_price,post_image_url)) stuff_for_front_end = { 'search': search, 'final_post_listing': final_post_listing } return render(request,'demoapp/new_search.html', stuff_for_front_end)
def current(city): city_input = quote_plus(city) token = random.choice(settings.KEYS) currentUrl = "https://api.openweathermap.org/data/2.5/weather?q={}&units=metric&appid={}".format( city_input, token) response = requests.get(currentUrl).json() returnDict = { 'city': city, "description": response['weather'][0]['description'], 'temperature': response['main']['temp'], 'icon': response['weather'][0]['icon'], 'time': time.strftime('%Y-%m-%d', time.localtime(response['dt'])), } return returnDict
def new_search(request): # Getting the search input search = request.POST.get('search') # Saving it to database models.Search.Mansouri.create(search=search) # Getting the url format final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) # getting the html source code of the url response = requests.get(final_url) data = response.text # Parsing Url soup = BeautifulSoup(data, features='html.parser') # GETTING OUR DATA post_listings = soup.find_all('li', {'class': 'result-row'}) final_postings = [] for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' post_date = post.find('time').get('title') if post.find(class_='result-image').get('data-ids'): post_image_id = post.find(class_='result-image').get( 'data-ids').split(',')[0].split(':')[1] post_image_url = BASE_IMAGE_URL.format(post_image_id) # print(post_image_url) else: post_image_url = 'https://craigslist.org/images/peace.jpg' final_postings.append( (post_title, post_url, post_price, post_date, post_image_url)) stuff_for_frontend = { 'search': search, 'final_postings': final_postings, } return render(request, 'my_app/new_search.html', stuff_for_frontend)
def secure_headers(self, url, post_data): """ Creates secure header for cryptopia private api. """ nonce = str(time.time()) md5 = hashlib.md5() jsonparams = post_data.encode('utf-8') md5.update(jsonparams) rcb64 = base64.b64encode(md5.digest()).decode('utf-8') signature = self.key + "POST" + quote_plus(url).lower() + nonce + rcb64 hmacsignature = base64.b64encode(hmac.new(base64.b64decode(self.secret), signature.encode('utf-8'), hashlib.sha256).digest()) header_value = "amx " + self.key + ":" + hmacsignature.decode('utf-8') + ":" + nonce return {'Authorization': header_value, 'Content-Type': 'application/json; charset=utf-8'}
def new_search(request): search = request.POST.get('search') models.Search.objects.create(search=search) # print(quote_plus(search)) final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) response = requests.get(final_url) data = response.text soup = BeautifulSoup(data, features='html.parser') # print(data) stuff_for_frontend = { 'search': search, } return render(request, 'games/new_search.html', stuff_for_frontend)
def _by_ids(self, service, ids): """ return multiple items by id """ if self.id and ids: values = ','.join(quote_plus(str(i)) for i in ids) term = '{}?id={}'.format(service.service_name, values) term = self._for_this_service(term) if ',' in values: return paginator(self._client, term, service.collection_name, service) res = self._client.get(term) return [ service(client=self._client, data=res[service.service_name]) ] return []
def search(request): if request.method == 'POST': form = SearchForm(request.POST) if form.is_valid(): search = form.cleaned_data['search'] if not Search.objects.filter(search=search): new_search = form.save(commit=False) new_search.save() final_url = BASE_SEARCH_URL.format(quote_plus(search)) response = requests.get(final_url) data = response.text soup = BeautifulSoup(data, features='html.parser') posts = soup.find_all('li', class_='result-row') final_posts = [] for post in posts: post_title = post.find('a', class_='result-title').text post_link = post.find('a', class_='result-title').get('href') if post.find('span', class_='result-price'): post_price = post.find('span', class_='result-price').text else: post_price = 'N/A' if post.find('a', class_='result-image gallery'): image_ids = post.find( 'a', class_='result-image gallery').get('data-ids') image_id = image_ids.split(',')[0].split(':')[1] post_image = BASE_IMG_URL.format(image_id) else: post_image = "https://www.craigslist.org/images/peace.jpg" final_posts.append((post_title, post_link, post_price, post_image)) form = SearchForm() frontend = { 'form': form, 'search': search, 'final_posts': final_posts, } return render(request, 'radzlist/search.html', frontend) else: return redirect('index')
def get_songs_list(self): result = [] search_content = self.artist + ' ' + self.song response = requests.get( SOUNDCLOUD_SEARCH_URL.format(quote_plus(search_content))) soup = BeautifulSoup(response.content, 'html.parser') for link in soup.find_all('a'): data = {} if link.parent.name == 'h2': data['title'] = link.text data['url'] = SOUNDCLOUD_URL + link['href'] result.append(data) return result[0]
def main(): module = AnsibleModule(argument_spec=dict(user=dict(required=True, aliases=['username']), password=dict(required=True), ip=dict(required=True), ptr=dict( required=True, aliases=['value', 'name'])), supports_check_mode=True) user = module.params['user'] password = module.params['password'] ip = module.params['ip'] ptr = module.params['ptr'] try: url = urljoin('https://robot-ws.your-server.de/rdns/', quote_plus(ip)) # Verify current status response = requests.get(url, auth=(user, password)) if response.status_code == 200: result = response.json() if result['ip'] == 'ip' and result['ptr'] == ptr: module.exit_json(changed=False, msg="OK", value=response.json()) if module.check_mode: module.exit_json(changed=True, msg="OK", value=response.json()) else: raise RuntimeError('Incorrect response from Hetzner WS: %s -> %d' % (url, response.status_code)) # Update record response = requests.post(url, json={'rdns': { 'ip': ip, 'ptr': ptr }}, auth=(user, password)) if response.status_code in (200, 201): module.exit_json(changed=True, msg="OK", value=response.json()) else: raise RuntimeError('Incorrect response from Hetzner WS: %s -> %d' % (url, response.status_code)) except Exception as e: module.fail_json(changed=False, msg='Failed in call to Hetzner WS: %s' % e.message)
def new_search(request): search = request.POST.get('search') # Gets the search variable from the python dictionary. is then POSTING the models.Search.objects.create(search=search) final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) # print(final_url) response = requests.get(final_url) data = response.text soup = BeautifulSoup(data, features='html.parser') post_listings = soup.find_all('li', {'class': 'result-row'}) # post_title = post_listings[0].find(class_='result-title').text # post_url = post_listings[0].find('a').get('href') # post_price = post_listings[0].find(class_='result-price').text # print(post_title) # print(post_url) # print(post_price) final_postings = [] for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' if post.find(class_='result-image').get('data-ids'): post_image_id = post.find(class_='result-image').get('data-ids').split(',')[0].split(':')[1] post_image_url = BASE_IMAGE_URL.format(post_image_id) print(post_image_url) else: post_image_url = 'https://craigslist.org/images/peace.jpg' final_postings.append((post_title, post_url, post_price, post_image_url)) # print(post_titles[0].text) # post_title = soup.find_all("a", {'class': 'result-title'}) # print(post_titles[0].get('href')) # print(data) # result! stuff_for_frontend = { 'search': search, 'final_postings': final_postings, } return render(request, 'my_app/new_search.html', stuff_for_frontend)
def new_search(request): search = request.POST.get('search') models.Search.objects.create(search=search) s=quote_plus(search) final_url = base_url.format(s) result = requests.get(final_url) src = result.content soup = BeautifulSoup( src , 'lxml') product_list =soup.find_all("div",{"class" : "product-thumb"}) product_info = [] for product in product_list: product_title = product.find(class_='product-name').text product_url = product.find('a').get('href') if product.find(class_='price'): product_price = product.find(class_='price').text else: product_price = 'N/A' if product.find(class_='img-holder'): product_image_url = product.find('img').get('src') product_info.append((product_title, product_url, product_price,product_image_url)) #jason_format = jason.dumps(jason_dict) #print(jason_format) stuff_for_frontend = { 'search': search, 'final_postings': product_info, } return render(request, 'my_app/new_search.html', stuff_for_frontend)
def new_search(request): # Getting the details of the specific search search = request.POST.get('search') # saving the search to the database models.Search.objects.create(search=search) final_url = BASE_CRAIGSLIST_URL.format(quote_plus(search)) # Extracting the source code of the page response = requests.get(final_url) data = response.text # Passung the source code to beautiful soup to create a Beautiful soup object for it soup = BeautifulSoup(data, features='html.parser') # Extracting all the <a> tags whose class name is 'result-title' into a list post_listings = soup.find_all('li', {'class': 'result-row'}) final_postings = [] for post in post_listings: post_title = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: new_response = requests.get(post_url) new_data = new_response.text new_soup = BeautifulSoup(new_data, features='html.parser') post_text = new_soup.find(id='postingbody').text r1 = re.findall(r'\$\w+', post_text) if r1: post_price = r1[0] else: post_price = 'N/A' if post.find(class_='result-image').get('data-ids'): post_image_id = post.find( class_='result-image').get('data-ids').split(',')[0].split(':')[1] post_image_url = 'https://images.craigslist.org/{}_300x300.jpg'.format( post_image_id) else: post_image_url = 'https://craigslist.org/images/peace.jpg' final_postings.append( (post_title, post_url, post_price, post_image_url)) stuff_for_frontend = { 'search': search, 'final_postings': final_postings, } return render(request, 'my_app/new_search.html', stuff_for_frontend)
def post_file(session, domain='', file='', login=LOGIN): """ posts file to the cloud's upload server param: file - string filename with path """ assert domain is not None, 'no domain' assert file is not None, 'no file' filetype = guess_type(file)[0] if not filetype: filetype = DEFAULT_FILETYPE if LOGGER: LOGGER.warning('File {} type is unknown, using default: {}'.format(file, DEFAULT_FILETYPE)) filename = os.path.basename(file) quoted_login = quote_plus(login) timestamp = str(int(time.mktime(datetime.datetime.now().timetuple()))) + TIME_AMEND url = urljoin(domain, '?cloud_domain=' + str(CLOUD_DOMAIN_ORD) + '&x-email=' + quoted_login + '&fileapi' + timestamp) m = MultipartEncoder(fields={'file': (quote_plus(filename), open(file, 'rb'), filetype)}) try: r = session.post(url, data=m, headers={'Content-Type': m.content_type}, verify = VERIFY_SSL) except Exception as e: if LOGGER: LOGGER.error('Post file HTTP request error: {}'.format(e)) return (None, None) if r.status_code == requests.codes.ok: if len(r.content): hash = r.content[:40].decode() size = int(r.content[41:-2]) return (hash, size) elif LOGGER: LOGGER.error('File {} post error, no hash and size received'.format(file)) elif LOGGER: LOGGER.error('File {} post error, http code: {}, msg: {}'.format(file, r.status_code, r.text)) return (None, None)
def top10_largest_contries(request): data = requests.get( 'https://the-top-10.fandom.com/wiki/Top_10_Biggest_Countries').text soup = BeautifulSoup(data, features='html.parser') data = soup.find('table') data = pandas.read_html(str(data), header=0)[0] final_list = data.values.tolist() data2 = [] for i in final_list[0:10]: image = requests.get(image_url.format(quote_plus(f'{i[1]} flag'))).text image_soup = BeautifulSoup(image, 'html.parser') image_src = image_soup.find('img', {'class': 't0fcAb'})['src'] data2.append(i + [image_src]) return render(request, 'top/largest_contries.html', {'data': data2})
def odata_encode_str(s): """ String eq comparison in OData requires special characters to be escaped, like &. ALSO, single quotes need to be doubled up, so we do that before encoding. """ s = s.replace("'", "''") return quote_plus(s.encode('utf8'))
def search(self, search_params, age=0, ep_obj=None): # pylint: disable=too-many-locals results = [] search_url = self.urls['search'] download_url = self.urls['download'] if self.custom_url: if not validators.url(self.custom_url): logger.log("Invalid custom url: {0}".format(self.custom_url), logger.WARNING) return results search_url = urljoin(self.custom_url, search_url.split(self.url)[1]) download_url = urljoin(self.custom_url, download_url.split(self.url)[1]) if not self.login(): return results for mode in search_params: items = [] logger.log('Search Mode: {0}'.format(mode), logger.DEBUG) for search_string in search_params[mode]: if mode != 'RSS': logger.log('Search string: {0}'.format (search_string.decode('utf-8')), logger.DEBUG) get_params = {} get_params.update(self.categories[mode]) get_params["q"] = quote_plus(search_string.decode('utf-8', 'ignore')) try: torrents = self.get_url(search_url, params=get_params, returns='json') # Handle empty string response or None #4304 if not torrents: raise # Make sure it is iterable #4304 iter(torrents) except Exception: logger.log('Data returned from provider does not contain any torrents', logger.DEBUG) continue for torrent in torrents: title = re.sub(r'\[.*\=.*\].*\[/.*\]', '', torrent['name']) if torrent['name'] else None torrent_url = urljoin(download_url, '{0}/{1}.torrent'.format(torrent['t'], torrent['name'])) if torrent['t'] and torrent['name'] else \ None if not all([title, torrent_url]): continue seeders = try_int(torrent['seeders']) leechers = try_int(torrent['leechers']) # Filter unseeded torrent if seeders < self.minseed or leechers < self.minleech: if mode != 'RSS': logger.log('Discarding torrent because it doesn\'t meet the minimum seeders or leechers: {0} (S:{1} L:{2})'.format(title, seeders, leechers), logger.DEBUG) continue torrent_size = torrent['size'] size = convert_size(torrent_size) or -1 item = {'title': title, 'link': torrent_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'hash': ''} if mode != 'RSS': logger.log('Found result: {0} with {1} seeders and {2} leechers'.format (title, seeders, leechers), logger.DEBUG) items.append(item) # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results
def authenticate(self): config = ConfigParser.ConfigParser() config.read(TOKENS_FILE) if config.has_option("hubic", "refresh_token"): oauth_token = self._refresh_access_token() else: r = requests.get( OAUTH_ENDPOINT + 'auth/?client_id={0}&redirect_uri={1}' '&scope=credentials.r,account.r&response_type=code&state={2}'.format( quote(self._client_id), quote_plus(self._redirect_uri), pyrax.utils.random_ascii() # csrf ? wut ?.. ), allow_redirects=False ) if r.status_code != 200: raise exc.AuthenticationFailed("Incorrect/unauthorized " "client_id (%s)" % str(self._parse_error(r))) try: from lxml import html as lxml_html except ImportError: lxml_html = None if lxml_html: oauth = lxml_html.document_fromstring(r.content).xpath('//input[@name="oauth"]') oauth = oauth[0].value if oauth else None else: oauth = re.search(r'<input\s+[^>]*name=[\'"]?oauth[\'"]?\s+[^>]*value=[\'"]?(\d+)[\'"]?>', r.content) oauth = oauth.group(1) if oauth else None if not oauth: raise exc.AuthenticationFailed("Unable to get oauth_id from authorization page") if self._email is None or self._password is None: raise exc.AuthenticationFailed("Cannot retrieve email and/or password. Please run expresslane-hubic-setup.sh") r = requests.post( OAUTH_ENDPOINT + 'auth/', data={ 'action': 'accepted', 'oauth': oauth, 'login': self._email, 'user_pwd': self._password, 'account': 'r', 'credentials': 'r', }, allow_redirects=False ) try: query = urlparse.urlsplit(r.headers['location']).query code = dict(urlparse.parse_qsl(query))['code'] except: raise exc.AuthenticationFailed("Unable to authorize client_id, invalid login/password ?") oauth_token = self._get_access_token(code) if oauth_token['token_type'].lower() != 'bearer': raise exc.AuthenticationFailed("Unsupported access token type") r = requests.get( API_ENDPOINT + 'account/credentials', auth=BearerTokenAuth(oauth_token['access_token']), ) swift_token = r.json() self.authenticated = True self.token = swift_token['token'] self.expires = swift_token['expires'] self.services['object_store'] = Service(self, { 'name': 'HubiC', 'type': 'cloudfiles', 'endpoints': [ {'public_url': swift_token['endpoint']} ] }) self.username = self.password = None
def _magnet_from_result(info_hash, title): return 'magnet:?xt=urn:btih:{hash}&dn={title}&tr={trackers}'.format( hash=info_hash, title=quote_plus(title), trackers='http://tracker.tntvillage.scambioetico.org:2710/announce')
def authenticate(self): # import httplib # httplib.HTTPConnection.debuglevel = 1 r = requests.get( OAUTH_ENDPOINT+'auth/?client_id={0}&redirect_uri={1}' '&scope=credentials.r,account.r&response_type=code&state={2}'.format( quote(self._client_id), quote_plus(self._redirect_uri), pyrax.utils.random_ascii() # csrf ? wut ?.. ), allow_redirects=False ) if r.status_code != 200: raise exc.AuthenticationFailed("Incorrect/unauthorized " "client_id (%s)"%str(self._parse_error(r))) try: from lxml import html as lxml_html except ImportError: lxml_html = None if lxml_html: oauth = lxml_html.document_fromstring(r.content).xpath('//input[@name="oauth"]') oauth = oauth[0].value if oauth else None else: oauth = re.search(r'<input\s+[^>]*name=[\'"]?oauth[\'"]?\s+[^>]*value=[\'"]?(\d+)[\'"]?>', r.content) oauth = oauth.group(1) if oauth else None if not oauth: raise exc.AuthenticationFailed("Unable to get oauth_id from authorization page") r = requests.post( OAUTH_ENDPOINT+'auth/', data = { 'action': 'accepted', 'oauth': oauth, 'login': self._email, 'user_pwd': self._password, 'account': 'r', 'credentials': 'r', }, allow_redirects=False ) if r.status_code == 302 and r.headers['location'].startswith(self._redirect_uri): query = urlparse.urlsplit(r.headers['location']).query code = dict(urlparse.parse_qsl(query))['code'] else: raise exc.AuthenticationFailed("Unable to authorize client_id, invalid login/password ?") r = requests.post( OAUTH_ENDPOINT+'token/', data={ 'code': code, 'redirect_uri': self._redirect_uri, 'grant_type': 'authorization_code', }, auth=(self._client_id, self._client_secret) ) if r.status_code != 200: try: err = r.json() err['code'] = r.status_code except: err = {} raise exc.AuthenticationFailed("Unable to get oauth access token, " "wrong client_id or client_secret ? (%s)"%str(err)) oauth_token = r.json() if oauth_token['token_type'].lower() != 'bearer': raise exc.AuthenticationFailed("Unsupported access token type") r = requests.get( API_ENDPOINT+'account/credentials', auth=BearerTokenAuth(oauth_token['access_token']), ) swift_token = r.json() self.authenticated = True self.token = swift_token['token'] self.expires = swift_token['expires'] self.services['object_store'] = Service(self, { 'name': 'HubiC', 'type': 'cloudfiles', 'endpoints': [ {'public_url': swift_token['endpoint']} ] }) self.username = self.password = None
def search(self, search_strings, age=0, ep_obj=None): # pylint: disable=too-many-locals, too-many-branches, too-many-statements results = [] if not self.login(): return results for mode in search_strings: items = [] logger.log(u"Search Mode: {}".format(mode), logger.DEBUG) for search_string in search_strings[mode]: if mode != 'RSS': search_url = self.urls['search'] % (quote_plus(search_string), self.categories) logger.log(u"Search string: {}".format(search_string.decode("utf-8")), logger.DEBUG) else: search_url = self.urls['rss'] % self.categories if self.freeleech: search_url = search_url.replace('active=1', 'active=5') data = self.get_url(search_url, returns='text') if not data or 'please try later' in data: logger.log(u"No data returned from provider", logger.DEBUG) continue if data.find('No torrents here') != -1: logger.log(u"Data returned from provider does not contain any torrents", logger.DEBUG) continue # Search result page contains some invalid html that prevents html parser from returning all data. # We cut everything before the table that contains the data we are interested in thus eliminating # the invalid html portions try: index = data.lower().index('<table class="mainblockcontenttt"') except ValueError: logger.log(u"Could not find table of torrents mainblockcontenttt", logger.DEBUG) continue data = data[index:] with BS4Parser(data, 'html5lib') as html: if not html: logger.log(u"No html data parsed from provider", logger.DEBUG) continue torrent_rows = [] torrent_table = html.find('table', class_='mainblockcontenttt') if torrent_table: torrent_rows = torrent_table('tr') if not torrent_rows: logger.log(u"Could not find results in returned data", logger.DEBUG) continue # Cat., Active, Filename, Dl, Wl, Added, Size, Uploader, S, L, C labels = [label.a.get_text(strip=True) if label.a else label.get_text(strip=True) for label in torrent_rows[0]('td')] # Skip column headers for result in torrent_rows[1:]: try: cells = result.findChildren('td')[:len(labels)] if len(cells) < len(labels): continue title = cells[labels.index(u'Filename')].a.get_text(strip=True) seeders = try_int(cells[labels.index(u'S')].get_text(strip=True)) leechers = try_int(cells[labels.index(u'L')].get_text(strip=True)) torrent_size = cells[labels.index(u'Size')].get_text() size = convert_size(torrent_size) or -1 download_url = self.url + '/' + cells[labels.index(u'Dl')].a['href'] except (AttributeError, TypeError, KeyError, ValueError, IndexError): continue if not all([title, download_url]): continue # Filter unseeded torrent if seeders < min(self.minseed, 1): if mode != 'RSS': logger.log(u"Discarding torrent because it doesn't meet the minimum seeders: {0}. Seeders: {1})".format (title, seeders), logger.DEBUG) continue item = {'title': title, 'link': download_url, 'size': size, 'seeders': seeders, 'leechers': leechers, 'pubdate': None, 'hash': None} if mode != 'RSS': logger.log(u"Found result: %s with %s seeders and %s leechers" % (title, seeders, leechers), logger.DEBUG) items.append(item) # For each search mode sort all the items by seeders if available items.sort(key=lambda d: try_int(d.get('seeders', 0)), reverse=True) results += items return results