def _request_check_location(r, *args, **kwargs): """ hook的回调函数 :param r: :param args: :param kwargs: :return: """ if not r.is_redirect: return url = r.headers['location'] # The scheme should be lower case... parsed = urlparse.urlparse(url) url = parsed.geturl() # Facilitate relative 'location' headers, as allowed by RFC 7231. # (e.g. '/path/to/resource' instead of 'http://domain.tld/path/to/resource') # Compliant with RFC3986, we percent encode the url. if not parsed.netloc: url = urlparse.urljoin(r.url, requote_uri(url)) else: url = requote_uri(url) succ, errstr = check_ssrf(url) if not succ: print "SSRF Attack: %s" % (errstr) raise requests.exceptions.InvalidURL("SSRF Attack: %s" % (errstr,))
def search(self, keyword): if type(keyword) is not str: raise Exception( 'Error while processing search keyword, keyword must be str type!' ) search_url = self.search_url_datasource() if not search_url: raise Exception( 'Error while building search query, unknown URL returned!') html_contents = self.helper_get_contents( search_url.format(requote_uri(keyword))) parsed_object = self.search_results_datasource(html_contents) return parsed_object if type(parsed_object) is list else []
def recursive_folder(path): try: uri = fs_api + path folder = client.GET(requote_uri(uri)) # folder = client.GET(fs_api + urllib.quote(path.encode('utf8'))) folder_attributes = folder.json() # if folder_attributes.has_key('files'): if "files" in folder_attributes: print(type(folder_attributes['files'])) print("\n ****************** FILES FOUND ******************\n") print(folder_attributes['files']) print( "\n****************** FILES FOUND ENDED ******************\n") for f in folder_attributes['files']: versions = f['num_versions'] file_path = f['path'] uploaded_by = f['uploaded_by'] modified_date = datetime.strptime(f['last_modified'][5:16], '%d %b %Y') if versions > 1 and uploaded_by in user_restrict and modified_date == restrict_date: with codecs.open('files_to_delete.txt', 'ab', 'utf8') as fout: fout.write('%s\n' % file_path) #with codecs.open('data.log', 'ab', 'utf-8') as log: #log.write('{Path: %s, Uploaded_by: %s, last_modified: %s, num_versions: %s}\n' %(file_path, uploaded_by, f['last_modified'], versions)) # if folder_attributes.has_key('folders'): if 'folders' in folder_attributes: print("\n****************** FOLDERS FOUND ******************\n") print(folder_attributes['folders']) print( "\n****************** FOLDERS FOUND ENDED ******************\n" ) for dirs in folder_attributes['folders']: print('Walking through: ' + dirs['path']) recursive_folder(dirs['path']) except Exception as e: print('Error on path: %s' % path) with open('error.txt', 'ab') as error: error.write('%s\t %r\n' % (path, e)) print( "\nPlease look at 'files_to_delete.txt' for the files found. You may also look at error.txt for any errors" )
async def upload_to_gdrive(file_upload, message): subprocess.Popen(('touch', 'rclone.conf'), stdout=subprocess.PIPE) with open('rclone.conf', 'a', newline="\n") as fole: fole.write("[DRIVE]\n") fole.write(f"{RCLONE_CONFIG}") destination = f'{DESTINATION_FOLDER}' if os.path.isfile(file_upload): tmp = subprocess.Popen([ 'rclone', 'copy', '--config=rclone.conf', f'{file_upload}', 'DRIVE:' f'{destination}', '-v' ], stdout=subprocess.PIPE) out = tmp.communicate() print(out) indexurl = f"{INDEX_LINK}/{file_upload}" g_link = requote_uri(indexurl) await message.edit_text( f'{file_upload} has been Uploaded successfully to your cloud 🤒\n\n Direct Downlaod link: <a href="{g_link}">here</a>' ) os.remove(file_upload) else: tt = os.path.join(destination, file_upload) print(tt) tmp = subprocess.Popen([ 'rclone', 'copy', '--config=rclone.conf', f'{file_upload}', 'DRIVE:' f'{tt}', '-v' ], stdout=subprocess.PIPE) out = tmp.communicate() print(out) indexurl = f"{INDEX_LINK}/{file_upload}/" g_link = requote_uri(indexurl) await message.edit_text( f'Folder has been Uploaded successfully to {tt} in your cloud 🤒\n\n Direct Downlaod link: <a href="{g_link}">here</a>' ) shutil.rmtree(file_upload)
def getCountryNameFromPincode(self, pincode): if (re.fullmatch("\w{2}\d{2}|\d{5,6}", pincode)): url = self.baseUrl + self.pincodeUrl url = url.replace("#pincode", pincode) url = requote_uri(url) print(url) response = requests.get(url) if (response.status_code == 200): print(response.text) return response.text else: return "Sorry..!! I could not reach my internal api to get the details by country" else: return "Pincode is not on the appropriate format..!! Please give the picode as 110001 "
def profile(): """Handle Elevation profile requests.""" url = request.args.get('url', default='', type=str) step = request.args.get('step', default=0.30, type=float) url = requote_uri(url) if not url: raise TilerError("Missing 'url' parameter") # Gives an list of string x = request.args.get('x') y = request.args.get('y') x = np.array(x.split(','), dtype=float) y = np.array(y.split(','), dtype=float) print('data:', x) print('length:', len(x)) coord_x = [] coord_y = [] # Checking length of x and y if len(x) != len(y): raise TilerError('Error: Length of X and Y parameters are not equal') elif len(x) < 2: raise TilerError( 'Error: Length of parameters should be greater than 1') for i in range(len(x)-1): print('Generating line from points') temp_x, temp_y = get_profile.get_point2line( x[i], y[i], x[i+1], y[i+1], step=step) if temp_x is None or temp_y is None: raise TilerError( 'Error: Distance between points should be less than 10KM') for j in range(len(temp_x)): coord_x.append(temp_x[j]) coord_y.append(temp_y[j]) print('Generated %d number of points' % (len(coord_x))) # Initializing dataset data = [] info = get_value.get_value( address=url, coord_x=coord_x, coord_y=coord_y) return (jsonify(info))
def sendEmailToRest(self, entity): data = {} data['name'] = "" data['mailid'] = entity data['mobile'] = "" url = self.baseUrl + self.mailURL url = requote_uri(url) data = json.dumps(data) response = requests.post(url, json=data) if (response.status_code == 200): return response.text else: return response.text
def new_search(request): # store the user input into a variable search = request.POST.get('search') # insert the search text into the search table in the database models.Search.objects.create(search_text=search) final_url = BASE_CRAIGLIST_URl.format(requote_uri(search)) response_from_craiglist_website = requests.get(final_url) data = response_from_craiglist_website.text # Parsing the beautiful soup to create a BeautifulSoup object for it. soup = BeautifulSoup(data, features='html.parser') # Extracting all <a> tags whose class name is 'result-row' into a list post_listings = soup.find_all('li', class_='result-row') #print('posts: ', post_listings) final_postings = [] for post in post_listings: post_title = post.find('a', class_='result-title').text post_url = post.find('a', class_='result-title').get('href') #post_price = post.find(class_='result-price').text if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' # check if data-ids is present in the result-image if post.find(class_='result-image').get('data-ids'): CRAIGLIST_IMAGES_URL = 'https://images.craigslist.org/{}' post_image_id = post.find(class_='result-image').get( 'data-ids').split(',')[0].split(':')[1] post_image_url = CRAIGLIST_IMAGES_URL.format( post_image_id) + '_600x450.jpg' #print(post_image_url) else: post_image_url = 'https://london.craigslist.org/images/peace.jpg' final_postings.append( (post_title, post_url, post_price, post_image_url)) #print(final_postings) # if post.find(class_='result-price'): # post_price = post.find(class_='result-price').text # else: # new_response = request.get(post_url) # new_data = new_response.text # new_soup = BeautifulSoup(new_data, features='html.parser') # post_text = new_soup.find(id='postingbody').text # # r1 = re. frontend_context = {'search': search, 'final_postings': final_postings} return render(request, 'my_app/new_search.html', frontend_context)
async def encode_url(self, ctx, *, input: commands.clean_content = None): if not input: e = discord.Embed(description=":no_entry_sign: You must give an input string", colour=0xE74C3C) await ctx.send(embed=e) return e = discord.Embed(title="Result", colour=0x2ECC71) result = requote_uri(str(input)) e.add_field(name="Input", value=f"`{input}`") e.add_field(name="Output", value=f"`{result}`") e.set_footer(text="Made with ❤️ by Roxiun") await ctx.send(embed=e)
def start_request(self): url = requote_uri( f'http://www.51bc.net/cy/serach.php?f_key={self.encode_key()}&f_type=chengyu&f_type2=' ) headers = { 'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/77.0.3865.120 Safari/537.36', 'Referer': 'http://www.51bc.net/cy/serach.php' } r = requests.get(url, headers=headers) content = r.content.decode('gbk') doc = pq(content) idioms = doc("u").text() return idioms.split(" ")
async def npmSearch(self, ctx, *, args): parsed = requote_uri(args) req = requests.get(f"https://api.npms.io/v2/search?q={parsed}") if req.status_code != 200: embed = discord.Embed( title="Error API STATUS", description=f"Código de la API: {req.status_code}", color=discord.Color.red(), ) embed.set_thumbnail( url="https://media1.tenor.com/images/46ce1235c5697ce170c6e84f4b4fb4e7/tenor.gif" ) await ctx.send(embed=embed) elif req.status_code == 200: try: resp = req.json() results_info = resp["results"] resultsGet_info = results_info[0] pkgInfo = resultsGet_info["package"] linksPkg = pkgInfo["links"] embed = discord.Embed( title=f"La información sobre el paquete **{args}** es: ", description=f""" Nombre: {pkgInfo["name"]} Versión: {pkgInfo["version"]} Descripción: {pkgInfo["description"]} Fecha: {pkgInfo["date"]} Link: {linksPkg["npm"]} """, color=discord.Color.random(), ) embed.set_thumbnail( url="https://authy.com/wp-content/uploads/npm-logo.png" ) await ctx.send(embed=embed) except Exception as err: embed = discord.Embed( title="Error API STATUS", description=f"Código de la API: {err}", color=discord.Color.red(), ) embed.set_thumbnail( url="https://media1.tenor.com/images/46ce1235c5697ce170c6e84f4b4fb4e7/tenor.gif" ) await ctx.send(embed=embed) else: print(req.status_code)
def create_time_schedule_url(campus, quarter, department): """ Returns a URL for the UW Time Schedule for the given quarter and campus. """ quarter = validate_quarter_name(quarter) campus = validate_campus_name(campus) department = department.lower() url = f'{TIME_SCHEDULE_URL}/{campus}{quarter}/{department}.html' if not is_url(url): msg = f'Generated URL is an invalid URL. ' \ f'Given arguments are invalid: {campus}, {quarter}, {department}' raise ValueError(msg) return requote_uri(url)
def getCompanyIDByName(company): http = urllib3.PoolManager() url = requote_uri(companies_url + "?format=json&name=" + quote(company.name)) r = http.request('GET', url) jsondata = json.loads(r.data.decode('utf-8')) if len(jsondata) == 0: # Company name not found company.company_id = -1 else: # More than one company with the same name found company.company_id = jsondata[0]['id'] return company
def new_search(request): search = request.POST.get('search') models.Search.objects.create(search=search) # print(requote_uri(search)) final_url = BASE_CRAGSLIST_URL.format(requote_uri(search)) # print(final_url) response = requests.get(final_url) data = response.text # print(data) soup = BeautifulSoup(data, features='html.parser') post_listing = soup.find_all('li', {'class': 'result-row'}) # post_titles = post_listing[0].find(class_= 'result-title').text # post_url = post_listing[0].find('a').get('href') # post_price = post_listing[0].find(class_= 'result-price').text # print(post_titles) # print(post_url) # print(post_price) final_postings = [] for post in post_listing: post_titles = post.find(class_='result-title').text post_url = post.find('a').get('href') if post.find(class_='result-price'): post_price = post.find(class_='result-price').text else: post_price = 'N/A' if post.find(class_='result-image').get('data-ids'): post_image_id = post.find( class_='result-image').get('data-ids').split(':')[1].replace( ",3", "") print(post_image_id) post_image_url = BASE_IMAGE_URL.format(post_image_id) print(post_image_url) else: post_image_url = 'https://craigslist.org/images/peace.jpg' final_postings.append( (post_titles, post_url, post_price, post_image_url)) stuff_for_frontend = { 'search': search, 'final_postings': final_postings, } return render(request, 'my_app/new_search.html', stuff_for_frontend)
def post(self): token = request.args.get('token') verify = VerifyKey(KEYS_PATH, token) if(verify.isAuthorized()): url = request.json.get('url') url = requote_uri(url) full = request.json['options'].get('fullPage') formatType = request.json['options'].get('type') quality = request.json['options'].get('quality') tor = request.json['options'].get('tor') timeout = request.json['options'].get('timeout') browser = request.json['options'].get('browser') height = request.json['options'].get('height') width = request.json['options'].get('width') # Set defaults values if(quality == None): quality = 100 if(tor == None): tor = False if(height == None): height = 600 if(width == None): width = 800 checker = Checker(url, full, formatType, quality, tor, timeout, browser, height, width) checkerAnswer = checker.verifyAll() if(checkerAnswer != 0): return {'error':checkerAnswer.first , 'error-description':checkerAnswer.second} netloc = urlparse(url).netloc netloc = netloc.replace('.', '_') netloc = netloc.replace(':', '_') ts = calendar.timegm(time.gmtime()) filename = 'mps_{}_{}'.format(ts, netloc) screenshot = Screenshot(SCREENSHOT_PATH, FIREFOX_PATH, CHROME_PATH, TOR_PROFILE, TOR_URL) answer = screenshot.getImage(full, filename, url, formatType, tor, timeout, browser, height=height, width=width) if(answer == 0): mimeType = 'image/{}'.format(formatType) filename = '{}/{}.{}'.format(SCREENSHOT_PATH ,filename, formatType) return send_file(filename, mimetype=mimeType) else: return {'error':answer.first, 'error-description':answer.second} else: return {'error': Errors.UNAUTHORIZED.first, 'error-description': Errors.UNAUTHORIZED.second}
def creating_file_list(app, exception): ''' Creates a files containing the path to every html file that was compiled. This files are `.doclist` and the sitemap. ''' if app.builder.name == 'html': build_path = app.outdir separator = '\n' with open(build_path+'/.doclist', 'w') as doclist_file: list_text = separator.join(list_compiled_html) doclist_file.write(list_text) sitemap = '<?xml version=\'1.0\' encoding=\'utf-8\'?>'+separator sitemap += '<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">'+separator for compiled_html in list_compiled_html: sitemap += '\t<url><loc>' + requote_uri(html_theme_options.get('wazuh_doc_url') + '/' + version + '/' + compiled_html) + '</loc></url>' + separator sitemap += '</urlset>' with open(build_path+'/'+version+'-sitemap.xml', 'w') as sitemap_file: sitemap_file.write(sitemap)
def expandUrl(url: str): header = { "Authorization": settings["bitlyAuth"], "Content-Type": "application/json" } params = {"bitlink_id": requote_uri(url)} try: response = post("https://api-ssl.bitly.com/v4/expand", json=params, headers=header) data = response.json() long = data["long_url"] except Exception: long = url return long
def data_get_entry_history(self, entryTypeId, modifiedSince): """ Get Record Ids for entries that were modified or deleted. :param entryTypeId: entryListID or entry apiName. Required. :param modifiedSince: Date since modified :type modifiedSince: datetime. :returns: response.json """ modifiedSince = requote_uri(modifiedSince) endpoint = f"/api/rest/v4/data/entrydata/{entryTypeId}/entries/history?modifiedSince={modifiedSince}" response = self._get_response(method='get', endpoint=endpoint) return response.json()
def translate(self, word, return_all=False, **kwargs): """ function that uses PONS to translate a word @param word: word to translate @type word: str @param return_all: set to True to return all synonym of the translated word @type return_all: bool @return: str: translated word """ if self._validate_payload(word, max_chars=50): url = "{}{}-{}/{}".format(self.__base_url, self._source, self._target, word) url = requote_uri(url) response = requests.get(url) if response.status_code == 429: raise TooManyRequests() if response.status_code != 200: raise RequestError() soup = BeautifulSoup(response.text, 'html.parser') elements = soup.findAll(self._element_tag, self._element_query) if not elements: raise ElementNotFoundInGetRequest(word) filtered_elements = [] for el in elements: temp = '' for e in el.findAll('a'): if e.parent.name == 'div': if e and "/translate/{}-{}/".format( self._target, self._source) in e.get('href'): temp += e.get_text() + ' ' filtered_elements.append(temp) if not filtered_elements: raise ElementNotFoundInGetRequest(word) word_list = [ word for word in filtered_elements if word and len(word) > 1 ] if not word_list: raise TranslationNotFound(word) return word_list if return_all else word_list[0]
def __init__(self, record_to_request: str, image_as_base64: bool, input_type="name", temp_file="image"): ############################# # greenprint("[+] Running as Discord Attachment") # greenprint("[+] Not running as Discord Attachment") #print(str(os.environ['DISCORDAPP'])) if image_as_base64 == False: self.filename = temp_file + ".png" if search_validate(input_type): #in pubchem_search_types: greenprint("searching for an image : " + record_to_request) # fixes local code/context to work with url/remote context if input_type == "iupac_name": self.input_type = "name" else: self.input_type = input_type self.request_url = requote_uri("{}/compound/{}/{}/PNG".format(\ API_BASE_URL,self.input_type,record_to_request)) blueprint("[+] Requesting: " + makered(self.request_url)) self.rest_request = requests.get(self.request_url) # redprint("[-] Request failure at local level") # True means no error if self.was_there_was_an_error() == True: # request good # Store image self.image_storage = Image.open( BytesIO(self.rest_request.content)) if image_as_base64 == False: try: greenprint("[+] Saving image as {}".format( self.filename)) self.image_storage.save(self.filename, format="png") except Exception as blorp: redprint( "[-] Exception when opening or writing image file") print(blorp) elif image_as_base64 == True: print(self.rest_request.raw) greenprint("[+] Encoding Image as Base64") self.image_storage = base64.b64encode(self.image_storage) else: redprint("[-] Error with Class Variable self.base64_save") else: redprint("[-] Input type was wrong for Image Search") return None
def _lookup_ips(self, param): action_result = self.add_action_result(ActionResult(dict(param))) ret_val = self._check_apikey(action_result) if phantom.is_fail(ret_val): return ret_val try: ips = [x.strip() for x in param["ips"].split(",")] ips = list(filter(None, ips)) if not ips: return action_result.set_status( phantom.APP_ERROR, INVALID_COMMA_SEPARATED_VALUE_ERR_MSG.format(key='ips')) ips = ",".join(ips) ips_string = requote_uri(ips) except Exception as e: err = self._get_error_message_from_exception(e) err_msg = "Error occurred while processing 'ips' action parameter. {0}".format( err) return action_result.set_status(phantom.APP_ERROR, err_msg) ret_val, response_json, status_code = self._make_rest_call( action_result, "get", LOOKUP_IPS_URL.format(ips=ips_string), headers=self._headers) if phantom.is_fail(ret_val): return ret_val result_data = [] action_result.add_data(result_data) try: for result in response_json: if result["code"] in CODES: result["code_meaning"] = CODES[result["code"]] else: result["code_meaning"] = "This code is unmapped" result["visualization"] = VISUALIZATION_URL.format( ip=result["ip"]) result_data.append(result) return action_result.set_status(phantom.APP_SUCCESS) except Exception as e: err = self._get_error_message_from_exception(e) err_msg = "Error occurred while processing results: {0}".format( err) return action_result.set_status(phantom.APP_ERROR, err_msg)
def getopenedtickets(timespan, config): '''Need to fix - just here to make PyLint happy''' with open('queryOpened.json', 'r') as fileopened: queryjson = json.load(fileopened) # here we search for all tickets opened before the end of the search day queryjson["list_info"]["search_criteria"][0]["value"] = timespan baseurl = config["url"] + '/api/v3/requests?TECHNICIAN_KEY=' + config[ "technicianKey"] ticketcount = 0 for statuscode in config["statusCodes"]: queryjson["list_info"]["search_criteria"][1]["value"][ "id"] = statuscode more = True resultsindex = 1 while more: queryjson["list_info"]["start_index"] = resultsindex encodeduri = requote_uri(json.dumps(queryjson)) queryurl = baseurl + '&input_data=' + encodeduri response = fetchdata(queryurl) jsonresults = response.json() for request in jsonresults["requests"]: reqid = request["id"] if request["priority"]: if request["priority"]["name"] in config[ "priorityTranslations"]: reqpri = config["priorityTranslations"][ request["priority"]["name"]]["Priority"] else: reqpri = request["priority"]["name"] else: reqpri = "" if request["site"]: if request["site"]["name"] in config["siteTranslations"]: reqsite = config["siteTranslations"][request["site"] ["name"]]["Name"] else: reqsite = request["site"]["name"] else: reqsite = "NONE" ticketcount = ticketcount + 1 reqcreated = datetime.datetime.fromtimestamp( int(int(request["created_time"]["value"]) / 1000)).strftime("%Y-%m-%d %H:%M:%S") resultstring = reqid + "," + reqpri + "," + reqsite + "," + reqcreated + "\n" config["results"].append(resultstring) more = jsonresults["list_info"]["has_more_rows"] if more: resultsindex = resultsindex + 100
def get_attribute_value(selected_html, find_all_tabs, attribute, mode): try: values_list = [] if len(selected_html) > 0: for link in selected_html: all_tabs = link.find_all(find_all_tabs) # For ex:- a ,input for a in all_tabs: if mode == "url": values_list.append(requote_uri(a.get(attribute))) else: values_list.append(a.get(attribute)) return values_list except Exception as e: print("get_attribute_value " + str(e))
async def define(self, context): args = context.message.clean_content.split(maxsplit=1) if 2 > len(args): await utils.say( context.channel, content= f"Type `{context.prefix}{context.command.name} {context.command.usage}` to look up a term in the dictionary." ) return search_term = re.sub(r"\s+", " ", args[1].strip()) search_results = utils.dictionary.regular_lookup(search_term) if search_results: base_url = Settings.command_settings("define")["base_url"] search_url = requote_uri(f"{base_url}{search_term}") reply = discord.Embed(title=f'Define "{search_term}"', url=search_url) reply.set_footer(text=f"Source: {search_url}") try: num_entries = len(search_results) definitions = [] for i, entry in enumerate(search_results): if i > 2: break is_offensive = " *(offensive)*" if entry.is_offensive else "" term_type = entry.term_type definitions.append( f"**{search_term}** {i + 1}/{num_entries} *({term_type})*{is_offensive}" ) definitions.append("".join( ["*", "\n\n".join(entry.short_definitions), "*"])) definitions.append("\n") reply.description = "\n".join(definitions) except AttributeError: # Suggested search terms reply.url = "" suggestions = "\n".join(search_results) reply.description = f"**Did you mean...**\n*{suggestions}*" await utils.say(context.channel, embed=reply) else: await utils.say( context.channel, content=f"Couldn't find a definition for `{search_term}`.")
def post(self, request): url = requote_uri("https://shopee.vn/search?keyword=" + request.POST.get('keyword')) task = crawl_shopee_url.apply_async(kwargs={ 'url': url, 'required_class': 'shopee-search-item-result__item', 'scroll_to_bottom': True, 'label': 'shopee_crawling_search_result', }, link=get_shopee_products_url.s()) return Response({"task_id": task.id})
def getclosedtickets(timespan, config): '''Need to fix - just here to make PyLint happy''' with open('queryClosed.json', 'r') as fileclosed: queryjson = json.load(fileclosed) queryjson["list_info"]["search_criteria"][0]["value"] = timespan[0] queryjson["list_info"]["search_criteria"][1]["value"] = timespan[1] baseurl = config["url"] + '/api/v3/requests?TECHNICIAN_KEY=' + config[ "technicianKey"] more = True resultsindex = 1 ticketcount = 0 while more: queryjson["list_info"]["start_index"] = resultsindex encodeduri = requote_uri(json.dumps(queryjson)) queryurl = baseurl + '&input_data=' + encodeduri response = fetchdata(queryurl) jsonresults = response.json() for request in jsonresults["requests"]: #reqid = request["id"] #if (request["priority"]): # if (request["priority"]["name"] in config["priorityTranslations"]): # reqpri = config["priorityTranslations"][request["priority"]["name"]]["Priority"] # else: # reqpri = request["priority"]["name"] #else: # reqpri = "" if request["site"]: if request["site"]["name"] in config["siteTranslations"]: reqsite = config["siteTranslations"][request["site"] ["name"]]["Name"] else: reqsite = request["site"]["name"] else: reqsite = "NONE" if reqsite in config["results"]: config["results"][reqsite]["TotalClosed"] = str( int(config["results"][reqsite]["TotalClosed"]) + 1) else: config["UnknownSites"].append(reqsite) reqsite = "UNKNOWN" config["results"][reqsite]["TotalClosed"] = str( int(config["results"][reqsite]["TotalClosed"]) + 1) ticketcount = ticketcount + 1 more = jsonresults["list_info"]["has_more_rows"] if more: resultsindex = resultsindex + 100
def post(self, request): url = requote_uri("https://tiki.vn/search?q=" + request.POST.get('keyword')) task = craw_tiki_url.apply_async(kwargs={ 'url': url, 'required_class': 'product-box-list', 'scroll_to_bottom': False, 'label': 'tiki_crawling_search_result', }, link=get_tiki_products_url.s()) return Response({"task_id": task.id})
def getURL(): with open(URLS_PATH, 'r') as file: urls = '' for i, line in enumerate(file): if i == 0: url = line else: urls += line if not urls: # urls var is empty, so os.remove(URLS_PATH) else: with open(URLS_PATH, 'w') as file: file.write(urls) return requote_uri(url).rstrip('%0A')
def getTopCountyCases(self, entity=1, sortType="DESC", type=""): if (entity == "" or entity == None): entity = 1 url = self.baseUrl + self.topCountyDetailsUrl url = url.replace("#entity", str(entity)) url = url.replace("#sortType", sortType) url = url.replace("#type", type) url = requote_uri(url) print(url) response = requests.get(url) if (response.status_code == 200): responseMsg = response.text else: responseMsg = self.errMsg return responseMsg
def getListingIDBySymbol(listing): http = urllib3.PoolManager() url = requote_uri(ticker_url + "?format=json&ticker=" + quote(listing.ticker)) r = http.request('GET', url) jsondata = json.loads(r.data.decode('utf-8')) print("getListingIDBySymbol: retrieving: " + url) if len(jsondata) == 0: # Ticker symbol not found print("getListingIDBySymbol: Did not find ticker: " + listing.ticker) listing.listing_id = -1 else: # More than one lising with the same name found listing.listing_id = jsondata[0]['id'] print("getListingIDBySymbol: Found ticker: " + listing.ticker + " with ID: " + str(listing.listing_id)) return listing
def _send(self, endpoint: str, method: str = 'GET', data: Optional[Dict] = None) -> Optional[Dict]: """Send a HTTP request to Gandi Live DNS API. Args: endpoint (str): Live DNS API enpoint. method (str): HTTP method. data (str): payload to attach to the request. Returns: None on error. JSON object representing the response of Live DNS API Raises: None.""" headers = self._headers.copy() # parameters if not self._url.endswith('/') and not endpoint.startswith('/'): endpoint = f'/{endpoint}' url = requote_uri(f'{self._url}{endpoint}') if data: headers['Content-type'] = 'application/json' # request try: r = requests.request(method=method, url=url, headers=headers, json=data, timeout=60.0) except Timeout: return None if not r.ok: return None if r.status_code == 204: # HTTP/204 No content (on success) return {'code': 204, 'message': 'ok'} return r.json()
def test_requote_uri_with_unquoted_percents(uri, expected): """See: https://github.com/requests/requests/issues/2356""" assert requote_uri(uri) == expected