def test_request_kwargs(m): """Add request timeout #21""" headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0' } m.get('http://mock.com/', request_headers=headers, text='body') favicon.get('http://mock.com/', headers=headers) # Test deprecated header argument with pytest.warns(DeprecationWarning): favicon.get('http://mock.com/', headers)
def test_request_kwargs(m): """Add request timeout #21""" headers = { 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0' } m.get('http://mock.com/', request_headers=headers, text='body') # raises requests_mock.exceptions.NoMockAddress if headers do not match # https://requests-mock.readthedocs.io/en/latest/matching.html#request-headers with pytest.warns(None): favicon.get('http://mock.com/', headers=headers) # Test deprecated header argument with pytest.warns(None): favicon.get('http://mock.com/', headers=headers)
def downloadIcon(self, url, appDir): #Get all favicons icons = favicon.get(url) #Find the largest favicon icon = icons[0] if icon: iconPath = os.path.join(appDir, "icon") #Remove an existing icon if applicable if os.path.exists(iconPath): os.remove(iconPath) #Download the icon and save it response = requests.get(icon.url, stream=True) origIconPath = iconPath + '.' + icon.format with open(origIconPath, 'wb') as image: for chunk in response.iter_content(1024): image.write(chunk) self.onUpdate(16) #If not already an ico file, convert it finalIconPath = iconPath + '.ico' if icon.format != 'ico': img = Image.open(origIconPath) img.save(finalIconPath) #Return the path to the saved icon return finalIconPath return None
def get_favicons(self): for medium in models.Medium.objects.filter(favicon=None): print(medium.title) try: icons = favicon.get('http://' + medium.uri) except: pass png_icons = list(filter(lambda x: x.format == 'png', icons)) ico_icons = list(filter(lambda x: x.format == 'ico', icons)) ss_png_icons = list( filter(lambda x: x.width == x.height and x.width > 0, png_icons)) found = False for img in ss_png_icons: try: if requests.get(img.url).status_code == 200: medium.favicon = img.url medium.save() found = True except: pass if found: break if not found and ico_icons: print(ico_icons) ico_response = requests.get(ico_icons[0].url) if ico_response.status_code == 200: medium.favicon = ico_response.url medium.save()
def download_icon(request_uri, timeout=connection_timeout): icons = favicon.get(request_uri) if icons[0]: icon = get_data(icons[0].url, timeout=timeout) return icon else: return None
def download_favi_json(): with open('app.json') as json_file: json_data = json.load(json_file) for i in range(0, len(json_data) - 1): uri_list.append(json_data[i]['uri']) name_list.append(json_data[i]['app_name']) for i in range(0, len(json_data) - 1): icons = favicon.get(uri_list[i]) # icons(list) may not have .ico file for j in icons: if ".ico" in j.url: icon = j response = requests.get(icon.url, stream=True) with open('icon/' + name_list[i] + '.{}'.format(icon.format), 'wb') as image: for chunk in response.iter_content(1024): image.write(chunk) print("Favicon ICO Download Finished!") return name_list
def download_favi_xlsx(): wb = load_workbook('example.xlsx') ws = wb.active name_list = [] uri_list = [] # r[2] field = web site Name in Excel # r[3] field = web site URI in Excel for r in ws.rows: if "http" in str(r[3].value): name_list.append(r[2].value) uri_list.append(r[3].value) for i in range(0, len(uri_list)): icons = favicon.get(uri_list[i]) # icons(list) may not have .ico file for j in icons: if ".ico" in j.url: icon = j response = requests.get(icon.url, stream=True) with open('icon/' + name_list[i] + '.{}'.format(icon.format), 'wb') as image: for chunk in response.iter_content(1024): image.write(chunk) print("Favicon ICO Download Finished!") return name_list
def add(url: str): tags = typer.prompt("Any tags? - comma seperated list") tags = tags.lower() try: icon = favicon.get(url)[0][0] except: icon = None html = requests.get(url) soup = bs(html.content, 'html.parser') title = soup.find('title') title = title.get_text() db.insert({ "Title": title, "URL": url, "Icon": icon, "Tags": tags.split(",") }) typer.echo("{} saved with - Title: {} - Favicon: {}".format( typer.style(url, fg=typer.colors.GREEN, bold=False), typer.style(title, fg=typer.colors.GREEN, bold=True), typer.style(icon, fg=typer.colors.GREEN, bold=True)))
def find_icons_at_url(self): try: return favicon.get(self.url_to_search, allow_redirects=True, timeout=5) except: return []
def create_item_icon(item): icons = favicon.get(item.url) icon = icons[0] temp = NamedTemporaryFile(delete=True) temp.write(urlopen(icon.url).read()) temp.flush() item.icon.save(f"icon-{item.id}.png", File(temp)) item.save()
def test_invalid_meta_tag(m): m.get( 'http://mock.com/', text='<meta content="en-US" data-rh="true" itemprop="inLanguage"/>', ) icons = favicon.get('http://mock.com/') assert not icons
def test_link_tag_href_attribute(m, link, url): m.get('http://mock.com/', text=link) icons = favicon.get('http://mock.com/') assert icons icon = icons[0] assert icon.url == url
def test_link_tag_sizes_attribute(m, link, size): m.get('http://mock.com/', text=link) icons = favicon.get('http://mock.com/') assert icons icon = icons[0] assert icon.width == size[0] and icon.height == size[1]
def test_link_tag_empty_href_attribute(m): """'NoneType' object has no attribute 'strip' #22""" m.get('http://mock.com/', text='<link rel="icon" href="">') with pytest.warns(None): icons = favicon.get('http://mock.com/') assert not icons
def cacheDownloadAndRelinkImages(data): for index, link in enumerate(data["links"]): # parse icon name with and without extensions try: icon_url = link["iconUrl"] except KeyError as e: icon_url = None # use library to find iconUrl if none given icons = favicon.get(link["href"]) for icon in icons: # else have horde problem # gets first valid so best quality favicon if ".php?url=" not in icon.url: icon_url = icon.url break # found no valid iconUrl if not icon_url: print("Found no favicon for " + link["href"]) print(icons) continue # could also use new webp or other next gen formats # but webP not supported in safari # thanks to https://stackoverflow.com/a/27253809 icon_name_new_extension = b64encode( icon_url.encode()).decode() + '.png' webLink = '/static/' + icon_name_new_extension # create dist folder if new and construct path static_path = 'dist/static/' if not os.path.exists(static_path): os.makedirs(static_path) path = static_path + icon_name_new_extension # get and cache iconImages or relink if existing if icon_url in iconsDownloadedUrls: data["links"][index]["iconUrl"] = webLink else: response = requests.get(icon_url) if response and response.ok: print("Cached " + icon_url + "...") img = Image.open(BytesIO(response.content)) # resize to 150px width and height basewidth = 150 # would be to calculate height via aspect ratio # wpercent = (basewidth/float(img.size[0])) # hsize = int((float(img.size[1])*float(wpercent))) # img = img.resize((basewidth,hsize), Image.ANTIALIAS) img = img.resize((basewidth, basewidth), Image.ANTIALIAS) # convert to rgba for png file, save and relink img.convert('RGBA') img.save(path) data["links"][index]["iconUrl"] = webLink iconsDownloadedUrls.append(icon_url) return data
def Favicon(url): try: icons = favicon.get(url) if (icons[0][1] == 0 or icons[0][2] == 0): return 1 else: return -1 except: return 0
def check_favicon(url): icons = favicon.get(url) icon = icons[0] icondomain = urlparse(icon.url).netloc websitedomain = urlparse(url).netloc if(icondomain == websitedomain): return 1 else: return -1
def faviconCheck(url): try: icon = favicon.get(url) if icon: return 1 else: return -1 except: return -1
def get_fi_link(url): parsed_uri = urlparse(url) host = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri) icons = favicon.get(host) icon_link = "https://www.google.com/s2/favicons?domain=ukrnet.club" for icon in icons: if icon.format == 'ico': icon_link = icon.url return (icon_link)
def test_default(m, url, expected): m.get(url, text='body') m.head(expected, text='icon') m.get(expected, text='icon') icons = favicon.get(url) assert icons icon = icons[0] assert icon.url == expected
def get_data_by_url(url): try: html = urlopen(url) except: return None soup = bs4.BeautifulSoup(html, 'html.parser') text = re.sub(r'\s+', ' ', soup.get_text()) icon = favicon.get(url)[0][0] title = soup.title.text return {'text': text, 'icon': icon, 'title': title}
def check_favicon(self, dom): for proto in ['http', 'https']: try: fav = favicon.get(proto + '://' + dom + '/', stream=True, verify=False) r = requests.get(fav[0].url, verify=False) return hashlib.sha1(r.text.encode('utf-8')).hexdigest() except (ConnectionError, IndexError, HTTPError): pass
def test_default(m): m.get('http://mock.com/', text='body') m.head('http://mock.com/favicon.ico', text='icon') m.get('http://mock.com/favicon.ico', text='icon') icons = favicon.get('http://mock.com/') assert icons icon = icons[0] assert icon.url == 'http://mock.com/favicon.ico'
async def generate_embed(self, ctx, article, summary): source = self.db.get( where('Site').test(Smmry.domain_check, summary.sm_url)) with urlopen(Request(url=article, headers=Smmry.headers)) as page: soup = BeautifulSoup(page, features="lxml") icon_link = favicon.get(article)[0].url color = soup.find("meta", property="theme-color") embed = Embed(title=f"*{soup.find('title').text}*", url=summary.sm_url, description=f"{summary.sm_api_content}", colour=Colour(color['content']) if color else Colour( int(randint(0, 0xFFFFFF)))) embed.set_author( name=f"{source['Title'] if source else summary.sm_domain} ", url=f"http://www.{summary.sm_domain}", icon_url=icon_link) if source: leaning = " ".join(source['Leaning'].split("-")) bias_source = source['Link'] img = f"https://i.imgur.com/{Smmry.images.get(leaning)}.png" if leaning == "satire": embed.set_footer(text=f"{leaning.title()} | {bias_source}", icon_url=img) elif leaning == "fake-news": embed.set_footer( text= f"{leaning.title()} | Reasoning: {source['Factual']} | {bias_source}", icon_url=img) else: embed.set_footer( text= f"{leaning.title()} | Factuality: {source['Factual']} | {bias_source}", icon_url=img) embed.add_field(name="__Requests remaining__", value=f" {summary.sm_requests_remaining}/100", inline=True) embed.add_field(name="__Requested By__", value=f"{ctx.message.author.name}", inline=True) embed.add_field(name="__Reduced By__", value=f"{summary.sm_api_content_reduced}", inline=True) await ctx.send(embed=embed)
def favicon_domain(url, tldextract_output): try: icons = favicon.get(url) except: return 0 if len(icons) == 0: return 0 icon = icons[0] url_favicon = (icon.url) result_favicon = tldextract.extract(url_favicon) if result_favicon.domain == tldextract_output.domain: return 1 return -1
def Favicon(url): subDomain, domain, suffix = extract(url) b = domain try: icons = favicon.get(url) icon = icons[0] subDomain, domain, suffix = extract(icon.url) a = domain if (a == b): return 1 else: return -1 except: return -1
def get_icon_hash(url="https://fofa.so"): """计算网页favicon的hash值,如果出错返回False - 使用 favicon 库来获取网页favicon的URL - 计算hash值算法来自 https://github.com/Becivells/iconhash""" try: icons = favicon.get(url, verify=False) if len(icons) == 0: return False icon = icons[0] icon_content = requests.get(icon.url).content return mmh3.hash(codecs.lookup('base64').encode(icon_content)[0]) except Exception as e: logging.exception(e) return False
def shorten_url(request): url = request.POST.get('url', '') custom_alias = request.POST.get('custom_alias', '') max_length = request.POST.get('max_length', 6) response_data = {} if not (url == ''): start = time.time() while True: alias = custom_alias or randomstrings.alpha(max_length) try: Urls.objects.get(pk=alias) if not (custom_alias == ''): response_data['alias'] = alias response_data['ERR_CODE'] = '001' response_data[ 'description'] = 'CUSTOM ALIAS ALREADY EXISTS' return HttpResponse(json.dumps(response_data), content_type="application/json") except ObjectDoesNotExist: favicon_url = re.search( r'https?:\/\/(?:[-\w.]|(?:%[\da-fA-F]{2}))+', url).group(0) icon = favicon.get(favicon_url).pop() b = Urls(httpurl=url, alias=alias, favicon=icon.url) b.save() response_data['alias'] = alias response_data['url'] = settings.SITE_URL + "/" + alias response_data['statistics'] = { 'time_taken': int((time.time() - start) * 1000) } return HttpResponse(json.dumps(response_data), content_type="application/json") return HttpResponse(json.dumps({"error": "error occurs"}), content_type="application/json")
def write_image_file(): url = 'https://www.gallaudetathletics.com/landing/index' icons = favicon.get(url) print(icons) for n, i in enumerate(icons): response = requests.get(i.url, stream=True, headers=headers) print(response.status_code) if response.status_code != 200 or i.url.endswith('.com/') or i.url.endswith('.com') or i.url.endswith('.edu/') \ or i.url.endswith('.edu'): continue name = i.url.split('/')[-1] if name.__contains__('?'): name = name.split('?')[0] print(name) with open('junaid_new/{}_{}.{}'.format('346', '346', name), 'wb') as image: for chunk in response.iter_content(1024): image.write(chunk) break
def get_icon(url, row, index, writer, out_file): icons = favicon.get(url) print(icons) print('INDEX***', index) for n, i in enumerate(icons): response = requests.get(i.url, stream=True, headers=headers) if response.status_code != 200 or i.url.endswith('.com/') or i.url.endswith('.com') or i.url.endswith('.edu/')\ or i.url.endswith('.edu'): continue name = i.url.split('/')[-1] if name.__contains__('?'): name = name.split('?')[0] print(name) with open('fbs_fcs_missing/{}_{}.{}'.format(index, index, name), 'wb') as image: for chunk in response.iter_content(1024): image.write(chunk) row.append('{}_{}.{}'.format(index, index, name)) writer.writerow(row) out_file.flush() break
def download_favicon(url, file): icons = favicon.get(url) if len(icons) ==0: # # save the favicon, even if empty # # If empty, that will prevent from trying to download over again with open(file, "wb"): pass icon = icons[0] # por exemplo # Icon(url='https://static.publico.pt/files/site/assets/img/ico/apple-touch-icon.png?v=Km29lWbk4K', width=180, height=180, format='png') # supoonho que format é a extensao, sem o "." # print("download_favicon",url,file) # print("icon", icon) # print("fav url", icon.url) # print("format", icon.format) data = urlopen(icon.url).read() if "." + icon.format == ICON_EXTENSION: # ja esta na formato certa with open(file, "wb") as fd: fd.write(data) else: import tempfile # cria ficheiro temporaria com a extensao do icon para fazer download tmp =tempfile.mkstemp(suffix= "."+icon.format)[1] with open(tmp, "wb") as fd: fd.write(data) # o "[0]" é pq alguns .ico tem varias imagnes... e entao o convert iria adicionar os sufixos -1, -2 etc... subprocess.Popen(["convert", tmp+"[0]", file]).wait()