def test_request_kwargs(m):
"""Add request timeout #21"""
headers = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0'
}
m.get('http://mock.com/', request_headers=headers, text='body')
favicon.get('http://mock.com/', headers=headers)
# Test deprecated header argument
with pytest.warns(DeprecationWarning):
favicon.get('http://mock.com/', headers)
def test_request_kwargs(m):
"""Add request timeout #21"""
headers = {
'User-Agent':
'Mozilla/5.0 (X11; Linux x86_64; rv:68.0) Gecko/20100101 Firefox/68.0'
}
m.get('http://mock.com/', request_headers=headers, text='body')
# raises requests_mock.exceptions.NoMockAddress if headers do not match
# https://requests-mock.readthedocs.io/en/latest/matching.html#request-headers
with pytest.warns(None):
favicon.get('http://mock.com/', headers=headers)
# Test deprecated header argument
with pytest.warns(None):
favicon.get('http://mock.com/', headers=headers)
def downloadIcon(self, url, appDir):
#Get all favicons
icons = favicon.get(url)
#Find the largest favicon
icon = icons[0]
if icon:
iconPath = os.path.join(appDir, "icon")
#Remove an existing icon if applicable
if os.path.exists(iconPath):
os.remove(iconPath)
#Download the icon and save it
response = requests.get(icon.url, stream=True)
origIconPath = iconPath + '.' + icon.format
with open(origIconPath, 'wb') as image:
for chunk in response.iter_content(1024):
image.write(chunk)
self.onUpdate(16)
#If not already an ico file, convert it
finalIconPath = iconPath + '.ico'
if icon.format != 'ico':
img = Image.open(origIconPath)
img.save(finalIconPath)
#Return the path to the saved icon
return finalIconPath
return None
def get_favicons(self):
for medium in models.Medium.objects.filter(favicon=None):
print(medium.title)
try:
icons = favicon.get('http://' + medium.uri)
except:
pass
png_icons = list(filter(lambda x: x.format == 'png', icons))
ico_icons = list(filter(lambda x: x.format == 'ico', icons))
ss_png_icons = list(
filter(lambda x: x.width == x.height and x.width > 0,
png_icons))
found = False
for img in ss_png_icons:
try:
if requests.get(img.url).status_code == 200:
medium.favicon = img.url
medium.save()
found = True
except:
pass
if found:
break
if not found and ico_icons:
print(ico_icons)
ico_response = requests.get(ico_icons[0].url)
if ico_response.status_code == 200:
medium.favicon = ico_response.url
medium.save()
def download_icon(request_uri, timeout=connection_timeout):
icons = favicon.get(request_uri)
if icons[0]:
icon = get_data(icons[0].url, timeout=timeout)
return icon
else:
return None
def download_favi_json():
with open('app.json') as json_file:
json_data = json.load(json_file)
for i in range(0, len(json_data) - 1):
uri_list.append(json_data[i]['uri'])
name_list.append(json_data[i]['app_name'])
for i in range(0, len(json_data) - 1):
icons = favicon.get(uri_list[i])
# icons(list) may not have .ico file
for j in icons:
if ".ico" in j.url:
icon = j
response = requests.get(icon.url, stream=True)
with open('icon/' + name_list[i] + '.{}'.format(icon.format),
'wb') as image:
for chunk in response.iter_content(1024):
image.write(chunk)
print("Favicon ICO Download Finished!")
return name_list
def download_favi_xlsx():
wb = load_workbook('example.xlsx')
ws = wb.active
name_list = []
uri_list = []
# r[2] field = web site Name in Excel
# r[3] field = web site URI in Excel
for r in ws.rows:
if "http" in str(r[3].value):
name_list.append(r[2].value)
uri_list.append(r[3].value)
for i in range(0, len(uri_list)):
icons = favicon.get(uri_list[i])
# icons(list) may not have .ico file
for j in icons:
if ".ico" in j.url:
icon = j
response = requests.get(icon.url, stream=True)
with open('icon/' + name_list[i] + '.{}'.format(icon.format),
'wb') as image:
for chunk in response.iter_content(1024):
image.write(chunk)
print("Favicon ICO Download Finished!")
return name_list
def add(url: str):
tags = typer.prompt("Any tags? - comma seperated list")
tags = tags.lower()
try:
icon = favicon.get(url)[0][0]
except:
icon = None
html = requests.get(url)
soup = bs(html.content, 'html.parser')
title = soup.find('title')
title = title.get_text()
db.insert({
"Title": title,
"URL": url,
"Icon": icon,
"Tags": tags.split(",")
})
typer.echo("{} saved with - Title: {} - Favicon: {}".format(
typer.style(url, fg=typer.colors.GREEN, bold=False),
typer.style(title, fg=typer.colors.GREEN, bold=True),
typer.style(icon, fg=typer.colors.GREEN, bold=True)))
def find_icons_at_url(self):
try:
return favicon.get(self.url_to_search,
allow_redirects=True,
timeout=5)
except:
return []
def create_item_icon(item):
icons = favicon.get(item.url)
icon = icons[0]
temp = NamedTemporaryFile(delete=True)
temp.write(urlopen(icon.url).read())
temp.flush()
item.icon.save(f"icon-{item.id}.png", File(temp))
item.save()
def test_invalid_meta_tag(m):
m.get(
'http://mock.com/',
text='<meta content="en-US" data-rh="true" itemprop="inLanguage"/>',
)
icons = favicon.get('http://mock.com/')
assert not icons
def test_link_tag_href_attribute(m, link, url):
m.get('http://mock.com/', text=link)
icons = favicon.get('http://mock.com/')
assert icons
icon = icons[0]
assert icon.url == url
def test_link_tag_sizes_attribute(m, link, size):
m.get('http://mock.com/', text=link)
icons = favicon.get('http://mock.com/')
assert icons
icon = icons[0]
assert icon.width == size[0] and icon.height == size[1]
def test_link_tag_empty_href_attribute(m):
"""'NoneType' object has no attribute 'strip' #22"""
m.get('http://mock.com/', text='<link rel="icon" href="">')
with pytest.warns(None):
icons = favicon.get('http://mock.com/')
assert not icons
def cacheDownloadAndRelinkImages(data):
for index, link in enumerate(data["links"]):
# parse icon name with and without extensions
try:
icon_url = link["iconUrl"]
except KeyError as e:
icon_url = None
# use library to find iconUrl if none given
icons = favicon.get(link["href"])
for icon in icons:
# else have horde problem
# gets first valid so best quality favicon
if ".php?url=" not in icon.url:
icon_url = icon.url
break
# found no valid iconUrl
if not icon_url:
print("Found no favicon for " + link["href"])
print(icons)
continue
# could also use new webp or other next gen formats
# but webP not supported in safari
# thanks to https://stackoverflow.com/a/27253809
icon_name_new_extension = b64encode(
icon_url.encode()).decode() + '.png'
webLink = '/static/' + icon_name_new_extension
# create dist folder if new and construct path
static_path = 'dist/static/'
if not os.path.exists(static_path):
os.makedirs(static_path)
path = static_path + icon_name_new_extension
# get and cache iconImages or relink if existing
if icon_url in iconsDownloadedUrls:
data["links"][index]["iconUrl"] = webLink
else:
response = requests.get(icon_url)
if response and response.ok:
print("Cached " + icon_url + "...")
img = Image.open(BytesIO(response.content))
# resize to 150px width and height
basewidth = 150
# would be to calculate height via aspect ratio
# wpercent = (basewidth/float(img.size[0]))
# hsize = int((float(img.size[1])*float(wpercent)))
# img = img.resize((basewidth,hsize), Image.ANTIALIAS)
img = img.resize((basewidth, basewidth), Image.ANTIALIAS)
# convert to rgba for png file, save and relink
img.convert('RGBA')
img.save(path)
data["links"][index]["iconUrl"] = webLink
iconsDownloadedUrls.append(icon_url)
return data
def Favicon(url):
try:
icons = favicon.get(url)
if (icons[0][1] == 0 or icons[0][2] == 0):
return 1
else:
return -1
except:
return 0
def check_favicon(url):
icons = favicon.get(url)
icon = icons[0]
icondomain = urlparse(icon.url).netloc
websitedomain = urlparse(url).netloc
if(icondomain == websitedomain):
return 1
else:
return -1
def faviconCheck(url):
try:
icon = favicon.get(url)
if icon:
return 1
else:
return -1
except:
return -1
def get_fi_link(url):
parsed_uri = urlparse(url)
host = '{uri.scheme}://{uri.netloc}/'.format(uri=parsed_uri)
icons = favicon.get(host)
icon_link = "https://www.google.com/s2/favicons?domain=ukrnet.club"
for icon in icons:
if icon.format == 'ico':
icon_link = icon.url
return (icon_link)
def test_default(m, url, expected):
m.get(url, text='body')
m.head(expected, text='icon')
m.get(expected, text='icon')
icons = favicon.get(url)
assert icons
icon = icons[0]
assert icon.url == expected
def get_data_by_url(url):
try:
html = urlopen(url)
except:
return None
soup = bs4.BeautifulSoup(html, 'html.parser')
text = re.sub(r'\s+', ' ', soup.get_text())
icon = favicon.get(url)[0][0]
title = soup.title.text
return {'text': text, 'icon': icon, 'title': title}
def check_favicon(self, dom):
for proto in ['http', 'https']:
try:
fav = favicon.get(proto + '://' + dom + '/',
stream=True,
verify=False)
r = requests.get(fav[0].url, verify=False)
return hashlib.sha1(r.text.encode('utf-8')).hexdigest()
except (ConnectionError, IndexError, HTTPError):
pass
def test_default(m):
m.get('http://mock.com/', text='body')
m.head('http://mock.com/favicon.ico', text='icon')
m.get('http://mock.com/favicon.ico', text='icon')
icons = favicon.get('http://mock.com/')
assert icons
icon = icons[0]
assert icon.url == 'http://mock.com/favicon.ico'
async def generate_embed(self, ctx, article, summary):
source = self.db.get(
where('Site').test(Smmry.domain_check, summary.sm_url))
with urlopen(Request(url=article, headers=Smmry.headers)) as page:
soup = BeautifulSoup(page, features="lxml")
icon_link = favicon.get(article)[0].url
color = soup.find("meta", property="theme-color")
embed = Embed(title=f"*{soup.find('title').text}*",
url=summary.sm_url,
description=f"{summary.sm_api_content}",
colour=Colour(color['content']) if color else Colour(
int(randint(0, 0xFFFFFF))))
embed.set_author(
name=f"{source['Title'] if source else summary.sm_domain} ",
url=f"http://www.{summary.sm_domain}",
icon_url=icon_link)
if source:
leaning = " ".join(source['Leaning'].split("-"))
bias_source = source['Link']
img = f"https://i.imgur.com/{Smmry.images.get(leaning)}.png"
if leaning == "satire":
embed.set_footer(text=f"{leaning.title()} | {bias_source}",
icon_url=img)
elif leaning == "fake-news":
embed.set_footer(
text=
f"{leaning.title()} | Reasoning: {source['Factual']} | {bias_source}",
icon_url=img)
else:
embed.set_footer(
text=
f"{leaning.title()} | Factuality: {source['Factual']} | {bias_source}",
icon_url=img)
embed.add_field(name="__Requests remaining__",
value=f" {summary.sm_requests_remaining}/100",
inline=True)
embed.add_field(name="__Requested By__",
value=f"{ctx.message.author.name}",
inline=True)
embed.add_field(name="__Reduced By__",
value=f"{summary.sm_api_content_reduced}",
inline=True)
await ctx.send(embed=embed)
def favicon_domain(url, tldextract_output):
try:
icons = favicon.get(url)
except:
return 0
if len(icons) == 0:
return 0
icon = icons[0]
url_favicon = (icon.url)
result_favicon = tldextract.extract(url_favicon)
if result_favicon.domain == tldextract_output.domain:
return 1
return -1
def Favicon(url):
subDomain, domain, suffix = extract(url)
b = domain
try:
icons = favicon.get(url)
icon = icons[0]
subDomain, domain, suffix = extract(icon.url)
a = domain
if (a == b):
return 1
else:
return -1
except:
return -1
def get_icon_hash(url="https://fofa.so"):
"""计算网页favicon的hash值，如果出错返回False
- 使用 favicon 库来获取网页favicon的URL
- 计算hash值算法来自 https://github.com/Becivells/iconhash"""
try:
icons = favicon.get(url, verify=False)
if len(icons) == 0:
return False
icon = icons[0]
icon_content = requests.get(icon.url).content
return mmh3.hash(codecs.lookup('base64').encode(icon_content)[0])
except Exception as e:
logging.exception(e)
return False
def shorten_url(request):
url = request.POST.get('url', '')
custom_alias = request.POST.get('custom_alias', '')
max_length = request.POST.get('max_length', 6)
response_data = {}
if not (url == ''):
start = time.time()
while True:
alias = custom_alias or randomstrings.alpha(max_length)
try:
Urls.objects.get(pk=alias)
if not (custom_alias == ''):
response_data['alias'] = alias
response_data['ERR_CODE'] = '001'
response_data[
'description'] = 'CUSTOM ALIAS ALREADY EXISTS'
return HttpResponse(json.dumps(response_data),
content_type="application/json")
except ObjectDoesNotExist:
favicon_url = re.search(
r'https?:\/\/(?:[-\w.]|(?:%[\da-fA-F]{2}))+', url).group(0)
icon = favicon.get(favicon_url).pop()
b = Urls(httpurl=url, alias=alias, favicon=icon.url)
b.save()
response_data['alias'] = alias
response_data['url'] = settings.SITE_URL + "/" + alias
response_data['statistics'] = {
'time_taken': int((time.time() - start) * 1000)
}
return HttpResponse(json.dumps(response_data),
content_type="application/json")
return HttpResponse(json.dumps({"error": "error occurs"}),
content_type="application/json")
def write_image_file():
url = 'https://www.gallaudetathletics.com/landing/index'
icons = favicon.get(url)
print(icons)
for n, i in enumerate(icons):
response = requests.get(i.url, stream=True, headers=headers)
print(response.status_code)
if response.status_code != 200 or i.url.endswith('.com/') or i.url.endswith('.com') or i.url.endswith('.edu/') \
or i.url.endswith('.edu'):
continue
name = i.url.split('/')[-1]
if name.__contains__('?'):
name = name.split('?')[0]
print(name)
with open('junaid_new/{}_{}.{}'.format('346', '346', name),
'wb') as image:
for chunk in response.iter_content(1024):
image.write(chunk)
break
def get_icon(url, row, index, writer, out_file):
icons = favicon.get(url)
print(icons)
print('INDEX***', index)
for n, i in enumerate(icons):
response = requests.get(i.url, stream=True, headers=headers)
if response.status_code != 200 or i.url.endswith('.com/') or i.url.endswith('.com') or i.url.endswith('.edu/')\
or i.url.endswith('.edu'):
continue
name = i.url.split('/')[-1]
if name.__contains__('?'):
name = name.split('?')[0]
print(name)
with open('fbs_fcs_missing/{}_{}.{}'.format(index, index, name),
'wb') as image:
for chunk in response.iter_content(1024):
image.write(chunk)
row.append('{}_{}.{}'.format(index, index, name))
writer.writerow(row)
out_file.flush()
break
def download_favicon(url, file):
icons = favicon.get(url)
if len(icons) ==0:
# # save the favicon, even if empty
# # If empty, that will prevent from trying to download over again
with open(file, "wb"):
pass
icon = icons[0]
# por exemplo
# Icon(url='https://static.publico.pt/files/site/assets/img/ico/apple-touch-icon.png?v=Km29lWbk4K', width=180, height=180, format='png')
# supoonho que format é a extensao, sem o "."
# print("download_favicon",url,file)
# print("icon", icon)
# print("fav url", icon.url)
# print("format", icon.format)
data = urlopen(icon.url).read()
if "." + icon.format == ICON_EXTENSION:
# ja esta na formato certa
with open(file, "wb") as fd:
fd.write(data)
else:
import tempfile
# cria ficheiro temporaria com a extensao do icon para fazer download
tmp =tempfile.mkstemp(suffix= "."+icon.format)[1]
with open(tmp, "wb") as fd:
fd.write(data)
# o "[0]" é pq alguns .ico tem varias imagnes... e entao o convert iria adicionar os sufixos -1, -2 etc...
subprocess.Popen(["convert", tmp+"[0]", file]).wait()
