def fuzz_discover_helper(url, file):
current_url = utility.S.get(url).url
links = utility.get_links_from_html(url)
params = utility.parse_url(url)
inputs = utility.get_input_fields(url)
good_urls = utility.page_guess(file, url)
print("-------------------------------------------------------------")
print("CURRENT URL: " + current_url)
print("-------------------------------------------------------------")
print("REACHABLE URLS FROM COMMON WORDS LIST: ")
for good_url in good_urls:
print(good_url)
print("-------------------------------------------------------------")
print("INPUT FIELDS: ")
print("| NAME OF FIELD | TYPE OF FIELD |")
for input_field in inputs:
if input_field.get('name') is not None and input_field.get('type') is not None:
print("| " + input_field.get('name') + " | " + input_field.get('type') + " |")
elif input_field.get('type') is not None:
print("| NO NAME | " + input_field.get('type') + " |")
else:
print("| " + input_field.get('name') + " | NO TYPE | ")
print("-------------------------------------------------------------")
print("URL PARAMETERS: ")
print(params)
print("-------------------------------------------------------------")
print("")
print("")
links = utility.filter_urls_for_offsite(url, links)
return links
def update_rss_feeds(self):
"""
RSS Feed updater function that iterates over the rss feeds in the
database and associated subscribed users to send them new articles.
"""
for feed in RSS.objects:
if not feed.meta_info.fetched:
continue
parsed = utility.parse_url(feed.rss_link)
# even though a feed check is done when the it is added to the
# database, xml files can change any time
if not self._full_feed_check(parsed):
for user_id in feed.subscribed:
user = User.get_user(user_id)
user_lang = user.settings.language
self.bot.send_message(
text=self.txt['UPD_RSS']['bad_feed'][user_lang].format(
feed.rss_link, html.escape(feed.title)),
chat_id=user_id,
parse_mode='HTML')
feed.delete()
return
new_entries = self._get_new_entries(db_feed=feed, rss_parsed=parsed)
for user_id in feed.subscribed:
for entry in new_entries:
exists = self._send_rss_message(
self.txt['UPD_RSS']['formatted'].format(
html.escape(entry.title), entry.link), user_id)
if exists:
continue
if feed.check_subscription(user_id):
feed.subscribed.remove(user_id)
feed.save()
if not feed.check_subscribed():
feed.meta_info.fetched = False
feed.save()
break
def conn_from_env():
ec2_access = os.getenv("EC2_ACCESS_KEY")
ec2_secret = os.getenv("EC2_SECRET_KEY")
ec2_url = os.getenv("EC2_URL")
ec2_host, ec2_port, ec2_path = utility.parse_url(ec2_url)
region = boto.ec2.regioninfo.RegionInfo()
region.endpoint = ec2_host
conn = boto.ec2.connection.EC2Connection(
aws_access_key_id=ec2_access,
aws_secret_access_key=ec2_secret,
port=ec2_port,
path=ec2_path,
is_secure=False,
region=region
)
return conn
import boto
import boto.s3
import boto.s3.connection
import os,sys,re
import utility
##
## A simple demo of the Boto S3 interface.
## connect to futuregrid or OpenStack using the environment variables
## and then list the S3 buckets
##
EC2_ACCESS = os.getenv('EC2_ACCESS_KEY')
EC2_SECRET = os.getenv('EC2_SECRET_KEY')
S3_URL = utility.parse_url(os.getenv('S3_URL'))
EC2_URL = utility.parse_url(os.getenv('EC2_URL'))
calling_format=boto.s3.connection.OrdinaryCallingFormat()
connection = boto.s3.connection.S3Connection(
aws_access_key_id=EC2_ACCESS,
aws_secret_access_key=EC2_SECRET,
is_secure=False,
host=S3_URL[0],
port=S3_URL[1],
calling_format=calling_format,
path=S3_URL[2])
print "Connection is ", connection
def rss_compile(update, context, user, link) -> str:
"""
Handler: fsm:2.1 -> 3
:param user: mongoengine User object
:param link: the extracted entity from the message
"""
news = utility.parse_url(link)
language = user.settings.language
state = user.settings.fsm_state
# check the source for possible errors, such as bozo and format
if not utility.check_source(news):
return txt['CALLBACK']['error_link'][language]
# check the actual feed, i.e.:
# stuff like title, subtitle, link and such.
checked_feed = utility.check_parsed(
news.feed, config['SCRAPE']['RSS']['req_feed_keys'])
# implement the checking described above
if not checked_feed:
return txt['CALLBACK']['error_feed'][language]
# all entries must be checked for certain required elements
# this must strike a fine balance between universality and enough
# information for a good display of the RSS feed
checked_all_entries = all([
utility.check_parsed(x, config['SCRAPE']['RSS']['req_entry_keys'])
for x in news.entries
])
# implement the checking above
if not checked_all_entries:
return txt['CALLBACK']['error_entries'][language]
# if all the checks have so far been passed, then we create the RSS
# feed in our database and register it - unless it already exists for the
# user.
try:
db_news = RSS(
rss_link=link,
link=news.feed.link,
title=news.feed.title,
subtitle=news.feed.get('subtitle', ''),
summary=news.feed.get('summary', ''),
)
db_news.subscribed.append(user.user_id)
db_news.save()
# if an identical RSS feed exists instead of saving, we fetch the existing
except errors.NotUniqueError:
db_news = RSS.get_rss(rss_link=link)
if user.user_id in db_news.subscribed:
return txt['CALLBACK']['repeated_rss'][language]
db_news.subscribed.append(user.user_id)
db_news.meta_info.fetched = True
db_news.save()
user.subscribed.rss_list.append(db_news.pk)
user.subscribed.session_list.append(news.feed.title)
user.save()
feed_formatted = f"<a href=\"{db_news.link}\">" + \
f"{utility.escape(db_news.title)}</a>"
if not len(news.entries) > 0:
return txt['CALLBACK']['empty_feed'][language]
db_news.last_entry_link = news.entries[0].link
db_news.save()
# because this function is used both in the setup and post-setup, we assign
# a special 'b' sub-state that the user never takes on, but that contains
# the buttons required to move on to the next FSM state.
return txt['FSM'][f'{state}b']['text'][language].format(feed_formatted)