예제 #1
0
def fuzz_discover_helper(url, file):

    current_url = utility.S.get(url).url
    links = utility.get_links_from_html(url)
    params = utility.parse_url(url)
    inputs = utility.get_input_fields(url)
    good_urls = utility.page_guess(file, url)

    print("-------------------------------------------------------------")
    print("CURRENT URL: " + current_url)
    print("-------------------------------------------------------------")
    print("REACHABLE URLS FROM COMMON WORDS LIST: ")
    for good_url in good_urls:
        print(good_url)
    print("-------------------------------------------------------------")
    print("INPUT FIELDS: ")
    print("| NAME OF FIELD | TYPE OF FIELD |")
    for input_field in inputs:
        if input_field.get('name') is not None and input_field.get('type') is not None:
            print("| " + input_field.get('name') + " | " + input_field.get('type') + " |")
        elif input_field.get('type') is not None:
            print("| NO NAME | " + input_field.get('type') + " |")
        else:
            print("| " + input_field.get('name') + " | NO TYPE | ")
    print("-------------------------------------------------------------")
    print("URL PARAMETERS: ")
    print(params)
    print("-------------------------------------------------------------")
    print("")
    print("")

    links = utility.filter_urls_for_offsite(url, links)

    return links
예제 #2
0
def update_rss_feeds(self):
    """
    RSS Feed updater function that iterates over the rss feeds in the
    database and associated subscribed users to send them new articles.
    """
    for feed in RSS.objects:
        if not feed.meta_info.fetched:
            continue

        parsed = utility.parse_url(feed.rss_link)

        # even though a feed check is done when the it is added to the
        # database, xml files can change any time
        if not self._full_feed_check(parsed):
            for user_id in feed.subscribed:
                user = User.get_user(user_id)
                user_lang = user.settings.language

                self.bot.send_message(
                    text=self.txt['UPD_RSS']['bad_feed'][user_lang].format(
                        feed.rss_link, html.escape(feed.title)),
                    chat_id=user_id,
                    parse_mode='HTML')

            feed.delete()
            return

        new_entries = self._get_new_entries(db_feed=feed, rss_parsed=parsed)

        for user_id in feed.subscribed:
            for entry in new_entries:
                exists = self._send_rss_message(
                    self.txt['UPD_RSS']['formatted'].format(
                        html.escape(entry.title), entry.link), user_id)

                if exists:
                    continue
                if feed.check_subscription(user_id):
                    feed.subscribed.remove(user_id)
                    feed.save()
                if not feed.check_subscribed():
                    feed.meta_info.fetched = False
                    feed.save()
                break
예제 #3
0
def conn_from_env():

    ec2_access = os.getenv("EC2_ACCESS_KEY")
    ec2_secret = os.getenv("EC2_SECRET_KEY")
    ec2_url = os.getenv("EC2_URL")
    
    ec2_host, ec2_port, ec2_path = utility.parse_url(ec2_url)
    
    region = boto.ec2.regioninfo.RegionInfo()
    region.endpoint = ec2_host
    
    conn = boto.ec2.connection.EC2Connection(
        aws_access_key_id=ec2_access,
        aws_secret_access_key=ec2_secret,
        port=ec2_port,
        path=ec2_path,
        is_secure=False,
        region=region
        )
    
    return conn
예제 #4
0
import boto
import boto.s3
import boto.s3.connection
import os,sys,re

import utility

##
## A simple demo of the Boto S3 interface.
## connect to futuregrid or OpenStack using the environment variables
## and then list the S3 buckets
##

EC2_ACCESS  = os.getenv('EC2_ACCESS_KEY')
EC2_SECRET  = os.getenv('EC2_SECRET_KEY')
S3_URL = utility.parse_url(os.getenv('S3_URL'))
EC2_URL = utility.parse_url(os.getenv('EC2_URL'))

calling_format=boto.s3.connection.OrdinaryCallingFormat()

connection = boto.s3.connection.S3Connection(
    aws_access_key_id=EC2_ACCESS,
    aws_secret_access_key=EC2_SECRET,
    is_secure=False,
    host=S3_URL[0],
    port=S3_URL[1],
    calling_format=calling_format,
    path=S3_URL[2])

print "Connection is ", connection
예제 #5
0
def rss_compile(update, context, user, link) -> str:
    """
    Handler: fsm:2.1 -> 3

    :param user: mongoengine User object
    :param link: the extracted entity from the message
    """
    news = utility.parse_url(link)
    language = user.settings.language
    state = user.settings.fsm_state

    # check the source for possible errors, such as bozo and format
    if not utility.check_source(news):
        return txt['CALLBACK']['error_link'][language]

    # check the actual feed, i.e.:
    # stuff like title, subtitle, link and such.
    checked_feed = utility.check_parsed(
        news.feed, config['SCRAPE']['RSS']['req_feed_keys'])

    # implement the checking described above
    if not checked_feed:
        return txt['CALLBACK']['error_feed'][language]

    # all entries must be checked for certain required elements
    # this must strike a fine balance between universality and enough
    # information for a good display of the RSS feed
    checked_all_entries = all([
        utility.check_parsed(x, config['SCRAPE']['RSS']['req_entry_keys'])
        for x in news.entries
    ])

    # implement the checking above
    if not checked_all_entries:
        return txt['CALLBACK']['error_entries'][language]

    # if all the checks have so far been passed, then we create the RSS
    # feed in our database and register it - unless it already exists for the
    # user.
    try:
        db_news = RSS(
            rss_link=link,
            link=news.feed.link,
            title=news.feed.title,
            subtitle=news.feed.get('subtitle', ''),
            summary=news.feed.get('summary', ''),
        )
        db_news.subscribed.append(user.user_id)
        db_news.save()

    # if an identical RSS feed exists instead of saving, we fetch the existing
    except errors.NotUniqueError:
        db_news = RSS.get_rss(rss_link=link)

        if user.user_id in db_news.subscribed:
            return txt['CALLBACK']['repeated_rss'][language]

        db_news.subscribed.append(user.user_id)
        db_news.meta_info.fetched = True
        db_news.save()

    user.subscribed.rss_list.append(db_news.pk)
    user.subscribed.session_list.append(news.feed.title)
    user.save()

    feed_formatted = f"<a href=\"{db_news.link}\">" + \
        f"{utility.escape(db_news.title)}</a>"

    if not len(news.entries) > 0:
        return txt['CALLBACK']['empty_feed'][language]

    db_news.last_entry_link = news.entries[0].link
    db_news.save()

    # because this function is used both in the setup and post-setup, we assign
    # a special 'b' sub-state that the user never takes on, but that contains
    # the buttons required to move on to the next FSM state.
    return txt['FSM'][f'{state}b']['text'][language].format(feed_formatted)