Esempio n. 1
0
    def __init__(self, PluginClass=None):
        """
            @param PluginClass: as returned by handler.list('controller'). Must
                extend BasePlugin.
        """
        plugin = PluginClass()
        if plugin:

            self.name = plugin._meta.label

            if plugin.can_enumerate_plugins:
                self.plugins_can_enumerate = True
                self.plugins_wordlist_size = file_len(plugin.plugins_file)

            if plugin.can_enumerate_themes:
                self.themes_can_enumerate = True
                self.themes_wordlist_size = file_len(plugin.themes_file)

            if plugin.can_enumerate_interesting:
                self.interesting_can_enumerate = True
                self.interesting_url_size = len(plugin.interesting_urls)

            if plugin.can_enumerate_version:
                versions_file = VersionsFile(plugin.versions_file)

                self.version_can_enumerate = True
                hvm = versions_file.highest_version_major(plugin.update_majors)
                self.version_highest = ', '.join(hvm.values())
def main(args):
    total_followers = common.file_len(args.followers_file)
    with open(args.followers_file, 'r') as followers_file:
        with open(args.tweets_file, 'wb') as tweets_file:
            # get the tweets of the broadcaster
            tweets = common.get_timeline_in_range(args.handle, args.start_date, args.end_date, 3)
            # write the tweets of the broadcaster
            print "Processing broadcaster " + args.handle
            for tweet in tweets:
                timestamp = str(datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
                # format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
                tweets_file.write(timestamp + "," + args.handle + "," + str(-1) + "\n")

            i = 1
            for handle in followers_file:
                handle = handle.rstrip('\n')
                print "Processing follower " + handle + "... (" + str(i) + "/" + str(total_followers) + ")"
                # get the tweets of their followers
                tweets = common.get_timeline_in_range(handle, args.start_date, args.end_date, 3)
                # write the tweets of their followers
                for tweet in tweets:
                    timestamp = str(datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
                    # format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
                    tweets_file.write(timestamp + "," + handle + "," + (args.handle if args.handle in tweet['text'] else str(-1)) + "\n")
                i += 1
Esempio n. 3
0
def main(args):
    # this value is used to provide visual feedback on how the code is proceding, but isn't strictly necessary
    total_followers = common.file_len(args.users_file)
    a = common.UID_Assigner()
    with open(args.users_file, 'r') as users_file:
        with open(args.tweets_file, 'wb') as tweets_file:
            i = 1
            for handle in users_file:
                handle = handle.rstrip('\n')
                print "Processing user " + handle + "... (" + str(
                    i) + "/" + str(total_followers) + ")"
                # get the tweets of a user
                tweets = common.get_timeline_in_range(handle, args.start_date,
                                                      args.end_date, 3)
                # write down the tweets of a user
                for tweet in tweets:
                    timestamp = str(
                        datetime.strptime(
                            tweet['created_at'],
                            "%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
                    # we define a retweet as being both if a user directly retweets another tweet, and if they are merely replying to a previous tweet (that is, it merely shows cross-node interaction)
                    retweeted = (
                        tweet['retweeted_status']['user']['screen_name']
                        if 'retweeted_status' in tweet else
                        (tweet['in_reply_to_screen_name']
                         if 'in_reply_to_screen_name' in tweet else ('-1')))
                    # format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
                    tweets_file.write(timestamp + "," + handle + "," +
                                      str(retweeted) + "\n")
                i += 1
Esempio n. 4
0
    def enumerate_themes(self, url, base_url, scanning_method='forbidden',
            max_plugins=500, threads=10, verb='head', timeout=15,
            hide_progressbar=False, imu=None):

        iterator = self.themes_get
        iterator_len = file_len(self.themes_file)

        return self.enumerate(url, base_url, scanning_method, iterator,
                iterator_len, max_plugins, threads, verb, timeout,
                hide_progressbar, imu)
    def test_gets_modules(self):
        # unwrap the generator
        plugins_generator = self.scanner.plugins_get()
        plugins = []
        for plugin in plugins_generator:
            plugins.append(plugin)

        l = file_len(self.scanner.plugins_file)

        assert l == len(plugins), "Should have read the contents of the file."
Esempio n. 6
0
    def test_gets_modules(self):
        # unwrap the generator
        plugins_generator = self.scanner.plugins_get()
        plugins = []
        for plugin in plugins_generator:
            plugins.append(plugin)

        l = file_len(self.scanner.plugins_file)

        assert l == len(plugins), "Should have read the contents of the file."
    def enumerate_themes(self, url, base_url, scanning_method='forbidden',
            max_plugins=500, threads=10, verb='head', timeout=15,
            hide_progressbar=False, imu=None):

        iterator = self.themes_get
        iterator_len = file_len(self.themes_file)

        return self.enumerate(url, base_url, scanning_method, iterator,
                iterator_len, max_plugins, threads, verb, timeout,
                hide_progressbar, imu)
Esempio n. 8
0
def import_ccrs_file(db, ccrs_bed):
    input_file = open(ccrs_bed, 'rt')
    reader = csv.reader(input_file, delimiter='\t')
    headers = next(reader)

    chrom_index = headers.index('#chrom')
    start_index = headers.index('start')
    end_index = headers.index('end')
    ccr_pct_index = headers.index('ccr_pct')
    gene_index = headers.index('gene')
    ranges_index = headers.index('ranges')
    varflag_index = headers.index('varflag')
    syn_density_index = headers.index('syn_density')
    cpg_index = headers.index('cpg')
    cov_score_index = headers.index('cov_score')
    resid_index = headers.index('resid')
    resid_pctile_index = headers.index('resid_pctile')
    unique_key_index = headers.index('unique_key')

    bulk = db.gevir.ccrs.initialize_unordered_bulk_op()
    total_lines = file_len(ccrs_bed)
    line_number = 0
    bar = progressbar.ProgressBar(maxval=1.0).start()
    for row in reader:
        ccr = CCR()
        ccr.chrom = row[chrom_index]
        ccr.start = int(row[start_index])
        ccr.end = int(row[end_index])
        ccr.ccr_pct = float(row[ccr_pct_index])
        ccr.gene = row[gene_index]
        ccr.ranges = row[ranges_index]
        ccr.varflag = row[varflag_index]
        ccr.syn_density = float(row[syn_density_index])
        ccr.cpg = float(row[cpg_index])
        ccr.cov_score = float(row[cov_score_index])
        ccr.resid = float(row[resid_index])
        ccr.resid_pctile = float(row[resid_pctile_index])
        ccr.unique_key = int(row[unique_key_index])
        ccr = ccr.get_dictionary()

        bulk.insert(ccr)
        if (line_number % 10000 == 0):
            bulk.execute()
            bulk = db.gevir.ccrs.initialize_unordered_bulk_op()

        line_number += 1
        bar.update((line_number + 0.0) / total_lines)
    bar.finish()

    if (line_number % 10000 != 0):
        bulk.execute()
Esempio n. 9
0
    def __init__(self, plugin=None):
        """
            @param plugin as returned by handler.list('controller'). Must
                extend BasePlugin.
        """
        if plugin:

            self.name = plugin.__name__

            if plugin.can_enumerate_plugins:
                self.plugins_can_enumerate = True
                self.plugins_wordlist_size = file_len(plugin.plugins_file)
                # this can fail due to git not being installed on the system or
                # this not being a git repository.
                try:
                    self.plugins_mtime = self.file_mtime(plugin.plugins_file)
                except:
                    pass

            if plugin.can_enumerate_themes:
                self.themes_can_enumerate = True
                self.themes_wordlist_size = file_len(plugin.themes_file)
                try:
                    self.themes_mtime = self.file_mtime(plugin.themes_file)
                except:
                    pass

            if plugin.can_enumerate_interesting:
                self.interesting_can_enumerate = True
                self.interesting_url_size = len(plugin.interesting_urls)

            if plugin.can_enumerate_version:
                versions_file = VersionsFile(plugin.versions_file)

                self.version_can_enumerate = True
                self.version_highest = versions_file.highest_version()
def main(args):
    # this value is used to provide visual feedback on how the code is proceding, but isn't strictly necessary
    total_followers = common.file_len(args.users_file)
    a = common.UID_Assigner()
    with open(args.users_file, 'r') as users_file:
        with open(args.tweets_file, 'wb') as tweets_file:
            i = 1
            for handle in users_file:
                handle = handle.rstrip('\n')
                print "Processing user " + handle + "... (" + str(i) + "/" + str(total_followers) + ")"
                # get the tweets of a user
                tweets = common.get_timeline_in_range(handle, args.start_date, args.end_date, 3)
                # write down the tweets of a user
                for tweet in tweets:
                    timestamp = str(datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
                    # we define a retweet as being both if a user directly retweets another tweet, and if they are merely replying to a previous tweet (that is, it merely shows cross-node interaction)
                    retweeted = (tweet['retweeted_status']['user']['screen_name'] if 'retweeted_status' in tweet else (tweet['in_reply_to_screen_name'] if 'in_reply_to_screen_name' in tweet else ('-1')))
                    # format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
                    tweets_file.write(timestamp + "," + handle + "," + str(retweeted) + "\n")
                i += 1
Esempio n. 11
0
def learn(bot, update, args):
    message = ''
    if len(args) < 1:
        message = "Forgot to add which language to learn \nAvailable languages with message total:\n%s" % form_list(
                Dbhelper.get_languages_message_count())
    else:
        num_of_messages = file_len(args[0])
        try:
            if num_of_messages < 3000:
                raise ValueError
            ml_obj = min_char(args[0], args[1])
            message = ml_obj.learn()
        except ValueError:
            message = 'At least 3000 messages are needed, %s chat has %d messages.' % (args[0], num_of_messages)
        except IndexError:
            ml_obj = min_char(args[0], 5000)
            message = ml_obj.learn()
        except IOError:
            message = 'File for the language %s does not exist' % args[0]
    bot.sendMessage(chat_id=update.message.chat_id, text=message)
def main(args):
    total_followers = common.file_len(args.followers_file)
    with open(args.followers_file, 'r') as followers_file:
        with open(args.tweets_file, 'wb') as tweets_file:
            # get the tweets of the broadcaster
            tweets = common.get_timeline_in_range(args.handle, args.start_date,
                                                  args.end_date, 3)
            # write the tweets of the broadcaster
            print "Processing broadcaster " + args.handle
            for tweet in tweets:
                timestamp = str(
                    datetime.strptime(
                        tweet['created_at'],
                        "%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
                # format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
                tweets_file.write(timestamp + "," + args.handle + "," +
                                  str(-1) + "\n")

            i = 1
            for handle in followers_file:
                handle = handle.rstrip('\n')
                print "Processing follower " + handle + "... (" + str(
                    i) + "/" + str(total_followers) + ")"
                # get the tweets of their followers
                tweets = common.get_timeline_in_range(handle, args.start_date,
                                                      args.end_date, 3)
                # write the tweets of their followers
                for tweet in tweets:
                    timestamp = str(
                        datetime.strptime(
                            tweet['created_at'],
                            "%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
                    # format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
                    tweets_file.write(timestamp + "," + handle + "," +
                                      (args.handle if args.handle in
                                       tweet['text'] else str(-1)) + "\n")
                i += 1
Esempio n. 13
0
 def test_file_len_empty_file(self):
     m = mock_open()
     with patch('common.functions.open', m, create=True) as o:
         ln = file_len("test")
         print(o.call_args_list)
         assert ln == 0