def __init__(self, PluginClass=None):
"""
@param PluginClass: as returned by handler.list('controller'). Must
extend BasePlugin.
"""
plugin = PluginClass()
if plugin:
self.name = plugin._meta.label
if plugin.can_enumerate_plugins:
self.plugins_can_enumerate = True
self.plugins_wordlist_size = file_len(plugin.plugins_file)
if plugin.can_enumerate_themes:
self.themes_can_enumerate = True
self.themes_wordlist_size = file_len(plugin.themes_file)
if plugin.can_enumerate_interesting:
self.interesting_can_enumerate = True
self.interesting_url_size = len(plugin.interesting_urls)
if plugin.can_enumerate_version:
versions_file = VersionsFile(plugin.versions_file)
self.version_can_enumerate = True
hvm = versions_file.highest_version_major(plugin.update_majors)
self.version_highest = ', '.join(hvm.values())
def main(args):
total_followers = common.file_len(args.followers_file)
with open(args.followers_file, 'r') as followers_file:
with open(args.tweets_file, 'wb') as tweets_file:
# get the tweets of the broadcaster
tweets = common.get_timeline_in_range(args.handle, args.start_date, args.end_date, 3)
# write the tweets of the broadcaster
print "Processing broadcaster " + args.handle
for tweet in tweets:
timestamp = str(datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
# format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
tweets_file.write(timestamp + "," + args.handle + "," + str(-1) + "\n")
i = 1
for handle in followers_file:
handle = handle.rstrip('\n')
print "Processing follower " + handle + "... (" + str(i) + "/" + str(total_followers) + ")"
# get the tweets of their followers
tweets = common.get_timeline_in_range(handle, args.start_date, args.end_date, 3)
# write the tweets of their followers
for tweet in tweets:
timestamp = str(datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
# format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
tweets_file.write(timestamp + "," + handle + "," + (args.handle if args.handle in tweet['text'] else str(-1)) + "\n")
i += 1
def main(args):
# this value is used to provide visual feedback on how the code is proceding, but isn't strictly necessary
total_followers = common.file_len(args.users_file)
a = common.UID_Assigner()
with open(args.users_file, 'r') as users_file:
with open(args.tweets_file, 'wb') as tweets_file:
i = 1
for handle in users_file:
handle = handle.rstrip('\n')
print "Processing user " + handle + "... (" + str(
i) + "/" + str(total_followers) + ")"
# get the tweets of a user
tweets = common.get_timeline_in_range(handle, args.start_date,
args.end_date, 3)
# write down the tweets of a user
for tweet in tweets:
timestamp = str(
datetime.strptime(
tweet['created_at'],
"%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
# we define a retweet as being both if a user directly retweets another tweet, and if they are merely replying to a previous tweet (that is, it merely shows cross-node interaction)
retweeted = (
tweet['retweeted_status']['user']['screen_name']
if 'retweeted_status' in tweet else
(tweet['in_reply_to_screen_name']
if 'in_reply_to_screen_name' in tweet else ('-1')))
# format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
tweets_file.write(timestamp + "," + handle + "," +
str(retweeted) + "\n")
i += 1
def enumerate_themes(self, url, base_url, scanning_method='forbidden',
max_plugins=500, threads=10, verb='head', timeout=15,
hide_progressbar=False, imu=None):
iterator = self.themes_get
iterator_len = file_len(self.themes_file)
return self.enumerate(url, base_url, scanning_method, iterator,
iterator_len, max_plugins, threads, verb, timeout,
hide_progressbar, imu)
def test_gets_modules(self):
# unwrap the generator
plugins_generator = self.scanner.plugins_get()
plugins = []
for plugin in plugins_generator:
plugins.append(plugin)
l = file_len(self.scanner.plugins_file)
assert l == len(plugins), "Should have read the contents of the file."
def test_gets_modules(self):
# unwrap the generator
plugins_generator = self.scanner.plugins_get()
plugins = []
for plugin in plugins_generator:
plugins.append(plugin)
l = file_len(self.scanner.plugins_file)
assert l == len(plugins), "Should have read the contents of the file."
def enumerate_themes(self, url, base_url, scanning_method='forbidden',
max_plugins=500, threads=10, verb='head', timeout=15,
hide_progressbar=False, imu=None):
iterator = self.themes_get
iterator_len = file_len(self.themes_file)
return self.enumerate(url, base_url, scanning_method, iterator,
iterator_len, max_plugins, threads, verb, timeout,
hide_progressbar, imu)
def import_ccrs_file(db, ccrs_bed):
input_file = open(ccrs_bed, 'rt')
reader = csv.reader(input_file, delimiter='\t')
headers = next(reader)
chrom_index = headers.index('#chrom')
start_index = headers.index('start')
end_index = headers.index('end')
ccr_pct_index = headers.index('ccr_pct')
gene_index = headers.index('gene')
ranges_index = headers.index('ranges')
varflag_index = headers.index('varflag')
syn_density_index = headers.index('syn_density')
cpg_index = headers.index('cpg')
cov_score_index = headers.index('cov_score')
resid_index = headers.index('resid')
resid_pctile_index = headers.index('resid_pctile')
unique_key_index = headers.index('unique_key')
bulk = db.gevir.ccrs.initialize_unordered_bulk_op()
total_lines = file_len(ccrs_bed)
line_number = 0
bar = progressbar.ProgressBar(maxval=1.0).start()
for row in reader:
ccr = CCR()
ccr.chrom = row[chrom_index]
ccr.start = int(row[start_index])
ccr.end = int(row[end_index])
ccr.ccr_pct = float(row[ccr_pct_index])
ccr.gene = row[gene_index]
ccr.ranges = row[ranges_index]
ccr.varflag = row[varflag_index]
ccr.syn_density = float(row[syn_density_index])
ccr.cpg = float(row[cpg_index])
ccr.cov_score = float(row[cov_score_index])
ccr.resid = float(row[resid_index])
ccr.resid_pctile = float(row[resid_pctile_index])
ccr.unique_key = int(row[unique_key_index])
ccr = ccr.get_dictionary()
bulk.insert(ccr)
if (line_number % 10000 == 0):
bulk.execute()
bulk = db.gevir.ccrs.initialize_unordered_bulk_op()
line_number += 1
bar.update((line_number + 0.0) / total_lines)
bar.finish()
if (line_number % 10000 != 0):
bulk.execute()
def __init__(self, plugin=None):
"""
@param plugin as returned by handler.list('controller'). Must
extend BasePlugin.
"""
if plugin:
self.name = plugin.__name__
if plugin.can_enumerate_plugins:
self.plugins_can_enumerate = True
self.plugins_wordlist_size = file_len(plugin.plugins_file)
# this can fail due to git not being installed on the system or
# this not being a git repository.
try:
self.plugins_mtime = self.file_mtime(plugin.plugins_file)
except:
pass
if plugin.can_enumerate_themes:
self.themes_can_enumerate = True
self.themes_wordlist_size = file_len(plugin.themes_file)
try:
self.themes_mtime = self.file_mtime(plugin.themes_file)
except:
pass
if plugin.can_enumerate_interesting:
self.interesting_can_enumerate = True
self.interesting_url_size = len(plugin.interesting_urls)
if plugin.can_enumerate_version:
versions_file = VersionsFile(plugin.versions_file)
self.version_can_enumerate = True
self.version_highest = versions_file.highest_version()
def main(args):
# this value is used to provide visual feedback on how the code is proceding, but isn't strictly necessary
total_followers = common.file_len(args.users_file)
a = common.UID_Assigner()
with open(args.users_file, 'r') as users_file:
with open(args.tweets_file, 'wb') as tweets_file:
i = 1
for handle in users_file:
handle = handle.rstrip('\n')
print "Processing user " + handle + "... (" + str(i) + "/" + str(total_followers) + ")"
# get the tweets of a user
tweets = common.get_timeline_in_range(handle, args.start_date, args.end_date, 3)
# write down the tweets of a user
for tweet in tweets:
timestamp = str(datetime.strptime(tweet['created_at'], "%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
# we define a retweet as being both if a user directly retweets another tweet, and if they are merely replying to a previous tweet (that is, it merely shows cross-node interaction)
retweeted = (tweet['retweeted_status']['user']['screen_name'] if 'retweeted_status' in tweet else (tweet['in_reply_to_screen_name'] if 'in_reply_to_screen_name' in tweet else ('-1')))
# format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
tweets_file.write(timestamp + "," + handle + "," + str(retweeted) + "\n")
i += 1
def learn(bot, update, args):
message = ''
if len(args) < 1:
message = "Forgot to add which language to learn \nAvailable languages with message total:\n%s" % form_list(
Dbhelper.get_languages_message_count())
else:
num_of_messages = file_len(args[0])
try:
if num_of_messages < 3000:
raise ValueError
ml_obj = min_char(args[0], args[1])
message = ml_obj.learn()
except ValueError:
message = 'At least 3000 messages are needed, %s chat has %d messages.' % (args[0], num_of_messages)
except IndexError:
ml_obj = min_char(args[0], 5000)
message = ml_obj.learn()
except IOError:
message = 'File for the language %s does not exist' % args[0]
bot.sendMessage(chat_id=update.message.chat_id, text=message)
def main(args):
total_followers = common.file_len(args.followers_file)
with open(args.followers_file, 'r') as followers_file:
with open(args.tweets_file, 'wb') as tweets_file:
# get the tweets of the broadcaster
tweets = common.get_timeline_in_range(args.handle, args.start_date,
args.end_date, 3)
# write the tweets of the broadcaster
print "Processing broadcaster " + args.handle
for tweet in tweets:
timestamp = str(
datetime.strptime(
tweet['created_at'],
"%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
# format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
tweets_file.write(timestamp + "," + args.handle + "," +
str(-1) + "\n")
i = 1
for handle in followers_file:
handle = handle.rstrip('\n')
print "Processing follower " + handle + "... (" + str(
i) + "/" + str(total_followers) + ")"
# get the tweets of their followers
tweets = common.get_timeline_in_range(handle, args.start_date,
args.end_date, 3)
# write the tweets of their followers
for tweet in tweets:
timestamp = str(
datetime.strptime(
tweet['created_at'],
"%a %b %d %H:%M:%S +0000 %Y").strftime('%s'))
# format: timestamp, their username, username of who they're retweeting (-1 if not a retweet)
tweets_file.write(timestamp + "," + handle + "," +
(args.handle if args.handle in
tweet['text'] else str(-1)) + "\n")
i += 1
def test_file_len_empty_file(self):
m = mock_open()
with patch('common.functions.open', m, create=True) as o:
ln = file_len("test")
print(o.call_args_list)
assert ln == 0