def init(opts=AttribDict()):
banner()
# Optional options
dataTostdout('legal disclaimer: %s\n\n' % LEGAL_DISCLAIMER, True)
if not opts.plugin:
print 'Plugin not loaded'
else:
# Get path to plugin
dir, file = os.path.split(opts.plugin)
# Load module from his filename
if not os.path.exists(os.path.join(dir, file if file.endswith('.py') else '%s.py' % file)):
errMsg = 'the plugin %s cannot be found' % opts.plugin
logger.critical(errMsg)
close()
else:
dir = os.path.abspath(dir)
sys.path.insert(0, dir)
try:
module = __import__(file[:-3])
except ImportError:
errMsg = 'the plugin %s cannot be loaded' % opts.plugin
logger.critical(errMsg)
raise
logger.info('Starting plugin %s', file)
start(module, opts)
def post(self, postdata):
logger.debug('user = %s' % self.user)
auth = base64.encodestring(self.user.decode('ascii', 'replace')).strip()
logger.debug('auth = %s' % auth)
r = urllib2.Request(url='http://twitter.com/statuses/update.xml', data='status=%s' % postdata, headers={'Authorization': 'Basic %s' % auth})
try:
h = urllib2.urlopen(r).read()
except urllib2.HTTPError as e:
if e.code == 401:
logger.err('user %s cannot be logged' % self.user.split(':')[0])
logger.debug(e.info)
else:
logger.info('tweet is posted')
def __init__(self):
threading.Thread.__init__(self)
# Variables
global pwn
pwn.current = pwn.first
self.active = pwn.first + int(self.name[-1])
# Try to get line number of file opening it
try:
file = open(pwn.file, 'r')
self.count = len(file.readlines())
file.close()
except IOError:
errMsg = 'The option is not a file, please make sure that this options is that corrected...'
logger.info(errMsg)
raise
def start(module, args):
global pwn
pwn = args
pwn.follow = 0
if pwn.first > 0:
logger.info('skipping %s lines' % pwn.first)
threads = []
if args.postdata:
plugin = module.Plugin(args.target, args.verbose)
plugin.post(args.postdata)
close()
for i in xrange(pwn.threads):
thread = ConnectionThread()
if sys.version.split()[0] >= "2.6":
thread.daemon = True
else:
thread.setDaemon(True)
thread.plugin = module.Plugin(args.target, args.verbose)
thread.start()
threads.append(thread)
# Threading
alive = True
try:
while alive:
alive = False
for thread in threads:
if thread.isAlive():
alive = True
time.sleep(0.1)
except KeyboardInterrupt:
errMsg = 'user aborted'
print ''
logger.error(errMsg)
close()
def run(self):
global pwn
# Function to get actual line to usage
getline = lambda: linecache.getline(pwn.file, self.active).strip()
while self.active <= self.count:
line = getline()
if not line and self.active >= self.count:
logger.warning('line %s is empty or something occured so wrong', self.active)
logger.debug('line = %s' % line)
close()
if self.plugin.run(line):
pwn.follow += 1
if pwn.save:
s = open(pwn.save, 'a')
s.write('%s\n' % line)
s.close()
self.active += pwn.threads
pwn.current += 1
# Display current statistics
active_threads = threading.active_count()-1
dataTostdout('\r[%s] [INFO] %s/%s lines completed. %s thread%s running.' % (time.strftime('%X'), pwn.current, self.count, active_threads,
's' if active_threads > 1 else ''), forceOutput=True)
# Show statistics only one time
if pwn.info:
if int(self.name[-1]):
self.plugin.statistics()
# Display all information from completed usage
if pwn.purge:
logger.info('Purging file...')
os.remove(pwn.file)
logger.info('Purge file completed.')
close()
def run(kwargs, cli=False):
if cli:
logger.setLevel(logging.DEBUG if kwargs['verbose'] else logging.INFO)
else:
logger.setLevel(logging.NOTSET)
logger.info('Starting Hedwig')
start = time.time()
start_date = datetime.now().isoformat()
graph = build_graph(kwargs)
logger.info('Building the knowledge base')
score_func = getattr(scorefunctions, kwargs['score'])
kb = ExperimentKB(graph, score_func, instances_as_leaves=kwargs['leaves'])
validator = Validate(kb, significance_test=significance.apply_fisher,
adjustment=getattr(adjustment, kwargs['adjust']))
rules_per_target = run_learner(kwargs, kb, validator)
rules_report = generate_rules_report(kwargs, rules_per_target)
end = time.time()
time_taken = end-start
logger.info('Finished in %d seconds' % time_taken)
logger.info('Outputing results')
if kwargs['covered']:
with open(kwargs['covered'], 'w') as f:
examples = Rule.ruleset_examples_json(rules_per_target)
f.write(json.dumps(examples, indent=2))
parameters_report = _parameters_report(kwargs, start_date, time_taken)
if kwargs['output']:
with open(kwargs['output'], 'w') as f:
f.write(parameters_report)
f.write(rules_report)
elif cli:
print parameters_report
print rules_report
return rules_per_target
def load_graph(paths, def_format='n3'):
logger.info('Calculating data checksum')
md5 = _md5_checksum(filter(lambda path: path.endswith(def_format), paths))
cached_fn = '.%s' % md5
if os.path.exists(cached_fn):
logger.info('Loading cached graph structure')
g = _load_cached_graph(cached_fn)
else:
logger.info('Building graph structure')
g = rdf(paths, def_format=def_format)
_save_graph_to_cache(g, cached_fn)
return g
def run_learner(kwargs, kb, validator):
if kb.is_discrete_target():
targets = kb.class_values if not kwargs['target'] else [kwargs['target']]
else:
targets = [None]
rules_report = ''
rules_per_target = []
for target in targets:
if target:
logger.info('Starting learner for target \'%s\'' % target)
else:
logger.info('Ranks detected - starting learner.')
learner_cls = {
'heuristic': HeuristicLearner,
'optimal': OptimalLearner
} [kwargs['learner']]
learner = learner_cls(kb,
n=kwargs['beam'],
min_sup=int(kwargs['support']*kb.n_examples()),
target=target,
depth=kwargs['depth'],
sim=0.9,
use_negations=kwargs['negations'],
optimal_subclass=kwargs['optimalsubclass'])
rules = learner.induce()
if kb.is_discrete_target():
if kwargs['adjust'] == 'fdr':
logger.info('Validating rules, FDR = %.3f' % kwargs['FDR'])
elif kwargs['adjust'] == 'fwer':
logger.info('Validating rules, alpha = %.3f' % kwargs['alpha'])
rules = validator.test(rules, alpha=kwargs['alpha'], q=kwargs['FDR'])
rules_per_target.append((target, rules))
return rules_per_target
def statistics(self):
logger.info('Completed. Statistics: %s/%s following now (%s)' % (twitterinfo.follow, twitterinfo.count, '{0:.1f}%'.format(float(twitterinfo.follow)/twitterinfo.count*100)))