def getPosts(self, uri, min=None, max=None, text=None):
"""
Get mailing list's posts
@param uri: mailing list's uri
@param min: min date
@param max: max date
@param text: text to search
"""
if (self.cache == None):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
else:
if (uri != self.cache.uri or self.cache.bad):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
min, max = self.getDates()
if (not self.cache.bad):
posts = self.cache.query()
if (posts == None):
self.messageBar('unknow problem parsing RDF at ' + self.uri)
return None
else:
if (min != None or max != None or text != None):
posts = self.cache.filterPosts(posts, min, max, text)
return posts
else:
self.alert('An exception ocurred parsing this URI')
return None
def getPosts(self, uri, min=None, max=None, text=None):
"""
Get mailing list's posts
@param uri: mailing list's uri
@param min: min date
@param max: max date
@param text: text to search
"""
if (self.cache == None):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
else:
if (uri!=self.cache.uri or self.cache.bad):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
min, max = self.getDates()
if (not self.cache.bad):
posts = self.cache.query()
if (posts == None):
self.messageBar('unknow problem parsing RDF at ' + self.uri)
return None
else:
if (min!=None or max!=None or text!=None):
posts = self.cache.filterPosts(posts, min, max, text)
return posts
else:
self.alert('An exception ocurred parsing this URI')
return None
def run(schedule_name, jobs, do_fixes=False):
heuristics = get_job_heuristics()
if len(jobs) == 0:
return
print("^ {} running {} jobs".format(time.strftime("%m/%d/%Y %H:%M:%S"),
len(jobs)))
return_string = ""
# def run_module_job(name, module):
# start_time = time.time()
# return_string = ''
# return_string += "\n\n################\n"
# return_string += name
# if hasattr(module, 'problem_data'):
# job = getattr(module, 'problem_data')
# else:
# job = getattr(module, 'main')
# try:
# list(sessions.values())[0].console.new_line('Running {}'.format(name))
# if job.__name__ == 'main':
# result = str(job(sessions))
# elif job.__name__ == 'problem_data':
# result = run_job(name, module.problem_data, module.fix, module.sessions, do_fix=False, download_result=False, sf_sessions=sessions, sql_sessions=None)
# if len(result) > 0:
# return_string += "\n"
# return_string += result
# else:
# return_string = ''
# except Exception as e:
# return_string += "\n>>>>> Job failed: {}\n{}\n".format(name,str(e))
# duration = math.floor(time.time() - start_time)
# return name, return_string, duration
threads = []
# sorted_jobs =
for name, module in jobs.items():
threads.append(
threading.new(run_module_job,
sessions,
name,
module,
do_fix=do_fixes))
for thread in threads:
job_name, result_string, duration = thread.result()
return_string += '\n' + result_string
heur = heuristics.get(
job_name, {
'name': job_name,
'num_runs': 0,
'avg_duration': 0,
'last_error_message': None
})
heur['avg_duration'] = (heur['num_runs'] * heur['avg_duration'] +
duration) / (heur['num_runs'] + 1)
heur['num_runs'] = heur['num_runs'] + 1
heur['last_error_message'] = result_string
heuristics[job_name] = heur
Cache.set('scheduled_jobs', heuristics)
return print_report(schedule_name, return_string, len(jobs))
def __init__(self, args):
urls = None
if args.input_file is not None:
args.quiet = True
with open(args.input_file, "r") as input_file:
urls = []
for url in input_file.readlines():
url = url.strip()
urls.append(url if "://" in url else "http://" + url)
else:
args.url = args.url.lower()
if "://" not in args.url:
args.url = "http://" + args.url
text_printer = Printer(args.verbosity)
cache = Cache()
cache.printer = text_printer
self.options = {
"url": args.url,
"urls": urls,
"quiet": args.quiet,
"prefix": "",
"user_agent": args.user_agent,
"proxy": args.proxy,
"verbosity": args.verbosity,
"threads": 10,
"batch_size": 20,
"run_all": args.run_all,
"match_all": args.match_all,
"stop_after": args.stop_after,
"no_cache_load": args.no_cache_load,
"no_cache_save": args.no_cache_save,
"write_file": args.output_file,
"subdomains": args.subdomains,
}
self.data = {
"cache": cache,
"results": Results(self.options),
"fingerprints": Fingerprints(),
"matcher": Match(),
"printer": text_printer,
"detected_cms": set(),
"error_pages": set(),
"requested": queue.Queue(),
}
if self.options["write_file"] is not None:
self.json_outputter = OutputJSON(self.options, self.data)
self.data["printer"].print_logo()
self.results = None
def __init__(self, args):
urls = None
if args.input_file is not None:
args.quiet = True
with open(args.input_file, 'r') as input_file:
urls = []
for url in input_file.readlines():
url = url.strip()
urls.append(url if '://' in url else 'http://'+url)
else:
args.url = args.url.lower()
if '://' not in args.url:
args.url = 'http://' + args.url
text_printer = Printer(args.verbosity)
cache = Cache()
cache.printer = text_printer
self.options = {
'url': args.url,
'urls': urls,
'quiet': args.quiet,
'prefix': '',
'user_agent': args.user_agent,
'proxy': args.proxy,
'verbosity': args.verbosity,
'threads': 10,
'batch_size': 20,
'run_all': args.run_all,
'match_all': args.match_all,
'stop_after': args.stop_after,
'no_cache_load': args.no_cache_load,
'no_cache_save': args.no_cache_save,
'write_file': args.output_file,
'subdomains': args.subdomains
}
self.data = {
'cache': cache,
'results': Results(self.options),
'fingerprints': Fingerprints(),
'matcher': Match(),
'printer': text_printer,
'detected_cms': set(),
'error_pages': set(),
'requested': queue.Queue()
}
if self.options['write_file'] is not None:
self.json_outputter = OutputJSON(self.options, self.data)
self.data['printer'].print_logo()
self.results = None
def __init__(self, args):
urls = None
if args.input_file is not None:
args.quiet = True
with open(args.input_file, 'r') as input_file:
urls = []
for url in input_file.readlines():
url = url.strip()
urls.append(url if '://' in url else 'http://' + url)
else:
args.url = args.url.lower()
if '://' not in args.url:
args.url = 'http://' + args.url
text_printer = Printer(args.verbosity)
cache = Cache()
cache.printer = text_printer
self.options = {
'url': args.url,
'urls': urls,
'quiet': args.quiet,
'prefix': '',
'user_agent': args.user_agent,
'proxy': args.proxy,
'verbosity': args.verbosity,
'threads': 10,
'batch_size': 20,
'run_all': args.run_all,
'match_all': args.match_all,
'stop_after': args.stop_after,
'no_cache_load': args.no_cache_load,
'no_cache_save': args.no_cache_save,
'write_file': args.output_file,
'subdomains': args.subdomains
}
self.data = {
'cache': cache,
'results': Results(self.options),
'fingerprints': Fingerprints(),
'matcher': Match(),
'printer': text_printer,
'detected_cms': set(),
'error_pages': set(),
'requested': queue.Queue()
}
if self.options['write_file'] is not None:
self.json_outputter = OutputJSON(self.options, self.data)
self.data['printer'].print_logo()
self.results = None
def __init__(self, args):
self.options = {
'url': args.url,
'prefix': '',
'user_agent': args.user_agent,
'proxy': args.proxy,
'verbosity': args.verbosity,
'threads': 10,
'chunk_size': 10, # same as threads
'run_all': args.run_all,
'match_all': args.match_all,
'stop_after': args.stop_after,
'no_cache_load': args.no_cache_load,
'no_cache_save': args.no_cache_save,
}
self.data = {
'cache': Cache(),
'results': Results(self.options),
'fingerprints': Fingerprints(),
'matcher': Match(),
'colorizer': Color(),
'printer': Printer(args.verbosity, Color()),
'detected_cms': set(),
'error_pages': set(),
'queue': queue.Queue(),
'requested': queue.Queue()
}
self.data['results'].set_printer(self.data['printer'])
self.data['requester'] = Requester(self.options, self.data)
def __init__(self, host, profile, verbose, desperate):
self.plugins = self.load_plugins()
self.host = host
self.results = Results()
self.cache = Cache()
self.profile = Profile(profile)
self.colorizer = Color()
self.logs = Log()
self.verbose = verbose
self.check_url()
self.redirect()
self.cache.set_host(self.host)
if desperate:
self.desperate = Desperate()
else:
self.desperate = None
def __init__(self, host, profile, verbose): self.plugins = self.load_plugins() self.host = host self.results = Results() self.cache = Cache() self.profile = Profile(profile) self.check_url() self.redirect() self.colorizer = Color() self.logs = Log() self.verbose = verbose
def getPosts(self, uri, min=None, max=None, text=None):
if (self.cache == None):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
else:
if (uri!=self.cache.uri or self.cache.bad):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
min, max = self.getDates()
if (not self.cache.bad):
posts = self.cache.query()
if (min!=None or max!=None or text!=None):
posts = self.cache.filterPosts(posts, min, max, text)
return posts
else:
self.alert('An exception ocurred parsing this URI')
return None
def getPosts(self, uri, min=None, max=None, text=None):
if (self.cache == None):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
else:
if (uri != self.cache.uri or self.cache.bad):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
min, max = self.getDates()
if (not self.cache.bad):
posts = self.cache.query()
if (min != None or max != None or text != None):
posts = self.cache.filterPosts(posts, min, max, text)
return posts
else:
self.alert('An exception ocurred parsing this URI')
return None
def __init__(self, args):
urls = None
interactive = True
if args.input_file is not None:
interactive = False
with open(args.input_file, 'r') as input_file:
urls = []
for url in input_file.readlines():
u = url.strip()
urls.append(u if '://' in u else 'http://' + u)
elif '://' not in args.url:
args.url = 'http://' + args.url
self.options = {
'url': args.url,
'urls': urls,
'interactive': interactive,
'prefix': '',
'user_agent': args.user_agent,
'proxy': args.proxy,
'verbosity': args.verbosity,
'threads': 10,
'batch_size': 20,
'run_all': args.run_all,
'match_all': args.match_all,
'stop_after': args.stop_after,
'no_cache_load': args.no_cache_load,
'no_cache_save': args.no_cache_save,
'write_file': args.output_file
}
self.data = {
'cache': Cache(),
'results': Results(self.options),
'fingerprints': Fingerprints(),
'matcher': Match(),
'colorizer': Color(),
'printer': Printer(args.verbosity, Color()),
'detected_cms': set(),
'error_pages': set(),
'requested': queue.Queue()
}
if self.options['write_file'] is not None:
self.json_outputter = OutputJSON(self.options, self.data)
def __init__(self, host, cache, results):
super().__init__(requests)
self.threads = 10
self.queue = queue.Queue()
self.requested = queue.Queue()
self.workers = []
self.host = host
# set cache
if not cache: self.cache = Cache()
else: self.cache = cache
# set results
if not results: self.results = Results()
else: self.results = results
def get_folder_jobs(folder_name):
expected = {'daily', 'hourly', 'every10mins'}
assert folder_name in expected, f'folder_name must be one of: {expected}'
path = f'./scheduled/{folder_name}'
jobs = {}
heuristics = Cache.get('scheduled_jobs', {})
for file_name in os.listdir(path):
if not is_job_to_run(f'{path}/{file_name}'):
continue
job_name = file_name.replace('.py', '')
import_path_string = f'scheduled.{folder_name}.{job_name}'
module = __import__(import_path_string, globals(), locals(),
['main', 'problem_data', 'fix'])
jobs[job_name] = module
# jobs.get_heuristics = lambda: Cache.get(job_name, {'name': job_name, 'num_runs': 0, 'avg_duration': 0})
# jobs.set_heuristics = lambda: Cache.get(job_name, {'name': job_name, 'num_runs': 0, 'avg_duration': 0})
return jobs
def main():
# a = Cache.set('abc', {'a':1})
# b = Cache.set(('abc', 123), {'a':1})
def func_value():
print('waiting...')
time.sleep(5)
print('done!')
return 12
# c = Cache.define('defg', func_value, calc_on_func_change=func_value)
d = Cache.define(
'defg',
func_value,
calc_if_true=lambda: prompt('Do you want to re-calculate the value?',
boolean=True))
return
def __init__(self, host, profile, verbose, desperate): self.plugins = self.load_plugins() self.host = host self.results = Results() self.cache = Cache() self.profile = Profile(profile) self.colorizer = Color() self.logs = Log() self.verbose = verbose self.check_url() self.redirect() self.cache.set_host(self.host) if desperate: self.desperate = Desperate() else: self.desperate = None
def fix_event_artists_field():
events = Cache.get('fix_event_artists_field')
# events = sf.select("""
# SELECT Id, EventName__c, PrimaryHeadlinerArtist__c, Artists__c, Venue__r.VenueName__c, Venue__r.BillingCity, EventYear__c
# , IsTouringApp__c, TourLeg__r.Tour__r.TourName__c, TourLeg__r.LegName__c, TourLeg__r.Tour__r.AppScope__c
# , (SELECT Id, Type__c, Artist__r.Name, Rank__c FROM Deals__r WHERE RecordType.Name = 'Artist' AND StageType__c = 'Plan')
# FROM Event__c
# CreatedDate >= THIS_YEAR
# """)
def iterator():
for item in events:
snapshot = item.copy()
item.Deals__r.sort(key=lambda x: (
(0 if x.Type__c == 'Primary Headliner' else 1
if x.Type__c == 'Co-Headliner' else 2), x.Rank__c))
primaryheadlinerlist = [
x for x in item.Deals__r if x['Type__c'] == 'Primary Headliner'
]
# headlinername = primaryheadlinerlist[0].Artist__r.Name if len(primaryheadlinerlist) > 0 else ''
# item.PrimaryHeadlinerArtist__c = headlinername
new_artists = '\r\n'.join((x.Artist__r.Name for x in item.Deals__r
if x.Artist__r is not None))
if new_artists != item.Artists__c:
item.OldArtists = item.Artists__c
item.Artists__c = new_artists
yield item
new_events = list(iterator())
pdh.to_excel(new_events, 'Events with incorrect Artist(s) field')
# with sf.bypass_settings():
# sf.update(new_events)
return
class Wig():
def __init__(self, host, profile, verbose, desperate, plugin_name=None):
self.plugins = self.load_plugins()
self.host = host
self.results = Results()
self.cache = Cache()
self.profile = Profile(profile)
self.colorizer = Color()
self.logs = Log()
self.verbose = verbose
self.plugin_name = plugin_name
self.check_url()
self.redirect()
self.cache.set_host(self.host)
if desperate:
self.desperate = Desperate()
else:
self.desperate = None
def redirect(self):
# detects redirection if this happend
try:
r = requests.get(self.host, verify=False)
except:
print("Invalid URL or host not found. Exiting...")
sys.exit(0)
if not r.url == self.host:
# ensure that sub-folders and files are removed
parts = r.url.split('//')
http, url = parts[0:2]
# remove subfolders and/or files
# http://example.com/test -> http://example.com/
if '/' in url:
redirected = http + '//' + url.split('/')[0] + '/'
else:
redirected = http + '//' + url + '/'
self.host = redirected
def check_url(self):
# adds http:// to input if not present
if not self.host.startswith("http"):
self.host = "http://" + self.host
def load_plugins(self):
# load all the plugins listed in plugins/__init__.py
all_plugins = []
for p in plugins.__all__:
plugin_path = "plugins." + p
__import__(plugin_path)
all_plugins.append(sys.modules[plugin_path])
return all_plugins
def run(self):
t = time.time()
num_fps = 0
num_plugins = 0
# loops over all the plugins loaded
for plugin in self.plugins:
# a loaded plugin might have more than one plugin, so 'ps' is a list
ps = plugin.get_instances(self.host, self.cache, self.results)
num_plugins += len(ps)
for p in ps:
# give a status of which plugin is run
print(p.name,
end=" \r")
sys.stdout.flush()
# applies the choosen profile by removing fingerprints from the
# fingerprint set if these do not match the choosen profile
p.set_profile(self.profile, self.plugin_name)
# the main plugin method
p.run()
num_fps += p.get_num_fps()
# check if running desperate mode.
if self.desperate:
# add the plugins fingerprints to the global fingerprint database
self.desperate.add_fingerprints(
p.get_items_for_desperate_mode())
# add logs
self.logs.add(p.get_logs())
if self.desperate:
self.desperate.set_cache(self.cache)
self.desperate.run()
for i in self.desperate.get_matches():
self.results.add('Desperate', i['cms'], i, i['count'])
# check the response headers for information
ch = CheckHeaders(self.cache, self.results, self.logs)
ch.run()
run_time = "%.1f" % (time.time() - t)
num_urls = self.cache.get_num_urls()
status = "Time: %s sec | Plugins: %s | Urls: %s | Fingerprints: %s" % (
run_time, num_plugins, num_urls, num_fps)
bar = "_" * len(status)
self.results.set_width(len(status))
print(self.results)
print(bar)
print(status + "\n")
if self.verbose:
print(bar)
print(self.logs)
class Buxon(GtkUI):
def clear(self):
"""
Clear all GTK components on Buxon
"""
#tree
self.treeTranslator = {}
for column in self.treeView.get_columns():
self.treeView.remove_column(column)
#text
self.text.get_buffer().set_text('')
def clearSearchForm(self):
"""
Clear search form
"""
widgets.get_widget('searchInput').set_text('')
widgets.get_widget('fromEntry').set_text('01/01/1995')
widgets.get_widget('toEntry').set_text('31/31/2010')
def showPost(self):
"""
Show post selected at gtk.TreeView
"""
selection = self.treeView.get_selection()
(model, iter) = selection.get_selected()
uri = model.get_value(iter, 0)
author, authorUri, listName, listUri, title, date, content = self.cache.getPost(
uri)
self.messageBar('loaded post ' + uri)
self.writePost(uri, author, authorUri, listName, listUri, title, date,
content)
def writePost(self,
uri,
author=None,
authorUri='',
listName=None,
listUri='',
title='',
date='',
content=''):
"""
Write a post on the gtkTextView
@param uri: post uri
@param author: author's name
@param authorUri: author's uri
@param listName: mailing list's name
@param listUri: mailing list's uri
@param title: post subject
@param date: post date
@param content: post body
"""
PANGO_SCALE = 1024
buffer = self.text.get_buffer()
buffer.set_text('')
iter = buffer.get_iter_at_offset(0)
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Post URI: \t', 'bold')
buffer.insert_with_tags_by_name(iter, uri, 'monospace')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'From: \t', 'bold')
if (author == None):
buffer.insert_with_tags_by_name(iter, authorUri, 'monospace')
else:
buffer.insert(iter, author)
buffer.insert(iter, ' <')
buffer.insert_with_tags_by_name(iter, authorUri, 'monospace')
buffer.insert(iter, '>')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'To: \t\t', 'bold')
if (listName == None):
buffer.insert_with_tags_by_name(iter, listUri, 'monospace')
else:
buffer.insert(iter, listName)
buffer.insert(iter, ' <')
buffer.insert_with_tags_by_name(iter, listUri, 'monospace')
buffer.insert(iter, '>')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Subject: \t', 'bold')
buffer.insert(iter, title)
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Date: \t', 'bold')
buffer.insert(iter, date)
buffer.insert(iter, '\n\n')
buffer.insert_with_tags_by_name(iter, content, 'wrap_mode')
buffer.insert(iter, '\n')
def getDates(self):
"""
Get selected dates
@return: dates
@rtype: tuple
"""
#min date
fromDate = widgets.get_widget('fromEntry').get_text().split('/')
min = float(fromDate[2]) * 10000000000
min += float(fromDate[1]) * 100000000
min += float(fromDate[0]) * 1000000
#max date
toDate = widgets.get_widget('toEntry').get_text().split('/')
max = float(toDate[2]) * 10000000000
max += float(toDate[1]) * 100000000
max += float(toDate[0]) * 1000000
return min, max
def getPosts(self, uri, min=None, max=None, text=None):
"""
Get mailing list's posts
@param uri: mailing list's uri
@param min: min date
@param max: max date
@param text: text to search
"""
if (self.cache == None):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
else:
if (uri != self.cache.uri or self.cache.bad):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
min, max = self.getDates()
if (not self.cache.bad):
posts = self.cache.query()
if (posts == None):
self.messageBar('unknow problem parsing RDF at ' + self.uri)
return None
else:
if (min != None or max != None or text != None):
posts = self.cache.filterPosts(posts, min, max, text)
return posts
else:
self.alert('An exception ocurred parsing this URI')
return None
def drawTree(self, posts):
"""
Draw post on gtk.TreeView
@param posts: posts
@type posts: tuple
"""
if (posts != None and len(posts) > 0):
#create tree
self.treeStore = gtk.TreeStore(str, str)
self.treeView.set_model(self.treeStore)
#append items
parent = None
for (post, title, date, creator, content, parent) in posts:
self.treeTranslator[post] = self.treeStore.append(
self.__getParent(parent),
[str(post), str(title)])
#print 'drawing post', post, 'on tree'
#and show it
treeColumn = gtk.TreeViewColumn('Posts')
self.treeView.append_column(treeColumn)
cell = gtk.CellRendererText()
treeColumn.pack_start(cell, True)
treeColumn.add_attribute(cell, 'text', 1)
treeColumn.set_sort_column_id(0)
self.messageBar('loaded ' + self.cache.uri)
else:
self.messageBar('none posts founded at ' + self.cache.uri)
def __getParent(self, uri):
"""
Get the parent post
@param uri: post uri
@return: parent uri
"""
if (uri in self.treeTranslator):
return self.treeTranslator[uri]
else:
return None
def messageBar(self, text):
"""
Write a message on the status bar
@param text: text
"""
self.statusbar.push(0, text)
def insertBufferTag(self, buffer, name, property, value):
"""
Insert a new tag on buffer
@param buffer: buffer
@param name: tag name
@param property: property to customize
@param value: property value
"""
tag = gtk.TextTag(name)
tag.set_property(property, value)
table = buffer.get_tag_table()
table.add(tag)
def getUri(self):
"""
Get actual URI
@return: actual uri
"""
if (self.cache == None):
return None
else:
return self.cache.uri
def destroy(self):
"""
Destoy all the infraestructure
"""
print 'Exiting...'
#if (self.cache != None):
# self.cache.dump(self.base + 'buxon.cache')
gtk.main_quit()
return gtk.FALSE
def main(self, uri=None):
"""
Main bucle
@param uri: uri
"""
#widgets
self.treeView = widgets.get_widget('postsTree')
self.text = widgets.get_widget('buxonTextView')
buffer = self.text.get_buffer()
self.insertBufferTag(buffer, 'bold', 'weight', pango.WEIGHT_BOLD)
self.insertBufferTag(buffer, 'monospace', 'family', 'monospace')
self.insertBufferTag(buffer, 'wrap_mode', 'wrap_mode', gtk.WRAP_WORD)
self.input = widgets.get_widget('urlInput')
self.statusbar = widgets.get_widget('buxonStatusbar')
self.messageBar('ready')
#main window
self.window = widgets.get_widget('buxon')
self.window.set_icon_from_file(self.base + 'includes/images/rdf.ico')
self.window.show()
if (uri != None):
self.input.set_text(uri)
gtk.main()
def __init__(self, base='./'):
"""
Buxon constructor
@param base: base directory
"""
GtkUI.__init__(self, 'buxon', base)
self.base = base
self.cache = None
self.treeTranslator = {}
class Buxon:
def clear(self):
#tree
self.treeTranslator = {}
for column in self.treeView.get_columns():
self.treeView.remove_column(column)
#text
self.text.get_buffer().set_text('')
def clearSearchForm(self):
widgets.get_widget('searchInput').set_text('')
widgets.get_widget('fromEntry').set_text('01/01/1995')
widgets.get_widget('toEntry').set_text('31/31/2010')
def showPost(self):
selection = self.treeView.get_selection()
(model, iter) = selection.get_selected()
uri = model.get_value(iter, 0)
author, authorUri, listName, listUri, title, date, content = self.cache.getPost(uri)
self.messageBar('loaded post ' + uri)
self.writePost(uri, author, authorUri, listName, listUri, title, date, content)
def writePost(self, uri, author=None, authorUri='', listName=None, listUri='', title='', date='', content=''):
PANGO_SCALE = 1024
buffer = self.text.get_buffer()
buffer.set_text('')
iter = buffer.get_iter_at_offset(0)
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Post URI: \t', 'bold')
buffer.insert_with_tags_by_name(iter, uri, 'monospace')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'From: \t', 'bold')
if (author == None):
buffer.insert_with_tags_by_name(iter, authorUri, 'monospace')
else:
buffer.insert(iter, author)
buffer.insert(iter, ' <')
buffer.insert_with_tags_by_name(iter, authorUri, 'monospace')
buffer.insert(iter, '>')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'To: \t\t', 'bold')
if (listName == None):
buffer.insert_with_tags_by_name(iter, listUri, 'monospace')
else:
buffer.insert(iter, listName)
buffer.insert(iter, ' <')
buffer.insert_with_tags_by_name(iter, listUri, 'monospace')
buffer.insert(iter, '>')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Subject: \t', 'bold')
buffer.insert(iter, title)
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Date: \t', 'bold')
buffer.insert(iter, date)
buffer.insert(iter, '\n\n')
buffer.insert_with_tags_by_name(iter, content, 'wrap_mode')
buffer.insert(iter, '\n')
def getDates(self):
#min date
fromDate = widgets.get_widget('fromEntry').get_text().split('/')
min = float(fromDate[2]) * 10000000000
min += float(fromDate[1]) * 100000000
min += float(fromDate[0]) * 1000000
#max date
toDate = widgets.get_widget('toEntry').get_text().split('/')
max = float(toDate[2]) * 10000000000
max += float(toDate[1]) * 100000000
max += float(toDate[0]) * 1000000
return min, max
def getPosts(self, uri, min=None, max=None, text=None):
if (self.cache == None):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
else:
if (uri!=self.cache.uri or self.cache.bad):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
min, max = self.getDates()
if (not self.cache.bad):
posts = self.cache.query()
if (min!=None or max!=None or text!=None):
posts = self.cache.filterPosts(posts, min, max, text)
return posts
else:
self.alert('An exception ocurred parsing this URI')
return None
def drawTree(self, posts):
if (posts!=None and len(posts)>0):
#create tree
self.treeStore = gtk.TreeStore(str, str)
self.treeView.set_model(self.treeStore)
#append items
parent = None
for (post, title, date, creator, content, parent) in posts:
self.treeTranslator[post] = self.treeStore.append(self.__getParent(parent), [str(post), str(title)])
print 'drawing post', post, 'on tree'
#and show it
treeColumn = gtk.TreeViewColumn('Posts')
self.treeView.append_column(treeColumn)
cell = gtk.CellRendererText()
treeColumn.pack_start(cell, True)
treeColumn.add_attribute(cell, 'text', 1)
treeColumn.set_sort_column_id(0)
self.messageBar('loaded ' + self.cache.uri)
else:
self.messageBar('none posts founded at ' + self.cache.uri)
def __getParent(self, uri):
if (uri in self.treeTranslator):
return self.treeTranslator[uri]
else:
return None
def alert(self, text):
self.alertWindow = gtk.Window(gtk.WINDOW_POPUP)
self.alertWindow.set_position(gtk.WIN_POS_CENTER_ALWAYS)
self.alertWindow.set_modal(True)
self.alertWindow.set_resizable(False)
self.alertWindow.set_border_width(0)
vbox = gtk.VBox(False, 5)
vbox.set_border_width(10)
self.alertWindow.add(vbox)
vbox.show()
align1 = gtk.Alignment(0.5, 0.5, 0, 0)
vbox.pack_start(align1, False, False, 5)
align1.show()
label = gtk.Label(text)
align1.add(label)
label.show()
align2 = gtk.Alignment(0.5, 0.5, 0, 0)
vbox.pack_start(align2, False, False, 5)
align2.show()
button = gtk.Button('OK')
button.connect('clicked', destroyAlert, 'cool button')
align2.add(button)
button.show()
self.alertWindow.show()
def messageBar(self, text):
self.statusbar.push(0, text)
def insertBufferTag(self, buffer, name, property, value):
tag = gtk.TextTag(name)
tag.set_property(property, value)
table = buffer.get_tag_table()
table.add(tag)
def getUri(self):
if (self.cache == None):
return None
else:
return self.cache.uri
def main(self, uri=None):
if (uri != None):
self.input.set_text(uri)
gtk.main()
def __init__(self):
self.cache = None
self.treeTranslator = {}
#widgets
self.treeView = widgets.get_widget('postsTree')
self.text = widgets.get_widget('buxonTextView')
buffer = self.text.get_buffer()
self.insertBufferTag(buffer, 'bold', 'weight', pango.WEIGHT_BOLD)
self.insertBufferTag(buffer, 'monospace', 'family', 'monospace')
self.insertBufferTag(buffer, 'wrap_mode', 'wrap_mode', gtk.WRAP_WORD)
self.input = widgets.get_widget('urlInput')
self.statusbar = widgets.get_widget('buxonStatusbar')
self.messageBar('ready')
#main window
self.window = widgets.get_widget('buxon')
self.window.set_icon_from_file('includes/rdf.ico')
self.window.show()
class Buxon:
def clear(self):
#tree
self.treeTranslator = {}
for column in self.treeView.get_columns():
self.treeView.remove_column(column)
#text
self.text.get_buffer().set_text('')
def clearSearchForm(self):
widgets.get_widget('searchInput').set_text('')
widgets.get_widget('fromEntry').set_text('01/01/1995')
widgets.get_widget('toEntry').set_text('31/31/2010')
def showPost(self):
selection = self.treeView.get_selection()
(model, iter) = selection.get_selected()
uri = model.get_value(iter, 0)
author, authorUri, listName, listUri, title, date, content = self.cache.getPost(
uri)
self.messageBar('loaded post ' + uri)
self.writePost(uri, author, authorUri, listName, listUri, title, date,
content)
def writePost(self,
uri,
author=None,
authorUri='',
listName=None,
listUri='',
title='',
date='',
content=''):
PANGO_SCALE = 1024
buffer = self.text.get_buffer()
buffer.set_text('')
iter = buffer.get_iter_at_offset(0)
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Post URI: \t', 'bold')
buffer.insert_with_tags_by_name(iter, uri, 'monospace')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'From: \t', 'bold')
if (author == None):
buffer.insert_with_tags_by_name(iter, authorUri, 'monospace')
else:
buffer.insert(iter, author)
buffer.insert(iter, ' <')
buffer.insert_with_tags_by_name(iter, authorUri, 'monospace')
buffer.insert(iter, '>')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'To: \t\t', 'bold')
if (listName == None):
buffer.insert_with_tags_by_name(iter, listUri, 'monospace')
else:
buffer.insert(iter, listName)
buffer.insert(iter, ' <')
buffer.insert_with_tags_by_name(iter, listUri, 'monospace')
buffer.insert(iter, '>')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Subject: \t', 'bold')
buffer.insert(iter, title)
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Date: \t', 'bold')
buffer.insert(iter, date)
buffer.insert(iter, '\n\n')
buffer.insert_with_tags_by_name(iter, content, 'wrap_mode')
buffer.insert(iter, '\n')
def getDates(self):
#min date
fromDate = widgets.get_widget('fromEntry').get_text().split('/')
min = float(fromDate[2]) * 10000000000
min += float(fromDate[1]) * 100000000
min += float(fromDate[0]) * 1000000
#max date
toDate = widgets.get_widget('toEntry').get_text().split('/')
max = float(toDate[2]) * 10000000000
max += float(toDate[1]) * 100000000
max += float(toDate[0]) * 1000000
return min, max
def getPosts(self, uri, min=None, max=None, text=None):
if (self.cache == None):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
else:
if (uri != self.cache.uri or self.cache.bad):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
min, max = self.getDates()
if (not self.cache.bad):
posts = self.cache.query()
if (min != None or max != None or text != None):
posts = self.cache.filterPosts(posts, min, max, text)
return posts
else:
self.alert('An exception ocurred parsing this URI')
return None
def drawTree(self, posts):
if (posts != None and len(posts) > 0):
#create tree
self.treeStore = gtk.TreeStore(str, str)
self.treeView.set_model(self.treeStore)
#append items
parent = None
for (post, title, date, creator, content, parent) in posts:
self.treeTranslator[post] = self.treeStore.append(
self.__getParent(parent),
[str(post), str(title)])
print 'drawing post', post, 'on tree'
#and show it
treeColumn = gtk.TreeViewColumn('Posts')
self.treeView.append_column(treeColumn)
cell = gtk.CellRendererText()
treeColumn.pack_start(cell, True)
treeColumn.add_attribute(cell, 'text', 1)
treeColumn.set_sort_column_id(0)
self.messageBar('loaded ' + self.cache.uri)
else:
self.messageBar('none posts founded at ' + self.cache.uri)
def __getParent(self, uri):
if (uri in self.treeTranslator):
return self.treeTranslator[uri]
else:
return None
def alert(self, text):
self.alertWindow = gtk.Window(gtk.WINDOW_POPUP)
self.alertWindow.set_position(gtk.WIN_POS_CENTER_ALWAYS)
self.alertWindow.set_modal(True)
self.alertWindow.set_resizable(False)
self.alertWindow.set_border_width(0)
vbox = gtk.VBox(False, 5)
vbox.set_border_width(10)
self.alertWindow.add(vbox)
vbox.show()
align1 = gtk.Alignment(0.5, 0.5, 0, 0)
vbox.pack_start(align1, False, False, 5)
align1.show()
label = gtk.Label(text)
align1.add(label)
label.show()
align2 = gtk.Alignment(0.5, 0.5, 0, 0)
vbox.pack_start(align2, False, False, 5)
align2.show()
button = gtk.Button('OK')
button.connect('clicked', destroyAlert, 'cool button')
align2.add(button)
button.show()
self.alertWindow.show()
def messageBar(self, text):
self.statusbar.push(0, text)
def insertBufferTag(self, buffer, name, property, value):
tag = gtk.TextTag(name)
tag.set_property(property, value)
table = buffer.get_tag_table()
table.add(tag)
def getUri(self):
if (self.cache == None):
return None
else:
return self.cache.uri
def main(self, uri=None):
if (uri != None):
self.input.set_text(uri)
gtk.main()
def __init__(self):
self.cache = None
self.treeTranslator = {}
#widgets
self.treeView = widgets.get_widget('postsTree')
self.text = widgets.get_widget('buxonTextView')
buffer = self.text.get_buffer()
self.insertBufferTag(buffer, 'bold', 'weight', pango.WEIGHT_BOLD)
self.insertBufferTag(buffer, 'monospace', 'family', 'monospace')
self.insertBufferTag(buffer, 'wrap_mode', 'wrap_mode', gtk.WRAP_WORD)
self.input = widgets.get_widget('urlInput')
self.statusbar = widgets.get_widget('buxonStatusbar')
self.messageBar('ready')
#main window
self.window = widgets.get_widget('buxon')
self.window.set_icon_from_file('includes/rdf.ico')
self.window.show()
def save_cookie(driver):
cookies = driver.get_cookies()
Cache.set(('browser_cookie', driver.current_url), cookies)
def load_cookie(driver):
cookies = Cache.get(('browser_cookie', driver.current_url))
for cookie in cookies:
driver.add_cookie(cookie)
class Buxon(GtkUI):
def clear(self):
"""
Clear all GTK components on Buxon
"""
#tree
self.treeTranslator = {}
for column in self.treeView.get_columns():
self.treeView.remove_column(column)
#text
self.text.get_buffer().set_text('')
def clearSearchForm(self):
"""
Clear search form
"""
widgets.get_widget('searchInput').set_text('')
widgets.get_widget('fromEntry').set_text('01/01/1995')
widgets.get_widget('toEntry').set_text('31/31/2010')
def showPost(self):
"""
Show post selected at gtk.TreeView
"""
selection = self.treeView.get_selection()
(model, iter) = selection.get_selected()
uri = model.get_value(iter, 0)
author, authorUri, listName, listUri, title, date, content = self.cache.getPost(uri)
self.messageBar('loaded post ' + uri)
self.writePost(uri, author, authorUri, listName, listUri, title, date, content)
def writePost(self, uri, author=None, authorUri='', listName=None, listUri='', title='', date='', content=''):
"""
Write a post on the gtkTextView
@param uri: post uri
@param author: author's name
@param authorUri: author's uri
@param listName: mailing list's name
@param listUri: mailing list's uri
@param title: post subject
@param date: post date
@param content: post body
"""
PANGO_SCALE = 1024
buffer = self.text.get_buffer()
buffer.set_text('')
iter = buffer.get_iter_at_offset(0)
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Post URI: \t', 'bold')
buffer.insert_with_tags_by_name(iter, uri, 'monospace')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'From: \t', 'bold')
if (author == None):
buffer.insert_with_tags_by_name(iter, authorUri, 'monospace')
else:
buffer.insert(iter, author)
buffer.insert(iter, ' <')
buffer.insert_with_tags_by_name(iter, authorUri, 'monospace')
buffer.insert(iter, '>')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'To: \t\t', 'bold')
if (listName == None):
buffer.insert_with_tags_by_name(iter, listUri, 'monospace')
else:
buffer.insert(iter, listName)
buffer.insert(iter, ' <')
buffer.insert_with_tags_by_name(iter, listUri, 'monospace')
buffer.insert(iter, '>')
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Subject: \t', 'bold')
buffer.insert(iter, title)
buffer.insert(iter, '\n')
buffer.insert_with_tags_by_name(iter, 'Date: \t', 'bold')
buffer.insert(iter, date)
buffer.insert(iter, '\n\n')
buffer.insert_with_tags_by_name(iter, content, 'wrap_mode')
buffer.insert(iter, '\n')
def getDates(self):
"""
Get selected dates
@return: dates
@rtype: tuple
"""
#min date
fromDate = widgets.get_widget('fromEntry').get_text().split('/')
min = float(fromDate[2]) * 10000000000
min += float(fromDate[1]) * 100000000
min += float(fromDate[0]) * 1000000
#max date
toDate = widgets.get_widget('toEntry').get_text().split('/')
max = float(toDate[2]) * 10000000000
max += float(toDate[1]) * 100000000
max += float(toDate[0]) * 1000000
return min, max
def getPosts(self, uri, min=None, max=None, text=None):
"""
Get mailing list's posts
@param uri: mailing list's uri
@param min: min date
@param max: max date
@param text: text to search
"""
if (self.cache == None):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
else:
if (uri!=self.cache.uri or self.cache.bad):
pb = LoadProgressBar()
self.cache = Cache(uri, pb)
pb.destroy()
min, max = self.getDates()
if (not self.cache.bad):
posts = self.cache.query()
if (posts == None):
self.messageBar('unknow problem parsing RDF at ' + self.uri)
return None
else:
if (min!=None or max!=None or text!=None):
posts = self.cache.filterPosts(posts, min, max, text)
return posts
else:
self.alert('An exception ocurred parsing this URI')
return None
def drawTree(self, posts):
"""
Draw post on gtk.TreeView
@param posts: posts
@type posts: tuple
"""
if (posts!=None and len(posts)>0):
#create tree
self.treeStore = gtk.TreeStore(str, str)
self.treeView.set_model(self.treeStore)
#append items
parent = None
for (post, title, date, creator, content, parent) in posts:
self.treeTranslator[post] = self.treeStore.append(self.__getParent(parent), [str(post), str(title)])
#print 'drawing post', post, 'on tree'
#and show it
treeColumn = gtk.TreeViewColumn('Posts')
self.treeView.append_column(treeColumn)
cell = gtk.CellRendererText()
treeColumn.pack_start(cell, True)
treeColumn.add_attribute(cell, 'text', 1)
treeColumn.set_sort_column_id(0)
self.messageBar('loaded ' + self.cache.uri)
else:
self.messageBar('none posts founded at ' + self.cache.uri)
def __getParent(self, uri):
"""
Get the parent post
@param uri: post uri
@return: parent uri
"""
if (uri in self.treeTranslator):
return self.treeTranslator[uri]
else:
return None
def messageBar(self, text):
"""
Write a message on the status bar
@param text: text
"""
self.statusbar.push(0, text)
def insertBufferTag(self, buffer, name, property, value):
"""
Insert a new tag on buffer
@param buffer: buffer
@param name: tag name
@param property: property to customize
@param value: property value
"""
tag = gtk.TextTag(name)
tag.set_property(property, value)
table = buffer.get_tag_table()
table.add(tag)
def getUri(self):
"""
Get actual URI
@return: actual uri
"""
if (self.cache == None):
return None
else:
return self.cache.uri
def destroy(self):
"""
Destoy all the infraestructure
"""
print 'Exiting...'
#if (self.cache != None):
# self.cache.dump(self.base + 'buxon.cache')
gtk.main_quit()
return gtk.FALSE
def main(self, uri=None):
"""
Main bucle
@param uri: uri
"""
#widgets
self.treeView = widgets.get_widget('postsTree')
self.text = widgets.get_widget('buxonTextView')
buffer = self.text.get_buffer()
self.insertBufferTag(buffer, 'bold', 'weight', pango.WEIGHT_BOLD)
self.insertBufferTag(buffer, 'monospace', 'family', 'monospace')
self.insertBufferTag(buffer, 'wrap_mode', 'wrap_mode', gtk.WRAP_WORD)
self.input = widgets.get_widget('urlInput')
self.statusbar = widgets.get_widget('buxonStatusbar')
self.messageBar('ready')
#main window
self.window = widgets.get_widget('buxon')
self.window.set_icon_from_file(self.base + 'includes/images/rdf.ico')
self.window.show()
if (uri != None):
self.input.set_text(uri)
gtk.main()
def __init__(self, base='./'):
"""
Buxon constructor
@param base: base directory
"""
GtkUI.__init__(self, 'buxon', base)
self.base = base
self.cache = None
self.treeTranslator = {}
ap.add_argument("-d",
"--debug",
dest="log_level",
action="store_const",
const=logging.DEBUG,
default=logging.INFO,
help="Show debug log messages")
args = ap.parse_args()
logging.getLogger().setLevel(args.log_level)
ngrok_service = NgrokService()
ngrok_service.start(settings.API_PORT)
ngrok_url = ngrok_service.get_url()
postback_api = PostbackApi(ngrok_url)
catappult_api = CatappultApi(settings.CATAPPULT_API_HOST, Cache())
one_step_validator = OneStepPaymentValidator(postback_api, catappult_api)
logging.info("Testing One-Step Payment...")
purchase_info = get_purchase_info(ngrok_url)
try:
is_verified = one_step_validator.validate(purchase_info)
except ValidatorException as e:
logging.error(e)
is_verified = False
if is_verified:
logging.info("Transaction: '{}' was successfully verified".format(
purchase_info))
else:
logging.error(
class Wig():
def __init__(self, host, profile, verbose):
self.plugins = self.load_plugins()
self.host = host
self.results = Results()
self.cache = Cache()
self.profile = Profile(profile)
self.check_url()
self.redirect()
self.colorizer = Color()
self.logs = Log()
self.verbose = verbose
def redirect(self):
# detects redirection if this happend
r = requests.get(self.host, verify=False)
if not r.url == self.host:
# ensure that sub-folders and files are removed
parts = r.url.split('//')
http, url = parts[0:2]
# remove subfolders and/or files
# http://example.com/test -> http://example.com/
if '/' in url:
redirected = http + '//' + url.split('/')[0] + '/'
else:
redirected = http + '//' + url + '/'
self.host = redirected
def check_url(self):
# adds http:// to input if not present
if not self.host.startswith("http"):
self.host = "http://" + self.host
def load_plugins(self):
# load all the plugins listed in plugins/__init__.py
all_plugins = []
for p in plugins.__all__:
plugin_path = "plugins." + p
__import__(plugin_path)
all_plugins.append(sys.modules[plugin_path])
return all_plugins
def run(self):
t = time.time()
num_fps = 0
num_plugins = 0
# loops over all the plugins loaded
for plugin in self.plugins:
# a loaded plugin might have more than one plugin, so 'ps' is a list
ps = plugin.get_instances(self.host, self.cache, self.results)
num_plugins += len(ps)
for p in ps:
# give a status of which plugin is run
print(p.name, end=" \r")
sys.stdout.flush()
# applies the choosen profile by removing fingerprints from the
# fingerprint set if these do not match the choosen profile
p.set_profile(self.profile)
# the main plugin method
p.run()
num_fps += p.get_num_fps()
# add logs
self.logs.add( p.get_logs() )
run_time = "%.1f" % (time.time() - t)
num_urls = self.cache.get_num_urls()
status = "Time: %s sec | Plugins: %s | Urls: %s | Fingerprints: %s" % (run_time, num_plugins, num_urls, num_fps)
bar = "_"*len(status)
self.results.set_width(len(status))
print(self.results)
print(bar)
print(status + "\n")
if self.verbose:
print(bar)
print(self.logs)
class Wig():
def __init__(self, host, profile, verbose, desperate, plugin_name=None):
self.plugins = self.load_plugins()
self.host = host
self.results = Results()
self.cache = Cache()
self.profile = Profile(profile)
self.colorizer = Color()
self.logs = Log()
self.verbose = verbose
self.plugin_name = plugin_name
self.check_url()
self.redirect()
self.cache.set_host(self.host)
if desperate:
self.desperate = Desperate()
else:
self.desperate = None
def redirect(self):
# detects redirection if this happend
try:
r = requests.get(self.host, verify=False)
except:
print("Invalid URL or host not found. Exiting...")
sys.exit(0)
if not r.url == self.host:
# ensure that sub-folders and files are removed
parts = r.url.split('//')
http, url = parts[0:2]
# remove subfolders and/or files
# http://example.com/test -> http://example.com/
if '/' in url:
redirected = http + '//' + url.split('/')[0] + '/'
else:
redirected = http + '//' + url + '/'
self.host = redirected
def check_url(self):
# adds http:// to input if not present
if not self.host.startswith("http"):
self.host = "http://" + self.host
def load_plugins(self):
# load all the plugins listed in plugins/__init__.py
all_plugins = []
for p in plugins.__all__:
plugin_path = "plugins." + p
__import__(plugin_path)
all_plugins.append(sys.modules[plugin_path])
return all_plugins
def run(self):
t = time.time()
num_fps = 0
num_plugins = 0
# loops over all the plugins loaded
for plugin in self.plugins:
# a loaded plugin might have more than one plugin, so 'ps' is a list
ps = plugin.get_instances(self.host, self.cache, self.results)
num_plugins += len(ps)
for p in ps:
# give a status of which plugin is run
print(p.name, end=" \r")
sys.stdout.flush()
# applies the choosen profile by removing fingerprints from the
# fingerprint set if these do not match the choosen profile
p.set_profile(self.profile, self.plugin_name)
# the main plugin method
p.run()
num_fps += p.get_num_fps()
# check if running desperate mode.
if self.desperate:
# add the plugins fingerprints to the global fingerprint database
self.desperate.add_fingerprints(p.get_items_for_desperate_mode())
# add logs
self.logs.add( p.get_logs() )
if self.desperate:
self.desperate.set_cache(self.cache)
self.desperate.run()
for i in self.desperate.get_matches():
self.results.add('Desperate', i['cms'], i, i['count'])
# check the response headers for information
ch = CheckHeaders(self.cache, self.results, self.logs)
ch.run()
run_time = "%.1f" % (time.time() - t)
num_urls = self.cache.get_num_urls()
status = "Time: %s sec | Plugins: %s | Urls: %s | Fingerprints: %s" % (run_time, num_plugins, num_urls, num_fps)
bar = "_"*len(status)
self.results.set_width(len(status))
print(self.results)
print(bar)
print(status + "\n")
if self.verbose:
print(bar)
print(self.logs)
def reset(self):
self.data['results'] = Results(self.options)
self.data['cache'] = Cache()
def get_job_heuristics():
return Cache.get('scheduled_jobs', {})
def main():
# test = sf.select('''SELECT Id, Event__r.Venue__c, Event__r.Venue__r.VenueName__c,
# Event__r.Venue__r.BillingCity, Event__r.Venue__r.BillingCountry, Type__c,RateType__c,
# AmountRate__c, BeforeAfter__c
# FROM Deduction__c LIMIT 50000''', mode='bulk', return_type='dataframe')
data = sf.fselect("""
--sql
SELECT
Event__c
, Event__r.Venue__c AS VenueId
, Event__r.Venue__r.VenueName__c AS VenueName
, Event__r.Venue__r.BillingCity + ', ' + Event__r.Venue__r.BillingCountryCode AS City
, Event__r.Venue__r.BillingCountry AS Country
, Type__c
, RateType__c
, AmountRate__c
, BeforeAfter__c
FROM Deduction__c
WHERE ApplyTicketScale__c NOT IN ('Selected Tickets','Other Amount')
AND StageType__c = Event__r.StatusStage__c
AND Type__c != 'FMF/Venue Charge'
AND Event__r.Status__c IN ('Confirmed','Flash Started','Flash Complete')
-- AND RateType__c != 'Flat'
-- AND Event__r.Venue__r.VenueName__c = 'The Wiltern'
-- LIMIT 50000
--endsql
""", mode='bulk', content_type='JSON')
Cache.set('deductionsanalysis', data)
# data = Cache.get('deductionsanalysis')
by_event_id = {}
venue_ids = set()
venue_fields = {}
venue_counts = {}
for i, item in enumerate(data):
if isinstance(item.VenueId, SalesforceFormulaParser.NullValue): continue
curr_list = by_event_id.get(item.Event__c, [])
curr_list.append(Deduction(item.VenueId, item.VenueName or '', item.BeforeAfter__c or '', item.Type__c or '', item.RateType__c or '', item.AmountRate__c or 0))
by_event_id[item.Event__c] = curr_list
venue_ids.add(item.VenueId)
venue_fields[item.VenueId] = Venue(item.VenueName, item.City, item.Country)
sets = []
sets_by_venue = {}
for lst in by_event_id.values():
new_set = tuple(sorted(lst, key=lambda item: (item.VenueId, item.VenueName, item.BeforeAfter__c, item.Type__c, item.RateType__c, item.AmountRate__c)))
venue_id = new_set[0].VenueId
if venue_id not in sets_by_venue:
sets_by_venue[venue_id] = set()
venue_counts[venue_id] = 0
sets.append(new_set)
sets_by_venue[venue_id].add(new_set)
venue_counts[venue_id] += 1
counter = {}
for tup in sets:
val = counter.get(tup, 0) + 1
counter[tup] = val
top_n = 3
top_n_sets_by_venue = {
k: sorted([(counter[s], s) for s in v], key=lambda x: x[0], reverse=True)[:top_n]
for k, v in sets_by_venue.items()
}
top_n_sets_by_venue_text = {
k: [set_to_text(x[1], x[0]) for x in v]
for k, v in top_n_sets_by_venue.items()
}
def get_n(venue_id, n):
lst = top_n_sets_by_venue[venue_id]
if len(lst) > n:
return lst[n][0]
return None
def get_n_str(venue_id, n):
lst = top_n_sets_by_venue_text[venue_id]
if len(lst) > n:
return lst[n]
return ''
def set_output(row):
x = row.VenueId
row['Total Events'] = venue_counts[x]
percent = lambda x: None if not x else (x / row['Total Events'] * 100).round()
row['Venue Name'] = venue_fields[x].Name
row['City'] = venue_fields[x].City
row['Country'] = venue_fields[x].Country
row['1st'] = get_n(x, 0)
row['1st %'] = percent(row['1st'])
row['1st Deduction Setup'] = get_n_str(x, 0)
row['2nd'] = get_n(x, 1)
row['2nd %'] = percent(row['2nd'])
row['2nd Deduction Setup'] = get_n_str(x, 1)
row['3rd'] = get_n(x, 2)
row['3rd %'] = percent(row['3rd'])
row['3rd Deduction Setup'] = get_n_str(x, 2)
return row
df = (
pd.DataFrame({'VenueId': list(venue_ids)})
.apply(set_output, axis=1, result_type='expand')
# .assign(**{
# 'Total Events': lambda df: df.VenueId.apply(lambda x: venue_counts[x]),
# 'Venue Name': lambda df: df.VenueId.apply(lambda x: venue_fields[x].Name),
# 'Venue City': lambda df: df.VenueId.apply(lambda x: venue_fields[x].City),
# '1st': lambda df: df.VenueId.apply(lambda x: get_n(x, 0)),
# '1st Deduction Setup': lambda df: df.VenueId.apply(lambda x: get_n_str(x, 0)),
# '2nd': lambda df: df.VenueId.apply(lambda x: get_n(x, 1)),
# '2nd Deduction Setup': lambda df: df.VenueId.apply(lambda x: get_n_str(x, 1)),
# '3rd': lambda df: df.VenueId.apply(lambda x: get_n(x, 2)),
# '3rd Deduction Setup': lambda df: df.VenueId.apply(lambda x: get_n_str(x, 2)),
# })
.sort_values(['1st %', '1st'], ascending=False)
)
pdh.to_excel(df, 'Common Deductions Analysis.xlsx', word_wrap=True)
return
def eos_venues():
def get_eos_dupes_analysis():
records = sql.query("""
SELECT v.Id, v.Name, v.RomeId, v.IsActive
FROM Venue v
LEFT JOIN TownCity t
ON v.TownCityId = t.Id
LEFT JOIN Region r
ON t.RegionId = r.Id
LEFT JOIN Country c
ON r.CountryId = c.Id
WHERE c.Name IN ('United Kingdom')
""")
return pdh.find_duplicates(records, 'Name', None,
['Id', 'RomeId', 'IsActive'])
def get_deletion_log():
return pd.DataFrame(
sql.query("""
SELECT l.DeletedRecordId__c
, l.MergedAccount__c
, l.RecordTypeName__c
, v1.Id AS EOSIdToDelete
, v1.Name AS EOSNameToDelete
, v2.Id AS EOSIdToMergeInto
, v2.Name AS EOSNameToMergeInto
, CASE WHEN v1.Id < v2.Id THEN 'Double check this - older ID being deleted' ELSE NULL END AS SpotCheck
FROM RomeDeletionLog l
LEFT JOIN Venue v1
ON v1.RomeId = l.DeletedRecordId__c
AND l.RecordTypeName__c = 'Venue'
LEFT JOIN Venue v2
ON v2.RomeId = l.MergedAccount__c
AND l.RecordTypeName__c = 'Venue'WHERE v1.Id IS NOT NULL
"""))
eos_dupes = Cache.get('eosdupes', None)
if eos_dupes is None or prompt('Re-run EOS Venue Dupe Analysis?',
boolean=True):
eos_dupes = get_eos_dupes_analysis()
Cache.set('eosdupes', eos_dupes)
deletion_log = get_deletion_log()
deleted_ids = set(deletion_log.DeletedRecordId__c)
def note(row):
r1, r2 = row['RomeId'], row['m_RomeId']
if pdh.isnull(r1) and pdh.isnull(r2):
return 'EOS Merge - No Rome links exist'
if r1 in deleted_ids and r2 in deleted_ids:
return 'Both RomeIds are deleted in Rome'
elif r2 in deleted_ids:
return 'EOS Merge RIGHT into LEFT - RIGHT Id is deleted in Rome'
elif r1 in deleted_ids:
return 'EOS Merge LEFT into RIGHT - LEFT Id is deleted in Rome'
if pdh.isnull(r1) and pdh.notnull(r2):
return 'EOS Merge LEFT into RIGHT'
if pdh.isnull(r2) and pdh.notnull(r1):
return 'EOS Merge RIGHT into LEFT'
if r1 == r2:
return 'EOS Merge - Both EOS Venues have the same RomeId... merge one into the other'
if r1 not in deleted_ids and r2 not in deleted_ids:
return 'ROME Merge first, then EOS can be merged'
eos_dupes.query("`m_%` > 70", inplace=True)
eos_dupes['Note'] = eos_dupes.apply(note, axis=1)
eos_dupes[
'RomeMergeLink'] = 'https://lne.my.salesforce.com/merge/accmergewizard.jsp?goNext=+Next+&cid=' + eos_dupes[
'RomeId'] + '&cid=' + eos_dupes['m_RomeId']
output = {
'EOS Dupe Analysis': eos_dupes,
'RomeDeletionLog': deletion_log,
}
file_name = loc.uk_folder_path + '/Master Data - Output/Venue Dupe Analysis/EOS Venue Duplicate Analysis ' + datetime.now(
).strftime('%Y-%m-%d %H.%M') + '.xlsx'
pdh.to_excel(output, file_name)
return
