def getPosts(self, uri, min=None, max=None, text=None): """ Get mailing list's posts @param uri: mailing list's uri @param min: min date @param max: max date @param text: text to search """ if (self.cache == None): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() else: if (uri != self.cache.uri or self.cache.bad): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() min, max = self.getDates() if (not self.cache.bad): posts = self.cache.query() if (posts == None): self.messageBar('unknow problem parsing RDF at ' + self.uri) return None else: if (min != None or max != None or text != None): posts = self.cache.filterPosts(posts, min, max, text) return posts else: self.alert('An exception ocurred parsing this URI') return None
def getPosts(self, uri, min=None, max=None, text=None): """ Get mailing list's posts @param uri: mailing list's uri @param min: min date @param max: max date @param text: text to search """ if (self.cache == None): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() else: if (uri!=self.cache.uri or self.cache.bad): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() min, max = self.getDates() if (not self.cache.bad): posts = self.cache.query() if (posts == None): self.messageBar('unknow problem parsing RDF at ' + self.uri) return None else: if (min!=None or max!=None or text!=None): posts = self.cache.filterPosts(posts, min, max, text) return posts else: self.alert('An exception ocurred parsing this URI') return None
def run(schedule_name, jobs, do_fixes=False): heuristics = get_job_heuristics() if len(jobs) == 0: return print("^ {} running {} jobs".format(time.strftime("%m/%d/%Y %H:%M:%S"), len(jobs))) return_string = "" # def run_module_job(name, module): # start_time = time.time() # return_string = '' # return_string += "\n\n################\n" # return_string += name # if hasattr(module, 'problem_data'): # job = getattr(module, 'problem_data') # else: # job = getattr(module, 'main') # try: # list(sessions.values())[0].console.new_line('Running {}'.format(name)) # if job.__name__ == 'main': # result = str(job(sessions)) # elif job.__name__ == 'problem_data': # result = run_job(name, module.problem_data, module.fix, module.sessions, do_fix=False, download_result=False, sf_sessions=sessions, sql_sessions=None) # if len(result) > 0: # return_string += "\n" # return_string += result # else: # return_string = '' # except Exception as e: # return_string += "\n>>>>> Job failed: {}\n{}\n".format(name,str(e)) # duration = math.floor(time.time() - start_time) # return name, return_string, duration threads = [] # sorted_jobs = for name, module in jobs.items(): threads.append( threading.new(run_module_job, sessions, name, module, do_fix=do_fixes)) for thread in threads: job_name, result_string, duration = thread.result() return_string += '\n' + result_string heur = heuristics.get( job_name, { 'name': job_name, 'num_runs': 0, 'avg_duration': 0, 'last_error_message': None }) heur['avg_duration'] = (heur['num_runs'] * heur['avg_duration'] + duration) / (heur['num_runs'] + 1) heur['num_runs'] = heur['num_runs'] + 1 heur['last_error_message'] = result_string heuristics[job_name] = heur Cache.set('scheduled_jobs', heuristics) return print_report(schedule_name, return_string, len(jobs))
def __init__(self, args): urls = None if args.input_file is not None: args.quiet = True with open(args.input_file, "r") as input_file: urls = [] for url in input_file.readlines(): url = url.strip() urls.append(url if "://" in url else "http://" + url) else: args.url = args.url.lower() if "://" not in args.url: args.url = "http://" + args.url text_printer = Printer(args.verbosity) cache = Cache() cache.printer = text_printer self.options = { "url": args.url, "urls": urls, "quiet": args.quiet, "prefix": "", "user_agent": args.user_agent, "proxy": args.proxy, "verbosity": args.verbosity, "threads": 10, "batch_size": 20, "run_all": args.run_all, "match_all": args.match_all, "stop_after": args.stop_after, "no_cache_load": args.no_cache_load, "no_cache_save": args.no_cache_save, "write_file": args.output_file, "subdomains": args.subdomains, } self.data = { "cache": cache, "results": Results(self.options), "fingerprints": Fingerprints(), "matcher": Match(), "printer": text_printer, "detected_cms": set(), "error_pages": set(), "requested": queue.Queue(), } if self.options["write_file"] is not None: self.json_outputter = OutputJSON(self.options, self.data) self.data["printer"].print_logo() self.results = None
def __init__(self, args): urls = None if args.input_file is not None: args.quiet = True with open(args.input_file, 'r') as input_file: urls = [] for url in input_file.readlines(): url = url.strip() urls.append(url if '://' in url else 'http://'+url) else: args.url = args.url.lower() if '://' not in args.url: args.url = 'http://' + args.url text_printer = Printer(args.verbosity) cache = Cache() cache.printer = text_printer self.options = { 'url': args.url, 'urls': urls, 'quiet': args.quiet, 'prefix': '', 'user_agent': args.user_agent, 'proxy': args.proxy, 'verbosity': args.verbosity, 'threads': 10, 'batch_size': 20, 'run_all': args.run_all, 'match_all': args.match_all, 'stop_after': args.stop_after, 'no_cache_load': args.no_cache_load, 'no_cache_save': args.no_cache_save, 'write_file': args.output_file, 'subdomains': args.subdomains } self.data = { 'cache': cache, 'results': Results(self.options), 'fingerprints': Fingerprints(), 'matcher': Match(), 'printer': text_printer, 'detected_cms': set(), 'error_pages': set(), 'requested': queue.Queue() } if self.options['write_file'] is not None: self.json_outputter = OutputJSON(self.options, self.data) self.data['printer'].print_logo() self.results = None
def __init__(self, args): urls = None if args.input_file is not None: args.quiet = True with open(args.input_file, 'r') as input_file: urls = [] for url in input_file.readlines(): url = url.strip() urls.append(url if '://' in url else 'http://' + url) else: args.url = args.url.lower() if '://' not in args.url: args.url = 'http://' + args.url text_printer = Printer(args.verbosity) cache = Cache() cache.printer = text_printer self.options = { 'url': args.url, 'urls': urls, 'quiet': args.quiet, 'prefix': '', 'user_agent': args.user_agent, 'proxy': args.proxy, 'verbosity': args.verbosity, 'threads': 10, 'batch_size': 20, 'run_all': args.run_all, 'match_all': args.match_all, 'stop_after': args.stop_after, 'no_cache_load': args.no_cache_load, 'no_cache_save': args.no_cache_save, 'write_file': args.output_file, 'subdomains': args.subdomains } self.data = { 'cache': cache, 'results': Results(self.options), 'fingerprints': Fingerprints(), 'matcher': Match(), 'printer': text_printer, 'detected_cms': set(), 'error_pages': set(), 'requested': queue.Queue() } if self.options['write_file'] is not None: self.json_outputter = OutputJSON(self.options, self.data) self.data['printer'].print_logo() self.results = None
def __init__(self, args): self.options = { 'url': args.url, 'prefix': '', 'user_agent': args.user_agent, 'proxy': args.proxy, 'verbosity': args.verbosity, 'threads': 10, 'chunk_size': 10, # same as threads 'run_all': args.run_all, 'match_all': args.match_all, 'stop_after': args.stop_after, 'no_cache_load': args.no_cache_load, 'no_cache_save': args.no_cache_save, } self.data = { 'cache': Cache(), 'results': Results(self.options), 'fingerprints': Fingerprints(), 'matcher': Match(), 'colorizer': Color(), 'printer': Printer(args.verbosity, Color()), 'detected_cms': set(), 'error_pages': set(), 'queue': queue.Queue(), 'requested': queue.Queue() } self.data['results'].set_printer(self.data['printer']) self.data['requester'] = Requester(self.options, self.data)
def __init__(self, host, profile, verbose, desperate): self.plugins = self.load_plugins() self.host = host self.results = Results() self.cache = Cache() self.profile = Profile(profile) self.colorizer = Color() self.logs = Log() self.verbose = verbose self.check_url() self.redirect() self.cache.set_host(self.host) if desperate: self.desperate = Desperate() else: self.desperate = None
def __init__(self, host, profile, verbose): self.plugins = self.load_plugins() self.host = host self.results = Results() self.cache = Cache() self.profile = Profile(profile) self.check_url() self.redirect() self.colorizer = Color() self.logs = Log() self.verbose = verbose
def getPosts(self, uri, min=None, max=None, text=None): if (self.cache == None): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() else: if (uri!=self.cache.uri or self.cache.bad): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() min, max = self.getDates() if (not self.cache.bad): posts = self.cache.query() if (min!=None or max!=None or text!=None): posts = self.cache.filterPosts(posts, min, max, text) return posts else: self.alert('An exception ocurred parsing this URI') return None
def getPosts(self, uri, min=None, max=None, text=None): if (self.cache == None): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() else: if (uri != self.cache.uri or self.cache.bad): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() min, max = self.getDates() if (not self.cache.bad): posts = self.cache.query() if (min != None or max != None or text != None): posts = self.cache.filterPosts(posts, min, max, text) return posts else: self.alert('An exception ocurred parsing this URI') return None
def __init__(self, args): urls = None interactive = True if args.input_file is not None: interactive = False with open(args.input_file, 'r') as input_file: urls = [] for url in input_file.readlines(): u = url.strip() urls.append(u if '://' in u else 'http://' + u) elif '://' not in args.url: args.url = 'http://' + args.url self.options = { 'url': args.url, 'urls': urls, 'interactive': interactive, 'prefix': '', 'user_agent': args.user_agent, 'proxy': args.proxy, 'verbosity': args.verbosity, 'threads': 10, 'batch_size': 20, 'run_all': args.run_all, 'match_all': args.match_all, 'stop_after': args.stop_after, 'no_cache_load': args.no_cache_load, 'no_cache_save': args.no_cache_save, 'write_file': args.output_file } self.data = { 'cache': Cache(), 'results': Results(self.options), 'fingerprints': Fingerprints(), 'matcher': Match(), 'colorizer': Color(), 'printer': Printer(args.verbosity, Color()), 'detected_cms': set(), 'error_pages': set(), 'requested': queue.Queue() } if self.options['write_file'] is not None: self.json_outputter = OutputJSON(self.options, self.data)
def __init__(self, host, cache, results): super().__init__(requests) self.threads = 10 self.queue = queue.Queue() self.requested = queue.Queue() self.workers = [] self.host = host # set cache if not cache: self.cache = Cache() else: self.cache = cache # set results if not results: self.results = Results() else: self.results = results
def get_folder_jobs(folder_name): expected = {'daily', 'hourly', 'every10mins'} assert folder_name in expected, f'folder_name must be one of: {expected}' path = f'./scheduled/{folder_name}' jobs = {} heuristics = Cache.get('scheduled_jobs', {}) for file_name in os.listdir(path): if not is_job_to_run(f'{path}/{file_name}'): continue job_name = file_name.replace('.py', '') import_path_string = f'scheduled.{folder_name}.{job_name}' module = __import__(import_path_string, globals(), locals(), ['main', 'problem_data', 'fix']) jobs[job_name] = module # jobs.get_heuristics = lambda: Cache.get(job_name, {'name': job_name, 'num_runs': 0, 'avg_duration': 0}) # jobs.set_heuristics = lambda: Cache.get(job_name, {'name': job_name, 'num_runs': 0, 'avg_duration': 0}) return jobs
def main(): # a = Cache.set('abc', {'a':1}) # b = Cache.set(('abc', 123), {'a':1}) def func_value(): print('waiting...') time.sleep(5) print('done!') return 12 # c = Cache.define('defg', func_value, calc_on_func_change=func_value) d = Cache.define( 'defg', func_value, calc_if_true=lambda: prompt('Do you want to re-calculate the value?', boolean=True)) return
def fix_event_artists_field(): events = Cache.get('fix_event_artists_field') # events = sf.select(""" # SELECT Id, EventName__c, PrimaryHeadlinerArtist__c, Artists__c, Venue__r.VenueName__c, Venue__r.BillingCity, EventYear__c # , IsTouringApp__c, TourLeg__r.Tour__r.TourName__c, TourLeg__r.LegName__c, TourLeg__r.Tour__r.AppScope__c # , (SELECT Id, Type__c, Artist__r.Name, Rank__c FROM Deals__r WHERE RecordType.Name = 'Artist' AND StageType__c = 'Plan') # FROM Event__c # CreatedDate >= THIS_YEAR # """) def iterator(): for item in events: snapshot = item.copy() item.Deals__r.sort(key=lambda x: ( (0 if x.Type__c == 'Primary Headliner' else 1 if x.Type__c == 'Co-Headliner' else 2), x.Rank__c)) primaryheadlinerlist = [ x for x in item.Deals__r if x['Type__c'] == 'Primary Headliner' ] # headlinername = primaryheadlinerlist[0].Artist__r.Name if len(primaryheadlinerlist) > 0 else '' # item.PrimaryHeadlinerArtist__c = headlinername new_artists = '\r\n'.join((x.Artist__r.Name for x in item.Deals__r if x.Artist__r is not None)) if new_artists != item.Artists__c: item.OldArtists = item.Artists__c item.Artists__c = new_artists yield item new_events = list(iterator()) pdh.to_excel(new_events, 'Events with incorrect Artist(s) field') # with sf.bypass_settings(): # sf.update(new_events) return
class Wig(): def __init__(self, host, profile, verbose, desperate, plugin_name=None): self.plugins = self.load_plugins() self.host = host self.results = Results() self.cache = Cache() self.profile = Profile(profile) self.colorizer = Color() self.logs = Log() self.verbose = verbose self.plugin_name = plugin_name self.check_url() self.redirect() self.cache.set_host(self.host) if desperate: self.desperate = Desperate() else: self.desperate = None def redirect(self): # detects redirection if this happend try: r = requests.get(self.host, verify=False) except: print("Invalid URL or host not found. Exiting...") sys.exit(0) if not r.url == self.host: # ensure that sub-folders and files are removed parts = r.url.split('//') http, url = parts[0:2] # remove subfolders and/or files # http://example.com/test -> http://example.com/ if '/' in url: redirected = http + '//' + url.split('/')[0] + '/' else: redirected = http + '//' + url + '/' self.host = redirected def check_url(self): # adds http:// to input if not present if not self.host.startswith("http"): self.host = "http://" + self.host def load_plugins(self): # load all the plugins listed in plugins/__init__.py all_plugins = [] for p in plugins.__all__: plugin_path = "plugins." + p __import__(plugin_path) all_plugins.append(sys.modules[plugin_path]) return all_plugins def run(self): t = time.time() num_fps = 0 num_plugins = 0 # loops over all the plugins loaded for plugin in self.plugins: # a loaded plugin might have more than one plugin, so 'ps' is a list ps = plugin.get_instances(self.host, self.cache, self.results) num_plugins += len(ps) for p in ps: # give a status of which plugin is run print(p.name, end=" \r") sys.stdout.flush() # applies the choosen profile by removing fingerprints from the # fingerprint set if these do not match the choosen profile p.set_profile(self.profile, self.plugin_name) # the main plugin method p.run() num_fps += p.get_num_fps() # check if running desperate mode. if self.desperate: # add the plugins fingerprints to the global fingerprint database self.desperate.add_fingerprints( p.get_items_for_desperate_mode()) # add logs self.logs.add(p.get_logs()) if self.desperate: self.desperate.set_cache(self.cache) self.desperate.run() for i in self.desperate.get_matches(): self.results.add('Desperate', i['cms'], i, i['count']) # check the response headers for information ch = CheckHeaders(self.cache, self.results, self.logs) ch.run() run_time = "%.1f" % (time.time() - t) num_urls = self.cache.get_num_urls() status = "Time: %s sec | Plugins: %s | Urls: %s | Fingerprints: %s" % ( run_time, num_plugins, num_urls, num_fps) bar = "_" * len(status) self.results.set_width(len(status)) print(self.results) print(bar) print(status + "\n") if self.verbose: print(bar) print(self.logs)
class Buxon(GtkUI): def clear(self): """ Clear all GTK components on Buxon """ #tree self.treeTranslator = {} for column in self.treeView.get_columns(): self.treeView.remove_column(column) #text self.text.get_buffer().set_text('') def clearSearchForm(self): """ Clear search form """ widgets.get_widget('searchInput').set_text('') widgets.get_widget('fromEntry').set_text('01/01/1995') widgets.get_widget('toEntry').set_text('31/31/2010') def showPost(self): """ Show post selected at gtk.TreeView """ selection = self.treeView.get_selection() (model, iter) = selection.get_selected() uri = model.get_value(iter, 0) author, authorUri, listName, listUri, title, date, content = self.cache.getPost( uri) self.messageBar('loaded post ' + uri) self.writePost(uri, author, authorUri, listName, listUri, title, date, content) def writePost(self, uri, author=None, authorUri='', listName=None, listUri='', title='', date='', content=''): """ Write a post on the gtkTextView @param uri: post uri @param author: author's name @param authorUri: author's uri @param listName: mailing list's name @param listUri: mailing list's uri @param title: post subject @param date: post date @param content: post body """ PANGO_SCALE = 1024 buffer = self.text.get_buffer() buffer.set_text('') iter = buffer.get_iter_at_offset(0) buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Post URI: \t', 'bold') buffer.insert_with_tags_by_name(iter, uri, 'monospace') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'From: \t', 'bold') if (author == None): buffer.insert_with_tags_by_name(iter, authorUri, 'monospace') else: buffer.insert(iter, author) buffer.insert(iter, ' <') buffer.insert_with_tags_by_name(iter, authorUri, 'monospace') buffer.insert(iter, '>') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'To: \t\t', 'bold') if (listName == None): buffer.insert_with_tags_by_name(iter, listUri, 'monospace') else: buffer.insert(iter, listName) buffer.insert(iter, ' <') buffer.insert_with_tags_by_name(iter, listUri, 'monospace') buffer.insert(iter, '>') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Subject: \t', 'bold') buffer.insert(iter, title) buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Date: \t', 'bold') buffer.insert(iter, date) buffer.insert(iter, '\n\n') buffer.insert_with_tags_by_name(iter, content, 'wrap_mode') buffer.insert(iter, '\n') def getDates(self): """ Get selected dates @return: dates @rtype: tuple """ #min date fromDate = widgets.get_widget('fromEntry').get_text().split('/') min = float(fromDate[2]) * 10000000000 min += float(fromDate[1]) * 100000000 min += float(fromDate[0]) * 1000000 #max date toDate = widgets.get_widget('toEntry').get_text().split('/') max = float(toDate[2]) * 10000000000 max += float(toDate[1]) * 100000000 max += float(toDate[0]) * 1000000 return min, max def getPosts(self, uri, min=None, max=None, text=None): """ Get mailing list's posts @param uri: mailing list's uri @param min: min date @param max: max date @param text: text to search """ if (self.cache == None): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() else: if (uri != self.cache.uri or self.cache.bad): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() min, max = self.getDates() if (not self.cache.bad): posts = self.cache.query() if (posts == None): self.messageBar('unknow problem parsing RDF at ' + self.uri) return None else: if (min != None or max != None or text != None): posts = self.cache.filterPosts(posts, min, max, text) return posts else: self.alert('An exception ocurred parsing this URI') return None def drawTree(self, posts): """ Draw post on gtk.TreeView @param posts: posts @type posts: tuple """ if (posts != None and len(posts) > 0): #create tree self.treeStore = gtk.TreeStore(str, str) self.treeView.set_model(self.treeStore) #append items parent = None for (post, title, date, creator, content, parent) in posts: self.treeTranslator[post] = self.treeStore.append( self.__getParent(parent), [str(post), str(title)]) #print 'drawing post', post, 'on tree' #and show it treeColumn = gtk.TreeViewColumn('Posts') self.treeView.append_column(treeColumn) cell = gtk.CellRendererText() treeColumn.pack_start(cell, True) treeColumn.add_attribute(cell, 'text', 1) treeColumn.set_sort_column_id(0) self.messageBar('loaded ' + self.cache.uri) else: self.messageBar('none posts founded at ' + self.cache.uri) def __getParent(self, uri): """ Get the parent post @param uri: post uri @return: parent uri """ if (uri in self.treeTranslator): return self.treeTranslator[uri] else: return None def messageBar(self, text): """ Write a message on the status bar @param text: text """ self.statusbar.push(0, text) def insertBufferTag(self, buffer, name, property, value): """ Insert a new tag on buffer @param buffer: buffer @param name: tag name @param property: property to customize @param value: property value """ tag = gtk.TextTag(name) tag.set_property(property, value) table = buffer.get_tag_table() table.add(tag) def getUri(self): """ Get actual URI @return: actual uri """ if (self.cache == None): return None else: return self.cache.uri def destroy(self): """ Destoy all the infraestructure """ print 'Exiting...' #if (self.cache != None): # self.cache.dump(self.base + 'buxon.cache') gtk.main_quit() return gtk.FALSE def main(self, uri=None): """ Main bucle @param uri: uri """ #widgets self.treeView = widgets.get_widget('postsTree') self.text = widgets.get_widget('buxonTextView') buffer = self.text.get_buffer() self.insertBufferTag(buffer, 'bold', 'weight', pango.WEIGHT_BOLD) self.insertBufferTag(buffer, 'monospace', 'family', 'monospace') self.insertBufferTag(buffer, 'wrap_mode', 'wrap_mode', gtk.WRAP_WORD) self.input = widgets.get_widget('urlInput') self.statusbar = widgets.get_widget('buxonStatusbar') self.messageBar('ready') #main window self.window = widgets.get_widget('buxon') self.window.set_icon_from_file(self.base + 'includes/images/rdf.ico') self.window.show() if (uri != None): self.input.set_text(uri) gtk.main() def __init__(self, base='./'): """ Buxon constructor @param base: base directory """ GtkUI.__init__(self, 'buxon', base) self.base = base self.cache = None self.treeTranslator = {}
class Buxon: def clear(self): #tree self.treeTranslator = {} for column in self.treeView.get_columns(): self.treeView.remove_column(column) #text self.text.get_buffer().set_text('') def clearSearchForm(self): widgets.get_widget('searchInput').set_text('') widgets.get_widget('fromEntry').set_text('01/01/1995') widgets.get_widget('toEntry').set_text('31/31/2010') def showPost(self): selection = self.treeView.get_selection() (model, iter) = selection.get_selected() uri = model.get_value(iter, 0) author, authorUri, listName, listUri, title, date, content = self.cache.getPost(uri) self.messageBar('loaded post ' + uri) self.writePost(uri, author, authorUri, listName, listUri, title, date, content) def writePost(self, uri, author=None, authorUri='', listName=None, listUri='', title='', date='', content=''): PANGO_SCALE = 1024 buffer = self.text.get_buffer() buffer.set_text('') iter = buffer.get_iter_at_offset(0) buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Post URI: \t', 'bold') buffer.insert_with_tags_by_name(iter, uri, 'monospace') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'From: \t', 'bold') if (author == None): buffer.insert_with_tags_by_name(iter, authorUri, 'monospace') else: buffer.insert(iter, author) buffer.insert(iter, ' <') buffer.insert_with_tags_by_name(iter, authorUri, 'monospace') buffer.insert(iter, '>') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'To: \t\t', 'bold') if (listName == None): buffer.insert_with_tags_by_name(iter, listUri, 'monospace') else: buffer.insert(iter, listName) buffer.insert(iter, ' <') buffer.insert_with_tags_by_name(iter, listUri, 'monospace') buffer.insert(iter, '>') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Subject: \t', 'bold') buffer.insert(iter, title) buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Date: \t', 'bold') buffer.insert(iter, date) buffer.insert(iter, '\n\n') buffer.insert_with_tags_by_name(iter, content, 'wrap_mode') buffer.insert(iter, '\n') def getDates(self): #min date fromDate = widgets.get_widget('fromEntry').get_text().split('/') min = float(fromDate[2]) * 10000000000 min += float(fromDate[1]) * 100000000 min += float(fromDate[0]) * 1000000 #max date toDate = widgets.get_widget('toEntry').get_text().split('/') max = float(toDate[2]) * 10000000000 max += float(toDate[1]) * 100000000 max += float(toDate[0]) * 1000000 return min, max def getPosts(self, uri, min=None, max=None, text=None): if (self.cache == None): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() else: if (uri!=self.cache.uri or self.cache.bad): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() min, max = self.getDates() if (not self.cache.bad): posts = self.cache.query() if (min!=None or max!=None or text!=None): posts = self.cache.filterPosts(posts, min, max, text) return posts else: self.alert('An exception ocurred parsing this URI') return None def drawTree(self, posts): if (posts!=None and len(posts)>0): #create tree self.treeStore = gtk.TreeStore(str, str) self.treeView.set_model(self.treeStore) #append items parent = None for (post, title, date, creator, content, parent) in posts: self.treeTranslator[post] = self.treeStore.append(self.__getParent(parent), [str(post), str(title)]) print 'drawing post', post, 'on tree' #and show it treeColumn = gtk.TreeViewColumn('Posts') self.treeView.append_column(treeColumn) cell = gtk.CellRendererText() treeColumn.pack_start(cell, True) treeColumn.add_attribute(cell, 'text', 1) treeColumn.set_sort_column_id(0) self.messageBar('loaded ' + self.cache.uri) else: self.messageBar('none posts founded at ' + self.cache.uri) def __getParent(self, uri): if (uri in self.treeTranslator): return self.treeTranslator[uri] else: return None def alert(self, text): self.alertWindow = gtk.Window(gtk.WINDOW_POPUP) self.alertWindow.set_position(gtk.WIN_POS_CENTER_ALWAYS) self.alertWindow.set_modal(True) self.alertWindow.set_resizable(False) self.alertWindow.set_border_width(0) vbox = gtk.VBox(False, 5) vbox.set_border_width(10) self.alertWindow.add(vbox) vbox.show() align1 = gtk.Alignment(0.5, 0.5, 0, 0) vbox.pack_start(align1, False, False, 5) align1.show() label = gtk.Label(text) align1.add(label) label.show() align2 = gtk.Alignment(0.5, 0.5, 0, 0) vbox.pack_start(align2, False, False, 5) align2.show() button = gtk.Button('OK') button.connect('clicked', destroyAlert, 'cool button') align2.add(button) button.show() self.alertWindow.show() def messageBar(self, text): self.statusbar.push(0, text) def insertBufferTag(self, buffer, name, property, value): tag = gtk.TextTag(name) tag.set_property(property, value) table = buffer.get_tag_table() table.add(tag) def getUri(self): if (self.cache == None): return None else: return self.cache.uri def main(self, uri=None): if (uri != None): self.input.set_text(uri) gtk.main() def __init__(self): self.cache = None self.treeTranslator = {} #widgets self.treeView = widgets.get_widget('postsTree') self.text = widgets.get_widget('buxonTextView') buffer = self.text.get_buffer() self.insertBufferTag(buffer, 'bold', 'weight', pango.WEIGHT_BOLD) self.insertBufferTag(buffer, 'monospace', 'family', 'monospace') self.insertBufferTag(buffer, 'wrap_mode', 'wrap_mode', gtk.WRAP_WORD) self.input = widgets.get_widget('urlInput') self.statusbar = widgets.get_widget('buxonStatusbar') self.messageBar('ready') #main window self.window = widgets.get_widget('buxon') self.window.set_icon_from_file('includes/rdf.ico') self.window.show()
class Buxon: def clear(self): #tree self.treeTranslator = {} for column in self.treeView.get_columns(): self.treeView.remove_column(column) #text self.text.get_buffer().set_text('') def clearSearchForm(self): widgets.get_widget('searchInput').set_text('') widgets.get_widget('fromEntry').set_text('01/01/1995') widgets.get_widget('toEntry').set_text('31/31/2010') def showPost(self): selection = self.treeView.get_selection() (model, iter) = selection.get_selected() uri = model.get_value(iter, 0) author, authorUri, listName, listUri, title, date, content = self.cache.getPost( uri) self.messageBar('loaded post ' + uri) self.writePost(uri, author, authorUri, listName, listUri, title, date, content) def writePost(self, uri, author=None, authorUri='', listName=None, listUri='', title='', date='', content=''): PANGO_SCALE = 1024 buffer = self.text.get_buffer() buffer.set_text('') iter = buffer.get_iter_at_offset(0) buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Post URI: \t', 'bold') buffer.insert_with_tags_by_name(iter, uri, 'monospace') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'From: \t', 'bold') if (author == None): buffer.insert_with_tags_by_name(iter, authorUri, 'monospace') else: buffer.insert(iter, author) buffer.insert(iter, ' <') buffer.insert_with_tags_by_name(iter, authorUri, 'monospace') buffer.insert(iter, '>') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'To: \t\t', 'bold') if (listName == None): buffer.insert_with_tags_by_name(iter, listUri, 'monospace') else: buffer.insert(iter, listName) buffer.insert(iter, ' <') buffer.insert_with_tags_by_name(iter, listUri, 'monospace') buffer.insert(iter, '>') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Subject: \t', 'bold') buffer.insert(iter, title) buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Date: \t', 'bold') buffer.insert(iter, date) buffer.insert(iter, '\n\n') buffer.insert_with_tags_by_name(iter, content, 'wrap_mode') buffer.insert(iter, '\n') def getDates(self): #min date fromDate = widgets.get_widget('fromEntry').get_text().split('/') min = float(fromDate[2]) * 10000000000 min += float(fromDate[1]) * 100000000 min += float(fromDate[0]) * 1000000 #max date toDate = widgets.get_widget('toEntry').get_text().split('/') max = float(toDate[2]) * 10000000000 max += float(toDate[1]) * 100000000 max += float(toDate[0]) * 1000000 return min, max def getPosts(self, uri, min=None, max=None, text=None): if (self.cache == None): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() else: if (uri != self.cache.uri or self.cache.bad): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() min, max = self.getDates() if (not self.cache.bad): posts = self.cache.query() if (min != None or max != None or text != None): posts = self.cache.filterPosts(posts, min, max, text) return posts else: self.alert('An exception ocurred parsing this URI') return None def drawTree(self, posts): if (posts != None and len(posts) > 0): #create tree self.treeStore = gtk.TreeStore(str, str) self.treeView.set_model(self.treeStore) #append items parent = None for (post, title, date, creator, content, parent) in posts: self.treeTranslator[post] = self.treeStore.append( self.__getParent(parent), [str(post), str(title)]) print 'drawing post', post, 'on tree' #and show it treeColumn = gtk.TreeViewColumn('Posts') self.treeView.append_column(treeColumn) cell = gtk.CellRendererText() treeColumn.pack_start(cell, True) treeColumn.add_attribute(cell, 'text', 1) treeColumn.set_sort_column_id(0) self.messageBar('loaded ' + self.cache.uri) else: self.messageBar('none posts founded at ' + self.cache.uri) def __getParent(self, uri): if (uri in self.treeTranslator): return self.treeTranslator[uri] else: return None def alert(self, text): self.alertWindow = gtk.Window(gtk.WINDOW_POPUP) self.alertWindow.set_position(gtk.WIN_POS_CENTER_ALWAYS) self.alertWindow.set_modal(True) self.alertWindow.set_resizable(False) self.alertWindow.set_border_width(0) vbox = gtk.VBox(False, 5) vbox.set_border_width(10) self.alertWindow.add(vbox) vbox.show() align1 = gtk.Alignment(0.5, 0.5, 0, 0) vbox.pack_start(align1, False, False, 5) align1.show() label = gtk.Label(text) align1.add(label) label.show() align2 = gtk.Alignment(0.5, 0.5, 0, 0) vbox.pack_start(align2, False, False, 5) align2.show() button = gtk.Button('OK') button.connect('clicked', destroyAlert, 'cool button') align2.add(button) button.show() self.alertWindow.show() def messageBar(self, text): self.statusbar.push(0, text) def insertBufferTag(self, buffer, name, property, value): tag = gtk.TextTag(name) tag.set_property(property, value) table = buffer.get_tag_table() table.add(tag) def getUri(self): if (self.cache == None): return None else: return self.cache.uri def main(self, uri=None): if (uri != None): self.input.set_text(uri) gtk.main() def __init__(self): self.cache = None self.treeTranslator = {} #widgets self.treeView = widgets.get_widget('postsTree') self.text = widgets.get_widget('buxonTextView') buffer = self.text.get_buffer() self.insertBufferTag(buffer, 'bold', 'weight', pango.WEIGHT_BOLD) self.insertBufferTag(buffer, 'monospace', 'family', 'monospace') self.insertBufferTag(buffer, 'wrap_mode', 'wrap_mode', gtk.WRAP_WORD) self.input = widgets.get_widget('urlInput') self.statusbar = widgets.get_widget('buxonStatusbar') self.messageBar('ready') #main window self.window = widgets.get_widget('buxon') self.window.set_icon_from_file('includes/rdf.ico') self.window.show()
def save_cookie(driver): cookies = driver.get_cookies() Cache.set(('browser_cookie', driver.current_url), cookies)
def load_cookie(driver): cookies = Cache.get(('browser_cookie', driver.current_url)) for cookie in cookies: driver.add_cookie(cookie)
class Buxon(GtkUI): def clear(self): """ Clear all GTK components on Buxon """ #tree self.treeTranslator = {} for column in self.treeView.get_columns(): self.treeView.remove_column(column) #text self.text.get_buffer().set_text('') def clearSearchForm(self): """ Clear search form """ widgets.get_widget('searchInput').set_text('') widgets.get_widget('fromEntry').set_text('01/01/1995') widgets.get_widget('toEntry').set_text('31/31/2010') def showPost(self): """ Show post selected at gtk.TreeView """ selection = self.treeView.get_selection() (model, iter) = selection.get_selected() uri = model.get_value(iter, 0) author, authorUri, listName, listUri, title, date, content = self.cache.getPost(uri) self.messageBar('loaded post ' + uri) self.writePost(uri, author, authorUri, listName, listUri, title, date, content) def writePost(self, uri, author=None, authorUri='', listName=None, listUri='', title='', date='', content=''): """ Write a post on the gtkTextView @param uri: post uri @param author: author's name @param authorUri: author's uri @param listName: mailing list's name @param listUri: mailing list's uri @param title: post subject @param date: post date @param content: post body """ PANGO_SCALE = 1024 buffer = self.text.get_buffer() buffer.set_text('') iter = buffer.get_iter_at_offset(0) buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Post URI: \t', 'bold') buffer.insert_with_tags_by_name(iter, uri, 'monospace') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'From: \t', 'bold') if (author == None): buffer.insert_with_tags_by_name(iter, authorUri, 'monospace') else: buffer.insert(iter, author) buffer.insert(iter, ' <') buffer.insert_with_tags_by_name(iter, authorUri, 'monospace') buffer.insert(iter, '>') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'To: \t\t', 'bold') if (listName == None): buffer.insert_with_tags_by_name(iter, listUri, 'monospace') else: buffer.insert(iter, listName) buffer.insert(iter, ' <') buffer.insert_with_tags_by_name(iter, listUri, 'monospace') buffer.insert(iter, '>') buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Subject: \t', 'bold') buffer.insert(iter, title) buffer.insert(iter, '\n') buffer.insert_with_tags_by_name(iter, 'Date: \t', 'bold') buffer.insert(iter, date) buffer.insert(iter, '\n\n') buffer.insert_with_tags_by_name(iter, content, 'wrap_mode') buffer.insert(iter, '\n') def getDates(self): """ Get selected dates @return: dates @rtype: tuple """ #min date fromDate = widgets.get_widget('fromEntry').get_text().split('/') min = float(fromDate[2]) * 10000000000 min += float(fromDate[1]) * 100000000 min += float(fromDate[0]) * 1000000 #max date toDate = widgets.get_widget('toEntry').get_text().split('/') max = float(toDate[2]) * 10000000000 max += float(toDate[1]) * 100000000 max += float(toDate[0]) * 1000000 return min, max def getPosts(self, uri, min=None, max=None, text=None): """ Get mailing list's posts @param uri: mailing list's uri @param min: min date @param max: max date @param text: text to search """ if (self.cache == None): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() else: if (uri!=self.cache.uri or self.cache.bad): pb = LoadProgressBar() self.cache = Cache(uri, pb) pb.destroy() min, max = self.getDates() if (not self.cache.bad): posts = self.cache.query() if (posts == None): self.messageBar('unknow problem parsing RDF at ' + self.uri) return None else: if (min!=None or max!=None or text!=None): posts = self.cache.filterPosts(posts, min, max, text) return posts else: self.alert('An exception ocurred parsing this URI') return None def drawTree(self, posts): """ Draw post on gtk.TreeView @param posts: posts @type posts: tuple """ if (posts!=None and len(posts)>0): #create tree self.treeStore = gtk.TreeStore(str, str) self.treeView.set_model(self.treeStore) #append items parent = None for (post, title, date, creator, content, parent) in posts: self.treeTranslator[post] = self.treeStore.append(self.__getParent(parent), [str(post), str(title)]) #print 'drawing post', post, 'on tree' #and show it treeColumn = gtk.TreeViewColumn('Posts') self.treeView.append_column(treeColumn) cell = gtk.CellRendererText() treeColumn.pack_start(cell, True) treeColumn.add_attribute(cell, 'text', 1) treeColumn.set_sort_column_id(0) self.messageBar('loaded ' + self.cache.uri) else: self.messageBar('none posts founded at ' + self.cache.uri) def __getParent(self, uri): """ Get the parent post @param uri: post uri @return: parent uri """ if (uri in self.treeTranslator): return self.treeTranslator[uri] else: return None def messageBar(self, text): """ Write a message on the status bar @param text: text """ self.statusbar.push(0, text) def insertBufferTag(self, buffer, name, property, value): """ Insert a new tag on buffer @param buffer: buffer @param name: tag name @param property: property to customize @param value: property value """ tag = gtk.TextTag(name) tag.set_property(property, value) table = buffer.get_tag_table() table.add(tag) def getUri(self): """ Get actual URI @return: actual uri """ if (self.cache == None): return None else: return self.cache.uri def destroy(self): """ Destoy all the infraestructure """ print 'Exiting...' #if (self.cache != None): # self.cache.dump(self.base + 'buxon.cache') gtk.main_quit() return gtk.FALSE def main(self, uri=None): """ Main bucle @param uri: uri """ #widgets self.treeView = widgets.get_widget('postsTree') self.text = widgets.get_widget('buxonTextView') buffer = self.text.get_buffer() self.insertBufferTag(buffer, 'bold', 'weight', pango.WEIGHT_BOLD) self.insertBufferTag(buffer, 'monospace', 'family', 'monospace') self.insertBufferTag(buffer, 'wrap_mode', 'wrap_mode', gtk.WRAP_WORD) self.input = widgets.get_widget('urlInput') self.statusbar = widgets.get_widget('buxonStatusbar') self.messageBar('ready') #main window self.window = widgets.get_widget('buxon') self.window.set_icon_from_file(self.base + 'includes/images/rdf.ico') self.window.show() if (uri != None): self.input.set_text(uri) gtk.main() def __init__(self, base='./'): """ Buxon constructor @param base: base directory """ GtkUI.__init__(self, 'buxon', base) self.base = base self.cache = None self.treeTranslator = {}
ap.add_argument("-d", "--debug", dest="log_level", action="store_const", const=logging.DEBUG, default=logging.INFO, help="Show debug log messages") args = ap.parse_args() logging.getLogger().setLevel(args.log_level) ngrok_service = NgrokService() ngrok_service.start(settings.API_PORT) ngrok_url = ngrok_service.get_url() postback_api = PostbackApi(ngrok_url) catappult_api = CatappultApi(settings.CATAPPULT_API_HOST, Cache()) one_step_validator = OneStepPaymentValidator(postback_api, catappult_api) logging.info("Testing One-Step Payment...") purchase_info = get_purchase_info(ngrok_url) try: is_verified = one_step_validator.validate(purchase_info) except ValidatorException as e: logging.error(e) is_verified = False if is_verified: logging.info("Transaction: '{}' was successfully verified".format( purchase_info)) else: logging.error(
class Wig(): def __init__(self, host, profile, verbose): self.plugins = self.load_plugins() self.host = host self.results = Results() self.cache = Cache() self.profile = Profile(profile) self.check_url() self.redirect() self.colorizer = Color() self.logs = Log() self.verbose = verbose def redirect(self): # detects redirection if this happend r = requests.get(self.host, verify=False) if not r.url == self.host: # ensure that sub-folders and files are removed parts = r.url.split('//') http, url = parts[0:2] # remove subfolders and/or files # http://example.com/test -> http://example.com/ if '/' in url: redirected = http + '//' + url.split('/')[0] + '/' else: redirected = http + '//' + url + '/' self.host = redirected def check_url(self): # adds http:// to input if not present if not self.host.startswith("http"): self.host = "http://" + self.host def load_plugins(self): # load all the plugins listed in plugins/__init__.py all_plugins = [] for p in plugins.__all__: plugin_path = "plugins." + p __import__(plugin_path) all_plugins.append(sys.modules[plugin_path]) return all_plugins def run(self): t = time.time() num_fps = 0 num_plugins = 0 # loops over all the plugins loaded for plugin in self.plugins: # a loaded plugin might have more than one plugin, so 'ps' is a list ps = plugin.get_instances(self.host, self.cache, self.results) num_plugins += len(ps) for p in ps: # give a status of which plugin is run print(p.name, end=" \r") sys.stdout.flush() # applies the choosen profile by removing fingerprints from the # fingerprint set if these do not match the choosen profile p.set_profile(self.profile) # the main plugin method p.run() num_fps += p.get_num_fps() # add logs self.logs.add( p.get_logs() ) run_time = "%.1f" % (time.time() - t) num_urls = self.cache.get_num_urls() status = "Time: %s sec | Plugins: %s | Urls: %s | Fingerprints: %s" % (run_time, num_plugins, num_urls, num_fps) bar = "_"*len(status) self.results.set_width(len(status)) print(self.results) print(bar) print(status + "\n") if self.verbose: print(bar) print(self.logs)
class Wig(): def __init__(self, host, profile, verbose, desperate, plugin_name=None): self.plugins = self.load_plugins() self.host = host self.results = Results() self.cache = Cache() self.profile = Profile(profile) self.colorizer = Color() self.logs = Log() self.verbose = verbose self.plugin_name = plugin_name self.check_url() self.redirect() self.cache.set_host(self.host) if desperate: self.desperate = Desperate() else: self.desperate = None def redirect(self): # detects redirection if this happend try: r = requests.get(self.host, verify=False) except: print("Invalid URL or host not found. Exiting...") sys.exit(0) if not r.url == self.host: # ensure that sub-folders and files are removed parts = r.url.split('//') http, url = parts[0:2] # remove subfolders and/or files # http://example.com/test -> http://example.com/ if '/' in url: redirected = http + '//' + url.split('/')[0] + '/' else: redirected = http + '//' + url + '/' self.host = redirected def check_url(self): # adds http:// to input if not present if not self.host.startswith("http"): self.host = "http://" + self.host def load_plugins(self): # load all the plugins listed in plugins/__init__.py all_plugins = [] for p in plugins.__all__: plugin_path = "plugins." + p __import__(plugin_path) all_plugins.append(sys.modules[plugin_path]) return all_plugins def run(self): t = time.time() num_fps = 0 num_plugins = 0 # loops over all the plugins loaded for plugin in self.plugins: # a loaded plugin might have more than one plugin, so 'ps' is a list ps = plugin.get_instances(self.host, self.cache, self.results) num_plugins += len(ps) for p in ps: # give a status of which plugin is run print(p.name, end=" \r") sys.stdout.flush() # applies the choosen profile by removing fingerprints from the # fingerprint set if these do not match the choosen profile p.set_profile(self.profile, self.plugin_name) # the main plugin method p.run() num_fps += p.get_num_fps() # check if running desperate mode. if self.desperate: # add the plugins fingerprints to the global fingerprint database self.desperate.add_fingerprints(p.get_items_for_desperate_mode()) # add logs self.logs.add( p.get_logs() ) if self.desperate: self.desperate.set_cache(self.cache) self.desperate.run() for i in self.desperate.get_matches(): self.results.add('Desperate', i['cms'], i, i['count']) # check the response headers for information ch = CheckHeaders(self.cache, self.results, self.logs) ch.run() run_time = "%.1f" % (time.time() - t) num_urls = self.cache.get_num_urls() status = "Time: %s sec | Plugins: %s | Urls: %s | Fingerprints: %s" % (run_time, num_plugins, num_urls, num_fps) bar = "_"*len(status) self.results.set_width(len(status)) print(self.results) print(bar) print(status + "\n") if self.verbose: print(bar) print(self.logs)
def reset(self): self.data['results'] = Results(self.options) self.data['cache'] = Cache()
def get_job_heuristics(): return Cache.get('scheduled_jobs', {})
def main(): # test = sf.select('''SELECT Id, Event__r.Venue__c, Event__r.Venue__r.VenueName__c, # Event__r.Venue__r.BillingCity, Event__r.Venue__r.BillingCountry, Type__c,RateType__c, # AmountRate__c, BeforeAfter__c # FROM Deduction__c LIMIT 50000''', mode='bulk', return_type='dataframe') data = sf.fselect(""" --sql SELECT Event__c , Event__r.Venue__c AS VenueId , Event__r.Venue__r.VenueName__c AS VenueName , Event__r.Venue__r.BillingCity + ', ' + Event__r.Venue__r.BillingCountryCode AS City , Event__r.Venue__r.BillingCountry AS Country , Type__c , RateType__c , AmountRate__c , BeforeAfter__c FROM Deduction__c WHERE ApplyTicketScale__c NOT IN ('Selected Tickets','Other Amount') AND StageType__c = Event__r.StatusStage__c AND Type__c != 'FMF/Venue Charge' AND Event__r.Status__c IN ('Confirmed','Flash Started','Flash Complete') -- AND RateType__c != 'Flat' -- AND Event__r.Venue__r.VenueName__c = 'The Wiltern' -- LIMIT 50000 --endsql """, mode='bulk', content_type='JSON') Cache.set('deductionsanalysis', data) # data = Cache.get('deductionsanalysis') by_event_id = {} venue_ids = set() venue_fields = {} venue_counts = {} for i, item in enumerate(data): if isinstance(item.VenueId, SalesforceFormulaParser.NullValue): continue curr_list = by_event_id.get(item.Event__c, []) curr_list.append(Deduction(item.VenueId, item.VenueName or '', item.BeforeAfter__c or '', item.Type__c or '', item.RateType__c or '', item.AmountRate__c or 0)) by_event_id[item.Event__c] = curr_list venue_ids.add(item.VenueId) venue_fields[item.VenueId] = Venue(item.VenueName, item.City, item.Country) sets = [] sets_by_venue = {} for lst in by_event_id.values(): new_set = tuple(sorted(lst, key=lambda item: (item.VenueId, item.VenueName, item.BeforeAfter__c, item.Type__c, item.RateType__c, item.AmountRate__c))) venue_id = new_set[0].VenueId if venue_id not in sets_by_venue: sets_by_venue[venue_id] = set() venue_counts[venue_id] = 0 sets.append(new_set) sets_by_venue[venue_id].add(new_set) venue_counts[venue_id] += 1 counter = {} for tup in sets: val = counter.get(tup, 0) + 1 counter[tup] = val top_n = 3 top_n_sets_by_venue = { k: sorted([(counter[s], s) for s in v], key=lambda x: x[0], reverse=True)[:top_n] for k, v in sets_by_venue.items() } top_n_sets_by_venue_text = { k: [set_to_text(x[1], x[0]) for x in v] for k, v in top_n_sets_by_venue.items() } def get_n(venue_id, n): lst = top_n_sets_by_venue[venue_id] if len(lst) > n: return lst[n][0] return None def get_n_str(venue_id, n): lst = top_n_sets_by_venue_text[venue_id] if len(lst) > n: return lst[n] return '' def set_output(row): x = row.VenueId row['Total Events'] = venue_counts[x] percent = lambda x: None if not x else (x / row['Total Events'] * 100).round() row['Venue Name'] = venue_fields[x].Name row['City'] = venue_fields[x].City row['Country'] = venue_fields[x].Country row['1st'] = get_n(x, 0) row['1st %'] = percent(row['1st']) row['1st Deduction Setup'] = get_n_str(x, 0) row['2nd'] = get_n(x, 1) row['2nd %'] = percent(row['2nd']) row['2nd Deduction Setup'] = get_n_str(x, 1) row['3rd'] = get_n(x, 2) row['3rd %'] = percent(row['3rd']) row['3rd Deduction Setup'] = get_n_str(x, 2) return row df = ( pd.DataFrame({'VenueId': list(venue_ids)}) .apply(set_output, axis=1, result_type='expand') # .assign(**{ # 'Total Events': lambda df: df.VenueId.apply(lambda x: venue_counts[x]), # 'Venue Name': lambda df: df.VenueId.apply(lambda x: venue_fields[x].Name), # 'Venue City': lambda df: df.VenueId.apply(lambda x: venue_fields[x].City), # '1st': lambda df: df.VenueId.apply(lambda x: get_n(x, 0)), # '1st Deduction Setup': lambda df: df.VenueId.apply(lambda x: get_n_str(x, 0)), # '2nd': lambda df: df.VenueId.apply(lambda x: get_n(x, 1)), # '2nd Deduction Setup': lambda df: df.VenueId.apply(lambda x: get_n_str(x, 1)), # '3rd': lambda df: df.VenueId.apply(lambda x: get_n(x, 2)), # '3rd Deduction Setup': lambda df: df.VenueId.apply(lambda x: get_n_str(x, 2)), # }) .sort_values(['1st %', '1st'], ascending=False) ) pdh.to_excel(df, 'Common Deductions Analysis.xlsx', word_wrap=True) return
def eos_venues(): def get_eos_dupes_analysis(): records = sql.query(""" SELECT v.Id, v.Name, v.RomeId, v.IsActive FROM Venue v LEFT JOIN TownCity t ON v.TownCityId = t.Id LEFT JOIN Region r ON t.RegionId = r.Id LEFT JOIN Country c ON r.CountryId = c.Id WHERE c.Name IN ('United Kingdom') """) return pdh.find_duplicates(records, 'Name', None, ['Id', 'RomeId', 'IsActive']) def get_deletion_log(): return pd.DataFrame( sql.query(""" SELECT l.DeletedRecordId__c , l.MergedAccount__c , l.RecordTypeName__c , v1.Id AS EOSIdToDelete , v1.Name AS EOSNameToDelete , v2.Id AS EOSIdToMergeInto , v2.Name AS EOSNameToMergeInto , CASE WHEN v1.Id < v2.Id THEN 'Double check this - older ID being deleted' ELSE NULL END AS SpotCheck FROM RomeDeletionLog l LEFT JOIN Venue v1 ON v1.RomeId = l.DeletedRecordId__c AND l.RecordTypeName__c = 'Venue' LEFT JOIN Venue v2 ON v2.RomeId = l.MergedAccount__c AND l.RecordTypeName__c = 'Venue'WHERE v1.Id IS NOT NULL """)) eos_dupes = Cache.get('eosdupes', None) if eos_dupes is None or prompt('Re-run EOS Venue Dupe Analysis?', boolean=True): eos_dupes = get_eos_dupes_analysis() Cache.set('eosdupes', eos_dupes) deletion_log = get_deletion_log() deleted_ids = set(deletion_log.DeletedRecordId__c) def note(row): r1, r2 = row['RomeId'], row['m_RomeId'] if pdh.isnull(r1) and pdh.isnull(r2): return 'EOS Merge - No Rome links exist' if r1 in deleted_ids and r2 in deleted_ids: return 'Both RomeIds are deleted in Rome' elif r2 in deleted_ids: return 'EOS Merge RIGHT into LEFT - RIGHT Id is deleted in Rome' elif r1 in deleted_ids: return 'EOS Merge LEFT into RIGHT - LEFT Id is deleted in Rome' if pdh.isnull(r1) and pdh.notnull(r2): return 'EOS Merge LEFT into RIGHT' if pdh.isnull(r2) and pdh.notnull(r1): return 'EOS Merge RIGHT into LEFT' if r1 == r2: return 'EOS Merge - Both EOS Venues have the same RomeId... merge one into the other' if r1 not in deleted_ids and r2 not in deleted_ids: return 'ROME Merge first, then EOS can be merged' eos_dupes.query("`m_%` > 70", inplace=True) eos_dupes['Note'] = eos_dupes.apply(note, axis=1) eos_dupes[ 'RomeMergeLink'] = 'https://lne.my.salesforce.com/merge/accmergewizard.jsp?goNext=+Next+&cid=' + eos_dupes[ 'RomeId'] + '&cid=' + eos_dupes['m_RomeId'] output = { 'EOS Dupe Analysis': eos_dupes, 'RomeDeletionLog': deletion_log, } file_name = loc.uk_folder_path + '/Master Data - Output/Venue Dupe Analysis/EOS Venue Duplicate Analysis ' + datetime.now( ).strftime('%Y-%m-%d %H.%M') + '.xlsx' pdh.to_excel(output, file_name) return