def get_folder_jobs(folder_name): expected = {'daily', 'hourly', 'every10mins'} assert folder_name in expected, f'folder_name must be one of: {expected}' path = f'./scheduled/{folder_name}' jobs = {} heuristics = Cache.get('scheduled_jobs', {}) for file_name in os.listdir(path): if not is_job_to_run(f'{path}/{file_name}'): continue job_name = file_name.replace('.py', '') import_path_string = f'scheduled.{folder_name}.{job_name}' module = __import__(import_path_string, globals(), locals(), ['main', 'problem_data', 'fix']) jobs[job_name] = module # jobs.get_heuristics = lambda: Cache.get(job_name, {'name': job_name, 'num_runs': 0, 'avg_duration': 0}) # jobs.set_heuristics = lambda: Cache.get(job_name, {'name': job_name, 'num_runs': 0, 'avg_duration': 0}) return jobs
def fix_event_artists_field(): events = Cache.get('fix_event_artists_field') # events = sf.select(""" # SELECT Id, EventName__c, PrimaryHeadlinerArtist__c, Artists__c, Venue__r.VenueName__c, Venue__r.BillingCity, EventYear__c # , IsTouringApp__c, TourLeg__r.Tour__r.TourName__c, TourLeg__r.LegName__c, TourLeg__r.Tour__r.AppScope__c # , (SELECT Id, Type__c, Artist__r.Name, Rank__c FROM Deals__r WHERE RecordType.Name = 'Artist' AND StageType__c = 'Plan') # FROM Event__c # CreatedDate >= THIS_YEAR # """) def iterator(): for item in events: snapshot = item.copy() item.Deals__r.sort(key=lambda x: ( (0 if x.Type__c == 'Primary Headliner' else 1 if x.Type__c == 'Co-Headliner' else 2), x.Rank__c)) primaryheadlinerlist = [ x for x in item.Deals__r if x['Type__c'] == 'Primary Headliner' ] # headlinername = primaryheadlinerlist[0].Artist__r.Name if len(primaryheadlinerlist) > 0 else '' # item.PrimaryHeadlinerArtist__c = headlinername new_artists = '\r\n'.join((x.Artist__r.Name for x in item.Deals__r if x.Artist__r is not None)) if new_artists != item.Artists__c: item.OldArtists = item.Artists__c item.Artists__c = new_artists yield item new_events = list(iterator()) pdh.to_excel(new_events, 'Events with incorrect Artist(s) field') # with sf.bypass_settings(): # sf.update(new_events) return
def get_job_heuristics(): return Cache.get('scheduled_jobs', {})
def load_cookie(driver): cookies = Cache.get(('browser_cookie', driver.current_url)) for cookie in cookies: driver.add_cookie(cookie)
def eos_venues(): def get_eos_dupes_analysis(): records = sql.query(""" SELECT v.Id, v.Name, v.RomeId, v.IsActive FROM Venue v LEFT JOIN TownCity t ON v.TownCityId = t.Id LEFT JOIN Region r ON t.RegionId = r.Id LEFT JOIN Country c ON r.CountryId = c.Id WHERE c.Name IN ('United Kingdom') """) return pdh.find_duplicates(records, 'Name', None, ['Id', 'RomeId', 'IsActive']) def get_deletion_log(): return pd.DataFrame( sql.query(""" SELECT l.DeletedRecordId__c , l.MergedAccount__c , l.RecordTypeName__c , v1.Id AS EOSIdToDelete , v1.Name AS EOSNameToDelete , v2.Id AS EOSIdToMergeInto , v2.Name AS EOSNameToMergeInto , CASE WHEN v1.Id < v2.Id THEN 'Double check this - older ID being deleted' ELSE NULL END AS SpotCheck FROM RomeDeletionLog l LEFT JOIN Venue v1 ON v1.RomeId = l.DeletedRecordId__c AND l.RecordTypeName__c = 'Venue' LEFT JOIN Venue v2 ON v2.RomeId = l.MergedAccount__c AND l.RecordTypeName__c = 'Venue'WHERE v1.Id IS NOT NULL """)) eos_dupes = Cache.get('eosdupes', None) if eos_dupes is None or prompt('Re-run EOS Venue Dupe Analysis?', boolean=True): eos_dupes = get_eos_dupes_analysis() Cache.set('eosdupes', eos_dupes) deletion_log = get_deletion_log() deleted_ids = set(deletion_log.DeletedRecordId__c) def note(row): r1, r2 = row['RomeId'], row['m_RomeId'] if pdh.isnull(r1) and pdh.isnull(r2): return 'EOS Merge - No Rome links exist' if r1 in deleted_ids and r2 in deleted_ids: return 'Both RomeIds are deleted in Rome' elif r2 in deleted_ids: return 'EOS Merge RIGHT into LEFT - RIGHT Id is deleted in Rome' elif r1 in deleted_ids: return 'EOS Merge LEFT into RIGHT - LEFT Id is deleted in Rome' if pdh.isnull(r1) and pdh.notnull(r2): return 'EOS Merge LEFT into RIGHT' if pdh.isnull(r2) and pdh.notnull(r1): return 'EOS Merge RIGHT into LEFT' if r1 == r2: return 'EOS Merge - Both EOS Venues have the same RomeId... merge one into the other' if r1 not in deleted_ids and r2 not in deleted_ids: return 'ROME Merge first, then EOS can be merged' eos_dupes.query("`m_%` > 70", inplace=True) eos_dupes['Note'] = eos_dupes.apply(note, axis=1) eos_dupes[ 'RomeMergeLink'] = 'https://lne.my.salesforce.com/merge/accmergewizard.jsp?goNext=+Next+&cid=' + eos_dupes[ 'RomeId'] + '&cid=' + eos_dupes['m_RomeId'] output = { 'EOS Dupe Analysis': eos_dupes, 'RomeDeletionLog': deletion_log, } file_name = loc.uk_folder_path + '/Master Data - Output/Venue Dupe Analysis/EOS Venue Duplicate Analysis ' + datetime.now( ).strftime('%Y-%m-%d %H.%M') + '.xlsx' pdh.to_excel(output, file_name) return