def get_folder_jobs(folder_name):
expected = {'daily', 'hourly', 'every10mins'}
assert folder_name in expected, f'folder_name must be one of: {expected}'
path = f'./scheduled/{folder_name}'
jobs = {}
heuristics = Cache.get('scheduled_jobs', {})
for file_name in os.listdir(path):
if not is_job_to_run(f'{path}/{file_name}'):
continue
job_name = file_name.replace('.py', '')
import_path_string = f'scheduled.{folder_name}.{job_name}'
module = __import__(import_path_string, globals(), locals(),
['main', 'problem_data', 'fix'])
jobs[job_name] = module
# jobs.get_heuristics = lambda: Cache.get(job_name, {'name': job_name, 'num_runs': 0, 'avg_duration': 0})
# jobs.set_heuristics = lambda: Cache.get(job_name, {'name': job_name, 'num_runs': 0, 'avg_duration': 0})
return jobs
def fix_event_artists_field():
events = Cache.get('fix_event_artists_field')
# events = sf.select("""
# SELECT Id, EventName__c, PrimaryHeadlinerArtist__c, Artists__c, Venue__r.VenueName__c, Venue__r.BillingCity, EventYear__c
# , IsTouringApp__c, TourLeg__r.Tour__r.TourName__c, TourLeg__r.LegName__c, TourLeg__r.Tour__r.AppScope__c
# , (SELECT Id, Type__c, Artist__r.Name, Rank__c FROM Deals__r WHERE RecordType.Name = 'Artist' AND StageType__c = 'Plan')
# FROM Event__c
# CreatedDate >= THIS_YEAR
# """)
def iterator():
for item in events:
snapshot = item.copy()
item.Deals__r.sort(key=lambda x: (
(0 if x.Type__c == 'Primary Headliner' else 1
if x.Type__c == 'Co-Headliner' else 2), x.Rank__c))
primaryheadlinerlist = [
x for x in item.Deals__r if x['Type__c'] == 'Primary Headliner'
]
# headlinername = primaryheadlinerlist[0].Artist__r.Name if len(primaryheadlinerlist) > 0 else ''
# item.PrimaryHeadlinerArtist__c = headlinername
new_artists = '\r\n'.join((x.Artist__r.Name for x in item.Deals__r
if x.Artist__r is not None))
if new_artists != item.Artists__c:
item.OldArtists = item.Artists__c
item.Artists__c = new_artists
yield item
new_events = list(iterator())
pdh.to_excel(new_events, 'Events with incorrect Artist(s) field')
# with sf.bypass_settings():
# sf.update(new_events)
return
def get_job_heuristics():
return Cache.get('scheduled_jobs', {})
def load_cookie(driver):
cookies = Cache.get(('browser_cookie', driver.current_url))
for cookie in cookies:
driver.add_cookie(cookie)
def eos_venues():
def get_eos_dupes_analysis():
records = sql.query("""
SELECT v.Id, v.Name, v.RomeId, v.IsActive
FROM Venue v
LEFT JOIN TownCity t
ON v.TownCityId = t.Id
LEFT JOIN Region r
ON t.RegionId = r.Id
LEFT JOIN Country c
ON r.CountryId = c.Id
WHERE c.Name IN ('United Kingdom')
""")
return pdh.find_duplicates(records, 'Name', None,
['Id', 'RomeId', 'IsActive'])
def get_deletion_log():
return pd.DataFrame(
sql.query("""
SELECT l.DeletedRecordId__c
, l.MergedAccount__c
, l.RecordTypeName__c
, v1.Id AS EOSIdToDelete
, v1.Name AS EOSNameToDelete
, v2.Id AS EOSIdToMergeInto
, v2.Name AS EOSNameToMergeInto
, CASE WHEN v1.Id < v2.Id THEN 'Double check this - older ID being deleted' ELSE NULL END AS SpotCheck
FROM RomeDeletionLog l
LEFT JOIN Venue v1
ON v1.RomeId = l.DeletedRecordId__c
AND l.RecordTypeName__c = 'Venue'
LEFT JOIN Venue v2
ON v2.RomeId = l.MergedAccount__c
AND l.RecordTypeName__c = 'Venue'WHERE v1.Id IS NOT NULL
"""))
eos_dupes = Cache.get('eosdupes', None)
if eos_dupes is None or prompt('Re-run EOS Venue Dupe Analysis?',
boolean=True):
eos_dupes = get_eos_dupes_analysis()
Cache.set('eosdupes', eos_dupes)
deletion_log = get_deletion_log()
deleted_ids = set(deletion_log.DeletedRecordId__c)
def note(row):
r1, r2 = row['RomeId'], row['m_RomeId']
if pdh.isnull(r1) and pdh.isnull(r2):
return 'EOS Merge - No Rome links exist'
if r1 in deleted_ids and r2 in deleted_ids:
return 'Both RomeIds are deleted in Rome'
elif r2 in deleted_ids:
return 'EOS Merge RIGHT into LEFT - RIGHT Id is deleted in Rome'
elif r1 in deleted_ids:
return 'EOS Merge LEFT into RIGHT - LEFT Id is deleted in Rome'
if pdh.isnull(r1) and pdh.notnull(r2):
return 'EOS Merge LEFT into RIGHT'
if pdh.isnull(r2) and pdh.notnull(r1):
return 'EOS Merge RIGHT into LEFT'
if r1 == r2:
return 'EOS Merge - Both EOS Venues have the same RomeId... merge one into the other'
if r1 not in deleted_ids and r2 not in deleted_ids:
return 'ROME Merge first, then EOS can be merged'
eos_dupes.query("`m_%` > 70", inplace=True)
eos_dupes['Note'] = eos_dupes.apply(note, axis=1)
eos_dupes[
'RomeMergeLink'] = 'https://lne.my.salesforce.com/merge/accmergewizard.jsp?goNext=+Next+&cid=' + eos_dupes[
'RomeId'] + '&cid=' + eos_dupes['m_RomeId']
output = {
'EOS Dupe Analysis': eos_dupes,
'RomeDeletionLog': deletion_log,
}
file_name = loc.uk_folder_path + '/Master Data - Output/Venue Dupe Analysis/EOS Venue Duplicate Analysis ' + datetime.now(
).strftime('%Y-%m-%d %H.%M') + '.xlsx'
pdh.to_excel(output, file_name)
return
