def run(batch_id, source_file_name, output_file_name):
data_gen = DataGenerator()
# load source file
source_columns = [
'External_Id__c', 'Owner.External_Id__c', 'CreatedDate__c',
'EndTime__c', 'EndedBy__c', 'Status__c'
]
data_gen.load_source_file(source_file_name, source_columns)
data_gen.rename_column('Owner.External_Id__c', 'Agent.External_Id__c')
data_gen.add_copy_column('LiveChatTranscript.External_Id__c',
'External_Id__c')
data_gen.add_copy_column('Time__c', 'CreatedDate__c')
data_gen.add_constant_column('Type__c', '')
data_gen.add_constant_column('Detail__c', '')
# add a UUID for each row that is created in this batch
data_gen.add_constant_column('analyticsdemo_batch_id__c', batch_id)
data_gen.apply_transformations()
type_detail_map = {
"ChatRequest": "Visitor requested chat.",
"ChoiceRoute":
"Choice chat request routed to all available qualified agents.",
"CancelNoAgent":
"Chat request canceled because no qualifying agents were available.",
"Accept": "Chat request accepted by agent.",
"CancelVisitor": "Visitor clicked Cancel Chat.",
"LeaveAgent": "Agent left chat.",
"EndAgent": "Agent clicked End Chat.",
"LeaveVisitor": "Visitor left chat.",
"EndVisitor": "Visitor clicked End Chat."
}
current_count = 1
new_rows = []
row_count = len(data_gen.rows)
for i in range(row_count):
row = data_gen.rows.pop()
column_values = data_gen.row_to_column_values(row)
live_chat = column_values['LiveChatTranscript.External_Id__c']
agent = column_values['Agent.External_Id__c']
create_date = dateutil.parser.parse(column_values['CreatedDate__c'])
end_date = dateutil.parser.parse(column_values['EndTime__c'])
ended_by = column_values['EndedBy__c']
status = column_values['Status__c']
# initialize chat request
new_column_values = {
'External_Id__c':
'W_LiveChatTranscriptEvent.' + str(current_count),
'LiveChatTranscript.External_Id__c': live_chat,
'Agent.External_Id__c': agent,
'CreatedDate__c': create_date.isoformat(sep=' '),
'Time__c': create_date.isoformat(sep=' '),
'Type__c': 'ChatRequest',
'Detail__c': 'Visitor requested chat.',
'analyticsdemo_batch_id__c': batch_id
}
current_count += 1
new_rows.append(data_gen.column_values_to_row(new_column_values))
if status == 'Missed':
type__c = choice(['CancelVisitor', 'CancelNoAgent'])
if type__c == 'CancelNoAgent':
# no agents
create_date = fake.date_time_between_dates(
create_date, end_date)
new_column_values = {
'External_Id__c':
'W_LiveChatTranscriptEvent.' + str(current_count),
'LiveChatTranscript.External_Id__c':
live_chat,
'Agent.External_Id__c':
agent,
'CreatedDate__c':
create_date.isoformat(sep=' '),
'Time__c':
create_date.isoformat(sep=' '),
'Type__c':
'ChoiceRoute',
'Detail__c':
'Choice chat request routed to all available qualified agents.',
'analyticsdemo_batch_id__c':
batch_id
}
current_count += 1
new_rows.append(
data_gen.column_values_to_row(new_column_values))
create_date = fake.date_time_between_dates(
create_date, end_date)
new_column_values = {
'External_Id__c':
'W_LiveChatTranscriptEvent.' + str(current_count),
'LiveChatTranscript.External_Id__c':
live_chat,
'Agent.External_Id__c':
agent,
'CreatedDate__c':
create_date.isoformat(sep=' '),
'Time__c':
create_date.isoformat(sep=' '),
'Type__c':
type__c,
'Detail__c':
type_detail_map[type__c],
'analyticsdemo_batch_id__c':
batch_id
}
current_count += 1
new_rows.append(
data_gen.column_values_to_row(new_column_values))
type__c = choice(['LeaveVisitor', 'EndVisitor'])
new_column_values = {
'External_Id__c':
'W_LiveChatTranscriptEvent.' + str(current_count),
'LiveChatTranscript.External_Id__c':
live_chat,
'Agent.External_Id__c':
agent,
'CreatedDate__c':
end_date.isoformat(sep=' '),
'Time__c':
end_date.isoformat(sep=' '),
'Type__c':
type__c,
'Detail__c':
type_detail_map[type__c],
'analyticsdemo_batch_id__c':
batch_id
}
current_count += 1
new_rows.append(
data_gen.column_values_to_row(new_column_values))
else:
# visitor canceled
new_column_values = {
'External_Id__c':
'W_LiveChatTranscriptEvent.' + str(current_count),
'LiveChatTranscript.External_Id__c':
live_chat,
'Agent.External_Id__c':
agent,
'CreatedDate__c':
end_date.isoformat(sep=' '),
'Time__c':
end_date.isoformat(sep=' '),
'Type__c':
type__c,
'Detail__c':
type_detail_map[type__c],
'analyticsdemo_batch_id__c':
batch_id
}
current_count += 1
new_rows.append(
data_gen.column_values_to_row(new_column_values))
else:
type__c = 'ChoiceRoute'
new_column_values = {
'External_Id__c':
'W_LiveChatTranscriptEvent.' + str(current_count),
'LiveChatTranscript.External_Id__c': live_chat,
'Agent.External_Id__c': agent,
'CreatedDate__c': create_date.isoformat(sep=' '),
'Time__c': create_date.isoformat(sep=' '),
'Type__c': type__c,
'Detail__c': type_detail_map[type__c],
'analyticsdemo_batch_id__c': batch_id
}
current_count += 1
new_rows.append(data_gen.column_values_to_row(new_column_values))
type__c = 'Accept'
create_date = fake.date_time_between_dates(create_date, end_date)
new_column_values = {
'External_Id__c':
'W_LiveChatTranscriptEvent.' + str(current_count),
'LiveChatTranscript.External_Id__c': live_chat,
'Agent.External_Id__c': agent,
'CreatedDate__c': create_date.isoformat(sep=' '),
'Time__c': create_date.isoformat(sep=' '),
'Type__c': type__c,
'Detail__c': type_detail_map[type__c],
'analyticsdemo_batch_id__c': batch_id
}
current_count += 1
new_rows.append(data_gen.column_values_to_row(new_column_values))
if ended_by == 'Visitor':
type__c = choice(['LeaveVisitor', 'EndVisitor'])
new_column_values = {
'External_Id__c':
'W_LiveChatTranscriptEvent.' + str(current_count),
'LiveChatTranscript.External_Id__c':
live_chat,
'Agent.External_Id__c':
agent,
'CreatedDate__c':
end_date.isoformat(sep=' '),
'Time__c':
end_date.isoformat(sep=' '),
'Type__c':
type__c,
'Detail__c':
type_detail_map[type__c],
'analyticsdemo_batch_id__c':
batch_id
}
current_count += 1
new_rows.append(
data_gen.column_values_to_row(new_column_values))
else:
type__c = choice(['LeaveAgent', 'EndAgent'])
new_column_values = {
'External_Id__c':
'W_LiveChatTranscriptEvent.' + str(current_count),
'LiveChatTranscript.External_Id__c':
live_chat,
'Agent.External_Id__c':
agent,
'CreatedDate__c':
end_date.isoformat(sep=' '),
'Time__c':
end_date.isoformat(sep=' '),
'Type__c':
type__c,
'Detail__c':
type_detail_map[type__c],
'analyticsdemo_batch_id__c':
batch_id
}
current_count += 1
new_rows.append(
data_gen.column_values_to_row(new_column_values))
data_gen.rows = new_rows
# apply transformations and write file
output_columns = [
'External_Id__c', 'LiveChatTranscript.External_Id__c',
'Agent.External_Id__c', 'Type__c', 'Detail__c', 'CreatedDate__c',
'Time__c', 'analyticsdemo_batch_id__c'
]
data_gen.write(output_file_name, output_columns)
def run(batch_id,
source_file_name,
output_file_name,
reference_date=today_datetime):
data_gen = DataGenerator()
# load source file
source_columns = [
'External_Id__c', 'StageName', 'Amount', 'ForecastCategory',
'CloseDate', 'CreatedDate__c', 'SalesStageCount__c'
]
data_gen.load_source_file(source_file_name, source_columns)
data_gen.rename_columns({
'StageName': 'StageName__c',
'Amount': 'Amount__c',
'ForecastCategory': 'ForecastCategory__c',
'CloseDate': 'CloseDate__c'
})
data_gen.add_copy_column('Opportunity.External_Id__c', 'External_Id__c')
# add a UUID for each row that is created in this batch
data_gen.add_constant_column('analyticsdemo_batch_id__c', batch_id)
data_gen.apply_transformations()
stages = ['Qualification', 'Discovery', 'Proposal/Quote', 'Negotiation']
forecast_categories = ['BestCase', 'Pipeline', 'Commit']
pipe_bucket = [
'No Change', 'Reopen', 'Expand', 'Reduce', 'Moved Out', 'Moved In',
'Stage Change'
]
pipe_bucket_ratio = [0.10, 0.05, 0.15, 0.15, 0.30, 0.10, 0.15]
qualification_pipe_bucket = [
'No Change', 'Reopen', 'Expand', 'Reduce', 'Moved Out', 'Moved In'
]
qualification_pipe_bucket_ratio = [0.20, 0.05, 0.20, 0.10, 0.35, 0.10]
zero_amount_pipe_bucket = [
'No Change', 'Reopen', 'Moved Out', 'Moved In', 'Stage Change'
]
zero_amount_pipe_bucket_ratio = [0.20, 0.05, 0.35, 0.10, 0.30]
current_count = 1
new_rows = []
row_count = len(data_gen.rows)
for i in range(row_count):
row = data_gen.rows.pop()
column_values = data_gen.row_to_column_values(row)
opportunity_id = column_values['Opportunity.External_Id__c']
close_date = dateutil.parser.parse(column_values['CloseDate__c'])
create_date = dateutil.parser.parse(column_values['CreatedDate__c'])
final_amount = int(column_values['Amount__c'])
final_forecast_category = column_values['ForecastCategory__c']
final_stage_name = column_values['StageName__c']
stage_count = int(column_values['SalesStageCount__c'])
# initialize most recent event date to reference_date or earlier
event_date_range_start = create_date + (close_date - create_date) / 2
event_date_range_end = close_date
if close_date > reference_date:
event_date_range_end = reference_date
event_date_range_start = create_date + (reference_date -
create_date) / 2
# ensure event happens on or after opportunity create_date
event_date = fake.date_time_between_dates(event_date_range_start,
event_date_range_end)
# create final state
column_values['CreatedDate__c'] = event_date
column_values['External_Id__c'] = 'W_OpportunityHistory.' + str(
current_count)
current_count += 1
new_rows.append(data_gen.column_values_to_row(column_values))
next_create_date = event_date
next_stage_name = final_stage_name
next_forecast_category = final_forecast_category
next_close_date = close_date
next_amount = final_amount
movedOut = False
movedIn = False
expand = False
reduce = False
reopen = False
initialized = False
# generate events in reverse order until create_date
for current_stage_count in range(stage_count):
# choose the proper bucket depending on the scenario
bucket = pipe_bucket
ratio = pipe_bucket_ratio
if next_amount <= 0:
bucket = zero_amount_pipe_bucket
ratio = zero_amount_pipe_bucket_ratio
elif next_stage_name == 'Qualification':
bucket = qualification_pipe_bucket
ratio = qualification_pipe_bucket_ratio
event = choice(bucket, p=ratio)
event_date_range_end = event_date
event_date_range_start = create_date + (event_date -
create_date) / 2
event_date = fake.date_time_between_dates(event_date_range_start,
event_date_range_end)
# if next stage is closed, make the previous event a stage change
if 'Closed' in next_stage_name:
event = 'Stage Change'
# if the event date is the create date, create the initial state
if current_stage_count == stage_count - 1:
event_date = create_date
event = 'Initial State'
if event != 'No Change':
curr_close_date = next_close_date
curr_amount = next_amount
curr_stage_name = next_stage_name
curr_forecast_category = next_forecast_category
if event == 'Reopen' and not reopen:
curr_stage_name = 'Closed Lost'
curr_forecast_category = 'Omitted'
reopen = True
elif event == 'Initial State':
curr_stage_name = 'Qualification'
curr_forecast_category = 'Pipeline'
initialized = True
elif event == 'Expand' and not expand:
curr_amount = next_amount - int(
uniform(.15, .45) * final_amount)
if curr_amount <= 0:
# reduce instead
curr_amount = next_amount + int(
uniform(.15, .45) * final_amount)
expand = True
elif event == 'Reduce' and not reduce:
curr_amount = next_amount + int(
uniform(.15, .45) * final_amount)
reduce = True
elif event == 'Moved In' and not movedIn:
curr_close_date = curr_close_date + timedelta(
days=randint(0, 30))
movedIn = True
elif event == 'Moved Out' and not movedOut:
curr_close_date = curr_close_date - timedelta(
days=randint(30, 90))
movedOut = True
elif event == 'Stage Change':
# if next stage is not closed, use previous stage
if 'Closed' not in next_stage_name and stages.index(
next_stage_name) - 1 > 0:
curr_stage_name = stages[stages.index(next_stage_name)
- 1]
# if next stage is closed, use any stage
elif 'Closed' in next_stage_name:
curr_stage_name = stages[randint(1, len(stages) - 1)]
else:
curr_stage_name = stages[0]
curr_forecast_category = forecast_categories[randint(
0,
len(forecast_categories) - 1)]
new_column_values = {
'External_Id__c':
'W_OpportunityHistory.' + str(current_count),
'Opportunity.External_Id__c': opportunity_id,
'StageName__c': curr_stage_name,
'Amount__c': curr_amount,
'ForecastCategory__c': curr_forecast_category,
'CreatedDate__c': event_date.isoformat(sep=' '),
'CloseDate__c': curr_close_date.date().isoformat(),
'analyticsdemo_batch_id__c': batch_id
}
current_count += 1
new_rows.append(
data_gen.column_values_to_row(new_column_values))
next_stage_name = curr_stage_name
next_forecast_category = curr_forecast_category
next_close_date = curr_close_date
next_amount = curr_amount
data_gen.rows = new_rows
data_gen.reverse()
data_gen.write(output_file_name, [
'External_Id__c', 'Amount__c', 'StageName__c', 'ForecastCategory__c',
'CloseDate__c', 'CreatedDate__c', 'Opportunity.External_Id__c',
'analyticsdemo_batch_id__c'
])
def run(batch_id, source_file_name, output_file_name, reference_date=today_datetime, filter_function=None):
def get_close_date(values):
return dateutil.parser.parse(values['CloseDate'])
def get_create_date(values):
return dateutil.parser.parse(values['CreatedDate__c'])
data_gen = DataGenerator()
# load source file
data_gen.load_source_file(source_file_name)
# add a UUID for each row that is created in this batch
data_gen.add_constant_column('analyticsdemo_batch_id__c', batch_id)
# add an age column
data_gen.add_copy_column('Age__c', 'TimeToClose__c')
# generate a close date
def close_date_formula(column_values):
last_day = date(date.today().year, 12, 31)
offset = column_values['close_date_offset__c']
# last day of current year - offset
close_date = last_day - timedelta(days=int(offset))
return str(close_date)
data_gen.add_formula_column('CloseDate', close_date_formula)
# generate a create date
def create_date_formula(column_values):
close_date = dateutil.parser.parse(column_values['CloseDate'])
offset = column_values['TimeToClose__c']
create_date = close_date - timedelta(days=int(offset))
return create_date.isoformat(sep=' ')
data_gen.add_formula_column('CreatedDate__c', create_date_formula)
# generate last activity date
def last_activity_date_formula(column_values):
create_date = get_create_date(column_values)
close_date = get_close_date(column_values)
if close_date > reference_date:
close_date = reference_date
if create_date > reference_date:
create_date = reference_date
return fake.date_time_between_dates(create_date, close_date).date()
data_gen.add_formula_column('LastActivityDate__c', formula=last_activity_date_formula)
data_gen.apply_transformations()
if filter_function:
data_gen.filter(filter_function)
new_rows = []
row_count = len(data_gen.rows)
for i in range(row_count):
row = data_gen.rows.pop()
column_values = data_gen.row_to_column_values(row)
close_day = get_close_date(column_values)
create_day = get_create_date(column_values)
# if close date is before reference date keep it exactly as is
if close_day <= reference_date:
new_rows.append(row)
# if create date is before reference date, but the close date is after reference date
elif (create_day <= reference_date) and (close_day > reference_date):
# set age
age = (reference_date - create_day).days
column_values['Age__c'] = age
ttc = float(column_values['TimeToClose__c'])
pct = age / ttc
# set IsClosed to blank
column_values['IsClosed'] = ''
# set IsWon to blank
column_values['IsWon'] = ''
# set a stage name
stage_name_index = int(floor(pct * 4) + choice([-1, 0, 1], p=[.2, .7, .1]))
# adjust the stage name index
if stage_name_index < 0:
stage_name_index = 0
if stage_name_index > 3:
stage_name_index = 3
column_values['StageName'] = definitions.stage_name[stage_name_index]
column_values['Probability'] = definitions.probabilities[stage_name_index]
column_values['ForecastCategory'] = definitions.forecast_category[choice([1, 2, 4], p=[.625, .25, .125])]
column_values['ForecastCategoryName'] = definitions.forecast_category_name[column_values['ForecastCategory']]
column_values['SalesStageCount__c'] = ceil(pct * float(column_values['SalesStageCount__c']))
new_rows.append(data_gen.column_values_to_row(column_values))
data_gen.rows = new_rows
data_gen.reverse()
data_gen.write(output_file_name)
def run(batch_id,
source_file_name,
output_file_name,
reference_datetime=today_datetime):
case_status = ['Escalated', 'Waiting on Customer', 'On Hold', 'Working']
data_gen = DataGenerator()
# load source file
source_columns = [
'External_Id__c', 'Owner.External_Id__c', 'CreatedDate__c',
'ClosedDate__c', 'First_Contact_Close__c', 'Status'
]
data_gen.load_source_file(source_file_name, source_columns)
data_gen.rename_column('External_Id__c', 'Case.External_Id__c')
data_gen.rename_column('Owner.External_Id__c', 'CreatedById__c')
data_gen.add_formula_column('External_Id__c', '')
data_gen.add_constant_column('Field__c', 'created')
data_gen.add_constant_column('OldValue__c', '')
data_gen.add_constant_column('NewValue__c', '')
# add a UUID for each row that is created in this batch
data_gen.add_constant_column('analyticsdemo_batch_id__c', batch_id)
data_gen.apply_transformations()
current_count = 1
new_rows = []
row_count = len(data_gen.rows)
for i in range(row_count):
row = data_gen.rows.pop()
column_values = data_gen.row_to_column_values(row)
column_values['External_Id__c'] = 'W_CaseHistory.' + str(current_count)
current_count += 1
case_id = column_values['Case.External_Id__c']
created_by = column_values['CreatedById__c']
created_date = dateutil.parser.parse(column_values['CreatedDate__c'])
closed_date = dateutil.parser.parse(column_values['ClosedDate__c'])
if closed_date > reference_datetime:
closed_date = reference_datetime
first_contact_close = column_values['First_Contact_Close__c']
status = column_values['Status']
# include initial created row
new_rows.append(data_gen.column_values_to_row(column_values))
# include new status
new_column_values = {
'External_Id__c': 'W_CaseHistory.' + str(current_count),
'Case.External_Id__c': case_id,
'CreatedById__c': created_by,
'CreatedDate__c': created_date,
'Field__c': 'Status',
'OldValue__c': '',
'NewValue__c': 'New',
'ClosedDate__c': '',
'First_Contact_Close__c': ''
}
new_rows.append(data_gen.column_values_to_row(new_column_values))
current_count += 1
old_value = 'New'
next_event_date = created_date
while next_event_date <= closed_date:
next_event_date = next_event_date + timedelta(days=randint(0, 30))
if first_contact_close == 'true' and status == 'Closed':
next_event_date = closed_date
new_column_values = {
'External_Id__c': 'W_CaseHistory.' + str(current_count),
'Case.External_Id__c': case_id,
'CreatedById__c': created_by,
'CreatedDate__c': next_event_date,
'Field__c': 'Status',
'OldValue__c': old_value,
'NewValue__c': 'Closed',
'ClosedDate__c': '',
'First_Contact_Close__c': ''
}
new_rows.append(
data_gen.column_values_to_row(new_column_values))
current_count += 1
break
elif next_event_date >= closed_date:
next_event_date = closed_date
new_column_values = {
'External_Id__c': 'W_CaseHistory.' + str(current_count),
'Case.External_Id__c': case_id,
'CreatedById__c': created_by,
'CreatedDate__c': next_event_date,
'Field__c': 'Status',
'OldValue__c': old_value,
'NewValue__c': status,
'ClosedDate__c': '',
'First_Contact_Close__c': ''
}
new_rows.append(
data_gen.column_values_to_row(new_column_values))
current_count += 1
break
else:
new_value = case_status[randint(0, len(case_status) - 1)]
while old_value == new_value:
new_value = case_status[randint(0, len(case_status) - 1)]
new_column_values = {
'External_Id__c': 'W_CaseHistory.' + str(current_count),
'Case.External_Id__c': case_id,
'CreatedById__c': created_by,
'CreatedDate__c': next_event_date,
'Field__c': 'Status',
'OldValue__c': old_value,
'NewValue__c': new_value,
'ClosedDate__c': '',
'First_Contact_Close__c': '',
'analyticsdemo_batch_id__c': batch_id
}
new_rows.append(
data_gen.column_values_to_row(new_column_values))
old_value = new_value
current_count += 1
data_gen.rows = new_rows
data_gen.reverse()
output_columns = [
'External_Id__c', 'Case.External_Id__c', 'CreatedById__c',
'CreatedDate__c', 'Field__c', 'OldValue__c', 'NewValue__c',
'analyticsdemo_batch_id__c'
]
data_gen.write(output_file_name, output_columns)
def run(batch_id, source_file_name, output_file_name, reference_date=today_datetime):
def get_close_date(values):
return dateutil.parser.parse(values['ClosedDate__c'])
def get_create_date(values):
return dateutil.parser.parse(values['CreatedDate__c'])
data_gen = DataGenerator()
# load source file
data_gen.load_source_file(source_file_name)
# calculate dates
def close_date_formula(column_values):
last_day = date(date.today().year, 12, 31)
last_day = datetime.combine(last_day, datetime.min.time())
offset = column_values['close_date_offset']
# last day of current year - offset
close_date = last_day - timedelta(days=int(offset))
close_date = close_date + timedelta(hours=int(choice([9, 10, 11, 12, 13, 14, 15, 16, 17],
p=[.12, .13, .13, .07, .09, .13, .13, .11, .09])),
minutes=randint(0, 60),
seconds=randint(0, 60))
return close_date.isoformat(sep=' ')
data_gen.add_formula_column('ClosedDate__c', close_date_formula)
def created_date_formula(column_values):
time_open = int(column_values['Time_Open__c'])
date_closed = dateutil.parser.parse(column_values['ClosedDate__c'])
return (date_closed - timedelta(days=time_open)).isoformat(sep=' ')
data_gen.add_formula_column('CreatedDate__c', created_date_formula)
# generate last activity date
def last_activity_date_formula(column_values):
create_date = dateutil.parser.parse(column_values['CreatedDate__c'])
close_date = dateutil.parser.parse(column_values['ClosedDate__c'])
if close_date > today_datetime:
close_date = today_datetime
if create_date > today_datetime:
create_date = today_datetime
return fake.date_time_between_dates(create_date, close_date).date()
data_gen.add_formula_column('LastActivityDate__c', formula=last_activity_date_formula)
data_gen.apply_transformations()
new_rows = []
row_count = len(data_gen.rows)
for i in range(row_count):
row = data_gen.rows.pop()
column_values = data_gen.row_to_column_values(row)
close_day = get_close_date(column_values)
create_day = get_create_date(column_values)
# if close date is before reference date keep it exactly as is
if close_day <= reference_date:
new_rows.append(row)
# if create date is before reference date, but the close date is after reference date
elif (create_day <= reference_date) and (close_day > reference_date):
column_values['Status'] = choice([
'New',
'Working',
'Waiting on Customer',
'Response Received',
'Escalated',
'Warning',
'Attention',
'On Hold',
'Closed in Community'], p=[
0.20,
0.30,
0.10,
0.05,
0.10,
0.05,
0.05,
0.10,
0.05
])
new_rows.append(data_gen.column_values_to_row(column_values))
data_gen.rows = new_rows
data_gen.reverse()
def milestone_status_formula(column_values):
status = column_values['Status']
if status != 'Closed':
status = 'Open'
sla = column_values['SLA']
return status + ' - ' + sla
data_gen.add_formula_column('MilestoneStatus__c', formula=milestone_status_formula)
data_gen.add_formula_column('External_Id__c', formula=lambda: 'W_Case.' + str(data_gen.current_row + 1))
# add a UUID for each row that is created in this batch
data_gen.add_constant_column('analyticsdemo_batch_id__c', batch_id)
data_gen.apply_transformations()
data_gen.write(output_file_name)
def generate_status_file(source_file,
original_status_file,
tmp_folder=today.isoformat(),
file_name=''):
"""Takes a CSV file and generates another file containing mappings of External_Ids and intermediate and final statuses for the records.
Some MFG objects require intermediate status.
Parameters
----------
source_file : str
The name of the file (including the path) to be processed. File must have Id, External_Id__c and Status.
original_status_file : str
The file name (including path) of the CSV file containing the orginal Id and Status.
tmp_folder : str
Name of the folder for archive path. Default = today's date.
file_name : str
Name of Status file.
Returns
-------
None
Generates file(s) with corresponding statuses. Note that for SalesAgreements multiple files are generated and the name of these files will depend on the intermediate status name.
"""
def get_original_final_status(status_by_id, record_id):
return status_by_id[record_id][0].get('Status')
data_gen = DataGenerator()
data_gen.load_source_file(source_file, ['Id', 'External_Id__c', 'Status'])
object_status = data_gen.load_dataset('object_status',
original_status_file,
['Id', 'Status'])
status_by_id = object_status.group_by('Id')
## Order and Contract records can go from DRAFT directly to their final status
## That is why only one file containing all final statuses is needed
## Note: For reference, an ALL status file, containing SAs and their final status, is created for SalesAgreements
new_rows = []
row_count = len(data_gen.rows)
for r in range(row_count):
row = data_gen.rows.pop()
column_values = data_gen.row_to_column_values(row)
column_values['Status'] = get_original_final_status(
status_by_id, column_values['Id'])
new_rows.append(data_gen.column_values_to_row(column_values))
data_gen.rows = new_rows
data_gen.reverse()
# write file containing all statuses
status_file = definitions.mfg_temporal_path.format(tmp_folder) + file_name
data_gen.write(status_file, columns=['External_Id__c', 'Id', 'Status'])
# clear objects
new_rows.clear()
status_by_id.clear()
data_gen.remove_dataset('object_status')
if file_name == 'SalesAgreement.status.ALL.csv':
## SalesAgreements require to be updated in steps up to their final status
## 1> Draft (already inserted)
## 2> Approved (all records but the ones with final_status=Draft must be updated to Approved)
## 3> Discarded (only records with final status as Discarded)
## 4> Cancelled (only records with final status as Cancelled)
## 5> Expired (only records with final status as Expired)
## Note: Some status like Activated are automatically handled in the Org once a record is Approved
data_gen.load_source_file(status_file)
data_gen.add_dataset('sa_all_status', copy.deepcopy(data_gen.rows))
for status_name in ['Approved', 'Discarded', 'Cancelled', 'Expired']:
status_file = definitions.mfg_temporal_path.format(
tmp_folder) + 'SalesAgreement.status.' + status_name.upper(
) + '.csv'
if status_name == 'Approved':
data_gen.add_constant_column('Status', 'Approved')
data_gen.apply_transformations()
else:
data_gen.rows = copy.deepcopy(
data_gen.datasets['sa_all_status'])
data_gen.filter(lambda cv: cv['Status'] == status_name)
if data_gen.row_count:
data_gen.write(status_file,
columns=['External_Id__c', 'Status'])
else:
print("No records for {}".format(status_file))
