def run(batch_id, source_file_name, output_file_name): data_gen = DataGenerator() # load source file source_columns = [ 'External_Id__c', 'Owner.External_Id__c', 'CreatedDate__c', 'EndTime__c', 'EndedBy__c', 'Status__c' ] data_gen.load_source_file(source_file_name, source_columns) data_gen.rename_column('Owner.External_Id__c', 'Agent.External_Id__c') data_gen.add_copy_column('LiveChatTranscript.External_Id__c', 'External_Id__c') data_gen.add_copy_column('Time__c', 'CreatedDate__c') data_gen.add_constant_column('Type__c', '') data_gen.add_constant_column('Detail__c', '') # add a UUID for each row that is created in this batch data_gen.add_constant_column('analyticsdemo_batch_id__c', batch_id) data_gen.apply_transformations() type_detail_map = { "ChatRequest": "Visitor requested chat.", "ChoiceRoute": "Choice chat request routed to all available qualified agents.", "CancelNoAgent": "Chat request canceled because no qualifying agents were available.", "Accept": "Chat request accepted by agent.", "CancelVisitor": "Visitor clicked Cancel Chat.", "LeaveAgent": "Agent left chat.", "EndAgent": "Agent clicked End Chat.", "LeaveVisitor": "Visitor left chat.", "EndVisitor": "Visitor clicked End Chat." } current_count = 1 new_rows = [] row_count = len(data_gen.rows) for i in range(row_count): row = data_gen.rows.pop() column_values = data_gen.row_to_column_values(row) live_chat = column_values['LiveChatTranscript.External_Id__c'] agent = column_values['Agent.External_Id__c'] create_date = dateutil.parser.parse(column_values['CreatedDate__c']) end_date = dateutil.parser.parse(column_values['EndTime__c']) ended_by = column_values['EndedBy__c'] status = column_values['Status__c'] # initialize chat request new_column_values = { 'External_Id__c': 'W_LiveChatTranscriptEvent.' + str(current_count), 'LiveChatTranscript.External_Id__c': live_chat, 'Agent.External_Id__c': agent, 'CreatedDate__c': create_date.isoformat(sep=' '), 'Time__c': create_date.isoformat(sep=' '), 'Type__c': 'ChatRequest', 'Detail__c': 'Visitor requested chat.', 'analyticsdemo_batch_id__c': batch_id } current_count += 1 new_rows.append(data_gen.column_values_to_row(new_column_values)) if status == 'Missed': type__c = choice(['CancelVisitor', 'CancelNoAgent']) if type__c == 'CancelNoAgent': # no agents create_date = fake.date_time_between_dates( create_date, end_date) new_column_values = { 'External_Id__c': 'W_LiveChatTranscriptEvent.' + str(current_count), 'LiveChatTranscript.External_Id__c': live_chat, 'Agent.External_Id__c': agent, 'CreatedDate__c': create_date.isoformat(sep=' '), 'Time__c': create_date.isoformat(sep=' '), 'Type__c': 'ChoiceRoute', 'Detail__c': 'Choice chat request routed to all available qualified agents.', 'analyticsdemo_batch_id__c': batch_id } current_count += 1 new_rows.append( data_gen.column_values_to_row(new_column_values)) create_date = fake.date_time_between_dates( create_date, end_date) new_column_values = { 'External_Id__c': 'W_LiveChatTranscriptEvent.' + str(current_count), 'LiveChatTranscript.External_Id__c': live_chat, 'Agent.External_Id__c': agent, 'CreatedDate__c': create_date.isoformat(sep=' '), 'Time__c': create_date.isoformat(sep=' '), 'Type__c': type__c, 'Detail__c': type_detail_map[type__c], 'analyticsdemo_batch_id__c': batch_id } current_count += 1 new_rows.append( data_gen.column_values_to_row(new_column_values)) type__c = choice(['LeaveVisitor', 'EndVisitor']) new_column_values = { 'External_Id__c': 'W_LiveChatTranscriptEvent.' + str(current_count), 'LiveChatTranscript.External_Id__c': live_chat, 'Agent.External_Id__c': agent, 'CreatedDate__c': end_date.isoformat(sep=' '), 'Time__c': end_date.isoformat(sep=' '), 'Type__c': type__c, 'Detail__c': type_detail_map[type__c], 'analyticsdemo_batch_id__c': batch_id } current_count += 1 new_rows.append( data_gen.column_values_to_row(new_column_values)) else: # visitor canceled new_column_values = { 'External_Id__c': 'W_LiveChatTranscriptEvent.' + str(current_count), 'LiveChatTranscript.External_Id__c': live_chat, 'Agent.External_Id__c': agent, 'CreatedDate__c': end_date.isoformat(sep=' '), 'Time__c': end_date.isoformat(sep=' '), 'Type__c': type__c, 'Detail__c': type_detail_map[type__c], 'analyticsdemo_batch_id__c': batch_id } current_count += 1 new_rows.append( data_gen.column_values_to_row(new_column_values)) else: type__c = 'ChoiceRoute' new_column_values = { 'External_Id__c': 'W_LiveChatTranscriptEvent.' + str(current_count), 'LiveChatTranscript.External_Id__c': live_chat, 'Agent.External_Id__c': agent, 'CreatedDate__c': create_date.isoformat(sep=' '), 'Time__c': create_date.isoformat(sep=' '), 'Type__c': type__c, 'Detail__c': type_detail_map[type__c], 'analyticsdemo_batch_id__c': batch_id } current_count += 1 new_rows.append(data_gen.column_values_to_row(new_column_values)) type__c = 'Accept' create_date = fake.date_time_between_dates(create_date, end_date) new_column_values = { 'External_Id__c': 'W_LiveChatTranscriptEvent.' + str(current_count), 'LiveChatTranscript.External_Id__c': live_chat, 'Agent.External_Id__c': agent, 'CreatedDate__c': create_date.isoformat(sep=' '), 'Time__c': create_date.isoformat(sep=' '), 'Type__c': type__c, 'Detail__c': type_detail_map[type__c], 'analyticsdemo_batch_id__c': batch_id } current_count += 1 new_rows.append(data_gen.column_values_to_row(new_column_values)) if ended_by == 'Visitor': type__c = choice(['LeaveVisitor', 'EndVisitor']) new_column_values = { 'External_Id__c': 'W_LiveChatTranscriptEvent.' + str(current_count), 'LiveChatTranscript.External_Id__c': live_chat, 'Agent.External_Id__c': agent, 'CreatedDate__c': end_date.isoformat(sep=' '), 'Time__c': end_date.isoformat(sep=' '), 'Type__c': type__c, 'Detail__c': type_detail_map[type__c], 'analyticsdemo_batch_id__c': batch_id } current_count += 1 new_rows.append( data_gen.column_values_to_row(new_column_values)) else: type__c = choice(['LeaveAgent', 'EndAgent']) new_column_values = { 'External_Id__c': 'W_LiveChatTranscriptEvent.' + str(current_count), 'LiveChatTranscript.External_Id__c': live_chat, 'Agent.External_Id__c': agent, 'CreatedDate__c': end_date.isoformat(sep=' '), 'Time__c': end_date.isoformat(sep=' '), 'Type__c': type__c, 'Detail__c': type_detail_map[type__c], 'analyticsdemo_batch_id__c': batch_id } current_count += 1 new_rows.append( data_gen.column_values_to_row(new_column_values)) data_gen.rows = new_rows # apply transformations and write file output_columns = [ 'External_Id__c', 'LiveChatTranscript.External_Id__c', 'Agent.External_Id__c', 'Type__c', 'Detail__c', 'CreatedDate__c', 'Time__c', 'analyticsdemo_batch_id__c' ] data_gen.write(output_file_name, output_columns)
def run(batch_id, source_file_name, output_file_name, reference_date=today_datetime): data_gen = DataGenerator() # load source file source_columns = [ 'External_Id__c', 'StageName', 'Amount', 'ForecastCategory', 'CloseDate', 'CreatedDate__c', 'SalesStageCount__c' ] data_gen.load_source_file(source_file_name, source_columns) data_gen.rename_columns({ 'StageName': 'StageName__c', 'Amount': 'Amount__c', 'ForecastCategory': 'ForecastCategory__c', 'CloseDate': 'CloseDate__c' }) data_gen.add_copy_column('Opportunity.External_Id__c', 'External_Id__c') # add a UUID for each row that is created in this batch data_gen.add_constant_column('analyticsdemo_batch_id__c', batch_id) data_gen.apply_transformations() stages = ['Qualification', 'Discovery', 'Proposal/Quote', 'Negotiation'] forecast_categories = ['BestCase', 'Pipeline', 'Commit'] pipe_bucket = [ 'No Change', 'Reopen', 'Expand', 'Reduce', 'Moved Out', 'Moved In', 'Stage Change' ] pipe_bucket_ratio = [0.10, 0.05, 0.15, 0.15, 0.30, 0.10, 0.15] qualification_pipe_bucket = [ 'No Change', 'Reopen', 'Expand', 'Reduce', 'Moved Out', 'Moved In' ] qualification_pipe_bucket_ratio = [0.20, 0.05, 0.20, 0.10, 0.35, 0.10] zero_amount_pipe_bucket = [ 'No Change', 'Reopen', 'Moved Out', 'Moved In', 'Stage Change' ] zero_amount_pipe_bucket_ratio = [0.20, 0.05, 0.35, 0.10, 0.30] current_count = 1 new_rows = [] row_count = len(data_gen.rows) for i in range(row_count): row = data_gen.rows.pop() column_values = data_gen.row_to_column_values(row) opportunity_id = column_values['Opportunity.External_Id__c'] close_date = dateutil.parser.parse(column_values['CloseDate__c']) create_date = dateutil.parser.parse(column_values['CreatedDate__c']) final_amount = int(column_values['Amount__c']) final_forecast_category = column_values['ForecastCategory__c'] final_stage_name = column_values['StageName__c'] stage_count = int(column_values['SalesStageCount__c']) # initialize most recent event date to reference_date or earlier event_date_range_start = create_date + (close_date - create_date) / 2 event_date_range_end = close_date if close_date > reference_date: event_date_range_end = reference_date event_date_range_start = create_date + (reference_date - create_date) / 2 # ensure event happens on or after opportunity create_date event_date = fake.date_time_between_dates(event_date_range_start, event_date_range_end) # create final state column_values['CreatedDate__c'] = event_date column_values['External_Id__c'] = 'W_OpportunityHistory.' + str( current_count) current_count += 1 new_rows.append(data_gen.column_values_to_row(column_values)) next_create_date = event_date next_stage_name = final_stage_name next_forecast_category = final_forecast_category next_close_date = close_date next_amount = final_amount movedOut = False movedIn = False expand = False reduce = False reopen = False initialized = False # generate events in reverse order until create_date for current_stage_count in range(stage_count): # choose the proper bucket depending on the scenario bucket = pipe_bucket ratio = pipe_bucket_ratio if next_amount <= 0: bucket = zero_amount_pipe_bucket ratio = zero_amount_pipe_bucket_ratio elif next_stage_name == 'Qualification': bucket = qualification_pipe_bucket ratio = qualification_pipe_bucket_ratio event = choice(bucket, p=ratio) event_date_range_end = event_date event_date_range_start = create_date + (event_date - create_date) / 2 event_date = fake.date_time_between_dates(event_date_range_start, event_date_range_end) # if next stage is closed, make the previous event a stage change if 'Closed' in next_stage_name: event = 'Stage Change' # if the event date is the create date, create the initial state if current_stage_count == stage_count - 1: event_date = create_date event = 'Initial State' if event != 'No Change': curr_close_date = next_close_date curr_amount = next_amount curr_stage_name = next_stage_name curr_forecast_category = next_forecast_category if event == 'Reopen' and not reopen: curr_stage_name = 'Closed Lost' curr_forecast_category = 'Omitted' reopen = True elif event == 'Initial State': curr_stage_name = 'Qualification' curr_forecast_category = 'Pipeline' initialized = True elif event == 'Expand' and not expand: curr_amount = next_amount - int( uniform(.15, .45) * final_amount) if curr_amount <= 0: # reduce instead curr_amount = next_amount + int( uniform(.15, .45) * final_amount) expand = True elif event == 'Reduce' and not reduce: curr_amount = next_amount + int( uniform(.15, .45) * final_amount) reduce = True elif event == 'Moved In' and not movedIn: curr_close_date = curr_close_date + timedelta( days=randint(0, 30)) movedIn = True elif event == 'Moved Out' and not movedOut: curr_close_date = curr_close_date - timedelta( days=randint(30, 90)) movedOut = True elif event == 'Stage Change': # if next stage is not closed, use previous stage if 'Closed' not in next_stage_name and stages.index( next_stage_name) - 1 > 0: curr_stage_name = stages[stages.index(next_stage_name) - 1] # if next stage is closed, use any stage elif 'Closed' in next_stage_name: curr_stage_name = stages[randint(1, len(stages) - 1)] else: curr_stage_name = stages[0] curr_forecast_category = forecast_categories[randint( 0, len(forecast_categories) - 1)] new_column_values = { 'External_Id__c': 'W_OpportunityHistory.' + str(current_count), 'Opportunity.External_Id__c': opportunity_id, 'StageName__c': curr_stage_name, 'Amount__c': curr_amount, 'ForecastCategory__c': curr_forecast_category, 'CreatedDate__c': event_date.isoformat(sep=' '), 'CloseDate__c': curr_close_date.date().isoformat(), 'analyticsdemo_batch_id__c': batch_id } current_count += 1 new_rows.append( data_gen.column_values_to_row(new_column_values)) next_stage_name = curr_stage_name next_forecast_category = curr_forecast_category next_close_date = curr_close_date next_amount = curr_amount data_gen.rows = new_rows data_gen.reverse() data_gen.write(output_file_name, [ 'External_Id__c', 'Amount__c', 'StageName__c', 'ForecastCategory__c', 'CloseDate__c', 'CreatedDate__c', 'Opportunity.External_Id__c', 'analyticsdemo_batch_id__c' ])
def run(batch_id, source_file_name, output_file_name, reference_date=today_datetime, filter_function=None): def get_close_date(values): return dateutil.parser.parse(values['CloseDate']) def get_create_date(values): return dateutil.parser.parse(values['CreatedDate__c']) data_gen = DataGenerator() # load source file data_gen.load_source_file(source_file_name) # add a UUID for each row that is created in this batch data_gen.add_constant_column('analyticsdemo_batch_id__c', batch_id) # add an age column data_gen.add_copy_column('Age__c', 'TimeToClose__c') # generate a close date def close_date_formula(column_values): last_day = date(date.today().year, 12, 31) offset = column_values['close_date_offset__c'] # last day of current year - offset close_date = last_day - timedelta(days=int(offset)) return str(close_date) data_gen.add_formula_column('CloseDate', close_date_formula) # generate a create date def create_date_formula(column_values): close_date = dateutil.parser.parse(column_values['CloseDate']) offset = column_values['TimeToClose__c'] create_date = close_date - timedelta(days=int(offset)) return create_date.isoformat(sep=' ') data_gen.add_formula_column('CreatedDate__c', create_date_formula) # generate last activity date def last_activity_date_formula(column_values): create_date = get_create_date(column_values) close_date = get_close_date(column_values) if close_date > reference_date: close_date = reference_date if create_date > reference_date: create_date = reference_date return fake.date_time_between_dates(create_date, close_date).date() data_gen.add_formula_column('LastActivityDate__c', formula=last_activity_date_formula) data_gen.apply_transformations() if filter_function: data_gen.filter(filter_function) new_rows = [] row_count = len(data_gen.rows) for i in range(row_count): row = data_gen.rows.pop() column_values = data_gen.row_to_column_values(row) close_day = get_close_date(column_values) create_day = get_create_date(column_values) # if close date is before reference date keep it exactly as is if close_day <= reference_date: new_rows.append(row) # if create date is before reference date, but the close date is after reference date elif (create_day <= reference_date) and (close_day > reference_date): # set age age = (reference_date - create_day).days column_values['Age__c'] = age ttc = float(column_values['TimeToClose__c']) pct = age / ttc # set IsClosed to blank column_values['IsClosed'] = '' # set IsWon to blank column_values['IsWon'] = '' # set a stage name stage_name_index = int(floor(pct * 4) + choice([-1, 0, 1], p=[.2, .7, .1])) # adjust the stage name index if stage_name_index < 0: stage_name_index = 0 if stage_name_index > 3: stage_name_index = 3 column_values['StageName'] = definitions.stage_name[stage_name_index] column_values['Probability'] = definitions.probabilities[stage_name_index] column_values['ForecastCategory'] = definitions.forecast_category[choice([1, 2, 4], p=[.625, .25, .125])] column_values['ForecastCategoryName'] = definitions.forecast_category_name[column_values['ForecastCategory']] column_values['SalesStageCount__c'] = ceil(pct * float(column_values['SalesStageCount__c'])) new_rows.append(data_gen.column_values_to_row(column_values)) data_gen.rows = new_rows data_gen.reverse() data_gen.write(output_file_name)
def run(batch_id, source_file_name, output_file_name, reference_datetime=today_datetime): case_status = ['Escalated', 'Waiting on Customer', 'On Hold', 'Working'] data_gen = DataGenerator() # load source file source_columns = [ 'External_Id__c', 'Owner.External_Id__c', 'CreatedDate__c', 'ClosedDate__c', 'First_Contact_Close__c', 'Status' ] data_gen.load_source_file(source_file_name, source_columns) data_gen.rename_column('External_Id__c', 'Case.External_Id__c') data_gen.rename_column('Owner.External_Id__c', 'CreatedById__c') data_gen.add_formula_column('External_Id__c', '') data_gen.add_constant_column('Field__c', 'created') data_gen.add_constant_column('OldValue__c', '') data_gen.add_constant_column('NewValue__c', '') # add a UUID for each row that is created in this batch data_gen.add_constant_column('analyticsdemo_batch_id__c', batch_id) data_gen.apply_transformations() current_count = 1 new_rows = [] row_count = len(data_gen.rows) for i in range(row_count): row = data_gen.rows.pop() column_values = data_gen.row_to_column_values(row) column_values['External_Id__c'] = 'W_CaseHistory.' + str(current_count) current_count += 1 case_id = column_values['Case.External_Id__c'] created_by = column_values['CreatedById__c'] created_date = dateutil.parser.parse(column_values['CreatedDate__c']) closed_date = dateutil.parser.parse(column_values['ClosedDate__c']) if closed_date > reference_datetime: closed_date = reference_datetime first_contact_close = column_values['First_Contact_Close__c'] status = column_values['Status'] # include initial created row new_rows.append(data_gen.column_values_to_row(column_values)) # include new status new_column_values = { 'External_Id__c': 'W_CaseHistory.' + str(current_count), 'Case.External_Id__c': case_id, 'CreatedById__c': created_by, 'CreatedDate__c': created_date, 'Field__c': 'Status', 'OldValue__c': '', 'NewValue__c': 'New', 'ClosedDate__c': '', 'First_Contact_Close__c': '' } new_rows.append(data_gen.column_values_to_row(new_column_values)) current_count += 1 old_value = 'New' next_event_date = created_date while next_event_date <= closed_date: next_event_date = next_event_date + timedelta(days=randint(0, 30)) if first_contact_close == 'true' and status == 'Closed': next_event_date = closed_date new_column_values = { 'External_Id__c': 'W_CaseHistory.' + str(current_count), 'Case.External_Id__c': case_id, 'CreatedById__c': created_by, 'CreatedDate__c': next_event_date, 'Field__c': 'Status', 'OldValue__c': old_value, 'NewValue__c': 'Closed', 'ClosedDate__c': '', 'First_Contact_Close__c': '' } new_rows.append( data_gen.column_values_to_row(new_column_values)) current_count += 1 break elif next_event_date >= closed_date: next_event_date = closed_date new_column_values = { 'External_Id__c': 'W_CaseHistory.' + str(current_count), 'Case.External_Id__c': case_id, 'CreatedById__c': created_by, 'CreatedDate__c': next_event_date, 'Field__c': 'Status', 'OldValue__c': old_value, 'NewValue__c': status, 'ClosedDate__c': '', 'First_Contact_Close__c': '' } new_rows.append( data_gen.column_values_to_row(new_column_values)) current_count += 1 break else: new_value = case_status[randint(0, len(case_status) - 1)] while old_value == new_value: new_value = case_status[randint(0, len(case_status) - 1)] new_column_values = { 'External_Id__c': 'W_CaseHistory.' + str(current_count), 'Case.External_Id__c': case_id, 'CreatedById__c': created_by, 'CreatedDate__c': next_event_date, 'Field__c': 'Status', 'OldValue__c': old_value, 'NewValue__c': new_value, 'ClosedDate__c': '', 'First_Contact_Close__c': '', 'analyticsdemo_batch_id__c': batch_id } new_rows.append( data_gen.column_values_to_row(new_column_values)) old_value = new_value current_count += 1 data_gen.rows = new_rows data_gen.reverse() output_columns = [ 'External_Id__c', 'Case.External_Id__c', 'CreatedById__c', 'CreatedDate__c', 'Field__c', 'OldValue__c', 'NewValue__c', 'analyticsdemo_batch_id__c' ] data_gen.write(output_file_name, output_columns)
def run(batch_id, source_file_name, output_file_name, reference_date=today_datetime): def get_close_date(values): return dateutil.parser.parse(values['ClosedDate__c']) def get_create_date(values): return dateutil.parser.parse(values['CreatedDate__c']) data_gen = DataGenerator() # load source file data_gen.load_source_file(source_file_name) # calculate dates def close_date_formula(column_values): last_day = date(date.today().year, 12, 31) last_day = datetime.combine(last_day, datetime.min.time()) offset = column_values['close_date_offset'] # last day of current year - offset close_date = last_day - timedelta(days=int(offset)) close_date = close_date + timedelta(hours=int(choice([9, 10, 11, 12, 13, 14, 15, 16, 17], p=[.12, .13, .13, .07, .09, .13, .13, .11, .09])), minutes=randint(0, 60), seconds=randint(0, 60)) return close_date.isoformat(sep=' ') data_gen.add_formula_column('ClosedDate__c', close_date_formula) def created_date_formula(column_values): time_open = int(column_values['Time_Open__c']) date_closed = dateutil.parser.parse(column_values['ClosedDate__c']) return (date_closed - timedelta(days=time_open)).isoformat(sep=' ') data_gen.add_formula_column('CreatedDate__c', created_date_formula) # generate last activity date def last_activity_date_formula(column_values): create_date = dateutil.parser.parse(column_values['CreatedDate__c']) close_date = dateutil.parser.parse(column_values['ClosedDate__c']) if close_date > today_datetime: close_date = today_datetime if create_date > today_datetime: create_date = today_datetime return fake.date_time_between_dates(create_date, close_date).date() data_gen.add_formula_column('LastActivityDate__c', formula=last_activity_date_formula) data_gen.apply_transformations() new_rows = [] row_count = len(data_gen.rows) for i in range(row_count): row = data_gen.rows.pop() column_values = data_gen.row_to_column_values(row) close_day = get_close_date(column_values) create_day = get_create_date(column_values) # if close date is before reference date keep it exactly as is if close_day <= reference_date: new_rows.append(row) # if create date is before reference date, but the close date is after reference date elif (create_day <= reference_date) and (close_day > reference_date): column_values['Status'] = choice([ 'New', 'Working', 'Waiting on Customer', 'Response Received', 'Escalated', 'Warning', 'Attention', 'On Hold', 'Closed in Community'], p=[ 0.20, 0.30, 0.10, 0.05, 0.10, 0.05, 0.05, 0.10, 0.05 ]) new_rows.append(data_gen.column_values_to_row(column_values)) data_gen.rows = new_rows data_gen.reverse() def milestone_status_formula(column_values): status = column_values['Status'] if status != 'Closed': status = 'Open' sla = column_values['SLA'] return status + ' - ' + sla data_gen.add_formula_column('MilestoneStatus__c', formula=milestone_status_formula) data_gen.add_formula_column('External_Id__c', formula=lambda: 'W_Case.' + str(data_gen.current_row + 1)) # add a UUID for each row that is created in this batch data_gen.add_constant_column('analyticsdemo_batch_id__c', batch_id) data_gen.apply_transformations() data_gen.write(output_file_name)
def generate_status_file(source_file, original_status_file, tmp_folder=today.isoformat(), file_name=''): """Takes a CSV file and generates another file containing mappings of External_Ids and intermediate and final statuses for the records. Some MFG objects require intermediate status. Parameters ---------- source_file : str The name of the file (including the path) to be processed. File must have Id, External_Id__c and Status. original_status_file : str The file name (including path) of the CSV file containing the orginal Id and Status. tmp_folder : str Name of the folder for archive path. Default = today's date. file_name : str Name of Status file. Returns ------- None Generates file(s) with corresponding statuses. Note that for SalesAgreements multiple files are generated and the name of these files will depend on the intermediate status name. """ def get_original_final_status(status_by_id, record_id): return status_by_id[record_id][0].get('Status') data_gen = DataGenerator() data_gen.load_source_file(source_file, ['Id', 'External_Id__c', 'Status']) object_status = data_gen.load_dataset('object_status', original_status_file, ['Id', 'Status']) status_by_id = object_status.group_by('Id') ## Order and Contract records can go from DRAFT directly to their final status ## That is why only one file containing all final statuses is needed ## Note: For reference, an ALL status file, containing SAs and their final status, is created for SalesAgreements new_rows = [] row_count = len(data_gen.rows) for r in range(row_count): row = data_gen.rows.pop() column_values = data_gen.row_to_column_values(row) column_values['Status'] = get_original_final_status( status_by_id, column_values['Id']) new_rows.append(data_gen.column_values_to_row(column_values)) data_gen.rows = new_rows data_gen.reverse() # write file containing all statuses status_file = definitions.mfg_temporal_path.format(tmp_folder) + file_name data_gen.write(status_file, columns=['External_Id__c', 'Id', 'Status']) # clear objects new_rows.clear() status_by_id.clear() data_gen.remove_dataset('object_status') if file_name == 'SalesAgreement.status.ALL.csv': ## SalesAgreements require to be updated in steps up to their final status ## 1> Draft (already inserted) ## 2> Approved (all records but the ones with final_status=Draft must be updated to Approved) ## 3> Discarded (only records with final status as Discarded) ## 4> Cancelled (only records with final status as Cancelled) ## 5> Expired (only records with final status as Expired) ## Note: Some status like Activated are automatically handled in the Org once a record is Approved data_gen.load_source_file(status_file) data_gen.add_dataset('sa_all_status', copy.deepcopy(data_gen.rows)) for status_name in ['Approved', 'Discarded', 'Cancelled', 'Expired']: status_file = definitions.mfg_temporal_path.format( tmp_folder) + 'SalesAgreement.status.' + status_name.upper( ) + '.csv' if status_name == 'Approved': data_gen.add_constant_column('Status', 'Approved') data_gen.apply_transformations() else: data_gen.rows = copy.deepcopy( data_gen.datasets['sa_all_status']) data_gen.filter(lambda cv: cv['Status'] == status_name) if data_gen.row_count: data_gen.write(status_file, columns=['External_Id__c', 'Status']) else: print("No records for {}".format(status_file))