def __row_to_contract(self, row): if not (row['Labor Category'] and row['Contract Year'] and row['Year 1/base']): return None contract = Contract() contract.idv_piid = row['CONTRACT .'] contract.labor_category = row['Labor Category'].strip().replace('\n', ' ') contract.vendor_name = row['COMPANY NAME'] contract.education_level = contract.get_education_code(row['Education']) contract.schedule = row['Schedule'] contract.business_size = row['Bus Size'] contract.contract_year = row['Contract Year'] contract.sin = row['SIN NUMBER'] contract.hourly_rate_year1 = contract.normalize_rate(str(row['Year 1/base'])) contract.contractor_site = row['Location'] if row['Begin Date']: contract.contract_start = datetime.strptime(row['Begin Date'], '%m/%d/%Y').date() if row['End Date']: contract.contract_end = datetime.strptime(row['End Date'], '%m/%d/%Y').date() contract.min_years_experience = int(row['MinExpAct']) if row['MinExpAct'].isdigit() else 0 for count, rate in enumerate(row[2:6]): if rate: setattr(contract, 'hourly_rate_year{}'.format(count + 2), contract.normalize_rate(str(rate))) self.__generate_contract_rate_years(row, contract) return contract
def __row_to_contract(self, row): if not (row['Labor Category'] and row['Contract Year'] and row['Year 1/base']): return None contract = Contract() contract.idv_piid = row['CONTRACT .'] contract.labor_category = row['Labor Category'].strip().replace( '\n', ' ') contract.vendor_name = row['COMPANY NAME'] contract.education_level = contract.get_education_code( row['Education']) contract.schedule = row['Schedule'] contract.business_size = row['Bus Size'] contract.contract_year = row['Contract Year'] contract.sin = row['SIN NUMBER'] contract.hourly_rate_year1 = contract.normalize_rate( str(row['Year 1/base'])) contract.contractor_site = row['Location'] if row['Begin Date']: contract.contract_start = datetime.strptime( row['Begin Date'], '%m/%d/%Y').date() if row['End Date']: contract.contract_end = datetime.strptime(row['End Date'], '%m/%d/%Y').date() contract.min_years_experience = int( row['MinExpAct']) if row['MinExpAct'].isdigit() else 0 for count, rate in enumerate(row[2:6]): if rate: setattr(contract, 'hourly_rate_year{}'.format(count + 2), contract.normalize_rate(str(rate))) self.__generate_contract_rate_years(row, contract) return contract
def handle(self, *args, **options): log = logging.getLogger(__name__) log.info("Begin load_data task") log.info("Deleting existing contract records") Contract.objects.all().delete() data_file = csv.reader( open( os.path.join(settings.BASE_DIR, 'contracts/docs/hourly_prices.csv'), 'r')) #skip header row next(data_file) #used for removing expired contracts today = date.today() contracts = [] log.info("Processing new datafile") for line in data_file: #replace annoying msft carraige return for num in range(0, len(line)): #also replace version with capital D line[num] = line[num].replace("_x000d_", "").replace("_x000D_", "") try: if line[0]: #create contract record, unique to vendor, labor cat idv_piid = line[11] vendor_name = line[10] labor_category = line[0].strip().replace('\n', ' ') contract = Contract() contract.idv_piid = idv_piid contract.labor_category = labor_category contract.vendor_name = vendor_name contract.education_level = contract.get_education_code( line[6]) contract.schedule = line[12] contract.business_size = line[8] contract.contract_year = line[14] contract.sin = line[13] if line[15] != '': contract.contract_start = datetime.strptime( line[15], '%m/%d/%Y').date() if line[16] != '': contract.contract_end = datetime.strptime( line[16], '%m/%d/%Y').date() if line[7].strip() != '': contract.min_years_experience = line[7] else: contract.min_years_experience = 0 if line[1] and line[1] != '': contract.hourly_rate_year1 = contract.normalize_rate( line[1]) else: #there's no pricing info continue for count, rate in enumerate(line[2:6]): if rate and rate.strip() != '': setattr(contract, 'hourly_rate_year' + str(count + 2), contract.normalize_rate(rate)) if line[14] and line[14] != '': price_fields = { 'current_price': getattr(contract, 'hourly_rate_year' + str(line[14]), 0) } current_year = int(line[14]) # we have up to five years of rate data if current_year < 5: price_fields['next_year_price'] = getattr( contract, 'hourly_rate_year' + str(current_year + 1), 0) if current_year < 4: price_fields['second_year_price'] = getattr( contract, 'hourly_rate_year' + str(current_year + 2), 0) # don't create display prices for records where the rate # is under the federal minimum contract rate for field in price_fields: price = price_fields.get(field) if price and price >= FEDERAL_MIN_CONTRACT_RATE: setattr(contract, field, price) contract.contractor_site = line[9] contracts.append(contract) except Exception as e: log.exception(e) log.warning(line) break log.info("Inserting records") Contract.objects.bulk_create(contracts) log.info("Updating search index") call_command('update_search_field', Contract._meta.app_label, Contract._meta.model_name) log.info("End load_data task")
def make_contract(cls, line, upload_source=None): if line[0]: # create contract record, unique to vendor, labor cat idv_piid = line[11] vendor_name = line[10] labor_category = line[0].strip().replace('\n', ' ') contract = Contract() contract.idv_piid = idv_piid contract.labor_category = labor_category contract.vendor_name = vendor_name contract.education_level = contract.get_education_code( line[6] ) contract.schedule = line[12] contract.business_size = line[8] current_contract_year = int(float(line[14])) contract.contract_year = current_contract_year contract.sin = line[13] if line[15] != '': contract.contract_start = datetime.strptime( line[15], '%m/%d/%Y').date() if line[16] != '': contract.contract_end = datetime.strptime( line[16], '%m/%d/%Y').date() if line[7].strip() != '': contract.min_years_experience = int(float(line[7])) else: contract.min_years_experience = 0 if line[1] and line[1] != '': contract.hourly_rate_year1 = contract.normalize_rate( line[1] ) else: # there's no pricing info raise ValueError('missing price') for count, rate in enumerate(line[2:6]): if rate and rate.strip() != '': setattr(contract, 'hourly_rate_year' + str(count + 2), contract.normalize_rate(rate)) if line[14] and line[14] != '': price_fields = { 'current_price': getattr(contract, 'hourly_rate_year' + str(current_contract_year), 0) } # we have up to five years of rate data if current_contract_year < 5: price_fields['next_year_price'] = getattr( contract, 'hourly_rate_year' + str(current_contract_year + 1), 0 ) if current_contract_year < 4: price_fields['second_year_price'] = getattr( contract, 'hourly_rate_year' + str(current_contract_year + 2), 0 ) # don't create display prices for records where the # rate is under the federal minimum contract rate for field in price_fields: price = price_fields.get(field) if price and price >= FEDERAL_MIN_CONTRACT_RATE: setattr(contract, field, price) contract.contractor_site = line[9] if upload_source: contract.upload_source = upload_source return contract