def __row_to_contract(self, row): if not (row['Labor Category'] and row['Contract Year'] and row['Year 1/base']): return None contract = Contract() contract.idv_piid = row['CONTRACT .'] contract.labor_category = row['Labor Category'].strip().replace('\n', ' ') contract.vendor_name = row['COMPANY NAME'] contract.education_level = contract.get_education_code(row['Education']) contract.schedule = row['Schedule'] contract.business_size = row['Bus Size'] contract.contract_year = row['Contract Year'] contract.sin = row['SIN NUMBER'] contract.hourly_rate_year1 = contract.normalize_rate(str(row['Year 1/base'])) contract.contractor_site = row['Location'] if row['Begin Date']: contract.contract_start = datetime.strptime(row['Begin Date'], '%m/%d/%Y').date() if row['End Date']: contract.contract_end = datetime.strptime(row['End Date'], '%m/%d/%Y').date() contract.min_years_experience = int(row['MinExpAct']) if row['MinExpAct'].isdigit() else 0 for count, rate in enumerate(row[2:6]): if rate: setattr(contract, 'hourly_rate_year{}'.format(count + 2), contract.normalize_rate(str(rate))) self.__generate_contract_rate_years(row, contract) return contract
def handle(self, *args, **options): data_file = csv.reader(open(os.path.join(settings.BASE_DIR, 'contracts/docs/hourly_prices.csv'), 'r')) #skip header row next(data_file) for line in data_file: try: if line[0]: #create contract record, unique to vendor, labor cat idv_piid = line[0] vendor_name = line[3] labor_category = line[4].strip().replace('\n', ' ') try: contract = Contract.objects.get(idv_piid=idv_piid, labor_category=labor_category, vendor_name=vendor_name) except Contract.DoesNotExist: contract = Contract() contract.idv_piid = idv_piid contract.labor_category = labor_category contract.vendor_name = vendor_name contract.education_level = contract.get_education_code(line[5]) if line[1] != '': contract.contract_start = datetime.strptime(line[1], '%m/%d/%Y').date() if line[2] != '': contract.contract_end = datetime.strptime(line[2], '%m/%d/%Y').date() if line[6].strip() != '': contract.min_years_experience = line[6] else: contract.min_years_experience = 0 if line[7] and line[7] != '': contract.hourly_rate_year1 = contract.normalize_rate(line[7]) else: #there's no pricing info continue for count, rate in enumerate(line[8:12]): if rate and rate.strip() != '': setattr(contract, 'hourly_rate_year' + str(count+2), contract.normalize_rate(rate)) contract.contractor_site = line[12] contract.save() except Exception as e: print(e) print(line) break
def __row_to_contract(self, row): if not (row['Labor Category'] and row['Contract Year'] and row['Year 1/base']): return None contract = Contract() contract.idv_piid = row['CONTRACT .'] contract.labor_category = row['Labor Category'].strip().replace( '\n', ' ') contract.vendor_name = row['COMPANY NAME'] contract.education_level = contract.get_education_code( row['Education']) contract.schedule = row['Schedule'] contract.business_size = row['Bus Size'] contract.contract_year = row['Contract Year'] contract.sin = row['SIN NUMBER'] contract.hourly_rate_year1 = contract.normalize_rate( str(row['Year 1/base'])) contract.contractor_site = row['Location'] if row['Begin Date']: contract.contract_start = datetime.strptime( row['Begin Date'], '%m/%d/%Y').date() if row['End Date']: contract.contract_end = datetime.strptime(row['End Date'], '%m/%d/%Y').date() contract.min_years_experience = int( row['MinExpAct']) if row['MinExpAct'].isdigit() else 0 for count, rate in enumerate(row[2:6]): if rate: setattr(contract, 'hourly_rate_year{}'.format(count + 2), contract.normalize_rate(str(rate))) self.__generate_contract_rate_years(row, contract) return contract
def handle(self, *args, **options): log = logging.getLogger(__name__) log.info("Begin load_data task") log.info("Deleting existing contract records") Contract.objects.all().delete() data_file = csv.reader( open( os.path.join(settings.BASE_DIR, 'contracts/docs/hourly_prices.csv'), 'r')) #skip header row next(data_file) #used for removing expired contracts today = date.today() contracts = [] log.info("Processing new datafile") for line in data_file: #replace annoying msft carraige return for num in range(0, len(line)): #also replace version with capital D line[num] = line[num].replace("_x000d_", "").replace("_x000D_", "") try: if line[0]: #create contract record, unique to vendor, labor cat idv_piid = line[11] vendor_name = line[10] labor_category = line[0].strip().replace('\n', ' ') contract = Contract() contract.idv_piid = idv_piid contract.labor_category = labor_category contract.vendor_name = vendor_name contract.education_level = contract.get_education_code( line[6]) contract.schedule = line[12] contract.business_size = line[8] contract.contract_year = line[14] contract.sin = line[13] if line[15] != '': contract.contract_start = datetime.strptime( line[15], '%m/%d/%Y').date() if line[16] != '': contract.contract_end = datetime.strptime( line[16], '%m/%d/%Y').date() if line[7].strip() != '': contract.min_years_experience = line[7] else: contract.min_years_experience = 0 if line[1] and line[1] != '': contract.hourly_rate_year1 = contract.normalize_rate( line[1]) else: #there's no pricing info continue for count, rate in enumerate(line[2:6]): if rate and rate.strip() != '': setattr(contract, 'hourly_rate_year' + str(count + 2), contract.normalize_rate(rate)) if line[14] and line[14] != '': price_fields = { 'current_price': getattr(contract, 'hourly_rate_year' + str(line[14]), 0) } current_year = int(line[14]) # we have up to five years of rate data if current_year < 5: price_fields['next_year_price'] = getattr( contract, 'hourly_rate_year' + str(current_year + 1), 0) if current_year < 4: price_fields['second_year_price'] = getattr( contract, 'hourly_rate_year' + str(current_year + 2), 0) # don't create display prices for records where the rate # is under the federal minimum contract rate for field in price_fields: price = price_fields.get(field) if price and price >= FEDERAL_MIN_CONTRACT_RATE: setattr(contract, field, price) contract.contractor_site = line[9] contracts.append(contract) except Exception as e: log.exception(e) log.warning(line) break log.info("Inserting records") Contract.objects.bulk_create(contracts) log.info("Updating search index") call_command('update_search_field', Contract._meta.app_label, Contract._meta.model_name) log.info("End load_data task")
def make_contract(cls, line, upload_source=None): if line[0]: # create contract record, unique to vendor, labor cat idv_piid = line[11] vendor_name = line[10] labor_category = line[0].strip().replace('\n', ' ') contract = Contract() contract.idv_piid = idv_piid contract.labor_category = labor_category contract.vendor_name = vendor_name contract.education_level = contract.get_education_code( line[6] ) contract.schedule = line[12] contract.business_size = line[8] current_contract_year = int(float(line[14])) contract.contract_year = current_contract_year contract.sin = line[13] if line[15] != '': contract.contract_start = datetime.strptime( line[15], '%m/%d/%Y').date() if line[16] != '': contract.contract_end = datetime.strptime( line[16], '%m/%d/%Y').date() if line[7].strip() != '': contract.min_years_experience = int(float(line[7])) else: contract.min_years_experience = 0 if line[1] and line[1] != '': contract.hourly_rate_year1 = contract.normalize_rate( line[1] ) else: # there's no pricing info raise ValueError('missing price') for count, rate in enumerate(line[2:6]): if rate and rate.strip() != '': setattr(contract, 'hourly_rate_year' + str(count + 2), contract.normalize_rate(rate)) if line[14] and line[14] != '': price_fields = { 'current_price': getattr(contract, 'hourly_rate_year' + str(current_contract_year), 0) } # we have up to five years of rate data if current_contract_year < 5: price_fields['next_year_price'] = getattr( contract, 'hourly_rate_year' + str(current_contract_year + 1), 0 ) if current_contract_year < 4: price_fields['second_year_price'] = getattr( contract, 'hourly_rate_year' + str(current_contract_year + 2), 0 ) # don't create display prices for records where the # rate is under the federal minimum contract rate for field in price_fields: price = price_fields.get(field) if price and price >= FEDERAL_MIN_CONTRACT_RATE: setattr(contract, field, price) contract.contractor_site = line[9] if upload_source: contract.upload_source = upload_source return contract
def handle(self, *args, **options): log = logging.getLogger(__name__) log.info("Begin load_data task") log.info("Deleting existing contract records") Contract.objects.all().delete() data_file = csv.reader(open(os.path.join(settings.BASE_DIR, "contracts/docs/hourly_prices.csv"), "r")) # skip header row next(data_file) # used for removing expired contracts today = date.today() contracts = [] log.info("Processing new datafile") for line in data_file: # replace annoying msft carraige return for num in range(0, len(line)): # also replace version with capital D line[num] = line[num].replace("_x000d_", "").replace("_x000D_", "") try: if line[0]: # create contract record, unique to vendor, labor cat idv_piid = line[11] vendor_name = line[10] labor_category = line[0].strip().replace("\n", " ") contract = Contract() contract.idv_piid = idv_piid contract.labor_category = labor_category contract.vendor_name = vendor_name contract.education_level = contract.get_education_code(line[6]) contract.schedule = line[12] contract.business_size = line[8] contract.contract_year = line[14] contract.sin = line[13] if line[15] != "": contract.contract_start = datetime.strptime(line[15], "%m/%d/%Y").date() if line[16] != "": contract.contract_end = datetime.strptime(line[16], "%m/%d/%Y").date() if line[7].strip() != "": contract.min_years_experience = line[7] else: contract.min_years_experience = 0 if line[1] and line[1] != "": contract.hourly_rate_year1 = contract.normalize_rate(line[1]) else: # there's no pricing info continue for count, rate in enumerate(line[2:6]): if rate and rate.strip() != "": setattr(contract, "hourly_rate_year" + str(count + 2), contract.normalize_rate(rate)) # don't create current price for records where the rate # is under the federal minimum contract rate current_price = getattr(contract, "hourly_rate_year" + str(line[14])) if current_price and current_price >= FEDERAL_MIN_CONTRACT_RATE: contract.current_price = current_price contract.contractor_site = line[9] contracts.append(contract) except Exception as e: log.exception(e) log.warning(line) break log.info("Inserting records") Contract.objects.bulk_create(contracts) log.info("Updating search index") call_command("update_search_field", Contract._meta.app_label, Contract._meta.model_name) log.info("End load_data task")
def make_contract(cls, line, upload_source=None): if line[0]: # create contract record, unique to vendor, labor cat idv_piid = line[11] vendor_name = line[10] labor_category = line[0].strip().replace('\n', ' ') contract = Contract() contract.idv_piid = idv_piid contract.labor_category = labor_category contract.vendor_name = vendor_name contract.education_level = contract.get_education_code( line[6] ) contract.schedule = line[12] contract.business_size = line[8] current_contract_year = int(float(line[14])) contract.contract_year = current_contract_year contract.sin = line[13] if line[15] != '': contract.contract_start = datetime.strptime( line[15], '%m/%d/%Y').date() if line[16] != '': contract.contract_end = datetime.strptime( line[16], '%m/%d/%Y').date() if line[7].strip() != '': contract.min_years_experience = int(float(line[7])) else: contract.min_years_experience = 0 if line[1] and line[1] != '': contract.hourly_rate_year1 = contract.normalize_rate( line[1] ) else: # there's no pricing info raise ValueError('missing price') for count, rate in enumerate(line[2:6]): if rate and rate.strip() != '': setattr(contract, 'hourly_rate_year' + str(count + 2), contract.normalize_rate(rate)) if line[14] and line[14] != '': price_fields = { 'current_price': getattr(contract, 'hourly_rate_year' + str(current_contract_year), 0) } # we have up to five years of rate data if current_contract_year < 5: price_fields['next_year_price'] = getattr( contract, 'hourly_rate_year' + str(current_contract_year + 1), 0 ) if current_contract_year < 4: price_fields['second_year_price'] = getattr( contract, 'hourly_rate_year' + str(current_contract_year + 2), 0 ) # don't create display prices for records where the # rate is under the federal minimum contract rate for field in price_fields: price = price_fields.get(field) if price and price >= FEDERAL_MIN_CONTRACT_RATE: setattr(contract, field, price) contract.contractor_site = line[9] if upload_source: contract.upload_source = upload_source return contract else: raise ValueError('missing labor category')
def handle(self, *args, **options): log = logging.getLogger(__name__) log.info("Begin load_data task") log.info("Deleting existing contract records") Contract.objects.all().delete() data_file = csv.reader(open(os.path.join(settings.BASE_DIR, 'contracts/docs/hourly_prices.csv'), 'r')) #skip header row next(data_file) #used for removing expired contracts today = date.today() contracts = [] log.info("Processing new datafile") for line in data_file: #replace annoying msft carraige return for num in range(0, len(line)): #also replace version with capital D line[num] = line[num].replace("_x000d_", "").replace("_x000D_", "") try: if line[0]: #create contract record, unique to vendor, labor cat idv_piid = line[11] vendor_name = line[10] labor_category = line[0].strip().replace('\n', ' ') contract = Contract() contract.idv_piid = idv_piid contract.labor_category = labor_category contract.vendor_name = vendor_name contract.education_level = contract.get_education_code(line[6]) contract.schedule = line[12] contract.business_size = line[8] contract.sin = line[13] if line[14] != '': contract.contract_start = datetime.strptime(line[14], '%m/%d/%Y').date() if line[15] != '': contract.contract_end = datetime.strptime(line[15], '%m/%d/%Y').date() if line[7].strip() != '': contract.min_years_experience = line[7] else: contract.min_years_experience = 0 if line[1] and line[1] != '': contract.hourly_rate_year1 = contract.normalize_rate(line[1]) else: #there's no pricing info continue for count, rate in enumerate(line[2:6]): if rate and rate.strip() != '': setattr(contract, 'hourly_rate_year' + str(count+2), contract.normalize_rate(rate)) if contract.contract_end > today and contract.contract_start < today: #it's a current contract, need to find which year we're in start_day = contract.contract_start for plus_year in range(0,5): if date(year=start_day.year + plus_year, month=start_day.month, day=start_day.day) < today: contract.current_price = getattr(contract, 'hourly_rate_year' + str(plus_year + 1)) contract.contractor_site = line[9] contracts.append(contract) except Exception as e: log.exception(e) log.warning(line) break log.info("Inserting records") Contract.objects.bulk_create(contracts) log.info("Updating search index") call_command('update_search_field', Contract._meta.app_label, Contract._meta.model_name) log.info("End load_data task")