def handle(self, *args, **options):
log = logging.getLogger(__name__)
log.info("Begin load_data task")
log.info("Deleting existing contract records")
Contract.objects.all().delete()
data_file = csv.reader(open(os.path.join(settings.BASE_DIR, "contracts/docs/hourly_prices.csv"), "r"))
# skip header row
next(data_file)
# used for removing expired contracts
today = date.today()
contracts = []
log.info("Processing new datafile")
for line in data_file:
# replace annoying msft carraige return
for num in range(0, len(line)):
# also replace version with capital D
line[num] = line[num].replace("_x000d_", "").replace("_x000D_", "")
try:
if line[0]:
# create contract record, unique to vendor, labor cat
idv_piid = line[11]
vendor_name = line[10]
labor_category = line[0].strip().replace("\n", " ")
contract = Contract()
contract.idv_piid = idv_piid
contract.labor_category = labor_category
contract.vendor_name = vendor_name
contract.education_level = contract.get_education_code(line[6])
contract.schedule = line[12]
contract.business_size = line[8]
contract.contract_year = line[14]
contract.sin = line[13]
if line[15] != "":
contract.contract_start = datetime.strptime(line[15], "%m/%d/%Y").date()
if line[16] != "":
contract.contract_end = datetime.strptime(line[16], "%m/%d/%Y").date()
if line[7].strip() != "":
contract.min_years_experience = line[7]
else:
contract.min_years_experience = 0
if line[1] and line[1] != "":
contract.hourly_rate_year1 = contract.normalize_rate(line[1])
else:
# there's no pricing info
continue
for count, rate in enumerate(line[2:6]):
if rate and rate.strip() != "":
setattr(contract, "hourly_rate_year" + str(count + 2), contract.normalize_rate(rate))
# don't create current price for records where the rate
# is under the federal minimum contract rate
current_price = getattr(contract, "hourly_rate_year" + str(line[14]))
if current_price and current_price >= FEDERAL_MIN_CONTRACT_RATE:
contract.current_price = current_price
contract.contractor_site = line[9]
contracts.append(contract)
except Exception as e:
log.exception(e)
log.warning(line)
break
log.info("Inserting records")
Contract.objects.bulk_create(contracts)
log.info("Updating search index")
call_command("update_search_field", Contract._meta.app_label, Contract._meta.model_name)
log.info("End load_data task")
def handle(self, *args, **options):
log = logging.getLogger(__name__)
log.info("Begin load_data task")
log.info("Deleting existing contract records")
Contract.objects.all().delete()
data_file = csv.reader(open(os.path.join(settings.BASE_DIR, 'contracts/docs/hourly_prices.csv'), 'r'))
#skip header row
next(data_file)
#used for removing expired contracts
today = date.today()
contracts = []
log.info("Processing new datafile")
for line in data_file:
#replace annoying msft carraige return
for num in range(0, len(line)):
#also replace version with capital D
line[num] = line[num].replace("_x000d_", "").replace("_x000D_", "")
try:
if line[0]:
#create contract record, unique to vendor, labor cat
idv_piid = line[11]
vendor_name = line[10]
labor_category = line[0].strip().replace('\n', ' ')
contract = Contract()
contract.idv_piid = idv_piid
contract.labor_category = labor_category
contract.vendor_name = vendor_name
contract.education_level = contract.get_education_code(line[6])
contract.schedule = line[12]
contract.business_size = line[8]
contract.sin = line[13]
if line[14] != '':
contract.contract_start = datetime.strptime(line[14], '%m/%d/%Y').date()
if line[15] != '':
contract.contract_end = datetime.strptime(line[15], '%m/%d/%Y').date()
if line[7].strip() != '':
contract.min_years_experience = line[7]
else:
contract.min_years_experience = 0
if line[1] and line[1] != '':
contract.hourly_rate_year1 = contract.normalize_rate(line[1])
else:
#there's no pricing info
continue
for count, rate in enumerate(line[2:6]):
if rate and rate.strip() != '':
setattr(contract, 'hourly_rate_year' + str(count+2), contract.normalize_rate(rate))
if contract.contract_end > today and contract.contract_start < today:
#it's a current contract, need to find which year we're in
start_day = contract.contract_start
for plus_year in range(0,5):
if date(year=start_day.year + plus_year, month=start_day.month, day=start_day.day) < today:
contract.current_price = getattr(contract, 'hourly_rate_year' + str(plus_year + 1))
contract.contractor_site = line[9]
contracts.append(contract)
except Exception as e:
log.exception(e)
log.warning(line)
break
log.info("Inserting records")
Contract.objects.bulk_create(contracts)
log.info("Updating search index")
call_command('update_search_field', Contract._meta.app_label, Contract._meta.model_name)
log.info("End load_data task")
