def __row_to_contract(self, row):
if not (row['Labor Category'] and row['Contract Year'] and row['Year 1/base']):
return None
contract = Contract()
contract.idv_piid = row['CONTRACT .']
contract.labor_category = row['Labor Category'].strip().replace('\n', ' ')
contract.vendor_name = row['COMPANY NAME']
contract.education_level = contract.get_education_code(row['Education'])
contract.schedule = row['Schedule']
contract.business_size = row['Bus Size']
contract.contract_year = row['Contract Year']
contract.sin = row['SIN NUMBER']
contract.hourly_rate_year1 = contract.normalize_rate(str(row['Year 1/base']))
contract.contractor_site = row['Location']
if row['Begin Date']:
contract.contract_start = datetime.strptime(row['Begin Date'], '%m/%d/%Y').date()
if row['End Date']:
contract.contract_end = datetime.strptime(row['End Date'], '%m/%d/%Y').date()
contract.min_years_experience = int(row['MinExpAct']) if row['MinExpAct'].isdigit() else 0
for count, rate in enumerate(row[2:6]):
if rate:
setattr(contract, 'hourly_rate_year{}'.format(count + 2), contract.normalize_rate(str(rate)))
self.__generate_contract_rate_years(row, contract)
return contract
def handle(self, *args, **options):
data_file = csv.reader(open(os.path.join(settings.BASE_DIR, 'contracts/docs/hourly_prices.csv'), 'r'))
#skip header row
next(data_file)
for line in data_file:
try:
if line[0]:
#create contract record, unique to vendor, labor cat
idv_piid = line[0]
vendor_name = line[3]
labor_category = line[4].strip().replace('\n', ' ')
try:
contract = Contract.objects.get(idv_piid=idv_piid, labor_category=labor_category, vendor_name=vendor_name)
except Contract.DoesNotExist:
contract = Contract()
contract.idv_piid = idv_piid
contract.labor_category = labor_category
contract.vendor_name = vendor_name
contract.education_level = contract.get_education_code(line[5])
if line[1] != '':
contract.contract_start = datetime.strptime(line[1], '%m/%d/%Y').date()
if line[2] != '':
contract.contract_end = datetime.strptime(line[2], '%m/%d/%Y').date()
if line[6].strip() != '':
contract.min_years_experience = line[6]
else:
contract.min_years_experience = 0
if line[7] and line[7] != '':
contract.hourly_rate_year1 = contract.normalize_rate(line[7])
else:
#there's no pricing info
continue
for count, rate in enumerate(line[8:12]):
if rate and rate.strip() != '':
setattr(contract, 'hourly_rate_year' + str(count+2), contract.normalize_rate(rate))
contract.contractor_site = line[12]
contract.save()
except Exception as e:
print(e)
print(line)
break
def __row_to_contract(self, row):
if not (row['Labor Category'] and row['Contract Year']
and row['Year 1/base']):
return None
contract = Contract()
contract.idv_piid = row['CONTRACT .']
contract.labor_category = row['Labor Category'].strip().replace(
'\n', ' ')
contract.vendor_name = row['COMPANY NAME']
contract.education_level = contract.get_education_code(
row['Education'])
contract.schedule = row['Schedule']
contract.business_size = row['Bus Size']
contract.contract_year = row['Contract Year']
contract.sin = row['SIN NUMBER']
contract.hourly_rate_year1 = contract.normalize_rate(
str(row['Year 1/base']))
contract.contractor_site = row['Location']
if row['Begin Date']:
contract.contract_start = datetime.strptime(
row['Begin Date'], '%m/%d/%Y').date()
if row['End Date']:
contract.contract_end = datetime.strptime(row['End Date'],
'%m/%d/%Y').date()
contract.min_years_experience = int(
row['MinExpAct']) if row['MinExpAct'].isdigit() else 0
for count, rate in enumerate(row[2:6]):
if rate:
setattr(contract, 'hourly_rate_year{}'.format(count + 2),
contract.normalize_rate(str(rate)))
self.__generate_contract_rate_years(row, contract)
return contract
def handle(self, *args, **options):
log = logging.getLogger(__name__)
log.info("Begin load_data task")
log.info("Deleting existing contract records")
Contract.objects.all().delete()
data_file = csv.reader(
open(
os.path.join(settings.BASE_DIR,
'contracts/docs/hourly_prices.csv'), 'r'))
#skip header row
next(data_file)
#used for removing expired contracts
today = date.today()
contracts = []
log.info("Processing new datafile")
for line in data_file:
#replace annoying msft carraige return
for num in range(0, len(line)):
#also replace version with capital D
line[num] = line[num].replace("_x000d_",
"").replace("_x000D_", "")
try:
if line[0]:
#create contract record, unique to vendor, labor cat
idv_piid = line[11]
vendor_name = line[10]
labor_category = line[0].strip().replace('\n', ' ')
contract = Contract()
contract.idv_piid = idv_piid
contract.labor_category = labor_category
contract.vendor_name = vendor_name
contract.education_level = contract.get_education_code(
line[6])
contract.schedule = line[12]
contract.business_size = line[8]
contract.contract_year = line[14]
contract.sin = line[13]
if line[15] != '':
contract.contract_start = datetime.strptime(
line[15], '%m/%d/%Y').date()
if line[16] != '':
contract.contract_end = datetime.strptime(
line[16], '%m/%d/%Y').date()
if line[7].strip() != '':
contract.min_years_experience = line[7]
else:
contract.min_years_experience = 0
if line[1] and line[1] != '':
contract.hourly_rate_year1 = contract.normalize_rate(
line[1])
else:
#there's no pricing info
continue
for count, rate in enumerate(line[2:6]):
if rate and rate.strip() != '':
setattr(contract,
'hourly_rate_year' + str(count + 2),
contract.normalize_rate(rate))
if line[14] and line[14] != '':
price_fields = {
'current_price':
getattr(contract,
'hourly_rate_year' + str(line[14]), 0)
}
current_year = int(line[14])
# we have up to five years of rate data
if current_year < 5:
price_fields['next_year_price'] = getattr(
contract,
'hourly_rate_year' + str(current_year + 1), 0)
if current_year < 4:
price_fields['second_year_price'] = getattr(
contract,
'hourly_rate_year' + str(current_year + 2),
0)
# don't create display prices for records where the rate
# is under the federal minimum contract rate
for field in price_fields:
price = price_fields.get(field)
if price and price >= FEDERAL_MIN_CONTRACT_RATE:
setattr(contract, field, price)
contract.contractor_site = line[9]
contracts.append(contract)
except Exception as e:
log.exception(e)
log.warning(line)
break
log.info("Inserting records")
Contract.objects.bulk_create(contracts)
log.info("Updating search index")
call_command('update_search_field', Contract._meta.app_label,
Contract._meta.model_name)
log.info("End load_data task")
def make_contract(cls, line, upload_source=None):
if line[0]:
# create contract record, unique to vendor, labor cat
idv_piid = line[11]
vendor_name = line[10]
labor_category = line[0].strip().replace('\n', ' ')
contract = Contract()
contract.idv_piid = idv_piid
contract.labor_category = labor_category
contract.vendor_name = vendor_name
contract.education_level = contract.get_education_code(
line[6]
)
contract.schedule = line[12]
contract.business_size = line[8]
current_contract_year = int(float(line[14]))
contract.contract_year = current_contract_year
contract.sin = line[13]
if line[15] != '':
contract.contract_start = datetime.strptime(
line[15], '%m/%d/%Y').date()
if line[16] != '':
contract.contract_end = datetime.strptime(
line[16], '%m/%d/%Y').date()
if line[7].strip() != '':
contract.min_years_experience = int(float(line[7]))
else:
contract.min_years_experience = 0
if line[1] and line[1] != '':
contract.hourly_rate_year1 = contract.normalize_rate(
line[1]
)
else:
# there's no pricing info
raise ValueError('missing price')
for count, rate in enumerate(line[2:6]):
if rate and rate.strip() != '':
setattr(contract, 'hourly_rate_year' +
str(count + 2),
contract.normalize_rate(rate))
if line[14] and line[14] != '':
price_fields = {
'current_price': getattr(contract,
'hourly_rate_year' +
str(current_contract_year), 0)
}
# we have up to five years of rate data
if current_contract_year < 5:
price_fields['next_year_price'] = getattr(
contract, 'hourly_rate_year' +
str(current_contract_year + 1), 0
)
if current_contract_year < 4:
price_fields['second_year_price'] = getattr(
contract, 'hourly_rate_year' +
str(current_contract_year + 2), 0
)
# don't create display prices for records where the
# rate is under the federal minimum contract rate
for field in price_fields:
price = price_fields.get(field)
if price and price >= FEDERAL_MIN_CONTRACT_RATE:
setattr(contract, field, price)
contract.contractor_site = line[9]
if upload_source:
contract.upload_source = upload_source
return contract
def handle(self, *args, **options):
log = logging.getLogger(__name__)
log.info("Begin load_data task")
log.info("Deleting existing contract records")
Contract.objects.all().delete()
data_file = csv.reader(open(os.path.join(settings.BASE_DIR, "contracts/docs/hourly_prices.csv"), "r"))
# skip header row
next(data_file)
# used for removing expired contracts
today = date.today()
contracts = []
log.info("Processing new datafile")
for line in data_file:
# replace annoying msft carraige return
for num in range(0, len(line)):
# also replace version with capital D
line[num] = line[num].replace("_x000d_", "").replace("_x000D_", "")
try:
if line[0]:
# create contract record, unique to vendor, labor cat
idv_piid = line[11]
vendor_name = line[10]
labor_category = line[0].strip().replace("\n", " ")
contract = Contract()
contract.idv_piid = idv_piid
contract.labor_category = labor_category
contract.vendor_name = vendor_name
contract.education_level = contract.get_education_code(line[6])
contract.schedule = line[12]
contract.business_size = line[8]
contract.contract_year = line[14]
contract.sin = line[13]
if line[15] != "":
contract.contract_start = datetime.strptime(line[15], "%m/%d/%Y").date()
if line[16] != "":
contract.contract_end = datetime.strptime(line[16], "%m/%d/%Y").date()
if line[7].strip() != "":
contract.min_years_experience = line[7]
else:
contract.min_years_experience = 0
if line[1] and line[1] != "":
contract.hourly_rate_year1 = contract.normalize_rate(line[1])
else:
# there's no pricing info
continue
for count, rate in enumerate(line[2:6]):
if rate and rate.strip() != "":
setattr(contract, "hourly_rate_year" + str(count + 2), contract.normalize_rate(rate))
# don't create current price for records where the rate
# is under the federal minimum contract rate
current_price = getattr(contract, "hourly_rate_year" + str(line[14]))
if current_price and current_price >= FEDERAL_MIN_CONTRACT_RATE:
contract.current_price = current_price
contract.contractor_site = line[9]
contracts.append(contract)
except Exception as e:
log.exception(e)
log.warning(line)
break
log.info("Inserting records")
Contract.objects.bulk_create(contracts)
log.info("Updating search index")
call_command("update_search_field", Contract._meta.app_label, Contract._meta.model_name)
log.info("End load_data task")
def make_contract(cls, line, upload_source=None):
if line[0]:
# create contract record, unique to vendor, labor cat
idv_piid = line[11]
vendor_name = line[10]
labor_category = line[0].strip().replace('\n', ' ')
contract = Contract()
contract.idv_piid = idv_piid
contract.labor_category = labor_category
contract.vendor_name = vendor_name
contract.education_level = contract.get_education_code(
line[6]
)
contract.schedule = line[12]
contract.business_size = line[8]
current_contract_year = int(float(line[14]))
contract.contract_year = current_contract_year
contract.sin = line[13]
if line[15] != '':
contract.contract_start = datetime.strptime(
line[15], '%m/%d/%Y').date()
if line[16] != '':
contract.contract_end = datetime.strptime(
line[16], '%m/%d/%Y').date()
if line[7].strip() != '':
contract.min_years_experience = int(float(line[7]))
else:
contract.min_years_experience = 0
if line[1] and line[1] != '':
contract.hourly_rate_year1 = contract.normalize_rate(
line[1]
)
else:
# there's no pricing info
raise ValueError('missing price')
for count, rate in enumerate(line[2:6]):
if rate and rate.strip() != '':
setattr(contract, 'hourly_rate_year' +
str(count + 2),
contract.normalize_rate(rate))
if line[14] and line[14] != '':
price_fields = {
'current_price': getattr(contract,
'hourly_rate_year' +
str(current_contract_year), 0)
}
# we have up to five years of rate data
if current_contract_year < 5:
price_fields['next_year_price'] = getattr(
contract, 'hourly_rate_year' +
str(current_contract_year + 1), 0
)
if current_contract_year < 4:
price_fields['second_year_price'] = getattr(
contract, 'hourly_rate_year' +
str(current_contract_year + 2), 0
)
# don't create display prices for records where the
# rate is under the federal minimum contract rate
for field in price_fields:
price = price_fields.get(field)
if price and price >= FEDERAL_MIN_CONTRACT_RATE:
setattr(contract, field, price)
contract.contractor_site = line[9]
if upload_source:
contract.upload_source = upload_source
return contract
else:
raise ValueError('missing labor category')
def handle(self, *args, **options):
log = logging.getLogger(__name__)
log.info("Begin load_data task")
log.info("Deleting existing contract records")
Contract.objects.all().delete()
data_file = csv.reader(open(os.path.join(settings.BASE_DIR, 'contracts/docs/hourly_prices.csv'), 'r'))
#skip header row
next(data_file)
#used for removing expired contracts
today = date.today()
contracts = []
log.info("Processing new datafile")
for line in data_file:
#replace annoying msft carraige return
for num in range(0, len(line)):
#also replace version with capital D
line[num] = line[num].replace("_x000d_", "").replace("_x000D_", "")
try:
if line[0]:
#create contract record, unique to vendor, labor cat
idv_piid = line[11]
vendor_name = line[10]
labor_category = line[0].strip().replace('\n', ' ')
contract = Contract()
contract.idv_piid = idv_piid
contract.labor_category = labor_category
contract.vendor_name = vendor_name
contract.education_level = contract.get_education_code(line[6])
contract.schedule = line[12]
contract.business_size = line[8]
contract.sin = line[13]
if line[14] != '':
contract.contract_start = datetime.strptime(line[14], '%m/%d/%Y').date()
if line[15] != '':
contract.contract_end = datetime.strptime(line[15], '%m/%d/%Y').date()
if line[7].strip() != '':
contract.min_years_experience = line[7]
else:
contract.min_years_experience = 0
if line[1] and line[1] != '':
contract.hourly_rate_year1 = contract.normalize_rate(line[1])
else:
#there's no pricing info
continue
for count, rate in enumerate(line[2:6]):
if rate and rate.strip() != '':
setattr(contract, 'hourly_rate_year' + str(count+2), contract.normalize_rate(rate))
if contract.contract_end > today and contract.contract_start < today:
#it's a current contract, need to find which year we're in
start_day = contract.contract_start
for plus_year in range(0,5):
if date(year=start_day.year + plus_year, month=start_day.month, day=start_day.day) < today:
contract.current_price = getattr(contract, 'hourly_rate_year' + str(plus_year + 1))
contract.contractor_site = line[9]
contracts.append(contract)
except Exception as e:
log.exception(e)
log.warning(line)
break
log.info("Inserting records")
Contract.objects.bulk_create(contracts)
log.info("Updating search index")
call_command('update_search_field', Contract._meta.app_label, Contract._meta.model_name)
log.info("End load_data task")
