def getDictionaryFromRevAuc(raw, datasource):
try:
d = datetime.datetime.strptime(raw.data[21].strip(' \t\n\r'),
"%m/%d/%Y")
return { \
DATASOURCE : datasource, \
UNITS : replaceUndumpableData(raw.data[37]) , \
PRICE : replaceUndumpableData(raw.data[38]), \
AGENCY : replaceUndumpableData(raw.data[3]) , \
VENDOR : replaceUndumpableData(raw.data[29]) , \
PSC : replaceUndumpableData(raw.data[13]) , \
DESCR : replaceUndumpableData(raw.data[24]), \
LONGDESCR : replaceUndumpableData(raw.data[35]) , \
# This needs to be put in a standard format and sorted properly.
DATE : replaceUndumpableData(d.date().isoformat()), \
# here begin some less-standard fields
AWARDIDIDV : replaceUndumpableData(raw.data[19]) \
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_traceback,
limit=2,
file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromRevAuc(raw, datasource):
try:
d = datetime.datetime.strptime(raw.data[21].strip(" \t\n\r"), "%m/%d/%Y")
return {
DATASOURCE: datasource,
UNITS: replaceUndumpableData(raw.data[37]),
PRICE: replaceUndumpableData(raw.data[38]),
AGENCY: replaceUndumpableData(raw.data[3]),
VENDOR: replaceUndumpableData(raw.data[29]),
PSC: replaceUndumpableData(raw.data[13]),
DESCR: replaceUndumpableData(raw.data[24]),
LONGDESCR: replaceUndumpableData(raw.data[35]),
# This needs to be put in a standard format and sorted properly.
DATE: replaceUndumpableData(d.date().isoformat()),
# here begin some less-standard fields
AWARDIDIDV: replaceUndumpableData(raw.data[19]),
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback, limit=2, file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromEDWGSAAdv(raw, datasource):
try:
# Choosing the "Charge Processing Date" as the official date"
#d = datetime.datetime.strptime(raw.data[6].strip(' \t\n\r'),"%m/%d/%Y")
d = datetime.datetime.strptime(raw.data[6].strip(' \t\n\r'),
"%b %d %Y")
return { \
DATASOURCE : datasource, \
UNITS : tryToInferUnitsFromDescriptionOrDefaultToOne(replaceUndumpableData(raw.data[3])), \
PRICE : replaceUndumpableData(raw.data[2]), \
AGENCY : replaceUndumpableData(raw.data[9]), \
VENDOR : replaceUndumpableData(raw.data[5]), \
# I know all of this data is office supplies---this may not be too accurate
# but it matches
DESCR : replaceUndumpableData(raw.data[7]), \
LONGDESCR : replaceUndumpableData(raw.data[7]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
AWARDIDIDV : "GSAAdv", \
"GSA Schedule Number" : replaceUndumpableData(raw.data[12]),\
"Special Item Number" : replaceUndumpableData(raw.data[13]),\
MANUFACTURER_NAME : replaceUndumpableData(raw.data[1]), \
MANUFACTURER_PART_NUMBER : replaceUndumpableData(raw.data[0]), \
BUREAU : replaceUndumpableData(raw.data[10]), \
CONTRACT_NUMBER : replaceUndumpableData(raw.data[11]), \
TO_ZIP_CODE : replaceUndumpableData(ensureZipCodeHasFiveDigits(raw.data[15])), \
FROM_ZIP_CODE : replaceUndumpableData(ensureZipCodeHasFiveDigits(raw.data[14])) \
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_traceback,
limit=2,
file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromGSAAdv(raw,datasource):
try:
d = datetime.datetime.strptime(raw.data[6].strip(' \t\n\r'),"%b %d %Y")
return { \
DATASOURCE : datasource, \
UNITS : replaceUndumpableData(raw.data[3]), \
PRICE : replaceUndumpableData(raw.data[2]), \
AGENCY : replaceUndumpableData(raw.data[10]), \
VENDOR : replaceUndumpableData(raw.data[5]), \
# We are loading the "SIN" special item number field as the PSC for now.
# I don't think this data contains a PSC code!
# PSC : '', \
DESCR : replaceUndumpableData(raw.data[7]), \
LONGDESCR : replaceUndumpableData(raw.data[8]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
AWARDIDIDV : "GSA Advantage", \
"GSA Schedule Number" : replaceUndumpableData(raw.data[13]),\
"Special Item Number" : replaceUndumpableData(raw.data[14]),\
UNIT_OF_ISSUE : replaceUndumpableData(raw.data[9]),\
MANUFACTURER_NAME : replaceUndumpableData(raw.data[1]), \
MANUFACTURER_PART_NUMBER : replaceUndumpableData(raw.data[0]), \
BUREAU : replaceUndumpableData(raw.data[11]), \
CONTRACT_NUMBER : replaceUndumpableData(raw.data[12]), \
TO_ZIP_CODE : replaceUndumpableData(ensureZipCodeHasFiveDigits(raw.data[16])), \
FROM_ZIP_CODE : replaceUndumpableData(ensureZipCodeHasFiveDigits(raw.data[15])) \
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromUSASpending(raw,datasource):
try:
# Choosing the "Charge Processing Date" as the official date"
d = datetime.datetime.strptime(raw.data[14].strip(' \t\n\r'),"%m/%d/%Y")
return { \
DATASOURCE : datasource, \
DESCR : replaceUndumpableData(raw.data[31]), \
UNITS : tryToInferUnitsFromDescriptionOrDefaultToOne(replaceUndumpableData(raw.data[32])), \
PRICE : replaceUndumpableData(raw.data[4]), \
AGENCY : replaceUndumpableData(raw.data[5]), \
VENDOR : replaceUndumpableData(raw.data[43]), \
# I know all of this data is office supplies---this may not be too accurate
# but it matches
PSC : replaceUndumpableData(raw.data[80]), \
"product_service_code" : replaceUndumpableData(raw.data[80]), \
"naics_code" : replaceUndumpableData(raw.data[109]), \
LONGDESCR : replaceUndumpableData(raw.data[31]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
TO_ZIP_CODE : replaceUndumpableData(raw.data[63]), \
"street_address" : replaceUndumpableData(raw.data[52]), \
"city" : replaceUndumpableData(raw.data[55]), \
"state" : replaceUndumpableData(raw.data[56]), \
"vendor_state_code" : replaceUndumpableData(raw.data[59]), \
"congressionaldistrict" : replaceUndumpableData(raw.data[58]), \
"duns_number" : replaceUndumpableData(raw.data[64]), \
"phoneno" : replaceUndumpableData(raw.data[67]), \
"extent_competed" : replaceUndumpableData(raw.data[103]), \
"reason_not_competed" : replaceUndumpableData(raw.data[104]), \
AWARDIDIDV : replaceUndumpableData("USASpending")
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromEDWGSAAdv(raw, datasource):
try:
# Choosing the "Charge Processing Date" as the official date"
#d = datetime.datetime.strptime(raw.data[5].strip(' \t\n\r'),"%m/%d/%Y")
d = datetime.datetime.strptime(raw.data[5].strip(' \t\n\r'),
"%b %d %Y")
return { \
AGENCY : replaceUndumpableData(raw.data[0]), \
AWARDIDIDV : replaceUndumpableData(raw.data[1]), \
BUREAU : replaceUndumpableData(raw.data[2]), \
CONTRACT_NUMBER : replaceUndumpableData(raw.data[3]), \
DATASOURCE : replaceUndumpableData(raw.data[4]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
DESCR : replaceUndumpableData(raw.data[6]), \
FROM_ZIP_CODE : replaceUndumpableData(ensureZipCodeHasFiveDigits(raw.data[7])), \
LONGDESCR : replaceUndumpableData(raw.data[8]),
MANUFACTURER_NAME : replaceUndumpableData(raw.data[9]), \
MANUFACTURER_PART_NUMBER : replaceUndumpableData(raw.data[10]), \
PRICE : replaceUndumpableData(raw.data[11]), \
PSC : replaceUndumpableData(raw.data[12]), \
TO_ZIP_CODE : replaceUndumpableData(ensureZipCodeHasFiveDigits(raw.data[13])), \
UNIT_OF_ISSUE : replaceUndumpableData(raw.data[14]), \
UNITS : tryToInferUnitsFromDescriptionOrDefaultToOne(replaceUndumpableData(raw.data[15])), \
VENDOR : replaceUndumpableData(raw.data[16]), \
EXTENDED_PRICE : replaceUndumpableData(raw.data[17]), \
PRODUCT_DESCRIPTION : replaceUndumpableData(raw.data[18]), \
QUANTITY : replaceUndumpableData(raw.data[19]),
UNSPSC_CODE : replaceUndumpableData(raw.data[20]),
ORDERING_PROCESS : replaceUndumpableData(raw.data[21]), \
PSC_DESCRIPTION : replaceUndumpableData(raw.data[22]), \
DUNS_NUMBER: replaceUndumpableData(raw.data[23]), \
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_traceback,
limit=2,
file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromStandard(raw,datasource):
logger.error('RAW:'+repr(raw.data))
try:
# d = datetime.datetime.strptime(raw.data[21].strip(' \t\n\r'),"%m/%d/%Y")
d = datetime.datetime.today()
logger.error('RAW0:'+replaceUndumpableData(raw.data[0]))
logger.error('RAW1:'+replaceUndumpableData(raw.data[1]))
logger.error('RAW2:'+replaceUndumpableData(raw.data[2]))
logger.error('RAW3:'+replaceUndumpableData(raw.data[3]))
logger.error('RAW4:'+replaceUndumpableData(raw.data[4]))
logger.error('RAW5:'+replaceUndumpableData(raw.data[5]))
logger.error('RAW6:'+replaceUndumpableData(raw.data[6]))
logger.error('RAW7:'+replaceUndumpableData(raw.data[7]))
logger.error('RAW8:'+replaceUndumpableData(raw.data[8]))
return { \
UNITS : replaceUndumpableData(raw.data[0]),\
PRICE : replaceUndumpableData(raw.data[1]),\
AGENCY : replaceUndumpableData(raw.data[2]),\
VENDOR : replaceUndumpableData(raw.data[3]),\
PSC : replaceUndumpableData(raw.data[4]),\
DESCR : replaceUndumpableData(raw.data[5]),\
LONGDESCR : replaceUndumpableData(raw.data[6]),\
DATE : replaceUndumpableData(raw.data[7]),\
AWARDIDIDV : replaceUndumpableData(raw.data[8]),\
# This needs to be put in a standard format and sorted properly.
DATE : replaceUndumpableData(d.date().isoformat()) \
}
except:
logger.error("don't know what went wrong here"+repr(sys.exc_info()[0]))
return {}
def getDictionaryFromLabEquipment(raw, datasource):
try:
d = datetime.datetime.strptime(raw.data[21].strip(' \t\n\r'),
"%m/%d/%Y")
return { \
DATASOURCE : datasource, \
UNITS : replaceUndumpableData(raw.data[37]), \
PRICE : replaceUndumpableData(raw.data[38]), \
AGENCY : replaceUndumpableData(raw.data[1]), \
VENDOR : replaceUndumpableData(raw.data[29]), \
PSC : replaceUndumpableData(raw.data[13]), \
DESCR : replaceUndumpableData(raw.data[35]), \
LONGDESCR : replaceUndumpableData(raw.data[36]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
AWARDIDIDV : replaceUndumpableData(raw.data[19]), \
"Point of Contact" : replaceUndumpableData(raw.data[2]), \
"Buyer_Division" : replaceUndumpableData(raw.data[3]), \
"Category" :replaceUndumpableData(raw.data[24]), \
"Seller Type" : replaceUndumpableData(raw.data[25]), \
"Seller Award Type" : replaceUndumpableData(raw.data[26]), \
"Purchase Description" : replaceUndumpableData(raw.data[31]), \
"Set Aside Type" : replaceUndumpableData(raw.data[32]), \
"Commodity Type" : replaceUndumpableData(raw.data[12]), \
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_traceback,
limit=2,
file=sys.stderr)
logger.error("don't know what went wrong here")
def getDictionaryFromEDWGSAAdv(raw,datasource):
try:
# Choosing the "Charge Processing Date" as the official date"
#d = datetime.datetime.strptime(raw.data[5].strip(' \t\n\r'),"%m/%d/%Y")
d = datetime.datetime.strptime(raw.data[5].strip(' \t\n\r'),"%b %d %Y")
return { \
AGENCY : replaceUndumpableData(raw.data[0]), \
AWARDIDIDV : replaceUndumpableData(raw.data[1]), \
BUREAU : replaceUndumpableData(raw.data[2]), \
CONTRACT_NUMBER : replaceUndumpableData(raw.data[3]), \
DATASOURCE : replaceUndumpableData(raw.data[4]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
DESCR : replaceUndumpableData(raw.data[6]), \
FROM_ZIP_CODE : replaceUndumpableData(ensureZipCodeHasFiveDigits(raw.data[7])), \
LONGDESCR : replaceUndumpableData(raw.data[8]),
MANUFACTURER_NAME : replaceUndumpableData(raw.data[9]), \
MANUFACTURER_PART_NUMBER : replaceUndumpableData(raw.data[10]), \
PRICE : replaceUndumpableData(raw.data[11]), \
PSC : replaceUndumpableData(raw.data[12]), \
TO_ZIP_CODE : replaceUndumpableData(ensureZipCodeHasFiveDigits(raw.data[13])), \
UNIT_OF_ISSUE : replaceUndumpableData(raw.data[14]), \
UNITS : tryToInferUnitsFromDescriptionOrDefaultToOne(replaceUndumpableData(raw.data[15])), \
VENDOR : replaceUndumpableData(raw.data[16]), \
EXTENDED_PRICE : replaceUndumpableData(raw.data[17]), \
PRODUCT_DESCRIPTION : replaceUndumpableData(raw.data[18]), \
QUANTITY : replaceUndumpableData(raw.data[19]),
UNSPSC_CODE : replaceUndumpableData(raw.data[20]),
ORDERING_PROCESS : replaceUndumpableData(raw.data[21]), \
PSC_DESCRIPTION : replaceUndumpableData(raw.data[22]), \
DUNS_NUMBER: replaceUndumpableData(raw.data[23]), \
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromGSAAdv(raw, datasource):
try:
d = datetime.datetime.strptime(raw.data[6].strip(' \t\n\r'),
"%b %d %Y")
return { \
DATASOURCE : datasource, \
UNITS : replaceUndumpableData(raw.data[3]), \
PRICE : replaceUndumpableData(raw.data[2]), \
AGENCY : replaceUndumpableData(raw.data[10]), \
VENDOR : replaceUndumpableData(raw.data[5]), \
# We are loading the "SIN" special item number field as the PSC for now.
# I don't think this data contains a PSC code!
# PSC : '', \
DESCR : replaceUndumpableData(raw.data[7]), \
LONGDESCR : replaceUndumpableData(raw.data[8]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
AWARDIDIDV : "GSA Advantage", \
"GSA Schedule Number" : replaceUndumpableData(raw.data[13]),\
"Special Item Number" : replaceUndumpableData(raw.data[14]),\
UNIT_OF_ISSUE : replaceUndumpableData(raw.data[9]),\
MANUFACTURER_NAME : replaceUndumpableData(raw.data[1]), \
MANUFACTURER_PART_NUMBER : replaceUndumpableData(raw.data[0]), \
BUREAU : replaceUndumpableData(raw.data[11]), \
CONTRACT_NUMBER : replaceUndumpableData(raw.data[12]), \
TO_ZIP_CODE : replaceUndumpableData(ensureZipCodeHasFiveDigits(raw.data[16])), \
FROM_ZIP_CODE : replaceUndumpableData(ensureZipCodeHasFiveDigits(raw.data[15])) \
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_traceback,
limit=2,
file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromLabEquipment (raw, datasource):
try:
d = datetime.datetime.strptime(raw.data[21].strip(' \t\n\r'),"%m/%d/%Y")
return { \
DATASOURCE : datasource, \
UNITS : replaceUndumpableData(raw.data[37]), \
PRICE : replaceUndumpableData(raw.data[38]), \
AGENCY : replaceUndumpableData(raw.data[1]), \
VENDOR : replaceUndumpableData(raw.data[29]), \
PSC : replaceUndumpableData(raw.data[13]), \
DESCR : replaceUndumpableData(raw.data[35]), \
LONGDESCR : replaceUndumpableData(raw.data[36]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
AWARDIDIDV : replaceUndumpableData(raw.data[19]), \
"Point of Contact" : replaceUndumpableData(raw.data[2]), \
"Buyer_Division" : replaceUndumpableData(raw.data[3]), \
"Category" :replaceUndumpableData(raw.data[24]), \
"Seller Type" : replaceUndumpableData(raw.data[25]), \
"Seller Award Type" : replaceUndumpableData(raw.data[26]), \
"Purchase Description" : replaceUndumpableData(raw.data[31]), \
"Set Aside Type" : replaceUndumpableData(raw.data[32]), \
"Commodity Type" : replaceUndumpableData(raw.data[12]), \
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stderr)
logger.error("don't know what went wrong here")
def getDictionaryFromOS2(raw, datasource):
try:
# Choosing the "Charge Processing Date" as the official date"
d = datetime.datetime.strptime(raw.data[57].strip(' \t\n\r'),
"%m-%d-%Y")
return { \
DATASOURCE : datasource, \
UNITS : replaceUndumpableData(raw.data[16]), \
PRICE : replaceUndumpableData(raw.data[19]), \
AGENCY : replaceUndumpableData(raw.data[48]), \
VENDOR : replaceUndumpableData(raw.data[64]), \
# I know all of this data is office supplies---this may not be too accurate
# but it matches
PSC : replaceUndumpableData(raw.data[17]), \
DESCR : replaceUndumpableData(raw.data[5]), \
# DANGER! HACK!
# I think the OS2 data has a better version than this!
LONGDESCR : replaceUndumpableData(raw.data[5]), \
# Choosing the "Charge Processing Date" as the official date"
DATE : replaceUndumpableData(d.date().isoformat()), \
AWARDIDIDV : replaceUndumpableData("GSA Schedule-75"), \
# here begin some less-standard fields
# This data has significantly more fields--I am simply
# selecting the most salient. I think the reality is this sort
# of analysis should be done in a crowd-source, "datapalooza" type approach.
"Order Number" : replaceUndumpableData(raw.data[4]), \
MANUFACTURER_NAME : replaceUndumpableData(raw.data[7]), \
"isAbitlityOne" : replaceUndumpableData(raw.data[8]), \
MANUFACTURER_PART_NUMBER : replaceUndumpableData(raw.data[10]), \
"Revised SubCategory" : replaceUndumpableData(raw.data[13]), \
"Revised Category" : replaceUndumpableData(raw.data[14]), \
UNIT_OF_ISSUE : replaceUndumpableData(raw.data[15]), \
"UNSPSC" : replaceUndumpableData(raw.data[18]), \
"Revised Dbt_Crdt_ind" : replaceUndumpableData(raw.data[37]), \
"EEP_Ind" : replaceUndumpableData(raw.data[38]), \
"CPG_Ind" : replaceUndumpableData(raw.data[39]), \
"Comp_Remain_Toner" : replaceUndumpableData(raw.data[40]), \
"Post_Cons_Percent" : replaceUndumpableData(raw.data[41]), \
"Tot_Recyc_Percent" : replaceUndumpableData(raw.data[42]), \
"Dlvry_Method" : replaceUndumpableData(raw.data[43]), \
"Freight Charge" : replaceUndumpableData(raw.data[44]), \
"Shipping Weight" : replaceUndumpableData(raw.data[45]), \
"Sub_Agency1" : replaceUndumpableData(raw.data[52]), \
"Sub_Agency2" : replaceUndumpableData(raw.data[53]), \
"Sub_Agency3" : replaceUndumpableData(raw.data[54]), \
"MAJCOM" : replaceUndumpableData(raw.data[55]), \
"DODACC" : replaceUndumpableData(raw.data[56]), \
"Pay Date" : replaceUndumpableData(raw.data[57]), \
"Charge Processing Date" : replaceUndumpableData(raw.data[62]), \
"Transaction Number" : replaceUndumpableData(raw.data[63]), \
"Revised Socio Status" : replaceUndumpableData(raw.data[66]), \
CONTRACT_NUMBER : replaceUndumpableData(raw.data[68]), \
"Revised Ord_date" : replaceUndumpableData(raw.data[1]), \
"Report Month" : replaceUndumpableData(raw.data[3]), \
"Ord_num" : replaceUndumpableData(raw.data[4]), \
"FSC" : replaceUndumpableData(raw.data[17]), \
"Dbt_Crdt_Ind" : replaceUndumpableData(raw.data[36]), \
"Tot_Recyc_Percent" : replaceUndumpableData(raw.data[42]), \
"Dlvry_Method" : replaceUndumpableData(raw.data[43]), \
"Pay_Method" : replaceUndumpableData(raw.data[56]), \
TO_ZIP_CODE : replaceUndumpableData(raw.data[60]), \
FROM_ZIP_CODE : replaceUndumpableData(raw.data[61]) \
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_traceback,
limit=2,
file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromOS2(raw,datasource):
try:
# Choosing the "Charge Processing Date" as the official date"
d = datetime.datetime.strptime(raw.data[57].strip(' \t\n\r'),"%m-%d-%Y")
return { \
DATASOURCE : datasource, \
UNITS : replaceUndumpableData(raw.data[16]), \
PRICE : replaceUndumpableData(raw.data[19]), \
AGENCY : replaceUndumpableData(raw.data[48]), \
VENDOR : replaceUndumpableData(raw.data[64]), \
# I know all of this data is office supplies---this may not be too accurate
# but it matches
PSC : replaceUndumpableData(raw.data[17]), \
DESCR : replaceUndumpableData(raw.data[5]), \
# DANGER! HACK!
# I think the OS2 data has a better version than this!
LONGDESCR : replaceUndumpableData(raw.data[5]), \
# Choosing the "Charge Processing Date" as the official date"
DATE : replaceUndumpableData(d.date().isoformat()), \
AWARDIDIDV : replaceUndumpableData("GSA Schedule-75"), \
# here begin some less-standard fields
# This data has significantly more fields--I am simply
# selecting the most salient. I think the reality is this sort
# of analysis should be done in a crowd-source, "datapalooza" type approach.
"Order Number" : replaceUndumpableData(raw.data[4]), \
MANUFACTURER_NAME : replaceUndumpableData(raw.data[7]), \
"isAbitlityOne" : replaceUndumpableData(raw.data[8]), \
MANUFACTURER_PART_NUMBER : replaceUndumpableData(raw.data[10]), \
"Revised SubCategory" : replaceUndumpableData(raw.data[13]), \
"Revised Category" : replaceUndumpableData(raw.data[14]), \
UNIT_OF_ISSUE : replaceUndumpableData(raw.data[15]), \
"UNSPSC" : replaceUndumpableData(raw.data[18]), \
"Revised Dbt_Crdt_ind" : replaceUndumpableData(raw.data[37]), \
"EEP_Ind" : replaceUndumpableData(raw.data[38]), \
"CPG_Ind" : replaceUndumpableData(raw.data[39]), \
"Comp_Remain_Toner" : replaceUndumpableData(raw.data[40]), \
"Post_Cons_Percent" : replaceUndumpableData(raw.data[41]), \
"Tot_Recyc_Percent" : replaceUndumpableData(raw.data[42]), \
"Dlvry_Method" : replaceUndumpableData(raw.data[43]), \
"Freight Charge" : replaceUndumpableData(raw.data[44]), \
"Shipping Weight" : replaceUndumpableData(raw.data[45]), \
"Sub_Agency1" : replaceUndumpableData(raw.data[52]), \
"Sub_Agency2" : replaceUndumpableData(raw.data[53]), \
"Sub_Agency3" : replaceUndumpableData(raw.data[54]), \
"MAJCOM" : replaceUndumpableData(raw.data[55]), \
"DODACC" : replaceUndumpableData(raw.data[56]), \
"Pay Date" : replaceUndumpableData(raw.data[57]), \
"Charge Processing Date" : replaceUndumpableData(raw.data[62]), \
"Transaction Number" : replaceUndumpableData(raw.data[63]), \
"Revised Socio Status" : replaceUndumpableData(raw.data[66]), \
CONTRACT_NUMBER : replaceUndumpableData(raw.data[68]), \
"Revised Ord_date" : replaceUndumpableData(raw.data[1]), \
"Report Month" : replaceUndumpableData(raw.data[3]), \
"Ord_num" : replaceUndumpableData(raw.data[4]), \
"FSC" : replaceUndumpableData(raw.data[17]), \
"Dbt_Crdt_Ind" : replaceUndumpableData(raw.data[36]), \
"Tot_Recyc_Percent" : replaceUndumpableData(raw.data[42]), \
"Dlvry_Method" : replaceUndumpableData(raw.data[43]), \
"Pay_Method" : replaceUndumpableData(raw.data[56]), \
TO_ZIP_CODE : replaceUndumpableData(raw.data[60]), \
FROM_ZIP_CODE : replaceUndumpableData(raw.data[61]) \
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromUSASpending(raw,datasource):
try:
# Choosing the "Charge Processing Date" as the official date"
d = datetime.datetime.strptime(raw.data[14].strip(' \t\n\r'),"%m/%d/%Y")
return { \
DATASOURCE : datasource, \
DESCR : replaceUndumpableData(raw.data[31]), \
UNITS : tryToInferUnitsFromDescriptionOrDefaultToOne(replaceUndumpableData(raw.data[32])), \
PRICE : replaceUndumpableData(raw.data[4]), \
AGENCY : replaceUndumpableData(raw.data[5]), \
VENDOR : replaceUndumpableData(raw.data[43]), \
# I know all of this data is office supplies---this may not be too accurate
# but it matches
PSC : replaceUndumpableData(raw.data[80]), \
"product_service_code" : replaceUndumpableData(raw.data[80]), \
"naics_code" : replaceUndumpableData(raw.data[109]), \
LONGDESCR : replaceUndumpableData(raw.data[31]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
TO_ZIP_CODE : replaceUndumpableData(raw.data[63]), \
"street_address" : replaceUndumpableData(raw.data[52]), \
"city" : replaceUndumpableData(raw.data[55]), \
"state" : replaceUndumpableData(raw.data[56]), \
"vendor_state_code" : replaceUndumpableData(raw.data[59]), \
"congressionaldistrict" : replaceUndumpableData(raw.data[58]), \
"duns_number" : replaceUndumpableData(raw.data[64]), \
"phoneno" : replaceUndumpableData(raw.data[67]), \
"extent_competed" : replaceUndumpableData(raw.data[103]), \
"reason_not_competed" : replaceUndumpableData(raw.data[104]), \
AWARDIDIDV : replaceUndumpableData("USASpending")
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromHoustonSAP(raw,datasource):
try:
# Choosing the "Charge Processing Date" as the official date"
d = datetime.datetime.strptime(raw.data[3].strip(' \t\n\r'),"%m/%d/%Y")
return { \
DATASOURCE : datasource, \
LONGDESCR : replaceUndumpableData(raw.data[9]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
"Invoice Id" : replaceUndumpableData(raw.data[0]), \
"Invoice Line Item Id" : replaceUndumpableData(raw.data[1]), \
"Invoice Doc Type Id - Descr" : replaceUndumpableData(raw.data[2]), \
"Invoice Date" : replaceUndumpableData(raw.data[3]), \
"Invoice Post Date" : replaceUndumpableData(raw.data[4]), \
"Fisc Year" : replaceUndumpableData(raw.data[5]), \
"Invoice Vendor Id" : replaceUndumpableData(raw.data[6]), \
VENDOR : replaceUndumpableData(raw.data[7]), \
"Material Id" : replaceUndumpableData(raw.data[8]), \
DESCR : replaceUndumpableData(raw.data[9]), \
"Material Group ID" : replaceUndumpableData(raw.data[10]), \
"Material Group Desc" : replaceUndumpableData(raw.data[11]), \
"Material Class ID" : replaceUndumpableData(raw.data[12]), \
"Material Class Desc" : replaceUndumpableData(raw.data[13]), \
"Invoice Line Item Amount" : replaceUndumpableData(raw.data[14]), \
PRICE : replaceUndumpableData(convertFullFormatNumber(raw.data[14])), \
UNITS : replaceUndumpableData(convertUnitsToClosestInteger(raw.data[15])), \
"Uom Id" : replaceUndumpableData(raw.data[16]), \
UNIT_OF_ISSUE : replaceUndumpableData(raw.data[17]), \
"PO" : replaceUndumpableData(raw.data[18]), \
"PO Line" : replaceUndumpableData(raw.data[19]), \
"PO Doc Type" : replaceUndumpableData(raw.data[20]), \
"Purch Org Id" : replaceUndumpableData(raw.data[21]), \
"Purch Org Descr" : replaceUndumpableData(raw.data[22]), \
"Ext Purch Itm Catg Id - Descr" : replaceUndumpableData(raw.data[23]), \
"PO Short Text" : replaceUndumpableData(raw.data[24]), \
"PO Vendor" : replaceUndumpableData(raw.data[25]), \
AGENCY : replaceUndumpableData(raw.data[26]), \
"PO Purch Grp" : replaceUndumpableData(raw.data[27]), \
"OA" : replaceUndumpableData(raw.data[28]), \
AWARDIDIDV : replaceUndumpableData(raw.data[22])
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type, exc_value, exc_traceback,
limit=2, file=sys.stderr)
logger.error("don't know what went wrong here")
return {}
def getDictionaryFromHoustonSAP(raw, datasource):
try:
# Choosing the "Charge Processing Date" as the official date"
d = datetime.datetime.strptime(raw.data[3].strip(' \t\n\r'),
"%m/%d/%Y")
return { \
DATASOURCE : datasource, \
LONGDESCR : replaceUndumpableData(raw.data[9]), \
DATE : replaceUndumpableData(d.date().isoformat()), \
"Invoice Id" : replaceUndumpableData(raw.data[0]), \
"Invoice Line Item Id" : replaceUndumpableData(raw.data[1]), \
"Invoice Doc Type Id - Descr" : replaceUndumpableData(raw.data[2]), \
"Invoice Date" : replaceUndumpableData(raw.data[3]), \
"Invoice Post Date" : replaceUndumpableData(raw.data[4]), \
"Fisc Year" : replaceUndumpableData(raw.data[5]), \
"Invoice Vendor Id" : replaceUndumpableData(raw.data[6]), \
VENDOR : replaceUndumpableData(raw.data[7]), \
"Material Id" : replaceUndumpableData(raw.data[8]), \
DESCR : replaceUndumpableData(raw.data[9]), \
"Material Group ID" : replaceUndumpableData(raw.data[10]), \
"Material Group Desc" : replaceUndumpableData(raw.data[11]), \
"Material Class ID" : replaceUndumpableData(raw.data[12]), \
"Material Class Desc" : replaceUndumpableData(raw.data[13]), \
"Invoice Line Item Amount" : replaceUndumpableData(raw.data[14]), \
PRICE : replaceUndumpableData(convertFullFormatNumber(raw.data[14])), \
UNITS : replaceUndumpableData(convertUnitsToClosestInteger(raw.data[15])), \
"Uom Id" : replaceUndumpableData(raw.data[16]), \
UNIT_OF_ISSUE : replaceUndumpableData(raw.data[17]), \
"PO" : replaceUndumpableData(raw.data[18]), \
"PO Line" : replaceUndumpableData(raw.data[19]), \
"PO Doc Type" : replaceUndumpableData(raw.data[20]), \
"Purch Org Id" : replaceUndumpableData(raw.data[21]), \
"Purch Org Descr" : replaceUndumpableData(raw.data[22]), \
"Ext Purch Itm Catg Id - Descr" : replaceUndumpableData(raw.data[23]), \
"PO Short Text" : replaceUndumpableData(raw.data[24]), \
"PO Vendor" : replaceUndumpableData(raw.data[25]), \
AGENCY : replaceUndumpableData(raw.data[26]), \
"PO Purch Grp" : replaceUndumpableData(raw.data[27]), \
"OA" : replaceUndumpableData(raw.data[28]), \
AWARDIDIDV : replaceUndumpableData(raw.data[22])
}
except:
exc_type, exc_value, exc_traceback = sys.exc_info()
traceback.print_exception(exc_type,
exc_value,
exc_traceback,
limit=2,
file=sys.stderr)
logger.error("don't know what went wrong here")
return {}