def process_file(filename):
print "Processing file %s" % (filename)
token = 'type'
chunker = yamlChunker(filename, token)
count = 0
hasnext = True
while (hasnext):
count += 1
yamlrecord = chunker.next()
print "got yaml record: %s" % yamlrecord
if not yamlrecord:
hasnext = False
else:
if (count % 100 == 0):
print "Processed %s lines" % count
committee_yaml = yaml.load(yamlrecord)[0]
thomas_id = committee_yaml['thomas_id']
try:
this_committee = Committee.objects.get(thomas_id=thomas_id)
except Committee.DoesNotExist:
this_committee = Committee.objects.create(
committee_type=committee_yaml['type'],
name=committee_yaml['name'],
thomas_id=committee_yaml['thomas_id'],
house_committee_id=committee_yaml.get(
'house_committee_id'),
url=committee_yaml.get('url') or '')
def process_file(filename):
print "Processing file %s" % (filename)
token = 'type'
chunker = yamlChunker(filename, token)
count = 0
hasnext=True
while(hasnext):
count += 1
yamlrecord = chunker.next()
print "got yaml record: %s" % yamlrecord
if not yamlrecord:
hasnext=False
else:
if (count%100==0):
print "Processed %s lines" % count
committee_yaml = yaml.load(yamlrecord)[0]
thomas_id = committee_yaml['thomas_id']
try:
this_committee = Committee.objects.get(thomas_id=thomas_id)
except Committee.DoesNotExist:
this_committee = Committee.objects.create(
committee_type = committee_yaml['type'],
name = committee_yaml['name'],
thomas_id = committee_yaml['thomas_id'],
house_committee_id = committee_yaml.get('house_committee_id'),
url = committee_yaml.get('url') or '')
def process_file(filename):
print "Processing file %s" % (filename)
token = 'id'
chunker = yamlChunker(filename, token)
count = 0
hasnext=True
while(hasnext):
count += 1
yamlrecord = chunker.next()
#print "got yaml record: %s" % yamlrecord
if not yamlrecord:
hasnext=False
else:
if (count%100==0):
print "Processed %s lines" % count
legislator_yaml = yaml.load(yamlrecord)[0]
bioguide = legislator_yaml['id']['bioguide']
this_legislator = None
try:
this_legislator = Legislator.objects.get(bioguide=bioguide)
except Legislator.DoesNotExist:
this_legislator = Legislator.objects.create(
bioguide=getmultidict(legislator_yaml, ('id','bioguide')),
thomas=getmultidict(legislator_yaml, ('id','thomas')),
lis=getmultidict(legislator_yaml, ('id','lis')),
govtrack=getmultidict(legislator_yaml, ('id','govtrack')),
opensecrets=getmultidict(legislator_yaml, ('id','opensecrets')),
votesmart=getmultidict(legislator_yaml, ('id','votesmart')),
icpsr=getmultidict(legislator_yaml, ('id','icpsr')),
cspan=getmultidict(legislator_yaml, ('id','cspan')),
wikipedia=getmultidict(legislator_yaml, ('id','wikipedia')),
#house_history=getmultidict(legislator_yaml, ('id','house_history')),
bioguide_previous=getmultidict(legislator_yaml, ('id','bioguide_previous')),
first_name=getmultidict(legislator_yaml, ('name','first')),
middle_name=getmultidict(legislator_yaml, ('name','middle')),
last_name=getmultidict(legislator_yaml, ('name','last')),
suffix=getmultidict(legislator_yaml, ('name','suffix')),
nickname=getmultidict(legislator_yaml, ('name','nickname')),
official_full=getmultidict(legislator_yaml, ('name','official_full')),
gender = getmultidict(legislator_yaml, ('bio','gender')),
birthday = getmultidict(legislator_yaml, ('bio','birthday')),
religion = getmultidict(legislator_yaml, ('bio','religion'))
)
#print "Added %s" % (this_legislator)
# fec id list:
fecs = getmultidict(legislator_yaml, ('id','fec'))
print "Got fecs: %s, %s" % (fecs, type(fecs))
if fecs:
for this_fec in fecs:
print "processing fec id: %s" % (this_fec)
this_id = this_fec
(this_fec, created) = fec.objects.get_or_create(legislator=this_legislator, fec_id=this_id)
# terms
terms = legislator_yaml.get('terms')
if terms:
for term in terms:
term_type = term.get('type')
start = term.get('start')
if start:
start = dateparse(start)
end = term.get('end')
if end:
end = dateparse(end)
state = term.get('state')
district = term.get('district')
party = term.get('party')
url = term.get('url')
address = term.get('address')
term_class = term.get('class')
(this_term, created) = Term.objects.get_or_create(legislator=this_legislator, term_type=term_type, start=start, end=end, state=state, district=district, term_class=term_class, party=party, url=url, address=address)
# other names
other_names = legislator_yaml.get('other_names')
if other_names:
for other_name in other_names:
first_name = other_name.get('first')
middle_name = other_name.get('middle')
last_name = other_name.get('last')
start = other_name.get('start')
if start:
start = dateparse(start)
end = other_name.get('end')
if end:
end = dateparse(end)
(this_other_name, created) = Other_Names.objects.get_or_create(legislator=this_legislator, first_name=first_name, last_name=last_name, middle_name=middle_name, start=start, end=end)
