Beispiel #1
0
    def test_name_property(self):
        candidate = Candidate()
        candidate.first_name = 'Juanito'
        candidate.last_name = 'Perez'

        expected_name = 'Juanito Perez'

        self.assertEqual(candidate.name, expected_name)
def process_candidates():
    reader = codecs.open("candidates.csv", 'r', encoding='utf-8')
    counter = 0
    candidates_ids = []
    for line in reader:
        row = line.split(u',')
        area_name = row[0].title().strip()
        area = area_getter(area_name)
        if area is None:
            print u'no encontré a ' + area_name
            return
        kind_of = row[1].title().strip()
        election_name = kind_of + u' por ' + area.name
        if not Election.objects.filter(name__iexact=election_name):
            print u"no pillé a " + election_name
            # Candidate.objects.filter(id__in=candidates_ids).delete()
            return
        else:
            e = Election.objects.get(name__iexact=election_name)
        name = row[2].title().strip()
        candidate = Candidate(name=name)

        candidate.save()
        candidates_ids.append(candidate.id)
        e.candidates.add(candidate)
        print e, candidate
        try:
            mail = row[6].strip().lower()
        except IndexError:
            mail = None
        if mail:
            contact = CandidacyContact.objects.create(candidate=candidate,
                                                      mail=mail)

        pacto = row[3].strip().title()
        sub_pacto = row[4].strip().title()
        partido = row[5].strip().title()
        if pacto:
            PersonalData.objects.create(candidate=candidate,
                                        label=u'Pacto',
                                        value=pacto)
        if sub_pacto:
            PersonalData.objects.create(candidate=candidate,
                                        label=u'Sub Pacto',
                                        value=sub_pacto)
        if partido:
            PersonalData.objects.create(candidate=candidate,
                                        label=u'Partido',
                                        value=partido)
        send_candidate_username_and_password(candidate)
        counter += 1
        if not counter % 1000:
            print u'van' + str(counter)
def process_candidates():
    reader = codecs.open("candidates.csv", 'r', encoding='utf-8')
    counter = 0
    candidates_ids = []
    for line in reader:
        row = line.split(u',')
        area_name = row[0].title().strip()
        area = area_getter(area_name)
        if area is None:
            print u'no encontré a '+ area_name
            return
        kind_of = row[1].title().strip()
        election_name = kind_of + u' por ' + area.name
        if not Election.objects.filter(name__iexact=election_name):
            print u"no pillé a " + election_name
            # Candidate.objects.filter(id__in=candidates_ids).delete()
            return
        else:
            e = Election.objects.get(name__iexact=election_name)
        name = row[2].title().strip()
        candidate = Candidate(name=name)

        candidate.save()
        candidates_ids.append(candidate.id)
        e.candidates.add(candidate)
        print e, candidate
        try:
            mail = row[6].strip().lower()
        except IndexError:
            mail = None
        if mail:
            contact = CandidacyContact.objects.create(candidate=candidate,
                                                      mail=mail)

        pacto = row[3].strip().title()
        sub_pacto = row[4].strip().title()
        partido = row[5].strip().title()
        if pacto:
            PersonalData.objects.create(candidate=candidate,
                                        label=u'Pacto',
                                        value=pacto)
        if sub_pacto:
            PersonalData.objects.create(candidate=candidate,
                                        label=u'Sub Pacto',
                                        value=sub_pacto)
        if partido:
            PersonalData.objects.create(candidate=candidate,
                                        label=u'Partido',
                                        value=partido)
        send_candidate_username_and_password(candidate)
        counter += 1
        if not counter % 1000:
            print u'van' + str(counter)
Beispiel #4
0
        counts = table.find_all(class_="read")
        days = table.find_all(class_="date")
        for tit, count, day in zip(t**s, counts, days):
            title = tit.a.get_text()
            link = tit.a.get('href')
            read = count.get_text()
            date = day.get_text()
            temp_dict = {
                'day': date,
                'title': title,
                'count': read,
                'link': link
            }
            temp_list.append(temp_dict)

    #toJson(temp_list)

    return temp_list


#parsing()

if __name__ == '__main__':
    parsed_data = parsing()
    for i in range(len(parsed_data)):
        new_candidate = Candidate(name=parsed_data[i]["day"],
                                  introduction=parsed_data[i]["title"],
                                  area=parsed_data[i]["count"],
                                  party_number=parsed_data[i]["link"])
        new_candidate.save()
Beispiel #5
0
            if container.find('video'):
                videotag = container.find('video')
                image =videotag.get('poster')
            elif container.find('img'):
                imgtag = container.find('img')
                image =imgtag.get('src')
            else:
                image = 'none'
            #
            read = count.get_text()
            date = day.get_text()
            temp_dict = {'day': date, 'title': title, 'count': read, 'link': link, 'image': image}
            temp_list.append(temp_dict)
    #toJson(temp_list)
    return temp_list
'''
if __name__=='__main__':
    parsed_data = []
    parsed_data = ygosu_parsing()
    #parsed_data1 = ou_parsing()
    #parsed_data.extend(parsed_data1)
    toJson(parsed_data)

    for i in range(len(parsed_data)):
        new_candidate = Candidate(date=parsed_data[i]["day"],
        title=parsed_data[i]["title"],
        count=parsed_data[i]["count"],
        link=parsed_data[i]["link"],
        image=parsed_data[i]["image"])
        new_candidate.save()
 def handle_label(self, label, **options):
     import csv
     bios = csv.reader(open(label, 'rb'), delimiter='|')
     for row in bios:
         row[0] = int(row[0]) #politician id
         if row[5]:
             row[5] = int(row[5]) #year first elected
         else:
             row[5] = None
         row[20] = datetime.datetime(*time.strptime(row[20], "%m-%d-%Y %I:%M:%S %p")[:6]) # timestamp
         row[13] = row[13][0] # gender
         if row[6]:
             row[6] = datetime.date(*time.strptime(row[6], "%Y-%m-%d")[:3]) # birthdate
         else:
             row[6] = None
         try:
             candidate = Candidate.objects.get(politician_id=row[0])
             if candidate.timestamp != row[20]:
                 candidate.first_name = row[1]
                 candidate.middle_name = row[2]
                 candidate.last_name = row[3]
                 candidate.junior = row[4]
                 candidate.year_first_elected = row[5]
                 candidate.birth_date = row[6]
                 candidate.birth_place = row[7]
                 candidate.birth_state = row[8]
                 candidate.birth_province = row[9]
                 candidate.birth_country = row[10]
                 candidate.residence_place = row[11]
                 candidate.residence_state = row[12]
                 candidate.gender = row[13]
                 candidate.ethnicity = row[14]
                 candidate.hispanic = row[15]
                 candidate.religion = row[16]
                 candidate.biography = row[17]
                 candidate.profile = row[18]
                 candidate.campaigns = row[19]
                 candidate.timestamp = row[20]
                 print 'Updating %s %s' % (row[1], row[3])
                 candidate.save()
             else:
                 print "Skipping %s %s. No change." % (row[1], row[3])
                 
         except Candidate.DoesNotExist:
             print 'Adding %s %s' % (row[1], row[3])
             candidate = Candidate()
             candidate.politician_id = row[0]
             candidate.ap_candidate_id = row[0]
             candidate.candidate_number = row[0]
             candidate.first_name = row[1]
             candidate.middle_name = row[2]
             candidate.last_name = row[3]
             candidate.junior = row[4]
             candidate.year_first_elected = row[5]
             candidate.birth_date = row[6]
             candidate.birth_place = row[7]
             candidate.birth_state = row[8]
             candidate.birth_province = row[9]
             candidate.birth_country = row[10]
             candidate.residence_place = row[11]
             candidate.residence_state = row[12]
             candidate.gender = row[13]
             candidate.ethnicity = row[14]
             candidate.hispanic = row[15]
             candidate.religion = row[16]
             candidate.biography = row[17]
             candidate.profile = row[18]
             candidate.campaigns = row[19]
             candidate.timestamp = row[20]
             
             candidate.save()