Esempio n. 1
0
def handle_row(src, q_info, row):
    name = parse_tools.fix_mp_name(row[0])
    mp = Member.objects.filter(name=name)
    if not mp:
        print "%s not found" % name
        return
    mp = mp[0]
    print "%s" % mp
    for q in q_info:
        q_idx, i_idx, order, txt = q
        if not row[q_idx]:
            print "\tMissing value for column %d" % q_idx
            continue
        val = int(row[q_idx])
        opt_idx = int(val * OPTION_STEPS // ANSWER_MAX)
        if opt_idx == OPTION_STEPS:
            opt_idx = OPTION_STEPS - 1
        que = Question.objects.get(source=src, order=order)
        opt = Option.objects.get(question=que, order=opt_idx)
        try:
            ans = Answer.objects.get(member=mp, question=que)
        except Answer.DoesNotExist:
            ans = Answer(member=mp, question=que)
        ans.option = opt
        ans.explanation = None
        ans.save()
Esempio n. 2
0
def handle_row(src, row, writer=None):
    (district, party, last_name, first_name, age, gender, is_indep) = row[0:7]
    (county, edu, intro, is_county_off, is_mp, is_mep, profession) = row[7:14]
    (home_page, rss_feed) = row[14:16]
    answer_list = row[16::3]
    importance_list = row[17::3]
    comment_list = row[18::3]

    name = ' '.join((last_name, first_name))
    name = parse_tools.fix_mp_name(name)

    if writer:
        if district.endswith(' vaalipiiri'):
            district = district.replace(' vaalipiiri', '')
        row = [name, district, party, age, gender, edu]
        row = [s.encode('utf8') if s else '' for s in row]
        writer.writerow(row)

    mp = Member.objects.filter(name=name)
    if not mp:
        if is_mp != '0':
            print(name)
        return
    mp = mp[0]

    for idx, ans in enumerate(answer_list):
        if idx in SKIP_QUESTIONS:
            continue
        comment = comment_list[idx]
        if not ans and not comment:
            continue
        que = Question.objects.get(source=src, order=idx)
        if ans:
            opt = Option.objects.get(question=que, name=ans)
        else:
            opt = None
        try:
            ans = Answer.objects.get(member=mp, question=que)
        except Answer.DoesNotExist:
            ans = Answer(member=mp, question=que)
        ans.option = opt
        ans.explanation = comment
        ans.save()
Esempio n. 3
0
def handle_row(src, q_info, row):
    name = parse_tools.fix_mp_name(row[0])
    mp = Member.objects.filter(name=name)
    if not mp:
        print("%s not found" % name)
        return
    mp = mp[0]
    print("%s" % mp)
    for q in q_info:
        q_idx, i_idx, order, txt = q
        if not row[q_idx]:
            print("\tMissing value for column %d" % q_idx)
            continue
        val = int(row[q_idx])
        opt_idx = int(val * OPTION_STEPS // ANSWER_MAX)
        if opt_idx == OPTION_STEPS:
            opt_idx = OPTION_STEPS - 1
        que = Question.objects.get(source=src, order=order)
        opt = Option.objects.get(question=que, order=opt_idx)
        try:
            ans = Answer.objects.get(member=mp, question=que)
        except Answer.DoesNotExist:
            ans = Answer(member=mp, question=que)
        ans.option = opt
        ans.explanation = None
        ans.save()
Esempio n. 4
0
def handle_row(src, row, writer=None):
    (district, party, last_name, first_name, age, gender, is_indep) = row[0:7]
    (county, edu, intro, is_county_off, is_mp, is_mep, profession) = row[7:14]
    (home_page, rss_feed) = row[14:16]
    answer_list = row[16::3]
    importance_list = row[17::3]
    comment_list = row[18::3]

    name = ' '.join((last_name, first_name))
    name = parse_tools.fix_mp_name(name)

    if writer:
        if district.endswith(' vaalipiiri'):
            district = district.replace(' vaalipiiri', '')
        row = [name, district, party, age, gender, edu]
        row = [s.encode('utf8') if s else '' for s in row]
        writer.writerow(row)

    mp = Member.objects.filter(name=name)
    if not mp:
        if is_mp != '0':
            print(name)
        return
    mp = mp[0]

    for idx, ans in enumerate(answer_list):
        if idx in SKIP_QUESTIONS:
            continue
        comment = comment_list[idx]
        if not ans and not comment:
            continue
        que = Question.objects.get(source=src, order=idx)
        if ans:
            opt = Option.objects.get(question=que, name=ans)
        else:
            opt = None
        try:
            ans = Answer.objects.get(member=mp, question=que)
        except Answer.DoesNotExist:
            ans = Answer(member=mp, question=que)
        ans.option = opt
        ans.explanation = comment
        ans.save()
Esempio n. 5
0
def parse_mp(src, lname, fname, href):
    name = "%s %s" % (lname, fname)
    name = parse_tools.fix_mp_name(name)
    mp = Member.objects.filter(name=name)
    if not mp:
        return
    mp = mp[0]

    if name in mp_dict:
        mp = mp_dict[name]
        mp.found = True

    print(mp)

    s = http_cache.open_url(href, 'opinions')
    doc = html.fromstring(s)
    q_list = doc.xpath(".//div[@class='em-compare-container']")
    for q_idx, q_el in enumerate(q_list):
        if q_idx in SKIP_QUESTIONS:
            continue

        el = q_el.xpath("./h3")
        assert len(el) == 1
        q_text = el[0].text.strip()
        m = re.match(r'\d+\.\s+(.+)', q_text, re.U)
        assert m
        q_text = m.groups()[0]

        a_list = q_el.xpath(".//td[@class='em-text']")
        a_text_list = []
        for a_idx, a_text in enumerate(a_list):
            a_text = a_text.text.strip()
            a_text_list.append(a_text)

        q_obj = add_question(src, q_text, q_idx, a_text_list)

        a_list = q_el.xpath(".//table[@class='em-compare-alts ']/tr")
        assert len(a_list) == len(a_text_list)
        chosen = None
        for a_idx, el in enumerate(a_list):
            if el.xpath(".//acronym"):
                assert not chosen
                chosen = a_idx
        if chosen == None:
            continue

        comm_el = q_el.xpath(".//div[@class='em-comment']")
        if comm_el:
            assert len(comm_el) == 1
            comm_el = comm_el[0]
            text_list = []
            for br in comm_el.xpath(".//br"):
                if not br.tail:
                    continue
                s = br.tail.strip()
                if s:
                    text_list.append(s)
            comm_text = '\n'.join(text_list)
            assert comm_text[0] == '"' and comm_text[-1] == '"'
            comm_text = comm_text[1:-1]
        else:
            comm_text = None

        opt = q_obj.opt_dict[chosen]
        try:
            ans = Answer.objects.get(member=mp, question=q_obj)
        except:
            ans = Answer(member=mp, question=q_obj)
        ans.option = opt
        ans.explanation = comm_text
        ans.save()
Esempio n. 6
0
def parse_mp(src, lname, fname, href):
    name = "%s %s" % (lname, fname)
    name = parse_tools.fix_mp_name(name)
    mp = Member.objects.filter(name=name)
    if not mp:
        return
    mp = mp[0]

    if name in mp_dict:
        mp = mp_dict[name]
        mp.found = True

    print mp

    s = http_cache.open_url(href, 'opinions')
    doc = html.fromstring(s)
    q_list = doc.xpath(".//div[@class='em-compare-container']")
    for q_idx, q_el in enumerate(q_list):
        if q_idx in SKIP_QUESTIONS:
            continue

        el = q_el.xpath("./h3")
        assert len(el) == 1
        q_text = el[0].text.strip()
        m = re.match(r'\d+\.\s+(.+)', q_text, re.U)
        assert m
        q_text = m.groups()[0]

        a_list = q_el.xpath(".//td[@class='em-text']")
        a_text_list = []
        for a_idx, a_text in enumerate(a_list):
            a_text = a_text.text.strip()
            a_text_list.append(a_text)

        q_obj = add_question(src, q_text, q_idx, a_text_list)

        a_list = q_el.xpath(".//table[@class='em-compare-alts ']/tr")
        assert len(a_list) == len(a_text_list)
        chosen = None
        for a_idx, el in enumerate(a_list):
            if el.xpath(".//acronym"):
                assert not chosen
                chosen = a_idx
        if chosen == None:
            continue

        comm_el = q_el.xpath(".//div[@class='em-comment']")
        if comm_el:
            assert len(comm_el) == 1
            comm_el = comm_el[0]
            text_list = []
            for br in comm_el.xpath(".//br"):
                if not br.tail:
                    continue
                s = br.tail.strip()
                if s:
                    text_list.append(s)
            comm_text = '\n'.join(text_list)
            assert comm_text[0] == '"' and comm_text[-1] == '"'
            comm_text = comm_text[1:-1]
        else:
            comm_text = None

        opt = q_obj.opt_dict[chosen]
        try:
            ans = Answer.objects.get(member=mp, question=q_obj)
        except:
            ans = Answer(member=mp, question=q_obj)
        ans.option = opt
        ans.explanation = comm_text
        ans.save()