예제 #1
0
def handle():
    connection = create_postgresql_connection()

    cursor = connection.cursor()
    cursor.execute("BEGIN;")
    cursor.execute("DELETE FROM core_ratequery;")
    cursor.execute("COMMIT;")
    cursor.close()

    queue = JoinableQueue()
    event = Event()

    age_ids = age_map(connection).values() + [None]
    sex_ids = sex_map(connection).values() + [None]
    education_ids = education_map(connection).values() + [None]
    province_ids = province_map(connection).values() + [None]

    cursor = connection.cursor()
    cursor.execute("SELECT DISTINCT cycle FROM core_microdata;");
    cycles = [row[0] for row in cursor]
    cursor.close()

    greenlets = []

    for i in range(50):
        gv = gevent.spawn(worker, queue, event)
        greenlets.append(gv)

    combs = itertools.product(age_ids, sex_ids, province_ids, education_ids, cycles)
    for c in combs:
        queue.put(c)

    queue.join()
    event.set()
    gevent.joinall(greenlets)
예제 #2
0
def worker(queue, event):
    global counter
    connection = create_postgresql_connection()

    _age_map = age_map(connection)
    _sex_map = sex_map(connection)
    _aoi_map = aoi_map(connection)
    _education_map = education_map(connection)
    _province_map = province_map(connection)

    cursor = connection.cursor()
    cursor.execute("BEGIN;")
    general_rate = calculate_rate(connection)

    while not event.is_set():
        greenlet = gevent.getcurrent()
        try:
            c = queue.get(block=True, timeout=1)
        except Empty:
            continue

        counter += 1
        sys.stdout.write("\r{0}".format(counter))

        c_age, c_sex, c_province, c_education, c_cycle = c

        rate = calculate_rate(connection, c_age, c_cycle, c_education,
                              c_province, c_sex)
        percent = general_rate * 20 / 100
        if rate is None:
            compared = 0
        elif rate > (general_rate + percent):
            compared = 1
        elif rate < (general_rate - percent):
            compared = 3
        else:
            compared = 2

        hash = generate_hash(connection,
                             age=c_age,
                             cycle=c_cycle,
                             education=c_education,
                             province=c_province,
                             sex=c_sex),

        params = [
            hash, rate, c_cycle, c_age, c_sex, c_education, c_province,
            compared
        ]
        cursor.execute(
            "INSERT INTO core_ratequery (query_hash, rate, cycle, age_id, sex_id, education_id, province_id, compared, date) "
            "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now())", params)

        queue.task_done()

    cursor.execute("COMMIT;")
    cursor.close()
예제 #3
0
def worker(queue, event):
    global counter
    connection = create_postgresql_connection()

    _age_map = age_map(connection)
    _sex_map = sex_map(connection)
    _aoi_map = aoi_map(connection)
    _education_map = education_map(connection)
    _province_map = province_map(connection)

    cursor = connection.cursor()
    cursor.execute("BEGIN;")
    general_rate = calculate_rate(connection)

    while not event.is_set():
        greenlet = gevent.getcurrent()
        try:
            c = queue.get(block=True, timeout=1)
        except Empty:
            continue

        counter +=1
        sys.stdout.write("\r{0}".format(counter))

        c_age, c_sex, c_province, c_education, c_cycle = c

        rate = calculate_rate(connection, c_age, c_cycle, c_education, c_province, c_sex)
        percent = general_rate * 20 / 100
        if rate is None:
            compared = 0
        elif rate > (general_rate + percent):
            compared = 1
        elif rate < (general_rate - percent):
            compared = 3
        else:
            compared = 2

        hash = generate_hash(connection, age=c_age, cycle=c_cycle,
                education=c_education, province=c_province, sex=c_sex),

        params = [hash, rate, c_cycle, c_age, c_sex, c_education, c_province, compared]
        cursor.execute("INSERT INTO core_ratequery (query_hash, rate, cycle, age_id, sex_id, education_id, province_id, compared, date) "
                       "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now())", params)

        queue.task_done()

    cursor.execute("COMMIT;")
    cursor.close()
예제 #4
0
def worker(queue, event):
    global counter
    connection = create_postgresql_connection()

    _age_map = age_map(connection)
    _sex_map = sex_map(connection)
    _aoi_map = aoi_map(connection)
    _education_map = education_map(connection)
    _province_map = province_map(connection)

    cursor = connection.cursor()
    cursor.execute("BEGIN;")

    while not event.is_set():
        greenlet = gevent.getcurrent()
        try:
            line = queue.get(block=True, timeout=1)
        except Empty:
            continue

        counter +=1
        sys.stdout.write("\r{0}".format(counter))

        try:
            cycle, age, sex, education, province, aoi, factorel = line.split('\t')
        except ValueError:
            continue

        age = _age_map[age]
        sex = _sex_map[sex]
        aoi = _aoi_map[aoi]

        education = _education_map[education]
        province = _province_map[province]

        params = [cycle, age, sex, education, province, aoi, factorel]

        cursor.execute("INSERT INTO core_microdata (cycle,age_id,sex_id,education_id,province_id,aoi_id,factorel)"
                       "VALUES(%s, %s, %s, %s, %s, %s, %s);", params)
        queue.task_done()

    cursor.execute("COMMIT;")
    cursor.close()
예제 #5
0
def handle():
    connection = create_postgresql_connection()

    cursor = connection.cursor()
    cursor.execute("BEGIN;")
    cursor.execute("DELETE FROM core_ratequery;")
    cursor.execute("COMMIT;")
    cursor.close()

    queue = JoinableQueue()
    event = Event()

    age_ids = age_map(connection).values() + [None]
    sex_ids = sex_map(connection).values() + [None]
    education_ids = education_map(connection).values() + [None]
    province_ids = province_map(connection).values() + [None]

    cursor = connection.cursor()
    cursor.execute("SELECT DISTINCT cycle FROM core_microdata;")
    cycles = [row[0] for row in cursor]
    cursor.close()

    greenlets = []

    for i in range(50):
        gv = gevent.spawn(worker, queue, event)
        greenlets.append(gv)

    combs = itertools.product(age_ids, sex_ids, province_ids, education_ids,
                              cycles)
    for c in combs:
        queue.put(c)

    queue.join()
    event.set()
    gevent.joinall(greenlets)