def worker(queue, event): global counter connection = create_postgresql_connection() _age_map = age_map(connection) _sex_map = sex_map(connection) _aoi_map = aoi_map(connection) _education_map = education_map(connection) _province_map = province_map(connection) cursor = connection.cursor() cursor.execute("BEGIN;") general_rate = calculate_rate(connection) while not event.is_set(): greenlet = gevent.getcurrent() try: c = queue.get(block=True, timeout=1) except Empty: continue counter += 1 sys.stdout.write("\r{0}".format(counter)) c_age, c_sex, c_province, c_education, c_cycle = c rate = calculate_rate(connection, c_age, c_cycle, c_education, c_province, c_sex) percent = general_rate * 20 / 100 if rate is None: compared = 0 elif rate > (general_rate + percent): compared = 1 elif rate < (general_rate - percent): compared = 3 else: compared = 2 hash = generate_hash(connection, age=c_age, cycle=c_cycle, education=c_education, province=c_province, sex=c_sex), params = [ hash, rate, c_cycle, c_age, c_sex, c_education, c_province, compared ] cursor.execute( "INSERT INTO core_ratequery (query_hash, rate, cycle, age_id, sex_id, education_id, province_id, compared, date) " "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now())", params) queue.task_done() cursor.execute("COMMIT;") cursor.close()
def worker(queue, event): global counter connection = create_postgresql_connection() _age_map = age_map(connection) _sex_map = sex_map(connection) _aoi_map = aoi_map(connection) _education_map = education_map(connection) _province_map = province_map(connection) cursor = connection.cursor() cursor.execute("BEGIN;") general_rate = calculate_rate(connection) while not event.is_set(): greenlet = gevent.getcurrent() try: c = queue.get(block=True, timeout=1) except Empty: continue counter +=1 sys.stdout.write("\r{0}".format(counter)) c_age, c_sex, c_province, c_education, c_cycle = c rate = calculate_rate(connection, c_age, c_cycle, c_education, c_province, c_sex) percent = general_rate * 20 / 100 if rate is None: compared = 0 elif rate > (general_rate + percent): compared = 1 elif rate < (general_rate - percent): compared = 3 else: compared = 2 hash = generate_hash(connection, age=c_age, cycle=c_cycle, education=c_education, province=c_province, sex=c_sex), params = [hash, rate, c_cycle, c_age, c_sex, c_education, c_province, compared] cursor.execute("INSERT INTO core_ratequery (query_hash, rate, cycle, age_id, sex_id, education_id, province_id, compared, date) " "VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now())", params) queue.task_done() cursor.execute("COMMIT;") cursor.close()
def worker(queue, event): global counter connection = create_postgresql_connection() _age_map = age_map(connection) _sex_map = sex_map(connection) _aoi_map = aoi_map(connection) _education_map = education_map(connection) _province_map = province_map(connection) cursor = connection.cursor() cursor.execute("BEGIN;") while not event.is_set(): greenlet = gevent.getcurrent() try: line = queue.get(block=True, timeout=1) except Empty: continue counter +=1 sys.stdout.write("\r{0}".format(counter)) try: cycle, age, sex, education, province, aoi, factorel = line.split('\t') except ValueError: continue age = _age_map[age] sex = _sex_map[sex] aoi = _aoi_map[aoi] education = _education_map[education] province = _province_map[province] params = [cycle, age, sex, education, province, aoi, factorel] cursor.execute("INSERT INTO core_microdata (cycle,age_id,sex_id,education_id,province_id,aoi_id,factorel)" "VALUES(%s, %s, %s, %s, %s, %s, %s);", params) queue.task_done() cursor.execute("COMMIT;") cursor.close()