def handle():
connection = create_postgresql_connection()
cursor = connection.cursor()
cursor.execute("BEGIN;")
cursor.execute("DELETE FROM core_ratequery;")
cursor.execute("COMMIT;")
cursor.close()
queue = JoinableQueue()
event = Event()
age_ids = age_map(connection).values() + [None]
sex_ids = sex_map(connection).values() + [None]
education_ids = education_map(connection).values() + [None]
province_ids = province_map(connection).values() + [None]
cursor = connection.cursor()
cursor.execute("SELECT DISTINCT cycle FROM core_microdata;");
cycles = [row[0] for row in cursor]
cursor.close()
greenlets = []
for i in range(50):
gv = gevent.spawn(worker, queue, event)
greenlets.append(gv)
combs = itertools.product(age_ids, sex_ids, province_ids, education_ids, cycles)
for c in combs:
queue.put(c)
queue.join()
event.set()
gevent.joinall(greenlets)
def worker(queue, event):
global counter
connection = create_postgresql_connection()
_age_map = age_map(connection)
_sex_map = sex_map(connection)
_aoi_map = aoi_map(connection)
_education_map = education_map(connection)
_province_map = province_map(connection)
cursor = connection.cursor()
cursor.execute("BEGIN;")
general_rate = calculate_rate(connection)
while not event.is_set():
greenlet = gevent.getcurrent()
try:
c = queue.get(block=True, timeout=1)
except Empty:
continue
counter += 1
sys.stdout.write("\r{0}".format(counter))
c_age, c_sex, c_province, c_education, c_cycle = c
rate = calculate_rate(connection, c_age, c_cycle, c_education,
c_province, c_sex)
percent = general_rate * 20 / 100
if rate is None:
compared = 0
elif rate > (general_rate + percent):
compared = 1
elif rate < (general_rate - percent):
compared = 3
else:
compared = 2
hash = generate_hash(connection,
age=c_age,
cycle=c_cycle,
education=c_education,
province=c_province,
sex=c_sex),
params = [
hash, rate, c_cycle, c_age, c_sex, c_education, c_province,
compared
]
cursor.execute(
"INSERT INTO core_ratequery (query_hash, rate, cycle, age_id, sex_id, education_id, province_id, compared, date) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now())", params)
queue.task_done()
cursor.execute("COMMIT;")
cursor.close()
def worker(queue, event):
global counter
connection = create_postgresql_connection()
_age_map = age_map(connection)
_sex_map = sex_map(connection)
_aoi_map = aoi_map(connection)
_education_map = education_map(connection)
_province_map = province_map(connection)
cursor = connection.cursor()
cursor.execute("BEGIN;")
general_rate = calculate_rate(connection)
while not event.is_set():
greenlet = gevent.getcurrent()
try:
c = queue.get(block=True, timeout=1)
except Empty:
continue
counter +=1
sys.stdout.write("\r{0}".format(counter))
c_age, c_sex, c_province, c_education, c_cycle = c
rate = calculate_rate(connection, c_age, c_cycle, c_education, c_province, c_sex)
percent = general_rate * 20 / 100
if rate is None:
compared = 0
elif rate > (general_rate + percent):
compared = 1
elif rate < (general_rate - percent):
compared = 3
else:
compared = 2
hash = generate_hash(connection, age=c_age, cycle=c_cycle,
education=c_education, province=c_province, sex=c_sex),
params = [hash, rate, c_cycle, c_age, c_sex, c_education, c_province, compared]
cursor.execute("INSERT INTO core_ratequery (query_hash, rate, cycle, age_id, sex_id, education_id, province_id, compared, date) "
"VALUES (%s, %s, %s, %s, %s, %s, %s, %s, now())", params)
queue.task_done()
cursor.execute("COMMIT;")
cursor.close()
def worker(queue, event):
global counter
connection = create_postgresql_connection()
_age_map = age_map(connection)
_sex_map = sex_map(connection)
_aoi_map = aoi_map(connection)
_education_map = education_map(connection)
_province_map = province_map(connection)
cursor = connection.cursor()
cursor.execute("BEGIN;")
while not event.is_set():
greenlet = gevent.getcurrent()
try:
line = queue.get(block=True, timeout=1)
except Empty:
continue
counter +=1
sys.stdout.write("\r{0}".format(counter))
try:
cycle, age, sex, education, province, aoi, factorel = line.split('\t')
except ValueError:
continue
age = _age_map[age]
sex = _sex_map[sex]
aoi = _aoi_map[aoi]
education = _education_map[education]
province = _province_map[province]
params = [cycle, age, sex, education, province, aoi, factorel]
cursor.execute("INSERT INTO core_microdata (cycle,age_id,sex_id,education_id,province_id,aoi_id,factorel)"
"VALUES(%s, %s, %s, %s, %s, %s, %s);", params)
queue.task_done()
cursor.execute("COMMIT;")
cursor.close()
def handle():
connection = create_postgresql_connection()
cursor = connection.cursor()
cursor.execute("BEGIN;")
cursor.execute("DELETE FROM core_ratequery;")
cursor.execute("COMMIT;")
cursor.close()
queue = JoinableQueue()
event = Event()
age_ids = age_map(connection).values() + [None]
sex_ids = sex_map(connection).values() + [None]
education_ids = education_map(connection).values() + [None]
province_ids = province_map(connection).values() + [None]
cursor = connection.cursor()
cursor.execute("SELECT DISTINCT cycle FROM core_microdata;")
cycles = [row[0] for row in cursor]
cursor.close()
greenlets = []
for i in range(50):
gv = gevent.spawn(worker, queue, event)
greenlets.append(gv)
combs = itertools.product(age_ids, sex_ids, province_ids, education_ids,
cycles)
for c in combs:
queue.put(c)
queue.join()
event.set()
gevent.joinall(greenlets)
