def get_kawal_detail_tps(hps_log, isupdate):
if hps_log == 1:
hapus_log()
if isupdate == 0:
raw_tps = colection_raw_tps()
raw_tps.delete_all()
raw_tps = colection_raw_tps()
jml = raw_tps.count({})
jml_page = jml // 1000
if (jml % 1000) > 0:
jml_page += 1
thread_list = []
for i in range(jml_page):
print("no_page : {}".format(i + 1))
p = threading.Thread(target=proses.kawal_tps,
name='proses{}'.format(i),
args=(i + 1, 1000),
daemon=True)
p.start()
# p.join()
thread_list.append(p)
if (len(thread_list) % 2) == 0:
j = 1
for thread in thread_list:
if thread.is_alive() is True:
thread.join()
j += 1
thread_list.clear()
def get_raw_detail_tps(self, hps_log, isupdate):
if hps_log == 1:
self.hapus_log()
if isupdate == 0:
raw_detail_tps = colection_raw_detail_tps()
raw_detail_tps.delete_all()
# raw_proses = colection_raw_proses()
raw_tps = colection_raw_tps()
jml = raw_tps.count({})
jml_page = self.get_jml_page(jml, 800)
thread_list = []
for i in range(jml_page):
print("no_page : {}".format(i + 1))
kd_proses = str(uuid.uuid4())
p = threading.Thread(target=proses.detail_tps,
name='proses{}'.format(i),
args=(i + 1, 800, kd_proses),
daemon=True)
p.start()
thread_list.append({'kd_proses': kd_proses, 'p': p})
if (len(thread_list) % 2) == 0:
j = 1
for thread in thread_list:
if thread['p'].is_alive() is True:
thread['p'].join((60 // j) * 60)
j += 1
thread_list.clear()
def tps(data):
kd_thread = str(uuid.uuid4())
raw_thread = colection_raw_thread(kd_thread, data["kode_kelurahan"])
raw_tps = colection_raw_tps()
kpu_tps = data_kpu()
kawal_tps = data_kawal()
data_kpu_tps = kpu_tps.get_data([
data["kode_provinsi"], data["kode_kabkota"], data["kode_kec"],
data["kode_kelurahan"]
])
err_msg = kpu_tps.err_msg
data_kawal_tps = kawal_tps.get_data(data["kode_kelurahan"])
hsl_olah = olah_data_kawal_tps(data_kawal_tps["data_kawal"])
err_msg += kawal_tps.err_msg
if err_msg == '':
tmp_data = {
"kode": str(data['kode_kelurahan']),
"kode_provinsi": str(data["kode_provinsi"]),
"kode_kabkota": str(data["kode_kabkota"]),
"kode_kec": str(data["kode_kec"]),
"data_kpu": data_kpu_tps["data_kpu"],
"data_kawal": hsl_olah
}
raw_tps.mycol.update_one({"kode": str(data['kode_kelurahan'])},
{"$set": tmp_data},
upsert=True)
# raw_tps.mycol.insert_one(tmp_data)
else:
raw_thread.set_error_msg(err_msg)
raw_thread.stop()
def get_raw_tps(self, hps_log, isupdate):
if hps_log == 1:
self.hapus_log()
if isupdate == 0:
raw_tps = colection_raw_tps()
raw_tps.delete_all()
raw_kel = colection_raw_kel()
jml = raw_kel.count({})
jml_page = self.get_jml_page(jml, 800)
thread_list = []
for i in range(jml_page):
print("no_page : {}".format(i + 1))
p = threading.Thread(target=proses.tps,
name='proses{}'.format(i),
args=(i + 1, 800),
daemon=True)
p.start()
thread_list.append(p)
if (len(thread_list) % 2) == 0:
j = 1
for thread in thread_list:
if thread.is_alive() is True:
thread.join((30 // j) * 60)
j += 1
thread_list.clear()
def kawal_tps(no_page, jml):
time.sleep(5)
kd_proses = str(uuid.uuid4())
raw_proses = colection_raw_proses(kd_proses, "baca_raw_kelurahan")
raw_proses.update({
"page": no_page,
"jml": jml,
"jml_tps": 0,
"jml_thread": 0
})
skip = jml * (no_page - 1)
raw_tps = colection_raw_tps()
data3 = raw_tps.get_data_kawal(skip, jml)
i = 0
k = 0
j = 0
thread_list = []
for x in data3:
for y in x["tps"]:
t = threading.Thread(target=mthread.kawal_tps,
name='thread{}'.format(str(y["kode_tps"])),
args=(y, ),
daemon=True)
t.start()
thread_list.append(t)
i += 1
k += 1
j += 1
raw_proses.update({"jml_tps": k, "jml_proses": j, "jml_thread": i})
for thread in thread_list:
if thread.is_alive() is True:
thread.join()
thread_list.clear()
raw_proses.stop()
def raw_tps(self):
raw_tps = colection_raw_tps()
raw_tps.delete_all()
for i in range(21):
with open(
"db/raw_tps/raw_tps_part_{}.json".format(i)) as json_file:
data = json.load(json_file)
raw_tps.mycol.insert_many(data)
def raw_tps(self):
for file_name in os.listdir('db/raw_tps/'):
if fnmatch.fnmatch(file_name, 'raw_tps_part_*.json'):
os.remove('db/raw_tps/' + file_name)
raw_tps = colection_raw_tps()
jml = raw_tps.count({})
jml_page = self.get_jml_page(jml, 4000)
for i in range(jml_page):
skip = 4000 * i
data = raw_tps.mycol.find({}, {'_id': 0}).skip(skip).limit(4000)
with open("db/raw_tps/raw_tps_part_{}.json".format(i),
'w') as outfile:
json.dump(list(data), outfile, indent=4)
def tps(no_page, jml):
time.sleep(5)
kd_proses = str(uuid.uuid4())
raw_proses = colection_raw_proses(kd_proses, "baca_raw_kelurahan")
raw_proses.update({
"page": no_page,
"jml": jml,
"jml_tps": 0,
"jml_thread": 0,
"jml_proses": 0
})
skip = jml * (no_page - 1)
raw_kel = colection_raw_kel()
raw_tps = colection_raw_tps()
data1 = raw_kel.get_lst_tps(skip, jml)
data2 = raw_tps.get_tps_inlst(data1)
data3 = raw_kel.get_tps_notinlst(skip, jml, data2)
i = 0
j = 0
k = len(data2)
thread_list = []
if (len(data1) != len(data2)):
for x in data3:
for y in x["tps"]:
data = {
'kode_provinsi': x["kode_provinsi"],
'kode_kabkota': x["kode_kabkota"],
'kode_kec': x["kode_kec"],
'kode_kelurahan': str(y)
}
t = threading.Thread(target=mthread.tps,
name='thread{}'.format(str(y)),
args=(data, ),
daemon=True)
t.start()
thread_list.append(t)
i += 1
k += 1
j += 1
raw_proses.update({"jml_tps": k, "jml_proses": j, "jml_thread": i})
jml_thread = len(thread_list)
if jml_thread > 0:
for thread in thread_list:
if thread.is_alive() is True:
thread.join(jml_thread * 2)
else:
jml_thread -= 1
thread_list.clear()
raw_proses.stop()
def detail_tps(no_page, jml, kd_proses):
time.sleep(5)
# raw_thread = colection_raw_thread()
raw_proses = colection_raw_proses(kd_proses, "baca_raw_tps")
raw_proses.update({
"page": no_page,
"jml": jml,
"jml_tps": 0,
"jml_thread": 0,
"jml_proses": 0
})
skip = jml * (no_page - 1)
raw_tps = colection_raw_tps()
raw_detail_tps = colection_raw_detail_tps()
lst_tps1 = raw_tps.get_lst_tps(skip, jml)
lst_tps2 = raw_detail_tps.get_tps_inlst(lst_tps1)
i = 1
j = 0
k = len(lst_tps2)
thread_list = []
if (len(lst_tps1) != len(lst_tps2)):
lst_tps3 = raw_tps.get_tps_notinlst(skip, jml, lst_tps2)
for x in lst_tps3:
for y in x['tps']:
data = {
'kode_provinsi': x["_id"]["kode_provinsi"],
'kode_kabkota': x["_id"]["kode_kabkota"],
'kode_kec': x["_id"]["kode_kec"],
'kode_kelurahan': x["_id"]["kode_kel"],
'kode_tps': str(y["json_wilayah"]["kode_tps"]),
'json_wilayah': y["json_wilayah"],
'data_kawal': y['data_kawal']
}
t = threading.Thread(target=mthread.detail_tps,
name='thread{}'.format(
str(y["json_wilayah"]["kode_tps"])),
args=(data, ),
daemon=True)
t.start()
thread_list.append(t)
i += 1
k += 1
raw_proses.update({
"jml_tps": k,
"jml_proses": j,
"jml_thread": (i - 1)
})
jml_thread = len(thread_list)
if jml_thread > 0:
for thread in thread_list:
if thread.is_alive() is True:
thread.join(jml_thread * 30)
else:
jml_thread -= 1
thread_list.clear()
j += 1
else:
lst_tps3 = raw_tps.get_tps_notinlst_org(skip, jml, lst_tps2)
for x in lst_tps3:
for y in x['tps']:
data = {
'kode_provinsi': x["_id"]["kode_provinsi"],
'kode_kabkota': x["_id"]["kode_kabkota"],
'kode_kec': x["_id"]["kode_kec"],
'kode_kelurahan': x["_id"]["kode_kel"],
'kode_tps': str(y["json_wilayah"]["kode_tps"]),
'json_wilayah': y["json_wilayah"]
}
t = threading.Thread(
target=mthread.detail_tps_org,
name='thread{}'.format(
str(y["json_wilayah"]["kode_tps"])),
args=(data, ),
daemon=True)
t.start()
thread_list.append(t)
i += 1
k += 1
raw_proses.update({
"jml_tps": k,
"jml_proses": j,
"jml_thread": (i - 1)
})
jml_thread = len(thread_list)
if jml_thread > 0:
for thread in thread_list:
if thread.is_alive() is True:
thread.join(jml_thread * 30)
else:
jml_thread -= 1
thread_list.clear()
j += 1
raw_proses.stop()