def kec(no_page, jml): time.sleep(5) kd_proses = str(uuid.uuid4()) raw_proses = colection_raw_proses(kd_proses, "baca_raw_kec") raw_proses.update({ "page": no_page, "jml": jml, "jml_kabkota": 0, "jml_thread": 0, "jml_proses": 0 }) skip = jml * (no_page - 1) raw_kabkota = colection_raw_kabkota() raw_kec = colection_raw_kec() data1 = raw_kabkota.get_lst_kec(skip, jml) data2 = raw_kec.get_kec_inlst(data1) data3 = raw_kabkota.get_kec_notinlst(skip, jml, data2) i = 0 j = 0 k = len(data2) thread_list = [] if len(data1) != len(data2): for x in data3: for y in x["kec"]: data = { 'kode_provinsi': x["kode_provinsi"], 'kode_kabkota': str(y) } t = threading.Thread(target=mthread.kec, name='thread{}'.format(str(y)), args=(data, )) t.start() thread_list.append(t) i += 1 k += 1 j += 1 raw_proses.update({ "jml_kabkota": k, "jml_proses": j, "jml_thread": i }) jml_thread = len(thread_list) if jml_thread > 0: for thread in thread_list: if thread.is_alive() is True: thread.join(jml_thread * 2) else: jml_thread -= 1 thread_list.clear() raw_proses.stop()
def get_raw_kabkota(self): col_kpu_kabkota = data_kpu() col_kawal_kabkota = data_kawal() raw_kabkota = colection_raw_kabkota() raw_kabkota.delete_all() raw_provinsi = colection_raw_provinsi() raw_provinsi = raw_provinsi.mycol.find({}) for x in raw_provinsi: for y in x["data_kpu"]["json_wilayah"]: kpu_kabkota = col_kpu_kabkota.get_data([y]) kawal_kabkota = col_kawal_kabkota.get_data(y) raw_kabkota.mycol.insert_one({ "kode": str(y), "data_kpu": kpu_kabkota["data_kpu"], "data_kawal": kawal_kabkota["data_kawal"] })
def get_raw_kec(self, hps_log, isupdate): if hps_log == 1: self.hapus_log() if isupdate == 0: raw_kec = colection_raw_kec() raw_kec.delete_all() raw_kabkota = colection_raw_kabkota() jml = raw_kabkota.count({}) jml_page = self.get_jml_page(jml, 10) thread_list = [] for i in range(jml_page): print("no_page : {}".format(i + 1)) p = threading.Thread(target=proses.kec, name='proses{}'.format(i), args=(i + 1, 10)) p.start() thread_list.append(p) if (len(thread_list) % 2) == 0: for thread in thread_list: if thread.is_alive() is True: thread.join(2 * 10) thread_list.clear()
def raw_kabkota(self): raw_kabkota = colection_raw_kabkota() raw_kabkota.delete_all() with open('db/raw_kabkota.json') as json_file: data = json.load(json_file) raw_kabkota.mycol.insert_many(data)
def raw_kabkota(self): raw_kabkota = colection_raw_kabkota() data = raw_kabkota.mycol.find({}, {'_id': 0}) with open('db/raw_kabkota.json', 'w') as outfile: json.dump(list(data), outfile, indent=4)