def single_output(stop_event: Event):
print("single output get queue:")
sum_limit = 1000
counter = 0
manager, output_q = get_queue_client(QueueManager.MachineSettingCrawler,
QueueManager.Method_Whois_Input)
while not stop_event.is_set():
try:
while not output_q.empty() or not stop_event.is_set():
result = output_q.get(False, 1)
counter += 1
if isinstance(result, list):
for item in result:
print("server queue output:", str(item), "count:",
counter)
else:
# print(result)
pass
if counter / sum_limit > 0 and counter % sum_limit == 0:
print("current output count is:", counter)
time.sleep(0.000001)
except Exception as ex:
pass
# manager, output_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Output)
finally:
print("going to sleep.")
time.sleep(1)
def single_output():
current_pid = os.getpid()
print("single output get queue:")
sum_limit = 1000
counter = 0
manager, output_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Output)
while True:
try:
while not output_q.empty():
result = output_q.get()
counter += 1
if isinstance(result, list):
for item in result:
# print(item)
pass
else:
# print(result)
pass
if counter / sum_limit > 0 and counter % sum_limit == 0:
print("current output count is:", counter)
time.sleep(0.000001)
except Exception as ex:
print("single_output with pid: ", current_pid, " error:", ex)
# manager, output_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Output)
time.sleep(0.001)
def single_output(stop_event: Event):
print("single output get queue:")
sum_limit = 1000
counter = 0
manager, output_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Input)
while not stop_event.is_set():
try:
while not output_q.empty() or not stop_event.is_set():
result = output_q.get(False, 1)
counter += 1
if isinstance(result, list):
for item in result:
print("server queue output:", str(item), "count:", counter)
else:
# print(result)
pass
if counter/sum_limit > 0 and counter % sum_limit==0:
print("current output count is:", counter)
time.sleep(0.000001)
except Exception as ex:
pass
# manager, output_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Output)
finally:
print("going to sleep.")
time.sleep(1)
def single_output():
current_pid = os.getpid()
print("single output get queue:")
sum_limit = 1000
counter = 0
manager, output_q = get_queue_client(QueueManager.MachineSettingCrawler,
QueueManager.Method_Whois_Output)
while True:
try:
while not output_q.empty():
result = output_q.get()
counter += 1
if isinstance(result, list):
for item in result:
# print(item)
pass
else:
# print(result)
pass
if counter / sum_limit > 0 and counter % sum_limit == 0:
print("current output count is:", counter)
time.sleep(0.000001)
except Exception as ex:
print("single_output with pid: ", current_pid, " error:", ex)
# manager, output_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Output)
time.sleep(0.001)
def single_input(thread_id: int, limit=1000000):
current_pid = os.getpid()
duration = 0.1
link_format = "{0:d}something{1:d}.com"
counter = 0
batch_size = 100
print("single input get queue:")
manager, input_q = get_queue_client(QueueManager.MachineSettingCrawler,
QueueManager.Method_Whois_Input)
print("single input got queue:")
while counter < limit:
batch = list()
for item in range(batch_size):
batch.append((link_format.format(thread_id,
counter), ResponseCode.DNSError))
counter += 1
try:
input_q.put(batch)
except Exception as ex:
# manager.shutdown()
print("single_input thread id:", thread_id, "with pid: ",
current_pid, "error: ", ex)
# manager, input_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Input)
finally:
counter += batch_size
time.sleep(duration)
def test_queue_init(self):
chunk_size = 100
queue_server_t = Process(target=run_queue_server, name="queue_server")
queue_server_t.start()
output_t = Process(target=single_output, name="output_thread")
output_t.start()
counter = 0
for i in range(1000):
print("init queue: ", i)
time.sleep(0.00001)
temp = list()
manager, input_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Output)
for j in range(chunk_size):
temp.append(("some_domain.com{0:d}".format(counter), 999))
counter += 1
input_q.put(temp)
def test_queue_init(self):
chunk_size = 100
queue_server_t = Process(target=run_queue_server, name="queue_server")
queue_server_t.start()
output_t = Process(target=single_output, name="output_thread")
output_t.start()
counter = 0
for i in range(1000):
print("init queue: ", i)
time.sleep(0.00001)
temp = list()
manager, input_q = get_queue_client(
QueueManager.MachineSettingCrawler,
QueueManager.Method_Whois_Output)
for j in range(chunk_size):
temp.append(("some_domain.com{0:d}".format(counter, ), 999))
counter += 1
input_q.put(temp)
def single_input(thread_id: int, limit=1000000):
current_pid = os.getpid()
duration = 0.1
link_format = "{0:d}something{1:d}.com"
counter = 0
batch_size = 100
print("single input get queue:")
manager, input_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Input)
print("single input got queue:")
while counter < limit:
batch = list()
for item in range(batch_size):
batch.append((link_format.format(thread_id, counter), ResponseCode.DNSError))
counter += 1
try:
input_q.put(batch)
except Exception as ex:
# manager.shutdown()
print("single_input thread id:", thread_id, "with pid: ", current_pid, "error: ", ex)
# manager, input_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Input)
finally:
counter += batch_size
time.sleep(duration)
