class MultiProcessRunner(BaseRunner):
JOIN_TIMEOUT = 60
def __init__(self, target, **kwargs):
super(MultiProcessRunner, self).__init__(target, **kwargs)
self.process = None # type: Process
@capture_monitor_exception
def start(self):
self.process = Process(target=self.target, kwargs=self.kwargs)
self.process.start()
@capture_monitor_exception
def stop(self):
if self.process and self.is_alive():
self.process.terminate()
self.process.join(MultiProcessRunner.JOIN_TIMEOUT)
if self.process.is_alive():
self.process.kill()
@capture_monitor_exception
def heartbeat(self):
# do we want to do something here?
pass
@capture_monitor_exception
def is_alive(self):
return self.process.is_alive()
def __str__(self):
s = super(MultiProcessRunner, self).__str__()
return f"{s}({self.process})"
def test_contract_manager() -> None:
aut = AirspaceManager()
stop_ev = Event()
conn, manager_conn = Pipe()
p = Process(target=run_as_process, kwargs={"aut": aut,
"conn": manager_conn,
"stop_ev": stop_ev})
p.start()
try:
uid = 0
for i in range(10):
if conn.poll(1.0):
act = conn.recv()
print(act)
elif i % 3 != 2:
uid = i % 5
target = Contract.from_stamped_rectangles([
(0.0, Rectangle(mins=[0, 0, 0], maxes=[1, 1, 0.5])),
(0.5, Rectangle(mins=[0, 0.5, 0], maxes=[2, 3, 0.5])),
(1.0, Rectangle(mins=[0.5, 0.5, 1.0], maxes=[1.5, 1.5, 1.5]))
])
conn.send(("request", {"uid": uid, "target": target}))
else:
releasable = Contract.from_stamped_rectangles([
(0.0, Rectangle(mins=[0, 0, 0], maxes=[1, 1, 0.5])),
(0.5, Rectangle(mins=[0, 0.5, 0], maxes=[2, 2, 0.5]))
])
print("Agent " + str(uid) + " release > " + str(releasable))
conn.send(("release", {"uid": uid, "releasable": releasable}))
finally:
stop_ev.set() # Stop all automatons
p.join()
conn.close()
manager_conn.close()
def test_disk_locking_processes(block_maker, storage_factory, redis_hostname):
def visit(storage, root):
ds = block_maker.first_ds(first_constant=2, ids_arg=3)
cached = ds >> Apply(image=sleeper(0.1)) >> CacheToDisk(
root, storage, JsonSerializer(), 'image')
for i in ds.ids:
assert ds.image(i) == cached.image(i)
assert ds.image(i) == cached.image(i)
for _ in range(5):
with tempfile.TemporaryDirectory() as temp, storage_factory(
) as temp_storage:
temp = Path(temp) / 'cache'
init_storage(temp,
algorithm={
'name': 'blake2b',
'digest_size': 64
},
levels=[1, 31, 32],
locker={
'name': 'RedisLocker',
'args': [redis_hostname],
'kwargs': {
'prefix': 'connectome.tests',
'expire': 10
}
})
th = Process(target=visit, args=(temp_storage, temp))
th.start()
visit(temp_storage, temp)
th.join()
def test_agent() -> None:
stop_ev = Event()
conn, agent_conn = Pipe()
waypoints = [(-80.0, -65.0, 3.0), (-45.0, -66.0, 3.0), (-33.0, -65.0, 3.0),
(-33.0, -65.0, 0.3)]
aut = Agent(uid=1, motion=MotionHectorQuad("/drone1"), waypoints=waypoints)
p = Process(target=run_as_process,
kwargs={
"aut": aut,
"conn": agent_conn,
"stop_ev": stop_ev
})
p.start()
try:
for i in range(10):
if conn.poll(1.0):
act = conn.recv()
print(act)
if act[0] == "request":
reply = input("> ")
conn.send(("reply", {
"uid": aut.uid,
"acquired": Contract()
}))
else:
print("Response timeout")
finally:
stop_ev.set() # Stop all automatons
p.join()
conn.close()
agent_conn.close()
def main():
# num_workers = int(mp.cpu_count()/3
num_workers = 1
for i in range(num_workers):
file_ids = get_files_for_worker(i, num_workers)
process = Process(target=worker, args=(file_ids, i))
process.start()
def start():
builder = cluster_crawler.ClusterCrawler()
start_block_id = int(sys.argv[1])
block_id = start_block_id
process = None
while builder.crawl_block(block_id):
print("Block %d crawled" % block_id)
if block_id - start_block_id > 0 and (block_id - start_block_id) % settings.block_crawling_limit == 0:
builder.network_graph.check_integrity()
while process is not None and process.is_alive():
print("Waiting for insertion thread to complete...")
process.join()
if process is not None and process.exitcode > 0 : #error
raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode)
process = Process(target=builder.network_graph.synchronize_mongo_db)
process.start()
builder.network_graph = cluster_network.ClusterNetwork(settings.db_server, settings.db_port) #Starting a new graph while other graph data is inserted.
builder.connect_to_bitcoind_rpc()
if process is not None and not process.is_alive() and process.exitcode > 0 : #error
raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode)
block_id+=1
#Finished Crawling, Flushing to DB.
#Waiting for any previous DB Sync
while process is not None and process.is_alive():
print("Waiting for insertion thread to complete...")
process.join()
#Sync the rest
process = Process(target=builder.network_graph.synchronize_mongo_db)
process.start()
process.join()
def func(*args, **kwargs):
deltas, before, after = [], 0, 0
sys.stdout.write(" " * 147)
for item in iterator:
if isinstance(item, tuple):
args = list(item) + list(args)
item = item[0]
else:
args = [item] + args
delta = after - before
eta = 0
if delta > 0:
deltas.append(delta)
eta = (length - item) * (sum(deltas) / len(deltas))
percentage = 100 * item / length
p = Process(target=draw_progress_bar,
args=(eta, percentage,
f"{item + 1} of {length}..."))
before = time.time()
p.start()
function(*args, **kwargs)
p.terminate()
after = time.time()
p = Process(target=draw_progress_bar, args=(0, 100, "Ready!"))
p.start()
time.sleep(0.1)
p.terminate()
print()
class AIOProcess:
""" Execute a coroutine on a separate process """
def __init__(self,
coroutine: Callable = None,
*args,
daemon: bool = False,
target_override: Callable = None,
**kwargs):
if not asyncio.iscoroutinefunction(coroutine):
raise ValueError("target must be a coroutine function")
self.aio_process = Process(target=target_override
or partial(AIOProcess.run_async, coroutine),
args=args,
kwargs=kwargs,
daemon=daemon)
@staticmethod
def run_async(coroutine: Callable, *args, **kwargs):
try:
loop = uvloop.new_event_loop()
asyncio.set_event_loop(loop)
result = loop.run_until_complete(coroutine(*args, **kwargs))
return result
except BaseException:
log.exception(f"aio process {os.getpid()} failed")
raise
def start(self):
self.aio_process.start()
async def join(self, timeout=None):
if not self.is_alive() and self.exit_code is None:
raise ValueError("must start process before joining")
if timeout is not None:
return await asyncio.wait_for(self.join(), timeout)
while self.exit_code is None:
await asyncio.sleep(0.005)
@property
def pid(self):
return self.aio_process.pid
@property
def daemon(self):
return self.aio_process.daemon
@property
def exit_code(self):
return self.aio_process.exitcode
def is_alive(self):
return self.aio_process.is_alive()
def terminate(self):
self.aio_process.terminate()
def start_logging_process():
"""
Starts the logging Process that does the database Logging
"""
tcpserver = LoggingSocketReceiver()
process = Process(target=tcpserver.serve_until_stopped)
process.start()
return process
def main():
sys.stdout = open('log_print.txt', 'a')
#num_workers = int(mp.cpu_count()/3
num_workers = 1
for i in range(num_workers):
file_ids = get_files_for_worker(i, num_workers)
process = Process(target=worker, args=(file_ids, i))
process.start()
def proof_of_work(self, last_block):
"""
Simple Proof of Work Algorithm:
- Find a number p' such that hash(pp') contains leading 4 zeroes
- Where p is the previous proof, and p' is the new proof
:param last_block: <dict> last Block
:return: <int>
"""
last_proof = last_block['proof']
last_hash = self.hash(last_block)
# Flags
flags = Value(c_bool, False)
proof_result = Value(c_int, 0)
# Proof
proof1 = 0
proof2 = 75001
proof3 = 150001
proof4 = 225001
proof5 = 300001
start_time = time.time()
p1 = Process(target=self.find_proof,
args=(flags, 0, last_proof, proof1, last_hash,
proof_result, 0, 75001))
p2 = Process(target=self.find_proof,
args=(flags, 1, last_proof, proof2, last_hash,
proof_result, 75001, 150001))
p3 = Process(target=self.find_proof,
args=(flags, 2, last_proof, proof3, last_hash,
proof_result, 150001, 225001))
p4 = Process(target=self.find_proof,
args=(flags, 3, last_proof, proof4, last_hash,
proof_result, 225001, 300001))
p5 = Process(target=self.find_proof,
args=(flags, 4, last_proof, proof5, last_hash,
proof_result, 300001, sys.maxsize))
p1.start()
p2.start()
p3.start()
p4.start()
p5.start()
p1.join()
p2.join()
p3.join()
p4.join()
p5.join()
end_time = time.time()
print('TIMETIMETIMETIME : ', end_time - start_time)
print('proof_result : ', proof_result.value)
return proof_result.value
def do_cfg_multiprocess(processes=1):
print('processes:: ', str(processes))
procs = []
for i in range(processes):
print('_')
print('_process #:: ', str(i))
proc = Process(target=do_lite_function, args=(i, ))
procs.append(proc)
proc.start()
def run_app_as_process(command,
daemon=False,
shell=False,
state_queue=None) -> Process:
p = Process(target=run_app,
args=(command, shell, state_queue),
daemon=daemon)
p.start()
return p
def process_file(file_path_input, file_path_output, i):
if os.path.exists(file_path_output):
return print('{} {} Already done'.format(now(), file_path_output))
input_file = open(file_path_input, 'r')
output_file = open(file_path_output, 'a')
# every worker create separate file for a input file
writer = csv.writer(output_file, delimiter='\t')
# for every file we create seperate driver (100 URLs)
driver = webdriver.PhantomJS(executable_path=path_to_phantomjs)
for line in input_file:
splited = line.split('\t')
# property_type = splited[0]
url = splited[1]
print('{} Process={} Current url: {}'.format(now(), i, url))
# start process for getting microformat properties
temp_queue = Queue()
# p = Process(target=get_microformat_properties_by_type, args=(url, property_type, temp_queue, i))
p = Process(target=get_element_features,
args=(url, driver, temp_queue, i))
print("{} {} Process={} {} {}".format(now(), i, "Started: ",
"feature extraction", url))
p.start()
event_features = temp_queue.get(timeout=TIME_OUT_FEATURE)
# try:
# pass
# except Empty:
# print("{} {} Process={} {} {}".format(now(), i, "Timed out on: ", "feature extraction", url))
if p.is_alive():
p.terminate()
print("Event features:" + str(event_features))
if event_features is not None:
print("{} Process={} Got properties for {}".format(
now(), i, url))
# start process for feature extraction and writing to separate file
# p_event_features = Process(target=get_event_features_and_write,
# args=(event_features, driver, writer, i, output_file))
#
p_event_features = Process(target=write_element_features,
args=(event_features, writer, i,
output_file))
p.start()
# start_with_timeout(p_event_features, TIME_OUT_LOAD, "feature writing", url, i)
if p_event_features.is_alive():
p.terminate()
driver.service.process.send_signal(signal.SIGTERM)
driver.quit()
return 'done'
def do_main_driver_multiprocess(processes=1, reps=1):
print('processes:: ', str(processes))
procs = []
for i in range(processes):
print('_')
print('_process #:: ', str(i))
proc = Process(target=main_driver, args=(reps, ))
procs.append(proc)
proc.start()
def compute_hub_SALSA(inlinks, inlinks_count, outlinks, outlinks_count,
rootSet):
print("Size of rootset is " + str(len(rootSet)))
# computing hub scores
salsa_hub_scores = dict()
countBaseSetWithOutlinks = 0
for page_id in rootSet:
if page_id in outlinks:
if len(outlinks[page_id]) > 0:
countBaseSetWithOutlinks += 1
print(countBaseSetWithOutlinks)
for page_id in rootSet:
if page_id in outlinks:
if len(outlinks[page_id]) > 0:
salsa_hub_scores[page_id] = 1 / countBaseSetWithOutlinks
else:
salsa_hub_scores[page_id] = 0
else:
salsa_hub_scores[page_id] = 0
for iter in range(0, 3):
print(iter)
global temp_salsa_hub_scores
count = 0
page_list = list()
procs = list()
for page_id in rootSet:
page_list.append(page_id)
count += 1
if count % 100 == 0:
print(count)
proc = Process(target=multi_scoring,
args=(
page_list,
inlinks,
inlinks_count,
outlinks,
outlinks_count,
temp_salsa_hub_scores,
salsa_hub_scores,
))
page_list = list()
procs.append(proc)
proc.start()
print("Total processes " + str(len(procs)))
for proc in procs:
proc.join()
for page_id in temp_salsa_hub_scores:
salsa_hub_scores[page_id] = temp_salsa_hub_scores[page_id]
temp_salsa_hub_scores = dict()
return salsa_hub_scores
def dashboard():
# This fixture guarantees the proper termination of all spawned subprocesses
# after the tests.
dashboard = Process(target=run_detection)
dashboard.start()
yield
for child in psutil.Process(dashboard.pid).children(recursive=True):
child.kill()
dashboard.terminate()
dashboard.join()
def run(configfile):
"""
This method is called after initializing the framework and starts the configured servers.
@param configfile: Path referring to the xml configfile.
"""
logger.debug('run() method started!')
for role in config.get_roles(configfile):
process = Process(target=role)
print('Starting Process!')
process.start()
logger.debug('run() is done!')
def main():
size = os.path.getsize("dict.txt")
half_size = size // 2
p1 = Process(target=read_part1, args=("dict.txt", half_size))
p2 = Process(target=read_part2, args=("dict.txt", half_size))
p1.start()
p2.start()
p1.join()
p2.join()
def start_up_app(self):
self.show_splash_screen()
try:
# load db in separate process
process_startup = Process(target=LucteriosRefreshAll)
process_startup.start()
while process_startup.is_alive():
# print('updating')
self.splash.update()
finally:
self.remove_splash_screen()
def keyboard_interrupt(cls, path_file_input: Path, path_file_output: Path) -> None:
"""Test process of keyboard interrupt."""
process = Process(target=cls.report_raises_keyboard_interrupt, args=(path_file_input, path_file_output))
process.start()
assert LocalSocket.receive() == "Ready"
time.sleep(SECOND_SLEEP_FOR_TEST_KEYBOARD_INTERRUPT_CTRL_C_POSIX)
cls.simulate_ctrl_c_in_posix(process)
assert LocalSocket.receive() == "Test succeed"
psutil_process = psutil.Process(process.pid)
assert psutil_process.wait() == 0
# Reason: Requires to enhance types-psutil
assert not psutil_process.is_running() # type: ignore
def do_sanity_multiprocess():
names = ['A', 'B', 'C']
procs = []
# proc = Process(target=do_lite_function)
# procs.append(proc)
# proc.start()
for count, name in enumerate(names, 1):
print(count, name)
proc = Process(target=do_lite_function, args=(name, ))
procs.append(proc)
proc.start()
def __enter__(self):
self.listener_ = Listener(self.address_, family='AF_INET')
for child in range(self.nb_children_):
process = Process(target=creator,
args=(
self.data_,
self.models_,
self.address_,
))
process.start()
self.children_.append(process)
return self
def serve(routing: dict, open_browser=False, timeout=None, filename=''):
p = Process(target=WebServer.serve_and_browse,
args=(
routing,
open_browser,
filename,
))
p.start()
wait_for_server_seconds = timeout
logging.info("Waiting for server %d seconds" % wait_for_server_seconds)
time.sleep(wait_for_server_seconds)
p.terminate()
def test_sigterm(manager_queue: "queue.Queue[LogRecord]") -> None:
"""ProcessTaskPoolExecutor should raise keyboard interrupt."""
process = Process(target=example_use_case_cancel_repost_process_id, kwargs={"queue_main": manager_queue})
process.start()
LocalSocket.receive()
time.sleep(SECOND_SLEEP_FOR_TEST_SHORT)
psutil_process = psutil.Process(process.pid)
psutil_process.send_signal(SIGTERM)
psutil_process.wait()
# Reason: Requires to enhance types-psutil
assert not psutil_process.is_running() # type: ignore
assert_graceful_shutdown(manager_queue)
def run():
# while True:
for name in ['jd','sn']: #对应存储再config中的JSON配置文件
custom_settings = get_config(name) # 爬取使用的 Spider 名称
spider = custom_settings.get('spider', 'aCrawler')
project_settings = get_project_settings()
settings = dict(project_settings.copy()) # 合并配置
settings.update(custom_settings.get('settings'))
process = Process(target=crawl, kwargs={'settings': settings, 'spider': spider, 'name': name,
'custom_settings': custom_settings})
process.start()
process.join()
def start_up_app(self):
self.show_splash_screen()
# load db in separate process
process_startup = Process(target=LucteriosRefreshAll)
process_startup.start()
while process_startup.is_alive():
# print('updating')
self.splash.update()
self.remove_splash_screen()
def test(args):
model = Rockfish.load_from_checkpoint(checkpoint_path=args.checkpoint)
model.freeze()
test_ds = Fast5Data(args.test_path, args.recursive, args.reseg_path,
args.norm_method, args.motif, args.sample_size, args.window)
if args.n_workers > 0:
test_dl = DataLoader(test_ds, batch_size=args.batch_size,
num_workers=args.n_workers, pin_memory=True,
worker_init_fn=worker_init_fn,
prefetch_factor=args.prefetch_factor)
else:
test_dl = DataLoader(test_ds, batch_size=args.batch_size,
num_workers=args.n_workers, pin_memory=True,
worker_init_fn=worker_init_fn)
n_gpus = torch.cuda.device_count()
if n_gpus > 0:
model = DataParallel(model, device_ids=list(range(n_gpus)))
model.to(f'cuda:{model.device_ids[0]}')
model.eval()
output_queue = mp.Queue()
consumers = []
abs_out_path = str(args.out_path.absolute())
for i in range(args.output_workers):
worker_path = TMP_PATH.format(final=abs_out_path, id=i)
process = Process(target=output_worker, args=(worker_path, output_queue))
process.start()
consumers.append(process)
with torch.no_grad():
for info, sig, k_mer in tqdm(test_dl):
pred = model(sig, k_mer).squeeze(-1)
pred = pred.cpu().numpy()
output_queue.put((info, pred))
for _ in range(len(consumers)):
output_queue.put(None)
for c in consumers:
c.join()
with args.out_path.open('w') as out:
for i in range(len(consumers)):
worker_path = TMP_PATH.format(final=abs_out_path, id=i)
with open(worker_path, 'r') as tmp_f:
out.write(tmp_f.read())
os.remove(worker_path)
def run():
while True:
for name in ['sina', 'xinhua']: #对应存储再config中的JSON配置文件
custom_settings = get_config(name) # 爬取使用的 Spider 名称
spider = custom_settings.get('spider', 'sina')
project_settings = get_project_settings()
settings = dict(project_settings.copy()) # 合并配置
settings.update(custom_settings.get('settings'))
process = Process(target=crawl, kwargs={'settings': settings, 'spider': spider, 'name': name,
'custom_settings': custom_settings})
process.start()
process.join()
time.sleep(60 * 60 * 24) #一天运行一次
def test_keyboard_interrupt_on_linux(self) -> None:
"""
- Keyboard interrupt should reach to all descendant processes.
- Keyboard interrupt should shutdown ProcessTaskPoolExecutor gracefully.
"""
process = Process(target=self.report_raises_keyboard_interrupt)
process.start()
LocalSocket.receive()
time.sleep(SECOND_SLEEP_FOR_TEST_SHORT)
self.simulate_ctrl_c_in_posix(process)
assert LocalSocket.receive() == "Test succeed"
process.join()
assert process.exitcode == 0
assert not process.is_alive()
def create_app():
app = Flask(__name__)
logger.info("Server starting...")
from app.routes import bp
app.register_blueprint(bp)
starting_port = 50052
for port in range(starting_port, starting_port + 8):
process = Process(target=serve_runner, args=(port, ))
process.start()
return app
def terminate(cls, path_file_input: Path, path_file_output: Path) -> None:
"""Test process of keyboard interrupt."""
process = Process(target=cls.report_raises_cencelled_error, args=(path_file_input, path_file_output))
process.start()
assert LocalSocket.receive() == "Ready"
time.sleep(SECOND_SLEEP_FOR_TEST_KEYBOARD_INTERRUPT_CTRL_C_POSIX)
psutil_process = psutil.Process(process.pid)
psutil_process.terminate()
# Return code seems change when:
# run this test only: -15
# run all tests: 1
assert psutil_process.wait() in [-15, 1]
# Reason: Requires to enhance types-psutil
assert not psutil_process.is_running() # type: ignore
class ActorRunner:
def __init__(self, actors: Iterable[Actor]):
self._process = Process(target=self._run, args=(actors, ))
self._process.start()
def _run(self, actors: Iterable[Actor]):
asyncio.run(self._loop(actors))
async def _loop(self, actors: Iterable[Actor]):
loop = get_running_loop()
for actor in actors:
loop.create_task(actor.runner())
while loop.is_running():
await asyncio.sleep(1)
class SubprocessEnv(Env):
def __init__(self, factory: Callable[[], Env], blocking: bool = True):
self._blocking = blocking
self._parent_conn, child_conn = Pipe()
self._process = Process(target=self._start, args=(factory, child_conn))
self._process.start()
self.observation_space, self.action_space = self._parent_conn.recv()
def _start(self, factory: Callable[[], Env], connection: Connection):
env = factory()
_ = env.reset()
connection.send((env.observation_space, env.action_space))
terminate = False
while not terminate:
command, kwargs = connection.recv()
if command == 'render':
rendering = env.render(**kwargs)
connection.send(rendering)
elif command == 'step':
step = env.step(**kwargs)
connection.send(step)
elif command == 'reset':
obs = env.reset(**kwargs)
connection.send(obs)
elif command == 'close':
terminate = True
connection.close()
def step(self, action):
self._parent_conn.send(('step', dict(action=action)))
return self._return()
def reset(self, **kwargs):
self._parent_conn.send(('reset', kwargs))
return self._return()
def render(self, mode: str = 'human', **kwargs):
self._parent_conn.send(('render', {'mode': mode, **kwargs}))
return self._return()
def close(self):
self._parent_conn.send(('close', False))
self._parent_conn.close()
def _return(self) -> Any:
if self._blocking:
return self._parent_conn.recv()
else:
return lambda: self._parent_conn.recv()
def get_data_from_sn(self, search_list, database_name):
if database_name == 'neo':
sn_node = graph.find_one("SN", "name", "SOCIALNETWORKS")
twitter_node = graph.merge_one("Twitter", "name", "TWITTER")
sn_has_twitter = Relationship(sn_node, "HAS", twitter_node)
graph.create_unique(sn_has_twitter)
twitter_searcher = TwitterSearcher()
twitter_searcher_process = Process(target=twitter_searcher.run_twitter_search, args=(search_list, database_name,))
twitter_streamer = TwitterStreamer()
twitter_streamer_process = Process(target=twitter_streamer.run_twitter_stream, args=(search_list, database_name,))
twitter_searcher_process.start()
twitter_streamer_process.start()
twitter_streamer_process.join()
twitter_searcher_process.join()
def start():
builder = GraphBuilder()
start_block_id = int(sys.argv[1])
block_id = start_block_id
process = None
try:
while builder.crawl_block(block_id):
print("Block %d crawled" % block_id)
if block_id - start_block_id > 0 and (block_id - start_block_id) % Settings.block_crawling_limit == 0:
builder.network_graph.check_integrity()
while process is not None and process.is_alive():
print("Waiting for insertion thread to complete...")
process.join()
if process is not None and process.exitcode > 0 : #error
raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode)
process = Process(target=builder.network_graph.synchronize_mongo_db)
process.start()
builder.network_graph = NetworkGraph.Network(Settings.db_server,Settings.db_port) #Starting a new graph while other graph data is inserted.
if process is not None and not process.is_alive() and process.exitcode > 0 : #error
raise Exception("Errorcode %d in DB Sync Thread, aborting" % process.exitcode)
block_id+=1
#Finished Crawling, Flushing to DB.
#Waiting for any previous DB Sync
while process is not None and process.is_alive():
print("Waiting for insertion thread to complete...")
process.join()
#Sync the rest
process = Process(target=builder.network_graph.synchronize_mongo_db)
process.start()
process.join()
#DONE!
#For Debugging purpose
except:
input("An exception will rise ")
raise
def get_data_from_sn(self, search_list, database_name):
if database_name == 'neo':
sn_node = graph.find_one("SN", "name", "SOCIALNETWORKS")
facebook_node = graph.merge_one("Facebook", "name", "FACEBOOK")
sn_has_facebook = Relationship(sn_node, "HAS", facebook_node)
graph.create_unique(sn_has_facebook)
event_searcher = EventSearcher()
event_process = Process(target=event_searcher.run_facebook_events, args=(search_list, database_name,))
group_searcher = GroupSearcher()
group_process = Process(target=group_searcher.run_facebook_groups, args=(search_list, database_name,))
page_searcher = PageSearcher()
page_process = Process(target=page_searcher.run_facebook_pages, args=(search_list, database_name,))
event_process.start()
group_process.start()
page_process.start()
event_process.join()
group_process.join()
page_process.join()
class TestOrchestrator(BaseTestCase):
def setUp(self):
self.test_directory = os.path.dirname(os.path.realpath(__file__))
self.test_config = os.path.join(self.test_directory,
"packager_config.cfg")
self.packager_config = PackagerConfig(self.test_config)
"""Spin up a server in a seperate process"""
def start_test_server():
app = Orchestrator(self.packager_config)
from werkzeug.serving import run_simple
run_simple('127.0.0.1', 5000, app)
self.p = Process(target=start_test_server, daemon=True)
self.p.start()
def tearDown(self):
"""Make sure we tear down the process properly at the end"""
self.p.terminate()
def testOrchestratorAcceptingInputCorrectly(self):
@exponential_back_off
def wait_for_process():
return requests.get("http://127.0.0.1:5000/health")
wait_seconds = 1
print("await process spin up for %d seconds" % wait_seconds)
time.sleep(wait_seconds)
wait_for_process()
with open(self.get_file('code'), 'r') as code, \
open(self.get_file('tests'), 'r') as test:
language = 'python'
question_name = 'foo'
code_payload = code.read()
test_payload = test.read()
payload = {
'question_name': question_name,
'language': language,
'code': code_payload,
'test': test_payload
}
# http:// required,
# requests library needs protocol scheme to connect
got_resp_raw = requests.post('http://127.0.0.1:5000/submit',
data=payload)
got_resp = json.loads(got_resp_raw.text)
exp_data = {
'language': language,
'code': code_payload,
'test': test_payload,
'question_name': question_name
}
exp_response_message = 'not tested for in this testcase'
exp_resp = {
'execution_result': exp_response_message,
'got_data': exp_data
}
self.assertEqual(exp_resp['got_data'], got_resp['got_data'])
# we just need to check if the result contains success
for testcase, test_result in got_resp['execution_result'].items():
self.assertTrue(test_result['success'],
'testcase {} failure with message {}'
.format(testcase, test_result['message']))
def show_info_on(a_topic):
''' Postconditions:
1. a_topic is on the monitor
2. The name of the module executing this is on the monitor
3. If available, the ID of the process parent to this is on the monitor
4. The ID of the process executing this is on the monitor
'''
print('Information on ' + a_topic) # 1.
print('Name of module executing this: ', __name__) # 2.
if hasattr(os, 'getppid'): # 3: if available on this OS
print('Parent process: ', os.getppid())
print('ID of process executing this: ', os.getpid(), '\n') # 4.
def say_hello(name):
''' Postconditions:
1. = Postconditions of get_info_on('Say-hello process')
2. "Hello <name>" is on the monitor
'''
show_info_on('Say-hello process')
print('Hello ', name)
if __name__ == '__main__':
''' Postconditions:
1. = Postconditions of get_info_on (this) main process
2. = Postconditions of get_info_on a new process executing say_hello('Hugh Person')
'''
show_info_on('main line')
p = Process(target=say_hello, args=('Hugh Person',))
p.start()
p.join()
def process_paid_order(self, order):
self.logger.info(order.order_number+" - process")
self.ioc.new_order_service().process_paid_order(order)
p = Process(target=self.inform_customer, args=(self.ioc, order,))
p.start()
p.join()
from multiprocessing.queues import Queue
import os, time, random
# 写数据进程执行的代码:
def write(q):
print('Process to write: %s' % os.getpid())
for value in ['A', 'B', 'C']:
print('Put %s to queue...' % value)
q.put(value)
time.sleep(random.random())
# 读数据进程执行的代码:
def read(q):
print('Process to read: %s' % os.getpid())
while True:
value = q.get(True)
print('Get %s from queue.' % value)
if __name__=='__main__':
# 父进程创建Queue，并传给各个子进程：
q = Queue()
pw = Process(target=write, args=(q,))
pr = Process(target=read, args=(q,))
# 启动子进程pw，写入:
pw.start()
# 启动子进程pr，读取:
pr.start()
# 等待pw结束:
pw.join()
# pr进程里是死循环，无法等待其结束，只能强行终止:
pr.terminate()
print('end matrix')
'''
# declare variables in shared memory
Smallest = Value('i', n)
TotalCount = Value('i', 0)
# get break down of how many rows per process
size = int(n / 4)
# take time
start_time = time.time()
# create each process
p0 = Process(target=subMatrixSmallestCount, args=(matrix, 0, size, n, Smallest, TotalCount, 1))
p1 = Process(target=subMatrixSmallestCount, args=(matrix, 1, size, n, Smallest, TotalCount, 2))
p2 = Process(target=subMatrixSmallestCount, args=(matrix, 2, size, n, Smallest, TotalCount, 3))
p3 = Process(target=subMatrixSmallestCount, args=(matrix, 3, size, n, Smallest, TotalCount, 4))
# start processes
p0.start()
p1.start()
p2.start()
p3.start()
# wait for processes to finish
p0.join()
p1.join()
p2.join()
p3.join()
print('Smallest value: {}'.format(Smallest.value))
print('Count of that value: {}'.format(TotalCount.value))
# print time
print('Total time: {}s'.format (time.time() - start_time))
class TestRemoteSession(TestCase):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.server_process = None
""":type: Process"""
def setUp(self):
self.test_file = TestFile(settings.test_file_size)
self.original_data = self.test_file.get_content()
self.ftp_root = self.test_file.path
self.remote_filename = uuid.uuid4().hex
tuned_server, self.port = server.get_tuned_server(self.ftp_root)
def server_func():
tuned_server.serve_forever(handle_exit=True)
self.server_process = Process(target=server_func)
""":type: Process"""
self.server_process.start()
def get_connected_client_in_binary_mode(self):
client = Client()
client.connect('localhost', self.port)
client.login(settings.ftp_user, settings.ftp_pass)
client.type('I') # Binary mode ('I'mage)
return client
def tearDown(self):
while self.server_process.is_alive():
# посылаем SIGINT процессу, pyftpd умеет его обрабатывать и закрывается корректно
os.kill(self.server_process.pid, signal.SIGINT)
self.server_process.join(timeout=1)
unlink(self.test_file.full_filename)
def test_receive_file(self):
"""
тест на получение заранее сгенерированных и сохраненных на диске данных клиентом через подключение
к локальному серверу.
"""
client = self.get_connected_client_in_binary_mode()
code, rest, data = client.retr(self.test_file.filename)
q_code, q_rest = client.quit()
self.assertEqual(code, 226)
self.assertTrue(self.original_data == data)
def test_send_file(self):
"""
Тест на отправку файла клиентом на локальный сервер и сравнение содержимого файла на диске
с отправленными данными
"""
client = self.get_connected_client_in_binary_mode()
client.stor(self.remote_filename, self.original_data)
with open(os.path.join(self.ftp_root, self.remote_filename), 'rb') as f:
stored_file_content = f.read()
code, rest = client.quit()
self.assertTrue(self.original_data == stored_file_content)
