def start( self ): self.dataQ = Queue() self.messageQ = Queue() self.shutdownQ = Queue() # Queue to tell the Impinj monitor to shut down. self.readerStatusCB() if self.useHostName.GetValue(): self.impinjProcess = Process( name='ImpinjProcess', target=ImpinjServer, args=( self.dataQ, self.messageQ, self.shutdownQ, ImpinjHostNamePrefix + self.impinjHostName.GetValue() + ImpinjHostNameSuffix, ImpinjInboundPort, self.getAntennaStr(), self.readerStatusCB, ) ) else: self.impinjProcess = Process( name='ImpinjProcess', target=ImpinjServer, args=( self.dataQ, self.messageQ, self.shutdownQ, self.impinjHost.GetAddress(), ImpinjInboundPort, self.getAntennaStr(), self.readerStatusCB, ) ) self.impinjProcess.daemon = True self.crossMgrProcess = Process( name='CrossMgrProcess', target=CrossMgrServer, args=(self.dataQ, self.messageQ, self.shutdownQ, self.getCrossMgrHost(), CrossMgrPort) ) self.crossMgrProcess.daemon = True self.impinjProcess.start() self.crossMgrProcess.start()
def start(self):
self.dataQ = Queue()
self.messageQ = Queue()
self.shutdownQ = Queue(
) # Queue to tell the Alien monitor to shut down.
self.alienProcess = Process(
name='AlienProcess',
target=AlienServer,
args=(self.dataQ, self.messageQ, self.shutdownQ,
self.getNotifyHost(), NotifyPort, HeartbeatPort,
self.getAntennaStr(), self.listenForHeartbeat.GetValue(),
self.cmdHost.GetAddress(), self.cmdPort.GetValue()))
self.alienProcess.daemon = True
self.crossMgrProcess = Process(name='CrossMgrProcess',
target=CrossMgrServer,
args=(self.dataQ, self.messageQ,
self.shutdownQ,
self.getCrossMgrHost(),
CrossMgrPort))
self.crossMgrProcess.daemon = True
self.alienProcess.start()
self.crossMgrProcess.start()
def __init__(self, common_params=None, dataset_params=None):
"""
Args:
common_params: A dict
dataset_params: A dict
"""
if common_params:
self.image_size = int(common_params['image_size'])
self.batch_size = int(common_params['batch_size'])
if dataset_params:
self.data_path = str(dataset_params['path'])
self.thread_num = int(int(dataset_params['thread_num']) / 2)
self.thread_num2 = int(int(dataset_params['thread_num']) / 2)
#record and image_label queue
self.record_queue = Queue(maxsize=10000)
self.image_queue = Queue(maxsize=5000)
self.batch_queue = Queue(maxsize=100)
self.record_list = []
# filling the record_list
input_file = open(self.data_path, 'r')
for line in input_file:
line = line.strip()
self.record_list.append(line)
self.record_point = 0
self.record_number = len(self.record_list)
self.num_batch_per_epoch = int(self.record_number / self.batch_size)
t_record_producer = Process(target=self.record_producer)
t_record_producer.daemon = True
t_record_producer.start()
# for i in range(self.thread_num):
# t = Process(target=self.record_customer)
# t.daemon = True
# t.start()
# for i in range(self.thread_num2):
# t = Process(target=self.image_customer)
# t.daemon = True
# t.start()
self.record_process = Process(target=self.record_customer)
self.record_process.daemon = True
self.record_process.start()
self.image_cus_process = Process(target=self.image_customer)
self.image_cus_process.daemon = True
self.image_cus_process.start()
def __init__(self, data_path, in_dir):
"""
Args:
common_params: A dict
dataset_params: A dict
"""
self.input_size = 224
self.batch_size = 32
self.data_path = data_path
self.gray_dir = in_dir
self.thread_num = 20
self.thread_num2 = 20
#record and image_label queue
self.record_queue = Queue(maxsize=16000)
self.image_queue = Queue(maxsize=4000)
self.batch_queue = Queue(maxsize=100)
self.record_list = []
input_file = open(self.data_path, 'r')
print('Dataset {}'.format(self.data_path))
for line in input_file:
line = line.strip()
# name = os.path.split(line)[1]
# class_name = name.split('_')[0]
# self.record_list.append(class_name+'/'+name)
self.record_list.append(line)
# self.record_list.sort()
self.record_point = 0
self.record_number = len(self.record_list)
print('Total: {}'.format(self.record_number))
t_record_producer = Process(target=self.record_producer)
t_record_producer.daemon = True
t_record_producer.start()
for i in range(self.thread_num):
t = Process(target=self.record_customer)
t.daemon = True
t.start()
for i in range(self.thread_num2):
t = Process(target=self.image_customer)
t.daemon = True
t.start()
def __init__(self, config, processor_class):
if config.DEBUG:
from threading import Thread as Process
else:
from multiprocessing import Process
self._config = config
self._clients = {}
self._idle_workers = SimpleQueue()
self._max_queue_size = getattr(config, 'TCP_MAX_QUEUE_SIZE', None)
self._running_workers = SimpleQueue()
self._waiting_tasks = SimpleQueue(self._max_queue_size)
self._worker_processes = []
max_buffer_size = getattr(config, 'TCP_MAX_BUFFER_SIZE', None)
self._worker_server = CallbackTcpServer(
on_connect=self._on_worker_connect,
max_buffer_size=max_buffer_size)
self._worker_server.listen(**config.WORK_ENDPOINT)
self._connection_server = CallbackTcpServer(
on_connect=self._on_client_connect,
max_buffer_size=max_buffer_size)
for listen_port in config.LISTEN_ENDPOINTS:
self._connection_server.listen(**listen_port)
for i in xrange(0, config.WORKER_COUNT):
processor = processor_class(i, config)
p = Process(target=processor.run)
p.processor = processor
self._worker_processes.append(p)
p.start()
def run(prod_list, topics=None, test_message=None):
tmessage = get_test_message(test_message)
if tmessage:
from threading import Thread as Process
from posttroll.message import Message
else:
from multiprocessing import Process
with open(prod_list) as fid:
config = yaml.load(fid.read(), Loader=BaseLoader)
topics = topics or config['product_list'].pop('subscribe_topics', None)
if not tmessage:
listener = ListenerContainer(topics=topics)
while True:
try:
if tmessage:
msg = Message(rawstr=tmessage)
else:
msg = listener.output_queue.get(True, 5)
except KeyboardInterrupt:
if not tmessage:
listener.stop()
return
except queue_empty:
continue
proc = Process(target=process, args=(msg, prod_list))
proc.start()
proc.join()
if tmessage:
break
def escuchar(self):
try:
sock = socket(AF_INET, SOCK_STREAM) # Creamos un socket
sock.bind((self.host,
self.port)) # Conectamos el socket al puerto y host
sock.listen(5) # Escuchamos a 5 maximo
print("Server is running on " + str(self.port))
print("Waiting for clients....")
trds = []
for i in range(5):
conn, addr = sock.accept() # Aceptamos la conexion
self.clients.append(
conn
) # Insertamos el objeto del socket en la lista clients
t = Process(target=self.clientHandler, args=(
conn,
addr)) # Creamos un thread por cada cliente que se conecta
trds.append(t) # Insertamos el thread en la lista trds
t.start() # Iniciamos el thread
for t in trds:
t.join() # Aceptamos el thread
except KeyboardInterrupt:
print("Exiting..")
sock.close() # Si ocurre alguna excepcion cerramos el socket
return
def reimport(self, parameters):
"""import_dataset's equivalent without the Django request validation and transformation to Importer's parameter dict.
Used mostly by Syncer or if reimport is added to GUI.
:param parameters: Dataset Importer parameters which have been validated during a previous run.
:type parameters: dict
"""
parameters = self._preprocess_reimport(parameters=parameters)
Process(target=_import_dataset, args=(parameters, self._n_processes, self._process_batch_size)).start()
def background_run(self):
"""Run the client in the background."""
self.streaming, self.started, self.can_add_streams = True, True, False
self._clear_killer() # Clear the kill queue
streaming_process = Process(target=self._bg_run, name="StreamProcess")
# streaming_process.daemon = True
streaming_process.start()
self.processes.append(streaming_process)
Logger.info("Started streaming")
def __init__(self, data_path, nthread=12):
"""
Args:
common_params: A dict
dataset_params: A dict
"""
self.image_size = 256
self.batch_size = 16
self.data_path = data_path
self.thread_num = nthread
self.thread_num2 = nthread
self.record_queue = Queue(maxsize=30000)
self.image_queue = Queue(maxsize=15000)
self.batch_queue = Queue(maxsize=300)
self.record_list = []
self.data_dir = '/srv/glusterfs/xieya/data/places365_standard'
# filling the record_list
input_file = open(self.data_path, 'r')
for line in input_file:
self.record_list.append(os.path.join(self.data_dir,
line[:-1])) # Strip newline.
self.record_point = 0
self.record_number = len(self.record_list)
self.data_size = self.record_number
self.num_batch_per_epoch = int(self.record_number / self.batch_size)
t_record_producer = Process(target=self.record_producer)
t_record_producer.daemon = True
t_record_producer.start()
for i in range(self.thread_num):
t = Process(target=self.record_customer)
t.daemon = True
t.start()
for i in range(self.thread_num2):
t = Process(target=self.image_customer)
t.daemon = True
t.start()
def _start_visdom(port):
p = Process(target=visdom.server.start_server,
kwargs={
"port": port,
"base_url": ''
})
p.start()
atexit.register(p.terminate)
time.sleep(20)
return True
def _colorize_data_train(data_path, in_dir, out_dir):
ds = DataSet(data_path, in_dir)
input_tensor = tf.placeholder(tf.float32,
shape=(_BATCH_SIZE, _INPUT_SIZE, _INPUT_SIZE,
1))
model = demo._get_model(input_tensor)
saver = tf.train.Saver()
config = tf.ConfigProto(allow_soft_placement=True)
config.gpu_options.allow_growth = True
save_queue = Queue(100)
lock = RLock()
def save_fn():
while True:
img_l_batch, img_313_rs_batch, img_name_batch = save_queue.get()
for idx in range(_BATCH_SIZE):
img_l = img_l_batch[idx]
img_rgb, _ = utils.decode(img_l, img_313_rs_batch[idx:idx + 1],
_T)
io.imsave(os.path.join(out_dir, img_name_batch[idx]), img_rgb)
lock.acquire()
global i
i += _BATCH_SIZE
lock.release()
for _ in range(20):
t = Process(target=save_fn)
t.daemon = True
t.start()
with tf.Session(config=config) as sess:
saver.restore(sess, _CKPT_PATH)
start_time = monotonic.monotonic()
while i < ds.record_number:
lock.acquire()
if i % (_BATCH_SIZE * _LOG_FREQ) == 0:
print("Image count: {0} Time: {1}".format(
i,
monotonic.monotonic() - start_time))
start_time = monotonic.monotonic()
lock.release()
img_l_batch, img_l_rs_batch, img_name_batch = ds.batch()
img_313_rs_batch = sess.run(
model, feed_dict={input_tensor: img_l_rs_batch})
save_queue.put((img_l_batch, img_313_rs_batch, img_name_batch))
print("Save queue size: {}".format(save_queue.qsize()))
print("Colorized: {}".format(i))
def __init__(self):
self.image_size = 224
self.batch_size = 128
self.data_path = 'E:/chengzirui/dataset/images256/'
self.thread_num = 10
self.thread_num2 = 10
# record and image_label queue
self.record_queue = Queue(maxsize=10000)
self.image_queue = Queue(maxsize=5000)
self.batch_queue = Queue(maxsize=150)
self.record_list = []
# filling the record_list
input_file = open('data_list/places_train.txt', 'r')
for line in input_file:
line = line.strip()
self.record_list.append(line)
self.record_point = 0
# 总record大小
self.record_number = len(self.record_list)
self.num_batch_per_epoch = int(self.record_number / self.batch_size)
t_record_producer = Process(target=self.record_producer)
t_record_producer.daemon = True
t_record_producer.start()
for i in range(self.thread_num):
t = Process(target=self.record_customer)
t.daemon = True
t.start()
for i in range(self.thread_num2):
t = Process(target=self.image_customer)
t.daemon = True
t.start()
def __init__(self, common_params=None, dataset_params=None):
"""
Args:
common_params: A dict
dataset_params: A dict
"""
if common_params:
self.image_size = int(common_params['image_size'])
self.batch_size = int(common_params['batch_size'])
if dataset_params:
self.data_path = str(dataset_params['path'])
self.thread_num = int(int(dataset_params['thread_num']))
# Create record and batch queue for multi-threading
self.record_queue = Queue(maxsize=10000)
self.batch_queue = Queue(maxsize=100)
# Fill in the record_list
self.record_list = []
input_file = open(self.data_path, 'r')
for line in input_file:
line = line.strip()
self.record_list.append(line)
self.record_point = 0
self.record_number = len(self.record_list)
self.num_batch_per_epoch = int(self.record_number / self.batch_size)
# Keep adding record into record_queue
t_record_producer = Process(target=self.record_producer)
t_record_producer.daemon = True
t_record_producer.start()
# (Multi-threads) Read/Process images and batch them
for i in range(self.thread_num):
t = Process(target=self.image_customer)
t.daemon = True
t.start()
def __init__(self, w=None, h=None, max_side=None):
self.w = w
self.h = h
self.max_side = max_side
self.queue = Queue(maxsize=20)
self.queue_pts = Queue(maxsize=20)
threads = [
Process(target=worker_display, args=(self.queue_pts, )),
]
for t in threads:
t.daemon = True
t.start()
def __init__(self,
exp_name="main",
server="http://localhost",
port=8080,
auto_close=True,
auto_start=False,
auto_start_ports=(8080, 8000),
**kwargs):
"""
Creates a new NumpyVisdomLogger object.
Args:
name: The name of the visdom environment
server: The address of the visdom server
port: The port of the visdom server
auto_close: Close all objects and kill process at the end of the python script
auto_start: Flag, if it should try to start a visdom server on the given ports
auto_start_ports: Ordered list of ports, to try to start a visdom server on (only on the first available
port)
"""
super(NumpyVisdomLogger, self).__init__(**kwargs)
if auto_start:
auto_port = start_visdom(auto_start_ports)
if auto_port != -1:
port = auto_port
server = "http://localhost"
self.name = exp_name
self.server = server
self.port = port
self.vis = ExtraVisdom(env=self.name,
server=self.server,
port=self.port)
self._value_counter = defaultdict(dict)
self._3d_histograms = dict()
self._queue = Queue()
self._process = Process(target=self.__show, args=(self._queue, ))
if auto_close:
# atexit.register(self.close_all)
if not IS_WINDOWS:
atexit.register(self.exit)
atexit.register(self.save_vis)
self._process.start()
def start(self, file_loc, audio_src):
# from rec_thread import rec_thread
try:
from billiard import forking_enable
forking_enable(0)
except ImportError:
pass
self.should_close.clear()
self.process = Process(target=rec_thread,
args=(file_loc, audio_src, self.should_close))
self.process.start()
try:
forking_enable(1)
except:
pass
def run(prod_list,
topics=None,
test_message=None,
nameserver='localhost',
addresses=None):
"""Spawn one or multiple subprocesses to run the jobs from the product list."""
LOG.info("Launching trollflow2")
tmessage = get_test_message(test_message)
if tmessage:
from threading import Thread as Process
from six.moves.queue import Queue
from posttroll.message import Message
else:
from multiprocessing import Process, Queue
with open(prod_list) as fid:
config = yaml.load(fid.read(), Loader=BaseLoader)
topics = topics or config['product_list'].pop('subscribe_topics', None)
if not tmessage:
listener = ListenerContainer(topics=topics,
nameserver=nameserver,
addresses=addresses)
while True:
try:
if tmessage:
msg = Message(rawstr=tmessage)
else:
msg = listener.output_queue.get(True, 5)
except KeyboardInterrupt:
if not tmessage:
listener.stop()
return
except Empty:
continue
produced_files = Queue()
proc = Process(target=process, args=(msg, prod_list, produced_files))
start_time = datetime.now()
proc.start()
proc.join()
try:
exitcode = proc.exitcode
except AttributeError:
exitcode = 0
check_results(produced_files, start_time, exitcode)
if tmessage:
break
def StartListener( startTime=datetime.datetime.now(), comPort=1, HOST=None, PORT=None ): # HOST, PORT not used. global q global shutdownQ global listener StopListener() if Model.race: comPort = (comPort or getattr(Model.race, 'comPort', 1)) q = Queue() shutdownQ = Queue() listener = Process( target = Server, args=(q, shutdownQ, comPort, startTime) ) listener.name = 'RaceResultUSB Listener' listener.daemon = True listener.start()
def StartListener( startTime = datetime.datetime.now(), HOST = DEFAULT_HOST, PORT = DEFAULT_PORT ): global q global shutdownQ global listener global dateToday dateToday = startTime.date() StopListener() q = Queue() shutdownQ = Queue() listener = Process( target = Server, args=(q, shutdownQ, HOST, PORT, startTime) ) listener.name = 'JChip Listener' listener.daemon = True listener.start()
def launch_server():
ip = gethostbyname(getfqdn())
port, with_threading = get_options()
sock = socket(AF_INET, SOCK_STREAM)
sock.bind(('', port))
print('Server on: ' + ip + ':' + str(port) + ' with ' + ('threads' if with_threading else 'processes'))
if with_threading:
from threading import Thread as Process
else:
from multiprocessing import Process
while True:
sock.listen(16)
client_sock, ip = sock.accept()
print('Connected from ' + ip[0])
process = Process(target=hash_text, args=(client_sock,))
process.start()
def StartListener( startTime=now(), HOST=None, PORT=None ): global q global shutdownQ global listener StopListener() if Model.race: HOST = (HOST or Model.race.chipReaderIpAddr) PORT = (PORT or Model.race.chipReaderPort) q = Queue() shutdownQ = Queue() listener = Process( target = Server, args=(q, shutdownQ, HOST, PORT, startTime) ) listener.name = 'Ultra Listener' listener.daemon = True listener.start()
def init_instruments(self):
for name, instr in self._instruments.items():
instr.configure_with_proxy(instr.proxy_obj)
self.digitizers = [
v for _, v in self._instruments.items()
if "Digitizer" in v.instrument_type
]
self.awgs = [
v for _, v in self._instruments.items()
if "AWG" in v.instrument_type
]
# Swap the master AWG so it is last in the list
try:
master_awg_idx = next(ct for ct, awg in enumerate(self.awgs)
if awg.master)
self.awgs[-1], self.awgs[master_awg_idx] = self.awgs[
master_awg_idx], self.awgs[-1]
except:
logger.warning("No AWG is specified as the master.")
for gen_proxy in self.generators:
gen_proxy.instr.output = True
# Start socket listening processes, store as keys in a dictionary with exit commands as values
self.dig_listeners = {}
ready = Value('i', 0)
self.dig_run = Event()
self.dig_exit = Event()
for chan, dig in self.chan_to_dig.items():
socket = dig.get_socket(chan)
oc = self.chan_to_oc[chan]
p = Process(target=dig.receive_data,
args=(chan, oc, self.dig_exit, ready, self.dig_run))
self.dig_listeners[p] = self.dig_exit
assert None not in self.dig_listeners.keys()
for listener in self.dig_listeners.keys():
listener.start()
while ready.value < len(self.chan_to_dig):
time.sleep(0.1)
if self.cw_mode:
for awg in self.awgs:
awg.run()
def __init__(self, policy, child_box, left, status_updater):
self.policy = policy
self.box = child_box
self.left = left
self.expected_size = 0
self.available_nodes = []
self.available_edges = []
self.status_updater = status_updater
self.in_queue = Queue()
self.out_queue = Queue()
self.proc = Process(target=self.__push,
args=(self.in_queue, self.out_queue))
self.proc.start()
self.last_out = ([], [])
self.last_in = (array([]), [], [])
self.retry = False
self.status_context = status_updater.get_context_id("666")
self.last_status_message_id = None
def import_dataset(self, request):
"""Performs a dataset import session with all of the workflow - data acquisition, extraction, reading,
preprocessing, and storing.
:param request: Django request
"""
self._validate_request(request)
parameters = self.django_request_to_import_parameters(request.POST)
parameters = self._preprocess_import(parameters, request.user, request.FILES)
process = Process(target=_import_dataset, args=(parameters, self._n_processes, self._process_batch_size)).start()
# process = None
# _import_dataset(parameters, n_processes=self._n_processes, process_batch_size=self._process_batch_size)
self._active_import_jobs[parameters['import_id']] = {
'process': process,
'parameters': parameters
}
def queue_joinif(this_queue, kill_signal=None, timeout=None):
assert (kill_signal is not None) or (
timeout is not None
), "queue_joinif requires either a kill_signal or timout argument."
t = Process(target=this_queue.join)
t.start()
start_time = time.time()
while True:
if kill_signal is not None:
if kill_signal.is_set():
return False
if timeout is not None:
if (time.time() - start_time > timeout):
return False
try:
t.join(1)
assert not t.is_alive()
return True
except AssertionError:
pass
return False
def main():
host = 'heidiz.ddns.net'
port = 38010 # 设置端口好
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((host, port))
recv_connection = s.recv(4096)
print(recv_connection.decode())
pre = Queue()
post = Queue()
pre_lock = Lock()
post_lock = Lock()
#"""
thread = dict()
thread['capture'] = Process(target=capture, args=(s, pre, pre_lock))
#thread['send'] = Process(target = send, args = (s, pre, pre_lock))
thread['recv'] = Process(target=recv, args=(s, post, post_lock))
"""
thread = [None] * 4
thread[0] = gevent.spawn(capture, s, pre, pre_lock)
thread[1] = gevent.spawn(send, s, pre, pre_lock)
thread[2] = gevent.spawn(recv, s, post, post_lock)
thread[3] = gevent.spawn(display, post, post_lock)
gevent.joinall(thread)
"""
#thread['display'] = threading.Thread(target = display, args = (post, post_lock))
start = [t[1].start() for t in thread.items()]
while True:
if post.empty():
time.sleep(1)
continue
post_lock.acquire()
encode = post.get()
post_lock.release()
frame = decoder(encode)
cv2.imshow('frame', frame)
if cv2.waitKey(1) & 0xFF == ord('q'):
break
join = [t[1].join() for t in thread.items()]
bcast("Couldn't play " + current)
#######################################################
# Play loaded track
pygame.mixer.music.play()
# Take user input for controlling player
while pygame.mixer.music.get_busy():
if console == False:
#font = pygame.font.Font(None, 36)
#out = font.render(text, 1, (10, 10, 10))
#textpos = out.get_rect()
##textpos.centerx = background.get_rect().centerx
screen.blit(background, (0, 0))
#pygame.display.flip()
control2()
else:
t = Process(None, control())
t.daemon = False
t.start()
#######################################################
if not current in played:
played.append(current)
i = i + 1
sleep(0.2)
songNum = songNum + 1
bcast("All songs have been played!")
log('All songs have been played')
shutdown()
#######################################################
except:
LogErr()
shutdown()
}
# Dict with the xcel data loaded as pandas Dataframes
pd_csv = {}
def create_pd_csv_pd(excel, pd_csv):
"""
Read the excel files as pandas DataFrame
"""
pd_csv[excel] = pd.read_pickle("src/" + excel)
processes = []
for val in os.listdir("src/"):
processes.append(Process(target=create_pd_csv_pd, args=(val, pd_csv,)))
for process in processes:
process.start()
for process in processes:
process.join()
# print pd_csv.keys()
def compute_data_sizes(pd_csv):
"""
Isolate the different Units available
Compute the sum with unit conversion
store output in MB
"""
def start(self):
if self.alive():
return
self.join()
self.process = Process(target=self.run)
self.process.start()
