def compute_async_process(self, input_queue):
raise DeprecationWarning()
assert (isinstance(input_queue, type(mQueue())))
output_queue = mQueue(5)
p = Process(target=self.compute_async_impl,
args=(input_queue, output_queue))
p.start()
return output_queue
def __init__(self, hostname):
self.cmds = {}
self.executed_cmds = mQueue()
self.default = None
self.output = ""
self.hostname = hostname
self.username = ""
self.password = ""
self.prompt = ""
self.protocol = "ssh"
self.log = None
def __init__(self):
super(UDPGeolocate, self).__init__()
# Constants
self.GEOLOCATION_API_URL = 'http://ip-api.com/json/'
self.WINDUMP_URL = 'https://www.winpcap.org/windump/install/bin/windump_3_9_5/WinDump.exe'
self.PORT_PROBING_ATTEMPTS = 5
self.CUR_DIR = os.path.dirname(os.path.realpath(__file__))
self.HOST_IP = self.get_host_IP()
# Initialise config attributes
self.conf = {'min_pack_len': 200, 'port': None, 'timeout': 1}
# Housekeeping and internal stuff
self.running = True
self.procs = []
self.q = mQueue()
# Ensure graceful exit
signal.signal(signal.SIGINT, signal.default_int_handler)
atexit.register(self.stop_app)
def _build_workers(self,
func,
n=1,
on_kill=None,
timeout=1,
max_queue_size=-1):
self.kill_workers(self.workers)
self.workers = []
self.queue = mQueue(max_queue_size)
for _ in range(n):
self.workers.append(
ProcessWorker(
self.queue,
func,
on_kill=on_kill,
on_start=on_start,
timeout=timeout,
))
self.workers[-1].start()
def __init__(self,
func,
n=1,
on_kill=None,
on_start=None,
timeout=1,
max_queue_size=-1):
self.queue = mQueue(max_queue_size)
self._timeout = timeout
self.workers = []
for _ in range(n):
self.workers.append(
ProcessWorker(
self.queue,
func,
on_kill=on_kill,
on_start=on_start,
timeout=timeout,
))
self.workers[-1].start()
def main():
# TODO: Argparse input
# p1 -> p2
q1 = mQueue()
# p2 -> p1
q2 = mQueue()
# p2 -> p3
q3 = mQueue()
# p3 -> p2
q4 = mQueue()
# p2 -> p4
q5 = mQueue()
# p4 -> p2
q6 = mQueue()
p1 = Process(target=process1_UDPsrv, args=(
q1,
q2,
))
p2 = Process(target=process2_CP,
args=(q1, q2, q3, q4, q5, q6, addECS, ECSMASK))
p3 = Process(target=process3_UDPsend,
args=(q3, q4, CDNSADDR, DNSTIMEOUT, DNSTRIES))
p4 = Process(target=process4_TCPsend,
args=(q5, q6, CDNSADDR, DNSTIMEOUT, DNSTRIES))
p1.start()
p2.start()
p3.start()
p4.start()
try:
p1.join()
p2.join()
p3.join()
p4.join()
except KeyboardInterrupt:
p1.terminate()
p2.terminate()
p3.terminate()
p4.terminate()
# TODO: Remember to flush IP tables
print("--Exiting public DNS server program (Ctrl-C)--\n")
sys.exit()
print("Exiting public DNS server program...\n")
def main():
"""Run the main program."""
parser = argparse.ArgumentParser(description="Custom DNS relay server")
parser.add_argument("-saddr",
"--dns_relay_server_address",
action=saddrAction,
help="Valid IPv4 address for the DNS relay server",
default="127.0.0.1")
parser.add_argument("-sport",
"--dns_relay_server_port",
help="Valid port for the DNS relay server",
action=sportAction,
type=int,
default=53)
parser.add_argument("-ecs",
"--forward_ecs",
action=ecsAction,
help="Forward ECS with DNS - yes/no",
default="yes")
parser.add_argument("-tcp",
"--use_tcp_security_step",
action=tcpAction,
help="Use TCP DNS security step - yes/no",
default="yes")
parser.add_argument("-cname",
"--use_cname_security_step",
action=cnameAction,
help="Use CNAME DNS security step - yes/no",
default="yes")
parser.add_argument("-randrgw",
"--randomize_destination_rgw",
action=randrgwAction,
help="Randomize destination RGW - yes/no",
default="yes")
parser.add_argument("-cnamestr",
"--rgw_cname_string_component",
help="Leftmost part of dest. RGW dns-cname-soa config",
default="cname")
parser.add_argument("-dnsto",
"--dns_timeout",
help="DNS request timeout towards RGW in seconds",
action=dnstoAction,
type=int,
default=3)
parser.add_argument("-dnstry",
"--dns_request_attempts",
help="Max. DNS request attempts towards RGW",
action=dnstryAction,
type=int,
default=3)
parser.add_argument('-rgws',
'--rgws_list',
nargs='+',
action=rgwsAction,
help='List of RGW address (str) and port (int) pairs')
args = parser.parse_args()
print("Starting the custom DNS relay server...\n")
print("Server IP address and port: {}, {}\n".format(
args.dns_relay_server_address, str(args.dns_relay_server_port)))
servaddr = (args.dns_relay_server_address, args.dns_relay_server_port)
if (args.forward_ecs == "yes"):
print("Client subnet forwarding with DNS ECS is ON.\n")
forwardECS = True
else:
print("Client subnet forwarding with DNS ECS is OFF.\n")
forwardECS = False
if (args.use_tcp_security_step == "yes"):
print("DNS TCP security step is ON.\n")
TCPstep = True
else:
print("DNS TCP security step is OFF.\n")
TCPstep = False
if (args.use_cname_security_step == "yes"):
print("DNS CNAME security step is ON.\n")
CNAMEstep = True
else:
print("DNS CNAME security step is OFF.\n")
CNAMEstep = False
if (args.randomize_destination_rgw == "yes"):
print("Destination RGW randomization is ON.\n")
randomizeRGW = True
else:
print("Destination RGW randomization is OFF.\n")
randomizeRGW = False
print("CNAME string component in use: ")
servicecname = args.rgw_cname_string_component
print(servicecname)
print("\n")
dnstimeout = args.dns_timeout
dnstries = args.dns_request_attempts
print("DNS request timeout in seconds: ")
print(str(dnstimeout))
print("\n")
print("Maximum additional DNS request attempts: ")
print(str(dnstries))
print("\n")
# Populating the destination RGW list
if args.rgws_list:
tempaddr = 0
tempport = 0
ipvalue = True
for x in args.rgws_list:
if ipvalue is True:
ipvalue = False
tempaddr = x
else:
ipvalue = True
tempport = x
rgwlist.append((tempaddr, int(tempport)))
print("Following destination RGWs were given:\n")
for x in rgwlist:
print(x)
print("\n")
else:
print("No destination RGWs given, using the default: \n")
print("addr 127.0.0.1 port 54\n ")
rgwlist.append(("127.0.0.1", 54))
print("Server serves forever; exit by pressing CTRL-C")
# Creating queues for communication between processes
# p1 -> p3 (From UDP _server_ to Data handler)
q1 = mQueue()
# p3 -> p1 (From Data handler to clientside UDP _sender_)
q2 = mQueue()
# p2 -> p3 (From TCP server to data handler)
q3 = mQueue()
# p3 -> p2 (From data handler to TCP server)
q4 = mQueue()
# p3 -> p4 (From data handler to rgwside UDP/TCP sender)
q5 = mQueue()
# p4 -> p3 (From rgwside UDP/TCP sender to data handler)
q6 = mQueue()
p1 = Process(target=process1_UDPsrv, args=(q1, q2, servaddr))
p2 = Process(target=process2_TCPsrv, args=(q3, q4, servaddr))
p3 = Process(target=process3_CP,
args=(q1, q2, q3, q4, q5, q6, servicecname, ECSMASK,
forwardECS))
p4 = Process(target=process4_fwdUDP,
args=(q5, q6, dnstimeout, dnstries, rgwlist, randomizeRGW))
p1.start()
p2.start()
p3.start()
p4.start()
try:
p1.join()
p2.join()
p3.join()
p4.join()
except KeyboardInterrupt:
p1.terminate()
p2.terminate()
p3.terminate()
p4.terminate()
# TODO: Remember to flush IP tables
print("--Exiting Custom DNS server program (Ctrl-C)--\n")
sys.exit()
print("Exiting Custom DNS server program...\n")
def start_workers(self):
""" start worker processes """
# setter config
setter_conf = {}
for attr in [
"analyze_move", "analyze_stale", "analyze_offsubnet",
"auto_clear_stale", "auto_clear_offsubnet",
"notify_stale_syslog", "notify_stale_email",
"notify_offsubnet_syslog", "notify_offsubnet_email",
"notify_move_syslog", "notify_move_email", "notify_email",
"notify_syslog", "notify_syslog_port", "max_ep_events",
"worker_hello", "worker_hello_multiplier",
"trust_subscription", "queue_interval",
"transitory_delete_time", "transitory_stale_time",
"transitory_xr_stale_time", "transitory_offsubnet_time",
"monitor_disable"
]:
if hasattr(self, attr): setter_conf[attr] = getattr(self, attr)
# manually stop any/all workers before starting workers
self.stop_workers()
self.all_workers = []
self.workers = {}
for wid in range(self.max_workers):
logger.debug("starting worker id(%s)" % wid)
w = {
"wid": wid,
"txQ": mQueue(),
"rxQ": mQueue(),
"prQ": mQueue(),
"last_hello": 0
}
p = mProcess(
target=start_ep_worker,
kwargs={
"wid": w["wid"],
"txQ":
w["rxQ"], # swap tx/rx queue from worker perspective
"rxQ": w["txQ"],
"prQ": w["prQ"],
"fabric": self.fabric,
"overlay_vnid": self.overlay_vnid
})
w["process"] = p
# enqueue any specific variables for this worker via job
w["txQ"].put(EPJob("setter", {}, data=setter_conf))
p.start()
self.workers[wid] = w
self.all_workers.append(w)
# setup/start watch worker
if not self.wworker_disable:
logger.debug("starting watcher worker")
self.wworker = {
"wid": "watcher",
"txQ": mQueue(),
"rxQ": mQueue(),
"prQ": mQueue(),
"last_hello": 0
}
p = mProcess(
target=start_ep_worker,
kwargs={
"wid": self.wworker["wid"],
"txQ": self.wworker["rxQ"], # swap tx/rx queue
"rxQ": self.wworker["txQ"],
"prQ": self.wworker["prQ"],
"fabric": self.fabric,
"overlay_vnid": self.overlay_vnid
})
self.wworker["process"] = p
self.wworker["txQ"].put(EPJob("setter", {}, data=setter_conf))
p.start()
self.all_workers.append(self.wworker)
else:
logger.debug("skipping watch worker")
if self.pworker_disable:
logger.debug("skipping priority worker")
return
# setup/start priority queue worker
logger.debug("starting priority worker")
bcastQ = []
bcastPrQ = []
for wid in self.workers:
bcastQ.append(self.workers[wid]["txQ"])
bcastPrQ.append(self.workers[wid]["prQ"])
if not self.wworker_disable:
bcastQ.append(self.wworker["txQ"])
bcastPrQ.append(self.wworker["prQ"])
self.pworker = {
"wid": "pri",
"txQ": mQueue(),
"rxQ": mQueue(),
"prQ": mQueue(),
"last_hello": 0
}
p = mProcess(
target=start_ep_priority_worker,
kwargs={
"wid": self.pworker["wid"],
"txQ": self.pworker["rxQ"], # swap tx/rq queue
"rxQ": self.pworker["txQ"],
"prQ": self.pworker["prQ"],
"bcastQ": bcastQ,
"bcastPrQ": bcastPrQ,
"fabric": self.fabric,
})
self.pworker["txQ"].put(EPJob("setter", {}, data=setter_conf))
self.pworker["txQ"].put(EPJob("init", {}))
self.pworker["process"] = p
p.start()
self.all_workers.append(self.pworker)
# wait for priority worker setter and init job to successfully complete
job = None
try:
job_setter = self.pworker["rxQ"].get(True, 3.0)
job = self.pworker["rxQ"].get(True, 3.0)
except Empty:
pass
if job is None or "success" not in job.key or not job.key["success"]:
err = "failed to initialize priority worker"
if job is not None and "error" in job.key: err = job.key["error"]
logger.warn(err)
self.stop_workers()
raise Exception(err)
else:
logger.debug("successfully initialized priority worker")
from Queue import Queue
from multiprocessing import Queue as mQueue
from messages import QUIT
# note: we always expect queue items to be tuples:
# [0] is a message type, [1] is an optional dict of arguments
gui_queue = Queue()
controller_queue = Queue()
upload_queue = Queue()
api_queue = Queue()
photo_queue = mQueue()
all_queues = ( gui_queue, photo_queue, controller_queue, upload_queue, api_queue )
def send_quit_all( except_for=None ):
if except_for:
for q in all_queues:
if not q in except_for:
q.put( QUIT )
else:
for q in all_queues:
q.put( QUIT )
def generate_masterless_pillars(ids_=None,
skip=None,
processes=None,
executable=None,
threads=None,
debug=False,
local=None,
timeout=None,
loglevel=None,
config_dir=None,
env=None,
*args,
**kwargs):
_s = __salt__
_o = __opts__
locs = _s['mc_locations.settings']()
if processes is None:
try:
grains = salt.loader.grains(_o)
processes = int(grains['num_cpus'])
except ValueError:
processes = 0
if processes < 2:
processes = 2
if not executable:
executable = os.path.join(locs['msr'], 'bin/salt-call')
if not config_dir:
config_dir = _o['config_dir']
if not loglevel:
loglevel = _o['log_level']
if local is None:
local = _o['file_client'] == 'local'
ids_ = get_hosts(ids_)
if isinstance(ids_, six.string_types):
ids_ = ids_.split(',')
if not threads:
threads = 0
if not skip:
skip = []
if not env:
env = {}
env = _s['mc_utils.dictupdate'](copy.deepcopy(dict(os.environ)), env)
input_queue = Queue.Queue()
if processes:
output_queue = mQueue()
else:
output_queue = Queue.Queue()
threads = int(threads)
for ix, id_ in enumerate(ids_):
if id_ in skip:
log.info('Skipping pillar generation for {0}'.format(id_))
continue
input_queue.put(id_)
# for debug
# if ix >= 2: break
workers = {}
results = {}
try:
size = input_queue.qsize()
i = 0
while not input_queue.empty():
i += 1
id_ = input_queue.get()
fargs = [id_, 'set_retcode=True']
pargs = {
'executable': executable,
'func': 'mc_remote_pillar.generate_masterless_pillar',
'args': fargs,
'out': 'json',
'timeout': timeout,
'no_display_ret': True,
'local': local,
'config_dir': config_dir,
'loglevel': loglevel
}
log.info('Getting pillar through saltcaller.call'
' for {0}'.format(id_))
log.debug('Arguments: {0}'.format(pargs))
pargs.update({'env': env, 'output_queue': output_queue})
log.info('ETA: {0}/{1}'.format(i, size))
if threads:
if len(workers) >= threads:
wait_pool(workers, output_queue, results)
workers[id_] = (threading.Thread(target=saltcaller.call,
kwargs=pargs))
workers[id_].start()
elif processes:
if len(workers) >= processes:
wait_processes_pool(workers, output_queue, results)
workers[id_] = (multiprocessing.Process(target=saltcaller.call,
kwargs=pargs))
workers[id_].start()
else:
saltcaller.call(**pargs)
while not output_queue.empty():
item = output_queue.get()
handle_result(results, item)
if threads:
wait_pool(workers, output_queue, results)
elif processes:
wait_processes_pool(workers, output_queue, results)
except (KeyboardInterrupt, Exception):
if threads:
for id_ in [a for a in workers]:
th = workers.pop(id_, None)
if th.is_alive() and th.ident:
th.join(0.01)
elif processes:
for id_ in [a for a in workers]:
th = workers.pop(id_, None)
if th.is_alive() and th.ident:
th.terminate()
raise
return results
baseq.put('brag')
baseq.put('facemask')
baseq.put('getoutofhere')
print('Queue-based queue: ', baseq)
print('Queue-based queue: ', baseq.get())
print('Queue-based queue: ', baseq.get())
print('Queue-based queue: ', baseq.get())
# mutliprocessing queue, shared job queues
# allow queuedj items to be processed in parallel by multiple concurrent
# workers. Process-based parallelization is popular in CPython due to
# the global interpreter lock(GIL) that prevent parallel executon on a
# single interpreter process.
# mutliprocessing.Queue work around the GIL limitations.
# This type of queue can store and transfer any pickle-able object across
# process boundaries.
from multiprocessing import Queue as mQueue
print('=' * 50)
mqueue = mQueue()
mqueue.put('moxiao')
mqueue.put('grab')
mqueue.put('bangkok')
print('multiprocessing Queue-based: ', mqueue)
mqueue.get()
print('multiprocessing Queue-based: ', mqueue)
print('multiprocessing Queue-based: ', mqueue.get())
print('multiprocessing Queue-based: ', mqueue.get())
# if mqueue is empty, get() would blocks the current process and wait
#print('multiprocessing Queue-based: ', mqueue.get())
def __init__(self, splited_keys, images, datasets, config_input, augmenter,
perception_interface):
# sample inputs
# splited_keys: _splited_keys_train[i_labels_per_division][i_steering_bins_perc][a list of keys]
# images: [i_sensor][i_file_number] = (lastidx, lastidx + x.shape[0], x)
# datasets: [i_target_name] = dim*batch matrix, where batch=#all_samples
# config_input: configInputs
# augmenter: config_input.augment
# save the inputs
self._splited_keys = splited_keys
self._images = images
self._targets = np.concatenate(
tuple(datasets),
axis=1) # Cat the datasets, The shape is totalnum*totaldim
self._config = config_input
self._augmenter = augmenter
self._batch_size = config_input.batch_size
# prepare all the placeholders: 3 sources: _queue_image_input, _queue_targets, _queue_inputs
self._queue_image_input = tf.placeholder(
tf.float32,
shape=[
config_input.batch_size, config_input.feature_input_size[0],
config_input.feature_input_size[1],
config_input.feature_input_size[2]
])
self._queue_shapes = [self._queue_image_input.shape]
# config.targets_names: ['wp1_angle', 'wp2_angle', 'Steer', 'Gas', 'Brake', 'Speed']
self._queue_targets = []
for i in range(len(self._config.targets_names)):
self._queue_targets.append(
tf.placeholder(tf.float32,
shape=[
config_input.batch_size,
self._config.targets_sizes[i]
]))
self._queue_shapes.append(self._queue_targets[-1].shape)
# self.inputs_names = ['Control', 'Speed']
self._queue_inputs = []
for i in range(len(self._config.inputs_names)):
self._queue_inputs.append(
tf.placeholder(tf.float32,
shape=[
config_input.batch_size,
self._config.inputs_sizes[i]
]))
self._queue_shapes.append(self._queue_inputs[-1].shape)
self._queue = tf.FIFOQueue(
capacity=config_input.queue_capacity,
dtypes=[tf.float32] + [tf.float32] *
(len(self._config.targets_names) + len(self._config.inputs_names)),
shapes=self._queue_shapes)
self._enqueue_op = self._queue.enqueue([self._queue_image_input] +
self._queue_targets +
self._queue_inputs)
self._dequeue_op = self._queue.dequeue()
#self.parallel_workers = Parallel(n_jobs=8, backend="threading")
self.input_queue = mQueue(5)
self.output_queue = mQueue(5)
self.perception_interface = perception_interface
if "mapping" in self._config.inputs_names:
version = "v1"
if hasattr(self._config, "mapping_version"):
version = self._config.mapping_version
self.mapping_helper = mapping_helper.mapping_helper(
output_height_pix=self._config.map_height, version=version
) # using the default values, 30 meters of width view, 50*75*1 output size
