class Counter(object):
"""
A process-safe counter providing atomic incrementAndGet() and value() functions
"""
def __init__(self, initval=0):
"""
Initialize this counter
Args:
initval (int): set the initialize value of the counter
"""
self.val = Value('i', initval)
def incrementAndGet(self):
"""
Atomically increment this counter, and return the new value stored.
Returns:
int: The updated value of this counter.
"""
with self.val.get_lock():
self.val.value += 1
return self.val.value
def value(self):
"""
Atomically get the current value of this counter.
Returns:
int: The current value of this counter.
"""
with self.val.get_lock():
return self.val.value
def execute_task(self, website: Website, busy: Value, post_id: str, comment_id: str):
busy.value = 1
if os.path.exists("data.json"):
os.remove("data.json")
print("Started crawling task")
process = CrawlerProcess(get_project_settings())
process.crawl("od_links", base_url=website.url)
process.start()
print("Done crawling")
self.db.import_json("data.json", website)
os.remove("data.json")
print("Imported in SQLite3")
if post_id:
# Reply to post
stats = self.db.get_website_stats(website.id)
comment = self.reddit_bot.get_comment({"": stats}, website.id)
print(comment)
if "total_size" in stats and stats["total_size"] > 10000000:
post = self.reddit_bot.reddit.submission(post_id)
self.reddit_bot.reply(post, comment)
pass
else:
self.reddit_bot.log_crawl(post_id)
elif comment_id:
# Reply to comment
stats = self.db.get_website_stats(website.id)
comment = self.reddit_bot.get_comment({"There you go!": stats}, website.id)
print(comment)
reddit_comment = self.reddit_bot.reddit.comment(comment_id)
self.reddit_bot.reply(reddit_comment, comment)
busy.value = 0
print("Done crawling task")
def main():
settings.setup()
try:
import miniupnpc
except:
safeprint("Dependency miniupnpc is not installed. Running in outbound only mode")
settings.config['outbound'] = True
safeprint("settings are:")
safeprint(settings.config)
queue = Queue()
live = Value('b',True)
ear = listener(settings.config['port'],settings.config['outbound'],queue,live,settings.config['server'])
ear.daemon = True
ear.start()
feedback = []
stamp = time()
while queue.empty():
if time() - 5 > stamp:
break #pragma: no cover
try:
feedback = queue.get(False)
except: #pragma: no cover
safeprint("No feedback received from listener")
ext_ip = "" #Does this affect peers?
ext_port = -1 #Does this affect peers?
if feedback != []:
settings.outbound = feedback[0]
if settings.outbound is not True:
ext_ip = feedback[1]
ext_port = feedback[2]
initializePeerConnections(settings.config['port'], ext_ip, ext_port)
live.value = False
def call(args, stdout=None, stderr=None, stdin=None, daemonize=False,
preexec_fn=None, shell=False, cwd=None, env=None):
"""
Run an external command in a separate process and detach it from the current process. Excepting
`stdout`, `stderr`, and `stdin` all file descriptors are closed after forking. If `daemonize`
is True then the parent process exits. All stdio is redirected to `os.devnull` unless
specified. The `preexec_fn`, `shell`, `cwd`, and `env` parameters are the same as their `Popen`
counterparts. Return the PID of the child process if not daemonized.
"""
stream = lambda s, m: s is None and os.open(os.devnull, m) or s
stdout = stream(stdout, os.O_WRONLY)
stderr = stream(stderr, os.O_WRONLY)
stdin = stream(stdin, os.O_RDONLY)
shared_pid = Value('i', 0)
pid = os.fork()
if pid > 0:
os.waitpid(pid, 0)
child_pid = shared_pid.value
del shared_pid
if daemonize:
sys.exit(0)
return child_pid
else:
os.setsid()
proc = subprocess.Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, close_fds=True,
preexec_fn=preexec_fn, shell=shell, cwd=cwd, env=env)
shared_pid.value = proc.pid
os._exit(0)
def setup_heartbeat(self, client_controller):
cond = multiprocessing.Condition()
s_init_finish = Value('i', 0)
do_sample = Value('i', 0)
do_sample_lock = Lock()
server_process = multiprocessing.Process(
target=self.server_heart_beat,
args=(cond, s_init_finish, do_sample, do_sample_lock))
server_process.daemon = False
server_process.start()
logger.info("Waiting for server init ...")
cond.acquire()
while (s_init_finish.value == 0):
cond.wait()
if s_init_finish.value == 5:
logger.error("Waiting for server init ... FAIL")
raise RuntimeError("server init failed.")
cond.release()
logger.info("Waiting for server init ... Done")
# let all clients start running the benchmark
client_controller.client_run(do_sample, do_sample_lock)
cond.acquire()
s_init_finish.value = 0
cond.release()
return server_process
def run_stop_test(self):
""" Subclass StoppableThread and stop method `run` """
class IncrementThread(StoppableThread):
""" Used to test _stop in `run` """
def __init__(self, *args, **kwargs):
self.x = args[0]
super(IncrementThread, self).__init__(*args[1:], **kwargs)
def run(self):
while not self._stop.is_set():
with self.x.get_lock():
self.x.value += 1
x = Value('i', 0)
st = IncrementThread(x)
st.start()
assert_equals(st.stopped, False)
assert_equals(st.is_alive(), True)
sleep(0.5)
st.stop()
assert_equals(st.stopped, True)
st.join()
assert_equals(st.is_alive(), False)
with x.get_lock():
assert_greater(x.value, 0)
def run_with_exception_except_test(self):
""" Subclass StoppableExceptionThread and raise exception in method `run_with_exception` """
class IncrementThread(StoppableExceptionThread):
""" Used to test _stop in `run` """
def __init__(self, *args, **kwargs):
self.x = args[0]
StoppableExceptionThread.__init__(self, *args[1:], **kwargs)
def run_with_exception(self):
while not self._stop.is_set():
with self.x.get_lock():
self.x.value += 1
if self.x.value > 5:
raise ValueError('x > 5')
x = Value('i', 0)
st = IncrementThread(x)
st.start()
sleep(1)
assert_equals(st.stopped, False)
with self.assertRaises(ValueError):
st.join()
assert_equals(st.is_alive(), False)
with x.get_lock():
assert_equals(x.value, 6)
def main():
#Begin Init
settings.setup()
from common.safeprint import safeprint
try:
import miniupnpc
except:
safeprint("Dependency miniupnpc is not installed. Running in outbound only mode")
settings.config['outbound'] = True
safeprint("settings are:")
safeprint(settings.config)
queue = Queue()
live = Value('b',True)
ear = listener(settings.config['port'],settings.config['outbound'],queue,live,settings.config['server'])
ear.daemon = True
ear.items = sync()
ear.start()
mouth = propagator(settings.config['port'] + 1, live)
mouth.daemon = True
mouth.items = ear.items
mouth.start()
feedback = []
stamp = time()
while queue.empty():
if time() - 5 > stamp:
break #pragma: no cover
try:
feedback = queue.get(False)
except: #pragma: no cover
safeprint("No feedback received from listener")
ext_ip = "" #Does this affect peers?
ext_port = -1 #Does this affect peers?
if feedback != []:
settings.outbound = feedback[0]
if settings.outbound is not True:
ext_ip = feedback[1]
ext_port = feedback[2]
initializePeerConnections(settings.config['port'], ext_ip, ext_port)
#End Init
#Begin main loop
if settings.config.get('seed'):
safeprint("Seed mode activated")
try:
while True and not settings.config.get('test'):
sleep(0.1)
except KeyboardInterrupt:
safeprint("Keyboard Interrupt")
elif settings.config.get('server'):
safeprint("Server mode activated")
else:
safeprint("Client mode activated")
#End main loop
#Begin shutdown
safeprint("Beginning exit process")
live.value = False
settings.saveSettings()
saveToFile()
bounty.saveToFile()
def _init_visualization_and_io(self, sim):
if self.config.output:
output_cls = io.format_name_to_cls[self.config.output_format]
else:
output_cls = io.LBOutput
if self.config.mode != 'visualization':
return lambda subdomain: output_cls(self.config, subdomain.id)
# basic_fields = sim.fields()
# XXX compute total storage requirements
for subdomain in self.subdomains:
size = reduce(operator.mul, subdomain.size)
vis_lock = mp.Lock()
vis_buffer = Array(ctypes.c_float, size, lock=vis_lock)
vis_geo_buffer = Array(ctypes.c_uint8, size, lock=vis_lock)
subdomain.set_vis_buffers(vis_buffer, vis_geo_buffer)
vis_lock = mp.Lock()
vis_config = Value(io.VisConfig, lock=vis_lock)
vis_config.iteration = -1
vis_config.field_name = ''
vis_config.all_blocks = False
# Start the visualizatione engine.
vis_class = None
for engine in util.get_visualization_engines():
if engine.name == self.config.vis_engine:
vis_class = engine
break
if vis_class is None:
self.config.logger.warning('Requested visualization engine not '
'available.')
try:
vis_class = util.get_visualization_engines().next()
except StopIteration:
self.config.logger.warning(
'No visualization backends available. Falling back to '
'batch mode.')
self.config.mode = 'batch'
return lambda subdomain: output_cls(self.config, subdomain.id)
# Event to signal that the visualization process should be terminated.
self._vis_quit_event = Event()
self._vis_process = Process(
target=lambda: vis_class(
self.config, self.subdomains, self._vis_quit_event,
self._quit_event, vis_config).run(),
name='VisEngine')
self._vis_process.start()
return lambda subdomain: io.VisualizationWrapper(
self.config, subdomain, vis_config, output_cls)
def main():
running = Value(c_int, 1)
readQueue = Queue()
reader = Process(target = Reader("/dev/ttyUSB0", 9600), args = (running, readQueue))
worker = Process(target = Worker(), args = (running, readQueue))
reader.start()
worker.start()
time.sleep(5)
running.value = 0
reader.join()
worker.join()
def __init__(self, nomenclature="", width=0., height=0.):
Device.__init__(self, nomenclature, width, height)
self.xpos_left = -0.5 * width
self.xpos_right = 0.5 * width
self.ypos_bottom = -0.5 * height
self.ypos_top = 0.5 * height
self.count_left = Value('i', 0)
self.count_right = Value('i', 0)
self.count_bottom = Value('i', 0)
self.count_top = Value('i', 0)
def _init_visualization_and_io(self, sim):
if self.config.output:
output_cls = io.format_name_to_cls[self.config.output_format]
else:
# Dummy output class. Does not actually save data, but does provide
# utility functions common to all output classes.
output_cls = io.LBOutput
if self.config.mode != 'visualization':
return lambda subdomain: output_cls(self.config, subdomain.id)
# XXX compute total storage requirements
self._vis_geo_queues = []
for subdomain in self.subdomain_specs:
self._vis_geo_queues.append(subdomain.init_visualization())
vis_lock = mp.Lock()
vis_config = Value(io.VisConfig, lock=vis_lock)
vis_config.iteration = -1
vis_config.field_name = ''
vis_config.all_subdomains = False
# Start the visualizatione engine.
vis_class = None
for engine in util.get_visualization_engines():
if engine.name == self.config.vis_engine:
vis_class = engine
break
if vis_class is None:
self.config.logger.warning('Requested visualization engine not '
'available.')
try:
vis_class = util.get_visualization_engines().next()
except StopIteration:
self.config.logger.warning(
'No visualization backends available. Falling back to '
'batch mode.')
self.config.mode = 'batch'
return lambda subdomain: output_cls(self.config, subdomain.id)
# Event to signal that the visualization process should be terminated.
self._vis_quit_event = Event()
self._vis_process = Process(
target=lambda: vis_class(
self.config, self.subdomain_specs, self._vis_quit_event,
self._quit_event, vis_config, self._vis_geo_queues).run(),
name='VisEngine')
self._vis_process.start()
return lambda subdomain: io.VisualizationWrapper(
self.config, subdomain, vis_config, output_cls)
class Control(object):
"""Shared (long) value for passing control information between main and
worker threads.
Args:
initial_value: Initial value of the shared control variable.
"""
def __init__(self, initial_value=CONTROL_ACTIVE):
self.control = Value('l', initial_value)
def check_value(self, value, lock=False):
"""Check that the current control value == `value`.
Args:
value: The value to check.
lock: Whether to lock the shared variable before checking.
Returns:
True if the values are equal.
"""
return self.get_value(lock=lock) == value
def check_value_positive(self, lock=False):
"""Check that the current control value is positive.
Args:
lock: Whether to lock the shared variable before checking.
"""
return self.get_value(lock=lock) > 0
def get_value(self, lock=True):
"""Returns the current control value.
Args:
lock: Whether to lock the shared variable before checking.
"""
if lock:
with self.control.get_lock():
return self.control.value
else:
return self.control.value
def set_value(self, value):
"""Set the control value. The shared variable is always locked.
Args:
value: The value to set.
"""
with self.control.get_lock():
self.control.value = value
def camstream():
print "CAMera STREAMer (OpenCV " + cv2.__version__ + ")"
print "main(): OS: {}".format(os.name)
# * Start CameraStreamer process
print "main(): Starting CameraStreamer process..."
if os.name == 'nt': # [Windows]
# ** Create shared objects (NOTE only necessary on Windows since it uses a different multiprocessing implementation)
print "main(): [Windows] Creating shared objects..."
# *** Stay alive flag
stayAliveObj = Value(c_bool, True)
# *** Frame counter
frameCountObj = Value('i', 0)
# *** Image array
image = np.zeros((camera_frame_height, camera_frame_width, camera_frame_depth), dtype=np.uint8)
imageShape = image.shape
imageSize = image.size
image.shape = imageSize # flatten numpy array
imageObj = Array(c_ubyte, image) # create a synchronized shared array
# *** Image shape
imageShapeObj = Array('i', imageShape)
cameraStreamerProcess = CameraStreamer(stayAliveObj, frameCountObj, imageObj, imageShapeObj)
else: # [POSIX]
cameraStreamerProcess = CameraStreamer()
# ** Grab generated shared objects to share with other child processes
print "main(): [POSIX] Getting shared objects from CameraStreamer..."
stayAliveObj = cameraStreamerProcess.stayAliveObj
frameCountObj = cameraStreamerProcess.frameCountObj
imageObj = cameraStreamerProcess.imageObj
imageShapeObj = cameraStreamerProcess.imageShapeObj
cameraStreamerProcess.start()
# * Start StreamViewer process
print "main(): Starting StreamViewer process..."
streamViewerProcess = StreamViewer(stayAliveObj, frameCountObj, imageObj, imageShapeObj)
streamViewerProcess.start()
# * Wait for child processes to finish
print "main(): Waiting for child processes to finish..."
try:
streamViewerProcess.join()
cameraStreamerProcess.join()
except KeyboardInterrupt:
stayAliveObj.value = False
streamViewerProcess.join()
cameraStreamerProcess.join()
print "main(): Done."
class Counter: def __init__(self): self.value = Value(ctypes.c_int) def __enter__(self): with self.value.get_lock(): self.value.value += 1 def __exit__(self, exc_type, exc_val, exc_tb): with self.value.get_lock(): self.value.value -= 1 def __repr__(self): return str(self.value.value)
def start(self, reload_from=None):
"""Start this server process.
:param int reload_from: Optional, the PID of a running game server
process that this process should reload from
:returns None:
"""
assert not self._process, "server instance already started"
pid = Value("i")
self._process = Process(target=self._start,
args=(pid, socket_queue),
kwargs={"reload_from": reload_from})
self._process.start()
pid.value = self._process.pid
def send_mldquery_regularly(self):
self.logger.debug("")
requraly_query_type = self.config[const.REGURALY_QUERY_TYPE]
reguraly_query_interval = self.config[const.REGURALY_QUERY_INTERVAL]
mc_query_interval = self.config[const.MC_QUERY_INTERVAL]
# 初回送信前に定期送信クエリの送信間隔の1/4秒待つ
time.sleep(reguraly_query_interval / 4)
# General Query
if requraly_query_type == self.GENERAL_QUERY:
self.logger.debug("create general query")
mc_info = {const.MC_TAG_MC_ADDR: const.DELIMIT_DOUBLE_COLON,
const.MC_TAG_SERV_IP: None}
while self.SEND_LOOP:
self.send_mldquery([mc_info])
# タイムアウトチェック
self.check_user_timeout()
time.sleep(reguraly_query_interval - self.QUERY_QRV)
# Specific Query
elif requraly_query_type == self.SPECIFIC_QUERY:
self.logger.debug("create specific query")
next_interval = Value(ctypes.c_bool, False)
while self.SEND_LOOP:
query_proc = Process(
target=self.wait_query_interval,
args=(next_interval, reguraly_query_interval))
query_proc.daemon = True
query_proc.start()
self.logger.debug(
"next_interval : %s", str(next_interval.value))
self.send_mldquery(
self.mc_info_list, mc_query_interval, next_interval)
# タイムアウトチェック
self.check_user_timeout()
# 定期送信クエリの送信間隔が過ぎていない場合は待ち
if not next_interval.value:
self.logger.debug(
"waiting query interval(%d sec)...",
reguraly_query_interval)
query_proc.join()
next_interval.value = False
query_proc.terminate()
def __init__(self, initval=0):
"""
Initialize this counter
Args:
initval (int): set the initialize value of the counter
"""
self.val = Value('i', initval)
def reset(self):
self._pipes = {}
self._pipe_processes = []
self._first_pipe = None
self._last_pipe = None
self._func_read_stream = (lambda: range(0))
self._cleanups = []
self._already_cleanup = Value(ctypes.c_bool, False)
self._running_status = Value(ctypes.c_int, Pipeline.RUNNING_STATUS_STANDBY)
self._interrupted_by_exception = False
self._thread_watching_running_status = None
self._thread_watching_remaining_processes = None
self._stream_reader_process = None
def __init__(self, name, timeout=0, onstart=None, ondone=None, params=None, stdout=sys.stdout, stderr=sys.stderr):
"""Initialize task, which is a group of jobs to be executed
name - task name
timeout - execution timeout. Default: 0, means infinity
onstart - callback which is executed on the task starting (before the execution
started) in the CONTEXT OF THE CALLER (main process) with the single argument,
the task. Default: None
ATTENTION: must be lightweight
ondone - callback which is executed on successful completion of the task in the
CONTEXT OF THE CALLER (main process) with the single argument, the task. Default: None
ATTENTION: must be lightweight
params - additional parameters to be used in callbacks
stdout - None or file name or PIPE for the buffered output to be APPENDED
stderr - None or file name or PIPE or STDOUT for the unbuffered error output to be APPENDED
ATTENTION: PIPE is a buffer in RAM, so do not use it if the output data is huge or unlimited
tstart - start time is filled automatically on the execution start (before onstart). Default: None
tstop - termination / completion time after ondone
"""
assert isinstance(name, str) and timeout >= 0, "Parameters validaiton failed"
self.name = name
self.timeout = timeout
self.params = params
self.onstart = types.MethodType(onstart, self) if onstart else None
self.ondone = types.MethodType(ondone, self) if ondone else None
self.stdout = stdout
self.stderr = stderr
self.tstart = None
self.tstop = None # SyncValue() # Termination / completion time after ondone
# Private attributes
self._jobsnum = Value(ctypes.c_uint)
# Graceful completion of all tasks or at least one of the tasks was terminated
self._graceful = Value(ctypes.c_bool)
self._graceful.value = True
def __init__(self):
parser = argparse.ArgumentParser()
parser.add_argument("thread", help="URL of thread to scrape")
parser.add_argument("--directory", "-d", help="Specify dir to save to (Default: ~/4chan)")
parser.add_argument("--name", "-n", help="Specify name of dir to download to (Default: Topic/OP Post number)")
parser.add_argument("--workers", type=int, help="Number of threads to run (Default: 10)")
parser.add_argument("--version", "-v", action="version", version=VERSION)
self.args = parser.parse_args()
save_path = self.args.directory or os.path.join(
os.path.expanduser('~'), "4chan")
self.header = {
'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11\
(KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11',
'Accept': 'text/html,application/xhtml+xml,\
application/xml;q=0.9,*/*;q=0.8'}
self.thread_url = self.args.thread
self.board = self.thread_url.split('/')[3]
self.thread_name = ""
self.downloads = []
self.filename = []
self.save_path = save_path
self.counter = Value('i', 0)
self.total_count = Value('i', 0)
self.workers = self.args.workers
self.down_dir = ""
class Transformator(Device):
def __init__(self, nomenclature="", width=0., height=0.):
Device.__init__(self, nomenclature, width, height)
self.count = Value('i', 0)
def __repr__(self):
r = str(self) + "("
r += "width=" + str(self.width) + "m, "
r += "height=" + str(self.height) + "m, "
r += "length=" + str(self.length) + "m, "
r += "count=" + str(self.count.value) + ")"
return r
def transport(self, particle):
if not self.is_particle_lost(particle):
with self.count.get_lock():
self.count.value += 1
if self.next:
return self.next.transport(particle)
def reset(self):
self.count.value = 0
if self.next:
self.next.reset()
def __init__(self, world_class, opt, agents):
self.inner_world = world_class(opt, agents)
self.queued_items = Semaphore(0) # counts num exs to be processed
self.epochDone = Condition() # notifies when exs are finished
self.terminate = Value('b', False) # tells threads when to shut down
self.cnt = Value('i', 0) # number of exs that remain to be processed
self.threads = []
for i in range(opt['numthreads']):
self.threads.append(HogwildProcess(i, world_class, opt,
agents, self.queued_items,
self.epochDone, self.terminate,
self.cnt))
for t in self.threads:
t.start()
def main():
guardian = Heimdall()
# zoeh = Zoehmacarena()
# guardian.add_bot(zoeh)
# guardian.main_loop()
cont = True
ispaused = False
# bobby = Bob()
state = Value("i", 0)
l = Lock()
# proc = Process(target=bobby.main, args=(l, state,))
print("starting")
# proc.start()
while cont:
if ispaused:
command = raw_input("Heimdall[paused]: ")
else:
command = raw_input("Heimdall[running]: ")
if command == "quit":
cont = False
state.value = -1
elif command == "pause":
l.acquire()
print("paused")
ispaused = True
elif command == "start":
print("unpausing")
l.release()
ispaused = False
else:
print("unknown command")
# proc.join()
print("finished")
return
def main():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('-f', dest='logfile', required=True)
arg_parser.add_argument('-t', dest='sleep', type=int, default=1)
args = arg_parser.parse_args()
count = Value('i', 0)
p = Process(target=log_counter, args=(args.logfile, count,))
p.start()
while True:
time.sleep(args.sleep)
print('{0} events/{1}s'.format(count.value, args.sleep))
count.value = 0
p.join()
class Counter(object):
def __init__(self):
self.val = Value('i', 0)
def increment(self, n=1):
with self.val.get_lock():
self.val.value += n
@property
def value(self):
return self.val.value
def __init__(self, config):
configFile = config
try:
self.readConfig(configFile)
except Exception as e:
print ('\n%s' % e)
print('The Configuration is incomplete, exiting')
exit(2)
self.jobs = Value('i', 0)
self.logger.info('Slave %s initialized' % self.host)
class Control(object):
def __init__(self, initial_value):
self.control = Value('l', initial_value)
def check_value(self, value, lock=False):
return self.get_value(lock=lock) == value
def check_value_positive(self, lock=False):
return self.get_value(lock=lock) > 0
def get_value(self, lock=True):
if lock:
with self.control.get_lock():
return self.control.value
else:
return self.control.value
def set_value(self, value):
with self.control.get_lock():
self.control.value = value
def __init__(self, alias=None):
self.logger = _get_logger(__name__)
self._alias = alias
self._pipe_builders = []
self._pipes = {}
self._pipe_processes = []
self._first_pipe = None
self._last_pipe = None
self._func_read_stream = (lambda: range(0))
self._cleanups = []
self._already_cleanup = Value(ctypes.c_bool, False)
self._running_status = Value(ctypes.c_int, Pipeline.RUNNING_STATUS_STANDBY)
self._interrupted_by_exception = False
self._thread_watching_running_status = None
self._thread_watching_remaining_processes = None
self._stream_reader_process = None
def hanging_get(my_id, messages, initial_peer_id):
remote_sender = None
remote_peer_id = Value("i", initial_peer_id)
remote_sender = Process(target=to_remote_server, args=(5550,my_id,remote_peer_id)).start()
while True:
r = requests.get('http://localhost:8888/wait?peer_id=' + str(my_id))
if r.status_code == 200:
if int(r.headers['pragma']) == my_id:
connected = r.text.split("\n")
for l in connected:
info = l.strip().split(",")
if len(info) == 3 and info[0] == "receiver" and info[2] == '1':
remote_peer_id.value = int(info[1])
if len(info) == 3 and info[0] == "receiver" and info[2] == '0':
remote_peer_id.value = -1
else:
messages.put(Message(int(r.headers['pragma']), r.text))
from flask import Flask, request, redirect, url_for, flash, jsonify
import numpy as np
import pickle as p
import pandas as pd
import json
import base64
from multiprocessing import Value
counter = Value('i', 0)
model_counter = 0
app = Flask(__name__)
@app.route('/', methods=['GET'])
def show_model_counter():
with counter.get_lock():
counter.value += 1
out = counter.value
return jsonify(count=out)
@app.route('/model_predict/', methods=['POST'])
def makecalc():
j_data = request.get_json()
j_data = j_data['data']
print('j_data: ', j_data)
prediction = np.array2string(model.predict(j_data))
predict_proba = np.array2string(model.predict_proba(j_data))
print(model.predict(j_data))
print(prediction)
class imagequeue:
"""
This class keeps a queue of images which may be worked on in threads.
:param SAXS.calibration Cal: The SAXS Calibration to use for the processing
:param optparser options: The object with the comandline options of the saxsdog
:param list args: List of command line options
"""
def __init__(self, Cals, options, directory, conf):
self.pool = []
self.cals = Cals
self.conf = conf
self.options = options
self.picturequeue = Queue()
self.histqueue = Queue(maxsize=10000)
self.plotdataqueue = Queue(maxsize=1)
self.directory = directory
self.allp = Value('i', 0)
self.stopflag = Value('i', 0)
self.dirwalker = None
self.observer = None
self.filelist_output = np.array(["filename", 0, 0, 0])
if not options.plotwindow:
plt.switch_backend("Agg")
self.fig = plt.figure()
if options.plotwindow:
plt.ion()
def getlastdata(self):
print "getdatata" + str(self.lastfile)
return self.lastfile, self.lastdata
def fillqueuewithexistingfiles(self):
"""
Fill the queue with the list of images that is already there.
"""
if self.options.walkdirinthreads:
self.dirwalker = Thread(target=filler,
args=(self.picturequeue, self.directory))
self.dirwalker.start()
else:
filler(self.picturequeue, self.directory)
def procimage(self, picture, threadid):
filelist = {}
max = 60
data = []
integparams = {}
'''Setting output directory paths'''
if self.options.outdir != "":
basename = self.options.outdir + os.sep + ('_'.join(
picture.replace('./', '').split(os.sep))[:-3]).replace(
'/', "_")
basename = basename.replace(':', '').replace('.', '')
else:
reldir = os.path.join(os.path.dirname(picture),
self.options.relpath)
if not os.path.isdir(reldir):
try:
os.mkdir(reldir)
except:
print "Problem creating WORK directory!!!"
return
basename = os.path.join(reldir, os.path.basename(picture)[:-4])
'''Check if image exists or we are in Gisaxs mode'''
skipfile = False
if self.options["OverwriteFiles"] == False:
for calnum, cal in enumerate(self.cals):
if len(list(enumerate(self.cals))) == 1 or calnum == 0:
filename = basename
else:
filename = basename + "_c" + cal.kind[0] + str(calnum)
chifilename = filename + ".chi"
if os.path.isfile(chifilename):
filelist[cal.kind + str(calnum)] = chifilename
skipfile = True
if self.options["livefilelist"] is not "xxx":
with open(self.options["livefilelist"],
'a') as f_handle:
file_path = os.path.normpath(chifilename)
file_path = str.split(
str(file_path),
str(
os.path.split(
self.options["watchdir"])[0]))[1]
output = file_path + ", " + str(0) + ", " + str(
0) + ", " + str(0) + "\n"
f_handle.write(output)
f_handle.close()
if self.options.GISAXSmode == True and calnum == 0: #pass on GISAXSmode information to calibration.integratechi
skipfile = True
'''Check if image can be opened'''
if skipfile == False:
if not self.options.silent:
print "[", threadid, "] open: ", picture
for i in range(max):
try:
image = misc.imread(picture)
except KeyboardInterrupt:
return
except IOError as e:
try:
print "cannot open ", picture, ", lets wait.", max - i, " s"
print e.message, sys.exc_info()[0]
time.sleep(1)
continue
except KeyboardInterrupt:
return
except:
print "############"
print sys.exc_info()
continue
if image.shape == tuple(
self.cals[0].config["Geometry"]["Imagesize"]):
break
print "cannot open ", picture, ", lets wait.", max - i, " s"
time.sleep(1)
else:
print "image ", picture, " has wrong format"
return
if skipfile == False:
imgMetaData = datamerge.readtiff(picture)
if "date" in imgMetaData:
imgTime = imgMetaData["date"]
else:
imgTime = ""
else:
imgTime = ""
if skipfile == False:
for calnum, cal in enumerate(self.cals):
if len(list(enumerate(self.cals))) == 1 or calnum == 0:
filename = basename
else:
filename = basename + "_c" + cal.kind[0] + str(calnum)
chifilename = filename + ".chi"
filelist[cal.kind + str(calnum)] = chifilename
if not self.options.resume or not os.path.isfile(chifilename):
result = cal.integratechi(image, chifilename, picture)
#print(chifilename, " has been integrated!")
result["Image"] = picture
if "Integparam" in result:
integparams[cal.kind[0] +
str(calnum)] = result["Integparam"]
data.append(result)
if self.options["livefilelist"] is not "xxx":
#print result["Integparam"]["I0"]
with open(self.options["livefilelist"],
'a') as f_handle:
file_path = os.path.normpath(chifilename)
file_path = str.split(
str(file_path),
str(
os.path.split(
self.options["watchdir"])[0]))[1]
if "Integparam" in result:
output = file_path +", "+str(result["Integparam"]["I0"])+ \
", "+str(result["Integparam"]["I1"])+", "+str(result["Integparam"]["I2"])+"\n"
else:
output = file_path +", "+str(0)+ \
", "+str(0)+", "+str(0)+"\n"
f_handle.write(output)
f_handle.close()
#np.savetxt(f_handle,self.filelist_output, delimiter=',', fmt="%s ")
if threadid == 0 and self.options.plotwindow:
# this is a hack it really schould be a proper GUI
cal.plot(image, fig=self.fig)
plt.draw()
if self.options.writesvg:
if not self.options.resume or not os.path.isfile(filename +
'.svg'):
cal.plot(image, filename + ".svg", fig=self.fig)
if self.options.writepng:
if not self.options.resume or not os.path.isfile(filename +
'.svg'):
misc.imsave(filename + ".png", image)
if self.options.silent:
if np.mod(self.allp.value, 100) == 0:
print "[", threadid, "] ", self.allp.value
else:
print "[", threadid, "] write: ", filename + ".chi"
with self.allp.get_lock():
self.allp.value += 1
filelist["JSON"] = basename + ".json"
try:
self.histqueue.put(
{
"Time": float(time.time()),
"ImgTime": imgTime,
"FileList": filelist,
"BaseName": basename,
"IntegralParameters": integparams
},
block=False)
except Full:
print "Full"
return basename, data
def clearqueue(self):
while self.histqueue.empty() == False:
self.histqueue.get()
print "History Queue cleared"
def start(self):
"""
Start threads and directory observer.
"""
#start threads
for threadid in range(1, self.options.threads):
print "start proc [", threadid, "]"
worker = Process(target=funcworker, args=(self, threadid))
worker.daemon = True
self.pool.append(worker)
worker.start()
#self.processimage(picture,options)
self.starttime = time.time()
if self.options.watch:
eventhandler = addtoqueue(self.picturequeue)
self.observer = Observer()
self.observer.schedule(eventhandler, self.args[0], recursive=True)
self.observer.start()
#We let the master process do some work because its useful for matplotlib.
if not self.options.nowalk:
self.fillqueuewithexistingfiles()
if self.options.servermode:
from Leash import addauthentication
try:
while (self.options.servermode or (not self.picturequeue.empty())
or (self.dirwalker and self.dirwalker.is_alive())
or self.options.watch):
try:
picture = self.picturequeue.get(timeout=1)
except Empty:
continue
#in Case something goes wrong
try:
lastfile, data = self.procimage(picture, 0)
except:
continue
if self.options.servermode:
request = {
"command": "putplotdata",
"argument": {
"data": {
"result": "plot",
"data": {
"filename": lastfile,
"graphs": data,
"stat": {}
}
}
}
}
self.plotdataqueue.put(request)
if np.mod(self.allp.value, 500) == 0:
self.timreport()
except KeyboardInterrupt:
pass
self.stop()
self.timreport()
return self.allp.value, time.time() - self.starttime
def stop(self):
print "\n\nWaiting for the processes to terminate."
if self.observer:
self.observer.stop()
self.observer.observer.join(1)
self.stopflag.value = 1
for worker in self.pool:
print "join worker"
worker.join(1)
if self.dirwalker:
self.dirwalker.join(1)
print "empty pic queue"
while True:
try:
self.picturequeue.get(False)
except Empty:
break
print "empty hist queue"
while True:
try:
self.histqueue.get(False)
except Empty:
break
print "empty plot queue"
while True:
try:
self.plotdataqueue.get(False)
except Empty:
break
if os.sys.platform != "win32":
try:
self.histqueue.close()
self.plotdataqueue.close()
except Exception as e:
print e
def timreport(self):
tottime = time.time() - self.starttime
count = self.allp.value
#print count
if count == 0:
print "We didn't do any pictures "
else:
print "\n\nelapsed time: ", tottime
print "\nProcessed: ", count, " pic"
print " time per pic: ", tottime / count, "[s]"
print " pic per second: ", count / tottime, "[/s]"
time.sleep(1)
try:
# Send data
nums = numbers[:]
sock.sendall(bytes(str(nums), encoding='utf8'))
data = sock.recv(4096)
log.info("Received: %s" % str(data))
except socket.error as e:
log.error("Socket error: %s" % str(e))
except Exception as e:
log.error("Other exception: %s" % str(e))
finally:
log.info("Closing connection to the server")
sock.close()
if __name__ == '__main__':
port = 2000
numbers = []
howmanynumbers = click.prompt(
'Enter number of integers you want to process', type=int)
for _ in range(howmanynumbers):
numbers.append(click.prompt('Please enter a valid integer', type=int))
num = Value('i', port)
arr = Array('i', numbers)
p = Process(target=echo_client, args=(num, arr))
p.start()
p.join()
import multiprocessing as mp
from multiprocessing import Value
import array
import os
import time
import memory_profiler
init = " " * int(1e10)
arr = array.array('c', init)
def f(a, v):
print "running", os.getpid(), len(a)
print "in process", memory_profiler.memory_usage()
#a[0] = 'x'
print "did arr get changed in process?", a[:5]
v.value = len(a)
time.sleep(5)
value = Value('L')
print value
print "before making process", memory_profiler.memory_usage()
p = mp.Process(target=f, args=(arr, value))
print "after making process", memory_profiler.memory_usage()
p.start()
print "after making process 2", memory_profiler.memory_usage()
p.join() # a sleep would work but this is neater
print "Value is:", value.value
print "did arr get changed?", arr[:5]
"""
value.py 开辟单一共享内存空间
注意: 共享内存只能有一个值
"""
from multiprocessing import Process, Value
import time
import random
# 创建共享内存
money = Value('i', 5000)
# 操作共享内存
def man():
for i in range(30):
time.sleep(0.2)
money.value += random.randint(1, 1000)
def girl():
for i in range(30):
time.sleep(0.15)
money.value -= random.randint(100, 800)
p1 = Process(target=man)
p2 = Process(target=girl)
p1.start()
p2.start()
p1.join()
p2.join()
def main():
"""Gets input hash and salt from command line,
spawns worker threads, initializes queue,
calls worker function"""
# Check user input
if len(sys.argv) != 2:
sys.exit("Usage: python crack.py hash")
# Extract hash and salt from user input
user_hash = sys.argv[1]
salt = user_hash[:2]
# Input lists
inputs = [HASH_INPUTS_1, HASH_INPUTS_2, HASH_INPUTS_3, HASH_INPUTS_4]
# Shared memory map found to share state between processes
found = Value('i', 0)
# DOES NOT WORK BECAUSE SHARED MEM CANNOT BE PASSED TO pool.apply_async()
# !!!!!!!!
# The function to crack the DES hashed password.
# It is the callback function to main's calls to set_tuple_permutations
# via pool.apply_async. When the pool of child processes return results,
# the main process will process them via the callback function DES_crack.
def print_n_return(result):
"""Callback function for worker_crack
that prints the password found by
one of the callers. The other callbacks
do not print their None values.
"""
# Only let the caller print, who has found the password
if result:
print(f"password: {result}")
return
# Spawn pool of child processes calling the worker
# function: worker_crack in parallel and
# applying async, with callback function: print_n_terminate
# to the result returned from the workers.
start_time = time.time()
pool = Pool(NUM_PHYS_CORES)
for aninput in inputs:
pool.apply_async(worker_crack,
args=(aninput, user_hash, salt, found),
callback=print_n_return)
pool.close()
pool.join()
# DOES NOT WORK BECAUSE SHARED MEM CANNOT BE PASSED TO pool.apply_async()
# !!!!!!!!
duration = time.time() - start_time
print(f"Cracking password took {duration:.2f}")
logging.debug(f"Found: {found.value}")
# Success
sys.exit(0)
elif v[1] == 0: #v[1] ==fq
frame1 = v[0]
#print(result)
#process v
except queue.Empty:
continue
except Exception as e:
raise e
cv2.destroyAllWindows()
if __name__ == "__main__":
# Retrieve singleton reference to system object
sharedK = Value('i', 2)
sharedBaseExposure = Value('d', 4)
system = PySpin.System.GetInstance()
# Retrieve list of cameras from the system
cam_list = system.GetCameras()
num_cameras = cam_list.GetSize()
print("Number of cameras detected:", num_cameras)
# Finish if there are no cameras
if num_cameras == 0:
# Clear camera list before releasing system
cam_list.Clear()
# Release system
def test_cbsc002_callbacks_generating_children(dash_duo):
"""Modify the DOM tree by adding new components in the callbacks."""
# some components don't exist in the initial render
app = dash.Dash(__name__, suppress_callback_exceptions=True)
app.layout = html.Div(
[dcc.Input(id="input", value="initial value"), html.Div(id="output")]
)
@app.callback(Output("output", "children"), [Input("input", "value")])
def pad_output(input):
return html.Div(
[
dcc.Input(id="sub-input-1", value="sub input initial value"),
html.Div(id="sub-output-1"),
]
)
call_count = Value("i", 0)
@app.callback(
Output("sub-output-1", "children"), [Input("sub-input-1", "value")]
)
def update_input(value):
call_count.value = call_count.value + 1
return value
dash_duo.start_server(app)
dash_duo.wait_for_text_to_equal("#sub-output-1", "sub input initial value")
assert call_count.value == 1, "called once at initial stage"
pad_input, pad_div = dash_duo.dash_innerhtml_dom.select_one(
"#output > div"
).contents
assert (
pad_input.attrs["value"] == "sub input initial value"
and pad_input.attrs["id"] == "sub-input-1"
)
assert pad_input.name == "input"
assert (
pad_div.text == pad_input.attrs["value"]
and pad_div.get("id") == "sub-output-1"
), "the sub-output-1 content reflects to sub-input-1 value"
dash_duo.percy_snapshot(name="callback-generating-function-1")
assert dash_duo.redux_state_paths == {
"input": ["props", "children", 0],
"output": ["props", "children", 1],
"sub-input-1": [
"props",
"children",
1,
"props",
"children",
"props",
"children",
0,
],
"sub-output-1": [
"props",
"children",
1,
"props",
"children",
"props",
"children",
1,
],
}, "the paths should include these new output IDs"
# editing the input should modify the sub output
dash_duo.find_element("#sub-input-1").send_keys("deadbeef")
assert (
dash_duo.find_element("#sub-output-1").text
== pad_input.attrs["value"] + "deadbeef"
), "deadbeef is added"
# the total updates is initial one + the text input changes
dash_duo.wait_for_text_to_equal(
"#sub-output-1", pad_input.attrs["value"] + "deadbeef"
)
rqs = dash_duo.redux_state_rqs
assert rqs, "request queue is not empty"
assert all((rq["status"] == 200 and not rq["rejected"] for rq in rqs))
dash_duo.percy_snapshot(name="callback-generating-function-2")
assert dash_duo.get_logs() == [], "console is clean"
class HTAPController:
# have the shared-memory primitives static as otherwise the multiprocessing
# inheritance scheme doesn't work. we want these primitives so we can use
# "simple" synchronized primitives.
latest_timestamp = Value('d', 0) # database record timestamp
next_tsx_timestamp = Value(
'd', 0) # rtc time at which we can do the next oltp tsx
stats_queue = Queue() # queue for communicating statistics
def __init__(self, args):
self.args = args
self.next_tsx_timestamp.value = time.time()
self.tsx_timestamp_increment = 1.0 / self.args.target_tps if self.args.target_tps is not None else 0
self.num_warehouses = self._query_num_warehouses()
self.range_delivery_date = self._query_range_delivery_date()
# update the shared value to the actual last ingested timestamp
self.latest_timestamp.value = self.range_delivery_date[1].timestamp()
self.csv_interval = args.csv_interval if 'csv' in args.output else None
self.stats = Stats(self.args.dsn, self.args.oltp_workers,
self.args.olap_workers, self.csv_interval,
self.args.ignored_queries)
self.monitor = Monitor(self.stats, self.args.oltp_workers,
self.args.olap_workers, self.num_warehouses,
self.range_delivery_date[0])
print(f'Warehouses: {self.num_warehouses}')
def oltp_sleep(self):
with self.next_tsx_timestamp.get_lock():
self.next_tsx_timestamp.value = self.next_tsx_timestamp.value + self.tsx_timestamp_increment
sleep_until = self.next_tsx_timestamp.value
time_now = time.time()
if time_now < sleep_until:
time.sleep(sleep_until - time_now)
def oltp_worker(self, worker_id):
# do NOT introduce timeouts for the oltp queries! this will make that
# the workload gets inbalanaced and eventually the whole benchmark stalls
with DBConn(self.args.dsn) as conn:
oltp_worker = TransactionalWorker(worker_id, self.num_warehouses,
self.latest_timestamp, conn,
self.args.dry_run)
next_reporting_time = time.time() + 0.1
while True:
self.oltp_sleep()
oltp_worker.next_transaction()
if next_reporting_time <= time.time():
# its beneficial to send in chunks so try to batch the stats by accumulating 0.1s of samples
self.stats_queue.put(('oltp', oltp_worker.stats()))
next_reporting_time += 0.1
def olap_worker(self, worker_id):
stream = AnalyticalStream(worker_id, self.args,
self.range_delivery_date[0],
self.latest_timestamp, self.stats_queue)
while True:
stream.run_next_query()
def analyze_worker(self):
tables = [
'customer', 'district', 'history', 'item', 'nation', 'new_orders',
'order_line', 'orders', 'region', 'stock', 'supplier', 'warehouse'
]
os.makedirs('results', exist_ok=True)
with open('results/analyze.csv', 'w+') as csv:
with DBConn(self.args.dsn) as conn:
while True:
for table in tables:
start = time.time()
conn.cursor.execute(f'ANALYZE {table}')
runtime = time.time() - start
csv.write(
f'{datetime.now()}, {table}, {runtime:.2f}\n')
csv.flush()
time.sleep(600)
def _sql_error(self, msg):
import sys
print(f'ERROR: {msg} Did you run `prepare_benchmark`?')
sys.exit(-1)
def _query_range_delivery_date(self):
with DBConn(self.args.dsn) as conn:
try:
conn.cursor.execute(
'SELECT min(ol_delivery_d), max(ol_delivery_d) FROM order_line'
)
return conn.cursor.fetchone()
except ProgrammingError:
self._sql_error('Could not query the latest delivery date.')
def _query_num_warehouses(self):
with DBConn(self.args.dsn) as conn:
try:
conn.cursor.execute(
'SELECT count(distinct(w_id)) from warehouse')
return conn.cursor.fetchone()[0]
except ProgrammingError:
self._sql_error('Could not query number of warehouses.')
def _prepare_stats_db(self):
dsn_url = urlparse(self.args.stats_dsn)
dbname = dsn_url.path[1:]
with DBConn(f'{dsn_url.scheme}://{dsn_url.netloc}/postgres') as conn:
try:
conn.cursor.execute(
f"CREATE DATABASE {dbname} TEMPLATE template0 ENCODING 'UTF-8'"
)
except DuplicateDatabase:
pass
with DBConn(self.args.stats_dsn) as conn:
stats_schema_path = os.path.join('benchmarks', 'htap',
'stats_schema.sql')
with open(stats_schema_path, 'r') as schema:
schema_sql = schema.read()
try:
conn.cursor.execute(schema_sql)
except DuplicateTable:
pass
def run(self):
begin = datetime.now()
elapsed = timedelta()
burnin_duration = None if not self.args.dont_wait_until_enough_data else timedelta(
)
if self.args.stats_dsn is not None:
print(f"Statistics will be collected in '{self.args.stats_dsn}'.")
self._prepare_stats_db()
stats_conn_holder = DBConn(self.args.stats_dsn)
else:
print(f'Database statistics collection is disabled.')
stats_conn_holder = nullcontext()
def worker_init():
signal.signal(signal.SIGINT, signal.SIG_IGN)
num_total_workers = self.args.oltp_workers + self.args.olap_workers + 1
with stats_conn_holder as stats_conn:
with Pool(num_total_workers, worker_init) as pool:
oltp_workers = pool.map_async(self.oltp_worker,
range(self.args.oltp_workers))
olap_workers = pool.map_async(self.olap_worker,
range(self.args.olap_workers))
analyze_worker = pool.apply_async(self.analyze_worker)
try:
update_interval = timedelta(seconds=min(
self.args.monitoring_interval, self.args.csv_interval))
display_interval = timedelta(
seconds=self.args.monitoring_interval)
next_display = datetime.now() + display_interval
next_update = datetime.now() + update_interval
while True:
# the workers are not supposed to ever stop.
# so test for errors by testing for ready() and if so propagate them
# by calling .get()
if self.args.oltp_workers > 0 and oltp_workers.ready():
oltp_workers.get()
if self.args.olap_workers > 0 and olap_workers.ready():
olap_workers.get()
if analyze_worker.ready():
analyze_worker.get()
while datetime.now() < next_update:
self.stats.process_queue(self.stats_queue)
time.sleep(0.1)
time_now = datetime.now()
available_data = datetime.fromtimestamp(
self.latest_timestamp.value
) - self.range_delivery_date[0]
if burnin_duration == None and available_data >= WANTED_RANGE:
burnin_duration = time_now - begin
elapsed = time_now - begin
if elapsed.total_seconds() >= self.args.duration:
break
self.stats.update()
next_update = time_now + update_interval
if 'print' in self.args.output and next_display <= time_now:
next_display += display_interval
self.monitor.update_display(
elapsed, burnin_duration, time_now, stats_conn,
datetime.fromtimestamp(
self.latest_timestamp.value))
except KeyboardInterrupt:
pass
finally:
if burnin_duration == None:
burnin_duration = elapsed
self.monitor.display_summary(elapsed, burnin_duration)
self.stats.write_summary(self.args.csv_file, elapsed)
def _create_factor_instance(cls, factor=1.0) -> "SyncedFloat":
value = Value("d", factor)
return SyncedFloat(value)
def create_process(fromaddr, toaddrs, msg, counter, username, password, host,
port, usessl):
if not options.multiprocessing:
return send_mail(fromaddr, toaddrs, msg, counter, username, password,
host, port, usessl)
else:
p = Process(target=send_mail,
args=(fromaddr, toaddrs, msg, counter, username, password,
host, port, usessl))
p.start()
return p
counter = Value('d', 0)
if options.sender:
fromaddr = options.sender
else:
fromaddr = '*****@*****.**'
if options.username:
username = options.username
else:
username = fromaddr
port = None
if options.host:
host = options.host
if len(host.split(":")) == 2:
host, port = host.split(":")
if __name__ == "__main__":
# Processor info
cpu = cpuinfo.get_cpu_info()
# Colorama
init(autoreset=True)
title(getString("duco_python_miner") + str(minerVersion) + ")")
try:
from multiprocessing import (Manager, Process, Value, cpu_count,
current_process)
manager = Manager()
# Multiprocessing fix for pyinstaller
freeze_support()
# Multiprocessing globals
khashcount = Value("i", 0)
accepted = Value("i", 0)
rejected = Value("i", 0)
hashrates_list = manager.dict()
totalhashrate_mean = manager.list()
except Exception:
prettyPrint(
"sys0", " Multiprocessing is not available. " +
"Please check permissions and/or your python installation. " +
"Exiting in 15s.", "error")
sleep(15)
_exit(1)
try:
# Load config file or create new one
loadConfig()
def load_from_bam(bam_path,
target_contig,
start_pos,
end_pos,
vcf_handler,
use_end_sentinels=False,
n_threads=1,
debug_reads=False,
debug_pos=False,
stepper="samtools"):
"""
Load variants observed in a :py:class:`pysam.AlignmentFile` to
an instance of :py:class:`hansel.hansel.Hansel`.
Parameters
----------
bam_path : str
Path to the BAM alignment
target_contig : str
The name of the contig for which to recover haplotypes.
start_pos : int
The 1-indexed genomic position from which to begin considering variants.
end_pos : int
The 1-indexed genomic position at which to stop considering variants.
vcf_handler : dict{str, any}
Variant metadata, as provided by :py:func:`gretel.gretel.process_vcf`.
use_end_sentinels : boolean, optional(default=False)
Whether or not to append an additional pairwise observation between
the final variant on a read towards a sentinel.
.. note:: Experimental
This feature is for testing purposes, currently it is recommended
that the flag be left at the default of `False`. However, some
data sets report minor performance improvements for some haplotypes
when set to `True`.
This flag may be removed at any time without warning.
n_threads : int, optional(default=1)
Number of threads to spawn for reading the BAM
debug_reads : list{str}, optional
A list of read names for which to print out debugging information
debug_pos : list{int}, optional
A list of positions for which to print out debugging information
stepper : str, optional(default=samtools)
The pysam pileup stepper to use
Returns
-------
Hansel : :py:class:`hansel.hansel.Hansel`
"""
hansel = Hansel.init_matrix(['A', 'C', 'G', 'T', 'N', "-", "_"],
['N', "_"], vcf_handler["N"])
if not debug_reads:
debug_reads = set([])
if not debug_pos:
debug_pos = set([])
import random
def progress_worker(progress_q, n_workers, slices, total_snps, crumbs):
worker_pos = []
worker_done = []
for _ in range(0, n_workers):
worker_pos.append(0)
worker_done.append(0)
while sum(worker_done) < n_workers:
work_block = progress_q.get()
worker_pos[work_block["worker_i"]] = work_block["pos"]
if work_block["pos"] is None:
worker_done[work_block["worker_i"]] = 1
crumbs.value += work_block["crumbs"]
slices.value += work_block["slices"]
total_snps.value += work_block["covered_snps"]
sys.stderr.write("%s\n" % ([
worker_pos[i] if status != 1 else None
for (i, status) in enumerate(worker_done)
]))
if random.random() < 0.1:
sys.stderr.write("%s\n" % ([
worker_pos[i] if status != 1 else None
for (i, status) in enumerate(worker_done)
]))
return (slices, total_snps, crumbs)
def bam_worker(bam_q, progress_q, worker_i):
worker = worker_i
slices = 0
crumbs = 0
covered_snps = 0
bam = pysam.AlignmentFile(bam_path)
while True:
work_block = bam_q.get()
if work_block is None:
progress_q.put({
"pos": None,
"worker_i": worker_i,
"slices": slices,
"crumbs": crumbs,
"covered_snps": covered_snps,
})
break
reads = {}
dreads = set([])
for p_col in bam.pileup(reference=target_contig,
start=work_block["start"] - 1,
stop=work_block["end"],
ignore_overlaps=False,
min_base_quality=0,
stepper=stepper):
if p_col.reference_pos + 1 > end_pos:
# Ignore positions beyond the end_pos
break
if vcf_handler["region"][p_col.reference_pos + 1] != 1:
# Ignore non-SNPs
continue
for p_read in p_col.pileups:
curr_read_1or2 = 0
if p_read.alignment.is_paired:
if p_read.alignment.is_read1:
curr_read_1or2 = 1
elif p_read.alignment.is_read2:
curr_read_1or2 = 2
else:
#TODO Probably indicative of bad data
pass
curr_read_name = "%s_%s_%d" % (p_read.alignment.query_name,
str(p_read.alignment.flag),
curr_read_1or2)
LEFTMOST_1pos = p_read.alignment.reference_start + 1 # Convert 0-based reference_start to 1-based position (to match region array and 1-based VCF)
# Special case: Consider reads that begin before the start_pos, but overlap the 0th block
if work_block["i"] == 0:
if LEFTMOST_1pos < start_pos:
# Read starts before the start_pos
if p_read.alignment.reference_start + 1 + p_read.alignment.query_alignment_length < start_pos:
# Read ends before the start_pos
continue
LEFTMOST_1pos = start_pos
else:
# This read begins before the start of the current (non-0) block
# and will have already been covered by the block that preceded it
if LEFTMOST_1pos < work_block["start"]:
continue
sequence = None
qual = None
if p_read.is_del:
# TODO Not sure about how to estimate quality of deletion?
sequence = "-" * (abs(p_read.indel) + 1)
qual = p_read.alignment.query_qualities[
p_read.query_position_or_next] * (
abs(p_read.indel) + 1)
elif p_read.indel > 0:
# p_read.indel peeks to next CIGAR and determines whether the base FOLLOWING this one is an insertion or not
sequence = p_read.alignment.query_sequence[
p_read.query_position:p_read.query_position +
p_read.indel + 1]
qual = p_read.alignment.query_qualities[
p_read.query_position:p_read.query_position +
p_read.indel + 1]
else:
sequence = p_read.alignment.query_sequence[
p_read.query_position]
qual = p_read.alignment.query_qualities[
p_read.query_position]
if not sequence:
print(
"[WARN] Sequence data seems to not be correctly salvaged from read %s"
% p_read.alignment.query_name)
continue
if curr_read_name not in reads:
reads[curr_read_name] = {
"rank": np.sum(
vcf_handler["region"]
[1:LEFTMOST_1pos]), # non-inclusive 1pos end
"seq": [],
"quals": [],
"refs_1pos": [],
"read_variants_0pos": [],
}
if p_read.alignment.query_name in debug_reads:
dreads.add(curr_read_name)
reads[curr_read_name]["seq"].append(sequence)
reads[curr_read_name]["quals"].append(qual)
reads[curr_read_name]["refs_1pos"].append(
p_col.reference_pos + 1)
reads[curr_read_name]["read_variants_0pos"].append(
p_read.query_position)
for dread in sorted(dreads):
r = reads[dread]
if r:
for snp_i, ref_pos in enumerate(r["refs_1pos"]):
print(dread, ref_pos, r["seq"][snp_i])
print("RANK", dread, r["rank"])
if debug_pos:
for read in reads:
for d_pos in set(reads[read]["refs_1pos"]) & debug_pos:
i = reads[read]["refs_1pos"].index(d_pos)
print(read, d_pos, reads[read]["seq"][i])
num_reads = len(reads)
for qi, qname in enumerate(reads):
progress_q.put({
"pos": num_reads - (qi + 1),
"worker_i": worker_i
})
if not len(reads[qname]["seq"]) > 1:
# Ignore reads without evidence
continue
slices += 1
rank = reads[qname]["rank"]
support_len = len(reads[qname]["seq"])
support_seq = "".join(
[b[0] for b in reads[qname]["seq"]]
) # b[0] has the affect of capturing the base before any insertion
covered_snps += len(
support_seq.replace("N", "").replace("_", ""))
# For each position in the supporting sequence (that is, each covered SNP)
for i in range(0, support_len):
snp_a = support_seq[i]
#if support_len == 1:
# if rank == 0:
# hansel.add_observation('_', snp_a, 0, 1)
# hansel.add_observation(snp_a, '_', 1, 2)
# else:
# hansel.add_observation(snp_a, '_', rank+1, rank+2)
# For each position in the supporting sequence following i
for j in range(i + 1, support_len):
snp_b = support_seq[j]
# Ignore observations who are from an invalid transition
if snp_a in ['_', 'N']:
continue
# Sentinel->A
if i == 0 and j == 1 and rank == 0:
# If this is the first position in the support (support_pos == 0)
# and rank > 0 (that is, this is not the first SNP)
# and SNPs a, b are adjacent
hansel.add_observation('_', snp_a, 0, 1)
hansel.add_observation(snp_a, snp_b, 1, 2)
crumbs += 1
# B->Sentinel
elif (j + rank +
1) == vcf_handler["N"] and abs(i - j) == 1:
# Last observation (abs(i-j)==1),
# that ends on the final SNP (j+rank+1 == N)
hansel.add_observation(snp_a, snp_b,
vcf_handler["N"] - 1,
vcf_handler["N"])
hansel.add_observation(snp_b, '_',
vcf_handler["N"],
vcf_handler["N"] + 1)
crumbs += 1
# A regular observation (A->B)
else:
hansel.add_observation(snp_a, snp_b, i + rank + 1,
j + rank + 1)
crumbs += 1
if use_end_sentinels:
if j == (support_len - 1) and abs(i - j) == 1:
# The last SNP on a read, needs a sentinel afterward
hansel.add_observation(
snp_b, '_', j + rank + 1, j + rank + 2)
bam_queue = Queue()
progress_queue = Queue()
# Queue the wokers
# TODO Evenly divide, but in future, consider the distn
# TODO Also consider in general block0 has more work to do
window_l = round((end_pos - start_pos) / float(n_threads))
for window_i, window_pos in enumerate(
range(start_pos, end_pos + 1, window_l)):
bam_queue.put({
"start": window_pos,
"end": window_pos + window_l -
1, # add -1 to stop end of window colliding with next window
"i": window_i,
"region_end": end_pos,
})
processes = []
for _ in range(n_threads):
p = Process(target=bam_worker, args=(bam_queue, progress_queue, _))
processes.append(p)
# ...and a progress process
n_reads = Value('i', 0)
n_observations = Value('i', 0)
total_covered_snps = Value('i', 0)
p = Process(target=progress_worker,
args=(progress_queue, n_threads, n_reads, total_covered_snps,
n_observations))
processes.append(p)
for p in processes:
p.start()
# Add sentinels
for _ in range(n_threads):
bam_queue.put(None)
# Wait for processes to complete work
for p in processes:
p.join()
hansel.n_slices = n_reads.value
hansel.n_crumbs = n_observations.value
sys.stderr.write("[NOTE] Loaded %d breadcrumbs from %d bread slices.\n" %
(hansel.n_crumbs, hansel.n_slices))
hansel.L = int(ceil(float(total_covered_snps.value) / n_reads.value))
sys.stderr.write("[NOTE] Setting Gretel.L to %d\n" % hansel.L)
return hansel
packet_buff = buff[header_position:header_position + RX_SIZE]
# check crc
crc = struct.unpack(CRC_PACKSTR, packet_buff[-CRC_SIZE:])[0]
if crc_fun(packet_buff[:-CRC_SIZE]) == crc:
packet[:] = packet_buff
with init_flag.get_lock():
init_flag.value = 1
else:
print('crc discrepancy')
time.sleep(max(0, latest_time + 1.0 / freq - time.time()))
packet = Array('c', RX_SIZE)
init_flag = Value('i', 0)
exit_flag = Value('i', 0)
established_flag = False
ser = None
def terminate_process():
exit_flag.value = 1
atexit.register(terminate_process)
# share serial instance / process among multiple environments
def establish_connection():
global established_flag, ser
def __init__(self):
self.lock = Lock()
self.n = Value('i', 0)
def main(ip):
# use fixed random state
rand_state = np.random.RandomState(1).get_state()
np.random.set_state(rand_state)
tf_set_seeds(np.random.randint(1, 2**31 - 1))
# Create UR5 Reacher2D environment
env = ReacherEnv(setup="UR5_6dof",
host=ip,
dof=6,
control_type="velocity",
target_type="position",
reset_type="zero",
reward_type="precision",
derivative_type="none",
deriv_action_max=5,
first_deriv_max=2,
accel_max=1.4,
speed_max=0.3,
speedj_a=1.4,
episode_length_time=4.0,
episode_length_step=None,
actuation_sync_period=1,
dt=0.04,
run_mode="multiprocess",
rllab_box=False,
movej_t=2.0,
delay=0.0,
random_state=rand_state)
env = NormalizedEnv(env)
# Start environment processes
env.start()
# Create baselines TRPO policy function
sess = U.single_threaded_session()
sess.__enter__()
def policy_fn(name, ob_space, ac_space):
return MlpPolicy(name=name,
ob_space=ob_space,
ac_space=ac_space,
hid_size=64,
num_hid_layers=2)
# Create and start plotting process
plot_running = Value('i', 1)
shared_returns = Manager().dict({
"write_lock": False,
"episodic_returns": [],
"episodic_lengths": [],
})
# Spawn plotting process
pp = Process(target=plot_ur5_reacher,
args=(env, 2048, shared_returns, plot_running))
pp.start()
# Create callback function for logging data from baselines TRPO learn
kindred_callback = create_callback(shared_returns)
# Train baselines TRPO
learn(env,
policy_fn,
max_timesteps=200000,
timesteps_per_batch=2048,
max_kl=0.05,
cg_iters=10,
cg_damping=0.1,
vf_iters=5,
vf_stepsize=0.001,
gamma=0.995,
lam=0.995,
callback=kindred_callback)
# Safely terminate plotter process
plot_running.value = 0 # shutdown ploting process
time.sleep(2)
pp.join()
env.close()
f3.write("" + str(rstack[i]) + " ")
f3.close()
path6 = 'stack.txt'
f6 = open(path6, 'w')
for i in range(0, len(value), 1):
f6.write("" + str(value[i]) + " ")
f6.close()
if __name__ == '__main__':
start_time = time.time()
start = []
end = []
tabledata = []
tablecount = Value('i', 0)
address = Array('i', 10)
value = Array('i', 10)
rstack = Array('i', 100000)
lstack = Array('i', 100000)
rtop = Value('i', 0)
ltop = Value('i', 0)
endflag = {}
endflag0 = Value('i', 0)
notlabelflag = 0
lock = {}
variable_region = []
mlock = Lock()
lockfree = Lock()
a = '1'
def __init__(self, opt, shared=None):
super().__init__(opt, shared)
if not hasattr(self, 'datatype'):
self.datatype = opt['datatype']
if not hasattr(self, 'random'):
self.random = self.datatype == 'train'
if not hasattr(self, 'training'):
self.training = self.datatype.startswith('train')
if not hasattr(self, 'datafile'):
self.datafile = opt.get('datafile', opt.get('pytorch_datafile'))
# set up support for multithreaded data loading
self.data_queue = queue.Queue()
if shared:
self.index = shared['index']
if 'data_loader' in shared:
self.data_loader = shared['data_loader']
if 'threadindex' in shared:
self.threadindex = shared['threadindex']
if 'examples' in shared:
self.examples = shared['examples']
else:
self.index = AttrDict(value=-1)
if not hasattr(self, 'data_loader'):
self.data_loader = DataLoader(opt)
self.data_loader.start()
# set up batching
self.bsz = opt.get('batchsize', 1)
self.batchindex = opt.get('batchindex', 0)
dt = opt.get('datatype', '').split(':')
self.use_batch_act = (opt.get('batch_sort', False) and self.bsz > 1
and 'stream' not in dt)
if self.use_batch_act:
if shared:
self.lastYs = shared['lastYs']
if 'sorted_data' in shared:
self.sorted_data = shared['sorted_data']
self.batches = shared['batches']
else:
self.lastYs = [None] * self.bsz
ordered_opt = opt.copy()
ordered_opt['datatype'] = ':'.join((dt[0], 'ordered'))
ordered_opt['batchsize'] = 1
ordered_opt['numthreads'] = 1
ordered_opt['hide_labels'] = False
ordered_teacher = create_task_agent_from_taskname(ordered_opt)[0]
clen = opt.get('context_length', -1)
incl = opt.get('include_labels', True)
if ordered_teacher.num_examples() > 1000000: # one million
print('WARNING: this dataset is large, and batch sorting '
'may use too much RAM or take too long to set up. '
'Consider disabling batch sorting, setting '
'context-length to a small integer (if this dataset '
'has episodes of multiple examples), or streaming '
'the data using a streamed data mode if supported.')
flatdata = flatten(ordered_teacher,
context_length=clen, include_labels=incl)
self.sorted_data = sort_data(flatdata)
self.batches = make_batches(self.sorted_data, self.bsz)
# one fixed-seed shuffle keeps determinism but makes sure that
# examples aren't presented in sorted order (bad for `-vme`)
random.Random(42).shuffle(self.batches)
class FixedDialogTeacher(Teacher):
"""A teacher agent for all teachers involved in tasks with fixed data.
This class provides the following functionality for its subclasses:
- Resets a teacher
- Provides an observe method
- Computes and retrieves the next episode index for a teacher
- Provides a threadpool option for loading data (especially useful for
large data, e.g. images)
In order to take advantage of the first few features, all a subclass has to
implement is three functions: ``num_episodes``, ``num_examples``, and
``get`` (which returns a specific example from a specific episode).
To utilize the DataLoader for threadpool loading, a teacher should
implement the ``submit_load_request`` function to send a load request
to the DataLoader by calling ``self.data_loader.request_load`` with the
appropriate arguments (``receive_fn, load_fn, args``). The DataLoader then
returns the data to the teacher's ``data_queue``, which the teacher can
poll in its ``act`` method.
The following is an example of the DataLoader usage in the VQA-V1 teacher.
1. In the teacher's ``init`` function, the teacher calls its
``submit_load_request`` function to preload an image.
2. The ``submit_load_request`` function gets the next ``episode_idx``,
and computes the image path for the load request.
3. At the end of ``submit_load_request``, the teacher calls
``self.data_loader.request_load`` with three args:
- ``self.receive_data`` - the function that the DataLoader calls to
return the the loaded object
- ``self.image_loader.load`` - the function used to load the image
from the image path
- ``[img_path]`` - a list of arguments for the load function, which
in this case is the path of the image.
4. In the teacher's ``act`` function, the teacher loads the data from
its data queue.
5. At the end of the ``act`` function, the teacher calls
``submit_load_request`` to preload an image for the next example.
To see this in action, take a look at this teacher in ``tasks.vqa_v1.agents``.
"""
def __init__(self, opt, shared=None):
super().__init__(opt, shared)
if not hasattr(self, 'datatype'):
self.datatype = opt['datatype']
if not hasattr(self, 'random'):
self.random = self.datatype == 'train'
if not hasattr(self, 'training'):
self.training = self.datatype.startswith('train')
if not hasattr(self, 'datafile'):
self.datafile = opt.get('datafile', opt.get('pytorch_datafile'))
# set up support for multithreaded data loading
self.data_queue = queue.Queue()
if shared:
self.index = shared['index']
if 'data_loader' in shared:
self.data_loader = shared['data_loader']
if 'threadindex' in shared:
self.threadindex = shared['threadindex']
if 'examples' in shared:
self.examples = shared['examples']
else:
self.index = AttrDict(value=-1)
if not hasattr(self, 'data_loader'):
self.data_loader = DataLoader(opt)
self.data_loader.start()
# set up batching
self.bsz = opt.get('batchsize', 1)
self.batchindex = opt.get('batchindex', 0)
dt = opt.get('datatype', '').split(':')
self.use_batch_act = (opt.get('batch_sort', False) and self.bsz > 1
and 'stream' not in dt)
if self.use_batch_act:
if shared:
self.lastYs = shared['lastYs']
if 'sorted_data' in shared:
self.sorted_data = shared['sorted_data']
self.batches = shared['batches']
else:
self.lastYs = [None] * self.bsz
ordered_opt = opt.copy()
ordered_opt['datatype'] = ':'.join((dt[0], 'ordered'))
ordered_opt['batchsize'] = 1
ordered_opt['numthreads'] = 1
ordered_opt['hide_labels'] = False
ordered_teacher = create_task_agent_from_taskname(ordered_opt)[0]
clen = opt.get('context_length', -1)
incl = opt.get('include_labels', True)
if ordered_teacher.num_examples() > 1000000: # one million
print('WARNING: this dataset is large, and batch sorting '
'may use too much RAM or take too long to set up. '
'Consider disabling batch sorting, setting '
'context-length to a small integer (if this dataset '
'has episodes of multiple examples), or streaming '
'the data using a streamed data mode if supported.')
flatdata = flatten(ordered_teacher,
context_length=clen, include_labels=incl)
self.sorted_data = sort_data(flatdata)
self.batches = make_batches(self.sorted_data, self.bsz)
# one fixed-seed shuffle keeps determinism but makes sure that
# examples aren't presented in sorted order (bad for `-vme`)
random.Random(42).shuffle(self.batches)
def _lock(self):
if hasattr(self.index, 'get_lock'):
return self.index.get_lock()
else:
return no_lock()
def reset(self):
"""Reset the dialog so that it is at the start of the epoch,
and all metrics are reset.
"""
super().reset()
self.metrics.clear()
self.lastY = None
self.episode_done = True
self.epochDone = False
self.data_queue = queue.Queue()
self.episode_idx = -1
with self._lock():
self.index.value = -1
if self.use_batch_act and self.random and hasattr(self, 'batches'):
random.shuffle(self.batches)
def submit_load_request(self):
"""An agent should implement this method to submit requests to the
data loader. At the end of this method, the agent should call
``self.data_loader.request_load()`` with the appropriate args.
By default, this method does nothing.
"""
pass
def receive_data(self, future):
"""Function for receiving data from the data loader.
:param future: result from the load request.
"""
data = future.result()
self.data_queue.put(data)
def share(self):
"""Shares data structures between other instances created for batching
or hogwild.
"""
shared = super().share()
if hasattr(self, 'lastYs'):
# share lastYs to communicate between batch_act and observe
shared['lastYs'] = self.lastYs
if hasattr(self, 'examples'):
shared['examples'] = self.examples
if self.opt.get('numthreads', 1) > 1:
if type(self.index) is not multiprocessing.sharedctypes.Synchronized:
# for multithreading need to move index into threadsafe memory
self.index = Value('l', -1)
if hasattr(self, 'sorted_data'):
shared['sorted_data'] = self.sorted_data
shared['batches'] = self.batches
else:
shared['data_loader'] = self.data_loader
shared['index'] = self.index
return shared
def next_episode_idx(self, num_eps=None, loop=None):
"""Returns the next episode index.
:param num_eps: default None uses ``num_episodes`` value.
:param loop: default None loops during training but not evaluation.
"""
if num_eps is None:
num_eps = self.num_episodes()
if loop is None:
loop = self.training
if self.random:
new_idx = random.randrange(num_eps)
else:
with self._lock():
self.index.value += 1
if loop:
self.index.value %= num_eps
new_idx = self.index.value
return new_idx
def next_example(self):
"""Returns the next example.
If there are multiple examples in the same episode, returns the next
one in that episode. If that episode is over, gets a new episode index
and returns the first example of that episode.
"""
if self.episode_done:
self.episode_idx = self.next_episode_idx()
self.entry_idx = 0
else:
self.entry_idx += 1
if self.episode_idx >= self.num_episodes():
return {'episode_done': True}, True
ex = self.get(self.episode_idx, self.entry_idx)
self.episode_done = ex.get('episode_done', False)
if (not self.random and self.episode_done
and self.episode_idx + self.opt.get("batchsize", 1) >= self.num_episodes()):
epoch_done = True
else:
epoch_done = False
return ex, epoch_done
def next_batch(self):
"""Returns the next batch of examples."""
# get next batch
with self._lock():
self.index.value += 1
if self.training:
self.index.value %= len(self.batches)
batch_idx = self.index.value
if batch_idx + 1 >= len(self.batches):
if self.random:
random.shuffle(self.batches)
self.epochDone = True
else:
self.epochDone = False
if batch_idx >= len(self.batches):
return [{'episode_done': True, 'id': self.getID()}] * self.bsz
return self.batches[batch_idx]
def num_episodes(self):
"""Get the number of episodes in this dataset."""
if self.use_batch_act:
# when using batch_act, this is length of sorted data
return len(self.sorted_data)
raise RuntimeError('"num_episodes" must be overriden by children.')
def num_examples(self):
"""Get the total number of examples in this dataset."""
if self.use_batch_act:
# when using batch_act, this is length of sorted data
return len(self.sorted_data)
raise RuntimeError('"num_examples" must be overriden by children.')
def get(self, episode_idx, entry_idx=0):
"""Get the specified episode and the specified entry in that episode.
Children must override this method in order to inherit the
`next_example` method.
:param episode_idx: which episode to return examples from
:param entry_idx: which example to return from the episode.
Many datasets have only single-entry episodes,
so this defaults tozero.
"""
raise RuntimeError('"Get" method must be overriden by children.')
def observe(self, observation):
"""Process observation for metrics."""
if self.use_batch_act:
self.lastY = self.lastYs[self.batchindex]
self.lastYs[self.batchindex] = None
if hasattr(self, 'lastY') and self.lastY is not None:
self.metrics.update(observation, self.lastY)
self.lastY = None
return observation
def batch_act(self, observations):
"""Returns an entire batch of examples instead of just one."""
# we ignore observations
if not hasattr(self, 'epochDone'):
# reset if haven't yet
self.reset()
batch = self.next_batch()
# pad batch
if len(batch) < self.bsz:
batch += [{'episode_done': True, 'id': self.getID()}] * (self.bsz - len(batch))
# remember correct answer if available (for padding, None)
for i, ex in enumerate(batch):
self.lastYs[i] = ex.get('labels', ex.get('eval_labels'))
return batch
def act(self):
"""Send new dialog message."""
if not hasattr(self, 'epochDone'):
# reset if haven't yet
self.reset()
# get next example, action is episode_done dict if already out of exs
action, self.epochDone = self.next_example()
action['id'] = self.getID()
# remember correct answer if available
self.lastY = action.get('labels', None)
if ((not self.datatype.startswith('train') or 'evalmode' in self.datatype)
and 'labels' in action):
# move labels to eval field so not used for training
# but this way the model can use the labels for perplexity or loss
labels = action.pop('labels')
if not self.opt.get('hide_labels', False):
action['eval_labels'] = labels
return action
import cv2
import os
from Step2.preProcessing.PreProcess import GrayImg, lightRemove, gamma_trans, letter_box
import numpy as np
import math
# The moving area
delta_x = 960
delta_y = 540
m = 0
n = 0
# global value to stop/start process
alive = Value('b', True)
flag = False
# Click the button then the window will move
def moveit():
global delta_x, delta_y, flag, m, n
window2 = tk.Tk()
# On the top
window2.attributes("-topmost", True)
window2.overrideredirect(True)
# Fill with red
Full_color = tk.Label(window2, bg='red', width=10, height=10)
Full_color.pack()
n = 0
# from(300,150)，steps(330. 195)，20 times in total
import math
from DeteccionObj_detob import *
import threading
import cv2
import serial # Importing the serial library to communicate with Arduino
from multiprocessing import Value, Process
from ctypes import c_bool
import numpy as np
from interpreter import Automata
import time
from map_grilla_detob import map_grilla
from solucion import generar_solucion
flag = True
i = 30
angulo = Value('i', 0)
x = Value('i', 0)
y = Value('i', 0)
t_ref = Value('i', 0) # es el ángulo de referencia
x_ref = Value('i', 0)
y_ref = Value('i', 0)
coord_x_ref = Value('i', 0)
coord_y_ref = Value('i', 0)
coord_x_ref_ant = Value('i', 0)
coord_y_ref_ant = Value('i', 0)
ready = Value(c_bool, False)
flag = Value(c_bool, True)
import os
import sys
from functools import partial
from multiprocessing import Pool, cpu_count, Value
from pathlib import Path
from typing import Dict
from google.appengine.api import datastore
from google.appengine.api.datastore_types import EmbeddedEntity
from google.appengine.datastore import entity_bytes_pb2 as entity_pb2
from converter import records
from converter.exceptions import BaseError, ValidationError
from converter.utils import embedded_entity_to_dict, get_dest_dict, serialize_json
num_files: Value = Value("i", 0)
num_files_processed: Value = Value("i", 0)
def main(args=None):
if args is None:
args = sys.argv[1:]
parser = argparse.ArgumentParser(
prog="fs_to_json", description="Firestore DB export to JSON"
)
parser.add_argument(
"source_dir",
help="Destination directory to store generated JSON",
type=str,
def run(rank, workers, model, save_path, train_data, test_data, global_lr):
# 获取ps端传来的模型初始参数
print(workers)
_group = [w for w in workers].append(0)
group = dist.new_group(_group)
for p in model.parameters():
tmp_p = torch.zeros_like(p)
dist.scatter(tensor=tmp_p, src=0, group=group)
p.data = tmp_p
print('Model recved successfully!')
temp_lr = global_lr.get()
if args.model in ['MnistCNN', 'AlexNet', 'ResNet18OnCifar10']:
optimizer = MySGD(model.parameters(), lr=temp_lr)
else:
optimizer = MySGD(model.parameters(), lr=temp_lr)
if args.model in ['MnistCNN', 'AlexNet']:
criterion = torch.nn.NLLLoss()
else:
criterion = torch.nn.CrossEntropyLoss()
print('Begin!')
# the parameters that will be transferred to the thread
model_cache = [p.data + 0.0 for p in model.parameters()]
global_update = [torch.zeros_like(p) for p in model.parameters()]
local_update = [torch.zeros_like(p) for p in model.parameters()]
it_count = Value(c_float,
0.) # count update times in an iteration by local worker
data_lock = Lock()
update_lock = Queue()
update_lock.put(1)
loss_t = torch.tensor(0.0)
receive_end = Value(c_bool, False)
batch_communication_interval = Value(c_float, 0.0)
stale_in_iteration = Value(c_float, 0.)
sender_td = Thread(target=sender,
args=(
model_cache,
global_update,
local_update,
it_count,
loss_t,
update_lock,
data_lock,
group,
receive_end,
batch_communication_interval,
stale_in_iteration,
),
daemon=True)
sender_td.start()
time_logs = open("./record" + str(rank), 'w')
osp_logs = open("./log" + str(rank), 'w')
Stale_Threshold = args.stale_threshold
for epoch in range(args.epochs):
batch_interval = 0.0
batch_comp_interval = 0.0
s_time = time.time()
model.train()
# AlexNet在指定epoch减少学习率LR
# learning rate should be decreased on server due to unmatched updating speed between local worker and server
if not global_lr.empty():
g_lr = global_lr.get()
if args.model == 'AlexNet':
for param_group in optimizer.param_groups:
param_group['lr'] = g_lr
print('LR Decreased! Now: {}'.format(param_group['lr']))
for batch_idx, (data, target) in enumerate(train_data):
batch_start_time = time.time()
data, target = Variable(data), Variable(target)
optimizer.zero_grad()
output = model(data)
loss = criterion(output, target)
loss.backward()
delta_ws = optimizer.get_delta_w()
optimizer.step()
# Aggregate local update
data_lock.acquire()
# aggregate loss
loss_t.data += loss.data
it_count.value += 1
for g_idx, update in enumerate(local_update):
update.data += delta_ws[g_idx].data
data_lock.release()
batch_computation_time = time.time()
# Open the lock once the local update has at least one gradient
if it_count.value == 1:
update_lock.put(1)
while it_count.value >= Stale_Threshold:
pass
if receive_end.value:
receive_end.value = False
for idx, param in enumerate(model.parameters()):
param.data = model_cache[idx] # without local update
# param.data = model_cache[idx] - global_update[idx] # with local update
batch_end_time = time.time()
batch_interval += batch_end_time - batch_start_time
batch_comp_interval += batch_computation_time - batch_start_time
osp_logs.write(
str(batch_end_time - batch_start_time) + "\t" +
str(batch_computation_time - batch_start_time) + "\n")
osp_logs.flush()
print('Rank {}, Epoch {}, Loss:{}'.format(rank, epoch,
loss.data.item()))
e_time = time.time()
# 训练结束后进行test
#test_loss, acc = test_model(rank, model, test_data, criterion=criterion)
acc = 0.0
batch_interval /= batch_idx
batch_comp_interval /= batch_idx
logs = torch.tensor([
acc, batch_interval, batch_comp_interval,
batch_communication_interval.value, stale_in_iteration.value
])
time_logs.write(str(logs) + '\n')
time_logs.flush()
# dist.gather(tensor=logs, dst = 0, group = group)
time_logs.close()
sender_td.join()
def create_should_run_instance(cls) -> Value:
return Value(c_bool, True)
def __init__(self):
self.val = Value('i', 0)
Power_Index[0] = Power_Index[0] + 1
Power_Is_Read.value = 1
if __name__ == "__main__":
PhotoDiod_Port = "AIN1"
Spec1 = SBO.DetectSpectrometer()
Integration_Time = 10 # Integration time in ms
Spec1.setTriggerMode(0) # It is set for free running mode
Spec1.setIntegrationTime(
Integration_Time *
1000) # Integration time is in microseconds when using the library
DAQ1 = DAQ.DetectDAQT7()
Power_meter = P100.DetectPM100D()
Spec_Is_Read = Value('i', 0)
Spec_Is_Read.value = 0
Spec_Is_Done = Value('i', 0)
Spec_Is_Done.value = 0
DAQ_Is_Read = Value('i', 0)
DAQ_Is_Read.value = 0
Power_Is_Read = Value('i', 0)
Power_Is_Read.value = 0
Timer_Is_Over = Value('i', 0)
Timer_Is_Over.value = 0
DurationOfReading = 1.12 # Duration of reading in seconds.
No_DAC_Sample = int(
round(DurationOfReading * 1000 / 0.5)
) # Number of samples for DAQ analogue to digital converter (AINx). Roughly DAQ can read AINx every 0.4 ms
No_Power_Sample = int(
import os
import time
from datetime import datetime
from multiprocessing import Process, Value
import sys
from watchdog.events import FileSystemEventHandler
from watchdog.observers import Observer
from constant import MAX_TIME
# 日志输出
from logger import logger as logging
from util import check_game_state, start_game, initDevice
waitTime = Value('i', 0)
def startGame(waitTime1):
initDevice()
global waitTime
waitTime = waitTime1
start_game()
check_game_state(waitTime)
def updateTime(event):
if "screen.png" in event.src_path:
global lastChangeTime
lastChangeTime = datetime.now()
logging.debug('{0}:{1}'.format(event.event_type, event.src_path))
# clean expired sessions oonce on first use in case lifetime was changed
def cleaner():
with cleaned.get_lock():
if not cleaned.value:
cleaned.value = True
app.logger.info('cleaning session store')
MailuSessionExtension.cleanup_sessions(app)
app.before_first_request(cleaner)
app.session_config = MailuSessionConfig(app)
app.session_interface = MailuSessionInterface()
cleaned = Value('i', False)
session = MailuSessionExtension()
# this is used by the webmail to authenticate IMAP/SMTP
def verify_temp_token(email, token):
try:
if token.startswith('token-'):
if sessid := app.session_store.get(token):
session = MailuSession(sessid, app)
if session.get('_user_id', '') == email:
return True
except:
pass
conversation.start()
video_engine.start()
control_engine.start()
video_engine.join()
control_engine.join()
def start_api(bot_state, is_face_detected):
# flask server
application.run()
if __name__ == "__main__":
# globals
global is_face_detected
is_face_detected = Value('d')
is_face_detected.value = 0
global bot_state
bot_state = Value('d')
bot_state.value = 1
engines = Process(target=start_engines, args=(bot_state, is_face_detected))
api = Process(target=start_api, args=(bot_state, is_face_detected))
engines.start()
api.start()
engines.join()
api.join()
from flask import Flask, request, send_from_directory, jsonify, make_response
from flask_cors import CORS
import os
from flask_jwt_extended import JWTManager, jwt_required
from kazoo.client import KazooClient
from multiprocessing import Value
from datetime import datetime
count_post_requests = Value('i', 0)
count_get_requests = Value('i', 0)
count_delete_requests = Value('i', 0)
app = Flask(__name__)
app.config.from_envvar('APP_CONFIG_FILE')
app.config['STORAGE_ID'] = os.environ.get('STORAGE_ID')
app.config['UPLOAD_FOLDER'] = "/app/images" #+app.config['STORAGE_PORT']
app.config['ZK_HOST'] = os.environ.get('ZK_HOST')
LOG_FILE = 'storage_accesses.txt'
jwt = JWTManager(app)
CORS(app)
client = KazooClient(hosts=app.config['ZK_HOST'])
client.start()
client.ensure_path("/storage")
if not client.exists('/storage/' + app.config['STORAGE_ID']):
client.create('/storage/' + app.config['STORAGE_ID'],