def execute_task(self, website: Website, busy: Value, post_id: str, comment_id: str): busy.value = 1 if os.path.exists("data.json"): os.remove("data.json") print("Started crawling task") process = CrawlerProcess(get_project_settings()) process.crawl("od_links", base_url=website.url) process.start() print("Done crawling") self.db.import_json("data.json", website) os.remove("data.json") print("Imported in SQLite3") if post_id: # Reply to post stats = self.db.get_website_stats(website.id) comment = self.reddit_bot.get_comment({"": stats}, website.id) print(comment) if "total_size" in stats and stats["total_size"] > 10000000: post = self.reddit_bot.reddit.submission(post_id) self.reddit_bot.reply(post, comment) pass else: self.reddit_bot.log_crawl(post_id) elif comment_id: # Reply to comment stats = self.db.get_website_stats(website.id) comment = self.reddit_bot.get_comment({"There you go!": stats}, website.id) print(comment) reddit_comment = self.reddit_bot.reddit.comment(comment_id) self.reddit_bot.reply(reddit_comment, comment) busy.value = 0 print("Done crawling task")
def setup_heartbeat(self, client_controller): cond = multiprocessing.Condition() s_init_finish = Value('i', 0) do_sample = Value('i', 0) do_sample_lock = Lock() server_process = multiprocessing.Process( target=self.server_heart_beat, args=(cond, s_init_finish, do_sample, do_sample_lock)) server_process.daemon = False server_process.start() logger.info("Waiting for server init ...") cond.acquire() while (s_init_finish.value == 0): cond.wait() if s_init_finish.value == 5: logger.error("Waiting for server init ... FAIL") raise RuntimeError("server init failed.") cond.release() logger.info("Waiting for server init ... Done") # let all clients start running the benchmark client_controller.client_run(do_sample, do_sample_lock) cond.acquire() s_init_finish.value = 0 cond.release() return server_process
def call(args, stdout=None, stderr=None, stdin=None, daemonize=False, preexec_fn=None, shell=False, cwd=None, env=None): """ Run an external command in a separate process and detach it from the current process. Excepting `stdout`, `stderr`, and `stdin` all file descriptors are closed after forking. If `daemonize` is True then the parent process exits. All stdio is redirected to `os.devnull` unless specified. The `preexec_fn`, `shell`, `cwd`, and `env` parameters are the same as their `Popen` counterparts. Return the PID of the child process if not daemonized. """ stream = lambda s, m: s is None and os.open(os.devnull, m) or s stdout = stream(stdout, os.O_WRONLY) stderr = stream(stderr, os.O_WRONLY) stdin = stream(stdin, os.O_RDONLY) shared_pid = Value('i', 0) pid = os.fork() if pid > 0: os.waitpid(pid, 0) child_pid = shared_pid.value del shared_pid if daemonize: sys.exit(0) return child_pid else: os.setsid() proc = subprocess.Popen(args, stdout=stdout, stderr=stderr, stdin=stdin, close_fds=True, preexec_fn=preexec_fn, shell=shell, cwd=cwd, env=env) shared_pid.value = proc.pid os._exit(0)
def main(): #Begin Init settings.setup() from common.safeprint import safeprint try: import miniupnpc except: safeprint("Dependency miniupnpc is not installed. Running in outbound only mode") settings.config['outbound'] = True safeprint("settings are:") safeprint(settings.config) queue = Queue() live = Value('b',True) ear = listener(settings.config['port'],settings.config['outbound'],queue,live,settings.config['server']) ear.daemon = True ear.items = sync() ear.start() mouth = propagator(settings.config['port'] + 1, live) mouth.daemon = True mouth.items = ear.items mouth.start() feedback = [] stamp = time() while queue.empty(): if time() - 5 > stamp: break #pragma: no cover try: feedback = queue.get(False) except: #pragma: no cover safeprint("No feedback received from listener") ext_ip = "" #Does this affect peers? ext_port = -1 #Does this affect peers? if feedback != []: settings.outbound = feedback[0] if settings.outbound is not True: ext_ip = feedback[1] ext_port = feedback[2] initializePeerConnections(settings.config['port'], ext_ip, ext_port) #End Init #Begin main loop if settings.config.get('seed'): safeprint("Seed mode activated") try: while True and not settings.config.get('test'): sleep(0.1) except KeyboardInterrupt: safeprint("Keyboard Interrupt") elif settings.config.get('server'): safeprint("Server mode activated") else: safeprint("Client mode activated") #End main loop #Begin shutdown safeprint("Beginning exit process") live.value = False settings.saveSettings() saveToFile() bounty.saveToFile()
def main(): settings.setup() try: import miniupnpc except: safeprint("Dependency miniupnpc is not installed. Running in outbound only mode") settings.config['outbound'] = True safeprint("settings are:") safeprint(settings.config) queue = Queue() live = Value('b',True) ear = listener(settings.config['port'],settings.config['outbound'],queue,live,settings.config['server']) ear.daemon = True ear.start() feedback = [] stamp = time() while queue.empty(): if time() - 5 > stamp: break #pragma: no cover try: feedback = queue.get(False) except: #pragma: no cover safeprint("No feedback received from listener") ext_ip = "" #Does this affect peers? ext_port = -1 #Does this affect peers? if feedback != []: settings.outbound = feedback[0] if settings.outbound is not True: ext_ip = feedback[1] ext_port = feedback[2] initializePeerConnections(settings.config['port'], ext_ip, ext_port) live.value = False
def hanging_get(my_id, messages, initial_peer_id): remote_sender = None remote_peer_id = Value("i", initial_peer_id) remote_sender = Process(target=to_remote_server, args=(5550,my_id,remote_peer_id)).start() while True: r = requests.get('http://localhost:8888/wait?peer_id=' + str(my_id)) if r.status_code == 200: if int(r.headers['pragma']) == my_id: connected = r.text.split("\n") for l in connected: info = l.strip().split(",") if len(info) == 3 and info[0] == "receiver" and info[2] == '1': remote_peer_id.value = int(info[1]) if len(info) == 3 and info[0] == "receiver" and info[2] == '0': remote_peer_id.value = -1 else: messages.put(Message(int(r.headers['pragma']), r.text))
def main(): running = Value(c_int, 1) readQueue = Queue() reader = Process(target = Reader("/dev/ttyUSB0", 9600), args = (running, readQueue)) worker = Process(target = Worker(), args = (running, readQueue)) reader.start() worker.start() time.sleep(5) running.value = 0 reader.join() worker.join()
def camstream(): print "CAMera STREAMer (OpenCV " + cv2.__version__ + ")" print "main(): OS: {}".format(os.name) # * Start CameraStreamer process print "main(): Starting CameraStreamer process..." if os.name == 'nt': # [Windows] # ** Create shared objects (NOTE only necessary on Windows since it uses a different multiprocessing implementation) print "main(): [Windows] Creating shared objects..." # *** Stay alive flag stayAliveObj = Value(c_bool, True) # *** Frame counter frameCountObj = Value('i', 0) # *** Image array image = np.zeros((camera_frame_height, camera_frame_width, camera_frame_depth), dtype=np.uint8) imageShape = image.shape imageSize = image.size image.shape = imageSize # flatten numpy array imageObj = Array(c_ubyte, image) # create a synchronized shared array # *** Image shape imageShapeObj = Array('i', imageShape) cameraStreamerProcess = CameraStreamer(stayAliveObj, frameCountObj, imageObj, imageShapeObj) else: # [POSIX] cameraStreamerProcess = CameraStreamer() # ** Grab generated shared objects to share with other child processes print "main(): [POSIX] Getting shared objects from CameraStreamer..." stayAliveObj = cameraStreamerProcess.stayAliveObj frameCountObj = cameraStreamerProcess.frameCountObj imageObj = cameraStreamerProcess.imageObj imageShapeObj = cameraStreamerProcess.imageShapeObj cameraStreamerProcess.start() # * Start StreamViewer process print "main(): Starting StreamViewer process..." streamViewerProcess = StreamViewer(stayAliveObj, frameCountObj, imageObj, imageShapeObj) streamViewerProcess.start() # * Wait for child processes to finish print "main(): Waiting for child processes to finish..." try: streamViewerProcess.join() cameraStreamerProcess.join() except KeyboardInterrupt: stayAliveObj.value = False streamViewerProcess.join() cameraStreamerProcess.join() print "main(): Done."
def start(self, reload_from=None): """Start this server process. :param int reload_from: Optional, the PID of a running game server process that this process should reload from :returns None: """ assert not self._process, "server instance already started" pid = Value("i") self._process = Process(target=self._start, args=(pid, socket_queue), kwargs={"reload_from": reload_from}) self._process.start() pid.value = self._process.pid
def send_mldquery_regularly(self): self.logger.debug("") requraly_query_type = self.config[const.REGURALY_QUERY_TYPE] reguraly_query_interval = self.config[const.REGURALY_QUERY_INTERVAL] mc_query_interval = self.config[const.MC_QUERY_INTERVAL] # 初回送信前に定期送信クエリの送信間隔の1/4秒待つ time.sleep(reguraly_query_interval / 4) # General Query if requraly_query_type == self.GENERAL_QUERY: self.logger.debug("create general query") mc_info = {const.MC_TAG_MC_ADDR: const.DELIMIT_DOUBLE_COLON, const.MC_TAG_SERV_IP: None} while self.SEND_LOOP: self.send_mldquery([mc_info]) # タイムアウトチェック self.check_user_timeout() time.sleep(reguraly_query_interval - self.QUERY_QRV) # Specific Query elif requraly_query_type == self.SPECIFIC_QUERY: self.logger.debug("create specific query") next_interval = Value(ctypes.c_bool, False) while self.SEND_LOOP: query_proc = Process( target=self.wait_query_interval, args=(next_interval, reguraly_query_interval)) query_proc.daemon = True query_proc.start() self.logger.debug( "next_interval : %s", str(next_interval.value)) self.send_mldquery( self.mc_info_list, mc_query_interval, next_interval) # タイムアウトチェック self.check_user_timeout() # 定期送信クエリの送信間隔が過ぎていない場合は待ち if not next_interval.value: self.logger.debug( "waiting query interval(%d sec)...", reguraly_query_interval) query_proc.join() next_interval.value = False query_proc.terminate()
def main(): arg_parser = argparse.ArgumentParser() arg_parser.add_argument('-f', dest='logfile', required=True) arg_parser.add_argument('-t', dest='sleep', type=int, default=1) args = arg_parser.parse_args() count = Value('i', 0) p = Process(target=log_counter, args=(args.logfile, count,)) p.start() while True: time.sleep(args.sleep) print('{0} events/{1}s'.format(count.value, args.sleep)) count.value = 0 p.join()
def main(): guardian = Heimdall() # zoeh = Zoehmacarena() # guardian.add_bot(zoeh) # guardian.main_loop() cont = True ispaused = False # bobby = Bob() state = Value("i", 0) l = Lock() # proc = Process(target=bobby.main, args=(l, state,)) print("starting") # proc.start() while cont: if ispaused: command = raw_input("Heimdall[paused]: ") else: command = raw_input("Heimdall[running]: ") if command == "quit": cont = False state.value = -1 elif command == "pause": l.acquire() print("paused") ispaused = True elif command == "start": print("unpausing") l.release() ispaused = False else: print("unknown command") # proc.join() print("finished") return
def server(backlog=5): server_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server_socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) server_socket.bind(('', PORT)) server_socket.listen(backlog) nclients = Value('i', 0) processes = [] schedule.every(1).seconds.do(update_active_clients, nclients) try: while 1: conn, addr = server_socket.accept() print "new client " + str(addr) proc = Process(target=client_process, args=(conn, server_socket, nclients)) processes.append(proc) proc.start() conn.close() nclients.value = len(multiprocessing.active_children()) finally: # Ctrl-C for proc in processes: proc.terminate() server_socket.close()
def run_test(*args): # logins = get_bm_config('ADMINS') # logins.extend([get_bm_config('API_USER_PREFIX') % i # for i in xrange(args['proc'])]) # msg_count = get_bm_config('MSG_COUNT') # host = args['host'] # port = args['port'] # visit_args = [(login, host, port, msg_count) for login in logins] visit_args = [[i] for i in [1, 2]] term = Value(c_int, 0) queue = Queue() logging.debug('users pool') pool = Pool(initializer=init, initargs=(term, queue), processes=10, maxtasksperchild=1) try: swarm = pool.map(visit, [i for i in range(10)]) logging.debug('finishing...') except KeyboardInterrupt: logging.info('terminating...') term.value = 1 pool.terminate()
def main(): with open("/media/extradikke/UbuntuData/wikipedia_data/data_dump/enwiki-20150112-pages-articles.xml", mode='r') as big_file: fsize = os.stat( "/media/extradikke/UbuntuData/wikipedia_data/data_dump/enwiki-20150112-pages-articles.xml").st_size stream = pulldom.parse(big_file) # sys.setrecursionlimit(10000) redirect_queue = Queue(100) pages_queue = Queue(100) rawnodes_queue = Queue(1000) counter = 1 redirect_counter = 0 running = Value('b', True) link_extractor1 = Process(target=link_extractor, args=(rawnodes_queue, redirect_queue, pages_queue, running, )) link_extractor2 = Process(target=link_extractor, args=(rawnodes_queue, redirect_queue, pages_queue, running, )) # link_extractor3 = Process(target=link_extractor, args=(rawnodes_queue, redirect_queue, pages_queue, running, )) # link_extractor4 = Process(target=link_extractor, args=(rawnodes_queue, redirect_queue, pages_queue, running, )) xml_openeger = Process(target=opener, args=(rawnodes_queue, pages_queue, redirect_queue)) redirects = Process(target=redirect_saver, args=(redirect_queue, running,)) page_saver = Process(target=save_pages, args=(pages_queue, running,)) # xml_openeger.start() processes = [link_extractor1, link_extractor2, redirects, page_saver] for process in processes: process.start() xml_openeger.join() while True: print("here???") if redirect_queue.empty() and pages_queue.empty() and rawnodes_queue.empty(): running.value = False for process in processes: process.join() break
def _hash_result(self, cursor, timeout_unix_time): """Returns a hash that is independent of row order.""" # A value of 1 indicates that the hash thread should continue to work. should_continue = Value("i", 1) def hash_result_impl(): try: current_thread().result = 1 while should_continue.value: LOG.debug("Fetching result for query with id %s" % cursor._last_operation_handle) rows = cursor.fetchmany(self.BATCH_SIZE) if not rows: return for row in rows: for idx, val in enumerate(row): # Floats returned by Impala may not be deterministic, the ending # insignificant digits may differ. Only the first 6 digits will be used # after rounding. if isinstance(val, float): sval = "%f" % val dot_idx = sval.find(".") val = round(val, 6 - dot_idx) current_thread().result += (idx + 1) * hash(val) # Modulo the result to Keep it "small" otherwise the math ops can be slow # since python does infinite precision math. current_thread().result %= maxint except Exception as e: current_thread().error = e hash_thread = create_and_start_daemon_thread(hash_result_impl) hash_thread.join(max(timeout_unix_time - time(), 0)) if hash_thread.is_alive(): should_continue.value = 0 raise QueryTimeout() if hash_thread.error: raise hash_thread.error return hash_thread.result
def main(): # getting password psswd = gethash(getpass(), mode="pass") # decrypting and loading settings settings = fdecrypt("files/vk.settings", psswd) # print(settings) login = "".join(re.findall(r"login=(.+)#endlogin", settings)) password = "".join(re.findall(r"password=(.+)#endpass", settings)) chatid = int("".join(re.findall(r"chatid=(\d+)#endchatid", settings))) albumid = int("".join(re.findall(r"album_id=(\d+)#endalbumid", settings))) userid = int("".join(re.findall(r"userid=(\d+)#enduserid", settings))) state_auth = False # getting session while not state_auth: try: vk_session = vk_api.VkApi(login, password, captcha_handler=captcha_handler) vk_session.authorization() vk = vk_session.get_api() state_auth = True except: state_auth = False tsleep(30) exception("smth goes wrong at geting api\n") # getting url url = geturl(psswd) # Lists of messages msg_man = Manager() list_of_cmds, list_of_alles, list_of_imnts, tl_msgs = msg_man.list(), msg_man.list(), msg_man.list(), msg_man.list() # accounting the speed iterations_vk = Value("i", 0) iterations_tl = Value("i", 0) # file`s safe work msgshistory = io.SharedFile("files/msgshistory.db") # stats Manager stat_man = Manager() curr_stat = stat_man.dict() curr_stat["temp"] = 0 curr_stat["iter_tl"] = 0 curr_stat["iter_vk"] = 0 curr_stat["PID_VK"] = 0 curr_stat["PID_TL"] = 0 # starting bot print("Logged in, starting bot...") vk_process = Process( target=vk_run.run_vk_bot, args=( vk, chatid, albumid, userid, msgshistory, tl_msgs, list_of_alles, list_of_imnts, list_of_cmds, iterations_vk, curr_stat, ), ) tl_process = Process( target=telegrambot.tlmain, args=(url, tl_msgs, msgshistory, list_of_alles, list_of_imnts, iterations_tl, curr_stat), ) print("Starting VK bot...") vk_process.start() print("Starting TL bot...") tl_process.start() # checking if admin gave argument "--screen" if len(argv) > 1 and argv[1] == "--screen": import curses stdscr = curses.initscr() curses.noecho() stdscr.keypad(True) # crutch iterations_tl.value = 1 iterations_vk.value = 1 while True: out_string = """Temp: {0} C; \nSpeed_TL: {1}; \nSpeed_VK: {2};""" tempfile = open("/sys/class/thermal/thermal_zone0/temp", "r") # print('Temp: ' + str(float(tempfile.read().strip())/1000) + ' C ', end='') ctemp = str(float(tempfile.read().strip()) / 1000) tempfile.close() curr_stat["temp"] = ctemp curr_stat["iter_tl"] = iterations_tl.value curr_stat["iter_vk"] = iterations_vk.value if len(argv) > 1 and argv[1] == "--screen": stdscr.clear() stdscr.addstr(out_string.format(ctemp, iterations_tl.value, iterations_vk.value)) stdscr.refresh() # checking if process is alive. If not, restarting process if iterations_tl.value == 0: tl_process.terminate() tl_process = Process( target=telegrambot.tlmain, args=(url, tl_msgs, msgshistory, list_of_alles, list_of_imnts, iterations_tl, curr_stat), ) print("Restarting TL bot...") tl_process.start() # iterations_tl.value = 1 if iterations_vk.value == 0: vk_process.terminate() vk_process = Process( target=vk_run.run_vk_bot, args=( vk, chatid, albumid, userid, msgshistory, tl_msgs, list_of_alles, list_of_imnts, list_of_cmds, iterations_vk, curr_stat, ), ) print("Restarting VK bot...") vk_process.start() # iterations_vk.value = 1 # stdout.flush() iterations_tl.value = 0 iterations_vk.value = 0 tsleep(60) tl_process.join() vk_process.join() curses.nocbreak() stdscr.keypad(False) curses.echo() print("Exit...")
# Corps de la tache, elle s'execute tant qu'il y a des donnees dans la pile if __name__ == '__main__': # ihm = subprocess.Popen(['nmake', '/F']) # ihm.wait() # Ajout : Il faut aussi pour les projets gerer le fichier AppVersion.h appversion_update() # On doit continuer a attendre qu'il n'y ait plus de taches dans la pool conf = sys.argv[1] if(conf == '1'): config = 'Release' else: config = 'Debug' while( liste or ( running.value > 0 ) ): if( ( running.value < 5 ) and liste ): if(liste): cur = liste.pop() p = Process( target = nmake, args = (cur,running,config,) ) p.start() running.value = running.value + 1 # print "**Librairie en cours de fab :" # print cur # print "Il y a ",running.value," en cours" # raw_input() print "\nOn lance l edition des liens\n" cfg = 'CFG=\"UCINEO_NANCY_CE - Win32 (WCE x86) '+ config + '\"' os.system('nmake' + '/F' + 'UCINEO_NANCY_CE.vcn ' + cfg) # En appelant une commande systeme, on bloque l execution du script
def serve(): """ This is the server entry point, where a while loop receives messages, parses them, and acts accordingly. Actually data processing may be spawned from here as a separate Process. """ # initiallize our bookkeeping state = Value('i', READY) proj = None scanNum = None refScan = True refScanNum = None filename = None # setup the server REPLY socket port = PORT context = zmq.Context() socket = context.socket(zmq.REP) socket.bind("tcp://*:%s" % port) # initialize the Process object p = None # initialize our 'memory' of what scans we # have processed scans = {} # TBF: terminate with a server command as well? # run till a keyboard interrupt run = True while run: try: #logger.debug ("waiting for message") msg = socket.recv() #logger.debug ("got msg") #logger.debug (msg) msgStr = "".join( chr(x) for x in msg) # ************ Process Data if msg == b"process": if state.value == READY: # record the current state of all parameters so # we know what scans are being processed filepath = getFITSFilePath(proj, filename) if proj not in scans: scans[proj] = {} scans[proj][scanNum] = { "scanNum": scanNum, "timestamp": datetime.now(), "refScan": refScan, "refScanNum": refScanNum, "filename": filename, "filepath": filepath, "filepathSmoothed": filepath.replace(".fits", ".smoothed.fits") } logger.debug("scans list: %s" % scans) if not refScan: # this is a signal scan, so get the filename of our # reference scan refScanFile = getRefScanFileName(scans, proj, refScanNum) else: refScanFile = None logger.debug("processing!") p = Process(target=process, args=(state, proj, scanNum, refScan, refScanNum, refScanFile, filename)) p.start() #state.value = PROCESSING # socket.send_string("Started Processing") logger.debug("Started Processing") socket.send_string("OK") else: logger.debug("processing already!") socket.send_string("Already processing") # ******** STOP processing data elif msg == b"stop": if state.value not in PROCESS_STATES: socket.send_string("Nothing to stop") else: if p is not None: logger.debug ("terminating process") logger.debug (stateMap[state.value]) p.terminate() state.value = READY logger.debug ("process terminated") socket.send_string("OK") else: logger.debug ("can't terminate, p is none") socket.send_string("can't terminate, p is none") # ************ return our current STATE elif msg == b"get_state": # or msg == "get_state": #socket.send_string("READY" if state.value is 0 else "PROCESSING") socket.send_string(stateMap[state.value]) # ************* SET a parameter elif msgStr[:4] == "set:": # set what to what? # set: key=value ps = msgStr[4:].split("=") if len(ps) != 2: socket.send_string("Cant understand: %s" % msgStr) else: key = ps[0] value = ps[1] if key == 'proj': proj = value elif key == 'scanNum': scanNum = int(value) elif key == 'refScan': # TBF: settle on bool or int type? # refScan = value == 'True' refScan = int(value) == 1 elif key == 'refScanNum': refScanNum = int(value) elif key == 'filename': filename = value else: logger.debug("unknonw key: %s" % key) # socket.send_string("setting %s to %s" % (key, value)) logger.debug("setting %s to %s" % (key, value)) socket.send_string("OK") # ********* RAISE an ERROR! else: logger.debug("what?") logger.debug(msg) socket.send_string("Dont' understand message") # no need for the server to spin too fast time.sleep(1) except KeyboardInterrupt: # exit gracefully logger.debug("KeyboardInterrupt") if p is not None: p.terminate() # socket.send_string("server exiting") run = False logger.debug("Exiting server")
help='interactive mode') parser.add_argument('-o', '--output', type=str, default="", help="the output file") parser.add_argument('file', metavar="FILE", help="the input file to convert") parser.add_argument('-d', '--dependencies', type=str, default="", help="the dependencies of the file to convert") args = parser.parse_args() if args.interactive: if not args.output: print("The ouput file is not optional when the interactive mode is activated") exit() dep = [f for f in args.dependencies.split(",") if f] print("Interactive mode activated " "(Press [enter] to quit).\n" "Waiting for modifications...") end_inter_mode = Value("i", False) p = Process(target=interactive, args=(args.file, dep, args.output, end_inter_mode)) try: p.start() input("") finally: end_inter_mode.value = True p.join() else: txt, frames = convert(args.file, root=True) if args.output: with open(args.output, "w") as f: f.write(txt) else: print(txt)
odo_cov_stack = [] gps_cov_stack = [] final_cov_stack = [] data_logger_odo = [] data_logger_gps = [] data_logger_eky = [] ################################################### # GLobal readness am_i_ready = False # handles terminating other threads and process global_finish = False # used across multiple running process global_finish_process = Value('i', 0) global_finish_process.value = 0 weAreDoneHere = False # for debugging Program_DEBUG = True ### GPS Server related variables LINETHRESHOLD = 800 reduceFactor = 0.35 # robot mode # mode 0 - Manual control # mode 1 - GPS control # mode 2 - Line Following robot_mode = 0
def read_barcode(): num = Value('i', 1) result_queue = Queue(1) key_queue = Queue(1) frame_queue = Queue(1) cond = Condition() dbr_proc = Process(target=dbr_run, args=( frame_queue, key_queue, cond, num, result_queue)) dbr_proc.start() vc = cv2.VideoCapture(0) vc.set(3, 640) #set width vc.set(4, 480) #set height if vc.isOpened(): # try to get the first frame rval, frame = vc.read() else: return windowName = "Robot View" try: while True: rval, frame = vc.read() cv2.imshow(windowName, frame) key = cv2.waitKey(1) & 0xFF if key == ord('q'): key_queue.put(key) dbr_proc.join() break elif key == ord('c'): frame_queue.put(frame) key_queue.put(key) elif key == ord('a') or key == ord('d') or key == ord('w') or key == ord('s') or key == ord('p'): key_queue.put(key) try: ret = result_queue.get_nowait() results = ret.results image = ret.image thickness = 2 color = (0,255,0) for result in results: print("barcode format: " + result[0]) print("barcode value: " + result[1]) x1 = result[2] y1 = result[3] x2 = result[4] y2 = result[5] x3 = result[6] y3 = result[7] x4 = result[8] y4 = result[9] cv2.line(image, (x1, y1), (x2, y2), color, thickness) cv2.line(image, (x2, y2), (x3, y3), color, thickness) cv2.line(image, (x3, y3), (x4, y4), color, thickness) cv2.line(image, (x4, y4), (x1, y1), color, thickness) cv2.putText(image, result[1], (min([x1, x2, x3, x4]), min([y1, y2, y3, y4])), cv2.FONT_HERSHEY_SIMPLEX, 1, (0,0,255), thickness) cv2.imshow("Localization", image) except: pass finally: num.value = 0 vc.release() cv2.destroyAllWindows()
def video_feed_counter(conf, mode, input, output, url, camera): # construct the argument parser and parse the arguments # load the configuration file conf = Conf(conf) count = 0 # initialize the MOG foreground background subtractor object # mog = cv2.bgsegm.createBackgroundSubtractorMOG() mog = cv2.createBackgroundSubtractorMOG2() # initialize and define the dilation kernel dKernel = cv2.getStructuringElement(cv2.MORPH_RECT, (3, 3)) # initialize the video writer process writerProcess = None # initialize the frame dimensions (we'll set them as soon as we read # the first frame from the video) W = None H = None # instantiate our centroid tracker and initialize a dictionary to # map each unique object ID to a trackable object ct = CentroidTracker(conf["max_disappeared"], conf["max_distance"]) trackableObjects = {} # if a video path was not supplied, grab a reference to the webcam # if not args.get("input", False): # if input: # print("[INFO] starting video stream...") # # vs = VideoStream(src=0).start() # vs = VideoStream(usePiCamera=True).start() # time.sleep(2.0) # otherwise, grab a reference to the video file # else: print("[INFO] opening video file...") vs = cv2.VideoCapture(url, cv2.CAP_FFMPEG) # vs = cv2.VideoCapture(args["input"]) # check if the user wants to use the difference flag feature if conf["diff_flag"]: # initialize the start counting flag and mouse click callback start = False cv2.namedWindow("set_points") cv2.setMouseCallback("set_points", set_points, [mode]) # otherwise, the user does not want to use it else: # set the start flag as true indicating to start traffic counting start = True # initialize the direction info variable (used to store information # such as up/down or left/right vehicle count) and the difference # point (used to differentiate between left and right lanes) directionInfo = None diffPt = None fps = FPS().start() # print('fbs') # loop over frames from the video stream while (vs.isOpened()): # grab the next frame and handle if we are reading from either # VideoCapture or VideoStream # frame = vs.read() ret, frame = vs.read() # import image # if not ret: # frame = cv2.VideoCapture(url) # continue # if ret: # frame = cv2.VideoCapture(url) # continue # if we are viewing a video and we did not grab a frame then we # have reached the end of the video if input is not None and frame is None: break #print("frame in while") # check if the start flag is set, if so, we will start traffic # counting if start: # if the frame dimensions are empty, grab the frame # dimensions, instantiate the direction counter, and set the # centroid tracker direction if W is None or H is None: # start the frames per second throughput estimator #fps = FPS().start() (H, W) = frame.shape[:2] dc = DirectionCounter(mode, W - conf["x_offset"], H - conf["y_offset"]) ct.direction = mode # check if the difference point is set, if it is, then # set it in the centroid tracker object if diffPt is not None: ct.diffPt = diffPt # begin writing the video to disk if required if output is not None and writerProcess is None: # set the value of the write flag (used to communicate when # to stop the process) writeVideo = Value('i', 1) # initialize a shared queue for the exhcange frames, # initialize a process, and start the process frameQueue = Queue() writerProcess = Process(target=write_video, args=(output, writeVideo, frameQueue, W, H)) writerProcess.start() # initialize a list to store the bounding box rectangles # returned by background subtraction model rects = [] # convert the frame to grayscale image and then blur it gray = cv2.cvtColor(frame, cv2.COLOR_BGR2GRAY) gray = cv2.GaussianBlur(gray, (5, 5), 0) # apply the MOG background subtraction model which returns # a mask mask = mog.apply(gray) # apply dilation dilation = cv2.dilate(mask, dKernel, iterations=2) # find contours in the mask cnts = cv2.findContours(dilation.copy(), cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE) cnts = imutils.grab_contours(cnts) # loop over each contour for c in cnts: # if the contour area is less than the minimum area # required then ignore the object if cv2.contourArea(c) < conf["min_area"]: continue # get the (x, y)-coordinates of the contour, along with # height and width (x, y, w, h) = cv2.boundingRect(c) # check if direction is *vertical and the vehicle is # further away from the line, if so then, no need to # detect it if mode == "vertical" and y < conf["limit"]: continue # otherwise, check if direction is horizontal and the # vehicle is further away from the line, if so then, # no need to detect it elif mode == "horizontal" and x > conf["limit"]: continue # add the bounding box coordinates to the rectangles list rects.append((x, y, x + w, y + h)) # check if the direction is vertical if mode == "vertical": # draw a horizontal line in the frame -- once an object # crosses this line we will determine whether they were # moving 'up' or 'down' cv2.line(frame, (0, H - conf["y_offset"]), (W, H - conf["y_offset"]), (0, 255, 255), 2) # check if a difference point has been set, if so, draw # a line diving the two lanes if diffPt is not None: cv2.line(frame, (diffPt, 0), (diffPt, H), (255, 0, 0), 2) # otherwise, the direction is horizontal else: # draw a vertical line in the frame -- once an object # crosses this line we will determine whether they were # moving 'left' or 'right' # print('ddds') cv2.line(frame, (W - conf["x_offset"], 0), (W - conf["x_offset"], H), (0, 255, 255), 2) # check if a difference point has been set, if so, draw a # line dividing the two lanes if diffPt is not None: cv2.line(frame, (0, diffPt), (W, diffPt), (255, 0, 0), 2) # use the centroid tracker to associate the (1) old object # centroids with (2) the newly computed object centroids objects = ct.update(rects) # loop over the tracked objects for (objectID, centroid) in objects.items(): # check to see if a trackable object exists for the # current object ID and initialize the color to = trackableObjects.get(objectID, None) color = (0, 0, 255) # create a new trackable object if needed if to is None: to = TrackableObject(objectID, centroid) # otherwise, there is a trackable object so we can # utilize it to determine direction else: # find the direction and update the list of centroids dc.find_direction(to, centroid) to.centroids.append(centroid) # check to see if the object has been counted or not if not to.counted: # find the direction of motion of the vehicles directionInfo = dc.count_object(to, centroid, camera) # otherwise, the object has been counted and set the # color to green indicate it has been counted else: color = (0, 255, 0) # store the trackable object in our dictionary trackableObjects[objectID] = to # draw both the ID of the object and the centroid of the # object on the output frame text = "ID {}".format(objectID) cv2.putText(frame, text, (centroid[0] - 10, centroid[1] - 10), cv2.FONT_HERSHEY_SIMPLEX, 0.5, color, 2) cv2.circle(frame, (centroid[0], centroid[1]), 4, color, -1) # extract the traffic counts and write/draw them if directionInfo is not None: for (i, (k, v)) in enumerate(directionInfo): text = "{}: {}".format(k, v) cv2.putText(frame, text, (10, ((i * 20) + 20)), cv2.FONT_HERSHEY_SIMPLEX, 0.6, (0, 0, 255), 2) # put frame into the shared queue for video writing if writerProcess is not None: frameQueue.put(frame) # show the output frame # cv2.imshow("Frame", frame) frames = cv2.imencode('.jpg', frame)[1].tobytes() yield (b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + frames + b'\r\n') key = cv2.waitKey(1) & 0xFF # if the `q` key was pressed, break from the loop if key == ord("q"): break # update the FPS counter fps.update() # otherwise, the user has to select a difference point else: # show the output frame # cv2.imshow("set_points", frame) frames = cv2.imencode('.jpg', frame)[1].tobytes() yield (b'--frame\r\n' b'Content-Type: image/jpeg\r\n\r\n' + frames + b'\r\n') key = cv2.waitKey(1) & 0xFF # if the `s` key was pressed, start traffic counting if key == ord("s"): # begin counting and eliminate the informational window start = True cv2.destroyWindow("set_points") # stop the timer and display FPS information fps.stop() print("[INFO] elapsed time: {:.2f}".format(fps.elapsed())) print("[INFO] approx. FPS: {:.2f}".format(fps.fps())) # terminate the video writer process if writerProcess is not None: writeVideo.value = 0 writerProcess.join() # if we are not using a video file, stop the camera video stream # if not args.get("input", False): # vs.stop() # otherwise, release the video file pointer else: vs.release() # close any open windows cv2.destroyAllWindows()
def baumWelchP(bitext_sd, s_count, t_table, sd_count): #L is the number of observations N = maxTargetSentenceLength(bitext_sd) print 'N', N N_max = N Y = bitext_sd yValues = s_count.keys() #computes all possible english words (indexMap, biword) = map_bitext_to_int(sd_count) sd_size = len(indexMap) lastLikelihood = 0 #a = zeros((N+1,N+1)) ''' a_array = Array(ct.c_double,((N+1)*(N+1))) a_array2 = np.frombuffer(a_array.get_obj()) # mp_arr and arr share the same memory a = a_array2.reshape((N+1,N+1)) # b and arr share the same memory ''' manager = Manager() a = manager.dict() #pi = zeros(N+1) pi = Array('d', N + 1) logLikelihood = Value('d', 0.0) lastLogLikelihood = Value('d', 0.0) L = len(Y) #N = len(Y[0][1]) #first sentence x length #(a,pi) = initializeUniformly(N) for iterations in range(0, 10): #E step #c = defaultdict(int) startTime = time.time() print 'iteration', iterations logLikelihood.value = 0.0 #totalGammaOverAllObservations = zeros(N+1) totalGammaOverAllObservations = Array('d', [0] * (N + 1)) #totalGammaDeltaOverAllObservations_t = zeros((N+1,sd_size)) totalGammaDeltaOverAllObservations_t = Array('d', [0] * ((N + 1) * (sd_size))) #totalGammaDeltaOverAllObservations_t_overall_states = zeros(sd_size) totalGammaDeltaOverAllObservations_t_overall_states = Array( 'd', [0] * sd_size) totalGammaDeltaOverAllObservations_t_overall_states_over_dest = defaultdict( int) #totalGamma1OverAllObservations = zeros(N+1) totalGamma1OverAllObservations = Array('d', [0] * (N + 1)) #totalC_j_Minus_iOverAllObservations = zeros((N+1,N+1)) ''' totalC_j_Minus_iOverAllObservations_array = Array(ct.c_double,(N+1)*(N+1)) totalC_j_Minus_iOverAllObservations_array2 = np.frombuffer(totalC_j_Minus_iOverAllObservations_array.get_obj()) totalC_j_Minus_iOverAllObservations = totalC_j_Minus_iOverAllObservations_array2.reshape((N+1,N+1)) for i in range(N+1): for j in range(N+1): totalC_j_Minus_iOverAllObservations[i,j] = 0.0 ''' manager = Manager() totalC_j_Minus_iOverAllObservations = manager.dict() #totalC_l_Minus_iOverAllObservations = zeros(N+1) ''' totalC_l_Minus_iOverAllObservations = Array('d',[0]*(N+1)) ''' manager2 = Manager() totalC_l_Minus_iOverAllObservations = manager2.dict() intervals = 10 jobs = [] lock = RLock() length_of_interval = L / intervals for i in range(0, intervals - 1): start = i * length_of_interval end = (i + 1) * length_of_interval #print start #print end p = Process(target=Expectation2, args=(lock, t_table, N, Y, sd_size, indexMap, iterations, totalGammaOverAllObservations, totalGammaDeltaOverAllObservations_t, totalGamma1OverAllObservations, totalC_j_Minus_iOverAllObservations, totalC_l_Minus_iOverAllObservations, start, end, a, pi, logLikelihood, lastLogLikelihood)) p.start() jobs.append(p) start = (intervals - 1) * length_of_interval end = L p = Process(target=Expectation2, args=(lock, t_table, N, Y, sd_size, indexMap, iterations, totalGammaOverAllObservations, totalGammaDeltaOverAllObservations_t, totalGamma1OverAllObservations, totalC_j_Minus_iOverAllObservations, totalC_l_Minus_iOverAllObservations, start, end, a, pi, logLikelihood, lastLogLikelihood)) p.start() jobs.append(p) for p in jobs: p.join() endTime = time.time() #print N print "%.2gs" % (endTime - startTime) #N = len(totalGamma1OverAllObservations)-1 #print N print 'last , new ', lastLogLikelihood.value, logLikelihood.value #print 'likelihood difference ', (logLikelihood.value - lastLogLikelihood.value) lastLogLikelihood.value = logLikelihood.value totalGammaOverAllObservationsOverAllStates = 0.0 sartTime = time.time() for i in range(1, N + 1): totalGammaOverAllObservationsOverAllStates += totalGammaOverAllObservations[ i] print 'likelihood ', logLikelihood #lastLikelihood = liklihood N = len(totalGamma1OverAllObservations) - 1 #print N # Create expected_counts(d,c) to be consistent with the Berg-Kirkpatrick et al. # To make it more memory efficient just keep either totalGammaDeltaOverAllObservations_t_overall_states or expected_counts expected_counts = defaultdict(int) for i in range(1, N + 1): for k in range(sd_size): address = i * sd_size + k totalGammaDeltaOverAllObservations_t_overall_states[ k] += totalGammaDeltaOverAllObservations_t[address] (f, e) = biword[k] totalGammaDeltaOverAllObservations_t_overall_states_over_dest[ e] += totalGammaDeltaOverAllObservations_t[address] for k in range(sd_size): (f, e) = biword[k] expected_counts[( f, e)] = totalGammaDeltaOverAllObservations_t_overall_states[k] totalGammaOverAllObservationsOverAllStates = 0.0 for i in range(1, N + 1): totalGammaOverAllObservationsOverAllStates += totalGammaOverAllObservations[ i] # M Step # We can clear a, b and pi here and then set the values for them #a = zeros((N+1,N+1)) manager = Manager() a = manager.dict() pi = zeros(N + 1) t_table = defaultdict(int) for i in range(1, N + 1): pi[i] = totalGamma1OverAllObservations[i] * (1.0 / L) ''' for j in range(1,N+1): a[(i,j)] = totalC_j_Minus_iOverAllObservations[(i,j)]/totalC_l_Minus_iOverAllObservations[i] ''' #a[(i,j,I)] = p(j|i,I) #print 'totalC_j_Minus_iOverAllObservations ', totalC_j_Minus_iOverAllObservations totalC_j_Minus_iOverAllObservations = dict( totalC_j_Minus_iOverAllObservations) for (i, j, I) in totalC_j_Minus_iOverAllObservations: #print x a[(i, j, I)] = totalC_j_Minus_iOverAllObservations[ (i, j, I)] / totalC_l_Minus_iOverAllObservations[(i, I)] #print 'a ', a #check_probability(a, N) for k in range(sd_size): (f, e) = biword[k] t_table[( f, e )] = totalGammaDeltaOverAllObservations_t_overall_states[ k] / totalGammaDeltaOverAllObservations_t_overall_states_over_dest[ e] print iterations return (a, t_table, pi)
def transfer_coin(): """ transfer money Arg: {"kribzz": {"transactionId":"643427363","streeAddress":"700 Rodeo Drive","cityName":"Beverly Hills", "stateCode":"CA","zipCode":"90210","latitude":"34.079678","longitude":"-118.413515", "transactionDateTime":"1523855778", "transactionTotal":"9975000.00", "sellerName":"John Clark", "buyerName":"Emily Stevens", }, "transfer": { "amount":12.54321, "destination_address":"ckbzzBGZpWBZiPyQmxN3NCK8DJ8S37Vhxb69a9pbYXgMAwKKPDGeZ93aZgSXfX1E3GMEbk6tgLGPK8gDAeGquLmASXKvRim7pzN", }, "smart_contract": { "agent_signature":"ckbzzBGZpWBZiPyQmxN3NCK8DJ8S37Vhxb69a9pbYXgMAwKKPDGeZ93aZgSXfX1E3GMEbk6tgLGPK8gDAeGquLmASXKvRim7pzN", "agent_pkey":"ckbzzBGZpWBZiPyQmxN3NCK8DJ8S37Vhxb69a9pbYXgMAwKKPDGeZ93aZgSXfX1E3GMEbk6tgLGPK8gDAeGquLmASXKvRim7pzN", "investor_signature":"ckbzzBGZpWBZiPyQmxN3NCK8DJ8S37Vhxb69a9pbYXgMAwKKPDGeZ93aZgSXfX1E3GMEbk6tgLGPK8gDAeGquLmASXKvRim7pzN", "investor_pkey":"ckbzzBGZpWBZiPyQmxN3NCK8DJ8S37Vhxb69a9pbYXgMAwKKPDGeZ93aZgSXfX1E3GMEbk6tgLGPK8gDAeGquLmASXKvRim7pzN", "owner_signature":"ckbzzBGZpWBZiPyQmxN3NCK8DJ8S37Vhxb69a9pbYXgMAwKKPDGeZ93aZgSXfX1E3GMEbk6tgLGPK8gDAeGquLmASXKvRim7pzN", "owner_pkey":"ckbzzBGZpWBZiPyQmxN3NCK8DJ8S37Vhxb69a9pbYXgMAwKKPDGeZ93aZgSXfX1E3GMEbk6tgLGPK8gDAeGquLmASXKvRim7pzN", }, } Return: reply(JSON) {"reply": {"msgId":cursor.lastrowid,"error":{"errorCode":PSI.errorCode,"errorMessage":PSI.errorMessage}} } """ psi_log_info(request.url) psi_log_info("POST: %s" % request.POST.dict) kr_data = None try: kr_data = parse_request("kribbz") # for key,val in kr_data.items(): # d = kr_data[key] # psi_log_debug( str(d) + ' = ' + str(val)) except Exception as errtxt: psi_log_error(str(errtxt)) pass print(kr_data) d_address = None amnt = None kr_amnt = None wallet = None pwd = None fee = 100000 mixin = 0 try: kr_amnt = parse_request("transfer") # for key,val in kr_data.items(): # d = kr_data[key] # psi_log_debug( str(d) + ' = ' + str(val)) amnt = kr_amnt["amount"] d_address = kr_amnt["destination_address"] fee = kr_amnt["fee"] mixin = kr_amnt["mixin"] except Exception as errtxt: psi_log_error(str(errtxt)) pass print(amnt, d_address) sec = None try: sec = parse_request("wallet") for key, val in sec.items(): d = sec[key] # psi_log_debug( str(d) + ' = ' + str(val)) pwd = sec["password"] wallet = sec["wallet_name"] except Exception as errtxt: psi_log_error(str(errtxt)) pass print(pwd, wallet) # run_wallet(wallet, pwd) from subprocess import Popen, PIPE from tempfile import SpooledTemporaryFile as tempfile f = tempfile() f.write('exit\n') f.seek(0) # Start wallet # print Popen(['/bin/grep','f'],stdout=PIPE,stdin=f).stdout.read() #./simple_wallet --wallet-file ./wal2 --password Vatson2008 --rpc-bind-port 18082 # ./simple_wallet --wallet-file /opt/kribbz/kribbz_wallet --password Password12345 --rpc-bind-port 18082 run_folder = APP_FOLDER cmd1 = "{0}simple_wallet".format(run_folder) cmd2 = "--wallet-file" cmd3 = "{0}{1}".format(WALLET_FOLDER, wallet) cmd4 = "--password" cmd5 = "{0}".format(pwd) cmd6 = "--rpc-bind-port" cmd7 = "18082" # proc = Popen([cmd1,cmd2,cmd3,cmd4,cmd5,cmd6,cmd7],stdout=PIPE,stdin=f).stdout.read() # out = prx.stdout.read() # print(out) # import sys # from subprocess import * # cmd = "{0}simple_wallet --wallet-file {1}{2} --password={3} --rpc-bind-port 18082".format(run_folder, WALLET_FOLDER, wallet, pwd) # proc = Popen(cmd, shell=True, stdout=PIPE) # cnt = 0 # while True: # cnt += 1 # if cnt > 40: # break # data = proc.stdout.readline() # Alternatively proc.stdout.read(1024) # if len(data) == 0: # break # sys.stdout.write(data) # sys.stdout.buffer.write(data) on Python 3.x s_stop = Value('d', 0.0) ret = start_wallet(run_folder, wallet, pwd, s_stop) if not ret: rez2 = { "tx_hash": None, "msg": 'wallet error', "error": True, "success": False } return (json.dumps(rez2, indent=4)) # simple wallet is running on the localhost and port of 18082 url = WALLET_URL # "http://localhost:18082/json_rpc" # standard json header headers = {'content-type': 'application/json'} # destination_address = "ckbzzBr9AquP7taMtmySt5PLPRAAxoC9ueGu2bqgveWf62d9X8DawW4gHBxmjLBuYDF5vspjRoQU37SgFYbHAKTS83vfWonzRKq" #wal5 destination_address = d_address # destination_address = "ckbzz7dYVt6T4sBWhjp8wqQUMgoE8hWyfFCcBjjdLrzEdJU9SQsXjm73rb4VzZt7RxGzuLCAs4ZtQQMJzeELHP841TbYDt4Rm9V" # wal6 # amount of xmr to send print("amnt=", amnt) amount = float(amnt) # amount = 12.54321 # cryptonote amount format is different then # that normally used by people. # thus the float amount must be changed to # something that cryptonote understands int_amount = int(get_amount(amount)) # just to make sure that amount->coversion->back # gives the same amount as in the initial number t_amount = float(get_money(str(int_amount))) print(t_amount, amount, int_amount) assert amount == float(get_money( str(int_amount))), "Amount conversion failed" # send specified xmr amount to the given destination_address recipents = [{"address": destination_address, "amount": int_amount}] # using given mixin # mixin = 0 # get some random payment_id payment_id = get_payment_id() if kr_data: kr1 = kr_data kribbz_info = json.dumps(kr1) # simplewallet' procedure/method to call rpc_input = { "method": "transfer", "params": { "destinations": recipents, "mixin": mixin, "kribbz_info": kribbz_info, "get_tx_key": True, "fee": fee, }, } else: rpc_input = { "method": "transfer", "params": { "destinations": recipents, "mixin": mixin, "get_tx_key": True, "fee": fee, }, } # add standard rpc values rpc_input.update({"jsonrpc": "2.0", "id": "0"}) try: # execute the rpc request response = requests.post(url, data=json.dumps(rpc_input), headers=headers) # rez = json.dumps(response.json()) rez = response.json() # f.close() save_wallet() time.sleep(2) # sleep s_stop.value = 1 stop_wallet() success = True except Exception as ex1: out = 'coin transfer not successful' error = "simple wallet not started" success = False tx_hash = None # {u'jsonrpc': u'2.0', u'id': u'0', u'result': {u'tx_hash': u'00000000000000001701000000000000320e5717afca5e5e0000000000000000'}} if success: try: tx_hash = rez[u'result'][u'tx_hash'] except: tx_hash = None if tx_hash is not None: out = 'coin transfer successful' success = True error = "0" else: out = 'coin transfer not successful' error = rez success = False rez2 = {"tx_hash": tx_hash, "msg": out, "error": error, "success": success} print(rez2) return (json.dumps(rez2, indent=4))
def main(): # use fixed random state rand_state = np.random.RandomState(1).get_state() np.random.set_state(rand_state) tf_set_seeds(np.random.randint(1, 2**31 - 1)) # Create UR5 Reacher2D environment env = ReacherEnv(setup="UR5_default", host=None, dof=2, control_type="velocity", target_type="position", reset_type="zero", reward_type="precision", derivative_type="none", deriv_action_max=5, first_deriv_max=2, accel_max=1.4, speed_max=0.3, speedj_a=1.4, episode_length_time=4.0, episode_length_step=None, actuation_sync_period=1, dt=0.04, run_mode="multiprocess", rllab_box=False, movej_t=2.0, delay=0.0, random_state=rand_state) env = NormalizedEnv(env) # Start environment processes env.start() # Create baselines TRPO policy function sess = U.single_threaded_session() sess.__enter__() def policy_fn(name, ob_space, ac_space): return MlpPolicy(name=name, ob_space=ob_space, ac_space=ac_space, hid_size=32, num_hid_layers=2) # Create and start plotting process plot_running = Value('i', 1) shared_returns = Manager().dict({ "write_lock": False, "episodic_returns": [], "episodic_lengths": [], }) # Spawn plotting process pp = Process(target=plot_ur5_reacher, args=(env, 2048, shared_returns, plot_running)) pp.start() # Create callback function for logging data from baselines TRPO learn kindred_callback = create_callback(shared_returns) # Train baselines TRPO learn(env, policy_fn, max_timesteps=150000, timesteps_per_batch=2048, max_kl=0.05, cg_iters=10, cg_damping=0.1, vf_iters=5, vf_stepsize=0.001, gamma=0.995, lam=0.995, callback=kindred_callback) # Safely terminate plotter process plot_running.value = 0 # shutdown ploting process time.sleep(2) pp.join() env.close()
finish_count = Value('i', 1) if (node_number == 0): # Start spawning the server server_p = Process(target=server_process, args=( rabbitmq_go, finish_count, )) rabbit_p = Process(target=cmd_process, args=("rabbitmq-server", )) index_p = Process(target=cmd_process, args=("index_server .", )) server_p.start() rabbit_p.start() sleep(2) index_p.start() sleep(0.5) rabbitmq_go.value = 1 else: print "I AM CLIENT " + hostname context = zmq.Context() socket = context.socket(zmq.REQ) socket.connect("tcp://" + servername + ":5566") response = False while not response: sleep(0.5) socket.send("Awake?") message = socket.recv() response = (message == "Yes") # Starting image2plate jobs
a = a['prices'] a.pop() plt.plot(a) plt.show() print("end") if __name__ == '__main__': print("Hello") running = Value('b', True) maxRequests = 4 print('Initialization of the weather') a = Array('f', range(3)) w = weather.Weather(a, running, 1) q = Queue(maxRequests) n = 12 pol = CVOUKIVOI h = Process(target=homes.homes, args=(a, q, running, n, pol)) m = market.Market(q, running, 2) m.start() w.start() h.start() print("gogogo") if input("Press any key") is not None: running.value = False print('STOPP') w.join() h.join() m.join() graph()
subject_cert = asymmetric.load_certificate(base64.b64decode(rawCert)) builder = OCSPRequestBuilder(subject_cert, issuerCerts[url]) ocsp_request = builder.build() url = urlparse(url) headers = {} conn = httplib.HTTPConnection(url.netloc) conn.request("POST", url.path, ocsp_request.dump(), headers) res = conn.getresponse().read() ocspResponseClass = ocsp.OCSPResponse.load(res) return (ocspResponseClass.response_data['responses'][0]['cert_status'].name != 'good') if __name__ == '__main__': q = Queue(workers * 16) check_finish = Value('i', 0) for i in range(workers): p = Process(target=doWork, args=(i, q, check_finish)) p.start() try: ctr = 0 for cert in open(infile, 'r'): q.put(cert) ctr += 1 if (ctr % 10000 == 0): print(str(ctr) + " certificates processed") check_finish.value = 1 print("End of put certificates into queue") except KeyboardInterrupt: sys.exit(1)
def run_server(tcp_ip, tcp_port, stream, password_range): q = JoinableQueue() found = Value('b', False) # Create a process responsible for generating password t = Process(target=generate, name="Password Generator", args=(q, password_range)) t.daemon = True t.start() # Run a socket listening for clients' hearthbeats t = threading.Thread(target=hearthbeat, name="Hearthbeat", args=(tcp_ip, found)) t.daemon = True t.start() # A separate thread to remove inactive clients t = threading.Thread(target=remove_inactive_clients, name="Client clean-up", args=(q, )) t.daemon = True t.start() # Initiate the listening socket on given port try: server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) server.bind((tcp_ip, tcp_port)) server.listen(5) except socket.error as ex: print "Error opening socket:", ex return global counter counter = 0 # Is used to keep track of currently processed password by clients. In case a client is invalidated # the data has to resend to another client global processed_passwords processed_passwords = {} start_time = time.time() # The server is terminated once the password is found, there are no more active clients and no # passwords are left to try or a KeyboardInterrupt (Ctrl + C) is received while True: print "Number of clients: " + str(len(clients)) print "Estimated speed: " + str( 1 / (time.time() - start_time) * counter) + " H/sec" message = prepare_data_for_transfer(q, stream) # There are no more passwords to send, but he have to wait for all clients to finish if (not message and not clients): print "Password is not in brute-forced space." server.close() return try: client, address = server.accept() except KeyboardInterrupt: print "Stoping server..." server.close() return # It is possible the queue was filled with passwords from inactivated clients # while the server was waiting for new connections if (not message): message = prepare_data_for_transfer(q, stream) result, counter = handle_connection(client, address, message) if (result): with found.get_lock(): found.value = True server.close() return
def __init__(self, model, device_ids=1, n_workers=None, max_batch_size=None, max_image_size=DEFAULT_MAX_IMAGE_SIZE, modder=None): """ Args: - model (PyMjModel): MuJoCo model to use for rendering - device_ids (int/list): list of device ids to use for rendering. One or more workers will be assigned to each device, depending on how many workers are requested. - n_workers (int): number of parallel processes in the pool. Defaults to the number of device ids. - max_batch_size (int): maximum number of states that can be rendered in batch using .render(). Defaults to the number of workers. - max_image_size (int): maximum number pixels in images requested by .render() - modder (Modder): modder to use for domain randomization. """ self._closed, self.pool = False, None if not (modder is None or inspect.isclass(modder)): raise ValueError("modder must be a class") if isinstance(device_ids, int): device_ids = list(range(device_ids)) else: assert isinstance(device_ids, list), ( "device_ids must be list of integer") n_workers = n_workers or 1 self._max_batch_size = max_batch_size or (len(device_ids) * n_workers) self._max_image_size = max_image_size array_size = self._max_image_size * self._max_batch_size self._shared_rgbs = Array(ctypes.c_uint8, array_size * 3) self._shared_depths = Array(ctypes.c_float, array_size) self._shared_rgbs_array = np.frombuffer( self._shared_rgbs.get_obj(), dtype=ctypes.c_uint8) assert self._shared_rgbs_array.size == (array_size * 3), ( "Array size is %d, expected %d" % ( self._shared_rgbs_array.size, array_size * 3)) self._shared_depths_array = np.frombuffer( self._shared_depths.get_obj(), dtype=ctypes.c_float) assert self._shared_depths_array.size == array_size, ( "Array size is %d, expected %d" % ( self._shared_depths_array.size, array_size)) worker_id = Value(ctypes.c_int) worker_id.value = 0 if get_start_method() != "spawn": raise RuntimeError( "Start method must be set to 'spawn' for the " "render pool to work. That is, you must add the " "following to the _TOP_ of your main script, " "before any other imports (since they might be " "setting it otherwise):\n" " import multiprocessing as mp\n" " if __name__ == '__main__':\n" " mp.set_start_method('spawn')\n") self.pool = Pool( processes=len(device_ids) * n_workers, initializer=MjRenderPool._worker_init, initargs=( model.get_mjb(), worker_id, device_ids, self._shared_rgbs, self._shared_depths, modder))
end = time() fpsTime += end - start fpsCnt += 1 sleep(max(0, 1.0/30 - (end - start))) except KeyboardInterrupt: pass print("joining") if VIDEO_OUT: outframe.fill(0) hdmi_out.writeframe(outframe) gpio.gpio_write(latch_pwr, 0) run.value = False procthread.join() statusthread.join() print("FPS: %.2f, PROC: %.2f" % (fpsCnt / fpsTime, procCnt.value / procTime.value)) if 'videoIn' in globals(): videoIn.release() del videoIn if VIDEO_OUT and 'hdmi_out' in globals(): hdmi_out.stop() hdmi_out.close() del hdmi_out if 'db' in globals():
def main(): # Create DXL Reacher1D environment env = DxlReacher1DEnv( setup='dxl_gripper_default', idn=1, baudrate=1000000, obs_history=1, dt=0.04, gripper_dt=0.01, rllab_box=False, episode_length_step=None, episode_length_time=2, max_torque_mag=100, control_type='torque', target_type='position', reset_type='zero', reward_type='linear', use_ctypes_driver=True, ) # The outputs of the policy function are sampled from a Gaussian. However, the actions in terms of torque # commands are in the range [-max_torque_mag, max_torque_mag]. NormalizedEnv wrapper scales action accordingly. # By default, it does not normalize observations or rewards. env = NormalizedEnv(env) # Start environment processes env.start() # Create baselines trpo policy function sess = U.single_threaded_session() sess.__enter__() def policy_fn(name, ob_space, ac_space): return MlpPolicy(name=name, ob_space=ob_space, ac_space=ac_space, hid_size=32, num_hid_layers=2) # create and start plotting process plot_running = Value('i', 1) shared_returns = Manager().dict({ "write_lock": False, "episodic_returns": [], "episodic_lengths": [], }) # Plotting process pp = Process(target=plot_dxl_reacher, args=(env, 2048, shared_returns, plot_running)) pp.start() # Create callback function for logging data from baselines PPO learn kindred_callback = create_callback(shared_returns) # Train baselines TRPO learn( env, policy_fn, max_timesteps=50000, timesteps_per_batch=2048, max_kl=0.05, cg_iters=10, cg_damping=0.1, vf_iters=5, vf_stepsize=0.001, gamma=0.995, lam=0.995, callback=kindred_callback, ) # Safely terminate plotter process plot_running.value = 0 # shutdown ploting process time.sleep(2) pp.join() # Shutdown the environment env.close()
def runperiodic(self, stop, periodOn, periodOff): "Invert current state periodicly" if self.debug: print("Starting periodic checking") while not stop.value: gpio.output(self.pin, not self.state) self.state = not self.state if self.debug: print(self.pin, "switched state") if self.state: time.sleep(periodOn) else: time.sleep(periodOff) if self.debug: print("Ending OnOff checking") if __name__ == '__main__': from threading import Thread test = OnOff("P8_15", 0.5,mode=False, debug=True) trigger = Value('b', 0) stop = Value('b', False) try: Thread(target=test.runperiodic, args=(stop, 3, 1)).start() except KeyboardInterrupt: print("That's it") stop.value = True
def calculate_impl(formula, result: Value, error: Value): try: result.value = str(calc(formula)) except Exception as e: result.value = e.args[0] error.value = True
print(" Total number of requests: {}").format( config["limit"] if config["limit"] != 0 else "unlimited") print("") print("Work details:") print(" Priority: {}").format(config["priority"]) print(" Weight: {}").format(config["weight"]) print(" Start delay: {} s").format(config["when"]) print(" Execution time: {} ms").format(config["delay"]) print(" Average queue depth: {}").format( (effective_rate * config["delay"]) / 1000) print("") print("Starting up...") # Create a set of processes, then start them all procs = [ Process(target=httploop, args=( ctr + 1, config, done, )) for ctr in range(config["numProcesses"]) ] map(lambda p: p.start(), procs) # Wait for user to cancel, then signal end and wait for processes to finish time.sleep(1) raw_input("\n\nEnd run (hit return)") done.value = 1 map(lambda p: p.join(), procs) print("\nRun complete")
processes = [ Process(name="thread %s" % x, target=thread_embed, args=(q_lock, q, v_lock, v)) for x in range(NUMBER_OF_THREADS) ] for process in processes: process.start() while not q.empty(): pass print("Done with embedding!") v.value = 0 for file in glob.glob(G_DATAPATH + "*"): q.put(file) processes = [ Process(name="thread %s" % x, target=thread_upload, args=(q_lock, q, v_lock, v)) for x in range(NUMBER_OF_THREADS) ] for process in processes: process.start() while not q.empty(): pass
return finish('success') cnt = 0 for class_wnid in classes_to_scrape: cnt += 1 class_name = class_info_dict[class_wnid]["class_name"] logging.info('********************** (% 3d/% 4d) Scraping images for class: %s' % (cnt, len(classes_to_scrape), class_name,) ) url_urls = IMAGENET_API_WNID_TO_URLS(class_wnid) time.sleep(0.05) resp = requests.get(url_urls) classsub = class_wnid + '___' + class_name class_folder = os.path.join(imagenet_images_folder, classsub) if not os.path.exists(class_folder): os.mkdir(class_folder) class_images.value = 0 urls = [url.decode('utf-8') for url in resp.content.splitlines()] logging.info('number of images for class: ' + str(len(urls))) with open( os.path.join(imglists, classsub+'.txt'), 'w') as f: for u in urls: f.write(u) f.write('\n') logging.info(" Multiprocessing workers: {w}".format(w=args.multiprocessing_workers)) with Pool(processes=args.multiprocessing_workers) as p: p.map(get_image, urls)
from time import sleep import random def runProcess(arr, remainActive): while remainActive.value: for idx in range(len(arr)): arr[idx] = random.randint(0,100) sleep(1) def runMonitor(arr, remainActive): while remainActive.value: print(arr[:]) sleep(2) if __name__ == '__main__': random.seed() arr = Array('i', range(5)) remainActive = Value('i', 1) process = Process(target = runProcess, args = (arr, remainActive)) monitor = Process(target = runMonitor, args = (arr, remainActive)) process.daemon = True monitor.daemon = True process.start() monitor.start() sleep(10) remainActive.value = 0 process.join() monitor.join()
def stop_value(): stop = Value('i', 0) yield stop stop.value = 1
server_name = "cluster" + os.sep + "server-" + str(server) if not os.path.exists(server_name): os.makedirs(server_name) filter_log_dir = 'cluster' + os.sep + 'server-0/filter' if not os.path.exists(filter_log_dir): os.makedirs(filter_log_dir) if __name__ == '__main__': #configura o formato do log utilizado para debug da aplicação FORMAT = '%(asctime)-15s %(type)s %(thread)d %(message)s' logging.basicConfig(filename='log-server',format=FORMAT, level=logging.DEBUG) #total de servidores simulados total_servers = 4 create_dir(total_servers) dead = Value('i', 0) servers = []; servers.append(Process(target=server_function, args=('cluster/server-0', 500000, dead))) servers.append(Process(target=server_function, args=('cluster/server-1', 500000, dead))) servers.append(Process(target=server_function, args=('cluster/server-2', 500000, dead))) servers.append(Process(target=server_function, args=('cluster/server-3', 500000, dead))) for server in range(total_servers): servers[server].start() time.sleep(900) dead.value = 1 for server in range(total_servers): servers[server].join() logger = logging.getLogger('log') logger.debug('%s', "Todos os servidores forem finalizados", extra={'type':'SERVER'})
# XXX handle over-sized requests except socket.error: pass if EOL1 in requests[fileno] or EOL2 in requests[fileno]: request_q.put({'id': fileno, 'raw': requests[fileno]}) elif event & select.EPOLLOUT: try: while len(responses[fileno]) > 0: byteswritten = connections[fileno].send( responses[fileno]) responses[fileno] = responses[fileno][ byteswritten:] except socket.error: pass if len(responses[fileno]) == 0: epoll.modify(fileno, select.EPOLLET) connections[fileno].shutdown(socket.SHUT_RDWR) elif event & select.EPOLLHUP: epoll.unregister(fileno) connections[fileno].close() del connections[fileno] finally: epoll.unregister(serversocket.fileno()) epoll.close() serversocket.close() is_running.value = 0 for x in request_handlers: x.join()
while run.value: pulse_rate = 60.0 / (bpm.value * 24) midi_output.send(clock_tick) t1 = perf_counter() if bpm.value <= 3000: sleep(pulse_rate * 0.8) t2 = perf_counter() while (t2 - t1) < pulse_rate: t2 = perf_counter() if __name__ == '__main__': midi_ports = mido.get_output_names() print(midi_ports) shared_bpm = Value('i', 60) run_code = Value('i', 1) midi_clock_generator_proc = Process(target=midi_clock_generator, args=(midi_ports[0], shared_bpm, run_code)) midi_clock_generator_proc.start() # sleep(.1) while run_code.value: bpm = input('Enter Tempo in BPM-> ') if bpm.isdigit(): shared_bpm.value = int(bpm) else: run_code.value = False print('Shutting down process') midi_clock_generator_proc.join() midi_clock_generator_proc.close() print('exiting')
replay_data_read_pipe, replay_data_write_pipe = Pipe(False) replay_data_distribution_pipes.append( (replay_data_read_pipe, replay_data_write_pipe)) p = Process(target=RunActor, args=(i, gamma, \ actor_cmd_write_pipe, weights_read_pipe, replay_data_write_pipe, \ cancelation_token, training_active_flag)) actor_processess.append(p) p.start() print("Awaiting buffer fill up") # 3. Fill up replay buffer while len(exp_buffer) < learner_batch_size * learner_prefetch_batches: sleep(1) # 4. Start learning buffer_ready.value = 1 input("training networks.\nPress enter to finish\n\n") cancelation_token.value = 1 actor_cmd_read_pipe.close() actor_cmd_write_pipe.close() actor_cmd_processor_thread.join() learner_cmd_read_pipe.close() learner_cmd_write_pipe.close() learner_replay_data_write_pipe.close() learner_replay_data_read_pipe.close() learner_priorities_read_pipe.close() learner_priorities_write_pipe.close()
args=(keym, weatherValue, cont)) homeList = [ multiprocessing.Process(target=home_p.home, args=(keyh, keym, weatherValue, cont)) for i in range(nbHomes) ] weather = multiprocessing.Process(target=weather_p.weather, args=(weatherValue, cont)) weather.start() market.start() for currentHome in homeList: currentHome.start() time.sleep(temps) cont.value = 0 market.join() #for currentHome in homeList: #currentHome.join() weather.join() #weather.join() mqm.remove() mqh.remove() print("--------------- Fin de la simulation ---------------")
DAQ_handle = DAQ.Init() DAQ.DAC_Write(DAQ_handle, Laser_Port, 0) #Laser is off DAQ.DAC_Write(DAQ_handle, Shutter_Port, 0) #Shutter is close # ############# Inititalizing the main loop for optrode ##############''' DAC_Sampl_Index = 0 Spec_Sampl_Index = 0 # ##################### Initializing the variables ###################''' Integration_list = [8000, 16000, 32000, 64000, 128000, 256000, 512000] No_DAC_Sample = 1400 # Number of samples for Photodiod per iteration of the laser exposer. Every sample takes ~3 ms. SB_Is_Done = Value('i', 0) SB_Current_Record = Array('f', np.zeros(shape=( len(Spec_handle.wavelengths()) ,1), dtype = float )) SB_Is_Done.value = 0 SB_Full_Records = np.zeros(shape=(len(Spec_handle.wavelengths()), len(Integration_list) ), dtype = float ) read_signal = np.zeros(No_DAC_Sample*len(Integration_list)) read_time = np.zeros(No_DAC_Sample*len(Integration_list)) # ########### The file containing the records (HDF5 format)###########''' File_name = "Opterode_Recording_At" + str('%i' %time.time())+ ".hdf5" f = h5py.File(File_name, "w") Spec_sub1 = f.create_group("Spectrumeter") Spec_specification = Spec_sub1.create_dataset("Spectrumeter", (10,), dtype='f') Spec_specification.attrs['Serial Number'] = np.string_(Spec_handle.serial_number) Spec_specification.attrs['Model'] = np.string_(Spec_handle.model) Spec_wavelength = f.create_dataset('Spectrumeter/Wavelength', data = Spec_handle.wavelengths())
def main(): parser = argparse.ArgumentParser(description='ImageNet image scraper') parser.add_argument('-scrape_only_flickr', default=True, type=lambda x: (str(x).lower() == 'true')) parser.add_argument('-number_of_classes', default=10, type=int) parser.add_argument('-images_per_class', default=10, type=int) parser.add_argument('-data_root', default='', type=str) parser.add_argument('-use_class_list', default=False, type=lambda x: (str(x).lower() == 'true')) parser.add_argument('-class_list', default=[], nargs='*') parser.add_argument('-debug', default=False, type=lambda x: (str(x).lower() == 'true')) parser.add_argument('-multiprocessing_workers', default=8, type=int) args, args_other = parser.parse_known_args() if args.debug: logging.basicConfig(filename='imagenet_scarper.log', level=logging.DEBUG) if len(args.data_root) == 0: logging.error("-data_root is required to run downloader!") exit() def imagenet_api_wnid_to_urls(wnid): return f'http://www.image-net.org/api/imagenet.synset.geturls?wnid={wnid}' current_folder = os.path.dirname(os.path.realpath(__file__)) class_info_json_filename = 'imagenet_class_info.json' class_info_json_filepath = os.path.join(current_folder, class_info_json_filename) with open(class_info_json_filepath) as class_info_json_f: class_info_dict = json.load(class_info_json_f) classes_to_scrape = [] if args.use_class_list: for item in args.class_list: classes_to_scrape.append(item) if item not in class_info_dict: logging.error(f'Class {item} not found in ImageNete') exit() elif not args.use_class_list: potential_class_pool = [] for key, val in class_info_dict.items(): if args.scrape_only_flickr: if int(val['flickr_img_url_count']) * 0.9 > args.images_per_class: potential_class_pool.append(key) else: if int(val['img_url_count']) * 0.8 > args.images_per_class: potential_class_pool.append(key) if len(potential_class_pool) < args.number_of_classes: logging.error( f"With {args.images_per_class} images per class there are {len(potential_class_pool)} to choose from.") logging.error(f"Decrease number of classes or decrease images per class.") exit() picked_classes_idxes = np.random.choice(len(potential_class_pool), args.number_of_classes, replace=False) for idx in picked_classes_idxes: classes_to_scrape.append(potential_class_pool[idx]) print("Picked the following clases: \nCount: %s" % len(classes_to_scrape)) print([class_info_dict[class_wnid]['class_name'] for class_wnid in classes_to_scrape]) if not os.path.isdir(args.data_root): os.mkdir(args.data_root) def add_debug_csv_row(row): with open('stats.csv', "a") as csv_f: csv_writer = csv.writer(csv_f, delimiter=",") csv_writer.writerow(row) class MultiStats: def __init__(self): self.lock = Lock() self.stats = dict( all=dict( tried=Value('d', 0), success=Value('d', 0), time_spent=Value('d', 0), ), is_flickr=dict( tried=Value('d', 0), success=Value('d', 0), time_spent=Value('d', 0), ), not_flickr=dict( tried=Value('d', 0), success=Value('d', 0), time_spent=Value('d', 0), ) ) def inc(self, cls, stat, val): with self.lock: self.stats[cls][stat].value += val def get(self, cls, stat): with self.lock: ret = self.stats[cls][stat].value return ret multi_stats = MultiStats() if args.debug: row = [ "all_tried", "all_success", "all_time_spent", "is_flickr_tried", "is_flickr_success", "is_flickr_time_spent", "not_flickr_tried", "not_flickr_success", "not_flickr_time_spent" ] add_debug_csv_row(row) def add_stats_to_debug_csv(): row = [ multi_stats.get('all', 'tried'), multi_stats.get('all', 'success'), multi_stats.get('all', 'time_spent'), multi_stats.get('is_flickr', 'tried'), multi_stats.get('is_flickr', 'success'), multi_stats.get('is_flickr', 'time_spent'), multi_stats.get('not_flickr', 'tried'), multi_stats.get('not_flickr', 'success'), multi_stats.get('not_flickr', 'time_spent'), ] add_debug_csv_row(row) def print_stats(cls, print_func): actual_all_time_spent = time.time() - scraping_t_start.value processes_all_time_spent = multi_stats.get('all', 'time_spent') if processes_all_time_spent == 0: actual_processes_ratio = 1.0 else: actual_processes_ratio = actual_all_time_spent / processes_all_time_spent print_func(f'STATS For class {cls}:') print_func(f' tried {multi_stats.get(cls, "tried")} urls with' f' {multi_stats.get(cls, "success")} successes') if multi_stats.get(cls, "tried") > 0: print_func( f'{100.0 * multi_stats.get(cls, "success") / multi_stats.get(cls, "tried")}% ' f'success rate for {cls} urls ') if multi_stats.get(cls, "success") > 0: print_func( f'{multi_stats.get(cls, "time_spent") * actual_processes_ratio / multi_stats.get(cls, "success")}' f' seconds spent per {cls} successful image download') lock = Lock() url_tries = Value('d', 0) scraping_t_start = Value('d', time.time()) class_folder = '' class_images = Value('d', 0) def get_image(img_url): def check(): with lock: cls_imgs = class_images.value if cls_imgs >= args.images_per_class: return True if len(img_url) <= 1: return if check(): return logging.debug(img_url) cls = '' if 'flickr' in img_url: cls = 'is_flickr' else: cls = 'not_flickr' if args.scrape_only_flickr: return t_start = time.time() def finish(status): t_spent = time.time() - t_start multi_stats.inc(cls, 'time_spent', t_spent) multi_stats.inc('all', 'time_spent', t_spent) multi_stats.inc(cls, 'tried', 1) multi_stats.inc('all', 'tried', 1) if status == 'success': multi_stats.inc(cls, 'success', 1) multi_stats.inc('all', 'success', 1) elif status == 'failure': pass else: logging.error(f'No such status {status}!!') exit() return with lock: url_tries.value += 1 if url_tries.value % 250 == 0: print(f'\nScraping stats:') print_stats('is_flickr', print) print_stats('not_flickr', print) print_stats('all', print) if args.debug: add_stats_to_debug_csv() try: img_resp = requests.get(img_url, timeout=1, verify=False) except ConnectionError: logging.debug(f"Connection Error for url {img_url}") return finish('failure') except ReadTimeout: logging.debug(f"Read Timeout for url {img_url}") return finish('failure') except TooManyRedirects: logging.debug(f"Too many redirects {img_url}") return finish('failure') except MissingSchema: return finish('failure') except InvalidURL: return finish('failure') if 'content-type' not in img_resp.headers: return finish('failure') if 'image' not in img_resp.headers['content-type']: logging.debug("Not an image") return finish('failure') if len(img_resp.content) < 1000: return finish('failure') logging.debug(img_resp.headers['content-type']) logging.debug(f'image size {len(img_resp.content)}') img_name = img_url.split('/')[-1] img_name = img_name.split("?")[0] if len(img_name) <= 1: return finish('failure') img_file_path = os.path.join(class_folder, img_name) logging.debug(f'Saving image in {img_file_path}') if check(): return with open(img_file_path, 'wb') as img_f: img_f.write(img_resp.content) with lock: class_images.value += 1 logging.debug(f'Scraping stats') print_stats('is_flickr', logging.debug) print_stats('not_flickr', logging.debug) print_stats('all', logging.debug) return finish('success') print(f"Multiprocessing workers: {args.multiprocessing_workers}") for class_wnid in tqdm(classes_to_scrape): class_name = class_info_dict[class_wnid]["class_name"] url_urls = imagenet_api_wnid_to_urls(class_wnid) time.sleep(0.05) resp = requests.get(url_urls, verify=False) class_folder = os.path.join(args.data_root, class_name) if not os.path.exists(class_folder): os.mkdir(class_folder) class_images.value = 0 urls = [url.decode('utf-8') for url in resp.content.splitlines()] with ThreadPool(processes=args.multiprocessing_workers) as p: p.map(get_image, urls)
# 'lfc-lhcb.grid.sara.nl', # 'lfclhcb.pic.es', # 'lhcb-lfc.gridpp.rl.ac.uk'] lfcHosts = ['prod-lfc-lhcb-ro.cern.ch'] # path = "/lhcb/LHCb" path = '/lhcb/user/c/chaen' print "Queueing task for directory", path, lfcHosts[0] writerProc = Process( target = writer, args = ( 'lfc_dfc.out', writerQueue, stopFlag ) ) writerProc.start() result = pPool.createAndQueueTask( processDir, [path , writerQueue, False, lfcHosts[0]], callback = finalizeDirectory ) if not result['OK']: print "Failed queueing", path for i in range(20): pPool.processResults() time.sleep(1) pPool.processAllResults( timeout = 300 ) stopFlag.value = 1 writerQueue.put( "Exit" ) writerProc.join()
def run(self, verbose = False, cmp_packet = None, scapy_packet = None): '''execute the test case. arguments: verbose -- if set to True, show debug output (default: False) cmp_packet -- packet of type CmpLayer (default: None) scapy_packet -- packet of type scapy.Packet (default: None) cmp_packet and scapy_packet are used by pre_run function, if such a function is given. return: True if the test case succeeded (access to the cmp_packet and the received packet via match_packets()) False if the test case failed (no given cmp_packet matched the received packet(s)) ''' if self.pre_run_processing is not None and callable(self.pre_run_processing): self.pre_run_processing(self, cmp_packet, scapy_packet) if len(self._tx_packets) == 0 and len(self._rx_packets) == 0: # the empty test case always succeeds return True max_rx_timeout = 0 rx_packets_by_if = {} rxps = [] ret_queue = Queue() ret_queue_lock = Lock() rx_timeout_start = Value('d', 0.0) rx_timeout_start_lock = Lock() self._match_packets = None if len(self._rx_packets) > 0: for rxp in self._rx_packets: if rxp.timeout > max_rx_timeout: max_rx_timeout = rxp.timeout if rxp.interface['name'] not in rx_packets_by_if: rx_packets_by_if[rxp.interface['name']] = [] rx_packets_by_if[rxp.interface['name']].append(rxp) for interface in rx_packets_by_if: rxps.append(Process(target=self._rx_worker, args=(interface, rx_packets_by_if[interface], rx_timeout_start, rx_timeout_start_lock, max_rx_timeout, ret_queue, ret_queue_lock))) if verbose is True: print('---') print('test case: %s' % (self.name)) print('send-via-tcp-socket: %s' % (str(self._use_tcp))) print('tx-packets: %d' % (len(self._tx_packets))) print('rx-packets: %d' % (len(self._rx_packets))) print('max-rx-timeout: %d' % (max_rx_timeout)) txp = None if len(self._tx_packets) > 0: txp = Process(target=self._tx_worker, args=(self,)) for rxp in rxps: rxp.start() ''' wait a small amount of time to give the sniffers a chance to settle down TODO: how to work around ?!? ''' time.sleep(0.1) if txp is not None: txp.start() # wait until all packets are send or we got a packet that matches our criteria while True: txp.join(1) if txp.is_alive() is False or ret_queue.empty() is False: break # wait if one of the rx processes catches a packet or we run in timeout if ret_queue.empty() is True: with rx_timeout_start_lock: rx_timeout_start.value = scapy.time.time() tstart = rx_timeout_start.value while True: for rxp in rxps: rxp.join(1) if (ret_queue.empty() is False) or ((scapy.time.time() - tstart)*1000 >= max_rx_timeout): break # stop all processes if txp is not None and txp.is_alive(): txp.terminate() with ret_queue_lock: for rxp in rxps: if rxp.is_alive(): rxp.terminate() if (self._use_tcp == True) and (self.sock_conf['socket'] is not None): self.sock_conf['socket'].close() self.sock_conf['socket'] = None if DO_DEBUG: print('TCP socket closed.') if ret_queue.empty() and len(self._rx_packets) > 0: if verbose is True: print('test-result: failed') return False else: if verbose is True: print('test-result: success') if len(self._rx_packets) > 0: self._match_packets = ret_queue.get() pckts = self._match_packets if DO_DEBUG: while True: print('---') print('received packet: ' + str(pckts['rx_packet'])) print('matched packet: ' + str(pckts['cmp_packet'])) if ret_queue.empty(): break pckts = ret_queue.get() return True
p2Speed = Value('d', 3) # Start Process to start first array of lights p1 = Process(target=lightsarray1, args=(p1Speed,)) p1.start() # Start Process to start second array of lights p2 = Process(target=lightsarray2, args=(p2Speed,)) p2.start() Process(target=printValues, args=(p1Speed, p2Speed, )).start() p1ButtonValue = 0 p2ButtonValue = 0 p3ButtonValue = 0 while True: if GPIO.input(21) == 1 and p1ButtonValue == 0: p1Speed.value = p1Speed.value - (p1Speed.value / 5) if GPIO.input(20) == 1 and p2ButtonValue == 0: p2Speed.value = p2Speed.value - (p2Speed.value / 5) if GPIO.input(16) == 1 and p3ButtonValue == 0: p2Speed.value = 1 p1Speed.value = 3 p1ButtonValue = GPIO.input(21) p2ButtonValue = GPIO.input(20) p3ButtonValue = GPIO.input(16) # Exit the program exit
Shutter_CloseDelay = Blue_Shutter_CloseDelay break else: print 'Wrong input!' # ##################### Initializing the variables ################### #Integration_list = [8000, 16000, 32000, 64000, 128000, 256000, 512000, 1024000, 2048000] Integration_list_sec = [0.008, 0.016, 0.032, 0.064, 0.128, 0.256, 0.512 ] Integration_marging = 0.2 #(In seconds) This is the duration before the external edge trigger is given to the spectrometer while the specrumeter started the integration period #Integration_base = Integration_list_sec[-1]*1000000 + Integration_marging*2000000 # This is the integration time applied for all the trials Integration_base = 2*1000000 # This is the integration time applied for all the trials in microseconds No_DAC_Sample = 10000 # Number of samples for Photodiod per iteration of the laser exposer. Every sample takes ~0.6 ms. SB_Is_Done = Value('i', 0) SB_Current_Record = Array('d', np.zeros(shape=( len(Spec_handle.wavelengths()) ,1), dtype = float )) SB_Is_Done.value = 0 Timer_Is_Done = Value('i', 0) Timer_Is_Done.value = 0 Timer_Is_Done2 = Value('i', 0) Timer_Is_Done2.value = 0 SB_Full_Records = np.zeros(shape=(len(Spec_handle.wavelengths()), len(Integration_list_sec)+1 ), dtype = float ) read_signal = np.zeros(No_DAC_Sample*len(Integration_list_sec)) read_time = np.zeros(No_DAC_Sample*len(Integration_list_sec)) ''' Open_delay = np.zeros(50) Close_delay = np.zeros(50) ''' read_signal_ref = np.zeros(No_DAC_Sample*len(Integration_list_sec)) read_time_ref = np.zeros(No_DAC_Sample*len(Integration_list_sec))
def analyze(): """ Analyze setup process to create children """ start = datetime.datetime.now() filename = "data/dataset.csv" # UFF!!! ALL THESE DECLARATIONS!!! # So integers are faster than dictionaries, that hash lookup # takes clock cycles. Next, we need to declare a set of counter # variables for each child process we're about to spawn. We need # return values from the proccess, so we need to allocate memory for # them. That's what we're doing here. I could have written a loop # to do this in less lines of code, but it results in more clock # cycles to complete the task. # I tried this with global variabels before this, will all process # trying to operate on them, and while it appeared to work, once # the process was lost so were the results. I also tried passing # a single set of shared memory variabes to all consume functions # this gave very strange results, the result would come up just a bit # different every run. I'd be expecting 8362, and I'd get 8320-8362. found_1 = Value("i", 0) _2013_1 = Value("i", 0) _2014_1 = Value("i", 0) _2015_1 = Value("i", 0) _2016_1 = Value("i", 0) _2017_1 = Value("i", 0) _2018_1 = Value("i", 0) found_2 = Value("i", 0) _2013_2 = Value("i", 0) _2014_2 = Value("i", 0) _2015_2 = Value("i", 0) _2016_2 = Value("i", 0) _2017_2 = Value("i", 0) _2018_2 = Value("i", 0) found_3 = Value("i", 0) _2013_3 = Value("i", 0) _2014_3 = Value("i", 0) _2015_3 = Value("i", 0) _2016_3 = Value("i", 0) _2017_3 = Value("i", 0) _2018_3 = Value("i", 0) # Loop was used to run the program multiple times for better results on # a higher-performing system. A single test of say .5 sec was less # reliable than 10 tests at 4.95 seconds. Reset to 1 loop for submission # in case grading is done on a potato. for _ in range(1): # Need to make sure our pile of variables are 0'd out. found_1.value = 0 _2013_1.value = 0 _2014_1.value = 0 _2015_1.value = 0 _2016_1.value = 0 _2017_1.value = 0 _2018_1.value = 0 found_2.value = 0 _2013_2.value = 0 _2014_2.value = 0 _2015_2.value = 0 _2016_2.value = 0 _2017_2.value = 0 _2018_2.value = 0 found_3.value = 0 _2013_3.value = 0 _2014_3.value = 0 _2015_3.value = 0 _2016_3.value = 0 _2017_3.value = 0 _2018_3.value = 0 # Here we define and spawn each process. process1 = Process( target=consume_1, args=( found_1, _2013_1, _2014_1, _2015_1, _2016_1, _2017_1, _2018_1, FIRST, filename, ), ) process1.start() process2 = Process( target=consume_2, args=( found_2, _2013_2, _2014_2, _2015_2, _2016_2, _2017_2, _2018_2, SECOND, FIRST, filename, ), ) process2.start() process3 = Process( target=consume_2, args=( found_3, _2013_3, _2014_3, _2015_3, _2016_3, _2017_3, _2018_3, THIRD, FIRST + SECOND, filename, ), ) process3.start() # Wait for the processes to finish process1.join() process2.join() process3.join() # Add up the values of all that shared ram found_total = found_1.value + found_2.value + found_3.value _2013_total = _2013_1.value + _2013_2.value + _2013_3.value _2014_total = _2014_1.value + _2014_2.value + _2014_3.value _2015_total = _2015_1.value + _2015_2.value + _2015_3.value _2016_total = _2016_1.value + _2016_2.value + _2016_3.value _2017_total = _2017_1.value + _2017_2.value + _2017_3.value _2018_total = _2018_1.value + _2018_2.value + _2018_3.value # And we get results display_results( found_total, _2013_total, _2014_total, _2015_total, _2016_total, _2017_total, _2018_total, ) end = datetime.datetime.now() return ( start, end, { "2013": _2013_total, "2014": _2014_total, "2015": _2015_total, "2016": _2016_total, "2017": _2017_total, "2018": _2018_total, }, found_total, )
def draw_panel_label(surf): def draw_version_label(screen): def get_component_color(component_pipe): dlog.info('create console') except Exception as e : except: exit(1) exit(1) exit(1) for i in range(256): global bigned global color_ut global color_ut global dlog global llog global main_breakout global screen globals()["bigned"].signal_extinguish() globals()["color_ut"] = None globals()["gfont"] = g.create_font() globals()["gselected_component"] = 1 globals()["llog"].info("extinguished") globals()["main_breakout"] = Value("d", 0) globals()["main_breakout"].value = 0 globals()["screen"] = g.init_pygame() init_color_tweeners() inverse = [abs(250-color[0]), abs(250-color[1]), abs(250-color[2])] inverse = [color[0]-a, color[1]-a, color[2]-a] key_calls={"d": extinguish_and_deload}) llog.error("error while init [ FAIL ] ") llog.error("failed to init tweeners: %s", e) llog.error("unknown error while init tweeners") llog.info("* PROCESS: Draw") llog.info("---> ABOUT TO DELOAD") llog.info("entering main draw loop") llog.info("exiting main draw loop...") llog.info("here we go") llog.info("init'd font successful [ SUCCESS ] ") llog.info("init'd screen [ SUCCESS ]") llog.info("signaled extinguish [ SUCCESS ]") llog.info("ut init'd successfully... [ SUCCESS ] ") panel_label_surf = pygame.Surface((256, 100)) panel_label_where = (20, 300) pass pass print e return return return inverse return inverse screen.fill((35, 35, 35)) try: try: while main_breakout.value == 1: """2 processes: NedProcess, DrawProcess""" """draw a visual AI window""" ## Innit is done #### Start here __version__ = "base" def Draw(theNed): def extinguish_and_deload(): def get_color_dimmer(color,a=20): def get_color_inverse(color): def init_color_tweeners(): def init_console(screen, key_calls): def setup_color_tweens(): ## This is a list of tweens set up by default... except Exception as e: except ValueError as e: except: from getmylogger import silent_logger,loud_logger from multiprocessing import Process, Value from Text import Text from time import sleep globals()["bigned"] = None globals()["dlog"] = silent_logger("drawing_examples") #silent drawing logger globals()["llog"] = loud_logger("drawing_examples") #loud logger (shows in console) globals()["llog"].info("log setup! [ SUCCESS ] ") globals()["selected_component_id"] = 0 globals()["user_console"] = init_console(globals()["screen"], if __name__ == '__main__': import exceptions import gui_helpers as g import os,sys import pygame,pyconsole import random import ut llog = globals()["llog"] try: try: USER_CONSOLE_ENABLED = True #only checked when drawing
mouse.setVisible(False) trialClock = core.Clock() stim1 = visual.PatchStim(win=mywin, mask='gauss', size=0.5, pos=[0,0], sf=0, color='black') stim2 = visual.PatchStim(win=mywin, mask='gauss', size=0.5, pos=[-20,0], sf=0, color='black') stim3 = visual.PatchStim(win=mywin, mask='gauss', size=0.5, pos=[0,10], sf=0, color='black') stim4 = visual.PatchStim(win=mywin, mask='gauss', size=0.5, pos=[20,0], sf=0, color='black') stim5 = visual.PatchStim(win=mywin, mask='gauss', size=0.5, pos=[0,-10], sf=0, color='black') stim = stim1 stims = [stim1,stim2,stim3,stim4] n_stims = len(stims) n_reps=10 detect.value=1 detector.start() #for n in range(n_reps): # vel=[] # stim = stims[np.mod(n,n_stims)] t = 0 trialClock.reset() while t < 3: stim.draw() mywin.flip() t=trialClock.getTime() detect.value=0 detector.join() v=[]
for i in range(0, int(CONFIG['BOT_CONFIG']['workers'])): worker_process.append(Process(target=process_updates)) worker_process[i].start() time_worker = ThreadProcess(target=check_time_args) time_worker.start() while RUNNING.value: time.sleep(30) for index, worker in enumerate(worker_process): if not worker.is_alive(): del worker_process[index] worker_process.append(Process(target=process_updates)) worker_process[-1].start() if not time_worker.is_alive(): time_worker = ThreadProcess(target=check_time_args) time_worker.start() if not get_update_process.is_alive(): get_update_process = Process(target=get_updates) get_update_process.start() get_update_process.join() time_worker.join() for worker in worker_process: worker.join() if __name__ == '__main__': try: main() except KeyboardInterrupt: print("\n\nShutting Down.....") RUNNING.value = 0
s.listen(1) conn, addr = s.accept() while True: try: # These three lines are usefull to debug wether to use MSB or LSB in the reading formats # for the first parameter of "hx.set_reading_format("LSB", "MSB")". # Comment the two lines "val = hx.get_weight(5)" and "print val" and uncomment these three lines to see what it prints. # np_arr8_string = hx.get_np_arr8_string() # binary_string = hx.get_binary_string() # print binary_string + " " + np_arr8_string lock3.acquire() load1_final_val.value = 0 load2_final_val.value = 0 load3_final_val.value = 0 load4_final_val.value = 0 lock1.acquire() try: load1_final_val.value = load1_per.value finally: lock1.release() lock2.acquire() try: load2_final_val.value = load2_per.value finally: lock2.release()
Power_Signal[Power_Index[0]], Power_Time[Power_Index[0]] = Power_meter.readPower() Power_Index[0] = Power_Index[0] + 1 Power_Is_Read.value = 1 if __name__ == "__main__": PhotoDiod_Port = "AIN1" Spec1 = SBO.open() Integration_Time = 2 # Integration time in ms Spec1.setTriggerMode(0) # It is set for free running mode Spec1.setIntegrationTime(Integration_Time*1000) # Integration time is in microseconds when using the library DAQ1 = DAQ.open() Power_meter = P100.open() Spec_Is_Read = Value('i', 0) Spec_Is_Read.value = 0 Spec_Is_Done = Value('i', 0) Spec_Is_Done.value = 0 DAQ_Is_Read = Value('i', 0) DAQ_Is_Read.value = 0 Power_Is_Read = Value('i', 0) Power_Is_Read.value = 0 Timer_Is_Over = Value('i', 0) Timer_Is_Over.value = 0 DurationOfReading = 3.12 # Duration of reading in seconds. No_DAC_Sample = int(round(DurationOfReading*1000/1.7)) # Number of samples for DAQ analogue to digital converter (AINx). Roughly DAQ can read AIN1 2 and 3 evry 1.5 ms and 2.4 ms for AIN0, No_Power_Sample = int(round(DurationOfReading*1000/5.1)) # Number of samples for P100D Power meter to read. Roughly P100 can read the power every 2.7 ms. No_Spec_Sample = int(round(DurationOfReading*1000/(Integration_Time))) # Number of samples for spectrometer to read. It takes integration time can read the power every 2.7 ms. Current_Spec_Record = Array('d', np.zeros(shape=( len(Spec1.Handle.wavelengths()) ,1), dtype = float ))
from multiprocessing import Value from multiprocessing import Process import time # 创建一个整数的共享内存值对象,初始值是10000 # 不加锁会引起逻辑混乱 shm_v = Value('i', 10000) def process_task2(): for _ in range(1000000): v = shm_v.value # 获取共享内存的值对象的值 v += 1 shm_v.value = v # 修改共享内存值对象 p = Process(target=process_task2) p.start() for _ in range(1000000): v = shm_v.value v -= 1 shm_v.value = v print("共享内存的值是:", shm_v.value) # 10000 p.join() print("程序正常退出")
gyro[1] = (shortdata[4] / gyro_divider - node.gyroBias[1]) * DEG2RAD gyro[2] = (shortdata[5] / gyro_divider - node.gyroBias[2]) * DEG2RAD gyro = np.asmatrix(gyro) #smoothing data (acc and gyro) - I found sma with buffer size 30 it is not enough for gyro filtterdData = SMA(np.concatenate((acc, gyro), axis=1), 30) if bufferFlag == 0: continue rawFilterdData = np.concatenate((acc, filtterdData[0, 0:3]), axis=1) ret = np.concatenate((ret, rawFilterdData), axis=0) if filtterdData.shape[0] != 0: if ret.shape[0] == numberOfPlot: ret = ret.transpose() # print ret plot1[0:numberOfPlot] = ret[0, :].tolist()[0] plot2[0:numberOfPlot] = ret[1, :].tolist()[0] plot3[0:numberOfPlot] = ret[2, :].tolist()[0] plot4[0:numberOfPlot] = ret[3, :].tolist()[0] plot5[0:numberOfPlot] = ret[4, :].tolist()[0] plot6[0:numberOfPlot] = ret[5, :].tolist()[0] Idx.value = numberOfPlot staticFlag.value = False ret = np.zeros((0, 6)) else: continue
#等待run结束 while not self.is_done: time.sleep(0.01) pass #结束 print('子进程结束') super(DataSource, self).close() def end(self): print('子进程准备结束') #通知run结束 self.need_end = False if __name__ == '__main__': #准备开始 need_end.value = False p1 = Process(target=one_channel, args=('1', need_end)) #p1 = DataSource('asdf') p1.start() print('主进程') time.sleep(5) print('主进程退出前关闭进程') #通知结束 need_end.value = True print('主进程中 need_end', need_end.value) #p1.need_end = False #p1.terminate() #print(p1.need_end) #p1.close() # 结束子进程