def join_shard(self, name): """Join a shard by name - LOCKS""" with self.lock: if name == b'': self.shard = None return print('%s (steamid %d) is now in shard %s' % (str(self.addr), self.steamid, name.decode('utf-8'))) self.last_shard_change = datetime.datetime.now() new_shard = self.server.get_shard(name) with new_shard.lock: new_shard.clients.append(self) self.shard = new_shard # Add client's cached sounds to shard's download list with self.server.cache_lock: for hash in self.sounds: if hash in self.server.cache and hash not in new_shard.sounds: new_shard.sounds.append(hash) # Send missing sounds list (as in the client doesn't have them) to client packet = SoundRequest() for hash in new_shard.sounds: if hash not in self.sounds: packet.sound_hash.append(hash) if len(packet.sound_hash) > 0: self.send(PacketInfo.SOUNDS_LIST, packet)
def createFile(self, port): try: f = open(resource_path('res/port.txt'), 'w') f.write(str(port)) f.close() except Exception as e: print('create_file ', e)
def find_count(self): if self.story_count is None: print('Fetching story count for', self.shelf) soup = self.first_page count = soup(class_="search_results_count")[0]('b')[2].string self.story_count = int(count) return self.story_count
def get_story_data(story): """ Get story data from fimfic: Returns a dict: { 'name': name of story, 'story_wc': word count of story, 'per_chapter_wc': list of word counts for each chapter, 'chapter_wc_sum': sum of all chapter word counts (sum(get_story_data(story)['per_chapter_wc'])) } story - url to story, assumed relative to FIMFICTION """ if story not in url_cache: #print('Loading story data of', story) soup = bs4.BeautifulSoup(get_url(FIMFICTION + story), 'lxml') name = str(soup(class_="story_name")[0].string) chapters = soup(class_="word_count") story_wc = int(deprettify(chapters[-1].b.string)) chapters = chapters[:-1] chapter_indiv_wc = [int(deprettify(x.get_text())) for x in chapters] chapter_wc_sum = sum(chapter_indiv_wc) if story_wc != chapter_wc_sum: print('WARNING: chapter word count ({}) did not match story word count ({}) for story {}'\ .format(story_wc, chapter_wc_sum, name)) url_cache[story] = { 'name': name, 'story_wc': story_wc, 'per_chapter_wc': chapter_indiv_wc, 'chapter_wc_sum': chapter_wc_sum } #print(story + "'s data:", url_cache[story]) return url_cache[story]
def load(self, filename_list, file_callback, error_callback): """Loads the sound list""" for filename in filename_list: file_stat = os.stat(filename) if file_stat.st_size > 2 * 1024 * 1024: error_callback( 'File %s is too large (over 2 Mb) and will not be loaded.' % filename) continue hash = hashlib.blake2b() with open(filename, 'rb') as infile: hash.update(infile.read()) digest = hash.digest() filepath = os.path.join('cache', hash.hexdigest()) with open(filepath, 'wb') as outfile: infile.seek(0) outfile.write(infile.read()) try: should_stream = self.name in [ 'MVP', 'Round lose', 'Round start', 'Round win', 'Timeout' ] file = pyglet.media.load(filename, streaming=should_stream) print(' + Loaded %s (%s)' % (small_hash(digest), filename)) except Exception as e: msg = 'Error while loading "%s":\n%s' % (filename, str(e)) error_callback(msg) else: self.samples.append(digest) file_callback(digest, file)
def run(self): self._client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) # self.connect() # self.listen() self.connected = False while not self.stopped(): if not self.connected: try: port = self.get_port() if port == 0: continue self._client.connect(('localhost', port)) self.connected = True except Exception as e: print('connect ', e) else: try: self.listen() except Exception as e: print('listen ', e) self._client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.connected = False self.barber_go_to_sleep() self.sleep(0.3)
def accept_connection(self): try: self._conn, addr = self._sock.accept() return True except Exception as e: print('accept_connecton ', e) return False
def handle(self, packet_type, raw_packet): print('Received %s packet' % PacketInfo.Type.Name(packet_type)) if packet_type == PacketInfo.PLAY_SOUND and self.state.is_ingame(): packet = PlaySound() packet.ParseFromString(raw_packet) if not self.sounds.play(packet): self.download_total = self.download_total + 1 self.threadripper.sounds_to_download.put(packet.sound_hash) elif packet_type == PacketInfo.SOUND_REQUEST: req = SoundRequest() req.ParseFromString(raw_packet) self.upload_total += len(req.sound_hash) for hash in req.sound_hash: self.threadripper.sounds_to_upload.put(hash) elif packet_type == PacketInfo.SOUND_RESPONSE: packet = SoundResponse() packet.ParseFromString(raw_packet) self.sounds.save(packet) self.sounds.play_received(packet.hash) elif packet_type == PacketInfo.SOUNDS_LIST: packet = SoundRequest() packet.ParseFromString(raw_packet) download_list = [] with self.sounds.cache_lock: for hash in packet.sound_hash: if hash not in self.sounds.cache: download_list.append(hash) self.download_total += len(download_list) for hash in download_list: self.threadripper.sounds_to_download.put(hash) else: print("Unhandled packet type!")
def listen(self): while not self.stopped() and self.connected: data = self._client.recv(1024) data = pickle.loads(data) if data[0] == 0: # clients count self._parent.clients = data[1] self._parent.clients_loaded.set() print(data)
def send(self, type, packet): raw_packet = packet.SerializeToString() header = PacketInfo() header.type = type header.length = len(raw_packet) with self.lock: print(f'Sending {PacketInfo.Type.Name(type)} to {str(self.addr)}') self.sock.sendall(header.SerializeToString()) self.sock.sendall(raw_packet)
def _exit(sig=None, frame=None): """ Interrupt handler. """ global run_proc if (run_proc): print("Recommendation engine exiting...") print("Finishing one last run so we can exit gracefully.") run_proc = False
def run_interleaved_hmc_with_leapfrog_steps(model_config, results_dir, num_leapfrog_steps_cp, num_leapfrog_steps_ncp, initial_step_size_cp, initial_step_size_ncp, initial_states_cp): (target, model, elbo, variational_parameters, learnable_parameters, actual_reparam) = create_target_graph(model_config, results_dir) target_cp, target_ncp = target (states, kernel_results, ess) = inference.hmc_interleaved( model_config, target_cp, target_ncp, num_leapfrog_steps_cp=num_leapfrog_steps_cp, num_leapfrog_steps_ncp=num_leapfrog_steps_ncp, step_size_cp=initial_step_size_cp, step_size_ncp=initial_step_size_ncp, initial_states_cp=initial_states_cp, ) init = tf.compat.v1.global_variables_initializer() with tf.compat.v1.Session() as sess: init.run() start_time = time.time() cp_accepted = kernel_results.cp_results.inner_results.is_accepted ncp_accepted = kernel_results.ncp_results.inner_results.is_accepted samples, is_accepted_cp, is_accepted_ncp, ess_final = sess.run( (states, cp_accepted, ncp_accepted, ess)) mcmc_time = time.time() - start_time normalized_ess_final = [] for ess_ in ess_final: # report effective samples per 1000 gradient evals normalized_ess_final.append( 1000 * ess_ / (FLAGS.num_samples * FLAGS.num_leapfrog_steps)) del ess_final ess_min, sem_min = util.get_min_ess(normalized_ess_final) util.print('ESS: {} +/- {}'.format(ess_min, sem_min)) acceptance_rate_cp = (np.sum(is_accepted_cp) * 100. / float(FLAGS.num_samples * FLAGS.num_chains)) acceptance_rate_ncp = (np.sum(is_accepted_ncp) * 100. / float(FLAGS.num_samples * FLAGS.num_chains)) return (ess_min, sem_min, acceptance_rate_cp, acceptance_rate_ncp, mcmc_time, samples, normalized_ess_final)
def get_port(self): while not self.stopped(): try: f = open(resource_path('res/port.txt')) port = int(f.readline()) f.close() return port except Exception as e: print('get_port ', e) self.sleep(0.3) return 0
def get_wordcount_per_chapter(self): if self.perchap_wc is None: print('Fetching word count per chapter for', self.shelf) self.find_count() self.deter_page_count() self.load_stories() self.perchap_wc = {} for x in self.stories: data = get_story_data(x) self.perchap_wc[x] = data['per_chapter_wc'] print('Loaded', len(self.perchap_wc), 'per-chapter word counts for', self.shelf) return self.perchap_wc
def send(self, data): try: self._client.send(pickle.dumps(data)) except Exception as e: print('send ', e) self.connected = False try: self._client.close() except: pass self._client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.barber_go_to_sleep()
def prev_theme(self, pyportal): """ Change the PyPortal display to the previous theme. Args: pyportal (adafruit_pyportal.PyPortal): A PyPortal instance. """ self.current_pos = (self.current_pos - 1) % len(self.themes) self._switch_themes(pyportal) print("Switching to previous theme.")
def init_sound_cache(self): self.cache_lock = Lock() self.cache = [] for filename in os.listdir('cache'): # Only add valid files if filename.startswith('.') or not os.path.isfile('cache/' + filename): continue with self.cache_lock: self.cache.append(bytes.fromhex(filename)) with self.cache_lock: print('%d sounds in cache.' % len(self.cache))
def deter_page_count(self): if self.pages is None: print('Determing pages for', self.shelf) soup = self.first_page pages = soup(class_="page_list")[0].ul('li') if len(pages) < 2: # only one page last_page = 1 else: last_page = pages[-2].string self.pages = int(last_page) print('Page count is', self.pages, 'for', self.shelf) return self.pages
def initialize(self, pyportal): """ Initialize the PyPortal with the default theme. Args: pyportal (adafruit_pyportal.PyPortal): A PyPortal instance. """ self.current_pos = 0 self.current_theme = None self._switch_themes(pyportal) print("Setting initial theme.")
def save(self, packet): filepath = os.path.join('cache', packet.hash.hex()) with open(filepath, 'wb') as outfile: outfile.write(packet.data) try: file = pyglet.media.load(filepath, streaming=True) print('Saved %s' % small_hash(packet.hash)) except Exception as e: print(" ! Failed to load \"" + small_hash(packet.hash) + "\": " + str(e)) else: with self.cache_lock: self.cache[packet.hash] = file
def run(self): self._client = ClientThread(self) self._client.start() self._draw = DrawThread(self, self._window) self._draw.start() while not self.stopped(): if not self._pause: self.get_clients() print('past_get_clietns') if self.clients_loaded.isSet() and self.clients > 0: # self._client.send([1]) # Стригу print('strigu') # self.sleep(0.1) self.status = 2 tmin = self._window.spinBox.value() tmax = self._window.spinBox_2.value() x = random.randint(tmin, tmax) self.count += 1 self._window.addColumn(self.count, x) self.sleep(x + 0.15) self.clients -= 1 print('decide') if self.clients > 0 and not self._pause: self.status = 1 else: self.status = 0 print('end_tick') self.sleep(0.1)
def _daemonize(): """ Daemonizes the module. Uses pub/sub Redis schema. """ global run_proc try: print("Persisting process...") signal.signal(signal.SIGTERM, _exit) signal.signal(signal.SIGINT, _exit) if (G.VERBOSE): print("Building engine...") en = engine.Engine() if (G.VERBOSE): print("Building wrapper...") ps = pubsub.Wrapper(en) print("Ready. Waiting for requests...") while run_proc: ps.heartbeat() time.sleep(10) except KeyboardInterrupt: _exit()
def play(self, steamid, hash): packet = PlaySound() packet.steamid = steamid packet.sound_hash = hash raw_packet = packet.SerializeToString() header = PacketInfo() header.type = PacketInfo.PLAY_SOUND header.length = len(raw_packet) raw_header = header.SerializeToString() print(f'{str(self)} Playing {small_hash(hash)} for {steamid}') with self.lock: for client in self.clients: with client.lock: print(f'steamid {client.steamid}') if client.steamid != steamid and client.steamid != 0: print(f'{str(self)} Playing {small_hash(hash)} @ {str(client.addr)}') client.sock.sendall(raw_header) client.sock.sendall(raw_packet) if client.round > self.round or client.round < self.round - 1: self.round = client.round self.round_events = [] print(f'Done playing {small_hash(hash)}')
def send_sound(self, packet): """Handles a SoundRequest packet - LOCKS""" with self.server.cache_lock: if not packet.sound_hash in self.server.cache: return res = SoundResponse() with open('cache/' + packet.sound_hash.hex(), 'rb') as infile: res.data = infile.read() res.hash = packet.sound_hash print(f'{self.steamid} is downloading {small_hash(packet.sound_hash)}') self.send(PacketInfo.SOUND_RESPONSE, res) print(f'{self.steamid} is done downloading {small_hash(packet.sound_hash)}')
def get_user_shelves(username, password): lib = [] load_from_site = user_bool(input('Use all bookshelves on the site (y/n)? ')) if load_from_site: return autharea.get_user_shelves(username, password) inp = '' while True: inp = input('Next bookshelf or return to finish\n%s: ' % lib) if inp.isdigit(): lib.append(int(inp)) print(lib) else: break return lib
def play_received(self, hash): """Try playing a sound if it was received quickly enough""" with self.cache_lock: if hash not in self.wanted_sounds: return wanted_time = self.wanted_sounds[hash] self.wanted_sounds.remove(hash) if wanted_time + 1000 > datetime.now(): return player = pyglet.media.Player() player.volume = math.pow(self.volume, 2) / 10000 player.queue(self.cache[hash]) player.play() print('[+] Playing %s (%f ms late)' % (small_hash(hash), datetime.now() - wanted_time))
def _switch_themes(self, pyportal): next_theme = self.themes[self.current_pos] if self.current_theme: next_theme.update_time( self.current_theme.days, self.current_theme.hours, self.current_theme.mins, ) self.current_theme.clear(pyportal) self.current_theme = next_theme self.current_theme.apply(pyportal) print("Setting theme to: ", self.current_theme.bg)
def update(self, packet): with self.lock: print(f'({self.addr}) {self.steamid} -> {packet.steamid}') self.steamid = packet.steamid if self.shard == None and packet.shard_code == b'': return if self.shard != None and self.shard.name == packet.shard_code: return # Don't allow client to switch shards more than 1x/second # TODO add restriction on client if (datetime.datetime.now() - self.last_shard_change).seconds < 1: return self.leave_shard() self.join_shard(packet.shard_code)
def listen(self): while not self.stopped(): try: data = self.connection().recv(1024) data = pickle.loads(data) if data[0] == 0: #self.sleep(0.05) self.send([0, self._parent.clients]) self._parent.clients = max(0, self._parent.clients - 1) elif data[0] == 1: self._parent.clients -= 1 print(data) except Exception as e: print('listen_ex ', e) if self._conn is not None: self._conn.close() self._conn = None return
def handle_event(self, packet): with self.lock: self.round = packet.round event_class = get_event_class(packet) if len(packet.proposed_sound_hash) != 64: print('This is not okay: %s' % str(packet)) print('Serialized: %s' % str(packet.SerializeToString())) self.check_or_request_sounds([packet.proposed_sound_hash]) with self.lock: if self.shard == None: return shard = self.shard steamid = self.steamid if event_class == 'normal' else 0 if event_class == 'shared': shard.play_shared(packet.update, packet.proposed_sound_hash) else: shard.play(steamid if event_class == 'normal' else 0, packet.proposed_sound_hash)
def main(): parsed_args = parser.parse_args() params = Parameters() util.write_parsed_args(params, parsed_args) if params.rand_seed != -1: random.seed(params.rand_seed) np.random.seed(params.rand_seed) torch.random.manual_seed(params.rand_seed) if params.stage == "train_analysts": train_analysts(params) elif params.stage == "pretrain_manipulators": pretrain_manipulators(params) elif params.stage == "train_manipulators": train_manipulators(params) elif params.stage == "playground": playground(params) else: print("Unrecognized stage: " + params.stage) exit()
def main(_): # tf.logging.set_verbosity(tf.logging.ERROR) np.warnings.filterwarnings('ignore') util.print('Loading model {} with dataset {}.'.format( FLAGS.model, FLAGS.dataset)) model_config = models.get_model_by_name(FLAGS.model, dataset=FLAGS.dataset) if FLAGS.results_dir == '': results_dir = FLAGS.model + '_' + FLAGS.dataset else: results_dir = FLAGS.results_dir if not tf.io.gfile.exists(results_dir): tf.io.gfile.makedirs(results_dir) filename = '{}{}{}{}{}.json'.format( FLAGS.method, ('_' + FLAGS.learnable_parameterisation_type if 'VIP' in FLAGS.method else ''), ('_tied' if FLAGS.tied_pparams else ''), ('_reparam_variational' if 'VIP' in FLAGS.method and FLAGS.reparameterise_variational else ''), ('_discrete_prior' if 'VIP' in FLAGS.method and FLAGS.discrete_prior else '')) file_path = os.path.join(results_dir, filename) if FLAGS.inference == 'VI': run_vi(model_config, results_dir, file_path) elif FLAGS.inference == 'HMC': if FLAGS.method == 'i': run_interleaved_hmc(model_config, results_dir, file_path) else: run_hmc(model_config, results_dir, file_path, tuning=False) elif FLAGS.inference == 'HMCtuning': run_hmc(model_config, results_dir, file_path, tuning=True)
def update_local_time_from_internet(pyportal, timezone="Etc/UTC", debug=False): """ Fetches the local time from the internet, and sets it on the PyPortal. Make sure you get the local time at the timezone you want, since the location set in your secrets file can override this value. Set debug to skip fetching time from the internet. Useful for faster startup time while reloading code. TODO NZ: Figure out why timezone doesn't match https://pythonclock.org/ TODO NZ: The pyportal library clobbers all exceptions, and sleeps. Rewrite for better error handling. Args: pyportal (adafruit_pyportal.PyPortal): PyPortal instance. timezone (str, optional): Timezone to fetch time from. Overwritten by value in secrets.py. Defaults to "Etc/UTC". debug (bool, optional): Use the rtc clock time if set. Defaults to False. Returns: float: Monotonic timestamp of the current time. """ is_rtc_clock_set = rtc.RTC().datetime.tm_year != 2000 if debug and is_rtc_clock_set: print("Debug mode. Using cached localtime.") else: print("Trying to update local time from internet.") pyportal.get_local_time(location=timezone) time_now = time.monotonic() print("Time last refreshed at", time_now) return time_now
def send(self, update_type, state): """Sends a sound to play for everybody""" if self.client == None: return hash = self.get_random(update_type, state) if hash != None: if hash == b'': print('?????????????????????????????') print('Update type: %s' % str(update_type)) print('?????????????????????????????') packet = GameEvent() packet.update = update_type packet.proposed_sound_hash = hash packet.kill_count = int(state.round_kills) packet.round = int(state.current_round) self.client.send(PacketInfo.GAME_EVENT, packet) # Normal event : play without waiting for server if get_event_class(packet) == 'normal': playpacket = PlaySound() playpacket.steamid = 0 playpacket.sound_hash = hash self.play(playpacket)
def play(self, packet): if self.loaded is False: return True if str(packet.steamid) != self.playerid and packet.steamid != 0: return True with self.cache_lock: # Sound is already loaded if packet.sound_hash in self.cache: print('[+] Playing %s' % small_hash(packet.sound_hash)) player = pyglet.media.Player() player.volume = math.pow(self.volume, 2) / 10000 player.queue(self.cache[packet.sound_hash]) player.play() return True else: filename = os.path.join('cache', packet.sound_hash.hex()) if os.path.isfile(filename): # Sound is downloaded but not loaded print('[+] Loading and playing %s' % small_hash(packet.sound_hash)) self.cache[packet.sound_hash] = pyglet.media.load( filename, streaming=True) player = pyglet.media.Player() player.volume = math.pow(self.volume, 2) / 10000 player.queue(self.cache[packet.sound_hash]) player.play() return True else: # Sound is not downloaded print('[!] Sound %s missing, requesting from server' % small_hash(packet.sound_hash)) self.wanted_sounds[packet.sound_hash] = datetime.now() return False
def main(method='', proxy=None, bookshelves=[], username=None, password=None): util.output.open() try: bookshelves = bookshelves or get_user_shelves(username, password) method = method or input('Choose a analyzer ' +\ str(ALL).replace('[', '(').replace(']', ')') +\ ':') # setup login lenbook = len(bookshelves) print('Connected to FimFiction, analyzing ' + number_objects(lenbook, 'bookshel(f|ves)') + '.') shelves = [] for shelf in bookshelves: obj = shelfmanager.Shelf(int(shelf), username, password) shelves.append(obj) # call the appropriate method globals()[method](shelves) input('Press enter to exit') except SystemExit: pass except KeyboardInterrupt: pass except BaseException as e: # scope things def do_fail(): debug = False try: fail('Error: ' + str(e).encode('ascii', errors='replace').decode('ascii')) except SystemExit: pass except AssertionError: debug = True return debug reraise = do_fail() if reraise: raise finally: util.output.close()
def get_wordcount(self): if self.wordcount is None: print('Fetching word count for', self.shelf) self.find_count() self.deter_page_count() self.load_stories() self.wordcount = 0 counted = 0 for story in self.stories: wc = get_story_data(story)['story_wc'] counted += 1 self.wordcount += wc print('{}/{} stories calculated, current word count is {}'.format( counted, len(self.stories), self.wordcount)) print('Loaded word count for', self.shelf) return self.wordcount
def load_stories(self): if self.stories is None: print('Loading story urls for', self.shelf) s = [] for page in range(self.pages): print('Loading page', page, 'out of', self.pages, 'for', self.shelf) soup = self.first_page if page == 0 else bs4.BeautifulSoup(get_page(self.shelf, page + 1, LIST_VIEW), 'lxml') bold_tags = soup(class_="search_results_count")[0]('b') from_ = int(bold_tags[0].string) to = int(bold_tags[1].string) # there are 1-60 stories on the first page which means 60, but 60-1=59 so we add one count = (to - from_) + 1 story_list = soup(class_="story-list")[0]('li') for story in story_list: s.append(story(class_="right")[0].h2.a['href']) self.stories = tuple(s) print(number_objects(len(self.stories), 'url(|s)'), 'loaded for', self.shelf) return self.stories
def read_by_story(allshelves): print('By Story NYI')
def read_by_chapter(allshelves): print('By Chapter NYI')
def prompt(choices, mode='*'): if mode not in PROMPT_MODES: raise ValueError("mode '{}' is invalid".format(mode)) if len(choices) > 26: raise ValueError("too many choices") if mode == '*': header = "select zero or more:" max, min = float('inf'), 0 elif mode == '+': header = "select one or more:" max, min = float('inf'), 1 elif mode in [1, '1']: header = "select one:" max, min = 1, 1 elif mode == '?': header = "select zero or one:" max, min = 1, 0 letters = list(map(lambda x: chr(ord('a') + x), range(len(choices)))) num_selections = 0 selections = [] # unique indices into choices list while num_selections < min or num_selections < max: util.print(util.green(header)) for i in range(len(choices)): if i in selections: choice = " × " else: choice = " " choice += str(letters[i]) + '. ' + str(choices[i]) if i in selections: choice = util.yellow(choice) util.print(choice) try: sel = input(util.green("make a selection (or ! to commit): ")) except KeyboardInterrupt: util.exit(util.ERR_INTERRUPTED) if sel == '!': if num_selections < min: util.error("can't stop now; you must make " "{} {}".format(min, util.plural("selection", min))) continue else: break try: if letters.index(sel) in selections: selections.remove(letters.index(sel)) continue selections.append(letters.index(sel)) num_selections += 1 except ValueError: if sel == '': util.print("make a selection (or ! to commit)") else: util.error("invalid selection: not in list") continue return selections
def total_words(allshelves: list): words = 0 for s in allshelves: words += s.get_wordcount() print('Total words for', allshelves, '=', words)