def deregister(self, usernames: List[str]): with self.lock: if tuple(usernames) not in self.user_blocks: mylog.warning("Tried to remove nonexistent usernames entry {}".format(usernames)) return self.user_blocks.remove(tuple(usernames)) mylog.info("Successfully removed {}".format(usernames))
def test_batching_x(n, batch): if not RUN_SLOW_TESTS: mylog.warning("[SKIP] Skipping slow test") return mylog.info("Testing batching.TweetBatcher:") conn = mq.PrintQueue.new('test_batching') batcher = mq.Batcher(conn) expected = [] for i in range(n): s = "Should be batch {batch}, message {i}/{n}" \ .format(batch=batch, i=i, n=n) expected.append(s) batcher.post(s) time.sleep(mq.BATCH_TIMEOUT / 2.0) if hasattr(conn, 'expect'): conn.expect([]) time.sleep(mq.BATCH_TIMEOUT) if hasattr(conn, 'expect'): # Expect precisely one message with all "parts" bundled up. conn.expect([expected]) expected = 'Check second run' batcher.post(expected) time.sleep(mq.BATCH_TIMEOUT / 2.0) if hasattr(conn, 'expect'): conn.expect([]) time.sleep(mq.BATCH_TIMEOUT) if hasattr(conn, 'expect'): batch = [expected] all_batches = [batch] conn.expect(all_batches)
def resolve_name(self, username: str): try: return str(self.api.get_user(username).id) except Exception as e: mylog.warning("Couldn't resolve username '{}': {}".format( username, e)) return None
def deregister(self, usernames: List[str]): with self.lock: if tuple(usernames) not in self.user_blocks: mylog.warning( "Tried to remove nonexistent usernames entry {}".format( usernames)) return self.user_blocks.remove(tuple(usernames)) mylog.info("Successfully removed {}".format(usernames))
def _remove_citizen(self, tid, token): with self.lock: mylog.info("Want to remove citizen {}, token {}".format(tid, token)) if tid not in self.citizens: mylog.warning("=> Already deleted (huh?)") elif self.citizens[tid]['token'] != token: mylog.info("=> Token mismatch, db has {}" .format(self.citizens[tid]['token'])) else: mylog.info("=> Yup") self.twitter.deregister([tid]) del self.citizens[tid] mylog.info("Remaining citizens: {}".format(self.citizens.keys()))
def _remove_citizen(self, tid, token): with self.lock: mylog.info("Want to remove citizen {}, token {}".format( tid, token)) if tid not in self.citizens: mylog.warning("=> Already deleted (huh?)") elif self.citizens[tid]['token'] != token: mylog.info("=> Token mismatch, db has {}".format( self.citizens[tid]['token'])) else: mylog.info("=> Yup") self.twitter.deregister([tid]) del self.citizens[tid] mylog.info("Remaining citizens: {}".format( self.citizens.keys()))
def test_messages_phrasing(): mylog.info("Testing messages.phrase:") expected = { 'twittername': 'heinzelmännchen', 'status': 'fail', 'reason': 'unknown-user', 'message': { "de": 'Konnte "heinzelmännchen" nicht auf Twitter finden.', "en": 'Could not find "heinzelmännchen" on twitter.' } } actual = messages.phrase("heinzelmännchen", 'unknown-user') assert actual == expected, (actual, expected) mylog.warning( "[MANU] Testing mq.RealQueue. Check http://localhost:15672/#/queues/%2F/test_mq" )
def test_mq(): def run_with(maker): conn = maker("test_mq") if hasattr(conn, 'expect'): conn.expect([]) for s in ["My lovely message", "Ümläöts Partyß! Or … what¿"]: conn.post(s) if hasattr(conn, 'expect'): conn.expect([s]) mylog.info("Testing mq.PrintQueue:") run_with(mq.PrintQueue.new) mylog.warning( "[MANU] Testing mq.RealQueue. Check http://localhost:15672/#/queues/%2F/test_mq" ) run_with(mq.RealQueue.new)
def run_short_poll(self): # Need to hold the lock *all* the time, as the self.api object might be modified # by the other objects. # WARNING: This opens up a convoluted but definite attack vector: # If you can manage to delay a REST call for a very long time, then the backend grinds to a halt. # Then again, with this level of network control you can always do that. # This is why I actually ignore this attack vector. with self.lock: if len(self.short_poll) > 2: retained = self.short_poll[-2:] # The two last entries dropped = self.short_poll[:-2] # Other entries mylog.warning('Lots of citizens! Dropping {}, retaining {} ...'.format(dropped, retained)) for e in dropped: self.consumer_updates.updateShortpoll(e['name'], 'del-toomany') self.short_poll = retained assert len(self.short_poll) <= 2, self.short_poll for e in self.short_poll: self.poll_user(e)
def addCitizen(self, twittername, birdid, tid=None): if tid is None: tid = self.twitter.resolve_name(twittername) if tid is None: mylog.warning("citizen user ignored, invalid name: " + twittername) self.consumer_updates.updateShortpoll(twittername, "unknown-user") return if self.polBack.getPolitician(tid) is not None: self.consumer_updates.updateShortpoll(twittername, "is-politician") return if birdid not in self.birdBack.bJson: mylog.warning("citizen user ignored, invalid bird: " + birdid) self.consumer_updates.updateShortpoll(twittername, "unknown-bird") return with self.lock: if tid in self.citizens: entry = self.citizens[tid] mylog.info( "Updating existing citizen's bird from {}".format(entry)) else: mylog.info("Creating new citizen's bird") entry = dict() entry["userId"] = tid entry["party"] = 'neutral' self.citizens[tid] = entry # Even if a tweet comes in instantly, getCitizen syncs on # self.lock, so it's fine. That's also why getCitizen() will # never see an incomplete citizen. self.twitter.register_shortlived(tid, twittername) entry["birdId"] = birdid token = poll_counter() entry["token"] = token mylog.debug("Resulting citizen entry: {}".format(entry)) timer = threading.Timer(REMOVE_CITIZEN_TIME, self._remove_citizen_wrap, [tid, token]) # Don't prevent shutting down timer.daemon = True timer.start() self.consumer_updates.updateShortpoll(twittername, "succ-resolved") return
def addCitizen(self, twittername, birdid, tid=None): if tid is None: tid = self.twitter.resolve_name(twittername) if tid is None: mylog.warning("citizen user ignored, invalid name: " + twittername) self.consumer_updates.updateShortpoll(twittername, "unknown-user") return if self.polBack.getPolitician(tid) is not None: self.consumer_updates.updateShortpoll(twittername, "is-politician") return if birdid not in self.birdBack.bJson: mylog.warning("citizen user ignored, invalid bird: " + birdid) self.consumer_updates.updateShortpoll(twittername, "unknown-bird") return with self.lock: if tid in self.citizens: entry = self.citizens[tid] mylog.info("Updating existing citizen's bird from {}".format(entry)) else: mylog.info("Creating new citizen's bird") entry = dict() entry["userId"] = tid entry["party"] = 'neutral' self.citizens[tid] = entry # Even if a tweet comes in instantly, getCitizen syncs on # self.lock, so it's fine. That's also why getCitizen() will # never see an incomplete citizen. self.twitter.register_shortlived(tid, twittername) entry["birdId"] = birdid token = poll_counter() entry["token"] = token mylog.debug("Resulting citizen entry: {}".format(entry)) timer = threading.Timer(REMOVE_CITIZEN_TIME, self._remove_citizen_wrap, [tid, token]) # Don't prevent shutting down timer.daemon = True timer.start() self.consumer_updates.updateShortpoll(twittername, "succ-resolved") return
def post(self, message, isRetry=False): mylog.info('Publishing on queue {name}: {data!r}' .format(name=self.name, data=message)) with self.lock: if self.connection.is_closed: mylog.warning("Whoops, connection is closed; reopen.") self.connection = connection() self.channel = self.connection.channel() try: self.channel.basic_publish(exchange='', routing_key=self.name, body=json.dumps(message)) self.maybe_log_message(message) except Exception as e: mylog.error("Connection failed anyway? Make sure RabbitMQ is running! (is_closed = {})" .format(self.connection.is_closed)) mylog.error(e.__repr__()) mylog.error(e.args) mylog.info("Message dropped.") if not isRetry: self.post(message, True)
def find_pair(bird: Union[None, str], mood: str, retweet: bool, length: int): if bird is None: return None candidates = [(bird, mood, retweet), (bird, 'neutral', retweet), (bird, 'fragend', retweet), (bird, 'aufgebracht', retweet), ('amsel', 'neutral', False)] verbose = False for (b, m, r) in candidates: candidSource = path_raw(b, m, r) if os.path.isfile(candidSource): if verbose: mylog.info("Found at {}".format(candidSource)) return candidSource, path_processed(b, m, r, length) else: mylog.warning("Source file {} missing, falling back ...".format(candidSource)) verbose = True mylog.error("All sources and fallbacks missing. Giving up.") return None
def run_short_poll(self): # Need to hold the lock *all* the time, as the self.api object might be modified # by the other objects. # WARNING: This opens up a convoluted but definite attack vector: # If you can manage to delay a REST call for a very long time, then the backend grinds to a halt. # Then again, with this level of network control you can always do that. # This is why I actually ignore this attack vector. with self.lock: if len(self.short_poll) > 2: retained = self.short_poll[-2:] # The two last entries dropped = self.short_poll[:-2] # Other entries mylog.warning( 'Lots of citizens! Dropping {}, retaining {} ...'.format( dropped, retained)) for e in dropped: self.consumer_updates.updateShortpoll( e['name'], 'del-toomany') self.short_poll = retained assert len(self.short_poll) <= 2, self.short_poll for e in self.short_poll: self.poll_user(e)
def deregister(self, usernames: List[str]): with self.lock: if tuple(usernames) in self.streams: s = self.streams[tuple(usernames)] del self.streams[tuple(usernames)] s.disconnect() else: # Technically, we don't have any timing guarantees about # when run_short_poll can prune the list down to size 2. # Practically, if we hit this limit, something has gone very wrong. assert len(self.short_poll) < 100, self.short_poll # Split list based on predicate "Should it be removed?" users_pruned = [] users_passed = [] for e in self.short_poll: l = users_pruned if e['user'] in usernames else users_passed l.append(e) if len(users_pruned) != len(usernames): mylog.warning("Tried to remove nonexistent usernames entries.") mylog.warning(" Actually pruned: {}", users_pruned) mylog.warning(" Wanted to prune: {}", usernames) if len(users_pruned) > 0: mylog.info("Some short-polls removed: {}".format(users_pruned)) self.short_poll = users_passed for e in users_pruned: self.consumer_updates.updateShortpoll(e['name'], 'del-timeout') return
def deregister(self, usernames: List[str]): with self.lock: if tuple(usernames) in self.streams: s = self.streams[tuple(usernames)] del self.streams[tuple(usernames)] s.disconnect() else: # Technically, we don't have any timing guarantees about # when run_short_poll can prune the list down to size 2. # Practically, if we hit this limit, something has gone very wrong. assert len(self.short_poll) < 100, self.short_poll # Split list based on predicate "Should it be removed?" users_pruned = [] users_passed = [] for e in self.short_poll: l = users_pruned if e['user'] in usernames else users_passed l.append(e) if len(users_pruned) != len(usernames): mylog.warning( "Tried to remove nonexistent usernames entries.") mylog.warning(" Actually pruned: {}", users_pruned) mylog.warning(" Wanted to prune: {}", usernames) if len(users_pruned) > 0: mylog.info( "Some short-polls removed: {}".format(users_pruned)) self.short_poll = users_passed for e in users_pruned: self.consumer_updates.updateShortpoll( e['name'], 'del-timeout') return
def post(self, message, isRetry=False): mylog.info('Publishing on queue {name}: {data!r}'.format( name=self.name, data=message)) with self.lock: if self.connection.is_closed: mylog.warning("Whoops, connection is closed; reopen.") self.connection = connection() self.channel = self.connection.channel() try: self.channel.basic_publish(exchange='', routing_key=self.name, body=json.dumps(message)) self.maybe_log_message(message) except Exception as e: mylog.error( "Connection failed anyway? Make sure RabbitMQ is running! (is_closed = {})" .format(self.connection.is_closed)) mylog.error(e.__repr__()) mylog.error(e.args) mylog.info("Message dropped.") if not isRetry: self.post(message, True)
def poli_modify(): check_writeback() set_skip_writeback(False) check_writeback() mylog.warning("Fiddling with politicianBackend.") pb = PoliticianBackend() # Do your thing here, e.g.: # pB.setBird('395912134', 'fitis', 'c') # # This is a no-op on the original pols.json # Or to purge everything French: # for poli in pb.poliList: # cv = poli['cv'] # if 'fr' in cv: # mylog.info('Purged the French out of ' + poli['name']) # del cv['fr'] # Note: # - setBird automatically calls dumpToFile. # In all other cases, you'll need to call __dumpToFile by hand, like this: pb._dumpToFile() mylog.warning("Now check by hand.")
def on_exception(self, exception): # Fun fact: even if no thread is runnable, # the existence of a Timer keeps Python alive. threading.Timer(RESPAWN_PERIOD, self.restarter.restart_now).start() # tweepy has lots of bugs. Backend and tweepy exception will # result in this code being called, so use it as a trampoline. # Log it 'only' as a warning, since if we get here, this isn't too bad anyway. mylog.warning("{} on_exception! Trying to print it:".format(self.desc)) mylog.warning("(You'll see the same error immediately again, but don't" " worry, I'm a Phoenix, I'll get revived in a few seconds.)") mylog.warning(exception)
def on_exception(self, exception): # Fun fact: even if no thread is runnable, # the existence of a Timer keeps Python alive. threading.Timer(RESPAWN_PERIOD, self.restarter.restart_now).start() # tweepy has lots of bugs. Backend and tweepy exception will # result in this code being called, so use it as a trampoline. # Log it 'only' as a warning, since if we get here, this isn't too bad anyway. mylog.warning("{} on_exception! Trying to print it:".format( self.desc)) mylog.warning( "(You'll see the same error immediately again, but don't" " worry, I'm a Phoenix, I'll get revived in a few seconds.)") mylog.warning(exception)
def _dumpToFile(self): if _SKIP_WRITEBACK: mylog.warning("=" * 77) mylog.warning("politicianBackend: skipping write-back <THIS SHOULD NOT HAPPEN IN PRODUCTION>") mylog.warning("=" * 77) return with open(BACKEND_POLI_DB, 'w') as outfile: json.dump(self.polByPid, outfile, sort_keys=True, indent=2) with open(FRONTEND_POLI_DB, "w") as out: out.write("@politicians = ") json.dump(self.polByPid, out, sort_keys=True, indent="\t") for line in fileinput.input([FRONTEND_POLI_DB], inplace=True): print('\t' + line.rstrip('\n')) # WHITELISTED PRINT
def handle_poli(self, tweet, msg, poli): # Careful: 'poli' is a copy, so any changes due to setBird aren't reflected! msg['partycolor'] = party_to_color(poli['party']) msg['party'] = poli['party'] pBird = poli['self_bird'] # In case it changed, use the one provided by twitter handle = msg['twitterName'] has_command = contains_command(tweet['hashtags']) # Check for any updates if 'house' in tweet['username'].lower( ) and tweet['content'].startswith('@'): mylog.warning( "Ignoring my own tweet for commands, as it starts with '@'") elif has_command: pid = poli['pid'] pBird_name = self.birdBack.getName(pBird) bird_id = find_bird(tweet['content'], self.birdBack) reply = None if bird_id is not None: # Ack bird_name = self.birdBack.getName(bird_id) mylog.info('politician "{}" ({}) gets new bird {}'.format( tweet['userscreen'], pid, bird_id)) msg['refresh'] = dict() msg['refresh']['politicianId'] = pid msg['refresh']['birdId'] = bird_id self.pb.setBird(pid, bird_id, actor='p') reply = responseBuilder.build_some_ack(handle, pBird_name, bird_name) # Again, 'poli' is a copy, so it wasn't updated by the call to 'setBird'. pBird = bird_id elif has_command != COMMAND_HASHTAGS_ACKONLY: # NACK mylog.warning('I saw that command, but no valid bird!') mylog.warning('pid={pid!r} content={ct}'.format( ct=tweet['content'], pid=pid)) reply = responseBuilder.build_some_nack(handle, pBird_name) if reply is not None: self.tw.twitter.maybe_reply(tweet['tweet_id'], reply) # In case of 'refresh', poli already contains the update: return [poli['citizen_bird'], pBird]
def handle_poli(self, tweet, msg, poli): # Careful: 'poli' is a copy, so any changes due to setBird aren't reflected! msg['partycolor'] = party_to_color(poli['party']) msg['party'] = poli['party'] pBird = poli['self_bird'] # In case it changed, use the one provided by twitter handle = msg['twitterName'] has_command = contains_command(tweet['hashtags']) # Check for any updates if 'house' in tweet['username'].lower() and tweet['content'].startswith('@'): mylog.warning("Ignoring my own tweet for commands, as it starts with '@'") elif has_command: pid = poli['pid'] pBird_name = self.birdBack.getName(pBird) bird_id = find_bird(tweet['content'], self.birdBack) reply = None if bird_id is not None: # Ack bird_name = self.birdBack.getName(bird_id) mylog.info('politician "{}" ({}) gets new bird {}' .format(tweet['userscreen'], pid, bird_id)) msg['refresh'] = dict() msg['refresh']['politicianId'] = pid msg['refresh']['birdId'] = bird_id self.pb.setBird(pid, bird_id, actor='p') reply = responseBuilder.build_some_ack(handle, pBird_name, bird_name) # Again, 'poli' is a copy, so it wasn't updated by the call to 'setBird'. pBird = bird_id elif has_command != COMMAND_HASHTAGS_ACKONLY: # NACK mylog.warning('I saw that command, but no valid bird!') mylog.warning('pid={pid!r} content={ct}' .format(ct=tweet['content'], pid=pid)) reply = responseBuilder.build_some_nack(handle, pBird_name) if reply is not None: self.tw.twitter.maybe_reply(tweet['tweet_id'], reply) # In case of 'refresh', poli already contains the update: return [poli['citizen_bird'], pBird]
def test_sound_gen(): # Don't even attempt to prevent writing to disk. Overwriting is perfectly # fine, as we don't overwrite anything important, and everything is in git. mylog.warning( "Please check by hand whether this generates the file on the") mylog.warning("first call and uses the cached version on the second:") actual = soundGenerator.generate_sound('Cheerio, buddy', False, 'amsel', 'amsel') # Set as 'warning' so that the user always sees both (or neither). mylog.warning("(end of manual part)") path_amsel = os.path.join(soundGenerator.SOUND_ROOT, 'processed', 'amsel-neutral-10000_v2.mp3') desc_amsel = {'natural': path_amsel, 'bid': 'amsel', 'duration': 10000} expected = {'citizen': desc_amsel, 'poli': desc_amsel} assert actual == expected, (actual, expected) content = "How can mirrors be real if our eyes aren't real?" actual = soundGenerator.generate_sound(content, True, 'zilpzalp', None) path_zz = os.path.join(soundGenerator.SOUND_ROOT, 'processed', 'zilpzalp-fragend-r-12000_v2.mp3') desc_zz = {'natural': path_zz, 'bid': 'zilpzalp', 'duration': 12000} expected = {'citizen': desc_zz} assert actual == expected, (actual, expected) soundGenerator.processed_tweets = 0
def resolve_name(self, username: str): try: return str(self.api.get_user(username).id) except Exception as e: mylog.warning("Couldn't resolve username '{}': {}".format(username, e)) return None
# except Exception,msg: # print msg # return False startpath = self.startpath # print startpath pid = self.check_process(startpath) if pid: try: psutil.Process(pid).kill() mylog.info('关闭服务成功,进程号为%s'%pid) return True except Exception,msg: mylog.error('关闭服务失败,%s'%msg) return False else: mylog.warning('关闭服务失败,服务未启动') return True def zip_dir(self,copypath,testpath): '''打包备份''' # copypath = self.copypath # testpath = self.testpath nowtime = time.strftime("%Y%m%d%H%M%S", time.localtime()) if os.path.isdir(copypath): zipFile1 = zipfile.ZipFile(os.path.join(copypath,os.path.basename(testpath)+nowtime+'.zip'),'w') else: mylog.error('备份失败,备份目录不合法') return False
def set_skip_writeback(to: bool): global _SKIP_WRITEBACK _SKIP_WRITEBACK = to mylog.warning("politicianBackend: UPDATE SKIP_WRITEBACK = {}".format(_SKIP_WRITEBACK))
def poli_modify(): check_writeback() set_skip_writeback(False) check_writeback() mylog.warning("Fiddling with politicianBackend.") pb = PoliticianBackend() # Do your thing here, e.g.: # pB.setBird('395912134', 'fitis', 'c') # # This is a no-op on the original pols.json # Or to purge everything French: # for poli in pb.poliList: # cv = poli['cv'] # if 'fr' in cv: # mylog.info('Purged the French out of ' + poli['name']) # del cv['fr'] # Note: # - setBird automatically calls dumpToFile. # In all other cases, you'll need to call __dumpToFile by hand, like this: pb._dumpToFile() mylog.warning("Now check by hand.") if __name__ == '__main__': mylog.warning("Are you sure you want to rewrite pols.json?") mylog.error("Uncomment me, I'm not gonna let ya!") # Uncomment to run: # poli_modify()
def test_twitter_citizenship(): # This test won't change anything about the politicians, # but still, better play it safe. politicianBackend.check_writeback() politicianBackend.set_skip_writeback(True) politicianBackend.check_writeback() birdBack = birdBackend.BirdBackend() polBack = politicianBackend.PoliticianBackend() follow = ["4718199753", "774336282101178368"] queue = mq.PrintQueue("twitter_conn_test") fakeTwitter = twitter.FakeTwitterInterface() twi = twitterConnection.TwitterConnection(queue, follow, polBack, birdBack, fakeTwitter, twitter.UpdatesPrinter()) queue.expect([]) twi.addCitizen("Heinz1", "Katzastrophe", tid="12345678") queue.expect([]) assert not twi.isPoli("12345678") assert twi.getCitizen("12345678") is None assert twi.citizens == dict() # Must be able to handle "decapitalization" twi.addCitizen("Heinz2", 'ara', tid="12345679") queue.expect([]) assert not twi.isPoli("12345679") assert twi.getCitizen("12345679") is not None assert twi.getCitizen("12345679")['birdId'] == 'ara' assert twi.citizens.keys() == {'12345679'} # Be able to deal with erroneous removals # noinspection PyProtectedMember twi._remove_citizen('123456', 666) # token of the beast assert twi.citizens.keys() == {'12345679'} queue.expect([]) if RUN_SLOW_TESTS: mylog.info("This wakeup should be completely silent.") long_wait(twitterConnection.REMOVE_CITIZEN_TIME / 2) twi.addCitizen("Heinz3", 'zilpzalp', tid="12345679") queue.expect([]) assert not twi.isPoli("12345679") assert twi.getCitizen("12345679") is not None assert twi.getCitizen("12345679")['birdId'] == 'zilpzalp' assert twi.citizens.keys() == {'12345679'} if not RUN_SLOW_TESTS: mylog.warning("The slow part of test_twitter_citizenship().") mylog.warning( "You might see several stray 'citizen removed' messages.") return mylog.info("This wakeup should be a no-op.") long_wait(twitterConnection.REMOVE_CITIZEN_TIME / 2 + 10) queue.expect([]) assert not twi.isPoli("12345679") assert twi.getCitizen("12345679") is not None assert twi.getCitizen("12345679")['birdId'] == 'zilpzalp' assert twi.citizens.keys() == {'12345679'} mylog.info("This wakeup should remove it, as citizen deletion") mylog.info("has been deferred when the bird was updated.") long_wait(twitterConnection.REMOVE_CITIZEN_TIME / 2 + 10) queue.expect([]) assert not twi.isPoli("12345679") assert twi.getCitizen("12345679") is None assert twi.citizens.keys() == set()
def on_warning(self, notice): mylog.warning("{} on_warning: {}".format(self.desc, notice))