Пример #1
0
  def __init__(self, node_index, geometry_handler):
    self.geom_handler = geometry_handler
    self.idx = node_index
    super(OSMHandler, self).__init__()

    self.counters = Counters()
    self.mp_cache = PendingMultipolyCache()
    self.receive_rels = True
Пример #2
0
 def __init__(self, run, targets, masks):
     self._run = run
     self._targets = targets
     self._masks = masks
     self.__adapter_t_rc = 0
     self.uses_tags = False
     self.counters = Counters(self._run)
     self._match_mask = self._match_mask_optimized if (run.masks[0] == 'RRRY' and run.masks[1] == 'YYYR') else self._match_mask_general
     if self._match_mask != self._match_mask_optimized:
         print("Warning: not using optimized mask match.")
     run.apply_config_restrictions()
     self.prepare()
Пример #3
0
def create_new_counter():
    if request.method == 'POST':
        name = request.form['name']
        counter = Counters(name=name)
        db.session.add(counter)
        db.session.commit()
        return redirect('/counters_list')
    return render_template('new_counter.html')
Пример #4
0
 def __init__(self, run, targets, masks):
     self._run = run
     self._targets = targets
     self._masks = masks
     self.__adapter_t_rc = 0
     self.uses_tags = False
     self.counters = Counters(self._run)
     self._match_mask = self._match_mask_optimized if (run.masks[0] == 'RRRY' and run.masks[1] == 'YYYR') else self._match_mask_general
     if self._match_mask != self._match_mask_optimized:
         print("Warning: not using optimized mask match.")
     run.apply_config_restrictions()
     self.prepare()
Пример #5
0
 def reset_counts(self):
     self.counters = Counters(self._run)
Пример #6
0
class PairProcessor(object):

    def __init__(self, run, targets, masks):
        self._run = run
        self._targets = targets
        self._masks = masks
        self.__adapter_t_rc = 0
        self.uses_tags = False
        self.counters = Counters(self._run)
        self._match_mask = self._match_mask_optimized if (run.masks[0] == 'RRRY' and run.masks[1] == 'YYYR') else self._match_mask_general
        if self._match_mask != self._match_mask_optimized:
            print("Warning: not using optimized mask match.")
        run.apply_config_restrictions()
        self.prepare()

    def exists(self):
        return True

    def reset_counts(self):
        self.counters = Counters(self._run)

    @property
    def _adapter_t_rc(self):
        if not self.__adapter_t_rc:
            self.__adapter_t_rc = reverse_complement(self._run.adapter_t)
        return self.__adapter_t_rc

    def _match_mask_general(self, pair):
        if pair.mask:
            self.counters.increment_mask(pair.mask, pair.multiplicity)
            return True
        seq = pair.r1.original_seq
        for mask in self._masks:
            if mask.matches(seq):
                pair.set_mask(mask)
                self.counters.increment_mask(mask, pair.multiplicity)
                return True
        self.counters.mask_failure += pair.multiplicity
        pair.failure = Failures.mask
        return False

    # optimized version for RRRY/YYYR
    def _match_mask_optimized(self, pair):
        if pair.mask:
            self.counters.increment_mask(pair.mask, pair.multiplicity)
            return True
        mask = match_mask_optimized(pair.r1.original_seq, self._masks)
        if mask:
            pair.set_mask(mask)
            self.counters.increment_mask(mask, pair.multiplicity)
            return True
        else:
            self.counters.mask_failure += pair.multiplicity
            pair.failure = Failures.mask
            return False

    def _check_indeterminate(self, pair):
        if not self._run.allow_indeterminate  and  not pair.is_determinate():
            pair.failure = Failures.indeterminate
            self.counters.indeterminate += pair.multiplicity
            return False
        return True

    def prepare(self):
        raise Exception("subclasses must override")

    def process_pair(self, pair):
        raise Exception("subclasses must override")
Пример #7
0
 def fromxml(hier_dir, root_node):
     cls_id = int(root_node.get('id'))
     name = str(root_node.get('name'))
     idf = Counters.import_idf(path.join(hier_dir, "%d.idf" % cls_id))
     trashhold = float(root_node.get('treshold'))
     return DocClass(cls_id, name, idf, trashhold)
Пример #8
0
class OSMHandler(o.SimpleHandler):
  @staticmethod
  def run(osm_file, geometry_handler, index_type = 'sparse_mem_array'):
    idx = o.index.create_map(index_type)
    lh = o.NodeLocationsForWays(idx)
    # lh.ignore_errors()

    handler = OSMHandler(idx, geometry_handler)

    relations_pass_reader = o.io.Reader(osm_file, o.osm.osm_entity_bits.RELATION)
    o.apply(relations_pass_reader, lh, handler)
    relations_pass_reader.close()

    print('**** REL PASS COMPLETE ****')
    handler.receive_rels = False # Hack for not being able to specify entity bits properly

    full_pass_reader = o.io.Reader(osm_file, o.osm.osm_entity_bits.ALL)
    o.apply(full_pass_reader, lh, handler)
    full_pass_reader.close()
    handler.geom_handler.run_complete()
    return handler

  def __init__(self, node_index, geometry_handler):
    self.geom_handler = geometry_handler
    self.idx = node_index
    super(OSMHandler, self).__init__()

    self.counters = Counters()
    self.mp_cache = PendingMultipolyCache()
    self.receive_rels = True

  def area(self, a):
    print('got an area:')
    print(a)

  def node(self, n):
    nodes = self.counters.inc('nodes')
    if nodes % 1000000 == 0:
      print("Processed %d nodes" % nodes)

  def relation(self, r):
    if not self.receive_rels:
      return None
    self.counters.inc('rels')
    if self.counters.get('rels') % 1000 == 0:
      print("Processed %d rels" % self.counters.get('rels'))
    if self.mp_cache.consider_rel(r):
      self.counters.inc('rels.multipolygon')

  def geom(self, way):
    return [[n.lon, n.lat] for n in way.nodes]

  def inspect_nodes(self, way):
    print("Way %d has %s nodes" %(way.id, len(way.nodes)))
    print(self.geom(way))

  def way(self, w):
    count = self.counters.inc('ways')
    # if count % 10000 == 0:
    #   print("Processed %d ways" % count)

    way = WayWrapper(w)

    if not way.has_valid_geometry():
      return None

    if way.is_building():
      self.counters.inc('building_counts.' + way.tags.get('building'))

    is_multipoly_member, completed_rel = self.mp_cache.consider_way(way)

    if is_multipoly_member and completed_rel:
      self.geom_handler.completed_entity(completed_rel.as_dict())
      self.mp_cache.remove_rel(completed_rel)
    elif not is_multipoly_member and way.is_building() and way.is_polygon():
      self.geom_handler.completed_entity(way.as_dict())
Пример #9
0
 def reset_counts(self):
     self.counters = Counters(self._run)
Пример #10
0
class PairProcessor(object):

    def __init__(self, run, targets, masks):
        self._run = run
        self._targets = targets
        self._masks = masks
        self.__adapter_t_rc = 0
        self.uses_tags = False
        self.counters = Counters(self._run)
        self._match_mask = self._match_mask_optimized if (run.masks[0] == 'RRRY' and run.masks[1] == 'YYYR') else self._match_mask_general
        if self._match_mask != self._match_mask_optimized:
            print("Warning: not using optimized mask match.")
        run.apply_config_restrictions()
        self.prepare()

    def exists(self):
        return True

    def reset_counts(self):
        self.counters = Counters(self._run)

    @property
    def _adapter_t_rc(self):
        if not self.__adapter_t_rc:
            self.__adapter_t_rc = reverse_complement(self._run.adapter_t)
        return self.__adapter_t_rc

    def _match_mask_general(self, pair):
        if pair.mask_label:
            self.counters.increment_mask(pair.mask_label, pair.multiplicity)
            return True
        seq = pair.r1.original_seq
        for mask in self._masks:
            if mask.matches(seq):
                pair.set_mask(mask)
                self.counters.increment_mask(pair.mask_label, pair.multiplicity)
                return True
        self.counters.mask_failure += pair.multiplicity
        pair.failure = Failures.mask
        return False

    # optimized version for RRRY/YYYR
    def _match_mask_optimized(self, pair):
        if pair.mask_label:
            self.counters.increment_mask(pair.mask_label, pair.multiplicity)
            return True
        mask = match_mask_optimized(pair.r1.original_seq, self._masks)
        if mask:
            pair.set_mask(mask)
            self.counters.increment_mask(pair.mask_label, pair.multiplicity)
            return True
        else:
            self.counters.mask_failure += pair.multiplicity
            pair.failure = Failures.mask
            return False

    def _check_indeterminate(self, pair):
        if not self._run.allow_indeterminate  and  not pair.is_determinate():
            pair.failure = Failures.indeterminate
            self.counters.indeterminate += pair.multiplicity
            return False
        return True

    def prepare(self):
        raise Exception("subclasses must override")

    def process_pair(self, pair):
        raise Exception("subclasses must override")
Пример #11
0
    def start(self,
              token,
              posts_file,
              group_ids,
              group_names,
              actions,
              period,
              post_every,
              profanity_reply=False):

        self.counters = Counters(post_every)
        self.filter = Filter()
        self.match = Match()
        self.profanity = Profanity(ignore=not profanity_reply)
        self.rate_limiters = RateLimiters(actions, period)
        self.replies = Replies(posts_file)
        self.sleep_wake = SleepWake()

        post_every_text = ""
        if post_every:
            post_every_text = f"- I reply to the last message every {post_every} messages posted."

        chee_text = "- I respond to messages that are just '/chee'."

        stats = self.replies.getStats()

        self.about_text = self.about_text.format(
            actions=actions,
            period=period,
            post_every_text=post_every_text,
            chee_text=chee_text,
            stats_total=stats["total"],
            stats_quotes=stats["quotes"],
            stats_images=stats["images"])
        #print("DEBUG: ", self.about_text) # Debugging

        self.allowed_group_ids = self.getAllowedIds(group_ids)
        self.allowed_group_names = self.getAllowedIds(group_names)
        if self.allowed_group_ids or self.allowed_group_names:
            logger.info(f"Allowed Group IDs: {self.allowed_group_ids}")
            logger.info(f"Allowed Group Names: {self.allowed_group_names}")
        else:
            logger.info(
                "No Group IDs or Group Names specified, the bot is open to ALL groups!"
            )

        bot = telegram.Bot(token=token)
        logger.info(f"Successfully authenticated! {bot.get_me()}")
        self.my_username = bot.get_me().username
        self.my_id = bot.get_me().id
        logger.info(f"My usernamne: {self.my_username}, My ID: {self.my_id}")

        updater = Updater(token=token)
        dispatcher = updater.dispatcher

        #
        # Catch errors
        #
        dispatcher.add_error_handler(self.errorHandler)

        #
        # We're just gonna reply to everything.
        #
        echo_handler = MessageHandler(Filters.all, self.echo)
        dispatcher.add_handler(echo_handler)

        updater.start_polling()
Пример #12
0
class CheetahBot():

    counters = None
    filter = None
    match = None
    profanity = None
    rate_limiters = None
    replies = None
    sleep_wake = None

    about_text = ""
    about_text = """I am Cheetah Bot -- a cybernetic organism: living spots and fur over a metal endoskeleton.\n
My mission is to chirp at you.  Add me to a Telegram group for cheetah sounds and pictures.\n
My directives are as follows:\n
{chee_text}
- I notice profanity and respond to it.
- If you @ me, I respond with cheetah pictures and noises.
- I have a quota of sending {actions} messages per {period} seconds.
{post_every_text}
- I can say {stats_total} different things, choosing from {stats_quotes} quotes and {stats_images} pictures.

@ me with 'help' to see this message again.

Made with 🙀 by Leopards.
"""

    def __init__(self):
        pass

    #
    # Our main entry point to start bot.  This function will never exit.
    #
    def start(self,
              token,
              posts_file,
              group_ids,
              group_names,
              actions,
              period,
              post_every,
              profanity_reply=False):

        self.counters = Counters(post_every)
        self.filter = Filter()
        self.match = Match()
        self.profanity = Profanity(ignore=not profanity_reply)
        self.rate_limiters = RateLimiters(actions, period)
        self.replies = Replies(posts_file)
        self.sleep_wake = SleepWake()

        post_every_text = ""
        if post_every:
            post_every_text = f"- I reply to the last message every {post_every} messages posted."

        chee_text = "- I respond to messages that are just '/chee'."

        stats = self.replies.getStats()

        self.about_text = self.about_text.format(
            actions=actions,
            period=period,
            post_every_text=post_every_text,
            chee_text=chee_text,
            stats_total=stats["total"],
            stats_quotes=stats["quotes"],
            stats_images=stats["images"])
        #print("DEBUG: ", self.about_text) # Debugging

        self.allowed_group_ids = self.getAllowedIds(group_ids)
        self.allowed_group_names = self.getAllowedIds(group_names)
        if self.allowed_group_ids or self.allowed_group_names:
            logger.info(f"Allowed Group IDs: {self.allowed_group_ids}")
            logger.info(f"Allowed Group Names: {self.allowed_group_names}")
        else:
            logger.info(
                "No Group IDs or Group Names specified, the bot is open to ALL groups!"
            )

        bot = telegram.Bot(token=token)
        logger.info(f"Successfully authenticated! {bot.get_me()}")
        self.my_username = bot.get_me().username
        self.my_id = bot.get_me().id
        logger.info(f"My usernamne: {self.my_username}, My ID: {self.my_id}")

        updater = Updater(token=token)
        dispatcher = updater.dispatcher

        #
        # Catch errors
        #
        dispatcher.add_error_handler(self.errorHandler)

        #
        # We're just gonna reply to everything.
        #
        echo_handler = MessageHandler(Filters.all, self.echo)
        dispatcher.add_handler(echo_handler)

        updater.start_polling()

    #
    # Our handler that is fired when a message comes in
    #
    def echo(self, update, context):

        results = self.echoParseMessage(update, context)

        if not results:
            return (None)

        (message, text, chat_id) = results[0], results[1], results[2]

        try:
            self.echoComposeReply(context, update, message, text, chat_id)
        except telegram.error.BadRequest as e:
            if "Have no rights to send a message" in str(e):
                logger.warning(
                    f"echo(): Unable to send message to chat_id={chat_id}: {str(e)}"
                )
            else:
                raise (e)

    #
    # Parse our message and figure out if we should reply.
    #
    def echoParseMessage(self, update, context):

        # Set some defaults
        reply = ""

        # Filter newlines out of our message for logging and matching purposes
        message = update.message
        text = ""
        if update.message.text:
            text = update.message.text.replace("\r", " ").replace("\n", " ")

        # How old is the message?
        age = time.time() - message.date.timestamp()

        logger.info(
            f"New Message: age={age:.3f}, chat_id={update.effective_chat.id}, chat_name='{update.effective_chat.title}', text={text[0:30]}..."
        )
        #logger.info(f"Update: {update}") # Debugging
        #logger.info(f"Message: {message}") # Debugging
        #logger.info(f"Effective chat: {update.effective_chat}") # Debugging

        # Bail if the message is too old, otherwise we risk spamming groups after a hiatus
        if age > 10:
            logger.info(f"Message is {age} > 10 seconds old, ignoring.")
            return (None)

        # Was this a bot add/remove?
        if self.filter.messageIsIgnorable(update, message, self.my_id):
            return (None)

        # Was this a DM?
        if self.filter.messageIsDm(update):
            logger.info(
                "This is a DM, talk about ourself and bail out (for now...)")
            context.bot.send_message(chat_id=update.effective_chat.id,
                                     text=self.about_text)
            return (None)

        #
        # Bail out if we're not in an allowed group
        #
        chat_id = update.effective_chat.id
        chat_name = update.effective_chat.title
        if not self.match.doesGroupMatch(self.allowed_group_ids,
                                         self.allowed_group_names, chat_id,
                                         chat_name):
            return (None)

        return ([message, text, chat_id])

    #
    # Compose our reply
    #
    def echoComposeReply(self, context, update, message, text, chat_id):

        reply = ""

        message_to_me = False
        if self.match.doesUserMatch(self.my_id, self.my_username,
                                    update.message, text):
            message_to_me = True

        #
        # Announce ourself it added to a group
        #
        #if self.filter.botWasAddedToGroup(update, message, self.my_id):
        #	logger.info(f"I was added to the chat '{chat_name}' ({chat_id}), let's say hi!")
        #	reply = self.about_text

        #
        # Get our rate limiter for this chat
        #
        limiter = self.rate_limiters.getRateLimiter(chat_id)

        #
        # Did the user ask us for help?
        #
        if message_to_me:
            #
            # Did the user ask us for help?
            #
            if self.filter.messageContainsHelp(text):
                logger.info("User asked us for help, give it.")
                reply = self.about_text

            elif self.filter.messageContainsStats(text):
                logger.info("User wants to know the bot stats")
                reply = self.getStats(limiter)

        #
        # See if anyone in the chat said "/chee"
        #
        if self.filter.messageIsChee(text):
            reply = "chee"
            logger.info("String '/chee' is exact match.")

        #
        # I'm not thrilled about calling checkForFoulLanguage() twice, but Python
        # doesn't let me do "if (value = func())" syntax like other languages do.
        # Once this goes into a class, I can have the function just set a classwide value instead.
        #
        if self.profanity.hasFoulLanguage(update, text):
            reply = self.profanity.getReply()
            logger.info("Profanity detected")

        #
        # If the message wasn't to the bot, and we're not replying to a user, stop.
        #
        if not reply:
            if not message_to_me:
                #
                # See if we should reply and do so.
                #
                should_post = self.counters.update(chat_id)
                if should_post:
                    delay = 10
                    #delay = 1 # Debugging
                    threading.Timer(delay,
                                    self.sendRandomMessageFromThread,
                                    args=(
                                        context.bot,
                                        limiter,
                                        chat_id,
                                    )).start()
                    logger.info(
                        f"We hit our num message threshold for posting, scheduled group post in {delay} seconds..."
                    )
                return (None)

        #
        # If we made it here, we're sending SOME kind of reply.
        #

        #
        # If we already have a reply, then send it out.
        # Otherwise, we have no idea what the user is talking about, so let's just
        # grab a random string of text or URL.
        #
        if reply:
            self.sendMessage(context.bot,
                             limiter,
                             chat_id,
                             reply=reply,
                             message_id=message.message_id)
        else:
            self.sendRandomReply(context.bot,
                                 limiter,
                                 chat_id,
                                 message_id=message.message_id)

    #
    # Handle our errors
    #
    def errorHandler(self, update: object, context: CallbackContext):

        error_string = str(context.error)

        if "terminated by other getUpdates request" in error_string:
            logger.warning(
                "Looks like another instance of this bot is running. Stop doing that."
            )

        else:
            logger.error(msg="Exception while handling an update:",
                         exc_info=context.error)

        #
        # Based on an example at
        # https://github.com/python-telegram-bot/python-telegram-bot/blob/master/examples/errorhandlerbot.py#L30
        #
        #errors = traceback.format_exception(None, context.error, context.error.__traceback__)
        #error= ''.join(errors)
        #logger.info(error)

    #
    # Split up our comma-delimited list of IDs, and filter out empty strings.
    #
    def getAllowedIds(self, group_ids):

        retval = [id for id in group_ids if id != ""]
        return (retval)

    #
    # Figure out which reply function to use, get a reply, and send it off!
    #
    def sendRandomReply(self, bot, limiter, chat_id, message_id):

        reply = self.replies.getRandomMessage()

        if "url" in reply:
            caption = ""
            if "caption" in reply:
                caption = reply["caption"]

            self.sendMessage(bot,
                             limiter,
                             chat_id,
                             image_url=reply["url"],
                             caption=caption,
                             message_id=message_id)

        elif "caption" in reply:
            self.sendMessage(bot,
                             limiter,
                             chat_id,
                             reply=reply["caption"],
                             message_id=message_id)

        else:
            logger.warn(
                f"Reply '{reply}' has neither caption not URL.  Probably a blank line!  I warned you at startup!"
            )

    #
    # Send a random message to the group.
    #
    def sendRandomMessage(self, bot, limiter, chat_id):

        reply = self.replies.getRandomMessage()

        if "url" in reply:
            self.sendMessage(bot,
                             limiter,
                             chat_id,
                             image_url=reply["url"],
                             caption=reply["caption"])
        else:
            self.sendMessage(bot, limiter, chat_id, reply=reply["caption"])

    #
    # Callback to send a random message from a thread, where there was a delay.
    #
    def sendRandomMessageFromThread(self, bot, limiter, chat_id):
        self.sendRandomMessage(bot, limiter, chat_id)

    #
    # Return the current stats for the bot in the current channel.
    #
    def getStats(self, limiter):
        stats = self.replies.getStats()
        retval = (
            f"I can send {limiter.actions} messages every {limiter.period} seconds."
            +
            f"I have {limiter.getQuota()-1:.1f} more messages left in my quota.\n\n"
            +
            f"I can say {stats['total']} different things, choosing from {stats['quotes']} quotes and {stats['images']} pictures."
        )

        return (retval)

    #
    # Send a message in a way that honors our rate limiter's quota.
    # All messages are either a reply or an image.
    #
    def sendMessage(self,
                    bot,
                    limiter,
                    chat_id,
                    reply=None,
                    image_url=None,
                    caption=None,
                    message_id=None):

        if self.sleep_wake.isAsleep(chat_id):
            logger.info(
                f"We're asleep, not sending reply to chat {chat_id}...")
            return (None)

        if limiter.action():

            if reply:
                logger.info(
                    f"Sending reply: {reply[0:40]}..., reply_to={message_id} quota_left={limiter.getQuota():.3f}"
                )
                if not message_id:
                    bot.send_message(chat_id=chat_id, text=reply)
                else:
                    bot.send_message(chat_id=chat_id,
                                     text=reply,
                                     reply_to_message_id=message_id)

            elif image_url:
                newline = "\n"
                if not message_id:
                    bot.send_photo(chat_id=chat_id,
                                   photo=image_url,
                                   caption=caption)
                    logger.info(
                        f"Sending image: {image_url}, caption: {caption.replace(newline, ' ')[0:20]}... quota_left={limiter.getQuota():.3f}"
                    )
                else:
                    logger.info(
                        f"Sending image: {image_url}, caption: {caption.replace(newline, ' ')[0:20]}... reply_to={message_id} quota_left={limiter.getQuota():.3f}"
                    )
                    bot.send_photo(chat_id=chat_id,
                                   photo=image_url,
                                   caption=caption,
                                   reply_to_message_id=message_id)

            else:
                raise Exception(
                    f"Not sure what to send with a message that is not a reply and doesn't have a URL"
                )

            #
            # Let the group know that we've gone over our quota.
            # Yes, this will only work with one group, even if we are listening in multiple groups.
            #
            if limiter.isQuotaExhausted():
                self.sleep_wake.goToSleep(bot, limiter, chat_id)

        else:
            logger.info(
                f"Not sending message, quota currently exhausted. quota_left={limiter.getQuota():.3f}"
            )
Пример #13
0
 def import_tfs(dirname):
     return [(f, Counters.import_weights(path.join(dirname, f), weights.tf)) for f in os.listdir(dirname)]
Пример #14
0
 def import_idfs(dirname):
     return [Counters.import_weights(path.join(dirname, f), weights.idf) for f in os.listdir(dirname)]
Пример #15
0
from os import path
from counters import Counters
from rating import RIC
from loader import Loader
import matplotlib.pyplot as plt
import weights

sets_dir = '/home/rsuvorov/projects/dcfs/data'
white_set = 'en_var_1'
black_set = 'en_porno_1'
white_dir = path.join(sets_dir, white_set)
black_dir = path.join(sets_dir, black_set)

white_docs = Loader.import_tfs(path.join(white_dir, 'tf'))
white_idf = Counters.import_weights(path.join(white_dir, white_set + '.idf'), weights.idf)
black_docs = Loader.import_tfs(path.join(black_dir, 'tf'))
black_idf = Counters.import_weights(path.join(black_dir, black_set + '.idf'), weights.idf)

rater = RIC(black_idf, white_idf)

white_ratings = [(fname, rater(tf)) for (fname, tf) in white_docs]
black_ratings = [(fname, rater(tf)) for (fname, tf) in black_docs]

white_dat = open(path.join(white_dir, "ratings.csv"), 'w')
for (fname, r) in white_ratings:
    print >> white_dat, fname, r

black_dat = open(path.join(black_dir, "ratings.csv"), 'w')    
for (fname, r) in black_ratings:
    print >> black_dat, fname, r