def handle(self, *args, **options):
     logger.info('Listening for package updates...')
     for name, endpoint, topic, msg in fedmsg.tail_messages():
         try:
             if topic.endswith('package.modified') or \
                     topic.endswith('package.removed') or \
                     topic.endswith('package.added'):
                 pkg = msg['msg']['package']
                 repoinfo_id = msg['msg'].get('upstream')
                 for package in Package.objects.filter(
                         name=pkg['name'], arch=pkg['arch']):
                     if package.version != pkg['version'] or \
                             package.release != pkg['release']:
                         logger.info('Package changed: %s %s'
                                     % (pkg, package))
                         images_qs = package.images.all()
                         if repoinfo_id is not None:
                             images_qs = images_qs.filter(
                                 repoinfo__id=repoinfo_id)
                         images_qs.update(to_build=True)
                         logger.info('Images marked for build: %s'
                                     % images_qs)
         except Exception as e:
             logger.critical('pkgupdatelistener errored out: %s' % e,
                             exc_info=True)
             sys.exit(1)
예제 #2
0
def listen(config):
    """
    Listens to activity on upstream repos on pagure and github \
    via fedmsg, and syncs new issues there to the JIRA instance \
    defined in 'fedmsg.d/sync2jira.py'

    :param Dict config: Config dict
    :returns: Nothing
    """
    if not config['sync2jira'].get('listen'):
        log.info("`listen` is disabled.  Exiting.")
        return

    log.info("Waiting for a relevant fedmsg message to arrive...")
    for _, _, topic, msg in fedmsg.tail_messages(**config):
        idx = msg['msg_id']
        suffix = ".".join(topic.split('.')[3:])
        log.debug("Encountered %r %r %r", suffix, topic, idx)

        if suffix not in issue_handlers and suffix not in pr_handlers:
            continue

        log.debug("Handling %r %r %r", suffix, topic, idx)

        handle_msg(msg, suffix, config)
예제 #3
0
def main():
    fedmsg_config = fedmsg.config.load_config()
    dictConfig(fedmsg_config.get('logging', {'version': 1}))

    log.info("Listening to the bus via fedmsg.tail_messages()")
    for _, _, topic, msg in fedmsg.tail_messages():

        # XXX - if you want to debug whether or not this is receiving fedmsg
        # messages, you can put a print statement here, before the 'continue'
        # statement.

        if not topic.endswith(target):
            continue

        log.info("A meeting just ended!  Sleeping 2s.  %r" % msg.get('msg_id'))
        time.sleep(2)

        teams_cmd = "/usr/local/bin/meetings_by_team.sh"
        log.info("Running %r" % teams_cmd)
        proc = sp.Popen(teams_cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE)
        stdout, stderr = proc.communicate()
        if proc.returncode:
            # Calling log.error in fedora infrastructure with fedmsg logging
            # configured, should send an email to the sysadmin-datanommer
            # group.
            log.error("Error %r running %r.\n  STDOUT:  %s\n  STDERR:  %s" % (
                proc.returncode, teams_cmd, stdout, stderr))

        log.info("Running soke.run()...")
        soke.run()

        log.info("Done.")
예제 #4
0
    def watch(self):
        config = fedmsg.config.load_config()
        fedmsg.init(mute=True, **config)
        fedmsg.meta.make_processors(**config)

        for name, endpoint, topic, msg in fedmsg.tail_messages():
            log.debug("received topic: {topic}".format(topic=topic))
            if not topic in self.topics:
                continue

            log.debug("match topic {topic}=>{data}".format(topic=topic,
                                                           data=msg['msg']))
            pargs = [topic]
            for parg in self.topics[topic]['args']:
                if hasattr(parg, '__call__'):
                    # run this as fedmsg.meta function
                    pargs.append(parg(msg, **config))
                elif '/' in parg:
                    # this is a dpath expression
                    try:
                        path, val = dpath.util.search(msg, parg, yielded=True).next()
                        pargs.append(val)
                    except StopIteration:
                        log.warning("Path {parg} does not exist in {topic}. Substituting empty string"
                        .format(parg=parg, topic=topic))
                        pargs.append('')
                elif parg in msg:
                    pargs.append(msg[parg])
                else:
                    log.warning("Path {parg} does not exist in {topic}. Substituting empty string"
                               .format(parg=parg, topic=topic))
                    pargs.append('')

            self.__run_scripts(self.script_dir, pargs)
예제 #5
0
def main():
    fedmsg_config = fedmsg.config.load_config()
    dictConfig(fedmsg_config.get('logging', {'version': 1}))

    log.info("Listening to the bus via fedmsg.tail_messages()")
    for _, _, topic, msg in fedmsg.tail_messages():

        # XXX - if you want to debug whether or not this is receiving fedmsg
        # messages, you can put a print statement here, before the 'continue'
        # statement.

        if not topic.endswith(target):
            continue

        log.info("A meeting just ended!  Sleeping 2s.  %r" % msg.get('msg_id'))
        time.sleep(2)

        teams_cmd = "/usr/local/bin/meetings_by_team.sh"
        log.info("Running %r" % teams_cmd)
        proc = sp.Popen(teams_cmd.split(), stdout=sp.PIPE, stderr=sp.PIPE)
        stdout, stderr = proc.communicate()
        if proc.returncode:
            # Calling log.error in fedora infrastructure with fedmsg logging
            # configured, should send an email to the sysadmin-datanommer
            # group.
            log.error("Error %r running %r.\n  STDOUT:  %s\n  STDERR:  %s" %
                      (proc.returncode, teams_cmd, stdout, stderr))

        log.info("Running soke.run()...")
        soke.run()

        log.info("Done.")
예제 #6
0
def tail(**kw):
    """ Watch all endpoints on the bus and print each message to stdout. """

    # Disable sending
    kw['publish_endpoint'] = None

    # Disable timeouts.  We want to tail forever!
    kw['timeout'] = 0

    # Even though fedmsg-tail won't be sending any messages, give it a name to
    # conform with the other commands.
    kw['name'] = 'relay_inbound'

    # Tail is never going to send any messages, so we suppress warnings about
    # having no publishing sockets established.
    kw['mute'] = True

    fedmsg.init(**kw)
    fedmsg.text.make_processors(**kw)

    # Build a message formatter
    formatter = lambda d: d
    if kw['pretty']:

        def formatter(d):
            d['timestamp'] = time.ctime(d['timestamp'])
            d = fedmsg.crypto.strip_credentials(d)
            return "\n" + pprint.pformat(d)

    if kw['really_pretty']:

        def formatter(d):
            d = fedmsg.crypto.strip_credentials(d)
            fancy = pygments.highlight(
                fedmsg.encoding.pretty_dumps(d),
                pygments.lexers.JavascriptLexer(),
                pygments.formatters.TerminalFormatter()).strip()
            return "\n" + fancy

    if kw['terse']:
        formatter = lambda d: "\n" + fedmsg.text.msg2repr(d, **kw)

    exclusive_regexp = re.compile(kw['exclusive_regexp'])
    inclusive_regexp = re.compile(kw['inclusive_regexp'])

    # The "proper" fedmsg way to do this would be to spin up or connect to an
    # existing Moksha Hub and register a consumer on the "*" topic that simply
    # prints out each message it consumes.  That seems like overkill, so we're
    # just going to directly access the endpoints ourself.

    for name, ep, topic, message in fedmsg.tail_messages(**kw):
        if exclusive_regexp.search(topic):
            continue

        if not inclusive_regexp.search(topic):
            continue

        print name, ep, topic, formatter(message)
예제 #7
0
파일: tail.py 프로젝트: pypingou/fedmsg
def tail(**kw):
    """ Watch all endpoints on the bus and print each message to stdout. """

    # Disable sending
    kw['publish_endpoint'] = None

    # Disable timeouts.  We want to tail forever!
    kw['timeout'] = 0

    # Even though fedmsg-tail won't be sending any messages, give it a name to
    # conform with the other commands.
    kw['name'] = 'relay_inbound'

    # Tail is never going to send any messages, so we suppress warnings about
    # having no publishing sockets established.
    kw['mute'] = True

    fedmsg.init(**kw)
    fedmsg.text.make_processors(**kw)

    # Build a message formatter
    formatter = lambda d: d
    if kw['pretty']:
        def formatter(d):
            d['timestamp'] = time.ctime(d['timestamp'])
            d = fedmsg.crypto.strip_credentials(d)
            return "\n" + pprint.pformat(d)

    if kw['really_pretty']:
        def formatter(d):
            d = fedmsg.crypto.strip_credentials(d)
            fancy = pygments.highlight(
                fedmsg.encoding.pretty_dumps(d),
                pygments.lexers.JavascriptLexer(),
                pygments.formatters.TerminalFormatter()
            ).strip()
            return "\n" + fancy

    if kw['terse']:
        formatter = lambda d: "\n" + fedmsg.text.msg2repr(d, **kw)

    exclusive_regexp = re.compile(kw['exclusive_regexp'])
    inclusive_regexp = re.compile(kw['inclusive_regexp'])

    # The "proper" fedmsg way to do this would be to spin up or connect to an
    # existing Moksha Hub and register a consumer on the "*" topic that simply
    # prints out each message it consumes.  That seems like overkill, so we're
    # just going to directly access the endpoints ourself.

    for name, ep, topic, message in fedmsg.tail_messages(**kw):
        if exclusive_regexp.search(topic):
            continue

        if not inclusive_regexp.search(topic):
            continue

        print name, ep, topic, formatter(message)
예제 #8
0
 def main(self):
     for _, _, topic, msg in fedmsg.tail_messages():
         self.notify_watchdog()
         try:
             if topic.startswith(get_config('fedmsg.topic') + '.'):
                 self.consume(topic, msg)
             plugin.dispatch_event('fedmsg_event', self.session, topic, msg)
         finally:
             self.db.rollback()
         self.memory_check()
def main():
    tweet = twitter_utils.Tweet()
    config = fedmsg.config.load_config([], None)
    fedmsg.meta.make_processors(**config)
    for name, endpoint, topic, msg in fedmsg.tail_messages():
        #currently only few topics are frequented
        #so sending tweets without filtering on topics
        print "Sending Tweet"
        title = fedmsg.meta.msg2title(msg, **config)
        tweet.send_tweet("Msg recieved on bus is titled:%s" % title)
예제 #10
0
def listen(config):
    """
    Listens to activity on upstream repos on pagure and github \
    via fedmsg, and syncs new issues there to the JIRA instance \
    defined in 'fedmsg.d/sync2jira.py'

    :param Dict config: Config dict
    :returns: Nothing
    """
    if not config['sync2jira'].get('listen'):
        log.info("`listen` is disabled.  Exiting.")
        return

    log.info("Waiting for a relevant fedmsg message to arrive...")
    for _, _, topic, msg in fedmsg.tail_messages(**config):
        idx = msg['msg_id']
        suffix = ".".join(topic.split('.')[3:])
        log.debug("Encountered %r %r %r", suffix, topic, idx)

        if suffix not in issue_handlers and suffix not in pr_handlers:
            continue

        log.debug("Handling %r %r %r", suffix, topic, idx)

        issue = None
        pr = None
        # Github '.issue.' is used for both PR and Issue
        # Check for that edge case
        if suffix == 'github.issue.comment':
            if 'pull_request' in msg['msg'][
                    'issue'] and msg['msg']['action'] != 'deleted':
                # pr_filter turns on/off the filtering of PRs
                pr = issue_handlers[suffix](msg, config, pr_filter=False)
                if not pr:
                    continue
                # Issues do not have suffix and reporter needs to be reformatted
                pr.suffix = suffix
                pr.reporter = pr.reporter.get('fullname')
                setattr(pr, 'match', matcher(pr.content, pr.comments))
            else:
                issue = issue_handlers[suffix](msg, config)
        elif suffix in issue_handlers:
            issue = issue_handlers[suffix](msg, config)
        elif suffix in pr_handlers:
            pr = pr_handlers[suffix](msg, config, suffix)

        if not issue and not pr:
            continue
        if issue:
            d_issue.sync_with_jira(issue, config)
        elif pr:
            d_pr.sync_with_jira(pr, config)
예제 #11
0
    def iterate_dg_pr_flags() -> Iterable[Tuple[str, dict]]:
        """
        Provide messages when a flag is added to a pull request in dist-git

        :return: tuple, (full topic name, dict with the message)
        """
        # we can watch for runs directly:
        # "org.centos.prod.ci.pipeline.allpackages.complete"
        topic = "org.fedoraproject.prod.pagure.pull-request.flag.added"

        logger.info("listening on fedmsg, topic=%s", topic)
        for name, endpoint, topic, msg in fedmsg.tail_messages(topic=topic):
            yield topic, msg
예제 #12
0
파일: watcher.py 프로젝트: W3SS/koschei
 def main(self):
     try:
         for _, _, topic, msg in fedmsg.tail_messages():
             self.notify_watchdog()
             try:
                 if topic.startswith(get_config('fedmsg.topic') + '.'):
                     self.consume(topic, msg)
                 plugin.dispatch_event('fedmsg_event', topic, msg, db=self.db,
                                       koji_sessions=self.koji_sessions)
             finally:
                 self.db.rollback()
     except requests.exceptions.ConnectionError:
         self.log.exception("Fedmsg watcher exception.")
         fedmsg.destroy()
         fedmsg.init()
예제 #13
0
파일: updater.py 프로젝트: asamalik/BPO
def main():
    # Fedmsg and Elasticsearch clients
    fedmsg_config = fedmsg.config.load_config()
    es = Elasticsearch(hosts=[{'host': config_es_url, 'port': config_es_port}])

    # Reading Fedmsg messages
    for name, endpoint, topic, msg in fedmsg.tail_messages(**fedmsg_config):
        log("\n\n==============================================\n")
        log("Fedmsg received!\n")
        log("Time:   {}".format(datetime.now()))
        log("Topic:  {}".format(topic))
        log("Message:\n{}\n".format(msg))

        if topic == config_topic_module_state_change:
            log ("Module state change: {}\n".format(topic))
            action_module_state_change(msg["msg"], es)
예제 #14
0
    def iterate_pull_requests() -> Iterable[Tuple[str, str, dict]]:
        """
        Provide messages for all github pull-request-related events

        Actions:
            https://developer.github.com/v3/activity/events/types/#events-api-payload-28

        :return: tuple, (full topic name, pull request action, dict with the message)
        """
        # https://github.com/fedora-infra/github2fedmsg/blob/a9c178b93aa6890e6b050e5f1c5e3297ceca463c/github2fedmsg/views/webhooks.py#L120
        topic_pre = "org.fedoraproject.prod.github.pull_request."
        for name, endpoint, topic, msg in fedmsg.tail_messages():
            # logger.debug("new message: %s", topic)
            # average load is about 5 messages a second
            if topic.startswith(topic_pre):
                logger.info("process message: %s", topic)
                action = topic.rsplit(".", 1)[1]
                yield topic, action, msg
예제 #15
0
    def run(self):
        # fedmsg configuration
        config = fedmsg.config.load_config([], None)
        config['mute'] = True
        config['timeout'] = 0

        # parse the koji-shadow configuration to ignore packages we know
        # we don't care about.
        ignorelist = (ks_config.get('rules', 'excludelist').split() + 
            ks_config.get('rules', 'ignorelist').split())

        logger.debug('Monitoring fedmsg.')
        for name, endpoint, topic, msg in fedmsg.tail_messages(**config):
            if (msg['topic'] == 
                'org.fedoraproject.prod.buildsys.build.state.change' and 
                msg['msg']['new'] == 1 and 
                msg['msg']['name'] not in ignorelist):
                buildqueue.append(msg['msg']['name']+'-'+msg['msg']['version']+'-'+msg['msg']['release'])
예제 #16
0
 def main(self):
     try:
         for _, _, topic, msg in fedmsg.tail_messages():
             self.notify_watchdog()
             try:
                 if topic.startswith(self.topic_name + '.'):
                     self.consume(topic, msg)
                 plugin.dispatch_event('fedmsg_event',
                                       topic,
                                       msg,
                                       db=self.db,
                                       koji_session=self.koji_session)
             finally:
                 self.db.rollback()
     except requests.exceptions.ConnectionError:
         self.log.exception("Fedmsg watcher exception.")
         fedmsg.destroy()
         fedmsg.init()
예제 #17
0
    def run(self):

        # This is a "required" option... :P
        if not self.config['command']:
            self.log.error("You must provide a --command to run.")
            sys.exit(1)

        # Disable sending
        self.config['publish_endpoint'] = None

        # Disable timeouts.  We want to tail forever!
        self.config['timeout'] = 0

        # Even though fedmsg-trigger won't be sending any messages, give it a
        # name to conform with the other commands.
        self.config['name'] = 'relay_inbound'

        # Tail is never going to send any messages, so we suppress warnings
        # about having no publishing sockets established.
        self.config['mute'] = True

        fedmsg.init(**self.config)

        exclusive_regexp = re.compile(self.config['exclusive_regexp'])
        inclusive_regexp = re.compile(self.config['inclusive_regexp'])

        for name, ep, topic, message in fedmsg.tail_messages(**self.config):
            if exclusive_regexp.search(topic):
                continue

            if not inclusive_regexp.search(topic):
                continue

            result = self.run_command(self.config['command'], message)

            if result != 0:
                self.log.info("Command returned error code %r" % result)
예제 #18
0
파일: trigger.py 프로젝트: arielb2/fedmsg
    def run(self):

        # This is a "required" option... :P
        if not self.config['command']:
            self.log.error("You must provide a --command to run.")
            sys.exit(1)

        # Disable sending
        self.config['publish_endpoint'] = None

        # Disable timeouts.  We want to tail forever!
        self.config['timeout'] = 0

        # Even though fedmsg-trigger won't be sending any messages, give it a
        # name to conform with the other commands.
        self.config['name'] = 'relay_inbound'

        # Tail is never going to send any messages, so we suppress warnings
        # about having no publishing sockets established.
        self.config['mute'] = True

        fedmsg.init(**self.config)

        exclusive_regexp = re.compile(self.config['exclusive_regexp'])
        inclusive_regexp = re.compile(self.config['inclusive_regexp'])

        for name, ep, topic, message in fedmsg.tail_messages(**self.config):
            if exclusive_regexp.search(topic):
                continue

            if not inclusive_regexp.search(topic):
                continue

            result = self.run_command(self.config['command'], message)

            if result != 0:
                self.log.info("Command returned error code %r" % result)
예제 #19
0
    def watch(self):
        config = fedmsg.config.load_config()
        fedmsg.init(mute=True, **config)
        fedmsg.meta.make_processors(**config)

        for name, endpoint, topic, msg in fedmsg.tail_messages():
            log.debug("received topic: {topic}".format(topic=topic))
            if not topic in self.topics:
                continue

            log.debug("match topic {topic}=>{data}".format(topic=topic,
                                                           data=msg['msg']))
            pargs = [topic]
            for parg in self.topics[topic]['args']:
                if hasattr(parg, '__call__'):
                    # run this as fedmsg.meta function
                    pargs.append(parg(msg, **config))
                elif '/' in parg:
                    # this is a dpath expression
                    try:
                        path, val = dpath.util.search(msg, parg,
                                                      yielded=True).next()
                        pargs.append(val)
                    except StopIteration:
                        log.warning(
                            "Path {parg} does not exist in {topic}. Substituting empty string"
                            .format(parg=parg, topic=topic))
                        pargs.append('')
                elif parg in msg:
                    pargs.append(msg[parg])
                else:
                    log.warning(
                        "Path {parg} does not exist in {topic}. Substituting empty string"
                        .format(parg=parg, topic=topic))
                    pargs.append('')

            self.__run_scripts(self.script_dir, pargs)
예제 #20
0
파일: gilmsg.py 프로젝트: Tubbz-alt/gilmsg
    def run(self):
        self.sanity_check()
        ack_topic = '.'.join([
            self.c['topic_prefix'],
            self.c['environment'],
            'gilmsg.ack',
        ])

        # Go into a loop, receiving gilmsg ACK messages from the fedmsg bus
        for n, e, t, msg in fedmsg.tail_messages(**self.c):

            # Did we run out of time?
            if self.time_is_up:
                return

            # Throw away anything that's not an ACK produced by a consumer.
            if t != ack_topic:
                continue

            # We should only know about ACKs.  Is this an ACK for *ours*?
            if not msg['msg']['ack_msg_id'] == self.msg_id:
                continue

            # We can declare that multiple other systems *must* receive our
            # message.  Write down who is acking this one.
            for signer in self.expectations:
                if not fedmsg.crypto.validate_signed_by(msg, signer, **self.c):
                    continue
                self.results.append(signer)

            # Check to see if we collected all the pokemon.
            # If all the people that we wanted to get the message have sent
            # us ACKs for this message_id, then hooray!  If not, go back into
            # tail_messages to wait for more ACKs.  The clock is ticking....
            if set(self.results) == set(self.expectations):
                return
예제 #21
0
 def looper(self):
     """
     Implementation of the generator to feed the event loop
     """
     for name, endpoint, topic, msg in fedmsg.tail_messages(mute=True):
         yield (topic, dict(msg))
예제 #22
0
    def run(self):
        # First, sanity checking.
        if not self.config.get('tweet_endpoints', None):
            raise ValueError("Not configured to tweet.")

        # Boilerplate..
        self.config['publish_endpoint'] = None
        self.config['name'] = 'relay_inbound'
        self.config['mute'] = True

        # Set up fedmsg
        fedmsg.init(**self.config)
        fedmsg.meta.make_processors(**self.config)

        # Set up twitter and statusnet.. multiple accounts if configured
        settings = self.config.get('tweet_endpoints', [])
        apis = [twitter_api.Api(**endpoint) for endpoint in settings]

        # Set up bitly
        settings = self.config['bitly_settings']
        bitly = bitlyapi.BitLy(
            settings['api_user'],
            settings['api_key'],
        )

        # How long to sleep if we spew too fast.
        hibernate_duration = self.config['tweet_hibernate_duration']
        # Sleep a second or two inbetween messages to try and avoid the hibernate
        intermessage_pause = self.config['tweet_intermessage_pause']

        def _post_to_api(api, message):
            try:
                api.PostUpdate(message)
            except Exception as e:
                if 'Too many notices too fast;' in str(e):
                    # Cool our heels then try again.
                    self.log.info("Sleeping for %i" % hibernate_duration)
                    time.sleep(hibernate_duration)
                    _post_to_api(api, message)
                elif 'json decoding' in str(e):
                    # Let it slide ... no idea what this one is.
                    pass
                elif 'duplicate' in str(e):
                    # Let it slide ...
                    pass
                else:
                    raise

        for name, ep, topic, msg in fedmsg.tail_messages(**self.config):
            message = fedmsg.meta.msg2subtitle(msg, **self.config)
            link = fedmsg.meta.msg2link(msg, **self.config)

            if link:
                link = bitly.shorten(longUrl=link)['url']
                message = message[:138 - len(link)] + " " + link
            else:
                message = message[:140]

            if not message:
                self.log.info("Not tweeting blank message.")
                continue

            self.log.info("Tweeting %r" % message)
            for api in apis:
                _post_to_api(api, message)

            time.sleep(intermessage_pause)
예제 #23
0
def main():
    parser = argparse.ArgumentParser(description = __doc__)
    parser.add_argument('config', help = 'path of configuration file')
    parser.add_argument('-f', '--fedmsg', action = 'store_true', help = 'poll fedmsg events')
    parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')

    global args
    args = parser.parse_args()
    logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)

    config_parser = ConfigParser.SafeConfigParser(dict(
        here = os.path.dirname(os.path.abspath(os.path.normpath(args.config))),
        ))
    config_parser.read(args.config)
    global conf
    conf = check(pipe(
        test_isinstance(dict),
        struct(
            {
                'ckan_of_worms.api_key': pipe(
                    cleanup_line,
                    not_none,
                    ),
                'ckan_of_worms.site_url': pipe(
                    make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    not_none,
                    ),
                'user_agent': pipe(
                    cleanup_line,
                    not_none,
                    ),
                },
            default = 'drop',
            ),
        not_none,
        ))(dict(config_parser.items('CowBots-Check-Datasets')), default_state)

    global headers
    headers = {
        'User-Agent': conf['user_agent'],
        }

    if args.fedmsg:
        import fedmsg

        fedmsg_conf = check(struct(
            dict(
                environment = pipe(
                    empty_to_none,
                    test_in(['dev', 'prod', 'stg']),
                    ),
                modname = pipe(
                    empty_to_none,
                    test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                    default('ckan_of_worms'),
                    ),
#                name = pipe(
#                    empty_to_none,
#                    default('ckan_of_worms.{}'.format(hostname)),
#                    ),
                topic_prefix = pipe(
                    empty_to_none,
                    test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                    ),
                ),
            default = 'drop',
            ))(dict(config_parser.items('fedmsg')))

        # Read in the config from /etc/fedmsg.d/.
        fedmsg_config = fedmsg.config.load_config([], None)
        # Disable a warning about not sending.  We know.  We only want to tail.
        fedmsg_config['mute'] = True
        # Disable timing out so that we can tail forever.  This is deprecated
        # and will disappear in future versions.
        fedmsg_config['timeout'] = 0
        # For the time being, don't require message to be signed.
        fedmsg_config['validate_signatures'] = False
        for key, value in fedmsg_conf.iteritems():
            if value is not None:
                fedmsg_config[key] = value

        expected_topic_prefix = '{}.{}.ckan_of_worms.'.format(fedmsg_config['topic_prefix'], fedmsg_config['environment'])
        for name, endpoint, topic, message in fedmsg.tail_messages(**fedmsg_config):
            if not topic.startswith(expected_topic_prefix):
                log.debug(u'Ignoring message: {}, {}'.format(topic, name))
                continue
            kind, action = topic[len(expected_topic_prefix):].split('.')
            if kind == 'dataset':
                if action in ('create', 'update'):
                    dataset = check(pipe(
                        cow_json_to_dataset,
                        not_none,
                        ))(message['msg'], state = default_state)
                    check_dataset(dataset)
                else:
                    log.debug(u'TODO: Handle {}, {} for {}'.format(kind, action, message))
            else:
                log.debug(u'TODO: Handle {}, {} for {}'.format(kind, action, message))
    else:
        request = urllib2.Request(urlparse.urljoin(conf['ckan_of_worms.site_url'], 'api/1/datasets'), headers = headers)
        response = urllib2.urlopen(request)
        datasets_id = check(pipe(
            cow_response_to_value,
            cow_json_to_ids,
            not_none,
            ))(response.read(), state = default_state)

        for dataset_id in datasets_id:
            request = urllib2.Request(urlparse.urljoin(conf['ckan_of_worms.site_url'],
                'api/1/datasets/{}'.format(dataset_id)), headers = headers)
            response = urllib2.urlopen(request)
            dataset = check(pipe(
                cow_response_to_value,
                cow_json_to_dataset,
                not_none,
                ))(response.read(), state = default_state)
            check_dataset(dataset)

    return 0
예제 #24
0
def main():
    parser = argparse.ArgumentParser(description = __doc__)
    parser.add_argument('config', help = 'path of configuration file')
    parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')

    global args
    args = parser.parse_args()
    logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)

    config_parser = ConfigParser.SafeConfigParser(dict(
        here = os.path.dirname(os.path.abspath(os.path.normpath(args.config))),
        ))
    config_parser.read(args.config)
    global conf
    conf = conv.check(conv.pipe(
        conv.test_isinstance(dict),
        conv.struct(
            {
                'dactylo.api_key': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'dactylo.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'user_agent': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                },
            default = 'drop',
            ),
        conv.not_none,
        ))(dict(config_parser.items('CowBots-Report-Activity')), conv.default_state)

    global headers
    headers = {
        'User-Agent': conf['user_agent'],
        }
    global request_headers
    request_headers = headers.copy()
    request_headers['Content-Type'] = 'application/json'

    fedmsg_conf = conv.check(conv.struct(
        dict(
            environment = conv.pipe(
                conv.empty_to_none,
                conv.test_in(['dev', 'prod', 'stg']),
                ),
            modname = conv.pipe(
                conv.empty_to_none,
                conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                conv.default('ckan_of_worms'),
                ),
#            name = conv.pipe(
#                conv.empty_to_none,
#                conv.default('ckan_of_worms.{}'.format(hostname)),
#                ),
            topic_prefix = conv.pipe(
                conv.empty_to_none,
                conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                ),
            ),
        default = 'drop',
        ))(dict(config_parser.items('fedmsg')))

    # Read in the config from /etc/fedmsg.d/.
    fedmsg_config = fedmsg.config.load_config([], None)
    # Disable a warning about not sending.  We know.  We only want to tail.
    fedmsg_config['mute'] = True
    # Disable timing out so that we can tail forever.  This is deprecated
    # and will disappear in future versions.
    fedmsg_config['timeout'] = 0
    # For the time being, don't require message to be signed.
    fedmsg_config['validate_signatures'] = False
    for key, value in fedmsg_conf.iteritems():
        if value is not None:
            fedmsg_config[key] = value

    expected_topic_prefix = '{}.{}.ckan_of_worms.'.format(fedmsg_config['topic_prefix'], fedmsg_config['environment'])
    for name, endpoint, topic, message in fedmsg.tail_messages(**fedmsg_config):
        if not topic.startswith(expected_topic_prefix):
            log.debug(u'Ignoring message: {}, {}'.format(topic, name))
            continue
        kind, action = topic[len(expected_topic_prefix):].split('.')
        if kind == 'related':
            activity = message['msg']
            dataset = activity['target']
            log.info(u'Adding activity for {} related in dataset "{}"'.format(action, dataset['name']))
            request = urllib2.Request(urlparse.urljoin(conf['dactylo.site_url'], 'api/1/activities'),
                headers = request_headers)
            request_data = dict(
                api_key = conf['dactylo.api_key'],
                value = activity,
                )
            try:
                response = urllib2.urlopen(request, json.dumps(request_data))
            except urllib2.HTTPError as response:
                log.error(u'An error occured while adding activity: {}'.format(activity))
                response_text = response.read()
                try:
                    response_dict = json.loads(response_text)
                except ValueError:
                    log.error(response_text)
                    raise
                for key, value in response_dict.iteritems():
                    print '{} = {}'.format(key, value)
                raise
            else:
                assert response.code == 200
                conv.check(cow_response_to_value)(response.read(), state = conv.default_state)
        else:
            log.debug(u'TODO: Handle {}, {} for {}'.format(kind, action, message))

    return 0
예제 #25
0
            time.sleep(30)
        dom.create()
    print("Domain %s started" % (domain))


if __name__ == '__main__':
    pidfile = open('/var/run/harness.pid', 'w')
    pid = str(os.getpid())
    pidfile.write(pid)
    pidfile.close()
    config = fedmsg.config.load_config([], None)
    config['mute'] = True
    config['timeout'] = 0
    fedmsg.meta.make_processors(**config)

    for name, endpoint, topic, msg in fedmsg.tail_messages(**config):
        if "buildsys.build.state.change" in topic:
            matchedmsg = fedmsg.meta.msg2repr(msg, **config)
            if "completed" in matchedmsg:
                if "kernel" in matchedmsg:
                    objectmsg = fedmsg.meta.msg2subtitle(msg,
                                                         legacy=False,
                                                         **config)
                    package = string.split(objectmsg, ' ')
                    fcrelease = string.split(package[1], '.')
                    domain = domainmap(fcrelease[-1])
                    logfile = open('/var/log/harness.log', 'a')
                    logfile.write('Testing ' + package[1] + '\n')
                    logfile.close()
                    writelatest(domain, package[1])
                    dom32 = domain + '32'
def main():
    parser = argparse.ArgumentParser(description = __doc__)
    parser.add_argument('config', help = 'path of configuration file')
    parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')

    global args
    args = parser.parse_args()
    logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)

    config_parser = ConfigParser.SafeConfigParser(dict(
        here = os.path.dirname(os.path.abspath(os.path.normpath(args.config))),
        ))
    config_parser.read(args.config)
    conf = conv.check(conv.pipe(
        conv.test_isinstance(dict),
        conv.struct(
            {
                'ckan.api_key': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'ckan.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'ckan_of_worms.api_key': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'ckan_of_worms.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'user_agent': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                },
            default = 'drop',
            ),
        conv.not_none,
        ))(dict(config_parser.items('CKAN-of-Worms-Harvesters')), conv.default_state)

    fedmsg_conf = conv.check(conv.struct(
        dict(
            environment = conv.pipe(
                conv.empty_to_none,
                conv.test_in(['dev', 'prod', 'stg']),
                ),
            modname = conv.pipe(
                conv.empty_to_none,
                conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                conv.default('ckan_of_worms'),
                ),
#            name = conv.pipe(
#                conv.empty_to_none,
#                conv.default('ckan_of_worms.{}'.format(hostname)),
#                ),
            topic_prefix = conv.pipe(
                conv.empty_to_none,
                conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                ),
            ),
        default = 'drop',
        ))(dict(config_parser.items('fedmsg')))

    source_headers = {
        'Authorization': conf['ckan.api_key'],  # API key is required to get full user profile.
        'User-Agent': conf['user_agent'],
        }
    source_site_url = conf['ckan.site_url']
    target_api_key = conf['ckan_of_worms.api_key']
    target_headers = {
        'User-Agent': conf['user_agent'],
        }
    target_site_url = conf['ckan_of_worms.site_url']

    # Read in the config from /etc/fedmsg.d/.
    fedmsg_config = fedmsg.config.load_config([], None)
    # Disable a warning about not sending.  We know.  We only want to tail.
    fedmsg_config['mute'] = True
    # Disable timing out so that we can tail forever.  This is deprecated
    # and will disappear in future versions.
    fedmsg_config['timeout'] = 0
    # For the time being, don't require message to be signed.
    fedmsg_config['validate_signatures'] = False
    for key, value in fedmsg_conf.iteritems():
        if value is not None:
            fedmsg_config[key] = value

    expected_topic_prefix = '{}.{}.ckan.'.format(fedmsg_config['topic_prefix'], fedmsg_config['environment'])
    for name, endpoint, topic, message in fedmsg.tail_messages(**fedmsg_config):
        if not topic.startswith(expected_topic_prefix):
            log.debug(u'Ignoring message: {}, {}'.format(topic, name))
            continue
        kind, action = topic[len(expected_topic_prefix):].split('.')
        if kind == 'group':
            if action in ('create', 'update'):
                group = message['msg']

                log.info(u'Upserting group: {}'.format(group['name']))
                request_headers = target_headers.copy()
                request_headers['Content-Type'] = 'application/json'
                request = urllib2.Request(urlparse.urljoin(target_site_url, 'api/1/groups/ckan'),
                    headers = request_headers)
                try:
                    response = urllib2.urlopen(request, json.dumps(dict(
                        api_key = target_api_key,
                        value = group,
                        )))
                except urllib2.HTTPError as response:
                    log.error(u'An error occured while upserting group: {}'.format(group))
                    response_text = response.read()
                    try:
                        response_dict = json.loads(response_text)
                    except ValueError:
                        log.error(response_text)
                        raise
                    for key, value in response_dict.iteritems():
                        print '{} = {}'.format(key, value)
                    raise
                else:
                    assert response.code == 200
                    response_dict = json.loads(response.read())
#                    group = response_dict['value']
#                    print group
            elif action == 'delete':
                group = message['msg']

                log.info(u'Deleting group: {}'.format(group['id']))
                request_headers = target_headers.copy()
                request_headers['Content-Type'] = 'application/json'
                request = urllib2.Request(urlparse.urljoin(target_site_url, 'api/1/groups/{}'.format(group['id'])),
                    headers = request_headers)
                request.get_method = lambda: 'DELETE'
                try:
                    response = urllib2.urlopen(request, json.dumps(dict(
                        api_key = target_api_key,
                        )))
                except urllib2.HTTPError as response:
                    if response.code == 404:
                        log.warning(u"Deleted group doesn't exist: {}".format(group['id']))
                        response_dict = json.loads(response.read())
                    else:
                        log.error(u'An error occured while deleting group: {}'.format(group))
                        response_text = response.read()
                        try:
                            response_dict = json.loads(response_text)
                        except ValueError:
                            log.error(response_text)
                            raise
                        for key, value in response_dict.iteritems():
                            print '{} = {}'.format(key, value)
                        raise
                else:
                    assert response.code == 200
                    response_dict = json.loads(response.read())
#                    group = response_dict['value']
#                    print group
            else:
                log.warning(u'TODO: Handle {}, {} for {}'.format(kind, action, message))
        elif kind == 'organization':
            if action in ('create', 'update'):
                organization = message['msg']

                log.info(u'Upserting organization: {}'.format(organization['name']))
                request_headers = target_headers.copy()
                request_headers['Content-Type'] = 'application/json'
                request = urllib2.Request(urlparse.urljoin(target_site_url, 'api/1/organizations/ckan'),
                    headers = request_headers)
                try:
                    response = urllib2.urlopen(request, json.dumps(dict(
                        api_key = target_api_key,
                        value = organization,
                        )))
                except urllib2.HTTPError as response:
                    log.error(u'An error occured while upserting organization: {}'.format(organization))
                    response_text = response.read()
                    try:
                        response_dict = json.loads(response_text)
                    except ValueError:
                        log.error(response_text)
                        raise
                    for key, value in response_dict.iteritems():
                        print '{} = {}'.format(key, value)
                    raise
                else:
                    assert response.code == 200
                    response_dict = json.loads(response.read())
#                    organization = response_dict['value']
#                    print organization
            elif action == 'delete':
                organization = message['msg']

                log.info(u'Deleting organization: {}'.format(organization['id']))
                request_headers = target_headers.copy()
                request_headers['Content-Type'] = 'application/json'
                request = urllib2.Request(urlparse.urljoin(target_site_url, 'api/1/organizations/{}'.format(
                    organization['id'])), headers = request_headers)
                request.get_method = lambda: 'DELETE'
                try:
                    response = urllib2.urlopen(request, json.dumps(dict(
                        api_key = target_api_key,
                        )))
                except urllib2.HTTPError as response:
                    if response.code == 404:
                        log.warning(u"Deleted organization doesn't exist: {}".format(organization['id']))
                        response_dict = json.loads(response.read())
                    else:
                        log.error(u'An error occured while deleting organization: {}'.format(organization))
                        response_text = response.read()
                        try:
                            response_dict = json.loads(response_text)
                        except ValueError:
                            log.error(response_text)
                            raise
                        for key, value in response_dict.iteritems():
                            print '{} = {}'.format(key, value)
                        raise
                else:
                    assert response.code == 200
                    response_dict = json.loads(response.read())
#                    organization = response_dict['value']
#                    print organization
            else:
                log.warning(u'TODO: Handle {}, {} for {}'.format(kind, action, message))
        elif kind == 'package':
            if action in ('create', 'update'):
                package = message['msg']

                log.info(u'Upserting package: {}'.format(package['name']))
                request_headers = target_headers.copy()
                request_headers['Content-Type'] = 'application/json'
                request = urllib2.Request(urlparse.urljoin(target_site_url, 'api/1/datasets/ckan'),
                    headers = request_headers)
                try:
                    response = urllib2.urlopen(request, json.dumps(dict(
                        api_key = target_api_key,
                        value = package,
                        )))
                except urllib2.HTTPError as response:
                    log.error(u'An error occured while upserting package: {}'.format(package))
                    response_text = response.read()
                    try:
                        response_dict = json.loads(response_text)
                    except ValueError:
                        log.error(response_text)
                        raise
                    for key, value in response_dict.iteritems():
                        print '{} = {}'.format(key, value)
                    raise
                else:
                    assert response.code == 200
                    response_dict = json.loads(response.read())
#                    dataset = response_dict['value']
#                    print dataset
            elif action == 'delete':
                package = message['msg']

                log.info(u'Deleting package: {}'.format(package['id']))
                request_headers = target_headers.copy()
                request_headers['Content-Type'] = 'application/json'
                request = urllib2.Request(urlparse.urljoin(target_site_url, 'api/1/datasets/{}'.format(package['id'])),
                    headers = request_headers)
                request.get_method = lambda: 'DELETE'
                try:
                    response = urllib2.urlopen(request, json.dumps(dict(
                        api_key = target_api_key,
                        )))
                except urllib2.HTTPError as response:
                    if response.code == 404:
                        log.warning(u"Deleted package doesn't exist: {}".format(package['id']))
                        response_dict = json.loads(response.read())
                    else:
                        log.error(u'An error occured while deleting package: {}'.format(package))
                        response_text = response.read()
                        try:
                            response_dict = json.loads(response_text)
                        except ValueError:
                            log.error(response_text)
                            raise
                        for key, value in response_dict.iteritems():
                            print '{} = {}'.format(key, value)
                        raise
                else:
                    assert response.code == 200
                    response_dict = json.loads(response.read())
#                    dataset = response_dict['value']
#                    print dataset
            else:
                log.warning(u'TODO: Handle {}, {} for {}'.format(kind, action, message))
        elif kind == 'related':
            if action in ('create', 'update'):
                related = message['msg']

                log.info(u'Upserting related: {}'.format(related['title']))
                request_headers = target_headers.copy()
                request_headers['Content-Type'] = 'application/json'
                request = urllib2.Request(urlparse.urljoin(target_site_url, 'api/1/datasets/ckan/related'),
                    headers = request_headers)
                try:
                    response = urllib2.urlopen(request, json.dumps(dict(
                        api_key = target_api_key,
                        value = related,
                        )))
                except urllib2.HTTPError as response:
                    log.error(u'An error occured while upserting related: {}'.format(related))
                    response_text = response.read()
                    try:
                        response_dict = json.loads(response_text)
                    except ValueError:
                        log.error(response_text)
                        raise
                    for key, value in response_dict.iteritems():
                        print '{} = {}'.format(key, value)
                    raise
                else:
                    assert response.code == 200
                    response_dict = json.loads(response.read())
#                    related = response_dict['value']
#                    print related
            elif action == 'delete':
                related = message['msg']

                log.info(u'Deleting related: {}'.format(related['id']))
                request_headers = target_headers.copy()
                request_headers['Content-Type'] = 'application/json'
                request = urllib2.Request(urlparse.urljoin(target_site_url, 'api/1/datasets/related/{}'.format(
                    related['id'])), headers = request_headers)
                request.get_method = lambda: 'DELETE'
                try:
                    response = urllib2.urlopen(request, json.dumps(dict(
                        api_key = target_api_key,
                        )))
                except urllib2.HTTPError as response:
                    if response.code == 404:
                        log.warning(u"Deleted related doesn't exist: {}".format(related['id']))
                        response_dict = json.loads(response.read())
                    else:
                        log.error(u'An error occured while deleting related: {}'.format(related))
                        response_text = response.read()
                        try:
                            response_dict = json.loads(response_text)
                        except ValueError:
                            log.error(response_text)
                            raise
                        for key, value in response_dict.iteritems():
                            print '{} = {}'.format(key, value)
                        raise
                else:
                    assert response.code == 200
                    response_dict = json.loads(response.read())
#                    related = response_dict['value']
#                    print related
            else:
                log.warning(u'TODO: Handle {}, {} for {}'.format(kind, action, message))
        elif kind == 'user':
            if action in ('create', 'update'):
                user = message['msg']

                # Retrieve full user profile (with email, etc).
                request = urllib2.Request(urlparse.urljoin(source_site_url, 'api/3/action/user_show'),
                    headers = source_headers)
                try:
                    response = urllib2.urlopen(request, urllib.quote(json.dumps(dict(
                        id = user['id'],
                        ))))  # CKAN < 2.0 requires a POST.
                except urllib2.HTTPError as response:
                    if response.code == 403:
                        # Private user => Keep incomplete user profile.
                        pass
                    else:
                        raise
                else:
                    response_text = response.read()
                    try:
                        response_dict = json.loads(response_text)
                    except ValueError:
                        log.error(response_text)
                        raise
                    user = response_dict['result']

                log.info(u'Upserting user: {}'.format(user['name']))
                request_headers = target_headers.copy()
                request_headers['Content-Type'] = 'application/json'
                request = urllib2.Request(urlparse.urljoin(target_site_url, 'api/1/accounts/ckan'),
                    headers = request_headers)
                try:
                    response = urllib2.urlopen(request, json.dumps(dict(
                        api_key = target_api_key,
                        value = user,
                        )))
                except urllib2.HTTPError as response:
                    log.error(u'An error occured while upserting user: {}'.format(user))
                    response_text = response.read()
                    try:
                        response_dict = json.loads(response_text)
                    except ValueError:
                        log.error(response_text)
                        raise
                    for key, value in response_dict.iteritems():
                        print '{} = {}'.format(key, value)
                    raise
                else:
                    assert response.code == 200
                    response_dict = json.loads(response.read())
#                    account = response_dict['value']
#                    print account
            elif action == 'delete':
                user = message['msg']

                log.info(u'Deleting user: {}'.format(user['id']))
                request_headers = target_headers.copy()
                request_headers['Content-Type'] = 'application/json'
                request = urllib2.Request(urlparse.urljoin(target_site_url, 'api/1/accounts/{}'.format(user['id'])),
                    headers = request_headers)
                request.get_method = lambda: 'DELETE'
                try:
                    response = urllib2.urlopen(request, json.dumps(dict(
                        api_key = target_api_key,
                        )))
                except urllib2.HTTPError as response:
                    if response.code == 404:
                        log.warning(u"Deleted user doesn't exist: {}".format(user['id']))
                        response_dict = json.loads(response.read())
                    else:
                        log.error(u'An error occured while deleting user: {}'.format(user))
                        response_text = response.read()
                        try:
                            response_dict = json.loads(response_text)
                        except ValueError:
                            log.error(response_text)
                            raise
                        for key, value in response_dict.iteritems():
                            print '{} = {}'.format(key, value)
                        raise
                else:
                    assert response.code == 200
                    response_dict = json.loads(response.read())
#                    account = response_dict['value']
#                    print account
            else:
                log.warning(u'TODO: Handle {}, {} for {}'.format(kind, action, message))
        else:
            log.debug(u'TODO: Handle {}, {} for {}'.format(kind, action, message))

    return 0
    if fullname != 'releng/fedora-scm-requests':
        print("Dropping %r.  Not scm request." % fullname)
        return False
    if 'close_status' not in fields:
        print("Dropping %r %r.  Not closed." % (fullname, fields))
        return False

    handle(content)


if __name__ == '__main__':
    config = fedmsg.config.load_config()
    logging.config.dictConfig(config['logging'])
    fedmsg.meta.make_processors(**config)
    topic = 'io.pagure.prod.pagure.issue.edit'
    for _, _, topic, msg in fedmsg.tail_messages(topic=topic):
        # Extract some useful information for debugging
        title, subtitle, link, idx = [None] * 4
        try:
            title = fedmsg.meta.msg2title(msg, **config)
            subtitle = fedmsg.meta.msg2subtitle(msg, **config)
            link = fedmsg.meta.msg2link(msg, **config)
            idx = msg.get('msg_id')
        except Exception as e:
            print("!! Failed to determine title, subtitle, link")
        print("Inspecting {title}, {subtitle}, {link}, {idx}".format(
            title=title, subtitle=subtitle, link=link, idx=idx))

        # Extract values we need to actually process the message
        fullname = msg['msg']['project']['fullname']
        fields = msg['msg']['fields']
예제 #28
0
파일: tail.py 프로젝트: anistark/fedmsg
    def run(self):
        # Disable sending
        self.config['publish_endpoint'] = None

        # Disable timeouts.  We want to tail forever!
        self.config['timeout'] = 0

        # Even though fedmsg-tail won't be sending any messages, give it a
        # name to conform with the other commands.
        self.config['name'] = 'relay_inbound'

        # Tail is never going to send any messages, so we suppress warnings
        # about having no publishing sockets established.
        self.config['mute'] = True

        fedmsg.init(**self.config)

        # Build a message formatter
        formatter = lambda d: d
        if self.config['pretty']:
            def formatter(d):
                d['timestamp'] = time.ctime(d['timestamp'])
                d = fedmsg.crypto.strip_credentials(d)
                return "\n" + pprint.pformat(d)

        if self.config['really_pretty']:
            def formatter(d):
                d = fedmsg.crypto.strip_credentials(d)
                fancy = pygments.highlight(
                    fedmsg.encoding.pretty_dumps(d),
                    pygments.lexers.JavascriptLexer(),
                    pygments.formatters.TerminalFormatter()
                ).strip()
                return "\n" + fancy

        if self.config['query']:
            def formatter(d):
                result = fedmsg.utils.dict_query(d, self.config['query'])
                return ", ".join([six.text_type(value) for value in result.values()])

        if self.config['terse']:
            formatter = lambda d: "\n" + fedmsg.meta.msg2repr(d, **self.config)

        if self.config['cowsay']:
            def formatter(d):
                result, error = cowsay_output(fedmsg.meta.msg2subtitle(d, **self.config))
                if not error:
                    return "\n" + result
                else:
                    return "\n" + error

        # Build regular expressions for use in our loop.
        exclusive_regexp = re.compile(self.config['exclusive_regexp'])
        inclusive_regexp = re.compile(self.config['inclusive_regexp'])

        # Build username and package filter sets for use in our loop.
        users, packages = set(), set()
        if self.config['users']:
            users = set(map(str.strip, self.config['users'].split(',')))
        if self.config['packages']:
            packages = set(map(str.strip, self.config['packages'].split(',')))

        # Only initialize this if we have to
        if users or packages or self.config['terse'] or self.config['cowsay']:
            fedmsg.meta.make_processors(**self.config)

        # Spin up a zmq.Poller and yield messages
        for name, ep, topic, message in fedmsg.tail_messages(**self.config):
            if exclusive_regexp.search(topic):
                continue

            if not inclusive_regexp.search(topic):
                continue

            if users:
                actual = fedmsg.meta.msg2usernames(message, **self.config)
                if not users.intersection(actual):
                    continue

            if packages:
                actual = fedmsg.meta.msg2packages(message, **self.config)
                if not packages.intersection(actual):
                    continue

            output = formatter(message)
            if output:
                self.log.info(output)
예제 #29
0
파일: tail.py 프로젝트: amitsaha/fedmsg
    def run(self):
        # Disable sending
        self.config['publish_endpoint'] = None

        # Disable timeouts.  We want to tail forever!
        self.config['timeout'] = 0

        # Even though fedmsg-tail won't be sending any messages, give it a
        # name to conform with the other commands.
        self.config['name'] = 'relay_inbound'

        # Tail is never going to send any messages, so we suppress warnings
        # about having no publishing sockets established.
        self.config['mute'] = True

        fedmsg.init(**self.config)
        fedmsg.meta.make_processors(**self.config)

        # Build a message formatter
        formatter = lambda d: d
        if self.config['pretty']:
            def formatter(d):
                d['timestamp'] = time.ctime(d['timestamp'])
                d = fedmsg.crypto.strip_credentials(d)
                return "\n" + pprint.pformat(d)

        if self.config['really_pretty']:
            def formatter(d):
                d = fedmsg.crypto.strip_credentials(d)
                fancy = pygments.highlight(
                    fedmsg.encoding.pretty_dumps(d),
                    pygments.lexers.JavascriptLexer(),
                    pygments.formatters.TerminalFormatter()
                ).strip()
                return "\n" + fancy

        if self.config['terse']:
            formatter = lambda d: "\n" + fedmsg.meta.msg2repr(d, **self.config)

        if self.config['gource']:
            # Output strings suitable for consumption by the "gource" tool.

            # We have 8 colors here and an unknown number of message types.
            # (There were 14 message types at the time this code was written).
            # Here we build a dict that maps message type names (a.k.a modnames
            # or services) to hex colors for usage in the gource graph.  We
            # wrap-around that dict if there are more message types than
            # there are colors (which there almost certainly are).
            procs = [proc.__name__.lower() for proc in fedmsg.meta.processors]
            colors = ["FFFFFF", "008F37", "FF680A", "CC4E00",
                      "8F0058", "8F7E00", "37008F", "7E008F"]
            n_wraps = 1 + int(math.ceil(len(colors) / float(len(procs))))
            colors = colors * n_wraps
            color_lookup = dict(zip(procs, colors))

            cache_directory = self.config['gource_user_image_dir']

            # After all that color trickiness, here is our formatter we'll use.
            def formatter(message):
                """ Use this like::

                  $ fedmsg-tail --gource | gource \
                          -i 0 \
                          --user-image-dir ~/.cache/gravatar/ \
                          --log-format custom -
                """
                proc = fedmsg.meta.msg2processor(message, **self.config)
                users = fedmsg.meta.msg2usernames(message, **self.config)
                objs = fedmsg.meta.msg2objects(message, **self.config)
                name = proc.__name__.lower()

                if not users:
                    users = [name]

                lines = []
                for user, obj in itertools.product(users, objs):
                    _grab_and_cache_avatar(user, cache_directory)
                    lines.append("%i|%s|A|%s|%s" % (
                        message['timestamp'],
                        user,
                        name + "/" + obj,
                        color_lookup[name],
                    ))
                return "\n".join(lines)

        exclusive_regexp = re.compile(self.config['exclusive_regexp'])
        inclusive_regexp = re.compile(self.config['inclusive_regexp'])

        # The "proper" fedmsg way to do this would be to spin up or connect to
        # an existing Moksha Hub and register a consumer on the "*" topic that
        # simply prints out each message it consumes.  That seems like
        # overkill, so we're just going to directly access the endpoints
        # ourself.
        for name, ep, topic, message in fedmsg.tail_messages(**self.config):
            if exclusive_regexp.search(topic):
                continue

            if not inclusive_regexp.search(topic):
                continue

            self.log.info(formatter(message))
예제 #30
0
def main():
    parser = argparse.ArgumentParser(description = __doc__)
    parser.add_argument('config', help = 'path of configuration file')
    parser.add_argument('-d', '--delete', action = 'store_true', help = 'force datastore re-creation')
    parser.add_argument('-f', '--fedmsg', action = 'store_true', help = 'poll fedmsg events')
    parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')

    global args
    args = parser.parse_args()
    logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)

    config_parser = ConfigParser.SafeConfigParser(dict(
        here = os.path.dirname(os.path.abspath(os.path.normpath(args.config))),
        ))
    config_parser.read(args.config)
    conf = conv.check(conv.pipe(
        conv.test_isinstance(dict),
        conv.struct(
            {
                'ckan.api_key': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'ckan.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'user_agent': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'weckan.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                },
            default = 'drop',
            ),
        conv.not_none,
        ))(dict(config_parser.items('CowBots-Datasets-to-Dataset')), conv.default_state)

    ckan_headers = {
        'Authorization': conf['ckan.api_key'],
        'User-Agent': conf['user_agent'],
        }

    organization_name = u'premier-ministre'
    request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'],
        '/api/3/action/organization_show?id={}'.format(organization_name)), headers = ckan_headers)
    response = urllib2.urlopen(request)
    response_dict = json.loads(response.read())
    organization = conv.check(conv.pipe(
        conv.make_ckan_json_to_organization(),
        conv.not_none,
        ))(response_dict['result'], state = conv.default_state)

    package_title = u'Jeux de données de data.gouv.fr'
    package_name = strings.slugify(package_title)

    # Try to retrieve exising package, to ensure that its resources will not be destroyed by package_update.
    request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'],
        '/api/3/action/package_show?id={}'.format(package_name)), headers = ckan_headers)
    try:
        response = urllib2.urlopen(request)
    except urllib2.HTTPError as response:
        if response.code == 404:
            package = {}
        else:
            raise
    else:
        assert response.code == 200
        response_dict = json.loads(response.read())
        package = conv.check(conv.pipe(
            conv.make_ckan_json_to_package(drop_none_values = True),
            conv.not_none,
            ))(response_dict['result'], state = conv.default_state)

    package.update(dict(
        author = u'Secrétariat général du Gouvernement',
        author_email = u'*****@*****.**',
        extras = [
            dict(
                key = u"Date de production des données",
                value = datetime.date.today().isoformat(),
                ),
            ],
        frequency = u'temps réel',
        groups = [
            dict(id = strings.slugify(u'État et collectivités')),
            ],
        license_id = u'fr-lo',
        name = package_name,
        notes = u"""Les jeux de données ouvertes collectés par la mission Etalab""",
        owner_org = organization['id'],
#        relationships_as_object (list of relationship dictionaries) – see package_relationship_create() for the format of relationship dictionaries (optional)
#        relationships_as_subject (list of relationship dictionaries) – see package_relationship_create() for the format of relationship dictionaries (optional)
#        resources = [
#            dict(
#                created = entry.get(u'Date de publication'),
#                format = data.get('Format'),
#                last_modified = entry.get(u'Date de dernière modification'),
#                name = data.get('Titre'),
#                # package_id (string) – id of package that the resource needs should be added to.
#                url = data['URL'],
##                        revision_id – (optional)
##                        description (string) – (optional)
##                        hash (string) – (optional)
##                        resource_type (string) – (optional)
##                        mimetype (string) – (optional)
##                        mimetype_inner (string) – (optional)
##                        webstore_url (string) – (optional)
##                        cache_url (string) – (optional)
##                        size (int) – (optional)
##                        cache_last_updated (iso date string) – (optional)
##                        webstore_last_updated (iso date string) – (optional)
#                )
#            for data in entry.get(u'Données', []) + entry.get(u'Documents annexes', [])
#            ],
        state = 'active',  # Undelete package if it was deleted.
#        tags = [
#            dict(
#                name = tag_name,
##                vocabulary_id (string) – the name or id of the vocabulary that the new tag should be added to, e.g. 'Genre'
#                )
#            for tag_name in (
#                strings.slugify(keyword)[:100]
#                for keyword in entry.get(u'Mots-clés', [])
#                if keyword is not None
#                )
#            if len(tag_name) >= 2
#            ],
        territorial_coverage = u'Country/FR',
        territorial_coverage_granularity = u'commune',
        title = package_title,
#        type (string) – the type of the dataset (optional), IDatasetForm plugins associate themselves with different dataset types and provide custom dataset handling behaviour for these types
#        url (string) – a URL for the dataset’s source (optional)
#        version (string, no longer than 100 characters) – (optional)
        ))

    if package.get('id') is None:
        request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'], '/api/3/action/package_create'),
            headers = ckan_headers)
        try:
            response = urllib2.urlopen(request, urllib.quote(json.dumps(package)))
        except urllib2.HTTPError as response:
            response_text = response.read()
            try:
                response_dict = json.loads(response_text)
            except ValueError:
                log.error(u'An exception occured while creating package: {0}'.format(package))
                log.error(response_text)
                raise
            for key, value in response_dict.iteritems():
                print '{} = {}'.format(key, value)
        else:
            assert response.code == 200
            response_dict = json.loads(response.read())
    else:
        request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'],
            '/api/3/action/package_update?id={}'.format(package_name)), headers = ckan_headers)
        try:
            response = urllib2.urlopen(request, urllib.quote(json.dumps(package)))
        except urllib2.HTTPError as response:
            response_text = response.read()
            try:
                response_dict = json.loads(response_text)
            except ValueError:
                log.error(u'An exception occured while updating package: {0}'.format(package))
                log.error(response_text)
                raise
            for key, value in response_dict.iteritems():
                print '{} = {}'.format(key, value)
        else:
            assert response.code == 200
            response_dict = json.loads(response.read())
    package = conv.check(conv.pipe(
        conv.make_ckan_json_to_package(),
        conv.not_none,
        ))(response_dict['result'], state = conv.default_state)
    package_id = package['id']

    resources = package['resources']
    if resources:
        assert len(resources) == 1, package
        existing_resource = resources[0].copy()
        existing_resource['package_id'] = package_id
    else:
        existing_resource = {}
    resource = existing_resource.copy()
    resource.update(dict(
        package_id = package_id,
#        format = data.get('Format'),
        name = u"Base de données",
        url = urlparse.urljoin(conf['weckan.site_url'], '/dataset/{}'.format(package_name)),
#        revision_id – (optional)
        description = u"""\
Base de données générée automatiquement à partir du contenu de data.gouv.fr\
""",
#        format (string) – (optional)
#        hash (string) – (optional)
#        resource_type (string) – (optional)
#        mimetype (string) – (optional)
#        mimetype_inner (string) – (optional)
#        webstore_url (string) – (optional)
#        cache_url (string) – (optional)
#        size (int) – (optional)
#        created (iso date string) – (optional)
#        last_modified (iso date string) – (optional)
#        cache_last_updated (iso date string) – (optional)
#        webstore_last_updated (iso date string) – (optional)
        ))
    if resource != existing_resource:
        if resource.get('id') is None:
            request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'], '/api/3/action/resource_create'),
                headers = ckan_headers)
            try:
                response = urllib2.urlopen(request, urllib.quote(json.dumps(resource)))
            except urllib2.HTTPError as response:
                response_dict = json.loads(response.read())
                for key, value in response_dict.iteritems():
                    print '{} = {}'.format(key, value)
                raise
            else:
                assert response.code == 200
                response_dict = json.loads(response.read())
        else:
            request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'],
                '/api/3/action/resource_update?id={}'.format(resource['id'])), headers = ckan_headers)
            try:
                response = urllib2.urlopen(request, urllib.quote(json.dumps(resource)))
            except urllib2.HTTPError as response:
                response_dict = json.loads(response.read())
                for key, value in response_dict.iteritems():
                    print '{} = {}'.format(key, value)
                raise
            else:
                assert response.code == 200
                response_dict = json.loads(response.read())
        resource = conv.check(conv.pipe(
            conv.make_ckan_json_to_resource(),
            conv.not_none,
            ))(response_dict['result'], state = conv.default_state)

    if args.delete:
        request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'], '/api/3/action/datastore_delete'),
            headers = ckan_headers)
        try:
            response = urllib2.urlopen(request, urllib.quote(json.dumps(dict(
                resource_id = resource['id'],
                ))))
        except urllib2.HTTPError as response:
            response_text = response.read()
            try:
                response_dict = json.loads(response_text)
            except ValueError:
                log.error(u'An exception occured while deleting datastore: {0}'.format(resource['id']))
                log.error(response_text)
                raise
            for key, value in response_dict.iteritems():
                print '{} = {}'.format(key, value)
            raise

    for index in range(2):
        datastore = dict(
            fields = [
                dict(id = 'author', type = 'text'),
                dict(id = 'author_email', type = 'text'),
                dict(id = 'ckan_url', type = 'text'),
                dict(id = 'extras', type = 'json'),
                dict(id = 'frequency', type = 'text'),
                dict(id = 'groups', type = 'json'),
                dict(id = 'id', type = 'text'),
#                dict(id = 'isopen', type = 'bool'),
                dict(id = 'license_id', type = 'text'),
#                dict(id = 'license_title', type = 'text'),
#                dict(id = 'license_url', type = 'text'),
                dict(id = 'maintainer', type = 'text'),
                dict(id = 'maintainer_email', type = 'text'),
                dict(id = 'metadata_created', type = 'date'),
                dict(id = 'metadata_modified', type = 'date'),
                dict(id = 'name', type = 'text'),
                dict(id = 'notes', type = 'text'),
                dict(id = 'organization', type = 'json'),
                dict(id = 'owner_org', type = 'text'),
#                dict(id = 'private', type = 'bool'),
                dict(id = 'relationships_as_object', type = 'json'),
                dict(id = 'relationships_as_subject', type = 'json'),
                dict(id = 'resources', type = 'json'),
#                dict(id = 'revision_id', type = 'text'),
                dict(id = 'revision_timestamp', type = 'timestamp'),
#                dict(id = 'state', type = 'text'),
                dict(id = 'supplier', type = 'json'),
                dict(id = 'supplier_id', type = 'text'),
                dict(id = 'tags', type = 'json'),
                dict(id = 'temporal_coverage_from', type = 'text'),  # not a "date", because it can be just a year
                dict(id = 'temporal_coverage_to', type = 'text'),  # not a "date", because it can be just a year
                dict(id = 'territorial_coverage', type = 'text'),
                dict(id = 'territorial_coverage_granularity', type = 'text'),
                dict(id = 'title', type = 'text'),
#                dict(id = 'tracking_summary', type = 'json'),
                dict(id = 'type', type = 'text'),
                dict(id = 'url', type = 'text'),
                dict(id = 'version', type = 'text'),
                ],
            primary_key = 'id',
            resource_id = resource['id'],
            )
        request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'], '/api/3/action/datastore_create'),
            headers = ckan_headers)
        try:
            response = urllib2.urlopen(request, urllib.quote(json.dumps(datastore)))
        except urllib2.HTTPError as response:
            response_text = response.read()
            try:
                response_dict = json.loads(response_text)
            except ValueError:
                log.error(u'An exception occured while creating datastore: {0}'.format(datastore))
                log.error(response_text)
                raise
            if response.code == 409 and index == 0:
                # Conflict: The fields may have changed. Delete datastore and recreate it.
                request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'], '/api/3/action/datastore_delete'),
                    headers = ckan_headers)
                try:
                    response = urllib2.urlopen(request, urllib.quote(json.dumps(dict(
                        resource_id = resource['id'],
                        ))))
                except urllib2.HTTPError as response:
                    response_text = response.read()
                    try:
                        response_dict = json.loads(response_text)
                    except ValueError:
                        log.error(u'An exception occured while deleting datastore: {0}'.format(resource['id']))
                        log.error(response_text)
                        raise
                    for key, value in response_dict.iteritems():
                        print '{} = {}'.format(key, value)
                    raise
                continue
            for key, value in response_dict.iteritems():
                print '{} = {}'.format(key, value)
            raise
        assert response.code == 200
        response_dict = json.loads(response.read())
        datastore = conv.check(conv.pipe(
            conv.make_ckan_json_to_datastore(),
            conv.not_none,
            ))(response_dict['result'], state = conv.default_state)
        break

    if args.fedmsg:
        import fedmsg

        fedmsg_conf = conv.check(conv.struct(
            dict(
                environment = conv.pipe(
                    conv.empty_to_none,
                    conv.test_in(['dev', 'prod', 'stg']),
                    ),
                modname = conv.pipe(
                    conv.empty_to_none,
                    conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                    conv.default('ckan'),
                    ),
#                name = conv.pipe(
#                    conv.empty_to_none,
#                    conv.default('ckan.{}'.format(hostname)),
#                    ),
                topic_prefix = conv.pipe(
                    conv.empty_to_none,
                    conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                    ),
                ),
            default = 'drop',
            ))(dict(config_parser.items('fedmsg')), state = conv.default_state)

        # Read in the config from /etc/fedmsg.d/.
        fedmsg_config = fedmsg.config.load_config([], None)
        # Disable a warning about not sending.  We know.  We only want to tail.
        fedmsg_config['mute'] = True
        # Disable timing out so that we can tail forever.  This is deprecated
        # and will disappear in future versions.
        fedmsg_config['timeout'] = 0
        # For the time being, don't require message to be signed.
        fedmsg_config['validate_signatures'] = False
        for key, value in fedmsg_conf.iteritems():
            if value is not None:
                fedmsg_config[key] = value

        expected_topic_prefix = '{}.{}.ckan.'.format(fedmsg_config['topic_prefix'], fedmsg_config['environment'])
        for name, endpoint, topic, message in fedmsg.tail_messages(**fedmsg_config):
            if not topic.startswith(expected_topic_prefix):
                log.debug(u'Ignoring message: {}, {}'.format(topic, name))
                continue
            kind, action = topic[len(expected_topic_prefix):].split('.')
            if kind == 'package':
                if action in ('create', 'update'):
                    package = conv.check(conv.pipe(
                        conv.make_ckan_json_to_package(drop_none_values = True),
                        conv.not_none,
                        conv.ckan_input_package_to_output_package,
                        ))(message['msg'], state = conv.default_state)
                    if package['id'] == package_id:
                        # Avoid infinite loop.
                        continue
                    log.info(u'Upserting package "{}".'.format(package['name']))
                    assert package.get('ckan_url') is None, package
                    package['ckan_url'] = urlparse.urljoin(conf['weckan.site_url'], '/dataset/{}'.format(package['name']))

                    request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'], '/api/3/action/datastore_upsert'),
                        headers = ckan_headers)
                    try:
                        response = urllib2.urlopen(request, urllib.quote(json.dumps(dict(
                            method = 'upsert',
                            records = [package],
                            resource_id = resource['id'],
                            ))))
                    except urllib2.HTTPError as response:
                        response_text = response.read()
                        log.error(u'An exception occured while upserting package in datastore: {0}'.format(package))
                        try:
                            response_dict = json.loads(response_text)
                        except ValueError:
                            log.error(response_text)
                            raise
                        for key, value in response_dict.iteritems():
                            print '{} = {}'.format(key, value)
                        raise
                    assert response.code == 200
                    response_dict = json.loads(response.read())
                    assert response_dict['success'] is True
                    # upsert = response_dict['result']
                elif action == 'delete':
                    package = message['msg']  # contains only "id".
                    if package['id'] == package_id:
                        # Avoid infinite loop.
                        continue
                    log.info(u'Deleting package "{}".'.format(package))
                    request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'], '/api/3/action/datastore_delete'),
                        headers = ckan_headers)
                    try:
                        response = urllib2.urlopen(request, urllib.quote(json.dumps(dict(
                            filters = dict(
                                id = package['id'],
                                ),
                            resource_id = resource['id'],
                            ))))
                    except urllib2.HTTPError as response:
                        if response.code != 404:
                            response_text = response.read()
                            log.error(u'An exception occured while deleting package from datastore: {0}'.format(package))
                            try:
                                response_dict = json.loads(response_text)
                            except ValueError:
                                log.error(response_text)
                                raise
                            for key, value in response_dict.iteritems():
                                print '{} = {}'.format(key, value)
                            raise
                    else:
                        assert response.code == 200
                        response_dict = json.loads(response.read())
                        assert response_dict['success'] is True
                        # delete = response_dict['result']
                else:
                    log.debug(u'TODO: Handle {}, {} for {}'.format(kind, action, message))
            else:
                log.debug(u'TODO: Handle {}, {} for {}'.format(kind, action, message))
    else:
        # Retrieve names of packages already existing in CKAN.
        request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'], '/api/3/action/package_list'),
            headers = ckan_headers)
        response = urllib2.urlopen(request)
        response_dict = json.loads(response.read())
        packages_name = conv.check(conv.pipe(
            conv.ckan_json_to_name_list,
            conv.not_none,
            ))(response_dict['result'], state = conv.default_state)

        for package_name in packages_name:
            request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'],
                '/api/3/action/package_show?id={}'.format(package_name)), headers = ckan_headers)
            response = urllib2.urlopen(request)
            response_dict = json.loads(response.read())
            package = conv.check(conv.pipe(
                conv.make_ckan_json_to_package(drop_none_values = True),
                conv.not_none,
                conv.ckan_input_package_to_output_package,
                ))(response_dict['result'], state = conv.default_state)
            assert package.get('ckan_url') is None, package
            package['ckan_url'] = urlparse.urljoin(conf['weckan.site_url'], '/dataset/{}'.format(package['name']))

            request = urllib2.Request(urlparse.urljoin(conf['ckan.site_url'], '/api/3/action/datastore_upsert'),
                headers = ckan_headers)
            try:
                response = urllib2.urlopen(request, urllib.quote(json.dumps(dict(
                    method = 'upsert',
                    records = [package],
                    resource_id = resource['id'],
                    ))))
            except urllib2.HTTPError as response:
                response_text = response.read()
                log.error(u'An exception occured while upserting package in datastore: {0}'.format(package))
                try:
                    response_dict = json.loads(response_text)
                except ValueError:
                    log.error(response_text)
                    raise
                for key, value in response_dict.iteritems():
                    print '{} = {}'.format(key, value)
                raise
            assert response.code == 200
            response_dict = json.loads(response.read())
            assert response_dict['success'] is True
            # upsert = response_dict['result']

    return 0
예제 #31
0
def main():
    parser = argparse.ArgumentParser(description = __doc__)
    parser.add_argument('config', help = 'path of configuration file')
    parser.add_argument('-f', '--fedmsg', action = 'store_true', help = 'poll fedmsg events')
    parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')

    global args
    args = parser.parse_args()
    logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)

    config_parser = ConfigParser.SafeConfigParser(dict(
        here = os.path.dirname(os.path.abspath(os.path.normpath(args.config))),
        ))
    config_parser.read(args.config)
    global conf
    conf = conv.check(conv.pipe(
        conv.test_isinstance(dict),
        conv.struct(
            {
                'ckan_of_worms.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'tumblr.access_token_key': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'tumblr.access_token_secret': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'tumblr.client_key': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'tumblr.client_secret': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'tumblr.hostname': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'user_agent': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                },
            default = 'drop',
            ),
        conv.not_none,
        ))(dict(config_parser.items('CowBots-Tumblr-Related')), conv.default_state)

    cache_dir = os.path.join(app_dir, 'cache')
    if not os.path.exists(cache_dir):
        os.makedirs(cache_dir)
    data_dir = os.path.join(app_dir, 'data')
    if not os.path.exists(data_dir):
        os.makedirs(data_dir)
    global db
    db = anydbm.open(os.path.join(data_dir, 'tumblr-posts'), 'c')
    global headers
    headers = {
        'User-Agent': conf['user_agent'],
        }
    global templates_lookup
    templates_lookup = mako.lookup.TemplateLookup(
        default_filters = ['h'],
        directories = [os.path.join(app_dir, 'tumblr-related-templates')],
        input_encoding = 'utf-8',
        module_directory = os.path.join(cache_dir, 'tumblr-related-templates'),
        strict_undefined = True,
        )

#    # To obtain access token, uncomment the following code, run it and put the results in configuration file.
#    oauth = requests_oauthlib.OAuth1Session(conf['tumblr.client_key'], client_secret = conf['tumblr.client_secret'])
#    fetch_response = oauth.fetch_request_token('http://www.tumblr.com/oauth/request_token')
#    request_token_key = fetch_response.get('oauth_token')
#    request_token_secret = fetch_response.get('oauth_token_secret')
#    authorization_url = oauth.authorization_url('http://www.tumblr.com/oauth/authorize')
#    print 'Please go here and authorize,', authorization_url

#    redirect_response = raw_input('Paste the full redirect URL here: ')
#    oauth_response = oauth.parse_authorization_response(redirect_response)
#    verifier = oauth_response.get('oauth_verifier')
#    access_token_url = 'http://www.tumblr.com/oauth/access_token'
#    oauth = requests_oauthlib.OAuth1Session(conf['tumblr.client_key'],
#        client_secret = conf['tumblr.client_secret'],
#        resource_owner_key = request_token_key,
#        resource_owner_secret = request_token_secret,
#        verifier = verifier,
#        )
#    oauth_tokens = oauth.fetch_access_token(access_token_url)
#    access_token_key = oauth_tokens.get('oauth_token')
#    print 'access_token_key =', access_token_key
#    access_token_secret = oauth_tokens.get('oauth_token_secret')
#    print 'access_token_secret =', access_token_secret
#    return 0

    global oauth
    oauth = requests_oauthlib.OAuth1(conf['tumblr.client_key'], conf['tumblr.client_secret'],
        conf['tumblr.access_token_key'], conf['tumblr.access_token_secret'])

    if args.fedmsg:
        import fedmsg

        fedmsg_conf = conv.check(conv.struct(
            dict(
                environment = conv.pipe(
                    conv.empty_to_none,
                    conv.test_in(['dev', 'prod', 'stg']),
                    ),
                modname = conv.pipe(
                    conv.empty_to_none,
                    conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                    conv.default('ckan_of_worms'),
                    ),
#                name = conv.pipe(
#                    conv.empty_to_none,
#                    conv.default('ckan_of_worms.{}'.format(hostname)),
#                    ),
                topic_prefix = conv.pipe(
                    conv.empty_to_none,
                    conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                    ),
                ),
            default = 'drop',
            ))(dict(config_parser.items('fedmsg')))

        # Read in the config from /etc/fedmsg.d/.
        fedmsg_config = fedmsg.config.load_config([], None)
        # Disable a warning about not sending.  We know.  We only want to tail.
        fedmsg_config['mute'] = True
        # Disable timing out so that we can tail forever.  This is deprecated
        # and will disappear in future versions.
        fedmsg_config['timeout'] = 0
        # For the time being, don't require message to be signed.
        fedmsg_config['validate_signatures'] = False
        for key, value in fedmsg_conf.iteritems():
            if value is not None:
                fedmsg_config[key] = value

        expected_topic_prefix = '{}.{}.ckan_of_worms.'.format(fedmsg_config['topic_prefix'], fedmsg_config['environment'])
        for name, endpoint, topic, message in fedmsg.tail_messages(**fedmsg_config):
            if not topic.startswith(expected_topic_prefix):
                log.debug(u'Ignoring message: {}, {}'.format(topic, name))
                continue
            kind, action = topic[len(expected_topic_prefix):].split('.')
            if kind == 'dataset':
                if action in ('create', 'update'):
                    dataset_upserted(message['msg'])
    else:
        request = requests.get(urlparse.urljoin(conf['ckan_of_worms.site_url'], 'api/1/datasets'),
            params = dict(
                related = 1,
                ),
            headers = headers,
            )
        datasets_id = conv.check(conv.pipe(
            cow_response_to_value,
            conv.not_none,
            ))(request.text, state = conv.default_state)

        for dataset_id in datasets_id:
            response = requests.get(urlparse.urljoin(conf['ckan_of_worms.site_url'],
                'api/1/datasets/{}'.format(dataset_id)), headers = headers)
            dataset = conv.check(conv.pipe(
                cow_response_to_value,
                conv.not_none,
                ))(response.text, state = conv.default_state)
            dataset_upserted(dataset)

    return 0
예제 #32
0
파일: trigger.py 프로젝트: yutiansut/fedmsg
    def run(self):

        # This is a "required" option... :P
        if not self.config['command']:
            self.log.error("You must provide a --command to run.")
            sys.exit(1)

        # Disable sending
        self.config['publish_endpoint'] = None

        # Disable timeouts.  We want to tail forever!
        self.config['timeout'] = 0

        # Even though fedmsg-trigger won't be sending any messages, give it a
        # name to conform with the other commands.
        self.config['name'] = 'relay_inbound'

        # Tail is never going to send any messages, so we suppress warnings
        # about having no publishing sockets established.
        self.config['mute'] = True

        fedmsg.init(**self.config)

        exclusive_regexp = re.compile(self.config['exclusive_regexp'])
        inclusive_regexp = re.compile(self.config['inclusive_regexp'])

        wait_for = int(self.config['wait_for'])
        max_queue_size = int(self.config['max_queue_size'])

        timer = None
        que = queue.Queue()

        def execute_queue():
            while not que.empty():
                message = que.get()

                result = self.run_command(self.config['command'], message)

                if result != 0:
                    self.log.info("Command returned error code %r" % result)

        try:
            for name, ep, topic, message in fedmsg.tail_messages(
                    **self.config):
                if exclusive_regexp.search(topic):
                    continue

                if not inclusive_regexp.search(topic):
                    continue

                que.put(message)

                if timer is not None:
                    # Try to cancel it
                    timer.cancel()
                if timer is None or not timer.is_alive():
                    # Either there was no timer yet, or it was still waiting
                    # -> Let's start a new one
                    if (max_queue_size > 0 and que.qsize() > max_queue_size):
                        # If the que is too big, let's just run it NOW
                        timer = threading.Timer(0, execute_queue)
                    else:
                        timer = threading.Timer(wait_for, execute_queue)
                    timer.start()
        except KeyboardInterrupt:
            if timer is not None:
                timer.cancel()
                # Let's wait for commands to finish
                timer.join()
예제 #33
0
파일: tail.py 프로젝트: axilleas/fedmsg
    def run(self):
        # Disable sending
        self.config['publish_endpoint'] = None

        # Disable timeouts.  We want to tail forever!
        self.config['timeout'] = 0

        # Even though fedmsg-tail won't be sending any messages, give it a
        # name to conform with the other commands.
        self.config['name'] = 'relay_inbound'

        # Tail is never going to send any messages, so we suppress warnings
        # about having no publishing sockets established.
        self.config['mute'] = True

        fedmsg.init(**self.config)
        fedmsg.meta.make_processors(**self.config)

        # Build a message formatter
        formatter = lambda d: d
        if self.config['pretty']:
            def formatter(d):
                d['timestamp'] = time.ctime(d['timestamp'])
                d = fedmsg.crypto.strip_credentials(d)
                return "\n" + pprint.pformat(d)

        if self.config['really_pretty']:
            def formatter(d):
                d = fedmsg.crypto.strip_credentials(d)
                fancy = pygments.highlight(
                    fedmsg.encoding.pretty_dumps(d),
                    pygments.lexers.JavascriptLexer(),
                    pygments.formatters.TerminalFormatter()
                ).strip()
                return "\n" + fancy

        if self.config['terse']:
            formatter = lambda d: "\n" + fedmsg.meta.msg2repr(d, **self.config)

        if self.config['gource']:
            # Output strings suitable for consumption by the "gource" tool.

            # We have 8 colors here and an unknown number of message types.
            # (There were 14 message types at the time this code was written).
            # Here we build a dict that maps message type names (a.k.a modnames
            # or services) to hex colors for usage in the gource graph.  We
            # wrap-around that dict if there are more message types than
            # there are colors (which there almost certainly are).
            procs = [proc.__name__.lower() for proc in fedmsg.meta.processors]
            colors = ["FFFFFF", "008F37", "FF680A", "CC4E00",
                      "8F0058", "8F7E00", "37008F", "7E008F"]
            n_wraps = 1 + int(math.ceil(len(colors) / float(len(procs))))
            colors = colors * n_wraps
            color_lookup = dict(zip(procs, colors))

            cache_directory = self.config['gource_user_image_dir']

            # After all that color trickiness, here is our formatter we'll use.
            def formatter(message):
                """ Use this like::

                  $ fedmsg-tail --gource | gource \
                          -i 0 \
                          --user-image-dir ~/.cache/avatar/ \
                          --log-format custom -
                """
                proc = fedmsg.meta.msg2processor(message, **self.config)
                avatars = fedmsg.meta.msg2avatars(message, **self.config)
                objs = fedmsg.meta.msg2objects(message, **self.config)
                name = proc.__name__.lower()

                lines = []
                for user, obj in itertools.product(avatars.keys(), objs):
                    _cache_avatar(user, avatars[user], cache_directory)
                    lines.append("%i|%s|A|%s|%s" % (
                        message['timestamp'],
                        user,
                        name + "/" + obj,
                        color_lookup[name],
                    ))
                return "\n".join(lines)

        # Build regular expressions for use in our loop.
        exclusive_regexp = re.compile(self.config['exclusive_regexp'])
        inclusive_regexp = re.compile(self.config['inclusive_regexp'])

        # Build username and package filter sets for use in our loop.
        users, packages = set(), set()
        if self.config['users']:
            users = set(map(str.strip, self.config['users'].split(',')))
        if self.config['packages']:
            packages = set(map(str.strip, self.config['packages'].split(',')))

        # Spin up a zmq.Poller and yield messages
        for name, ep, topic, message in fedmsg.tail_messages(**self.config):
            if exclusive_regexp.search(topic):
                continue

            if not inclusive_regexp.search(topic):
                continue

            actual_users = fedmsg.meta.msg2usernames(message, **self.config)
            if users and not users.intersection(actual_users):
                continue

            actual_packages = fedmsg.meta.msg2packages(message, **self.config)
            if packages and not packages.intersection(actual_packages):
                continue

            self.log.info(formatter(message))
예제 #34
0
def main():
    parser = argparse.ArgumentParser(description = __doc__)
    parser.add_argument('config', help = 'path of configuration file')
    parser.add_argument('-f', '--fedmsg', action = 'store_true', help = 'poll fedmsg events')
    parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')

    global args
    args = parser.parse_args()
    logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)

    config_parser = ConfigParser.SafeConfigParser(dict(
        here = os.path.dirname(os.path.abspath(os.path.normpath(args.config))),
        ))
    config_parser.read(args.config)
    global conf
    conf = conv.check(conv.pipe(
        conv.test_isinstance(dict),
        conv.struct(
            {
                'admin_email': conv.pipe(
                    conv.function(lambda emails: set(emails.split())),
                    conv.uniform_sequence(
                        conv.pipe(
                            conv.input_to_email,
                            conv.test_email(),
                            ),
                        constructor = lambda emails: sorted(set(emails)),
                        drop_none_items = True,
                        ),
                    conv.empty_to_none,
                    conv.not_none,
                    ),
                'ckan_of_worms.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'from_email': conv.pipe(
                    conv.input_to_email,
                    conv.test_email(),
                    conv.not_none,
                    ),
                'user_agent': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'weckan.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'wiki.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'youckan.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                },
            default = 'drop',
            ),
        conv.not_none,
        ))(dict(config_parser.items('CowBots-Email-Changes')), conv.default_state)

    cache_dir = os.path.join(app_dir, 'cache')
    if not os.path.exists(cache_dir):
        os.makedirs(cache_dir)
    global headers
    headers = {
        'User-Agent': conf['user_agent'],
        }
    global templates_lookup
    templates_lookup = mako.lookup.TemplateLookup(
        directories = [os.path.join(app_dir, 'email-changes-templates')],
        input_encoding = 'utf-8',
        module_directory = os.path.join(cache_dir, 'email-changes-templates'),
        )

    if args.fedmsg:
        import fedmsg

        fedmsg_conf = conv.check(conv.struct(
            dict(
                environment = conv.pipe(
                    conv.empty_to_none,
                    conv.test_in(['dev', 'prod', 'stg']),
                    ),
                modname = conv.pipe(
                    conv.empty_to_none,
                    conv.test(lambda value: value == value.strip('.'),
                        error = 'Value must not begin or end with a "."'),
                    conv.default('ckan_of_worms'),
                    ),
#                name = conv.pipe(
#                    conv.empty_to_none,
#                    conv.default('ckan_of_worms.{}'.format(hostname)),
#                    ),
                topic_prefix = conv.pipe(
                    conv.empty_to_none,
                    conv.test(lambda value: value == value.strip('.'),
                        error = 'Value must not begin or end with a "."'),
                    ),
                ),
            default = 'drop',
            ))(dict(config_parser.items('fedmsg')))

        # Read in the config from /etc/fedmsg.d/.
        fedmsg_config = fedmsg.config.load_config([], None)
        # Disable a warning about not sending.  We know.  We only want to tail.
        fedmsg_config['mute'] = True
        # Disable timing out so that we can tail forever.  This is deprecated
        # and will disappear in future versions.
        fedmsg_config['timeout'] = 0
        # For the time being, don't require message to be signed.
        fedmsg_config['validate_signatures'] = False
        for key, value in fedmsg_conf.iteritems():
            if value is not None:
                fedmsg_config[key] = value

        ckan_of_worms_topic_prefix = '{}.{}.ckan_of_worms.'.format(fedmsg_config['topic_prefix'],
            fedmsg_config['environment'])
        wiki_topic_prefix = '{}.{}.wiki.'.format(fedmsg_config['topic_prefix'], fedmsg_config['environment'])
        for name, endpoint, topic, message in fedmsg.tail_messages(**fedmsg_config):
            if topic.startswith(ckan_of_worms_topic_prefix):
                kind, action = topic[len(ckan_of_worms_topic_prefix):].split('.')
                if kind == 'account':
                    if action == 'create':
                        account_created(message['msg'])
                elif kind == 'dataset':
                    if action == 'create':
                        dataset_created(message['msg'])
                elif kind == 'group':
                    if action == 'create':
                        group_created(message['msg'])
                elif kind == 'organization':
                    if action == 'create':
                        organization_created(message['msg'])
                elif kind == 'related':
                    if action == 'create':
                        related_created(message['msg'])
            elif topic.startswith(wiki_topic_prefix):
                kind, action = topic[len(wiki_topic_prefix):].split('.')
                if kind == 'article':
                    if action == 'edit':
                        article_edited(message['msg'])
                elif kind == 'upload':
                    if action == 'complete':
                        upload_completed(message['msg'])
            else:
                log.debug(u'Ignoring message: {}, {}'.format(topic, name))
    else:
        pass  # TODO

    return 0
예제 #35
0
 def run(self):
     for name, endpoint, topic, msg in fedmsg.tail_messages():
         for observer in self._observers.get(topic, []):
             observer(name, endpoint, topic, msg)
예제 #36
0
def main():
    parser = argparse.ArgumentParser(description = __doc__)
    parser.add_argument('config', help = 'path of configuration file')
    parser.add_argument('-f', '--fedmsg', action = 'store_true', help = 'poll fedmsg events')
    parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')

    global args
    args = parser.parse_args()
    logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)

    config_parser = ConfigParser.SafeConfigParser(dict(
        here = os.path.dirname(os.path.abspath(os.path.normpath(args.config))),
        ))
    config_parser.read(args.config)
    global conf
    conf = conv.check(conv.pipe(
        conv.test_isinstance(dict),
        conv.struct(
            {
                'ckan_of_worms.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'twitter.access_token_key': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'twitter.access_token_secret': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'twitter.consumer_key': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'twitter.consumer_secret': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'user_agent': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'weckan.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                },
            default = 'drop',
            ),
        conv.not_none,
        ))(dict(config_parser.items('CowBots-Tweet-Changes')), conv.default_state)

    global headers
    headers = {
        'User-Agent': conf['user_agent'],
        }
    global twitter_api
    twitter_api = twitter.Api(
        consumer_key = conf['twitter.consumer_key'],
        consumer_secret = conf['twitter.consumer_secret'],
        access_token_key = conf['twitter.access_token_key'],
        access_token_secret = conf['twitter.access_token_secret'],
        )
    # print twitter_api.VerifyCredentials()
    thread.start_new_thread(tweet_messages, ())

    if args.fedmsg:
        import fedmsg

        fedmsg_conf = conv.check(conv.struct(
            dict(
                environment = conv.pipe(
                    conv.empty_to_none,
                    conv.test_in(['dev', 'prod', 'stg']),
                    ),
                modname = conv.pipe(
                    conv.empty_to_none,
                    conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                    conv.default('ckan_of_worms'),
                    ),
#                name = conv.pipe(
#                    conv.empty_to_none,
#                    conv.default('ckan_of_worms.{}'.format(hostname)),
#                    ),
                topic_prefix = conv.pipe(
                    conv.empty_to_none,
                    conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                    ),
                ),
            default = 'drop',
            ))(dict(config_parser.items('fedmsg')))

        # Read in the config from /etc/fedmsg.d/.
        fedmsg_config = fedmsg.config.load_config([], None)
        # Disable a warning about not sending.  We know.  We only want to tail.
        fedmsg_config['mute'] = True
        # Disable timing out so that we can tail forever.  This is deprecated
        # and will disappear in future versions.
        fedmsg_config['timeout'] = 0
        # For the time being, don't require message to be signed.
        fedmsg_config['validate_signatures'] = False
        for key, value in fedmsg_conf.iteritems():
            if value is not None:
                fedmsg_config[key] = value

        expected_topic_prefix = '{}.{}.ckan_of_worms.'.format(fedmsg_config['topic_prefix'], fedmsg_config['environment'])
        for name, endpoint, topic, message in fedmsg.tail_messages(**fedmsg_config):
            if not topic.startswith(expected_topic_prefix):
                log.debug(u'Ignoring message: {}, {}'.format(topic, name))
                continue
            kind, action = topic[len(expected_topic_prefix):].split('.')
            if kind == 'account':
                if action == 'create':
                    account_created(message['msg'])
            if kind == 'dataset':
                if action == 'create':
                    dataset_created(message['msg'])
            elif kind == 'group':
                if action == 'create':
                    group_created(message['msg'])
            elif kind == 'organizaton':
                if action == 'create':
                    organization_created(message['msg'])
    else:
        pass  # TODO

    return 0
예제 #37
0
           break
        time.sleep(30)
    dom.create()
    print "Domain %s started" % (domain)
 
if __name__ == '__main__':
    pidfile = open('/var/run/harness.pid', 'w')
    pid = str(os.getpid())
    pidfile.write(pid)
    pidfile.close()
    config = fedmsg.config.load_config([], None)
    config['mute'] = True
    config['timeout'] = 0
    fedmsg.meta.make_processors(**config)

    for name, endpoint, topic, msg in fedmsg.tail_messages(**config):
        if "buildsys.build.state.change" in topic:
            matchedmsg = fedmsg.meta.msg2repr(msg, **config)
            if "completed" in matchedmsg:
                if "kernel" in matchedmsg:
                    objectmsg = fedmsg.meta.msg2subtitle(msg, legacy=False, **config)
                    package = string.split(objectmsg, ' ')
                    fcrelease = string.split(package[1], '.')
                    domain = domainmap(fcrelease[-1])
                    logfile = open('/var/log/harness.log', 'a')
                    logfile.write('Testing ' + package[1] + '\n')
                    logfile.close()
                    writelatest(domain, package[1])
                    dom32 = domain + '32'
                    dom64 = domain + '64'
                    print "starting domain %s" % (dom32)
예제 #38
0
#          Tomas Hozza <*****@*****.**>


# It requires fedmsg package
from __future__ import print_function
import fedmsg
import requests
from rebasehelper.upstream_monitoring import UpstreamMonitoring

# Testing command line
# echo "{'package':'wget', 'version': '1.6.13'} | fedmsg-logger

# version update is
VERSION_UPDATE = 'anitya.project.version.update'
while True:
    try:
        message = fedmsg.tail_messages()
        for name, endpoint, topic, msg in message:
            # TODO Use logger instead of print like rebase-upstream.log file in /var/log/
            #print (topic)
            if topic.endswith(VERSION_UPDATE):
                #print (endpoint)
                #print (msg)
                try:
                    up = UpstreamMonitoring(name, endpoint, topic, msg)
                    up.process_messsage()
                except:
                    raise
    except requests.exceptions.ConnectionError:
        pass
예제 #39
0
badgebot_topic_filter = config.get('badgebot_topic_filter')
badgebot_username = config.get('badgebot_fas_username')

# This is mandatory, otherwise we'll get banned from twitter.
if not badgebot_topic_filter:
    print "No 'badgebot_topic_filter' found in the fedmsg config.  Bailing."
    sys.exit(3)

print "Looking for messages with %r in the topic" % badgebot_topic_filter
if badgebot_username:
    print "  Relating to the user %r" % badgebot_username
else:
    print "  Relating to any user at all!"

print "Posting up to listen on the fedmsg bus.  Waiting for a message..."
for name, endpoint, topic, msg in fedmsg.tail_messages():
    if badgebot_topic_filter not in topic:
        continue

    if badgebot_username:
        users = fedmsg.meta.msg2usernames(msg, **config)
        if badgebot_username not in users:
            print "%r is not among %r" % (badgebot_username, users)
            continue
    else:
        print "'badgebot_fas_username' not specified.  Proceeding."

    # So, we know we have a fedbadges message and that it is for us.
    icon = fedmsg.meta.msg2icon(msg, **config)
    subtitle = fedmsg.meta.msg2subtitle(msg, **config)
    link = fedmsg.meta.msg2link(msg, **config)
예제 #40
0
파일: gilmsg.py 프로젝트: Tubbz-alt/gilmsg
def tail_messages(topic="", passive=False, **kw):
    for n, e, t, m in fedmsg.tail_messages(topic=topic, passive=passive, **kw):
        # Only acknowledge gilmsg messages to avoid catastrophic spam storm
        _acknowledge(m, **kw)
        yield n, e, t, m
예제 #41
0
파일: tweet.py 프로젝트: silverrain/fedmsg
def tweet(**kw):
    """ Rebroadcast messages to twitter and statusnet

    New values in the fedmsg configuration are needed for this to work.  Lists
    and dicts of authentication credentials such as:

        - :term:`tweet_endpoints`
        - :term:`bitly_settings`

    And scalars to help with rate limiting such as:

        - :term:`tweet_hibernate_duration`
        - :term:`tweet_intermessage_pause`

    """

    # First, sanity checking.
    if not kw.get('tweet_endpoints', None):
        raise ValueError("Not configured to tweet.")

    # Boilerplate..
    kw['publish_endpoint'] = None
    kw['name'] = 'relay_inbound'
    kw['mute'] = True

    # Set up fedmsg
    fedmsg.init(**kw)
    fedmsg.text.make_processors(**kw)

    # Set up twitter and statusnet.. multiple accounts if configured
    settings = kw.get('tweet_endpoints', [])
    apis = [twitter_api.Api(**endpoint) for endpoint in settings]

    # Set up bitly
    settings = kw['bitly_settings']
    bitly = bitlyapi.BitLy(
        settings['api_user'],
        settings['api_key'],
    )

    # How long to sleep if we spew too fast.
    hibernate_duration = kw['tweet_hibernate_duration']
    # Sleep a second or two inbetween messages to try and avoid the hibernate
    intermessage_pause = kw['tweet_intermessage_pause']

    def _post_to_api(api, message):
        try:
            api.PostUpdate(message)
        except Exception as e:
            if 'Too many notices too fast;' in str(e):
                # Cool our heels then try again.
                print "Sleeping for", hibernate_duration
                time.sleep(hibernate_duration)
                _post_to_api(api, message)
            elif 'json decoding' in str(e):
                # Let it slide ... no idea what this one is.
                pass
            elif 'duplicate' in str(e):
                # Let it slide ...
                pass
            else:
                raise

    for name, ep, topic, msg in fedmsg.tail_messages(**kw):
        message = fedmsg.text.msg2subtitle(msg, **kw)
        link = fedmsg.text.msg2link(msg, **kw)

        if link:
            link = bitly.shorten(longUrl=link)['url']
            message = (message[:138] + " ")[:139 - len(link)] + link
        else:
            message = message[:140]

        print("Tweeting %r" % message)
        for api in apis:
            _post_to_api(api, message)

        time.sleep(intermessage_pause)
예제 #42
0
 def yield_all_messages() -> Iterable[Tuple[str, dict]]:
     logger.info("listening on fedmsg")
     for name, endpoint, topic, msg in fedmsg.tail_messages():
         yield topic, msg
예제 #43
0
 def _yield_messages(topic: str) -> Iterable[Tuple[str, dict]]:
     logger.info("listening on fedmsg, topic=%s", topic)
     for name, endpoint, topic, msg in fedmsg.tail_messages(topic=topic):
         yield topic, msg
예제 #44
0
파일: tail.py 프로젝트: yutiansut/fedmsg
    def run(self):
        # Disable sending
        self.config['publish_endpoint'] = None

        # Disable timeouts.  We want to tail forever!
        self.config['timeout'] = 0

        # Even though fedmsg-tail won't be sending any messages, give it a
        # name to conform with the other commands.
        self.config['name'] = 'relay_inbound'

        # Tail is never going to send any messages, so we suppress warnings
        # about having no publishing sockets established.
        self.config['mute'] = True

        fedmsg.init(**self.config)

        # Build a message formatter
        formatter = lambda d: d
        if self.config['pretty']:

            def formatter(d):
                d['timestamp'] = time.ctime(d['timestamp'])
                d = fedmsg.crypto.strip_credentials(d)
                return "\n" + pprint.pformat(d)

        if self.config['really_pretty']:

            def formatter(d):
                d = fedmsg.crypto.strip_credentials(d)
                fancy = pygments.highlight(
                    fedmsg.encoding.pretty_dumps(d),
                    pygments.lexers.JavascriptLexer(),
                    pygments.formatters.TerminalFormatter()).strip()
                return "\n" + fancy

        if self.config['query']:

            def formatter(d):
                result = fedmsg.utils.dict_query(d, self.config['query'])
                return ", ".join(
                    [six.text_type(value) for value in result.values()])

        if self.config['terse']:
            formatter = lambda d: "\n" + fedmsg.meta.msg2repr(d, **self.config)

        if self.config['cowsay']:

            def formatter(d):
                result, error = cowsay_output(
                    fedmsg.meta.msg2subtitle(d, **self.config))
                if not error:
                    return "\n" + result
                else:
                    return "\n" + error

        # Build regular expressions for use in our loop.
        exclusive_regexp = re.compile(self.config['exclusive_regexp'])
        inclusive_regexp = re.compile(self.config['inclusive_regexp'])

        # Build username and package filter sets for use in our loop.
        users, packages = set(), set()
        if self.config['users']:
            users = set(map(str.strip, self.config['users'].split(',')))
        if self.config['packages']:
            packages = set(map(str.strip, self.config['packages'].split(',')))

        # Only initialize this if we have to
        if users or packages or self.config['terse'] or self.config['cowsay']:
            fedmsg.meta.make_processors(**self.config)

        # Spin up a zmq.Poller and yield messages
        for name, ep, topic, message in fedmsg.tail_messages(**self.config):
            if exclusive_regexp.search(topic):
                continue

            if not inclusive_regexp.search(topic):
                continue

            if users:
                actual = fedmsg.meta.msg2usernames(message, **self.config)
                if not users.intersection(actual):
                    continue

            if packages:
                actual = fedmsg.meta.msg2packages(message, **self.config)
                if not packages.intersection(actual):
                    continue

            output = formatter(message)
            if output:
                self.log.info(output)
예제 #45
0
fedmsg.meta.make_processors(**config)

sck = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sck.connect((irc, port))
sck.send('NICK ' + nick + '\r\n')
sck.send('USER ' + nick + ' ' + nick + ' ' + nick + ' :Fedbadges bot\r\n')
sck.send('JOIN ' + channel + '\r\n')

data = ''
while True:
    data = sck.recv(2040)
    if data.find('PING') != -1:
        sck.send('PONG ' + data.split()[1] + '\r\n')
        break

print "Posting up to listen on the fedmsg bus.  Waiting for a message..."
for name, endpoint, topic, msg in fedmsg.tail_messages():
    if topic_filter not in topic:
        continue

    title = fedmsg.meta.msg2title(msg, **config)
    if title is not 'fedbadges.badge.award':
        continue

    subtitle = fedmsg.meta.msg2subtitle(msg, **config)
    link = fedmsg.meta.msg2link(msg, **config)

    sck.send('PRIVMSG ' + channel + '|' + nick + ' :' + subtitle +
             ". Congratulations. Checkout the badges here:" + link + '\r\n')
예제 #46
0
 def looper(self):
     """
     Implementation of the generator to feed the event loop
     """
     for name, endpoint, topic, msg in fedmsg.tail_messages(mute=True):
         yield (topic, dict(msg))
def main():
    # grab token and connec to pagure
    token = os.getenv('PAGURE_TOKEN')
    if token:
        logger.info("Using detected token to talk to pagure.")
        pg = Pagure(pagure_token=token)
    else:
        logger.info("No pagure token was detected.")
        logger.info(
            "This script will run but won't be able to create new issues.")
        pg = Pagure()

    # Set the repo to create new issues against
    pg.repo = PAGURE_REPO

    # Used for printing out a value when the day has changed
    date = datetime.date.today()

    # Grab messages from fedmsg and process them as we go
    logger.info("Starting listening for fedmsgs..")
    for name, endpoint, topic, msg in fedmsg.tail_messages():
        logger.debug(topic)

        # Print out a log statement if the day has changed
        today = datetime.date.today()
        if today != date:
            date = today
            logger.info('mark')

        if "pungi.compose.status.change" in topic:
            print('.', end='')  # some sort of indicator of progress
            print('.')  # some sort of indicator of progress

            status = msg['msg']['status']

            # If we are in good states then continue
            if status in ['FINISHED', 'STARTED']:
                continue

            # We have a compose that either failed or had missing artifacts
            # create a new issue.
            title = msg['msg']['compose_id'] + ' ' + status
            logfileurl = msg['msg'][
                'location'] + '/../logs/global/pungi.global.log'
            logger.info("%s\t%s" % (title, logfileurl))

            # variable to hold description for issue
            content = "[pungi.global.log](%s)\n\n" % logfileurl

            # If we fail to get the log file contents then we'll just
            # best effort put a message in the issue.
            try:
                lines = requests.get(logfileurl).text.splitlines()
            except:
                logger.info(
                    "Failed to retrieve log contents from server.. skipping analysis"
                )
                content += "Failed to retrieve log contents from server.. skipping analysis"
                lines = []
                pass

            for x in range(1, len(lines)):
                line = lines[x - 1][20:]  # trim date off log lines
                nextline = lines[x][20:]  # trim date off log lines

                # If this is a [FAIL] line then we take it and the
                # next line and add them in markdown format. Also grab
                # the taskid if we can and print a hyperlink to koji
                if re.search('\[FAIL\]', line):
                    content += get_supporting_text(nextline)
                    content += "```\n%s\n%s\n```\n\n" % (line, nextline)

                # If this is the Compose run failed line, then add it
                # to the description too
                if re.search('.*Compose run failed.*', line):
                    content += ("- Compose run failed because: %s\n" %
                                get_supporting_text(line))
                    content += "```\n%s\n```\n" % (line)

            logger.debug(content)

            # pull only part of the compose ID for the tag to set
            tag = re.search('(.*)-\d{8}', msg['msg']['compose_id']).group(1)
            #TODO figure out how to set tag on an issue

            if token:
                pg.create_issue(title=title, content=content)
예제 #48
0
def main():
    parser = argparse.ArgumentParser(description = __doc__)
    parser.add_argument('config', help = 'path of configuration file')
    parser.add_argument('-v', '--verbose', action = 'store_true', help = 'increase output verbosity')

    global args
    args = parser.parse_args()
    logging.basicConfig(level = logging.DEBUG if args.verbose else logging.WARNING, stream = sys.stdout)

    config_parser = ConfigParser.SafeConfigParser(dict(
        here = os.path.dirname(os.path.abspath(os.path.normpath(args.config))),
        ))
    config_parser.read(args.config)
    global conf
    conf = conv.check(conv.pipe(
        conv.test_isinstance(dict),
        conv.struct(
            {
                'ckan_of_worms.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'dactylo.api_key': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                'dactylo.site_url': conv.pipe(
                    conv.make_input_to_url(error_if_fragment = True, error_if_path = True, error_if_query = True,
                        full = True),
                    conv.not_none,
                    ),
                'user_agent': conv.pipe(
                    conv.cleanup_line,
                    conv.not_none,
                    ),
                },
            default = 'drop',
            ),
        conv.not_none,
        ))(dict(config_parser.items('CowBots-Report-State')), conv.default_state)

    global headers
    headers = {
        'User-Agent': conf['user_agent'],
        }
    global request_headers
    request_headers = headers.copy()
    request_headers['Content-Type'] = 'application/json'

    request = urllib2.Request(urlparse.urljoin(conf['ckan_of_worms.site_url'], 'api/1/metrics'),
        headers = headers)
    response = urllib2.urlopen(request)
    global metrics
    metrics = conv.check(conv.pipe(
        cow_response_to_value,
        conv.test_isinstance(dict),
        conv.not_none,
        ))(response.read(), state = conv.default_state)

    send_stats()

    fedmsg_conf = conv.check(conv.struct(
        dict(
            environment = conv.pipe(
                conv.empty_to_none,
                conv.test_in(['dev', 'prod', 'stg']),
                ),
            modname = conv.pipe(
                conv.empty_to_none,
                conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                conv.default('ckan_of_worms'),
                ),
#            name = conv.pipe(
#                conv.empty_to_none,
#                conv.default('ckan_of_worms.{}'.format(hostname)),
#                ),
            topic_prefix = conv.pipe(
                conv.empty_to_none,
                conv.test(lambda value: value == value.strip('.'), error = 'Value must not begin or end with a "."'),
                ),
            ),
        default = 'drop',
        ))(dict(config_parser.items('fedmsg')))

    # Read in the config from /etc/fedmsg.d/.
    fedmsg_config = fedmsg.config.load_config([], None)
    # Disable a warning about not sending.  We know.  We only want to tail.
    fedmsg_config['mute'] = True
    # Disable timing out so that we can tail forever.  This is deprecated
    # and will disappear in future versions.
    fedmsg_config['timeout'] = 0
    # For the time being, don't require message to be signed.
    fedmsg_config['validate_signatures'] = False
    for key, value in fedmsg_conf.iteritems():
        if value is not None:
            fedmsg_config[key] = value

    expected_topic_prefix = '{}.{}.ckan_of_worms.'.format(fedmsg_config['topic_prefix'], fedmsg_config['environment'])
    for name, endpoint, topic, message in fedmsg.tail_messages(**fedmsg_config):
        if not topic.startswith(expected_topic_prefix):
            log.debug(u'Ignoring message: {}, {}'.format(topic, name))
            continue
        kind, action = topic[len(expected_topic_prefix):].split('.')
        if kind == 'account':
            account = message['msg']
            name = account['name']
            if action == 'create':
                metrics['accounts_count'] += 1
                log.info(u'Updating stats after "{} account {}"'.format(action, name))
                send_stats()
            elif action == 'delete':
                metrics['accounts_count'] -= 1
                log.info(u'Updating stats after "{} account {}"'.format(action, name))
                send_stats()
            elif action != 'update':
                log.warning(u'Unknown action "{}" for account "{}"'.format(action, account))
        elif kind == 'dataset':
            dataset = message['msg']
            name = dataset['name']
            existing_dataset_metrics = metrics['datasets'].get(name, {})
            if action in ('create', 'update'):
                dataset_metrics = dict(
                    organization_name = (dataset.get('organization') or {}).get('name'),
                    related_count = len(dataset.get('related') or []),
                    resources = [
                        dict(
                            format = resource.get('format'),
                            )
                        for resource in (dataset.get('resources') or [])
                        ],
                    weight = dataset['weight'],
                    )
            elif action == 'delete':
                dataset_metrics = None
            else:
                log.warning(u'Unknown action "{}" for dataset "{}"'.format(action, dataset))
                continue
            if dataset_metrics != existing_dataset_metrics:
                log.info(u'Updating stats after "{} dataset {}"'.format(action, name))
                if dataset_metrics is None:
                    del metrics['datasets'][name]
                else:
                    metrics['datasets'][name] = dataset_metrics
                send_stats()
        elif kind == 'organization':
            organization = message['msg']
            name = organization['name']
            existing_organization_metrics = metrics['organizations'].get(name, {})
            if action in ('create', 'update'):
                organization_metrics = dict(
                    public_service = bool(organization.get('public_service')),
                    )
            elif action == 'delete':
                organization_metrics = None
            else:
                log.warning(u'Unknown action "{}" for organization "{}"'.format(action, organization))
                continue
            if organization_metrics != existing_organization_metrics:
                log.info(u'Updating stats after "{} organization {}"'.format(action, name))
                if organization_metrics is None:
                    del metrics['organizations'][name]
                else:
                    metrics['organizations'][name] = organization_metrics
                send_stats()
        else:
            log.debug(u'TODO: Handle {}, {} for {}'.format(kind, action, message))

    return 0
예제 #49
0
파일: trigger.py 프로젝트: AdamWill/fedmsg
    def run(self):

        # This is a "required" option... :P
        if not self.config['command']:
            self.log.error("You must provide a --command to run.")
            sys.exit(1)

        # Disable sending
        self.config['publish_endpoint'] = None

        # Disable timeouts.  We want to tail forever!
        self.config['timeout'] = 0

        # Even though fedmsg-trigger won't be sending any messages, give it a
        # name to conform with the other commands.
        self.config['name'] = 'relay_inbound'

        # Tail is never going to send any messages, so we suppress warnings
        # about having no publishing sockets established.
        self.config['mute'] = True

        fedmsg.init(**self.config)

        exclusive_regexp = re.compile(self.config['exclusive_regexp'])
        inclusive_regexp = re.compile(self.config['inclusive_regexp'])

        wait_for = int(self.config['wait_for'])
        max_queue_size = int(self.config['max_queue_size'])

        timer = None
        que = queue.Queue()

        def execute_queue():
            while not que.empty():
                message = que.get()

                result = self.run_command(self.config['command'], message)

                if result != 0:
                    self.log.info("Command returned error code %r" % result)

        try:
            for name, ep, topic, message in fedmsg.tail_messages(**self.config):
                if exclusive_regexp.search(topic):
                    continue

                if not inclusive_regexp.search(topic):
                    continue

                que.put(message)

                if timer is not None:
                    # Try to cancel it
                    timer.cancel()
                if timer is None or not timer.is_alive():
                    # Either there was no timer yet, or it was still waiting
                    # -> Let's start a new one
                    if (max_queue_size > 0 and
                            que.qsize() > max_queue_size):
                        # If the que is too big, let's just run it NOW
                        timer = threading.Timer(0, execute_queue)
                    else:
                        timer = threading.Timer(wait_for,
                                                execute_queue)
                    timer.start()
        except KeyboardInterrupt:
            if timer is not None:
                timer.cancel()
                # Let's wait for commands to finish
                timer.join()
예제 #50
0
파일: tail.py 프로젝트: pcdinh/fedmsg
    def run(self):
        # Disable sending
        self.config["publish_endpoint"] = None

        # Disable timeouts.  We want to tail forever!
        self.config["timeout"] = 0

        # Even though fedmsg-tail won't be sending any messages, give it a
        # name to conform with the other commands.
        self.config["name"] = "relay_inbound"

        # Tail is never going to send any messages, so we suppress warnings
        # about having no publishing sockets established.
        self.config["mute"] = True

        fedmsg.init(**self.config)
        fedmsg.meta.make_processors(**self.config)

        # Build a message formatter
        formatter = lambda d: d
        if self.config["pretty"]:

            def formatter(d):
                d["timestamp"] = time.ctime(d["timestamp"])
                d = fedmsg.crypto.strip_credentials(d)
                return "\n" + pprint.pformat(d)

        if self.config["really_pretty"]:

            def formatter(d):
                d = fedmsg.crypto.strip_credentials(d)
                fancy = pygments.highlight(
                    fedmsg.encoding.pretty_dumps(d),
                    pygments.lexers.JavascriptLexer(),
                    pygments.formatters.TerminalFormatter(),
                ).strip()
                return "\n" + fancy

        if self.config["terse"]:
            formatter = lambda d: "\n" + fedmsg.meta.msg2repr(d, **self.config)

        if self.config["gource"]:
            # Output strings suitable for consumption by the "gource" tool.

            # We have 8 colors here and an unknown number of message types.
            # (There were 14 message types at the time this code was written).
            # Here we build a dict that maps message type names (a.k.a modnames
            # or services) to hex colors for usage in the gource graph.  We
            # wrap-around that dict if there are more message types than
            # there are colors (which there almost certainly are).
            procs = [proc.__name__.lower() for proc in fedmsg.meta.processors]
            colors = ["FFFFFF", "008F37", "FF680A", "CC4E00", "8F0058", "8F7E00", "37008F", "7E008F"]
            n_wraps = 1 + int(math.ceil(len(colors) / float(len(procs))))
            colors = colors * n_wraps
            color_lookup = dict(zip(procs, colors))

            cache_directory = self.config["gource_user_image_dir"]

            # After all that color trickiness, here is our formatter we'll use.
            def formatter(message):
                """ Use this like::

                  $ fedmsg-tail --gource | gource \
                          -i 0 \
                          --user-image-dir ~/.cache/gravatar/ \
                          --log-format custom -
                """
                proc = fedmsg.meta.msg2processor(message, **self.config)
                users = fedmsg.meta.msg2usernames(message, **self.config)
                objs = fedmsg.meta.msg2objects(message, **self.config)
                name = proc.__name__.lower()

                if not users:
                    users = [name]

                lines = []
                for user, obj in itertools.product(users, objs):
                    _grab_and_cache_avatar(user, cache_directory)
                    lines.append("%i|%s|A|%s|%s" % (message["timestamp"], user, name + "/" + obj, color_lookup[name]))
                return "\n".join(lines)

        exclusive_regexp = re.compile(self.config["exclusive_regexp"])
        inclusive_regexp = re.compile(self.config["inclusive_regexp"])

        # The "proper" fedmsg way to do this would be to spin up or connect to
        # an existing Moksha Hub and register a consumer on the "*" topic that
        # simply prints out each message it consumes.  That seems like
        # overkill, so we're just going to directly access the endpoints
        # ourself.
        for name, ep, topic, message in fedmsg.tail_messages(**self.config):
            if exclusive_regexp.search(topic):
                continue

            if not inclusive_regexp.search(topic):
                continue

            self.log.info(formatter(message))