示例#1
0
    def run(self):
        m.init(self.config['datanommer.sqlalchemy.url'])
        config = self.config

        if config.get('topic', None):
            if config.get('category', None):
                query = m.session.query(
                    m.Message.topic, func.count(m.Message.topic)).filter(
                        m.Message.category == config.get('category'))
            else:
                query = m.session.query(m.Message.topic,
                                        func.count(m.Message.topic))
            query = query.group_by(m.Message.topic)
        else:
            if config.get('category', None):
                query = m.session.query(
                    m.Message.category, func.count(m.Message.category)).filter(
                        m.Message.category == config.get('category'))
            else:
                query = m.session.query(m.Message.category,
                                        func.count(m.Message.category))
            query = query.group_by(m.Message.category)

        results = query.all()

        if config.get('topic', None):
            for topic, count in results:
                self.log.info("%s has %s entries" % (topic, count))
        else:
            for category, count in results:
                self.log.info("%s has %s entries" % (category, count))
 def setUp(self):
     uri = "sqlite:///%s" % filename
     self.config = {
         'datanommer.sqlalchemy.url': uri,
         'logging': {
             'version': 1
         }
     }
     self.config.update(fedmsg.config.load_config())
     m.session = scoped_session(m.maker)
     m.init(uri=uri, create=True)
示例#3
0
    def run(self):
        m.init(self.config['datanommer.sqlalchemy.url'])
        config = self.config

        if config.get('topic', None):
            queries = [
                m.Message.query.filter(m.Message.topic == config.get('topic'))
            ]

        elif config.get('category', None):
            queries = [
                m.Message.query.filter(
                    m.Message.category == config.get('category'))
            ]
        elif not config.get('overall', False):
            # If no args..
            fedmsg.meta.make_processors(**config)
            categories = [p.__name__.lower() for p in fedmsg.meta.processors]
            queries = [
                m.Message.query.filter(m.Message.category == category)
                for category in categories
            ]
        else:
            # Show only the single latest message, regardless of type.
            queries = [m.Message.query]

        # Order and limit to the latest.
        queries = [
            q.order_by(m.Message.timestamp.desc()).limit(1) for q in queries
        ]

        def formatter(key, val):
            if config.get('timestamp', None) and config.get('human', None):
                return pretty_dumps(str(val.timestamp))
            elif config.get('timestamp', None):
                return pretty_dumps(time.mktime(val.timestamp.timetuple()))
            elif config.get('timesince', None) and config.get('human', None):
                return pretty_dumps(
                    str(datetime.datetime.now() - val.timestamp))
            elif config.get('timesince', None):
                timedelta = datetime.datetime.now() - val.timestamp
                return pretty_dumps(
                    str((timedelta.days * 86400) + timedelta.seconds))
            else:
                return "{%s: %s}" % (pretty_dumps(key), pretty_dumps(val))

        results = []
        for result in sum([query.all() for query in queries], []):
            results.append(formatter(result.category, result))

        self.log.info('[%s]' % ','.join(results))
示例#4
0
    def run(self):
        m.init(self.config['datanommer.sqlalchemy.url'])
        config = self.config

        query = m.Message.query
        if config.get('before', None):
            query = query.filter(m.Message.timestamp<=config.get('before'))

        if config.get('since', None):
            query = query.filter(m.Message.timestamp>=config.get('since'))

        results = query.all()

        self.log.info(pretty_dumps(results))
示例#5
0
    def run(self):
        m.init(self.config['datanommer.sqlalchemy.url'])
        config = self.config

        if config.get('topic', None):
            queries = [
                m.Message.query.filter(m.Message.topic == config.get('topic'))
            ]

        elif config.get('category', None):
            queries = [m.Message.query.filter(
                m.Message.category == config.get('category')
            )]
        elif not config.get('overall', False):
            # If no args..
            fedmsg.meta.make_processors(**config)
            categories = [
                p.__name__.lower() for p in fedmsg.meta.processors
            ]
            queries = [m.Message.query.filter(
                m.Message.category == category
            ) for category in categories]
        else:
            # Show only the single latest message, regardless of type.
            queries = [m.Message.query]

        # Order and limit to the latest.
        queries = [
            q.order_by(m.Message.timestamp.desc()).limit(1)
            for q in queries
        ]

        def formatter(key, val):
            if config.get('timestamp', None) and config.get('human', None):
                return pretty_dumps(str(val.timestamp))
            elif config.get('timestamp', None):
                return pretty_dumps(time.mktime(val.timestamp.timetuple()))
            elif config.get('timesince', None) and config.get('human', None):
                return pretty_dumps(str(datetime.datetime.now()-val.timestamp))
            elif config.get('timesince', None):
                timedelta = datetime.datetime.now() - val.timestamp
                return pretty_dumps(str((timedelta.days * 86400)+timedelta.seconds))
            else:
                return "{%s: %s}" % (pretty_dumps(key), pretty_dumps(val))

        results = []
        for result in sum([query.all() for query in queries], []):
            results.append(formatter(result.category, result))

        self.log.info('[%s]' % ','.join(results))
示例#6
0
    def run(self):
        m.init(self.config['datanommer.sqlalchemy.url'])
        config = self.config

        query = m.Message.query
        if config.get('before', None):
            query = query.filter(m.Message.timestamp <= config.get('before'))

        if config.get('since', None):
            query = query.filter(m.Message.timestamp >= config.get('since'))

        results = query.all()

        self.log.info(pretty_dumps(results))
示例#7
0
def upgrade():
    engine = op.get_bind().engine
    m.init(engine=engine)

    # git.branch
    branch = m.Message.topic.like(u'%.git.branch.%')
    for msg in m.Message.query.filter(branch).yield_per(100):
        prefix, suffix = msg.topic.split(u'.git.branch.')
        # Fix topic
        msg.topic = prefix + '.git.branch'
        # Fix message contents
        message = msg.msg
        message['name'] = '.'.join(suffix.split('.')[0:-1])
        message['branch'] = suffix.split('.')[-1]
        msg.msg = message
        # Drop cert and sig
        msg.certificate = None
        msg.signature = None
        m.session.add(msg)

    # git.lookaside.*.new
    lookaside = m.Message.topic.like(u'%.git.lookaside.%.new')
    for msg in m.Message.query.filter(lookaside).yield_per(100):
        prefix, suffix = msg.topic.split(u'.git.lookaside.')
        # Fix topic
        msg.topic = prefix + '.git.lookaside.new'
        # Drop cert and sig
        msg.certificate = None
        msg.signature = None
        m.session.add(msg)

    # git.receive
    receive = m.Message.topic.like(u'%.git.receive.%')
    for msg in m.Message.query.filter(receive).yield_per(100):
        prefix, suffix = msg.topic.split(u'.git.receive.')
        # Fix topic
        msg.topic = prefix + '.git.receive'
        # Fix message contents
        message = msg.msg
        message['commit']['repo'] = '.'.join(suffix.split('.')[0:-1])
        msg.msg = message
        # Drop cert and sig
        msg.certificate = None
        msg.signature = None
        m.session.add(msg)

    m.session.commit()
def upgrade():
    """ This takes a *really* long time.  Like, hours. """

    config_paths = context.config.get_main_option('fedmsg_config_dir')
    filenames = fedmsg.config._gather_configs_in(config_paths)

    config = fedmsg.config.load_config(filenames=filenames)

    make_processors(**config)

    engine = op.get_bind().engine
    m.init(engine=engine)
    for msg in _page(m.Message.query.order_by(m.Message.timestamp)):
        print "processing", msg.timestamp, msg.topic

        if msg.users and msg.packages:
            continue

        changed = False

        if not msg.users:
            new_usernames = msg2usernames(msg.__json__(), **config)
            print "Updating users to %r" % new_usernames
            changed = changed or new_usernames
            for new_username in new_usernames:
                new_user = m.User.get_or_create(new_username)
                msg.users.append(new_user)

        if not msg.packages:
            new_packagenames = msg2packages(msg.__json__(), **config)
            print "Updating packages to %r" % new_packagenames
            changed = changed or new_usernames
            for new_packagename in new_packagenames:
                new_package = m.Package.get_or_create(new_packagename)
                msg.packages.append(new_package)

        if changed and random.random() < 0.01:
            # Only save if something changed.. and only do it every so often.
            # We do this so that if we crash, we can kind of pick up where
            # we left off.  But if we do it on every change: too slow.
            print " * Saving!"
            m.session.commit()

    m.session.commit()
示例#9
0
    def run(self):
        m.init(self.config['datanommer.sqlalchemy.url'])
        config = self.config

        if config.get('topic', None):
            if config.get('category',None):
                query = m.session.query(
                    m.Message.topic, func.count(m.Message.topic)
                ).filter(
                    m.Message.category==config.get('category')
                )
            else:
                query = m.session.query(
                    m.Message.topic, func.count(m.Message.topic)
                )
            query = query.group_by(m.Message.topic)
        else:
            if config.get('category',None):
                query = m.session.query(
                    m.Message.category, func.count(m.Message.category)
                ).filter(
                    m.Message.category==config.get('category')
                )
            else:
                query = m.session.query(
                    m.Message.category, func.count(m.Message.category)
                )
            query = query.group_by(m.Message.category)


        results = query.all()

        if config.get('topic', None):
            for topic, count in results:
                self.log.info("%s has %s entries" % (topic, count))
        else:
            for category, count in results:
                self.log.info("%s has %s entries" % (category, count))
示例#10
0
 def run(self):
     m.init(
         self.config['datanommer.sqlalchemy.url'], create=True
     )
示例#11
0
 def __init__(self):
     # m.init(uri= 'postgresql://*****:*****@localhost/datanommer')
     m.init('postgresql+psycopg2://datanommer:datanommer@localhost:5432/datanommer')
     self.graph = Graph()
示例#12
0
import flask
import hashlib
import os

import shelve
import charts

import sqlalchemy
import datanommer.models as m
m.init('postgres://*****:*****@localhost/datanommer')

here = os.path.abspath(os.path.dirname(__file__))
datadir = os.path.join(here, '..', 'data')
gravatar_dir = os.path.join(datadir, 'gravatars')
montage_dir = os.path.join(datadir, 'montage')

app = flask.Flask(__name__)


def make_gravatar(username):
    email = username + "@fedoraproject.org"
    digest = hashlib.md5(email).hexdigest()
    gravatar = "http://www.gravatar.com/avatar/%s" % digest
    gravatar += "?s=128"
    return gravatar


@app.route('/')
def index():
    rows = 5
    columns = 12
示例#13
0
)
from pkg_resources import get_distribution


app = flask.Flask(__name__)
app.config.from_object('datagrepper.default_config')
app.config.from_envvar('DATAGREPPER_CONFIG')
app.config['CORS_DOMAINS'] = list(map(re.compile, app.config.get('CORS_DOMAINS', [])))
app.config['CORS_HEADERS'] = list(map(re.compile, app.config.get('CORS_HEADERS', [])))

# Read in the datanommer DB URL from /etc/fedmsg.d/ (or a local fedmsg.d/)
fedmsg_config = fedmsg.config.load_config()
fedmsg.meta.make_processors(**fedmsg_config)

# Initialize a datanommer session.
dm.init(fedmsg_config['datanommer.sqlalchemy.url'])

import datagrepper.widgets


@app.context_processor
def inject_variable():
    """ Inject some global variables into all templates
    """
    extras = {
        'models_version': get_distribution('datanommer.models').version,
        'grepper_version': get_distribution('datagrepper').version,
    }

    style = {
        'message_bus_link': 'http://fedmsg.com',
示例#14
0
def init():
    # Load stuff from /etc/fedmsg.d/ into a dict
    config = fedmsg.config.load_config(None, [])

    m.init(config['datanommer.sqlalchemy.url'])
示例#15
0
 def run(self):
     m.init(self.config['datanommer.sqlalchemy.url'], create=True)
def init():
    # Load stuff from /etc/fedmsg.d/ into a dict
    config = fedmsg.config.load_config(None, [])

    m.init(config['datanommer.sqlalchemy.url'])
示例#17
0
from pkg_resources import get_distribution

app = flask.Flask(__name__)
app.config.from_object('datagrepper.default_config')
app.config.from_envvar('DATAGREPPER_CONFIG')
app.config['CORS_DOMAINS'] = list(
    map(re.compile, app.config.get('CORS_DOMAINS', [])))
app.config['CORS_HEADERS'] = list(
    map(re.compile, app.config.get('CORS_HEADERS', [])))

# Read in the datanommer DB URL from /etc/fedmsg.d/ (or a local fedmsg.d/)
fedmsg_config = fedmsg.config.load_config()
fedmsg.meta.make_processors(**fedmsg_config)

# Initialize a datanommer session.
dm.init(fedmsg_config['datanommer.sqlalchemy.url'])

import datagrepper.widgets


@app.context_processor
def inject_variable():
    """ Inject some global variables into all templates
    """
    extras = {
        'models_version': get_distribution('datanommer.models').version,
        'grepper_version': get_distribution('datagrepper').version,
    }

    style = {
        'message_bus_link': 'http://fedmsg.com',
示例#18
0
文件: dots.py 项目: ralphbean/oculum
import calendar
from datetime import (
    datetime,
    timedelta,
)

import pygal
import pygal.style

from sqlalchemy.orm.exc import NoResultFound
import datanommer.models as m
m.init('postgres://*****:*****@localhost/datanommer')

title_template = 'Fedora Development Activity for %[email protected]'


def message_count(user, start):
    end = start + timedelta(days=1)

    if not user:
        return 0

    return len([
        msg for msg in user.messages
        if msg.timestamp >= start and msg.timestamp < end
    ])


def make_chart(username, style='default', **args):
    if isinstance(style, list):
        style = style[0]