Exemple #1
0
def save(data, stats):
    if not data: return stats
    #print jdump(data)
    res = db.eurlex.find_one({'id.celexid': data['id']['celexid']}) or {}
    d = diff(
        dict([(k, v) for k, v in res.items()
              if not k in ['_id', 'meta', 'changes']]),
        dict([(k, v) for k, v in data.items() if not k in [
            '_id',
            'meta',
            'changes',
        ]]))
    if d:
        now = unicode(datetime.utcnow().replace(microsecond=0).isoformat())
        if not res:
            logger.info(('adding %s' % (data['id']['celexid'])).encode('utf8'))
            data['meta']['created'] = now
            if stats: stats[0] += 1
        else:
            logger.info(
                ('updating %s' % (data['id']['celexid'])).encode('utf8'))
            #logger.warn(d)
            data['meta']['updated'] = now
            if stats: stats[1] += 1
            data['_id'] = res['_id']
        data['changes'] = res.get('changes', {})
        data['changes'][now] = d
        db.eurlex.save(data)
    if stats: return stats
    else: return data
Exemple #2
0
def save(data, stats):
    res=db.ep_meps2.find_one({ 'UserID' : data['UserID'] }) or {}
    if 'Gender' not in data and 'Gender' in res: data['Gender']=res['Gender']
    d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes', 'activities',]]),
           dict([(k,v) for k,v in data.items() if not k in ['_id', 'meta', 'changes', 'activities',]]))
    if d:
        now=datetime.utcnow().replace(microsecond=0)
        if not res:
            logger.info(('adding %s' % (data['Name']['full'])).encode('utf8'))
            data['meta']['created']=now
            if stats: stats[0]+=1
        else:
            logger.info(('updating %s' % (data['Name']['full'])).encode('utf8'))
            logger.warn(jdump(d))
            data['meta']['updated']=now
            if stats: stats[1]+=1
            data['_id']=res['_id']
        data['changes']=res.get('changes',{})
        data['changes'][now.isoformat()]=d
        db.ep_meps2.save(data)
    del res
    if stats: 
        del data
        return stats
    else: return data
Exemple #3
0
def save(data, stats):
    res = db.ep_meps2.find_one({"UserID": data["UserID"]}) or {}
    d = diff(
        dict([(k, v) for k, v in res.items() if not k in ["_id", "meta", "changes"]]),
        dict([(k, v) for k, v in data.items() if not k in ["_id", "meta", "changes"]]),
    )
    if d:
        now = datetime.utcnow().replace(microsecond=0)
        if not res:
            logger.info(("adding %s" % (data["Name"]["full"])).encode("utf8"))
            data["meta"]["created"] = now
            if stats:
                stats[0] += 1
        else:
            logger.info(("updating %s" % (data["Name"]["full"])).encode("utf8"))
            logger.warn(jdump(d))
            data["meta"]["updated"] = now
            if stats:
                stats[1] += 1
            data["_id"] = res["_id"]
        data["changes"] = res.get("changes", {})
        data["changes"][now.isoformat()] = d
        db.ep_meps2.save(data)
    if stats:
        return stats
    else:
        return data
Exemple #4
0
def save(data, stats):
    for item in data:
        if not 'committee' in item: continue
        query={'committee': item['committee'],
               'src': item['src'],
               'title': item['title']}
        if 'date' in data:
            query['date']= item['date']
            if 'end' in data:
                query['end']= item['end']
        else:
            query['seq_no']=item['seq_no']
        res=db.ep_comagendas.find_one(query) or {}
        d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes']]),
               dict([(k,v) for k,v in item.items() if not k in ['_id', 'meta', 'changes',]]))
        if d:
            now=datetime.utcnow().replace(microsecond=0)
            if not 'meta' in item: item[u'meta']={}
            if not res:
                logger.info((u'adding %s %s' % (item['committee'], item['title'])).encode('utf8'))
                item['meta']['created']=now
                if stats: stats[0]+=1
            else:
                logger.info((u'updating %s %s' % (item['committee'], item['title'])).encode('utf8'))
                logger.info(d)
                item['meta']['updated']=now
                if stats: stats[1]+=1
                item['_id']=res['_id']
            item['changes']=res.get('changes',{})
            item['changes'][now.isoformat()]=d
            db.ep_comagendas.save(item)
    if stats: return stats
    else: return data
Exemple #5
0
def save(data, stats):
    if not data: return stats
    src = data['meta']['source']
    res = db.dossiers2.find_one({'meta.source': src}) or {}
    d = diff(
        dict([(k, v) for k, v in res.items()
              if not k in ['_id', 'meta', 'changes']]),
        dict([(k, v) for k, v in data.items() if not k in [
            '_id',
            'meta',
            'changes',
        ]]))
    #logger.warn(pprint.pformat(d))
    if d:
        now = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
        if not res:
            logger.info(('adding %s - %s' %
                         (data['procedure']['reference'],
                          data['procedure']['title'])).encode('utf8'))
            data['meta']['created'] = data['meta']['timestamp']
            del data['meta']['timestamp']
            sys.stdout.flush()
            stats[0] += 1
        else:
            logger.info(('updating  %s - %s' %
                         (data['procedure']['reference'],
                          data['procedure']['title'])).encode('utf8'))
            data['meta']['updated'] = data['meta']['timestamp']
            del data['meta']['timestamp']
            sys.stdout.flush()
            stats[1] += 1
            data['_id'] = res['_id']
            logger.info(jdump(d))
        if not NOMAIL:
            m = db.notifications.find(
                {'dossiers': data['procedure']['reference']},
                ['active_emails'])
            for g in m:
                if len(g['active_emails']) == 0:
                    continue
                msg = Message("[PT] %s %s" % (data['procedure']['reference'],
                                              data['procedure']['title']),
                              sender="*****@*****.**",
                              bcc=g['active_emails'])
                #msg.html = htmldiff(data,d)
                msg.body = makemsg(data, d)
                mail.send(msg)
        #logger.info(htmldiff(data,d))
        #logger.info(makemsg(data,d))
        data['changes'] = res.get('changes', {})
        data['changes'][now] = d
        db.dossiers2.save(data)
    return stats
Exemple #6
0
def save(data, stats):
    for item in data:
        if not 'committee' in item: continue
        query = {
            'committee': item['committee'],
            'src': item['src'],
            'title': item['title']
        }
        if 'date' in data:
            query['date'] = item['date']
            if 'end' in data:
                query['end'] = item['end']
        else:
            query['seq_no'] = item['seq_no']
        res = db.ep_comagendas.find_one(query) or {}
        d = diff(
            dict([(k, v) for k, v in res.items()
                  if not k in ['_id', 'meta', 'changes']]),
            dict([(k, v) for k, v in item.items() if not k in [
                '_id',
                'meta',
                'changes',
            ]]))
        if d:
            now = datetime.utcnow().replace(microsecond=0)
            if not 'meta' in item: item[u'meta'] = {}
            if not res:
                logger.info(
                    (u'adding %s%s %s' %
                     (u'%s ' % item['epdoc'] if 'epdoc' in item else '',
                      item['committee'], item['title'])).encode('utf8'))
                item['meta']['created'] = now
                if stats: stats[0] += 1
                notify(item, None)
            else:
                logger.info(
                    (u'updating %s%s %s' %
                     (u'%s ' % item['epdoc'] if 'epdoc' in item else '',
                      item['committee'], item['title'])).encode('utf8'))
                logger.info(d)
                item['meta']['updated'] = now
                if stats: stats[1] += 1
                item['_id'] = res['_id']
                notify(item, d)
            item['changes'] = res.get('changes', {})
            item['changes'][now.isoformat()] = d
            db.ep_comagendas.save(item)
    if stats: return stats
    else: return data
Exemple #7
0
def save(data, stats):
    src = data["meta"]["source"]
    res = db.dossiers2.find_one({"meta.source": src}) or {}
    d = diff(
        dict([(k, v) for k, v in res.items() if not k in ["_id", "meta", "changes"]]),
        dict([(k, v) for k, v in data.items() if not k in ["_id", "meta", "changes"]]),
    )
    # logger.warn(d)
    if d:
        now = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
        if not res:
            logger.info(
                ("adding %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8")
            )
            data["meta"]["created"] = data["meta"]["timestamp"]
            del data["meta"]["timestamp"]
            sys.stdout.flush()
            stats[0] += 1
        else:
            logger.info(
                ("updating  %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8")
            )
            data["meta"]["updated"] = data["meta"]["timestamp"]
            del data["meta"]["timestamp"]
            sys.stdout.flush()
            stats[1] += 1
            data["_id"] = res["_id"]
            # print >> sys.stderr, (d)
        m = db.notifications.find({"dossiers": data["procedure"]["reference"]}, ["active_emails"])
        for g in m:
            if len(g["active_emails"]) == 0:
                continue
            msg = Message(
                "[PT] %s %s" % (data["procedure"]["reference"], data["procedure"]["title"]),
                sender="*****@*****.**",
                bcc=g["active_emails"],
            )
            msg.html = htmldiff(data, d)
            msg.body = makemsg(data, d)
            mail.send(msg)
        data["changes"] = res.get("changes", {})
        data["changes"][now] = d
        db.dossiers2.save(data)
    return stats
Exemple #8
0
def save(data, stats):
    res=db.ep_meps2.find_one({ 'UserID' : data['UserID'] }) or {}
    d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes']]),
           dict([(k,v) for k,v in data.items() if not k in ['_id', 'meta', 'changes',]]))
    if d:
        now=unicode(datetime.utcnow().replace(microsecond=0).isoformat())
        if not res:
            logger.info(('adding %s' % (data['Name']['full'])).encode('utf8'))
            data['meta']['created']=now
            stats[0]+=1
        else:
            logger.info(('updating %s' % (data['Name']['full'])).encode('utf8'))
            logger.warn(d)
            data['meta']['updated']=now
            stats[1]+=1
            data['_id']=res['_id']
        data['changes']=res.get('changes',{})
        data['changes'][now]=d
        db.ep_meps2.save(data)
    return stats
Exemple #9
0
def save(data, stats):
    if not data: return stats
    src=data['meta']['source']
    res=db.dossiers2.find_one({ 'meta.source' : src }) or {}
    d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes']]),
           dict([(k,v) for k,v in data.items() if not k in ['_id', 'meta', 'changes',]]))
    #logger.warn(pprint.pformat(d))
    if d:
        now=datetime.datetime.utcnow().replace(microsecond=0).isoformat()
        if not res:
            logger.info(('adding %s - %s' % (data['procedure']['reference'],data['procedure']['title'])).encode('utf8'))
            data['meta']['created']=data['meta']['timestamp']
            del data['meta']['timestamp']
            sys.stdout.flush()
            stats[0]+=1
        else:
            logger.info(('updating  %s - %s' % (data['procedure']['reference'],data['procedure']['title'])).encode('utf8'))
            data['meta']['updated']=data['meta']['timestamp']
            del data['meta']['timestamp']
            sys.stdout.flush()
            stats[1]+=1
            data['_id']=res['_id']
            logger.info(jdump(d))
        if not NOMAIL:
            m=db.notifications.find({'dossiers': data['procedure']['reference']},['active_emails'])
            for g in m:
                if len(g['active_emails'])==0:
                    continue
                msg = Message("[PT] %s %s" % (data['procedure']['reference'],data['procedure']['title']),
                              sender = "*****@*****.**",
                              bcc = g['active_emails'])
                #msg.html = htmldiff(data,d)
                msg.body = makemsg(data,d)
                mail.send(msg)
        #logger.info(htmldiff(data,d))
        #logger.info(makemsg(data,d))
        data['changes']=res.get('changes',{})
        data['changes'][now]=d
        db.dossiers2.save(data)
    return stats
Exemple #10
0
def save(data, stats):
    if not data: return stats
    res=db.eurlex.find_one({ 'id.celexid' : data['id']['celexid'] }) or {}
    d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes']]),
           dict([(k,v) for k,v in data.items() if not k in ['_id', 'meta', 'changes',]]))
    if d:
        now=unicode(datetime.utcnow().replace(microsecond=0).isoformat())
        if not res:
            logger.info(('adding %s' % (data['id']['celexid'])).encode('utf8'))
            data['meta']['created']=now
            if stats: stats[0]+=1
        else:
            logger.info(('updating %s' % (data['id']['celexid'])).encode('utf8'))
            logger.warn(d)
            data['meta']['updated']=now
            if stats: stats[1]+=1
            data['_id']=res['_id']
        data['changes']=res.get('changes',{})
        data['changes'][now]=d
        db.eurlex.save(data)
    if stats: return stats
    else: return data