def notify(data,d): if not 'epdoc' in data: return m=db.notifications.find({'dossiers': data['epdoc']},['active_emails']) for g in m: if len(g['active_emails'])==0: continue msg = Message("[PT-Com] %s: %s" % (data['committee'], data['title']), sender = "*****@*****.**", bcc = g['active_emails']) msg.body = (u"Parltrack has detected %s%s on the schedule of %s \n" u"\n on %s" u"\n%s" u"%s" u"\nsee the details here: %s\n" u"\nYour Parltrack team" % (u"a change on " if d else u'', data['epdoc'], data['committee'], data['date'] if 'date' in data else 'unknown date', ("\n - %s" % u'\n - '.join(data['list'])) if 'list' in data and len(data['list'])>0 else u"", "\n %s" % (textdiff(d) if d else ''), "%s/dossier/%s" % (ROOT_URL, data['epdoc']), )) mail.send(msg)
def save(data, stats): if not data: return stats src = data['meta']['source'] res = db.dossiers2.find_one({'meta.source': src}) or {} d = diff( dict([(k, v) for k, v in res.items() if not k in ['_id', 'meta', 'changes']]), dict([(k, v) for k, v in data.items() if not k in [ '_id', 'meta', 'changes', ]])) #logger.warn(pprint.pformat(d)) if d: now = datetime.datetime.utcnow().replace(microsecond=0).isoformat() if not res: logger.info(('adding %s - %s' % (data['procedure']['reference'], data['procedure']['title'])).encode('utf8')) data['meta']['created'] = data['meta']['timestamp'] del data['meta']['timestamp'] sys.stdout.flush() stats[0] += 1 else: logger.info(('updating %s - %s' % (data['procedure']['reference'], data['procedure']['title'])).encode('utf8')) data['meta']['updated'] = data['meta']['timestamp'] del data['meta']['timestamp'] sys.stdout.flush() stats[1] += 1 data['_id'] = res['_id'] logger.info(jdump(d)) if not NOMAIL: m = db.notifications.find( {'dossiers': data['procedure']['reference']}, ['active_emails']) for g in m: if len(g['active_emails']) == 0: continue msg = Message("[PT] %s %s" % (data['procedure']['reference'], data['procedure']['title']), sender="*****@*****.**", bcc=g['active_emails']) #msg.html = htmldiff(data,d) msg.body = makemsg(data, d) mail.send(msg) #logger.info(htmldiff(data,d)) #logger.info(makemsg(data,d)) data['changes'] = res.get('changes', {}) data['changes'][now] = d db.dossiers2.save(data) return stats
def save(data): data["activities"] = makeActivities(data) # print json.dumps(data,default=dateJSONhandler) # pprint.pprint(data) # return src = data["meta"]["source"] res = db.dossiers.find_one({"meta.source": src}) or {} d = diff( dict([(k, v) for k, v in data.items() if not k in ["_id", "meta", "changes"]]), dict([(k, v) for k, v in res.items() if not k in ["_id", "meta", "changes"]]), ) if d: now = datetime.datetime.utcnow().replace(microsecond=0).isoformat() if not res: print("\tadding %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8") data["meta"]["created"] = data["meta"]["timestamp"] del data["meta"]["timestamp"] sys.stdout.flush() stats[0] += 1 else: print("\tupdating %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8") data["meta"]["updated"] = data["meta"]["timestamp"] del data["meta"]["timestamp"] sys.stdout.flush() stats[1] += 1 data["_id"] = res["_id"] # print >> sys.stderr, (d) m = db.notifications.find({"dossiers": data["procedure"]["reference"]}, ["active_emails"]) for g in m: if len(g["active_emails"]) == 0: continue msg = Message( "Parltrack Notification for %s" % data["procedure"]["reference"], sender="*****@*****.**", bcc=g["active_emails"], ) # msg.body = "Parltrack has detected a change in %s %s on OEIL.\nfollow this URL: %s to see the dossier\n\nchanges below\n%s" % (data['procedure']['reference'], data['procedure']['title'],'%s/dossier/%s' % (ROOT_URL,data['procedure']['reference']), json.dumps(d,indent=1,default=dateJSONhandler)) msg.body = "Parltrack has detected a change in %s %s on OEIL.\nfollow this URL: %s to see the dossier\n" % ( data["procedure"]["reference"], data["procedure"]["title"], "%s/dossier/%s" % (ROOT_URL, data["procedure"]["reference"]), ) mail.send(msg) data["changes"] = res.get("changes", {}) data["changes"][now] = d db.dossiers.save(data) return stats
def save(data, stats): if len(data) < 1: return stats for item in data: db.ep_ams.save(item) m = db.notifications.find({"dossiers": data[0]["reference"]}, ["active_emails"]) for g in m: if len(g["active_emails"]) == 0: continue dtitle = db.dossiers2.find_one({"procedure.reference": data[0]["reference"]}, ["procedure.title"])["procedure"][ "title" ] msg = Message( "[PT-Am] %s %s" % (data[0]["reference"], dtitle), sender="*****@*****.**", bcc=g["active_emails"], ) msg.body = ( u"Parltrack has detected %s new amendments for \n" u"\n %s\n" u" %s\n" u"\n Committee(s): %s\n" u" Date: %s\n" u"\non OEIL.\n" u"\nPlease follow this URL:\n" u" %s/dossier/%s#ams\n" u" to see the dossier.\n" u"\nOr this URL to see the original PDF:\n" u" %s\n" u"\nsincerly,\n" u"Your Parltrack team" % ( len(data), data[0]["reference"], dtitle, ", ".join(data[0]["committee"]), data[0]["date"].isoformat().split("T")[0], ROOT_URL, quote_plus(data[0]["reference"]), data[0]["src"], ) ) mail.send(msg) if stats: stats[0] += len(data) return stats else: return data
def save(data, stats): src = data["meta"]["source"] res = db.dossiers2.find_one({"meta.source": src}) or {} d = diff( dict([(k, v) for k, v in res.items() if not k in ["_id", "meta", "changes"]]), dict([(k, v) for k, v in data.items() if not k in ["_id", "meta", "changes"]]), ) # logger.warn(d) if d: now = datetime.datetime.utcnow().replace(microsecond=0).isoformat() if not res: logger.info( ("adding %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8") ) data["meta"]["created"] = data["meta"]["timestamp"] del data["meta"]["timestamp"] sys.stdout.flush() stats[0] += 1 else: logger.info( ("updating %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8") ) data["meta"]["updated"] = data["meta"]["timestamp"] del data["meta"]["timestamp"] sys.stdout.flush() stats[1] += 1 data["_id"] = res["_id"] # print >> sys.stderr, (d) m = db.notifications.find({"dossiers": data["procedure"]["reference"]}, ["active_emails"]) for g in m: if len(g["active_emails"]) == 0: continue msg = Message( "[PT] %s %s" % (data["procedure"]["reference"], data["procedure"]["title"]), sender="*****@*****.**", bcc=g["active_emails"], ) msg.html = htmldiff(data, d) msg.body = makemsg(data, d) mail.send(msg) data["changes"] = res.get("changes", {}) data["changes"][now] = d db.dossiers2.save(data) return stats
def save(data, stats): if len(data)<1: return stats for item in data: db.ep_ams.save(item) m=db.notifications.find({'dossiers': data[0]['reference']},['active_emails']) for g in m: if len(g['active_emails'])==0: continue dtitle=db.dossiers2.find_one({'procedure.reference': data[0]['reference']}, ['procedure.title'])['procedure']['title'] msg = Message("[PT-Am] %s %s" % (data[0]['reference'], dtitle), sender = "*****@*****.**", bcc = g['active_emails']) msg.body = (u"Parltrack has detected %s new amendments for \n" u"\n %s\n" u" %s\n" u"\n Committee(s): %s\n" u" Date: %s\n" u"\non OEIL.\n" u"\nPlease follow this URL:\n" u" %s/dossier/%s#ams\n" u" to see the dossier.\n" u"\nOr this URL to see the original PDF:\n" u" %s\n" u"\nsincerly,\n" u"Your Parltrack team" % (len(data), data[0]['reference'], dtitle, ', '.join(data[0]['committee']), data[0]['date'].isoformat().split('T')[0], ROOT_URL, quote_plus(data[0]['reference']), data[0]['src']) ) mail.send(msg) if stats: stats[0]+=len(data) return stats else: return data
def save(data, stats): if not data: return stats src=data['meta']['source'] res=db.dossiers2.find_one({ 'meta.source' : src }) or {} d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes']]), dict([(k,v) for k,v in data.items() if not k in ['_id', 'meta', 'changes',]])) #logger.warn(pprint.pformat(d)) if d: now=datetime.datetime.utcnow().replace(microsecond=0).isoformat() if not res: logger.info(('adding %s - %s' % (data['procedure']['reference'],data['procedure']['title'])).encode('utf8')) data['meta']['created']=data['meta']['timestamp'] del data['meta']['timestamp'] sys.stdout.flush() stats[0]+=1 else: logger.info(('updating %s - %s' % (data['procedure']['reference'],data['procedure']['title'])).encode('utf8')) data['meta']['updated']=data['meta']['timestamp'] del data['meta']['timestamp'] sys.stdout.flush() stats[1]+=1 data['_id']=res['_id'] logger.info(jdump(d)) if not NOMAIL: m=db.notifications.find({'dossiers': data['procedure']['reference']},['active_emails']) for g in m: if len(g['active_emails'])==0: continue msg = Message("[PT] %s %s" % (data['procedure']['reference'],data['procedure']['title']), sender = "*****@*****.**", bcc = g['active_emails']) #msg.html = htmldiff(data,d) msg.body = makemsg(data,d) mail.send(msg) #logger.info(htmldiff(data,d)) #logger.info(makemsg(data,d)) data['changes']=res.get('changes',{}) data['changes'][now]=d db.dossiers2.save(data) return stats