def notify(data,d):
if not 'epdoc' in data: return
m=db.notifications.find({'dossiers': data['epdoc']},['active_emails'])
for g in m:
if len(g['active_emails'])==0:
continue
msg = Message("[PT-Com] %s: %s" %
(data['committee'],
data['title']),
sender = "*****@*****.**",
bcc = g['active_emails'])
msg.body = (u"Parltrack has detected %s%s on the schedule of %s \n"
u"\n on %s"
u"\n%s"
u"%s"
u"\nsee the details here: %s\n"
u"\nYour Parltrack team" %
(u"a change on " if d else u'',
data['epdoc'],
data['committee'],
data['date'] if 'date' in data else 'unknown date',
("\n - %s" % u'\n - '.join(data['list'])) if 'list' in data and len(data['list'])>0 else u"",
"\n %s" % (textdiff(d) if d else ''),
"%s/dossier/%s" % (ROOT_URL, data['epdoc']),
))
mail.send(msg)
def save(data, stats):
if not data: return stats
src = data['meta']['source']
res = db.dossiers2.find_one({'meta.source': src}) or {}
d = diff(
dict([(k, v) for k, v in res.items()
if not k in ['_id', 'meta', 'changes']]),
dict([(k, v) for k, v in data.items() if not k in [
'_id',
'meta',
'changes',
]]))
#logger.warn(pprint.pformat(d))
if d:
now = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
if not res:
logger.info(('adding %s - %s' %
(data['procedure']['reference'],
data['procedure']['title'])).encode('utf8'))
data['meta']['created'] = data['meta']['timestamp']
del data['meta']['timestamp']
sys.stdout.flush()
stats[0] += 1
else:
logger.info(('updating %s - %s' %
(data['procedure']['reference'],
data['procedure']['title'])).encode('utf8'))
data['meta']['updated'] = data['meta']['timestamp']
del data['meta']['timestamp']
sys.stdout.flush()
stats[1] += 1
data['_id'] = res['_id']
logger.info(jdump(d))
if not NOMAIL:
m = db.notifications.find(
{'dossiers': data['procedure']['reference']},
['active_emails'])
for g in m:
if len(g['active_emails']) == 0:
continue
msg = Message("[PT] %s %s" % (data['procedure']['reference'],
data['procedure']['title']),
sender="*****@*****.**",
bcc=g['active_emails'])
#msg.html = htmldiff(data,d)
msg.body = makemsg(data, d)
mail.send(msg)
#logger.info(htmldiff(data,d))
#logger.info(makemsg(data,d))
data['changes'] = res.get('changes', {})
data['changes'][now] = d
db.dossiers2.save(data)
return stats
def save(data):
data["activities"] = makeActivities(data)
# print json.dumps(data,default=dateJSONhandler)
# pprint.pprint(data)
# return
src = data["meta"]["source"]
res = db.dossiers.find_one({"meta.source": src}) or {}
d = diff(
dict([(k, v) for k, v in data.items() if not k in ["_id", "meta", "changes"]]),
dict([(k, v) for k, v in res.items() if not k in ["_id", "meta", "changes"]]),
)
if d:
now = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
if not res:
print("\tadding %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8")
data["meta"]["created"] = data["meta"]["timestamp"]
del data["meta"]["timestamp"]
sys.stdout.flush()
stats[0] += 1
else:
print("\tupdating %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8")
data["meta"]["updated"] = data["meta"]["timestamp"]
del data["meta"]["timestamp"]
sys.stdout.flush()
stats[1] += 1
data["_id"] = res["_id"]
# print >> sys.stderr, (d)
m = db.notifications.find({"dossiers": data["procedure"]["reference"]}, ["active_emails"])
for g in m:
if len(g["active_emails"]) == 0:
continue
msg = Message(
"Parltrack Notification for %s" % data["procedure"]["reference"],
sender="*****@*****.**",
bcc=g["active_emails"],
)
# msg.body = "Parltrack has detected a change in %s %s on OEIL.\nfollow this URL: %s to see the dossier\n\nchanges below\n%s" % (data['procedure']['reference'], data['procedure']['title'],'%s/dossier/%s' % (ROOT_URL,data['procedure']['reference']), json.dumps(d,indent=1,default=dateJSONhandler))
msg.body = "Parltrack has detected a change in %s %s on OEIL.\nfollow this URL: %s to see the dossier\n" % (
data["procedure"]["reference"],
data["procedure"]["title"],
"%s/dossier/%s" % (ROOT_URL, data["procedure"]["reference"]),
)
mail.send(msg)
data["changes"] = res.get("changes", {})
data["changes"][now] = d
db.dossiers.save(data)
return stats
def save(data, stats):
if len(data) < 1:
return stats
for item in data:
db.ep_ams.save(item)
m = db.notifications.find({"dossiers": data[0]["reference"]}, ["active_emails"])
for g in m:
if len(g["active_emails"]) == 0:
continue
dtitle = db.dossiers2.find_one({"procedure.reference": data[0]["reference"]}, ["procedure.title"])["procedure"][
"title"
]
msg = Message(
"[PT-Am] %s %s" % (data[0]["reference"], dtitle),
sender="*****@*****.**",
bcc=g["active_emails"],
)
msg.body = (
u"Parltrack has detected %s new amendments for \n"
u"\n %s\n"
u" %s\n"
u"\n Committee(s): %s\n"
u" Date: %s\n"
u"\non OEIL.\n"
u"\nPlease follow this URL:\n"
u" %s/dossier/%s#ams\n"
u" to see the dossier.\n"
u"\nOr this URL to see the original PDF:\n"
u" %s\n"
u"\nsincerly,\n"
u"Your Parltrack team"
% (
len(data),
data[0]["reference"],
dtitle,
", ".join(data[0]["committee"]),
data[0]["date"].isoformat().split("T")[0],
ROOT_URL,
quote_plus(data[0]["reference"]),
data[0]["src"],
)
)
mail.send(msg)
if stats:
stats[0] += len(data)
return stats
else:
return data
def save(data, stats):
src = data["meta"]["source"]
res = db.dossiers2.find_one({"meta.source": src}) or {}
d = diff(
dict([(k, v) for k, v in res.items() if not k in ["_id", "meta", "changes"]]),
dict([(k, v) for k, v in data.items() if not k in ["_id", "meta", "changes"]]),
)
# logger.warn(d)
if d:
now = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
if not res:
logger.info(
("adding %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8")
)
data["meta"]["created"] = data["meta"]["timestamp"]
del data["meta"]["timestamp"]
sys.stdout.flush()
stats[0] += 1
else:
logger.info(
("updating %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8")
)
data["meta"]["updated"] = data["meta"]["timestamp"]
del data["meta"]["timestamp"]
sys.stdout.flush()
stats[1] += 1
data["_id"] = res["_id"]
# print >> sys.stderr, (d)
m = db.notifications.find({"dossiers": data["procedure"]["reference"]}, ["active_emails"])
for g in m:
if len(g["active_emails"]) == 0:
continue
msg = Message(
"[PT] %s %s" % (data["procedure"]["reference"], data["procedure"]["title"]),
sender="*****@*****.**",
bcc=g["active_emails"],
)
msg.html = htmldiff(data, d)
msg.body = makemsg(data, d)
mail.send(msg)
data["changes"] = res.get("changes", {})
data["changes"][now] = d
db.dossiers2.save(data)
return stats
def save(data, stats):
if len(data)<1: return stats
for item in data:
db.ep_ams.save(item)
m=db.notifications.find({'dossiers': data[0]['reference']},['active_emails'])
for g in m:
if len(g['active_emails'])==0:
continue
dtitle=db.dossiers2.find_one({'procedure.reference': data[0]['reference']},
['procedure.title'])['procedure']['title']
msg = Message("[PT-Am] %s %s" %
(data[0]['reference'], dtitle),
sender = "*****@*****.**",
bcc = g['active_emails'])
msg.body = (u"Parltrack has detected %s new amendments for \n"
u"\n %s\n"
u" %s\n"
u"\n Committee(s): %s\n"
u" Date: %s\n"
u"\non OEIL.\n"
u"\nPlease follow this URL:\n"
u" %s/dossier/%s#ams\n"
u" to see the dossier.\n"
u"\nOr this URL to see the original PDF:\n"
u" %s\n"
u"\nsincerly,\n"
u"Your Parltrack team" %
(len(data),
data[0]['reference'],
dtitle,
', '.join(data[0]['committee']),
data[0]['date'].isoformat().split('T')[0],
ROOT_URL,
quote_plus(data[0]['reference']),
data[0]['src'])
)
mail.send(msg)
if stats:
stats[0]+=len(data)
return stats
else: return data
def save(data, stats):
if not data: return stats
src=data['meta']['source']
res=db.dossiers2.find_one({ 'meta.source' : src }) or {}
d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes']]),
dict([(k,v) for k,v in data.items() if not k in ['_id', 'meta', 'changes',]]))
#logger.warn(pprint.pformat(d))
if d:
now=datetime.datetime.utcnow().replace(microsecond=0).isoformat()
if not res:
logger.info(('adding %s - %s' % (data['procedure']['reference'],data['procedure']['title'])).encode('utf8'))
data['meta']['created']=data['meta']['timestamp']
del data['meta']['timestamp']
sys.stdout.flush()
stats[0]+=1
else:
logger.info(('updating %s - %s' % (data['procedure']['reference'],data['procedure']['title'])).encode('utf8'))
data['meta']['updated']=data['meta']['timestamp']
del data['meta']['timestamp']
sys.stdout.flush()
stats[1]+=1
data['_id']=res['_id']
logger.info(jdump(d))
if not NOMAIL:
m=db.notifications.find({'dossiers': data['procedure']['reference']},['active_emails'])
for g in m:
if len(g['active_emails'])==0:
continue
msg = Message("[PT] %s %s" % (data['procedure']['reference'],data['procedure']['title']),
sender = "*****@*****.**",
bcc = g['active_emails'])
#msg.html = htmldiff(data,d)
msg.body = makemsg(data,d)
mail.send(msg)
#logger.info(htmldiff(data,d))
#logger.info(makemsg(data,d))
data['changes']=res.get('changes',{})
data['changes'][now]=d
db.dossiers2.save(data)
return stats