def save(data, stats):
if not data: return stats
#print jdump(data)
res = db.eurlex.find_one({'id.celexid': data['id']['celexid']}) or {}
d = diff(
dict([(k, v) for k, v in res.items()
if not k in ['_id', 'meta', 'changes']]),
dict([(k, v) for k, v in data.items() if not k in [
'_id',
'meta',
'changes',
]]))
if d:
now = unicode(datetime.utcnow().replace(microsecond=0).isoformat())
if not res:
logger.info(('adding %s' % (data['id']['celexid'])).encode('utf8'))
data['meta']['created'] = now
if stats: stats[0] += 1
else:
logger.info(
('updating %s' % (data['id']['celexid'])).encode('utf8'))
#logger.warn(d)
data['meta']['updated'] = now
if stats: stats[1] += 1
data['_id'] = res['_id']
data['changes'] = res.get('changes', {})
data['changes'][now] = d
db.eurlex.save(data)
if stats: return stats
else: return data
def save(data, stats):
res=db.ep_meps2.find_one({ 'UserID' : data['UserID'] }) or {}
if 'Gender' not in data and 'Gender' in res: data['Gender']=res['Gender']
d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes', 'activities',]]),
dict([(k,v) for k,v in data.items() if not k in ['_id', 'meta', 'changes', 'activities',]]))
if d:
now=datetime.utcnow().replace(microsecond=0)
if not res:
logger.info(('adding %s' % (data['Name']['full'])).encode('utf8'))
data['meta']['created']=now
if stats: stats[0]+=1
else:
logger.info(('updating %s' % (data['Name']['full'])).encode('utf8'))
logger.warn(jdump(d))
data['meta']['updated']=now
if stats: stats[1]+=1
data['_id']=res['_id']
data['changes']=res.get('changes',{})
data['changes'][now.isoformat()]=d
db.ep_meps2.save(data)
del res
if stats:
del data
return stats
else: return data
def save(data, stats):
res = db.ep_meps2.find_one({"UserID": data["UserID"]}) or {}
d = diff(
dict([(k, v) for k, v in res.items() if not k in ["_id", "meta", "changes"]]),
dict([(k, v) for k, v in data.items() if not k in ["_id", "meta", "changes"]]),
)
if d:
now = datetime.utcnow().replace(microsecond=0)
if not res:
logger.info(("adding %s" % (data["Name"]["full"])).encode("utf8"))
data["meta"]["created"] = now
if stats:
stats[0] += 1
else:
logger.info(("updating %s" % (data["Name"]["full"])).encode("utf8"))
logger.warn(jdump(d))
data["meta"]["updated"] = now
if stats:
stats[1] += 1
data["_id"] = res["_id"]
data["changes"] = res.get("changes", {})
data["changes"][now.isoformat()] = d
db.ep_meps2.save(data)
if stats:
return stats
else:
return data
def save(data, stats):
for item in data:
if not 'committee' in item: continue
query={'committee': item['committee'],
'src': item['src'],
'title': item['title']}
if 'date' in data:
query['date']= item['date']
if 'end' in data:
query['end']= item['end']
else:
query['seq_no']=item['seq_no']
res=db.ep_comagendas.find_one(query) or {}
d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes']]),
dict([(k,v) for k,v in item.items() if not k in ['_id', 'meta', 'changes',]]))
if d:
now=datetime.utcnow().replace(microsecond=0)
if not 'meta' in item: item[u'meta']={}
if not res:
logger.info((u'adding %s %s' % (item['committee'], item['title'])).encode('utf8'))
item['meta']['created']=now
if stats: stats[0]+=1
else:
logger.info((u'updating %s %s' % (item['committee'], item['title'])).encode('utf8'))
logger.info(d)
item['meta']['updated']=now
if stats: stats[1]+=1
item['_id']=res['_id']
item['changes']=res.get('changes',{})
item['changes'][now.isoformat()]=d
db.ep_comagendas.save(item)
if stats: return stats
else: return data
def save(data, stats):
if not data: return stats
src = data['meta']['source']
res = db.dossiers2.find_one({'meta.source': src}) or {}
d = diff(
dict([(k, v) for k, v in res.items()
if not k in ['_id', 'meta', 'changes']]),
dict([(k, v) for k, v in data.items() if not k in [
'_id',
'meta',
'changes',
]]))
#logger.warn(pprint.pformat(d))
if d:
now = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
if not res:
logger.info(('adding %s - %s' %
(data['procedure']['reference'],
data['procedure']['title'])).encode('utf8'))
data['meta']['created'] = data['meta']['timestamp']
del data['meta']['timestamp']
sys.stdout.flush()
stats[0] += 1
else:
logger.info(('updating %s - %s' %
(data['procedure']['reference'],
data['procedure']['title'])).encode('utf8'))
data['meta']['updated'] = data['meta']['timestamp']
del data['meta']['timestamp']
sys.stdout.flush()
stats[1] += 1
data['_id'] = res['_id']
logger.info(jdump(d))
if not NOMAIL:
m = db.notifications.find(
{'dossiers': data['procedure']['reference']},
['active_emails'])
for g in m:
if len(g['active_emails']) == 0:
continue
msg = Message("[PT] %s %s" % (data['procedure']['reference'],
data['procedure']['title']),
sender="*****@*****.**",
bcc=g['active_emails'])
#msg.html = htmldiff(data,d)
msg.body = makemsg(data, d)
mail.send(msg)
#logger.info(htmldiff(data,d))
#logger.info(makemsg(data,d))
data['changes'] = res.get('changes', {})
data['changes'][now] = d
db.dossiers2.save(data)
return stats
def save(data, stats):
for item in data:
if not 'committee' in item: continue
query = {
'committee': item['committee'],
'src': item['src'],
'title': item['title']
}
if 'date' in data:
query['date'] = item['date']
if 'end' in data:
query['end'] = item['end']
else:
query['seq_no'] = item['seq_no']
res = db.ep_comagendas.find_one(query) or {}
d = diff(
dict([(k, v) for k, v in res.items()
if not k in ['_id', 'meta', 'changes']]),
dict([(k, v) for k, v in item.items() if not k in [
'_id',
'meta',
'changes',
]]))
if d:
now = datetime.utcnow().replace(microsecond=0)
if not 'meta' in item: item[u'meta'] = {}
if not res:
logger.info(
(u'adding %s%s %s' %
(u'%s ' % item['epdoc'] if 'epdoc' in item else '',
item['committee'], item['title'])).encode('utf8'))
item['meta']['created'] = now
if stats: stats[0] += 1
notify(item, None)
else:
logger.info(
(u'updating %s%s %s' %
(u'%s ' % item['epdoc'] if 'epdoc' in item else '',
item['committee'], item['title'])).encode('utf8'))
logger.info(d)
item['meta']['updated'] = now
if stats: stats[1] += 1
item['_id'] = res['_id']
notify(item, d)
item['changes'] = res.get('changes', {})
item['changes'][now.isoformat()] = d
db.ep_comagendas.save(item)
if stats: return stats
else: return data
def save(data, stats):
src = data["meta"]["source"]
res = db.dossiers2.find_one({"meta.source": src}) or {}
d = diff(
dict([(k, v) for k, v in res.items() if not k in ["_id", "meta", "changes"]]),
dict([(k, v) for k, v in data.items() if not k in ["_id", "meta", "changes"]]),
)
# logger.warn(d)
if d:
now = datetime.datetime.utcnow().replace(microsecond=0).isoformat()
if not res:
logger.info(
("adding %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8")
)
data["meta"]["created"] = data["meta"]["timestamp"]
del data["meta"]["timestamp"]
sys.stdout.flush()
stats[0] += 1
else:
logger.info(
("updating %s - %s" % (data["procedure"]["reference"], data["procedure"]["title"])).encode("utf8")
)
data["meta"]["updated"] = data["meta"]["timestamp"]
del data["meta"]["timestamp"]
sys.stdout.flush()
stats[1] += 1
data["_id"] = res["_id"]
# print >> sys.stderr, (d)
m = db.notifications.find({"dossiers": data["procedure"]["reference"]}, ["active_emails"])
for g in m:
if len(g["active_emails"]) == 0:
continue
msg = Message(
"[PT] %s %s" % (data["procedure"]["reference"], data["procedure"]["title"]),
sender="*****@*****.**",
bcc=g["active_emails"],
)
msg.html = htmldiff(data, d)
msg.body = makemsg(data, d)
mail.send(msg)
data["changes"] = res.get("changes", {})
data["changes"][now] = d
db.dossiers2.save(data)
return stats
def save(data, stats):
res=db.ep_meps2.find_one({ 'UserID' : data['UserID'] }) or {}
d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes']]),
dict([(k,v) for k,v in data.items() if not k in ['_id', 'meta', 'changes',]]))
if d:
now=unicode(datetime.utcnow().replace(microsecond=0).isoformat())
if not res:
logger.info(('adding %s' % (data['Name']['full'])).encode('utf8'))
data['meta']['created']=now
stats[0]+=1
else:
logger.info(('updating %s' % (data['Name']['full'])).encode('utf8'))
logger.warn(d)
data['meta']['updated']=now
stats[1]+=1
data['_id']=res['_id']
data['changes']=res.get('changes',{})
data['changes'][now]=d
db.ep_meps2.save(data)
return stats
def save(data, stats):
if not data: return stats
src=data['meta']['source']
res=db.dossiers2.find_one({ 'meta.source' : src }) or {}
d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes']]),
dict([(k,v) for k,v in data.items() if not k in ['_id', 'meta', 'changes',]]))
#logger.warn(pprint.pformat(d))
if d:
now=datetime.datetime.utcnow().replace(microsecond=0).isoformat()
if not res:
logger.info(('adding %s - %s' % (data['procedure']['reference'],data['procedure']['title'])).encode('utf8'))
data['meta']['created']=data['meta']['timestamp']
del data['meta']['timestamp']
sys.stdout.flush()
stats[0]+=1
else:
logger.info(('updating %s - %s' % (data['procedure']['reference'],data['procedure']['title'])).encode('utf8'))
data['meta']['updated']=data['meta']['timestamp']
del data['meta']['timestamp']
sys.stdout.flush()
stats[1]+=1
data['_id']=res['_id']
logger.info(jdump(d))
if not NOMAIL:
m=db.notifications.find({'dossiers': data['procedure']['reference']},['active_emails'])
for g in m:
if len(g['active_emails'])==0:
continue
msg = Message("[PT] %s %s" % (data['procedure']['reference'],data['procedure']['title']),
sender = "*****@*****.**",
bcc = g['active_emails'])
#msg.html = htmldiff(data,d)
msg.body = makemsg(data,d)
mail.send(msg)
#logger.info(htmldiff(data,d))
#logger.info(makemsg(data,d))
data['changes']=res.get('changes',{})
data['changes'][now]=d
db.dossiers2.save(data)
return stats
def save(data, stats):
if not data: return stats
res=db.eurlex.find_one({ 'id.celexid' : data['id']['celexid'] }) or {}
d=diff(dict([(k,v) for k,v in res.items() if not k in ['_id', 'meta', 'changes']]),
dict([(k,v) for k,v in data.items() if not k in ['_id', 'meta', 'changes',]]))
if d:
now=unicode(datetime.utcnow().replace(microsecond=0).isoformat())
if not res:
logger.info(('adding %s' % (data['id']['celexid'])).encode('utf8'))
data['meta']['created']=now
if stats: stats[0]+=1
else:
logger.info(('updating %s' % (data['id']['celexid'])).encode('utf8'))
logger.warn(d)
data['meta']['updated']=now
if stats: stats[1]+=1
data['_id']=res['_id']
data['changes']=res.get('changes',{})
data['changes'][now]=d
db.eurlex.save(data)
if stats: return stats
else: return data