def ouv():
depgp = dict((d['depute_shortid'],{'img':'http://www2.assemblee-nationale.fr/static/tribun/15/photos/'+d['depute_uid'][2:]+'.jpg','g':d['groupe_abrev'],'n':d['depute_nom']}) for d in mdb.deputes.find({},{'depute_nom':1,'depute_uid':1,'depute_shortid':1,'groupe_abrev':1,'_id':None}))
shortids = dict((d['depute_id'],d['depute_shortid']) for d in mdb.deputes.find({},{'depute_id':1,'depute_shortid':1,'_id':None}))
depsig = {}
for doc in mdb.documentsan.find({'signataires':{'$ne':None}},{'numero':1,'signataires':1,'_id':None}):
if doc['signataires']:
sig1 = doc['signataires'][0]
if sig1:
for sig in doc['signataires'][1:]:
if sig and depgp[sig1]['g']!=depgp[sig]['g'] and depgp[sig]['g']!='NI':
if sig=='mohamedlaqhila':
print doc
depsig[sig] = depsig.get(sig,0)+1
for amd in mdb.amendements.find({},{'numAmend':1,'signataires_ids':1,'_id':None}):
if not amd['signataires_ids']:
continue
sig1 = shortids.get(amd['signataires_ids'][0],None)
if sig1:
for sig in amd['signataires_ids'][1:]:
sig2 = shortids[sig]
if sig2 and depgp[sig1]['g']!=depgp[sig2]['g'] and depgp[sig2]!='NI':
if sig2=='mohamedlaqhila':
print amd
depsig[sig2] = depsig.get(sig2,0)+1
return json_response(sorted(depsig.iteritems(),key=lambda x:x[1],reverse=True))
def presences():
depute = request.args.get('depute',None)
commission = request.args.get('commission',None)
page = int(request.args.get('page','1'))-1
nb = int(request.args.get('itemsperpage','25'))
groupe = request.args.get('groupe',request.args.get('group',None))
skip = nb*page
filters = []
if depute:
filters.append({'depute_shortid': depute})
if groupe:
filters.append({'groupe_abrev':groupe})
if commission:
filters.append({'commission_sid':commission})
if len(filters)==0:
com_filter = {}
elif len(filters)==1:
com_filter = filters[0]
else:
com_filter = {'$and':filters}
prescom = list(mdb.presences.find(com_filter).sort([('presence_date',-1)]).skip(skip).limit(nb))
def countItems():
rcount = mdb.presences.find(com_filter).count()
return {'totalitems':rcount}
cachekey= u"comm%s_%s_%s" % (depute,groupe,commission)
counts = use_cache(cachekey,lambda:countItems(),expires=3600)
import math
nbpages = int(math.ceil(float(counts['totalitems'])/nb))
result = dict(nbitems=len(prescom),nbpages=nbpages, currentpage=1+page,itemsperpage=nb, items=prescom,**counts)
return json_response(result)
def votesgroupes():
#return json_response(list(mdb.groupes.find({},{'_id':None,'groupe_abrev':1,'groupe_uid':1})))
libelles = {'FI':u'France Insoumise',
'LAREM':u'République en Marche',
'UDI-AGIR': u'UDI, Agir et Indépendants',
'GDR':u'Gauche Démocratique et Républicaine',
'NG':u'Nouvelle Gauche',
'MODEM':u'Mouvement Démocrate',
'LR':u'Les Républicains',
'NI':u'Députés Non Inscrits'
}
#return json_response(mdb.votes.find_one())
pgroup = {}
pgroup['n'] = {'$sum':1}
pgroup['_id'] = { 'groupe':'$groupe_abrev','position':'$vote_position'}
pipeline = [{'$group':pgroup}]
grps = {}
for v in mdb.votes.aggregate(pipeline):
g = v['_id']['groupe']
p = v['_id']['position']
n = v['n']
if g=='LC':
g='UDI-AGIR'
if not g in grps:
grps[g] = { 'absent':0,'pour':0,'contre':0,'abstention':0 }
grps[g][p] += n
stats = []
for g in grps:
stat = dict(g=g)
tot = sum(grps[g].values())
for p in ('pour','contre','abstention','absent'):
stat[p]=100*float(grps[g][p])/tot
stats.append(stat)
stats.sort(key=lambda x:x['absent'],reverse=True)
from pygal.style import Style
custom_style = Style(
font_family="'Montserrat', sans-serif;",
major_label_font_size=15,
colors=['#25a87e','#e23d21','#213558','#bbbbbb']
)
histo_chart = pygal.HorizontalStackedBar(x_label_rotation=0,width=1024,height=512,human_readable=True, x_title='%',y_title="Groupes parlementaires",style=custom_style)
histo_chart.title = u'Votes des députés aux scrutins publics\n par groupe parlementaire (au %s)' % (datetime.datetime.now().strftime('%d/%m/%Y'))
for x in 'pour','contre','abstention','absent':
histo_chart.add('%s' % x, [stat[x] for stat in stats])
histo_chart.x_labels = [libelles[stat['g']] for stat in stats]
histo_chart.y_labels = []
#histo_chart.x_labels_major = majors
from StringIO import StringIO
chart = StringIO()
histo_chart.render_to_png(chart)
return image_response('png',chart.getvalue())
return json_response(stats)
def interventions():
nb = int(request.args.get('itemsperpage', '25'))
page = int(request.args.get('page', '1')) - 1
groupe = request.args.get('groupe', request.args.get('group', None))
search = request.args.get('requete', request.args.get('query', ''))
depute = request.args.get('depute', None)
session = request.args.get('session', None)
date = request.args.get('date', None)
skip = nb * page
filters = []
if depute:
filters.append({'depute_shortid': depute})
if groupe:
filters.append({'groupe_abrev': groupe})
if session:
filters.append({'session_id': session})
if date:
filters.append({'itv_date': date})
if search:
filters.append({'$text': {'$search': '"' + search + '"'}})
if len(filters) == 0:
itv_filter = {}
elif len(filters) == 1:
itv_filter = filters[0]
else:
itv_filter = {'$and': filters}
itvs = list(
mdb.interventions.find(itv_filter).sort([
('itv_date', -1), ('session_id', 1), ('itv_n', 1)
]).skip(skip).limit(nb))
def countItems():
rcount = mdb.interventions.find(itv_filter).count()
return {'totalitems': rcount}
cachekey = u"itv%s_%s_%s_%s_%s" % (depute, groupe, search, session, date)
counts = use_cache(cachekey, lambda: countItems(), expires=3600)
regx = re.compile(search, re.IGNORECASE)
if search:
for itv in itvs:
repl = regx.subn('<strong>' + search + '</strong>',
itv['itv_contenu'])
if repl[1]:
itv['itv_contenu'] = repl[0]
import math
nbpages = int(math.ceil(float(counts['totalitems']) / nb))
result = dict(nbitems=len(itvs),
nbpages=nbpages,
currentpage=1 + page,
itemsperpage=nb,
items=itvs,
**counts)
return json_response(result)
def deputes(func=""):
if func=='liste':
resp = _ajax('liste')
elif func=='top':
resp = _ajax('top')
else:
resp = deputeget(func)
return json_response(resp)
def groupes(func=""):
if func == 'liste':
resp = _ajax('liste')
elif func == 'top':
resp = _ajax('top')
else:
resp = groupeget(func)
return json_response(resp)
def interventions_last():
itvs = list(
mdb.interventions.find({
'$and': [{
'itv_president': False
}, {
'depute_shortid': {
'$ne': None
}
}]
}).sort([('itv_date', -1), ('session_id', -1),
('itv_n', -1)]).limit(1))
return json_response(itvs)
def classements():
return json_response(mdb.votes.find_one())
pgroup = {}
pgroup['n'] = {'$sum':1}
pgroup['_id'] = { 'mois':{'$concat':[{'$substr':['$scrutin_date',6,4]},'-',{'$substr':['$scrutin_date',3,2]}]},'dep':'$depute_shortid','position':'$vote_position'}
pipeline = [{'$group':pgroup}]
mois = {}
for agg in mdb.votes.aggregate(pipeline):
m = agg['_id']['mois']
d = agg['_id']['dep']
p = agg['_id']['position']
n = agg['n']
if not d in mois.keys():
mois[d] = {}
if not m in mois[d].keys():
mois[d][m] = {'absent':0,'abstention':0,'pour':0,'contre':0,'total':0}
mois[d][m][p] += n
mois[d][m]['total'] += n
return json_response(mois['charlottelecocq'])
def hatvpinter():
if 0:
r = requests.get('http://www.hatvp.fr/agora/opendata/agora_repertoire_opendata.json')
content = r.content
open('/tmp/decs','w').write(content)
else:
content = open('/tmp/decs','r').read()
import json
decs = json.loads(content)
nodes = []
links = []
for i,fiche in enumerate(decs['publications']):
print i
id = fiche['identifiantNational']
r =requests.get('http://www.hatvp.fr/fiche-organisation/?organisation='+fiche['identifiantNational']+'#')
if rien not in r.content:
print "-->",id
ids.append(id)
return json_response(ids)
def scrutinscles():
nb = int(request.args.get('nb', '0'))
scrutins_cles = use_cache('scrutins_cles',
lambda: getScrutinsCles(),
expires=3600)
scrutins_positions = use_cache('scrutins_positions',
lambda: getScrutinsPositions(),
expires=36000)
scrutins = mdb.scrutins.find({
'scrutin_num': {
'$in': scrutins_cles.keys()
}
}).sort([('scrutin_num', -1)])
if nb > 0:
scrutins = scrutins.limit(nb)
scles = []
for s in scrutins:
if s['scrutin_dossierLibelle'] == 'N/A' and scrutins_cles[
s['scrutin_num']]['dossier']:
dossier = scrutins_cles[s['scrutin_num']]['dossier']
else:
dossier = s['scrutin_dossierLibelle']
positions = dict(scrutins_positions[s['scrutin_num']])
if scrutins_cles[s['scrutin_num']]['inversion'] == 'oui':
positions['position'] = {
'pour': 'contre',
'contre': 'pour'
}.get(positions['position'], positions['position'])
positions['pour'] = scrutins_positions[s['scrutin_num']]['contre']
positions['contre'] = scrutins_positions[s['scrutin_num']]['pour']
scles.append(
dict(desc=s['scrutin_desc'],
date=s['scrutin_date'],
sort=s['scrutin_sort'],
dossierLibelle=dossier.replace(u'\u0092', "'"),
detail=scrutins_cles[s['scrutin_num']],
positions=positions))
return json_response(scles)
def testcompat():
#return json_response(list(mdb.scrutins.find({'$and':[{'scrutin_amendement_groupe':None},{'scrutin_typedetail':'amendement'}]},{'scrutin_num':1})))
#for gid in ['FI','GDR','UAI','LR','MODEM','NG','REM']:
pgroup = {'n':{'$sum':1}}
pgroup['_id'] = {'depute':'$depute_shortid'}
pgroup['_id']['position'] ='$vote_position'
pgroup['_id']['groupe'] = '$scrutin_groupe'
pipeline = [{'$match':{}}, {"$group": pgroup }] #'scrutin_typedetail':'amendement'
voteamdts = {}
for voteamdt in mdb.votes.aggregate(pipeline):
depuid = voteamdt['_id']['depute']
pos = voteamdt['_id']['position']
gp = voteamdt['_id']['groupe']
if not depuid in voteamdts.keys():
voteamdts[depuid] = {}
if not gp in voteamdts[depuid].keys():
voteamdts[depuid][gp] = {}
voteamdts[depuid][gp][pos] = voteamdts[depuid][gp].get(pos,0) + voteamdt['n']
return json_response(voteamdts['mariechristineverdierjouclas'])
def deputehasard():
from obsapis.controllers.scrutins import getScrutinsCles
scrutins_cles = use_cache('scrutins_cles',lambda:getScrutinsCles(),expires=3600)
nbdeputes = use_cache('nbdeputes',lambda:mdb.deputes.find({'depute_actif':True}).count(),expires=3600)
mfields = dict((f,1) for f in deputesfields)
mfields.update({'_id':None})
depute = mdb.deputes.find({'depute_actif':True},mfields).skip(int(random.random()*nbdeputes)).limit(1)[0]
photo_an='http://www2.assemblee-nationale.fr/static/tribun/15/photos/'+depute['depute_uid'][2:]+'.jpg'
depnumdep = depute['depute_departement_id'][1:] if depute['depute_departement_id'][0]=='0' else depute['depute_departement_id']
if depute['depute_region']==depute['depute_departement']:
depute_circo_complet = "%s (%s) / %se circ" % (depute['depute_departement'],depnumdep,depute['depute_circo'])
else:
depute_circo_complet = "%s / %s (%s) / %se circ" % (depute['depute_region'],depute['depute_departement'],depnumdep,depute['depute_circo'])
resp = dict(depute_circo_complet = depute_circo_complet,
depute_photo_an = photo_an,
id = depute['depute_shortid'],
**depute)
return json_response(resp)
def csa():
import requests
import lxml.html
from StringIO import StringIO
itvs = []
r = requests.get('http://www.csa.fr/csapluralisme/tableau?annee=2017')
html = lxml.html.fromstring(r.content)
for url in html.xpath('//tr[td[text()[contains(.,"PERSONNALITES POLITIQ7UES")]]]/td[3]/a/@href'):
print url
fullurl = 'http://www.csa.fr'+url
r = requests.get(fullurl)
parse = csa_pdf(StringIO(r.content))
if parse=='boom':
1/0
itvs += parse
f = open('test.csv','w')
for itv in itvs:
f.write(";".join([itv['chaine'],"%d-%d" % itv['date'],itv['type'],itv['nom'],itv['org'],itv['duree']])+'\n')
return "ok"
return json_response(itvs)
def amds():
import_amendements()
update_amendements()
#return "ok"
pgroup = {'n':{'$sum':1}}
pgroup['_id'] = {'depute':'$auteurs.id'}
pgroup['_id']['sort'] ='$sort'
pipeline = [{'$match':{}}, {'$unwind':'$auteurs'}, {"$group": pgroup }] #'scrutin_typedetail':'amendement'
stat_amdts = {}
for amdt in mdb.amendements.aggregate(pipeline):
if amdt['_id']['depute']=='lisemagnier':
print amdt['_id']['sort'],amdt['n']
amd = amdt['_id']
if not amd['depute'] in stat_amdts.keys():
stat_amdts[amd['depute']] = {'rediges':0,'adoptes':0,'cosignes':0}
if amd['sort']==u'Adopté':
stat_amdts[amd['depute']]['adoptes'] += amdt['n']
stat_amdts[amd['depute']]['rediges'] += amdt['n']
return json_response(stat_amdts)
def scrutinsdetail(num):
scrutin = mdb.scrutins.find_one({'scrutin_num': num})
scrutins_cles = use_cache('scrutins_cles',
lambda: getScrutinsCles(),
expires=3600)
scrutin.update(scrutins_cles.get(num, {}))
positions = {}
for dpos in mdb.votes.find({'scrutin_num': num}, {
'depute_shortid': 1,
'groupe_abrev': 1,
'vote_position': 1
}):
depid = dpos['depute_shortid']
gp = dpos['groupe_abrev']
pos = dpos['vote_position']
if not pos in positions.keys():
positions[pos] = {}
if not gp in positions[pos].keys():
positions[pos][gp] = []
positions[pos][gp].append(depid)
scrutin['positions'] = positions
return json_response(scrutin)
def connectionsjson2():
gsel = [0, 1, 2, 3, 4, 5, 7]
gps = {
'FI': 0,
'REM': 1,
'MODEM': 2,
'LR': 3,
'GDR': 4,
'NG': 5,
'NI': 6,
'UAI': 7
}
counts = {}
depgp = dict((d['depute_shortid'], {
'img':
'http://www2.assemblee-nationale.fr/static/tribun/15/photos/' +
d['depute_uid'][2:] + '.jpg',
'gn':
d['groupe_abrev'],
'g':
gps[d['groupe_abrev']],
'n':
d['depute_nom']
}) for d in mdb.deputes.find({}, {
'depute_nom': 1,
'depute_uid': 1,
'depute_shortid': 1,
'groupe_abrev': 1,
'_id': None
}))
shortids = dict((d['depute_id'], d['depute_shortid'])
for d in mdb.deputes.find({}, {
'depute_id': 1,
'depute_shortid': 1,
'_id': None
}))
liens = []
allitems = []
for doc in mdb.documentsan.find({'signataires': {
'$ne': None
}}, {
'numero': 1,
'signataires': 1,
'_id': None
}):
if doc['signataires']:
sig1 = doc['signataires'][0]
counts[sig1] = counts.get(sig1, [])
if sig1 and depgp[sig1]['g'] in gsel:
for sig in doc['signataires'][1:]:
if sig:
counts[sig] = counts.get(sig, [])
if depgp[sig1]['g'] in gsel:
if depgp[sig1]['g'] != depgp[sig]['g']:
counts[sig1].append(sig)
counts[sig].append(sig1)
allitems.append(sig)
liens.append((sig1, sig))
if len(counts[sig1]) > 0:
allitems.append(sig1)
for amd in mdb.amendements.find({}, {'signataires_ids': 1, '_id': None}):
if not amd['signataires_ids']:
continue
sig1 = shortids.get(amd['signataires_ids'][0], None)
counts[sig1] = counts.get(sig1, [])
if sig1 and depgp[sig1]['g'] in gsel:
for sig in amd['signataires_ids'][1:]:
sig2 = shortids[sig]
if sig2:
counts[sig2] = counts.get(sig2, [])
if depgp[sig1]['g'] in gsel:
if depgp[sig1]['g'] != depgp[sig2]['g']:
counts[sig1].append(sig2)
counts[sig2].append(sig1)
allitems.append(sig2)
liens.append((sig1, sig2))
if len(counts[sig1]) > 0:
allitems.append(sig1)
r = {'nodes': [], 'links': []}
c = Counter(frozenset(x) for x in liens).items()
mx = max([x[1] for x in c])
for g in gps.keys():
r['nodes'].append({
'id': g,
'name': g,
'group': gps[g],
'groupe': True,
'count': 400
})
counts = dict((c, len(list(set(v)))) for c, v in counts.iteritems())
allitems = list(set([it for it in allitems if counts[it] > 0]))
for i, d in enumerate(allitems):
r['nodes'].append({
'img': depgp[d]['img'],
'id': d,
'name': "%s (%s)" % (depgp[d]['n'], depgp[d]['gn']),
'group': depgp[d]['g'],
'count': counts.get(d, 0)
})
for d in depgp.keys():
if d in allitems:
r['links'].append({
'source': depgp[d]['gn'],
'target': d,
'value': 2000
})
for l, n in Counter(frozenset(x) for x in liens).items():
if len(list(l)) == 2:
s, t = list(l)
if (depgp[s]['g'] in gsel or depgp[t]['g']
in gsel) and s in allitems and t in allitems:
r['links'].append({'source': s, 'target': t, 'value': n})
return json_response(r)
def test():
gp = {}
for d in mdb.deputes.find({},{'stats.election':1,'groupe_abrev':1}):
g = d['groupe_abrev']
if not g in gp.keys():
gp[g] = []
gp[g].append(d['stats']['election']['inscrits'])
moy = {}
import numpy
for g,v in gp.iteritems():
moy[g] = numpy.median(numpy.array(v))
return json_response(moy)
stats = dict(groupe=0,dissidence=0,abstention=0)
for s in mdb.scrutins.find({'scrutin_num':{'$nin':[404,405,406]}},{'scrutin_positions':1}):
spos = s['scrutin_positions']['REM']
for pos in ['pour','contre','abstention']:
if pos in ['pour','contre']:
if spos['position']!=pos:
stats['dissidence'] += spos.get(pos,0)
else:
stats['groupe'] += spos.get(pos,0)
else:
stats[pos] += spos.get(pos,0)
return json_response(stats)
from obsapis.tools import parse_content
import requests
from lxml import etree
url = "http://www.assemblee-nationale.fr/15/dossiers/dons_jours_repos_aidants_familiaux.asp"
#url = "http://www.assemblee-nationale.fr/15/dossiers/soutien_collectivites_accueil_gens_voyage.asp"
r = requests.get(url)
xml = parse_content(r.content)
print xml.xpath('//a[text()[contains(.,"Proposition de loi")]]/@href')
from stripogram import html2text, html2safehtml
doc = html2text(r.content,page_width=10000).decode('iso8859-1').split(u'\n\n')
start = False
bloc = ""
done = False
for i,l in enumerate(doc):
l = l.replace(u'1ère',u'première')
if 'Proposition de loi' in l:
start = True
done = False
elif len(l)<4:
if start == True:
start = False
done = True
if start:
bloc += l
if done:
print bloc
done = False
start = False
m1 = re.search(r'n. *([0-9]+).*d\xe9pos\xe9e? le ([0-9]+ [^ ]+ [0-9]+).*mis en ligne le ([0-9]+ [^ ]+ [0-9]+).*renvoy\xe9e? \xe0 (.*)',bloc)
#m1 = re.search(r'n° *([0-9]+).*d\xe9pos\xe9e? le ([0-9]+ [^ ]+ [0-9]+).*mis en ligne le ([0-9]+ [^ ]+ [0-9]+).*renvoy\xe9e? \xe0 (.*)',bloc)
#print m1
if m1:
print m1.groups()
bloc = ""
return "ok"
gps = {}
import datetime
return json_response(mdb.amendements.find_one({'auteurs':None}))
for d in mdb.deputes.find({'depute_actif':True},{'depute_ddn':1,'groupe_abrev':1,'groupe_libelle':1}):
age = (datetime.datetime.now()-datetime.datetime.strptime(d['depute_ddn'],'%d/%m/%Y')).days/365.25
gps[d['groupe_libelle']] = gps.get(d['groupe_libelle'],[]) + [age]
from numpy import median,average
for k,v in gps.iteritems():
print v
print "%s - moyenne : %.2f, mediane : %.2f" % (k,average(v),median(v))
#for i,d in enumerate(mdb.documentsan.find()):
# d['contenu'] = d['titre'] + d.get('contenu','')
# mdbrw.documentsan.update_one({'id':d['id']},{'$set':{'contenu':d['contenu']}})
# print i
#import_amendements()
return json_response(gps)
#mdbrw.travaux.remove({'sort':'44'})
#update_travaux()
#return json_response(list(mdb.travaux.find({'sort':'44'})))
#return json_response(mdb.questions.find_one({}))
#import_qag()
return json_response(mdb.travaux.find_one())
#return json_response(mdb.interventions.find({'itv_rapporteur':None})))
#return json_response(mdb.interventions.find({'itv_rapporteur':None}).distinct('itv_date'))
#return json_response(mdb.interventions.find({'$and':[{'itv_rapporteur':True},{'depute_shortid':'ericcoquerel'}]}))
from obsapis.controllers.admin.updates.interventions import update_stats_interventions
deppdp = {}
#return json_response(update_stats_interventions())
for pdp in update_stats_interventions():
dep = pdp['_id'].get('depute',None)
if dep:
if not dep in deppdp.keys():
deppdp[dep]= dict(n=0,rap=0)
deppdp[dep]['rap' if pdp['_id']['rapporteur'] else 'n'] += pdp['n']
return json_response(', '.join('%d. %s (%d)' % (i+1,d[0],d[1]['n']+d[1]['rap']) for i,d in enumerate(sorted(deppdp.items(),key=lambda x:x[1]['n']+x[1]['rap'],reverse=True))))
counts = {}
nbmembres = dict((g['groupe_abrev'],g['groupe_nbmembres']) for g in mdb.groupes.find({},{'groupe_abrev':1,'groupe_nbmembres':1}))
for q in mdb.questions.find({'groupe':{'$ne':None}},{'groupe':1}):
g = q['groupe']
if not g in counts.keys():
counts[g] = 0
counts[g] += 1
return json_response([ "%s (%d)" % (g,n/nbmembres[g]) for g,n in sorted(counts.items(),key=lambda x:x[1]/nbmembres[x[0]],reverse=True)])
col = []
for d in mdb.deputes.find({},{'depute_collaborateurs_hatvp':1,'_id':None,'depute_shortid':1}):
col.append((d['depute_shortid'],len(d.get('depute_collaborateurs_hatvp',[]))))
return json_response(sorted(col,key=lambda x:x[1],reverse=True)[:20])
import datetime
#mdbrw.deputes.update_one({'depute_shortid':'michelevictory'},{'$unset':{'stats.commissions':""}})
return json_response(mdb.deputes.find_one({},{'depute_hatvp':1}))
return json_response([d['depute_shortid'] for d in mdb.deputes.find({'stats.commissions.present':0.0})])
#{'$and': [{'depute_actif': True}, ]} [('stats.nonclasse', 1), ('stats.ranks.down.exprimes', 1)]
return json_response(list(d['depute_shortid'] for d in mdb.deputes.find({'depute_mandat_debut':{'$gte':datetime.datetime(2017,5,21)}},{'depute_shortid':1})))
return json_response([d['depute_shortid'] for d in mdb.deputes.find({'$and':[{'$or':[{'depute_actif': True},{'depute_shortid':'michelevictory'}]},{u'stats.positions.exprimes': {'$ne': None}}]}).sort([('stats.nonclasse', 1), ('stats.ranks.down.exprimes', 1)]).limit(5)])
for d in mdb.deputes.find({'depute_election':None}):
circo = d['depute_circo_id']
titulaire = mdb.deputes.find_one({'$and':[{'depute_circo_id':circo},{'depute_election':{'$ne':None}}]})
mdbrw.deputes.update_one({'depute_shortid':d['depute_shortid']},{'$set':{'depute_election':titulaire['depute_election']}})
return "oj"
#mdbrw.questions.update_many({'legislature':None},{'$set':{'legislature':15}})
#update_travaux()
#return json_response(mdb.interventions.find_one({}))
return json_response(list(q['itv_contenu_texte'] for q in mdb.interventions.find({'depute_shortid':'mariechristineverdierjouclas'})))
return json_response(mdb.travaux.distinct('type'))
#for a in mdb.amendements.find({'suppression':True},{'id':1}):
# mdbrw.travaux.update_many({'idori':a['id']},{'$set':{'suppression':True}})
#mdbrw.travaux.remove({'idori':'S-AMANR5L15PO419610B155N7'})
#mdbrw.amendements.remove({'id':{'$in':amdlist}})
#mdbrw.travaux.remove({'idori':{'$in':amdlist}})
#import_amendements()
return json_response(list(q['description'] for q in mdb.travaux.find({'$and':[{'auteur':{'$ne':False}},{'type':'QE'},{'depute':'francoisruffin'}]})))
return json_response(list(mdb.travaux.find({'idori':'S-AMANR5L15PO419610B155N7'})))
print mdb.travaux.count()
return json_response(list(t['description'] for t in mdb.travaux.find({'groupe':'FI'})))
#updateDeputesTravaux()
#importdocs()
#import_qag()
return json_response(mdb.deputes.find_one({'depute_shortid':'francoisruffin'}))
#importdocs()
#return json_response(mdb.documentsan.find_one({'$and':[{'typeid':'propositionderesolution'},{'cosignataires.id':'francoisruffin'}]}))
ops = []
pgroup = {'n':{'$sum':1}}
pgroup['_id'] = {'depute':'$auteurs'}
pipeline = [{'$match':{}}, {'$unwind':'$auteurs'},{"$group": pgroup }] #'scrutin_typedetail':'amendement'
return json_response(sum(d['n'] for d in mdb.documentsan.aggregate(pipeline)))
print len(list(mdb.documentsan.aggregate(pipeline))),mdb.documentsan.count()
#return json_response(mdb.amendements.find({'suppression':True},{'dispositif':1}).count())
#mdbrw.scrutins.update_one({'scrutin_num':324},{'$set':{'scrutin_liendossier':'http://www.assemblee-nationale.fr/15/dossiers/deuxieme_collectif_budgetaire_2017.asp'}})
#return json_util.dumps(list(mdb.amendements.find({'numAmend':'426'})))
#mdbrw.scrutins.update_one({'scrutin_num':1},{'$set':{'scrutin_groupe':'Gouvernement','scrutin_lientexte':[(u'déclaration de politique générale',
# 'http://www.gouvernement.fr/partage/9296-declaration-de-politique-generale-du-premier-ministre-edouard-philippe',
#
#mdbrw.votes.update_many({'scrutin_num':1},{'$set':{'scrutin_groupe':'Gouvernement'}})
#return json_response([ (d['depute_shortid'],d['depute_mandat_fin_cause']) for d in mdb.deputes.find({'depute_actif':False},{'depute_shortid':1,'depute_mandat_fin_cause':1,'_id':None})])
#mdbrw.scrutins.update_one({'scrutin_num':357},{'$set':{'scrutin_lientexte.0.1':'http://www.assemblee-nationale.fr/15/dossiers/jeux_olympiques_paralympiques_2024.asp#'}})
#return json_response(mdb.scrutins.find_one({'scrutin_num':357}))
return json_response(mdb.documentsan.distinct('type'))
# visuels
pgroup = {}
pgroup['n'] = {'$sum':1}
pgroup['_id'] = { 'depute':'$depute'}
pipeline = [{'$match':{'name':'visuelstat'}},{'$group':pgroup}]
vdeps = []
for g in mdb.logs.aggregate(pipeline):
_g = g['_id']['depute']
if _g != None:
vdeps.append((_g,g['n']))
return ", ".join([ "%s (%s)" % i for i in sorted(vdeps,key=lambda x:x[1],reverse=True)])
#updateDeputesContacts()
return json_util.dumps(mdb.deputes.find_one({'depute_shortid':'nicolelepeih'},{'depute_contacts':1,'_id':None}))
#importdocs()
#return json_util.dumps(list(mdb.logs.find({'name':'visuelstat'})))
mts = list(mdb.scrutins.find({ '$text': { '$search': "rejet" } },{'scrutin_groupe':1,'scrutin_fulldesc':1,'scrutin_sort':1,'_id':None}))
_mts = "\n".join([";".join([m.get('scrutin_groupe',''),m['scrutin_sort'],m['scrutin_fulldesc']]) for m in mts])
print _mts
return json_util.dumps(mdb.deputes.find_one({'depute_shortid':'thierrysolere'},{'stats':1,'_id':None}))
return json_util.dumps([(d['depute_nom'],
d['stats']['positions']['exprimes'],
d['stats']['votesamdements']['pctpour'],
d['depute_shortid']) for d in mdb.deputes.find({'groupe_abrev':'REM','stats.positions.exprimes':{'$gt':20}},{'depute_nom':1,'depute_shortid':1,'stats.positions.exprimes':1,'stats.votesamdements.pctpour':1}).sort([('stats.votesamdements.pctpour',-1)]).limit(20)])
from fuzzywuzzy import fuzz
sdesc = [(s['scrutin_dossier'],s['scrutin_dossierLibelle'],s['scrutin_desc'][20:]) for s in mdb.scrutins.find({'scrutin_dossier':{'$ne':'N/A'}},{'scrutin_dossier':1,'scrutin_dossierLibelle':1,'scrutin_desc':1,'_id':None})]
r = []
for s in mdb.scrutins.find({'scrutin_dossier':'N/A'},{'scrutin_desc':1,'_id':None,'scrutin_id':1}):
for dos,doslib,d in sdesc:
fz = fuzz.partial_ratio(s['scrutin_desc'][20:],d)
if fz>97:
r.append((s['scrutin_id'],dos,doslib))
break
return json_util.dumps(r)
return json_util.dumps([(d['depute_shortid'],d['depute_suppleant'],d['depute_mandat_fin']) for d in mdb.deputes.find({'depute_actif':False})])
return json_util.dumps(list(mdb.amendements.find({'sort':u"Adopt\u00e9","signataires_groupes":{'$elemMatch':{'$eq':'FI'}}},{'_id':None,'numInit':1,'numAmend':1})))
def votes():
scrutins_data = use_cache('scrutins_data_new',
lambda: getScrutinsData(),
expires=300)
nb = int(request.args.get('itemsperpage', '25'))
page = int(request.args.get('page', '1')) - 1
groupe = request.args.get('groupe', request.args.get('group', None))
search = request.args.get('requete', request.args.get('query', ''))
scrutingroupe = request.args.get('scrutingroupe', None)
scrutin = request.args.get('scrutin', None)
csp = request.args.get('csp', None)
age = request.args.get('age', None)
region = request.args.get('region', None)
depute = request.args.get('depute', None)
position = request.args.get('position', None)
dissidence = int(request.args.get('dissidence', '0'))
skip = nb * page
filters = []
if scrutingroupe:
filters.append({'scrutin_groupe': scrutingroupe})
if dissidence:
filters.append({'vote_dissident': True})
if position:
filters.append({'vote_position': position})
if depute:
filters.append({'depute_shortid': depute})
if csp:
filters.append({'depute_csp': csp})
if age:
filters.append({'depute_classeage': age})
if groupe:
filters.append({'groupe_abrev': groupe})
if region:
filters.append({'depute_region': region})
if scrutin:
try:
scrutin = int(scrutin)
except:
pass
filters.append({'scrutin_num': scrutin})
if search:
filters.append({'$text': {'$search': '"' + search + '"'}})
if len(filters) == 0:
vote_filter = {}
elif len(filters) == 1:
vote_filter = filters[0]
else:
vote_filter = {'$and': filters}
votes = []
import re
for v in mdb.votes.find(vote_filter).sort('scrutin_num',
-1).skip(skip).limit(nb):
v['scrutin_sort'] = scrutins_data[v['scrutin_num']]['sort']
if scrutins_data[v['scrutin_num']]['urlAmendement']:
pass
#v['scrutin_desc'] = re.sub(r'([0-9]+)',r'<a target="_blank" href="'+scrutins_data[v['scrutin_num']]['urlAmendement']+r'">\1</a>',v['scrutin_desc'],1)
for i, lien in enumerate(
scrutins_data[v['scrutin_num']]['scrutin_lientexte']):
v['scrutin_desc'] = v['scrutin_desc'].replace(
lien[0], 'LIEN%d' % i)
for i, lien in enumerate(
scrutins_data[v['scrutin_num']]['scrutin_lientexte']):
v['scrutin_desc'] = v['scrutin_desc'].replace(
'LIEN%d' % i, '<a target="_blank" href="' + lien[1] + r'">' +
lien[0] + '</a>')
v['scrutin_dossierLibelle'] = v['scrutin_dossierLibelle'].replace(
u'\u0092', "'") # pb apostrophe
votes.append(v)
def countItems():
rcount = mdb.votes.find(vote_filter).count()
return {'totalitems': rcount}
cachekey = u"vot%s_%s_%s_%s_%s_%s_%s_%s_%s_%s" % (
depute, position, scrutingroupe, dissidence, scrutin, age,
csp if csp else csp, groupe, search, region if region else region)
counts = use_cache(cachekey, lambda: countItems(), expires=3600)
regx = re.compile(search, re.IGNORECASE)
if search:
for v in votes:
repl = regx.subn('<strong>' + search + '</strong>',
v['scrutin_desc'])
if repl[1]:
v['scrutin_desc'] = repl[0]
import math
nbpages = int(math.ceil(float(counts['totalitems']) / nb))
result = dict(nbitems=len(votes),
nbpages=nbpages,
currentpage=1 + page,
itemsperpage=nb,
items=votes,
**counts)
return json_response(result)
def hatvp():
from obsapis.controllers.admin.imports.hatvp import update_hatvp
update_hatvp()
return json_response(mdb.deputes.find_one({},{'depute_collaborateurs_hatvp':1}))
def view_updateDeputesContacts():
return json_response(updateDeputesContacts())
def obsgouv_getdata():
return json_response(import_obsgouv_gdoc())
def longs():
return json_response(maxis())
def view_travaux():
amd_sorts = {
'retire': u'Retiré',
'tombe': u'Tombé',
'adopte': u'Adopté',
'rejete': u'Rejeté',
'nonsoutenu': u'Non soutenu',
'nonrenseigne': u'Non renseigné'
}
nb = int(request.args.get('itemsperpage', '25'))
page = int(request.args.get('page', '1')) - 1
search = request.args.get('requete', request.args.get('query', ''))
depute = request.args.get('depute', None)
groupe = request.args.get('groupe', None)
cosig = request.args.get('cosignataire', None)
sort = request.args.get('sort', None)
ttype = request.args.get('type', None)
asupp = request.args.get('suppression', None)
skip = nb * page
filters = []
if 'suppression' in request.args.keys() and ttype == 'amendement':
filters.append({'suppression': (asupp not in ("", "0"))})
if sort and ttype == 'amendement' and sort in amd_sorts.keys():
filters.append({'sort': amd_sorts[sort]})
if depute:
filters.append({'depute': depute})
filters.append({'auteur': False if cosig else {'$in': [None, True]}})
elif groupe:
filters.append({'groupe': groupe})
else:
filters.append({'depute': None})
if ttype == 'question':
filters.append({'type': {'$in': ['QG', 'QE', 'QOSD']}})
elif ttype == 'amendement':
filters.append({'type': 'amendement'})
elif ttype in [
'QG', 'QE', 'QOSD', 'propositiondeloi', 'propositionderesolution',
'rapportdinformation', 'rapport', 'avis', 'projetdeloi'
]:
filters.append({'type': ttype})
elif ttype == 'document':
filters.append({
'type': {
'$nin': ['propositiondeloi', 'QG', 'QE', 'QOSD', 'amendement']
}
})
if search:
search = '"' + search + '"'
def searchText():
txt_amd = [
a['id']
for a in mdb.amendements.find({'$text': {
'$search': search
}})
]
txt_que = [
q['id']
for q in mdb.questions.find({'$text': {
'$search': search
}})
]
txt_doc = [
d['id']
for d in mdb.documentsan.find({'$text': {
'$search': search
}})
]
return txt_amd + txt_que + txt_doc
cachekey = u"trvtxt%s" % (search)
ids = use_cache(cachekey, lambda: searchText(), expires=3600)
filters.append({'idori': {'$in': ids}})
def makefilter(f):
if len(f) == 0:
mf = {}
elif len(f) == 1:
mf = f[0]
else:
mf = {'$and': f}
return mf
tfilter = makefilter(filters)
print filters, tfilter
travaux = list(
mdb.travaux.find(tfilter).sort('date', -1).skip(skip).limit(nb))
print travaux
def countItems():
rcount = mdb.travaux.find(tfilter).count()
return {'totalitems': rcount}
cachekey = u"trv%s_%s_%s_%s_%s_%s" % (depute, groupe, search, ttype, sort,
cosig)
counts = use_cache(cachekey, lambda: countItems(), expires=3600)
import math
for t in travaux:
t['description'] = t['description'].replace(u'\u0092', "'").replace(
u'\u2019', "'")
t['dossier'] = t['dossier'].replace(u'\u0092',
"'").replace(u'\u2019', "'")
nbpages = int(math.ceil(float(counts['totalitems']) / nb))
result = dict(nbitems=len(travaux),
nbpages=nbpages,
currentpage=1 + page,
itemsperpage=nb,
items=travaux,
**counts)
return json_response(result)
