def get_data_gvt_compo(row):
"""
FUNCTION
get a string (row from csv file) and put its content into an instance of GvtCompo
PARAMETERS
row: row from the csv file [row object]
RETURN
instance: instance of the model with the extracted data [GvtCompo model instance]
msg: id of the row, used to display an error message [string]
exist: True if the instance already exists, False otherwise [boolean]
"""
#used to identify the row
ids_row={}
ids_row["start_date"]=date_string_to_iso(row[0].strip())
ids_row["end_date"]=date_string_to_iso(row[1].strip())
gvt_compos=row[2].split(":")
ids_row["country"]=Country.objects.get(country_code=gvt_compos[0].strip())
#get instance or create instance if does not already exist
instance, created = GvtCompo.objects.get_or_create(**ids_row)
exist=not created
#if the row didn't exist, save the related models
#2009-avril-09 2010-juin-27 CZ: none
if not exist and gvt_compos[1].strip()!="none":
#split party and party_family
partys_temp=gvt_compos[1].split(",")
partys=[]
partys_family=[]
for party in partys_temp:
string=re.findall("[^()]+", party)
partys.append(string[0].strip())
partys_family.append(string[1].strip())
#save party and party_family
data={}
data["country"]=ids_row["country"]
for index in xrange(len(partys)):
#save party
data["party"]=save_get_object(Party, {"party": partys[index]})
instance.party.add(data["party"])
data["party_family"]=partys_family[index]
if data["party_family"]=="Conservative/ Christian Democracy":
data["party_family"]="Conservative/Christian Democracy"
#save party_family
try:
save_get_object(PartyFamily, data)
except Exception, e:
#inconsistency in data -> the source file must be fixed before saving the data again
print "get_data_gvt_compo exception", e
#~ msg="Cant save the row"
msg=str("Can't save the row " + str(ids_row) + ". '" + data["country"].country_code + "'+'" + data["party"].party + "' can't be associated to '" + str(data["party_family"]) + "' because they are already associated to '" + PartyFamily.objects.get(country=data["country"], party=data["party"]).party_family + "' !")
print msg
exist=True
instance.delete()
return instance, msg, exist
def get_save_act_ids(acts_ids):
"""
FUNCTION
get and save retrieved ids from eurlex and oeil in the the database (import or update act ids)
PARAMETERS
acts_ids: list of act ids to save (monthly summary ids) [list of dictionaries of releve_* variables]
RETURN
None
"""
fields={}
for releves in acts_ids:
act=Act.objects.get(**releves)
act_ids={}
#get or create the act ids instance for each source
for src in ["index", "eurlex", "oeil"]:
act_ids[src]=save_get_object(ActIds, {"act": act, "src": src})
#get ids
#eurlex
act_ids["eurlex"].__dict__.update(get_ids.check_get_ids_eurlex(act_ids["index"].no_celex.strip()))
#~ act_ids["eurlex"].__dict__.update(get_ids.check_get_ids_eurlex(act_ids["index"].no_celex))
#oeil
act_ids["oeil"].__dict__.update(get_ids.check_get_ids_oeil(act_ids["index"].no_unique_type, str(act_ids["index"].no_unique_annee), act_ids["index"].no_unique_chrono))
#~ act_ids["oeil"].__dict__.update(get_ids.check_get_ids_oeil(act_ids["index"].no_unique_type, str(act_ids["index"].no_unique_annee), act_ids["index"].no_unique_chrono))
#save ids
for src in act_ids:
act_ids[src].save()
def save_2_tables(row, fields, fields_fk, party_family=""):
"""
FUNCTION
open a csv file and save its variables variables in 2 joined tables:
code_sect_rep/config_cons
code_sect_rep/code_agenda
name/country
name/party
dg/dg_sigle
PARAMETERS
row: row to be processed (from the csv file) [list]
fields: model name, field name and position in file (num colomn) [list]
fields_fk: model name, field name and position in file (num colomn) for each foreign key [list of lists]
party_family: model name, field name and position in file (num colomn) for the party_family variable
RETURN
msg: row main variable, used to display a success or error [string]
exist: True if the instance already exists, False otherwise [boolean]
"""
#~ field=["Person", "name", <position>]
#~ fields_fk=[["Country", "country", <position>], ["Party", "party", <position>]]
src=""
#original lists are modified a few lines after in the program and should not be (the position is updated with the value of the field) -> use a copy of the list instead with deepcopy
field=copy.deepcopy(fields)
field_fk=copy.deepcopy(fields_fk)
#the position becomes the value (in order to call the save_get_field_and_fk function)
field[2]=row[field[2]].strip()
#if import of responsibles (eurlex) -> change name format of person: "LASTNAME, Firstname" -> "LASTNAME Firstname"
if field[1]=="name":
field[2]=field[2].replace(",","")
src="resp"
#delete the last dot for the code_sect if any
elif field[1]=="code_sect":
field[2]=field[2].rstrip(".")
msg=var_name_data.var_name[field[1]]+"="+field[2]
#for each foreign key
for fk in field_fk:
fk[2]=row[fk[2]].strip()
msg+=", "+fk[1]+"="+fk[2]
#save the field and its foreign keys
instance, exist=save_get_field_and_fk(field, field_fk, src)
#if dg import: we have to save dg_nb too
if field[1]=="dg":
dg_nb=row[2].strip()
#if the dg has an associate number
if dg_nb!="":
#save or get (if already exists) the dg_nb instance
dg_nb_instance=save_get_object(DGNb, {"dg_nb": dg_nb})
#link it to the current dg
instance.dg_nb.add(dg_nb_instance)
#import party_family
elif party_family!="":
party_family=row[party_family[2]].strip()
#save the party family
try:
PartyFamily.objects.create(party=instance.party, country=instance.country, party_family=party_family)
except Exception, e:
print "party_family already exists for this party and country"
msg+=", party_family="+party_family
