def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
city_code = options['city']
if not city_code:
raise Exception("Missing city parameter")
mapper = FLMapper()
city = mapper.get_city(city_code)
year = options['year']
preventivo_node = Voce.objects.get(slug='preventivo-spese-spese-per-investimenti-funzioni')
t = Territorio.objects.get(cod_finloc=city)
y = year
valori_bilancio = ValoreBilancio.objects.filter(territorio=t, anno=y).\
filter(voce__in=preventivo_node.get_descendants(include_self=True))
valori_bilancio_dict = dict(
(v['pk'], {'valore': v['valore'], 'valore_procapite': v['valore_procapite']})
for v in valori_bilancio.values('pk', 'valore', 'valore_procapite')
)
bilanci_tree = make_tree(preventivo_node, valori_bilancio_dict)
bilanci_tree.emit()
def set_cities(self, cities_codes, start_from):
# set considered cities
mapper = FLMapper()
if not cities_codes:
if start_from:
cities_codes = 'all'
all_cities = mapper.get_cities(cities_codes, logger=self.logger)
try:
cities_finloc = all_cities[all_cities.index(start_from):]
except ValueError:
raise Exception("Start-from city not found in cities complete list, use name--cod_finloc. "
"Example: ZUNGRI--4181030500")
else:
self.logger.info("Processing cities starting from: {0}".format(start_from))
else:
raise Exception("Missing cities parameter or start-from parameter")
else:
cities_finloc = mapper.get_cities(cities_codes, logger=self.logger)
finloc_numbers = [c[-10:] for c in cities_finloc]
slug_list = []
for numb in finloc_numbers:
slug_list.append(Territorio.objects.get(territorio="C", cod_finloc__endswith=numb).slug)
self.cities = Territorio.objects.filter(territorio="C", slug__in=slug_list)
def set_cities(self, cities_codes, start_from):
# set considered cities
mapper = FLMapper()
if not cities_codes:
if start_from:
cities_codes = 'all'
all_cities = mapper.get_cities(cities_codes,
logger=self.logger)
try:
cities_finloc = all_cities[all_cities.index(start_from):]
except ValueError:
raise Exception(
"Start-from city not found in cities complete list, use name--cod_finloc. "
"Example: ZUNGRI--4181030500")
else:
self.logger.info(
"Processing cities starting from: {0}".format(
start_from))
else:
raise Exception(
"Missing cities parameter or start-from parameter")
else:
cities_finloc = mapper.get_cities(cities_codes, logger=self.logger)
finloc_numbers = [c[-10:] for c in cities_finloc]
slug_list = []
for numb in finloc_numbers:
slug_list.append(
Territorio.objects.get(territorio="C",
cod_finloc__endswith=numb).slug)
self.cities = Territorio.objects.filter(territorio="C",
slug__in=slug_list)
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
mapper = FLMapper()
dryrun = options['dryrun']
delete = options['delete']
territori_type = options['territori_type']
###
# cities
###
cities_codes = options['cities']
if territori_type != '' and cities_codes != '':
self.logger.error("Cannot specify both territori_type and cities. Choose one")
return
self.apidomain = options['apidomain']
if options['auth']:
(user, pwd) = options['auth'].split(",")
self.baseurl = "http://{0}:{1}@{2}".format(user, pwd, self.apidomain)
else:
self.baseurl = "http://{0}".format(self.apidomain)
# sets cities_to_process: if territorio_type is set uses the macro-category: all, capoluoghi, other
# otherwise uses cities: single cities codes
if territori_type != '':
cities_to_process = self.get_territori_from_types(territori_type)
elif cities_codes != '':
cities = mapper.get_cities(cities_codes)
cities_to_process = self.get_territori_from_finloc(cities)
else:
self.logger.error("Must specify territory_type or cities")
return
self.logger.info(u"Start charges import with dryrun: {0}".format(dryrun))
if delete:
self.logger.info(u"Deleting all Incarico for considered cities")
Incarico.objects.filter(territorio__in=cities_to_process).delete()
self.logger.info(u"Done.")
self.process_cities(cities_to_process, dryrun)
self.log_errors()
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
# get the timestamp to ensure the document will be written in couchdb, this is a workaround for a bug,
# see later comment
timestamp = time.time()
dryrun = options['dryrun']
no_patch = options['no_patch']
if options['append'] is True:
self.logger = logging.getLogger('management_append')
# type option, different values are accepted:
# v, V, voce, Voce, VOCE or
# t, T, titolo, Titolo, TITOLO, Title
if 'type' not in options:
raise Exception("Missing type parameter")
if options['type'].lower()[0] not in ('v', 't'):
raise Exception("Wrong type parameter value (voce|titolo)")
translation_type = options['type'][0].lower()
force_google = options['force_google']
skip_existing = options['skip_existing']
design_documents = options['design_documents']
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing city parameter")
self.logger.info("Opening Lista Comuni")
mapper = FLMapper()
cities = mapper.get_cities(cities_codes)
if not cities:
self.logger.critical("Cities cannot be null!")
exit()
if cities_codes.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities))
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year) + 1)
else:
years = [int(y.strip()) for y in years.split(",") if
settings.APP_START_YEAR <= int(y.strip()) <= settings.APP_END_YEAR]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Processing years: {0}".format(years))
###
# couchdb connections
###
couchdb_server_alias = options['couchdb_server']
# set couch source and destination names
couchdb_source_name=''
couchdb_dest_name=''
if translation_type == 't':
couchdb_source_name = settings.COUCHDB_RAW_NAME
couchdb_dest_name = settings.COUCHDB_NORMALIZED_TITOLI_NAME
elif translation_type == 'v':
couchdb_source_name = settings.COUCHDB_NORMALIZED_TITOLI_NAME
couchdb_dest_name = settings.COUCHDB_NORMALIZED_VOCI_NAME
else:
self.logger.critical(u"Translation type not accepted:{}".format(translation_type))
exit()
if couchdb_server_alias not in settings.COUCHDB_SERVERS:
raise Exception("Unknown couchdb server alias.")
self.logger.info("Connecting to server: {}".format(couchdb_server_alias, ))
self.logger.info("Connecting source db: {}".format(couchdb_source_name))
try:
self.couchdb_source = couch.connect(
couchdb_source_name,
couchdb_server_settings=settings.COUCHDB_SERVERS[couchdb_server_alias]
)
except ResourceNotFound:
self.logger.error("Could not find source db. Quitting")
return
self.logger.info("Connecting to destination db: {0}".format(couchdb_dest_name))
couchdb_dest_settings = settings.COUCHDB_SERVERS[couchdb_server_alias]
try:
self.couchdb_dest = couch.connect(
couchdb_dest_name,
couchdb_server_settings=couchdb_dest_settings
)
except ResourceNotFound:
self.logger.error("Could not find destination db. Quitting")
return
self.logger.info("Compact destination db...")
self.couchdb_dest.compact()
self.logger.info("Done")
# create couch bulk writer
self.cbw = couch.CouchBulkWriter(logger=self.logger, couchdb_dest=self.couchdb_dest)
###
# Mapping files from gdoc
###
# connect to google account and fetch tree mapping and simple tree structure
normalized_map = gdocs.get_normalized_map(translation_type, n_header_lines=2, force_google=force_google)
normalized_titoli_sheet = {'preventivo': [row[2] for row in normalized_map['preventivo']],
'consuntivo': [row[2] for row in normalized_map['consuntivo']],
}
normalized_voci_sheet = {'preventivo': [(row[2], row[3]) for row in normalized_map['preventivo']],
'consuntivo': [(row[2], row[3]) for row in normalized_map['consuntivo']],
}
# copying design documents
if design_documents:
self.logger.info(u"Copying design documents")
source_design_docs = self.couchdb_source.view("_all_docs",
startkey="_design/", endkey="_design0",
include_docs=True
)
for row in source_design_docs.rows:
source_design_doc = row.doc
self.logger.info(u" document id: ".format(source_design_doc.id))
destination_document = {'_id': source_design_doc.id}
destination_document['language'] = source_design_doc['language']
destination_document['views'] = source_design_doc['views']
if not dryrun:
self.couchdb_dest.save(destination_document)
for city in cities:
self.logger.info(u"Updating {}".format(city))
for year in years:
doc_id = u"{0}_{1}".format(year, city)
if doc_id in self.couchdb_dest and skip_existing:
self.logger.info("Skipping city of {}, as already existing".format(city))
continue
# identify source document or skip
source_document = self.couchdb_source.get(doc_id)
if source_document is None:
self.logger.warning('"{0}" doc_id not found in source db. skipping.'.format(doc_id))
continue
# create destination document, to REPLACE old one
# NB: the useless timestamps serves the only function to work around a bug in COUCHDB that
# if the written doc is exactly the same as the new doc then it will not be written
destination_document = {'_id': doc_id, 'useless_timestamp': timestamp}
# if a doc with that id already exists on the destination document, gets the _rev value
# and insert it in the dest. document.
# this avoids document conflict on writing
# otherwise you should delete the old doc before writing the new one
old_destination_doc = self.couchdb_dest.get(doc_id, None)
if old_destination_doc:
revision = old_destination_doc.get('_rev', None)
if revision:
destination_document['_rev'] = revision
self.logger.debug("Adds rev value to doc:{}".format(doc_id))
for bilancio_type in ['preventivo', 'consuntivo']:
if bilancio_type in source_document.keys():
bilancio_object = source_document[bilancio_type]
destination_document[bilancio_type] = {}
for quadro_name, quadro_object in bilancio_object.iteritems():
destination_document[bilancio_type][quadro_name] = {}
for titolo_name, titolo_object in quadro_object.iteritems():
if translation_type == 't':
# for each titolo, apply translation_map, if valid
try:
idx = normalized_titoli_sheet[bilancio_type].index(titolo_name)
titolo_name = normalized_map[bilancio_type][idx][3]
except ValueError:
pass
# create dest doc titolo dictionary
destination_document[bilancio_type][quadro_name][titolo_name] = {}
# copy meta
if 'meta' in titolo_object.keys():
destination_document[bilancio_type][quadro_name][titolo_name]['meta'] = {}
destination_document[bilancio_type][quadro_name][titolo_name]['meta'] = \
titolo_object['meta']
# copy data (normalize voci if needed)
if 'data' in titolo_object.keys():
destination_document[bilancio_type][quadro_name][titolo_name]['data'] = {}
if translation_type == 'v':
# voci translation
for voce_name, voce_obj in titolo_object['data'].iteritems():
# voci are always translated into lowercase, unicode strings
# trailing dash is removed, if present
voce_name = unicode(voce_name.lower())
if voce_name.find("- ") == 0:
voce_name = voce_name.replace("- ", "")
# for each voce, apply translation_map, if valid
try:
idx = normalized_voci_sheet[bilancio_type].index(
(titolo_name, voce_name))
voce_name = normalized_map[bilancio_type][idx][4]
except ValueError:
pass
# create voice dictionary with normalized name
destination_document[bilancio_type][quadro_name][titolo_name]['data'][
voce_name] = {}
destination_document[bilancio_type][quadro_name][titolo_name]['data'][
voce_name] = voce_obj
else:
# copy all voci in data, with no normalization
destination_document[bilancio_type][quadro_name][titolo_name]['data'] = \
titolo_object['data']
if not dryrun:
# write doc to couchdb dest
ret = self.cbw.write(destination_document)
if ret is False:
email_utils.send_notification_email(msg_string='couch translate keys has encountered problems')
self.logger.critical("Write critical problem. Quit")
exit()
if not dryrun:
# if the buffer in CBW is non-empty, flushes the docs to the db
ret = self.cbw.close()
if ret is False:
email_utils.send_notification_email(msg_string='couch translate keys has encountered problems')
self.logger.critical("Write critical problem. Quit")
exit()
self.logger.info("Compact destination db...")
self.couchdb_dest.compact()
self.logger.info("Done compacting")
if not dryrun and couchdb_dest_name == settings.COUCHDB_NORMALIZED_VOCI_NAME and settings.INSTANCE_TYPE == 'production' or settings.INSTANCE_TYPE == 'staging' and no_patch is False:
self.logger.info(u"============Run patch 2013 for consuntivo======================")
call_command('consuntivo_13_patch', verbosity=2, interactive=False)
email_utils.send_notification_email(msg_string="Couch translate key has finished")
self.logger.info("Finished couch translate keys")
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
self.dryrun = options['dryrun']
input_file_path = options['input_file']
if options['append'] is True:
self.logger = logging.getLogger('management_append')
# open input file
try:
bilancio = BeautifulSoup(open(input_file_path), "xml")
except IOError:
self.logger.error("File {0} not found".format(input_file_path))
return
# get finloc, year, tipo bilancio
certificato = bilancio.certificato
self.anno = int(certificato['anno'])
tipo_certificato_code = certificato['tipoCertificato']
valuta = certificato['tipoValuta']
if valuta.lower() != 'e':
self.logger.error("Valuta is not EURO")
return
if tipo_certificato_code == "C":
self.tipo_certificato = 'consuntivo'
else:
self.tipo_certificato = 'preventivo'
self.rootnode_slug = self.tipo_certificato
# identifies the Comune from the finloc code
codiceente = certificato['codiceEnte']
mapper = FLMapper()
codfinloc = mapper.get_cities(codiceente)[0]
try:
self.territorio = Territorio.objects.get(cod_finloc=codfinloc)
self.logger.info(u"Comune: {0}".format(self.territorio.cod_finloc))
except ObjectDoesNotExist:
self.logger.error(u"Comune with codfinloc:{0} is not present on DB. Quitting".format(codfinloc))
return
# from finloc extracts only the numeric part, removing the eventual name
numeric_finloc = self.territorio.cod_finloc
if "--" in numeric_finloc:
split_finloc = numeric_finloc.split("--")
numeric_finloc = split_finloc[1]
# checks if Codice Voce (xml2slug mapping) is present in the DB for the current bilancio type
if CodiceVoce.get_bilancio_codes(anno=self.anno, tipo_certificato=self.tipo_certificato).count() == 0:
self.logger.error("Xml mapping is not present for current bilancio type. Run xml2slug command first")
exit()
# import bilancio data into Postgres db, calculate per capita values
self.import_bilancio(bilancio)
# log errors in a file, if any
self.log_errors()
##
# adds bilancio to the source table to mark this bilancio as coming from Comune xml source
##
import_data = {'territorio': self.territorio, 'anno': self.anno, 'tipologia': self.tipo_certificato}
if not self.dryrun:
try:
ImportXmlBilancio.objects.get(**import_data).delete()
except ObjectDoesNotExist:
pass
import_bilancio = ImportXmlBilancio(**import_data)
if not self.dryrun:
import_bilancio.save()
##
# Compute Indicators if bilancio is Consuntivo
##
if self.tipo_certificato == 'consuntivo':
self.logger.info(u"Compute indicators for Comune: {0}, year: {1}, tipo_bilancio: {2}...".format(
self.territorio.denominazione, self.anno, self.tipo_certificato))
if not self.dryrun:
call_command('indicators', verbosity=2, years=str(self.anno), cities=numeric_finloc, indicators='all',
interactive=False)
##
# Updates open data
##
# copy xml file to open data folder
xml_path = os.path.join(settings.OPENDATA_XML_ROOT, self.territorio.cod_finloc, certificato['anno'])
destination_file = xml_path+'/'+self.tipo_certificato + ".xml"
if not self.dryrun:
if not os.path.exists(xml_path):
os.makedirs(xml_path)
shutil.copyfile(src=input_file_path, dst=destination_file)
self.logger.info("Copied Xml file to {}".format(destination_file))
self.logger.info("** Update open data zip file for {} **".format(self.territorio.denominazione))
# updates open data zip file for considered Comune
if not self.dryrun:
years = "{0}-{1}".format(settings.APP_START_YEAR, settings.APP_END_YEAR)
call_command('update_opendata', verbosity=2, years=years, cities=numeric_finloc, compress=True,
interactive=False)
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
# get the timestamp to ensure the document will be written in couchdb, this is a workaround for a bug,
# see later comment
timestamp = time.time()
dryrun = options['dryrun']
no_patch = options['no_patch']
if options['append'] is True:
self.logger = logging.getLogger('management_append')
# type option, different values are accepted:
# v, V, voce, Voce, VOCE or
# t, T, titolo, Titolo, TITOLO, Title
if 'type' not in options:
raise Exception("Missing type parameter")
if options['type'].lower()[0] not in ('v', 't'):
raise Exception("Wrong type parameter value (voce|titolo)")
translation_type = options['type'][0].lower()
force_google = options['force_google']
skip_existing = options['skip_existing']
design_documents = options['design_documents']
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing city parameter")
self.logger.info("Opening Lista Comuni")
mapper = FLMapper()
cities = mapper.get_cities(cities_codes)
if not cities:
self.logger.critical("Cities cannot be null!")
exit()
if cities_codes.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities))
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year) + 1)
else:
years = [
int(y.strip()) for y in years.split(",")
if settings.APP_START_YEAR <= int(y.strip()) <=
settings.APP_END_YEAR
]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Processing years: {0}".format(years))
###
# couchdb connections
###
couchdb_server_alias = options['couchdb_server']
# set couch source and destination names
couchdb_source_name = ''
couchdb_dest_name = ''
if translation_type == 't':
couchdb_source_name = settings.COUCHDB_RAW_NAME
couchdb_dest_name = settings.COUCHDB_NORMALIZED_TITOLI_NAME
elif translation_type == 'v':
couchdb_source_name = settings.COUCHDB_NORMALIZED_TITOLI_NAME
couchdb_dest_name = settings.COUCHDB_NORMALIZED_VOCI_NAME
else:
self.logger.critical(
u"Translation type not accepted:{}".format(translation_type))
exit()
if couchdb_server_alias not in settings.COUCHDB_SERVERS:
raise Exception("Unknown couchdb server alias.")
self.logger.info("Connecting to server: {}".format(
couchdb_server_alias, ))
self.logger.info(
"Connecting source db: {}".format(couchdb_source_name))
try:
self.couchdb_source = couch.connect(
couchdb_source_name,
couchdb_server_settings=settings.
COUCHDB_SERVERS[couchdb_server_alias])
except ResourceNotFound:
self.logger.error("Could not find source db. Quitting")
return
self.logger.info(
"Connecting to destination db: {0}".format(couchdb_dest_name))
couchdb_dest_settings = settings.COUCHDB_SERVERS[couchdb_server_alias]
try:
self.couchdb_dest = couch.connect(
couchdb_dest_name,
couchdb_server_settings=couchdb_dest_settings)
except ResourceNotFound:
self.logger.error("Could not find destination db. Quitting")
return
self.logger.info("Compact destination db...")
self.couchdb_dest.compact()
self.logger.info("Done")
# create couch bulk writer
self.cbw = couch.CouchBulkWriter(logger=self.logger,
couchdb_dest=self.couchdb_dest)
###
# Mapping files from gdoc
###
# connect to google account and fetch tree mapping and simple tree structure
normalized_map = gdocs.get_normalized_map(translation_type,
n_header_lines=2,
force_google=force_google)
normalized_titoli_sheet = {
'preventivo': [row[2] for row in normalized_map['preventivo']],
'consuntivo': [row[2] for row in normalized_map['consuntivo']],
}
normalized_voci_sheet = {
'preventivo':
[(row[2], row[3]) for row in normalized_map['preventivo']],
'consuntivo':
[(row[2], row[3]) for row in normalized_map['consuntivo']],
}
# copying design documents
if design_documents:
self.logger.info(u"Copying design documents")
source_design_docs = self.couchdb_source.view("_all_docs",
startkey="_design/",
endkey="_design0",
include_docs=True)
for row in source_design_docs.rows:
source_design_doc = row.doc
self.logger.info(u" document id: ".format(
source_design_doc.id))
destination_document = {'_id': source_design_doc.id}
destination_document['language'] = source_design_doc[
'language']
destination_document['views'] = source_design_doc['views']
if not dryrun:
self.couchdb_dest.save(destination_document)
for city in cities:
self.logger.info(u"Updating {}".format(city))
for year in years:
doc_id = u"{0}_{1}".format(year, city)
if doc_id in self.couchdb_dest and skip_existing:
self.logger.info(
"Skipping city of {}, as already existing".format(
city))
continue
# identify source document or skip
source_document = self.couchdb_source.get(doc_id)
if source_document is None:
self.logger.warning(
'"{0}" doc_id not found in source db. skipping.'.
format(doc_id))
continue
# create destination document, to REPLACE old one
# NB: the useless timestamps serves the only function to work around a bug in COUCHDB that
# if the written doc is exactly the same as the new doc then it will not be written
destination_document = {
'_id': doc_id,
'useless_timestamp': timestamp
}
# if a doc with that id already exists on the destination document, gets the _rev value
# and insert it in the dest. document.
# this avoids document conflict on writing
# otherwise you should delete the old doc before writing the new one
old_destination_doc = self.couchdb_dest.get(doc_id, None)
if old_destination_doc:
revision = old_destination_doc.get('_rev', None)
if revision:
destination_document['_rev'] = revision
self.logger.debug(
"Adds rev value to doc:{}".format(doc_id))
for bilancio_type in ['preventivo', 'consuntivo']:
if bilancio_type in source_document.keys():
bilancio_object = source_document[bilancio_type]
destination_document[bilancio_type] = {}
for quadro_name, quadro_object in bilancio_object.iteritems(
):
destination_document[bilancio_type][
quadro_name] = {}
for titolo_name, titolo_object in quadro_object.iteritems(
):
if translation_type == 't':
# for each titolo, apply translation_map, if valid
try:
idx = normalized_titoli_sheet[
bilancio_type].index(titolo_name)
titolo_name = normalized_map[
bilancio_type][idx][3]
except ValueError:
pass
# create dest doc titolo dictionary
destination_document[bilancio_type][
quadro_name][titolo_name] = {}
# copy meta
if 'meta' in titolo_object.keys():
destination_document[bilancio_type][
quadro_name][titolo_name]['meta'] = {}
destination_document[bilancio_type][quadro_name][titolo_name]['meta'] = \
titolo_object['meta']
# copy data (normalize voci if needed)
if 'data' in titolo_object.keys():
destination_document[bilancio_type][
quadro_name][titolo_name]['data'] = {}
if translation_type == 'v':
# voci translation
for voce_name, voce_obj in titolo_object[
'data'].iteritems():
# voci are always translated into lowercase, unicode strings
# trailing dash is removed, if present
voce_name = unicode(
voce_name.lower())
if voce_name.find("- ") == 0:
voce_name = voce_name.replace(
"- ", "")
# for each voce, apply translation_map, if valid
try:
idx = normalized_voci_sheet[
bilancio_type].index(
(titolo_name,
voce_name))
voce_name = normalized_map[
bilancio_type][idx][4]
except ValueError:
pass
# create voice dictionary with normalized name
destination_document[
bilancio_type][quadro_name][
titolo_name]['data'][
voce_name] = {}
destination_document[
bilancio_type][quadro_name][
titolo_name]['data'][
voce_name] = voce_obj
else:
# copy all voci in data, with no normalization
destination_document[bilancio_type][quadro_name][titolo_name]['data'] = \
titolo_object['data']
if not dryrun:
# write doc to couchdb dest
ret = self.cbw.write(destination_document)
if ret is False:
email_utils.send_notification_email(
msg_string=
'couch translate keys has encountered problems')
self.logger.critical("Write critical problem. Quit")
exit()
if not dryrun:
# if the buffer in CBW is non-empty, flushes the docs to the db
ret = self.cbw.close()
if ret is False:
email_utils.send_notification_email(
msg_string='couch translate keys has encountered problems')
self.logger.critical("Write critical problem. Quit")
exit()
self.logger.info("Compact destination db...")
self.couchdb_dest.compact()
self.logger.info("Done compacting")
if not dryrun and couchdb_dest_name == settings.COUCHDB_NORMALIZED_VOCI_NAME and settings.INSTANCE_TYPE == 'production' or settings.INSTANCE_TYPE == 'staging' and no_patch is False:
self.logger.info(
u"============Run patch 2013 for consuntivo======================"
)
call_command('consuntivo_13_patch', verbosity=2, interactive=False)
email_utils.send_notification_email(
msg_string="Couch translate key has finished")
self.logger.info("Finished couch translate keys")
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
self.dryrun = options['dryrun']
input_file_path = options['input_file']
if options['append'] is True:
self.logger = logging.getLogger('management_append')
# open input file
try:
bilancio = BeautifulSoup(open(input_file_path), "xml")
except IOError:
self.logger.error("File {0} not found".format(input_file_path))
return
# get finloc, year, tipo bilancio
certificato = bilancio.certificato
self.anno = int(certificato['anno'])
tipo_certificato_code = certificato['tipoCertificato']
valuta = certificato['tipoValuta']
if valuta.lower() != 'e':
self.logger.error("Valuta is not EURO")
return
if tipo_certificato_code == "C":
self.tipo_certificato = 'consuntivo'
else:
self.tipo_certificato = 'preventivo'
self.rootnode_slug = self.tipo_certificato
# identifies the Comune from the finloc code
codiceente = certificato['codiceEnte']
mapper = FLMapper()
codfinloc = mapper.get_cities(codiceente)[0]
try:
self.territorio = Territorio.objects.get(cod_finloc=codfinloc)
self.logger.info(u"Comune: {0}".format(self.territorio.cod_finloc))
except ObjectDoesNotExist:
self.logger.error(
u"Comune with codfinloc:{0} is not present on DB. Quitting".
format(codfinloc))
return
# from finloc extracts only the numeric part, removing the eventual name
numeric_finloc = self.territorio.cod_finloc
if "--" in numeric_finloc:
split_finloc = numeric_finloc.split("--")
numeric_finloc = split_finloc[1]
# checks if Codice Voce (xml2slug mapping) is present in the DB for the current bilancio type
if CodiceVoce.get_bilancio_codes(
anno=self.anno,
tipo_certificato=self.tipo_certificato).count() == 0:
self.logger.error(
"Xml mapping is not present for current bilancio type. Run xml2slug command first"
)
exit()
# import bilancio data into Postgres db, calculate per capita values
self.import_bilancio(bilancio)
# log errors in a file, if any
self.log_errors()
##
# adds bilancio to the source table to mark this bilancio as coming from Comune xml source
##
import_data = {
'territorio': self.territorio,
'anno': self.anno,
'tipologia': self.tipo_certificato
}
if not self.dryrun:
try:
ImportXmlBilancio.objects.get(**import_data).delete()
except ObjectDoesNotExist:
pass
import_bilancio = ImportXmlBilancio(**import_data)
if not self.dryrun:
import_bilancio.save()
##
# Compute Indicators if bilancio is Consuntivo
##
if self.tipo_certificato == 'consuntivo':
self.logger.info(
u"Compute indicators for Comune: {0}, year: {1}, tipo_bilancio: {2}..."
.format(self.territorio.denominazione, self.anno,
self.tipo_certificato))
if not self.dryrun:
call_command('indicators',
verbosity=2,
years=str(self.anno),
cities=numeric_finloc,
indicators='all',
interactive=False)
##
# Updates open data
##
# copy xml file to open data folder
xml_path = os.path.join(settings.OPENDATA_XML_ROOT,
self.territorio.cod_finloc,
certificato['anno'])
destination_file = xml_path + '/' + self.tipo_certificato + ".xml"
if not self.dryrun:
if not os.path.exists(xml_path):
os.makedirs(xml_path)
shutil.copyfile(src=input_file_path, dst=destination_file)
self.logger.info("Copied Xml file to {}".format(destination_file))
self.logger.info("** Update open data zip file for {} **".format(
self.territorio.denominazione))
# updates open data zip file for considered Comune
if not self.dryrun:
years = "{0}-{1}".format(settings.APP_START_YEAR,
settings.APP_END_YEAR)
call_command('update_opendata',
verbosity=2,
years=years,
cities=numeric_finloc,
compress=True,
interactive=False)
#
# class Roma2006TestCase(SimplifyBaseTestCaseMixin, TestCase):
# code = "2006_ROMA--3120700900"
#
# class Roma2008TestCase(SimplifyBaseTestCaseMixin, TestCase):
# code = "2008_ROMA--3120700900"
#
# ...
#
# and to allow for *parametric testing*, a bit of meta-programming is used.
#
# This loop generates TestCase subclasses, so that the python manage.py test
# can discover and launch the test suite for all of them.
#
# Invocation:
# python manage.py test bilanci --settings=bilanci.settings.testnodb [-v2]
mapper = FLMapper()
for year in (2004, 2008, 2010, 2012, 2013, 2014):
for city_name in ('Roma', 'Milano', 'Torino', 'Napoli'):
name = "{}{}TestCase".format(city_name, year)
city = mapper.get_city(city_name)
code = "{}_{}".format(year, city)
Class = type(name, (BilanciSimpleBaseTestCaseMixin, TestCase),
dict(city=city, code=code))
globals()[name] = Class
# The Class variable contains a *TestCase type, at this point
# so we clear it, in order to avoid repeating an already
# performed TestCase
Class = None
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
dryrun = options['dryrun']
autocommit = options['autocommit']
skip_existing = options['skip_existing']
if options['append'] is True:
self.logger = logging.getLogger('management_append')
# check if debug is active: the task may fail
if settings.DEBUG is True and options['cities'] == 'all':
self.logger.error(
"DEBUG settings is True, task will fail. Disable DEBUG and retry"
)
exit()
# massaging cities option and getting cities finloc codes
cities_param = options['cities']
if not cities_param:
self.logger.error("Missing cities parameter")
exit()
if cities_param.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities_param))
mapper = FLMapper()
cities_codes = mapper.get_cities(cities_param)
if len(cities_codes) == 0:
self.logger.error(
"No cities found with id:{0}".format(cities_codes))
exit()
# massaging years option
self.set_years(options['years'])
self.logger.info("Processing years: {0}".format(self.years))
# transform cities codes into territori slug
cities = []
for city_code in cities_codes:
try:
if len(city_code) > 10:
cities.append(
Territorio.objects.get(
cod_finloc__endswith=city_code[-10:]))
else:
cities.append(
Territorio.objects.get(cod_finloc__endswith=city_code))
except ObjectDoesNotExist:
self.logger.warning(
u"Territorio '{}' not found in db, skip".format(city_code))
continue
# massaging indicators option
indicators_slugs = options['indicators']
indicators_slugs = [(i.strip()) for i in indicators_slugs.split(",")]
# meta-programming
# using the bilanci.indicators module to read indicators and their formulas
# generating records in the DB, if not existing
base_class = import_by_path('bilanci.indicators.BaseIndicator')
indicators_instances = []
for attr_name in dir(indicators):
if attr_name.endswith(
"Indicator") and attr_name != base_class.__name__:
indicator_class = import_by_path(
"bilanci.indicators.{0}".format(attr_name))
indicator_instance = indicator_class()
# skip indicators not in --indicators slugs
if indicators_slugs != [
'all'
] and not indicator_instance.slug in indicators_slugs:
continue
indicators_instances.append(indicator_instance)
# singleton - create indicator record in the DB, if non existing
# update existing denominazione, with label in class, if existing
indicator_obj, created = Indicatore.objects.get_or_create(
slug=indicator_instance.slug,
defaults={
'denominazione': indicator_instance.label,
'published': indicator_instance.published
})
if not created:
indicator_obj.denominazione = indicator_instance.label
indicator_obj.save()
# actual computation of the values
if autocommit is False:
set_autocommit(False)
for indicator in indicators_instances:
self.logger.info(u"Indicator: {0}".format(indicator.label))
if dryrun:
# no db storage
_ = indicator.compute(cities, self.years, logger=self.logger)
else:
# db storage
indicator.compute_and_commit(cities,
self.years,
logger=self.logger,
skip_existing=skip_existing)
if autocommit is False:
commit()
if autocommit is False:
set_autocommit(True)
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
dryrun = options['dryrun']
autocommit = options['autocommit']
skip_existing = options['skip_existing']
if options['append'] is True:
self.logger = logging.getLogger('management_append')
# check if debug is active: the task may fail
if settings.DEBUG is True and options['cities'] == 'all':
self.logger.error("DEBUG settings is True, task will fail. Disable DEBUG and retry")
exit()
# massaging cities option and getting cities finloc codes
cities_param = options['cities']
if not cities_param:
self.logger.error("Missing cities parameter")
exit()
if cities_param.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities_param))
mapper = FLMapper()
cities_codes = mapper.get_cities(cities_param)
if len(cities_codes) == 0 :
self.logger.error("No cities found with id:{0}".format(cities_codes))
exit()
# massaging years option
self.set_years(options['years'])
self.logger.info("Processing years: {0}".format(self.years))
# transform cities codes into territori slug
cities=[]
for city_code in cities_codes:
try:
if len(city_code)>10:
cities.append(Territorio.objects.get(cod_finloc__endswith=city_code[-10:]))
else:
cities.append(Territorio.objects.get(cod_finloc__endswith=city_code))
except ObjectDoesNotExist:
self.logger.warning(u"Territorio '{}' not found in db, skip".format(city_code))
continue
# massaging indicators option
indicators_slugs = options['indicators']
indicators_slugs = [(i.strip()) for i in indicators_slugs.split(",")]
# meta-programming
# using the bilanci.indicators module to read indicators and their formulas
# generating records in the DB, if not existing
base_class = import_by_path('bilanci.indicators.BaseIndicator')
indicators_instances = []
for attr_name in dir(indicators):
if attr_name.endswith("Indicator") and attr_name != base_class.__name__:
indicator_class = import_by_path("bilanci.indicators.{0}".format(attr_name))
indicator_instance = indicator_class()
# skip indicators not in --indicators slugs
if indicators_slugs != ['all'] and not indicator_instance.slug in indicators_slugs:
continue
indicators_instances.append(indicator_instance)
# singleton - create indicator record in the DB, if non existing
# update existing denominazione, with label in class, if existing
indicator_obj, created = Indicatore.objects.get_or_create(
slug=indicator_instance.slug,
defaults={
'denominazione': indicator_instance.label,
'published': indicator_instance.published
}
)
if not created:
indicator_obj.denominazione = indicator_instance.label
indicator_obj.save()
# actual computation of the values
if autocommit is False:
set_autocommit(False)
for indicator in indicators_instances:
self.logger.info(u"Indicator: {0}".format(
indicator.label
))
if dryrun:
# no db storage
_ = indicator.compute(cities, self.years, logger=self.logger)
else:
# db storage
indicator.compute_and_commit(cities, self.years, logger=self.logger, skip_existing=skip_existing)
if autocommit is False:
commit()
if autocommit is False:
set_autocommit(True)
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing city parameter")
mapper = FLMapper()
cities = mapper.get_cities(cities_codes)
if cities_codes.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities))
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year)+1)
else:
years = [int(y.strip()) for y in years.split(",") if 2001 < int(y.strip()) < 2014]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Processing years: {0}".format(years))
couchdb_server_name = options['couchdb_server']
if couchdb_server_name not in settings.COUCHDB_SERVERS:
raise Exception("Unknown couchdb server name.")
###
# Couchdb connections
###
couchdb_server_alias = options['couchdb_server']
if couchdb_server_alias not in settings.COUCHDB_SERVERS:
raise Exception("Unknown couchdb server alias.")
# hook to simple DB
simple_db_name = 'bilanci_simple'
simple_db = couch.connect(
simple_db_name,
couchdb_server_settings=settings.COUCHDB_SERVERS[couchdb_server_alias]
)
self.logger.info("Hooked to simple DB: {0}".format(simple_db_name))
# hook to normalized DB (for comparisons)
norm_db_name = 'bilanci_voci'
norm_db = couch.connect(
norm_db_name,
couchdb_server_settings=settings.COUCHDB_SERVERS[couchdb_server_alias]
)
self.logger.info("Hooked to normalized DB: {0}".format(norm_db_name))
entrate_sections = OrderedDict([
('Accertamenti', 0),
('Riscossioni in conto competenza', 1),
('Riscossioni in conto residui', 2),
])
spese_sections = OrderedDict([
('Impegni', 0),
('Pagamenti in conto competenza', 1),
('Pagamenti in conto residui', 2),
])
# totali_* will hold a list of all voices to be compared
# norm refers to the normalized tree
# simp refers to the simple tree
totali_preventivo_entrate = [
{'norm': ('preventivo', '02',
'quadro-2-entrate-entrate-tributarie',
'data', 'totale titolo i', 0),
'simp': ('preventivo', 'ENTRATE', 'Imposte e tasse', 'TOTALE')},
{'norm': ('preventivo', '02',
'quadro-2-entrate-entrate-derivanti-da-contributi-e-trasferimenti-correnti-dello-stato-della-regione-e-di-altri-enti-pubblici-anche-in-rapporto-funzioni-delegate-dalla-regione',
'data', 'totale titolo ii', 0),
'simp': ('preventivo', 'ENTRATE', 'Contributi pubblici', 'TOTALE')},
{'norm': ('preventivo', '02',
'quadro-2-entrate-entrate-extratributarie',
'data', 'totale titolo iii', 0),
'simp': ('preventivo', 'ENTRATE', 'Entrate extratributarie', 'TOTALE')},
{'norm': ('preventivo', '02',
'quadro-2-entrate-entrate-derivanti-da-alienazione-da-trasferimenti-di-capitali-e-da-riscossioni-di-crediti',
'data', 'totale titolo iv', 0),
'simp': ('preventivo', 'ENTRATE', 'Vendite e trasferimenti di capitali', 'TOTALE')},
{'norm': ('preventivo', '02',
'quadro-2-entrate-entrate-derivanti-da-accensioni-di-prestiti',
'data', 'totale titolo v', 0),
'simp': ('preventivo', 'ENTRATE', 'Prestiti')},
{'norm': ('preventivo', '02',
'quadro-2-entrate-entrate-derivanti-da-servizi-per-conto-di-terzi',
'data', 'totale titolo vi', 0),
'simp': ('preventivo', 'ENTRATE', 'Entrate per conto terzi')},
]
totali_consuntivo_entrate = []
for section_name, section_idx in entrate_sections.items():
totali_consuntivo_entrate.extend([
{'norm': ('consuntivo', '02',
'quadro-2-entrate-titolo-i-entrate-tributarie',
'data', 'totale entrate tributarie', section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name, 'Imposte e tasse', 'TOTALE')},
{'norm': ('consuntivo', '02',
'quadro-2-entrate-titolo-ii-entrate-derivanti-da-contributi-e-trasferimenti-correnti',
'data', 'totale entrate derivanti da contributi e trasferimenti correnti', section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name, 'Contributi pubblici', 'TOTALE')},
{'norm': ('consuntivo', '02',
'quadro-2-entrate-titolo-iii-entrate-extratributarie',
'data', 'totale entrate extratributarie', section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name, 'Entrate extratributarie', 'TOTALE')},
{'norm': ('consuntivo', '02',
'quadro-2-entrate-titolo-iv-entrate-derivanti-da-alienazione-da-trasfer-di-capitali-e-da-riscossioni-di-crediti',
'data', 'totale entrate derivanti da alienazione, trasferimenti di capitali e da riscossioni di crediti', section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name, 'Vendite e trasferimenti di capitali', 'TOTALE')},
{'norm': ('consuntivo', '02',
'quadro-2-entrate-titolo-v-entrate-derivanti-da-accensione-di-prestiti',
'data', 'totale entrate derivanti da accensione di prestiti', section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name, 'Prestiti')},
{'norm': ('consuntivo', '02',
'quadro-2-entrate-titolo-vi-entrate-da-servizi-per-conto-di-terzi',
'data', 'totale entrate da servizi per conto di terzi', section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name, 'Entrate per conto terzi')},
])
totali_consuntivo_spese = []
# quadro 3
# section_name and section_idx contains the Impegni/Competenze/Residui name and indexes
for section_name, section_idx in spese_sections.items():
totali_consuntivo_spese.extend([
{'norm': ('consuntivo', '03',
'quadro-3-riepilogo-generale-delle-spese',
'data', 'totale generale delle spese', section_idx),
'simp': ('consuntivo', 'SPESE', section_name, 'TOTALE')},
{'norm': ('consuntivo', '03',
'quadro-3-riepilogo-generale-delle-spese',
'data', 'titolo i - spese correnti', section_idx),
'simp': ('consuntivo', 'SPESE', section_name, 'Spese correnti', 'TOTALE')},
{'norm': ('consuntivo', '03',
'quadro-3-riepilogo-generale-delle-spese',
'data', 'titolo ii - spese in c/capitale', section_idx),
'simp': ('consuntivo', 'SPESE', section_name, 'Spese per investimenti', 'TOTALE')},
{'norm': ('consuntivo', '03',
'quadro-3-riepilogo-generale-delle-spese',
'data', 'titolo iii - spese per rimborso di prestiti', section_idx),
'simp': ('consuntivo', 'SPESE', section_name, 'Prestiti')},
{'norm': ('consuntivo', '03',
'quadro-3-riepilogo-generale-delle-spese',
'data', 'titolo iv - spese per servirzi per conto di terzi', section_idx),
'simp': ('consuntivo', 'SPESE', section_name, 'Spese per conto terzi')},
])
# quadro 4
totali_consuntivo_spese.extend([
{'norm': ('consuntivo', '04',
'quadro-4-a-impegni',
'data', 'totale', -1),
'simp': ('consuntivo', 'SPESE', 'Impegni', 'Spese correnti', 'TOTALE')},
{'norm': ('consuntivo', '04',
'quadro-4-b-pagamenti-in-conto-competenza',
'data', 'totali', -1),
'simp': ('consuntivo', 'SPESE', 'Pagamenti in conto competenza', 'Spese correnti', 'TOTALE')},
{'norm': ('consuntivo', '04',
'quadro-4-c-pagamenti-in-conto-residui',
'data', 'totali', -1),
'simp': ('consuntivo', 'SPESE', 'Pagamenti in conto residui', 'Spese correnti', 'TOTALE')},
])
# quadro 5
totali_consuntivo_spese.extend([
{'norm': ('consuntivo', '05',
'quadro-5-a-impegni',
'data', 'totale', -1),
'simp': ('consuntivo', 'SPESE', 'Impegni', 'Spese per investimenti', 'TOTALE')},
{'norm': ('consuntivo', '05',
'quadro-5-b-pagamenti-in-conto-competenza',
'data', 'totale', -1),
'simp': ('consuntivo', 'SPESE', 'Pagamenti in conto competenza', 'Spese per investimenti', 'TOTALE')},
{'norm': ('consuntivo', '05',
'quadro-5-c-pagamenti-in-conto-residui',
'data', 'totale', -1),
'simp': ('consuntivo', 'SPESE', 'Pagamenti in conto residui', 'Spese per investimenti', 'TOTALE')},
])
somme_consuntivo_nodes = []
for section_name in entrate_sections.keys():
somme_consuntivo_nodes.extend([
('consuntivo', 'ENTRATE', section_name, 'Imposte e tasse'),
('consuntivo', 'ENTRATE', section_name, 'Imposte e tasse', 'Imposte'),
('consuntivo', 'ENTRATE', section_name, 'Imposte e tasse', 'Tasse'),
('consuntivo', 'ENTRATE', section_name, 'Contributi pubblici'),
('consuntivo', 'ENTRATE', section_name, 'Entrate extratributarie'),
('consuntivo', 'ENTRATE', section_name, 'Entrate extratributarie', 'Servizi pubblici'),
('consuntivo', 'ENTRATE', section_name, 'Entrate extratributarie', 'Proventi di beni dell\'ente'),
('consuntivo', 'ENTRATE', section_name, 'Vendite e trasferimenti di capitali'),
('consuntivo', 'ENTRATE', section_name, 'Vendite e trasferimenti di capitali', 'Trasferimenti di capitali da privati'),
])
somme_preventivo_nodes = [
('preventivo', 'ENTRATE', 'Imposte e tasse'),
('preventivo', 'ENTRATE', 'Imposte e tasse', 'Imposte'),
('preventivo', 'ENTRATE', 'Imposte e tasse', 'Tasse'),
('preventivo', 'ENTRATE', 'Contributi pubblici'),
('preventivo', 'ENTRATE', 'Entrate extratributarie'),
('preventivo', 'ENTRATE', 'Vendite e trasferimenti di capitali'),
]
for city in cities:
for year in years:
self.logger.info("Processing city of {0}, year {1}".format(
city, year
))
code = "{}_{}".format(year, city)
norm_doc_id = "{}_{}".format(year, city)
simple_doc_id = city
# both documents need to exist in the dbs
self.assertTrue(self.test_couch_doc_exists(norm_db, norm_doc_id),
"Could not find {}".format(norm_doc_id))
self.assertTrue(self.test_couch_doc_exists(simple_db, simple_doc_id))
norm_doc = norm_db[norm_doc_id]
simple_doc = simple_db[simple_doc_id]
# preventivo tests
if len(simple_doc[str(year)]['preventivo'].keys()) > 0:
self.logger.debug("::::: Testing first level totals for preventivo entrate")
self.test_totali(totali_preventivo_entrate, simple_doc, norm_doc, year)
self.logger.debug("::::: Testing totale - funzioni - interventi for preventivo/spese")
for tipo_spese in (u'Spese correnti', u'Spese per investimenti'):
node = simple_doc[str(year)]['preventivo']['SPESE'][tipo_spese]
label = u"/Preventivo/{0}".format(tipo_spese)
self.test_totale_funzioni_interventi(label, node, year)
self.logger.debug("::::: Testing inner sums for preventivo entrate")
self.test_somme(somme_preventivo_nodes, simple_doc, year)
# consuntivo tests
if len(simple_doc[str(year)]['consuntivo'].keys()) > 0:
self.logger.debug("::::: Testing first level totals for consuntivo entrate")
self.test_totali(totali_consuntivo_entrate, simple_doc, norm_doc, year)
self.logger.debug("::::: Testing first level totals for consuntivo spese")
self.test_totali(totali_consuntivo_spese, simple_doc, norm_doc, year)
self.logger.debug("::::: Testing totale - funzioni - interventi for consuntivo/spese")
for section_name in spese_sections.keys():
for tipo_spese in ('Spese correnti', 'Spese per investimenti'):
node = simple_doc[str(year)]['consuntivo']['SPESE'][section_name][tipo_spese]
label = u"/Consuntivo/{0}/{1}".format(section_name, tipo_spese)
self.test_totale_funzioni_interventi(label, node, year)
self.logger.debug("::::: Testing inner sums for consuntivo entrate")
self.test_somme(somme_consuntivo_nodes, simple_doc, year)
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
valori_complete = []
dryrun = options['dryrun']
compress = options['compress']
single_file = options['single_file']
skip_existing = options['skip_existing']
csv_base_path = os.path.abspath(options['csv_base_dir'])
indicators_path = os.path.join(csv_base_path, "indicators")
if not os.path.exists(indicators_path):
os.makedirs(indicators_path)
###
# cities
###
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing cities parameter")
mapper = FLMapper()
# gets capoluoghi privincia finloc list from settings
if cities_codes == 'capoluoghi':
cities = Territorio.objects.\
filter(slug__in=settings.CAPOLUOGHI_PROVINCIA).\
order_by('cod_finloc').values_list('cod_finloc', flat=True)
else:
cities = mapper.get_cities(cities_codes)
if cities_codes.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities))
###
# years
###
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year) + 1)
else:
years = [int(y.strip()) for y in years.split(",") if 2001 < int(y.strip()) < 2013]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Processing years: {0}".format(years))
self.years = years
# massaging indicators option
indicators_slugs = options['indicators']
if indicators_slugs == 'all':
indicators_slugs = Indicatore.objects.all().values_list('slug', flat=True)
else:
indicators_slugs = [(i.strip()) for i in indicators_slugs.split(",")]
for indicator_slug in indicators_slugs:
indicator = Indicatore.objects.get(slug=indicator_slug)
self.logger.info(u"Indicatore:::: {0} ::::".format(indicator.denominazione))
if not single_file:
# check if files for this indicator exists and skip if
# the skip-existing option was required
csv_filename = os.path.join(indicators_path, "{0}.csv".format(indicator_slug))
if skip_existing and os.path.exists(csv_filename):
self.logger.info(u"Skipping indicator {}, as already processed".format(indicator_slug))
continue
valori = groupby(
indicator.valoreindicatore_set.values('territorio__cod_finloc', 'anno', 'valore').order_by(
'territorio__cod_finloc', 'anno'),
lambda x: (x['territorio__cod_finloc'])
)
valori_dict = {}
for k, g in valori:
valori_dict[k] = dict([(it['anno'], it['valore']) for it in g])
# open csv file
csv_file = open(csv_filename, 'w')
csv_writer = unicode_csv.UnicodeWriter(csv_file, dialect=unicode_csv.excel_semicolon)
# build and emit header
row = ['City', 'Cluster', 'Region']
row.extend(map(str, years))
csv_writer.writerow(row)
for city in cities:
if city not in valori_dict:
continue
try:
territorio = Territorio.objects.get(cod_finloc=city)
except ObjectDoesNotExist:
self.logger.warning(u"City {0} not found among territories in DB. Skipping.".format(city))
# emit
cluster = territorio.cluster if territorio.cluster else ''
region = territorio.regione if territorio.regione else ''
row = [city, cluster, region]
for year in years:
if year not in valori_dict[city]:
row.append('')
else:
row.append(str(valori_dict[city][year]))
csv_writer.writerow(row)
self.logger.debug(",".join(row))
else:
indicator_set = ValoreIndicatore.objects.filter(territorio__cod_finloc__in=cities,
indicatore=indicator). \
values_list('indicatore__slug', 'territorio__cod_finloc', 'territorio__istat_id', 'anno',
'valore').order_by('territorio__cod_finloc', 'anno')
valori_list = []
for t in indicator_set:
valori_list.append([str(element) for element in t])
valori_complete.extend(valori_list)
if single_file:
# write a single file with all indicators and values for cities
csv_filename = os.path.join(indicators_path, "indicators.csv")
# open csv file
csv_file = open(csv_filename, 'w')
csv_writer = unicode_csv.UnicodeWriter(csv_file, dialect=unicode_csv.excel_semicolon)
# build and emit header
row = ['indicatore', 'territorio', 'codice_istat', 'anno', 'valore']
csv_writer.writerow(row)
for row in valori_complete:
row[2] = row[2].zfill(6)
csv_writer.writerow(row)
csv_file.close()
self.logger.info("Written file {}".format(csv_filename))
if compress:
csv_path = os.path.join('data', 'csv')
zip_path = os.path.join('data', 'zip')
if not os.path.exists(zip_path):
os.mkdir(zip_path)
zipfilename = os.path.join(zip_path, "indicators.zip")
zipdir("indicators", zipfile.ZipFile(zipfilename, "w", zipfile.ZIP_DEFLATED), root_path=csv_path)
self.logger.info("Compressed file {}".format(zipfilename))
# remove all tree under city_path
# with security control
if 'data' in indicators_path and 'csv' in indicators_path:
shutil.rmtree(indicators_path)
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
offset = int(options['offset'])
limit = int(options['limit'])
self.dryrun = options['dryrun']
self.apidomain = options['apidomain']
if options['auth']:
(user, pwd) = options['auth'].split(",")
self.baseurl = "http://{0}:{1}@{2}".format(user, pwd, self.apidomain)
else:
self.baseurl = "http://{0}".format(self.apidomain)
self.logger.info(u"=== Starting ===")
# all cities in the DB
comuni = Territorio.objects.filter(territorio=Territorio.TERRITORIO.C)
mapper = FLMapper()
c = 0
for comune in comuni:
c += 1
if c < offset:
continue
if limit and c >= limit + offset:
break
city_url = "{0}/maps/places/{1}".format(self.baseurl, comune.slug)
self.logger.debug("CITY_URL: {0}".format(city_url))
place = requests.get(city_url).json()
# get identifiers needed and build the finloc code
identifiers = place['placeidentifiers']
macroregion_id = None
region_id = None
province_id = None
city_id = None
for i in identifiers:
identifier = i['identifier']
value = i['value']
if 'istat-macroregion-id' in identifier:
macroregion_id = int(value)
if 'minint-region-id' in identifier:
region_id = int(value)
if 'minint-province-id' in identifier:
province_id = int(value)
if 'minint-city-id' in identifier:
city_id = int(value)
# build numeric code for finanzalocale
num_cod_finloc = "{0:d}{1:02d}{2:03d}{3:04d}".format(
macroregion_id, region_id, province_id, city_id
)
# store complete finloc code inside the database
try:
comune.cod_finloc = mapper.get_city(num_cod_finloc)
except IndexError:
name = "-".join(comune.slug.split('-')[:-2])
try:
comune.cod_finloc = mapper.get_city(name)
except IndexError:
try:
# next try: (Sant'Antonio => sant-antonio)
# to fetch names with apostrophe
# that are not fetched with the preceding tries
denominazione = comune.denominazione.replace("'", " ")
name = slugify(denominazione)
comune.cod_finloc = mapper.get_city(name)
except IndexError:
self.logger.warning("Could not find city: {0}".format(comune.slug))
continue
except CityNameNotUnique:
# add the province code to get_city because this city
# name is not unique
name_prov = "{0}({1})".format(name,comune.prov)
comune.cod_finloc = mapper.get_city(name_prov)
self.logger.info(u"{0}, slug: {1.slug}, cod_finloc: {1.cod_finloc}".format(
c, comune
))
if not self.dryrun:
try:
comune.save()
except IntegrityError:
# given that finloc field is unique if the comune has a duplicated finloc code
# there is an error
self.logger.error("Finloc code:{0} for City: {1} is already present in DB, quitting...".\
format(comune.cod_finloc,comune.slug)
)
return
self.logger.info(u" === End ===")
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
dryrun = options['dryrun']
output_path = options['output_path']
compress = options['compress']
skip_existing = options['skip_existing']
if options['append'] is True:
self.logger = logging.getLogger('management_append')
###
# cities
###
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing cities parameter")
mapper = FLMapper()
cities = mapper.get_cities(cities_codes)
if not cities:
self.logger.error("Cities cannot be null")
return
if cities_codes.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities))
###
# years
###
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year) + 1)
else:
years = [
int(y.strip()) for y in years.split(",")
if settings.APP_START_YEAR < int(y.strip()) <
settings.APP_END_YEAR
]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Processing years: {0}".format(years))
self.years = years
# instantiate emitter
# to emit CSV files
emitter = FLCSVEmitter(self.logger)
###
# connect to couchdb
###
couchdb = self.connect_to_couch(
couchdb_server=options['couchdb_server'])
output_abs_path = os.path.abspath(output_path)
csv_path = os.path.join(output_abs_path, 'csv')
xml_path = os.path.join(output_abs_path, 'xml')
zip_path = os.path.join(output_abs_path, 'zip')
# build the map of slug to pk for the Voce tree
self.voci_dict = Voce.objects.get_dict_by_slug()
for city in cities:
# check city path (skip if existing and skip-existing active)
city_path = os.path.join(csv_path, city)
if not os.path.exists(city_path):
os.makedirs(city_path)
else:
# if the file for the city already exists and
# skip-existing was specified then skips
if skip_existing:
self.logger.info(
u"Skipping city of {}, as already processed".format(
city))
continue
self.logger.info(u"Processing city: {}".format(city))
for year in years:
# get year budgets for the city
key = u"{}_{}".format(year, city)
city_budget = couchdb.get(key)
if city_budget is None:
self.logger.warning(
u"Budget for:{} not found in couchdb. Skipping.".
format(key))
continue
self.logger.debug(u"Processing: {}".format(key))
# check year path
year_path = os.path.join(city_path, str(year))
if not os.path.exists(year_path):
os.mkdir(year_path)
# if for current city/year was imported a XML bilancio, then skips the Couchdb data, xml file
# will be provided instead
try:
ImportXmlBilancio.objects.\
get(territorio__cod_finloc=city, anno=year, tipologia=ImportXmlBilancio.TIPO_CERTIFICATO.preventivo)
except ObjectDoesNotExist:
pass
else:
self.logger.info(
u"Budget:{} for:{} will be provided only in xml".
format('preventivo', key))
# save preventivo
self.logger.debug(" Preventivo")
prev_path = os.path.join(year_path, 'preventivo')
if not os.path.exists(prev_path):
os.mkdir(prev_path)
preventivo = city_budget.get('preventivo', None)
if preventivo:
emitter.emit(q_data=preventivo, base_path=prev_path)
# if for current city/year was imported a XML bilancio, then skips the Couchdb data, xml file
# will be provided instead
try:
ImportXmlBilancio.objects.\
get(territorio__cod_finloc=city, anno=year, tipologia=ImportXmlBilancio.TIPO_CERTIFICATO.consuntivo)
except ObjectDoesNotExist:
pass
else:
self.logger.info(
u"Budget:{} for:{} will be provided only in xml".
format('consuntivo', key))
# save consuntivo
self.logger.debug(" Consuntivo")
cons_path = os.path.join(year_path, 'Consuntivo')
if not os.path.exists(cons_path):
os.mkdir(cons_path)
consuntivo = city_budget.get('consuntivo', None)
# emit q_data as CSV files in a directory tree
if consuntivo:
emitter.emit(q_data=consuntivo, base_path=cons_path)
# if the zip file is requested, creates the zip folder,
# creates zip file
if compress:
if not os.path.exists(zip_path):
os.mkdir(zip_path)
zipfilename = os.path.join(zip_path, "{}.zip".format(city))
# zips the csv and the xml folders and creates a zip file containing both
# zipdir(city, zipfile.ZipFile(zipfilename, "w", zipfile.ZIP_DEFLATED), root_path=csv_path)
opendata_zipfile = zipfile.ZipFile(zipfilename, "w",
zipfile.ZIP_DEFLATED)
zipdir_prefix(opendata_zipfile, csv_path, city, "csv")
zipdir_prefix(opendata_zipfile, xml_path, city, "xml")
self.logger.info("Created zip file: {}".format(zipfilename))
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
dryrun = options['dryrun']
output_path = options['output_path']
compress = options['compress']
skip_existing = options['skip_existing']
if options['append'] is True:
self.logger = logging.getLogger('management_append')
###
# cities
###
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing cities parameter")
mapper = FLMapper()
cities = mapper.get_cities(cities_codes)
if not cities:
self.logger.error("Cities cannot be null")
return
if cities_codes.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities))
###
# years
###
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year) + 1)
else:
years = [int(y.strip()) for y in years.split(",") if settings.APP_START_YEAR < int(y.strip()) < settings.APP_END_YEAR]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Processing years: {0}".format(years))
self.years = years
# instantiate emitter
# to emit CSV files
emitter = FLCSVEmitter(self.logger)
###
# connect to couchdb
###
couchdb = self.connect_to_couch(couchdb_server=options['couchdb_server'])
output_abs_path = os.path.abspath(output_path)
csv_path = os.path.join(output_abs_path, 'csv')
xml_path = os.path.join(output_abs_path, 'xml')
zip_path = os.path.join(output_abs_path, 'zip')
# build the map of slug to pk for the Voce tree
self.voci_dict = Voce.objects.get_dict_by_slug()
for city in cities:
# check city path (skip if existing and skip-existing active)
city_path = os.path.join(csv_path, city)
if not os.path.exists(city_path):
os.makedirs(city_path)
else:
# if the file for the city already exists and
# skip-existing was specified then skips
if skip_existing:
self.logger.info(u"Skipping city of {}, as already processed".format(city))
continue
self.logger.info(u"Processing city: {}".format(city))
for year in years:
# get year budgets for the city
key = u"{}_{}".format(year, city)
city_budget = couchdb.get(key)
if city_budget is None:
self.logger.warning(u"Budget for:{} not found in couchdb. Skipping.".format(key))
continue
self.logger.debug(u"Processing: {}".format(key))
# check year path
year_path = os.path.join(city_path, str(year))
if not os.path.exists(year_path):
os.mkdir(year_path)
# if for current city/year was imported a XML bilancio, then skips the Couchdb data, xml file
# will be provided instead
try:
ImportXmlBilancio.objects.\
get(territorio__cod_finloc=city, anno=year, tipologia=ImportXmlBilancio.TIPO_CERTIFICATO.preventivo)
except ObjectDoesNotExist:
pass
else:
self.logger.info(u"Budget:{} for:{} will be provided only in xml".format('preventivo', key))
# save preventivo
self.logger.debug(" Preventivo")
prev_path = os.path.join(year_path, 'preventivo')
if not os.path.exists(prev_path):
os.mkdir(prev_path)
preventivo = city_budget.get('preventivo', None)
if preventivo:
emitter.emit(q_data=preventivo, base_path=prev_path)
# if for current city/year was imported a XML bilancio, then skips the Couchdb data, xml file
# will be provided instead
try:
ImportXmlBilancio.objects.\
get(territorio__cod_finloc=city, anno=year, tipologia=ImportXmlBilancio.TIPO_CERTIFICATO.consuntivo)
except ObjectDoesNotExist:
pass
else:
self.logger.info(u"Budget:{} for:{} will be provided only in xml".format('consuntivo', key))
# save consuntivo
self.logger.debug(" Consuntivo")
cons_path = os.path.join(year_path, 'Consuntivo')
if not os.path.exists(cons_path):
os.mkdir(cons_path)
consuntivo = city_budget.get('consuntivo', None)
# emit q_data as CSV files in a directory tree
if consuntivo:
emitter.emit(q_data=consuntivo, base_path=cons_path)
# if the zip file is requested, creates the zip folder,
# creates zip file
if compress:
if not os.path.exists(zip_path):
os.mkdir(zip_path)
zipfilename = os.path.join(zip_path, "{}.zip".format(city))
# zips the csv and the xml folders and creates a zip file containing both
# zipdir(city, zipfile.ZipFile(zipfilename, "w", zipfile.ZIP_DEFLATED), root_path=csv_path)
opendata_zipfile = zipfile.ZipFile(zipfilename, "w", zipfile.ZIP_DEFLATED)
zipdir_prefix(opendata_zipfile, csv_path, city, "csv")
zipdir_prefix(opendata_zipfile, xml_path, city, "xml")
self.logger.info("Created zip file: {}".format(zipfilename))
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing city parameter")
mapper = FLMapper()
cities = mapper.get_cities(cities_codes)
if cities_codes.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities))
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year) + 1)
else:
years = [
int(y.strip()) for y in years.split(",")
if 2001 < int(y.strip()) < 2014
]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Processing years: {0}".format(years))
couchdb_server_name = options['couchdb_server']
if couchdb_server_name not in settings.COUCHDB_SERVERS:
raise Exception("Unknown couchdb server name.")
###
# Couchdb connections
###
couchdb_server_alias = options['couchdb_server']
if couchdb_server_alias not in settings.COUCHDB_SERVERS:
raise Exception("Unknown couchdb server alias.")
# hook to simple DB
simple_db_name = 'bilanci_simple'
simple_db = couch.connect(simple_db_name,
couchdb_server_settings=settings.
COUCHDB_SERVERS[couchdb_server_alias])
self.logger.info("Hooked to simple DB: {0}".format(simple_db_name))
# hook to normalized DB (for comparisons)
norm_db_name = 'bilanci_voci'
norm_db = couch.connect(norm_db_name,
couchdb_server_settings=settings.
COUCHDB_SERVERS[couchdb_server_alias])
self.logger.info("Hooked to normalized DB: {0}".format(norm_db_name))
entrate_sections = OrderedDict([
('Accertamenti', 0),
('Riscossioni in conto competenza', 1),
('Riscossioni in conto residui', 2),
])
spese_sections = OrderedDict([
('Impegni', 0),
('Pagamenti in conto competenza', 1),
('Pagamenti in conto residui', 2),
])
# totali_* will hold a list of all voices to be compared
# norm refers to the normalized tree
# simp refers to the simple tree
totali_preventivo_entrate = [
{
'norm':
('preventivo', '02', 'quadro-2-entrate-entrate-tributarie',
'data', 'totale titolo i', 0),
'simp': ('preventivo', 'ENTRATE', 'Imposte e tasse', 'TOTALE')
},
{
'norm':
('preventivo', '02',
'quadro-2-entrate-entrate-derivanti-da-contributi-e-trasferimenti-correnti-dello-stato-della-regione-e-di-altri-enti-pubblici-anche-in-rapporto-funzioni-delegate-dalla-regione',
'data', 'totale titolo ii', 0),
'simp':
('preventivo', 'ENTRATE', 'Contributi pubblici', 'TOTALE')
},
{
'norm': ('preventivo', '02',
'quadro-2-entrate-entrate-extratributarie', 'data',
'totale titolo iii', 0),
'simp':
('preventivo', 'ENTRATE', 'Entrate extratributarie', 'TOTALE')
},
{
'norm':
('preventivo', '02',
'quadro-2-entrate-entrate-derivanti-da-alienazione-da-trasferimenti-di-capitali-e-da-riscossioni-di-crediti',
'data', 'totale titolo iv', 0),
'simp': ('preventivo', 'ENTRATE',
'Vendite e trasferimenti di capitali', 'TOTALE')
},
{
'norm':
('preventivo', '02',
'quadro-2-entrate-entrate-derivanti-da-accensioni-di-prestiti',
'data', 'totale titolo v', 0),
'simp': ('preventivo', 'ENTRATE', 'Prestiti')
},
{
'norm':
('preventivo', '02',
'quadro-2-entrate-entrate-derivanti-da-servizi-per-conto-di-terzi',
'data', 'totale titolo vi', 0),
'simp': ('preventivo', 'ENTRATE', 'Entrate per conto terzi')
},
]
totali_consuntivo_entrate = []
for section_name, section_idx in entrate_sections.items():
totali_consuntivo_entrate.extend([
{
'norm':
('consuntivo', '02',
'quadro-2-entrate-titolo-i-entrate-tributarie', 'data',
'totale entrate tributarie', section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name,
'Imposte e tasse', 'TOTALE')
},
{
'norm':
('consuntivo', '02',
'quadro-2-entrate-titolo-ii-entrate-derivanti-da-contributi-e-trasferimenti-correnti',
'data',
'totale entrate derivanti da contributi e trasferimenti correnti',
section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name,
'Contributi pubblici', 'TOTALE')
},
{
'norm':
('consuntivo', '02',
'quadro-2-entrate-titolo-iii-entrate-extratributarie',
'data', 'totale entrate extratributarie', section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name,
'Entrate extratributarie', 'TOTALE')
},
{
'norm':
('consuntivo', '02',
'quadro-2-entrate-titolo-iv-entrate-derivanti-da-alienazione-da-trasfer-di-capitali-e-da-riscossioni-di-crediti',
'data',
'totale entrate derivanti da alienazione, trasferimenti di capitali e da riscossioni di crediti',
section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name,
'Vendite e trasferimenti di capitali', 'TOTALE')
},
{
'norm':
('consuntivo', '02',
'quadro-2-entrate-titolo-v-entrate-derivanti-da-accensione-di-prestiti',
'data',
'totale entrate derivanti da accensione di prestiti',
section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name, 'Prestiti')
},
{
'norm':
('consuntivo', '02',
'quadro-2-entrate-titolo-vi-entrate-da-servizi-per-conto-di-terzi',
'data', 'totale entrate da servizi per conto di terzi',
section_idx),
'simp': ('consuntivo', 'ENTRATE', section_name,
'Entrate per conto terzi')
},
])
totali_consuntivo_spese = []
# quadro 3
# section_name and section_idx contains the Impegni/Competenze/Residui name and indexes
for section_name, section_idx in spese_sections.items():
totali_consuntivo_spese.extend([
{
'norm': ('consuntivo', '03',
'quadro-3-riepilogo-generale-delle-spese', 'data',
'totale generale delle spese', section_idx),
'simp': ('consuntivo', 'SPESE', section_name, 'TOTALE')
},
{
'norm': ('consuntivo', '03',
'quadro-3-riepilogo-generale-delle-spese', 'data',
'titolo i - spese correnti', section_idx),
'simp': ('consuntivo', 'SPESE', section_name,
'Spese correnti', 'TOTALE')
},
{
'norm': ('consuntivo', '03',
'quadro-3-riepilogo-generale-delle-spese', 'data',
'titolo ii - spese in c/capitale', section_idx),
'simp': ('consuntivo', 'SPESE', section_name,
'Spese per investimenti', 'TOTALE')
},
{
'norm': ('consuntivo', '03',
'quadro-3-riepilogo-generale-delle-spese', 'data',
'titolo iii - spese per rimborso di prestiti',
section_idx),
'simp': ('consuntivo', 'SPESE', section_name, 'Prestiti')
},
{
'norm':
('consuntivo', '03',
'quadro-3-riepilogo-generale-delle-spese', 'data',
'titolo iv - spese per servirzi per conto di terzi',
section_idx),
'simp': ('consuntivo', 'SPESE', section_name,
'Spese per conto terzi')
},
])
# quadro 4
totali_consuntivo_spese.extend([
{
'norm': ('consuntivo', '04', 'quadro-4-a-impegni', 'data',
'totale', -1),
'simp':
('consuntivo', 'SPESE', 'Impegni', 'Spese correnti', 'TOTALE')
},
{
'norm': ('consuntivo', '04',
'quadro-4-b-pagamenti-in-conto-competenza', 'data',
'totali', -1),
'simp':
('consuntivo', 'SPESE', 'Pagamenti in conto competenza',
'Spese correnti', 'TOTALE')
},
{
'norm':
('consuntivo', '04', 'quadro-4-c-pagamenti-in-conto-residui',
'data', 'totali', -1),
'simp': ('consuntivo', 'SPESE', 'Pagamenti in conto residui',
'Spese correnti', 'TOTALE')
},
])
# quadro 5
totali_consuntivo_spese.extend([
{
'norm': ('consuntivo', '05', 'quadro-5-a-impegni', 'data',
'totale', -1),
'simp': ('consuntivo', 'SPESE', 'Impegni',
'Spese per investimenti', 'TOTALE')
},
{
'norm': ('consuntivo', '05',
'quadro-5-b-pagamenti-in-conto-competenza', 'data',
'totale', -1),
'simp':
('consuntivo', 'SPESE', 'Pagamenti in conto competenza',
'Spese per investimenti', 'TOTALE')
},
{
'norm':
('consuntivo', '05', 'quadro-5-c-pagamenti-in-conto-residui',
'data', 'totale', -1),
'simp': ('consuntivo', 'SPESE', 'Pagamenti in conto residui',
'Spese per investimenti', 'TOTALE')
},
])
somme_consuntivo_nodes = []
for section_name in entrate_sections.keys():
somme_consuntivo_nodes.extend([
('consuntivo', 'ENTRATE', section_name, 'Imposte e tasse'),
('consuntivo', 'ENTRATE', section_name, 'Imposte e tasse',
'Imposte'),
('consuntivo', 'ENTRATE', section_name, 'Imposte e tasse',
'Tasse'),
('consuntivo', 'ENTRATE', section_name, 'Contributi pubblici'),
('consuntivo', 'ENTRATE', section_name,
'Entrate extratributarie'),
('consuntivo', 'ENTRATE', section_name,
'Entrate extratributarie', 'Servizi pubblici'),
('consuntivo', 'ENTRATE', section_name,
'Entrate extratributarie', 'Proventi di beni dell\'ente'),
('consuntivo', 'ENTRATE', section_name,
'Vendite e trasferimenti di capitali'),
('consuntivo', 'ENTRATE', section_name,
'Vendite e trasferimenti di capitali',
'Trasferimenti di capitali da privati'),
])
somme_preventivo_nodes = [
('preventivo', 'ENTRATE', 'Imposte e tasse'),
('preventivo', 'ENTRATE', 'Imposte e tasse', 'Imposte'),
('preventivo', 'ENTRATE', 'Imposte e tasse', 'Tasse'),
('preventivo', 'ENTRATE', 'Contributi pubblici'),
('preventivo', 'ENTRATE', 'Entrate extratributarie'),
('preventivo', 'ENTRATE', 'Vendite e trasferimenti di capitali'),
]
for city in cities:
for year in years:
self.logger.info("Processing city of {0}, year {1}".format(
city, year))
code = "{}_{}".format(year, city)
norm_doc_id = "{}_{}".format(year, city)
simple_doc_id = city
# both documents need to exist in the dbs
self.assertTrue(
self.test_couch_doc_exists(norm_db, norm_doc_id),
"Could not find {}".format(norm_doc_id))
self.assertTrue(
self.test_couch_doc_exists(simple_db, simple_doc_id))
norm_doc = norm_db[norm_doc_id]
simple_doc = simple_db[simple_doc_id]
# preventivo tests
if len(simple_doc[str(year)]['preventivo'].keys()) > 0:
self.logger.debug(
"::::: Testing first level totals for preventivo entrate"
)
self.test_totali(totali_preventivo_entrate, simple_doc,
norm_doc, year)
self.logger.debug(
"::::: Testing totale - funzioni - interventi for preventivo/spese"
)
for tipo_spese in (u'Spese correnti',
u'Spese per investimenti'):
node = simple_doc[str(
year)]['preventivo']['SPESE'][tipo_spese]
label = u"/Preventivo/{0}".format(tipo_spese)
self.test_totale_funzioni_interventi(label, node, year)
self.logger.debug(
"::::: Testing inner sums for preventivo entrate")
self.test_somme(somme_preventivo_nodes, simple_doc, year)
# consuntivo tests
if len(simple_doc[str(year)]['consuntivo'].keys()) > 0:
self.logger.debug(
"::::: Testing first level totals for consuntivo entrate"
)
self.test_totali(totali_consuntivo_entrate, simple_doc,
norm_doc, year)
self.logger.debug(
"::::: Testing first level totals for consuntivo spese"
)
self.test_totali(totali_consuntivo_spese, simple_doc,
norm_doc, year)
self.logger.debug(
"::::: Testing totale - funzioni - interventi for consuntivo/spese"
)
for section_name in spese_sections.keys():
for tipo_spese in ('Spese correnti',
'Spese per investimenti'):
node = simple_doc[str(year)]['consuntivo'][
'SPESE'][section_name][tipo_spese]
label = u"/Consuntivo/{0}/{1}".format(
section_name, tipo_spese)
self.test_totale_funzioni_interventi(
label, node, year)
self.logger.debug(
"::::: Testing inner sums for consuntivo entrate")
self.test_somme(somme_consuntivo_nodes, simple_doc, year)
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
###
# dry run
###
dryrun = options['dryrun']
skip_existing = options['skip_existing']
if options['append'] is True:
self.logger = logging.getLogger('management_append')
###
# cities
###
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing cities parameter")
mapper = FLMapper()
cities = mapper.get_cities(cities_codes)
if cities_codes.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities))
###
# years
###
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year)+1)
else:
years = [int(y.strip()) for y in years.split(",") if 2001 < int(y.strip()) < 2013]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Processing years: {0}".format(years))
self.years = years
###
# couchdb
###
couchdb_server_alias = options['couchdb_server']
couchdb_dbname = settings.COUCHDB_NORMALIZED_VOCI_NAME
if couchdb_server_alias not in settings.COUCHDB_SERVERS:
self.logger.error(u"Unknown couchdb server alias.")
return
self.logger.info(u"Connecting to db: {0}".format(couchdb_dbname))
self.couchdb = couch.connect(
couchdb_dbname,
couchdb_server_settings=settings.COUCHDB_SERVERS[couchdb_server_alias]
)
# set contesto and filter out missing territories
missing_territories = []
recalculate_percapita_cities = []
for city in cities:
try:
territorio = Territorio.objects.get(cod_finloc=city)
except ObjectDoesNotExist:
self.logger.warning(u"City {0} not found among territories in DB. Skipping.".format(city))
missing_territories.append(city)
continue
# if skip_existing and the territorio has 1+ contesto then skip territorio
if skip_existing and Contesto.objects.filter(territorio=territorio).count() > 0:
self.logger.info(u"Skip Existing - City:{0} already has context, skipping".format(territorio.denominazione))
continue
self.logger.info(u"Setting context for city: {0}".format(territorio,))
# note: the following keys will not be stored in the db because
# the number format is not constant through the years
#
# "nuclei familiari (n)":"bil_nuclei_familiari",
# "superficie urbana (ha)":"bil_superficie_urbana",
# "superficie totale del comune (ha)":"bil_superficie_totale",
# "lunghezza delle strade esterne (km)":"bil_strade_esterne",
# "lunghezza delle strade interne centro abitato (km)":"bil_strade_interne",
# "di cui: in territorio montano (km)":"bil_strade_montane",
key_name = "popolazione residente (ab.)"
data_results = self.get_data(territorio, years, key_name)
for data_result in data_results:
contesto_pg = None
year, value = data_result
# if value is None it means that the year selected had a value for the key that is not acceptable or wrong
# then if the value for that specific year is already in the db, it has to be deleted
if value is None:
self.logger.warning(u"Deleting wrong value for city:{0} year:{1}".format(territorio.denominazione, year))
_ = Contesto.objects.filter(
anno = year,
territorio = territorio,
).delete()
if territorio not in recalculate_percapita_cities:
recalculate_percapita_cities.append(territorio)
continue
# if the contesto data is not present, inserts the data in the db
# otherwise skips
try:
contesto_pg = Contesto.objects.get(
anno = year,
territorio = territorio,
)
except ObjectDoesNotExist:
contesto_pg = Contesto()
pass
# write data on postgres
if dryrun is False:
contesto_pg.bil_popolazione_residente = value
self.logger.debug(u"year:{0}, value:{1}".format(year, value,))
contesto_pg.territorio = territorio
contesto_pg.anno = year
contesto_pg.save()
if len(missing_territories)>0:
self.logger.error(u"Following cities could not be found in Territori DB and could not be processed:")
for missing_city in missing_territories:
self.logger.error("{0}".format(missing_city))
percapita_cmd_string = u"python manage.py percapita -v2 --years={0} --cities=".format(options['years'])
if len(recalculate_percapita_cities)>0:
self.logger.error(u"Following cities had at least one wrong context data and percapita should be recalculated with this command:")
for missing_city in recalculate_percapita_cities:
numeric_codfinloc = missing_city.cod_finloc.split("--")[1]
percapita_cmd_string+=numeric_codfinloc+","
self.logger.error(percapita_cmd_string[:-1])
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
self.dryrun = options['dryrun']
# get the timestamp to ensure the document will be written in couchdb, this is a workaround for a bug,
# see later comment
timestamp = time.time()
###
# connect to couchdb
###
self.couchdb_dest = self.couch_connect(options['couchdb_server'])
# create couch bulk writer
self.cbw = couch.CouchBulkWriter(logger=self.logger, couchdb_dest=self.couchdb_dest)
mapper = FLMapper()
all_cities = mapper.get_cities('all', logger=self.logger)
counter = 0
for comune_slug in all_cities:
doc_key = "2013_{}".format(comune_slug)
bilancio_2013 = self.couchdb_dest.get(doc_key)
old_destination_doc = self.couchdb_dest.get(doc_key, None)
if old_destination_doc:
revision = old_destination_doc.get('_rev', None)
if revision:
bilancio_2013['_rev'] = revision
self.logger.debug("Adds rev value to doc:{}".format(doc_key))
if bilancio_2013 is None:
self.logger.error("Cannot find bilancio 2013 for {}".format(comune_slug))
continue
consuntivo = bilancio_2013.get('consuntivo', None)
if consuntivo is None:
self.logger.error("Cannot find consuntivo 2013 for {}".format(comune_slug))
continue
self.patch_totale_pareggio(consuntivo,comune_slug)
self.patch_residui_passivi(consuntivo,comune_slug)
self.patch_residui_attivi(consuntivo,comune_slug)
# writes back in couchdb
bilancio_2013['_id'] = doc_key
bilancio_2013['useless_timestamp'] = timestamp
if not self.dryrun:
# write doc to couchdb dest
ret = self.cbw.write(bilancio_2013)
if ret is False:
self.logger.critical("Write critical problem. Quit")
exit()
if not self.dryrun:
# if the buffer in CBW is non-empty, flushes the docs to the db
ret = self.cbw.close()
if ret is False:
self.logger.critical("Write critical problem. Quit")
self.logger.info(u"Done patch consuntivo 13")
return
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
# lista dei bilanci mancanti con citta,anno,tipo che viene usata per scrapy
missing_bilanci_tipo = []
# dizionari bilanci mancanti per citta e anno che viene usata per html2couch
missing_bilanci = OrderedDict([])
# open output filenames
if not 'output_path' in options:
raise Exception("output-path option is needed")
output_path = options['output_path']
scrapy_instructions_filename = os.path.join(output_path, "scrape_missing_bilanci")
missing_bilanci_args_filename = os.path.join(output_path, "missing_bilanci_args")
self.logger.info("Opening output files in : {0}".format(output_path))
scrapy_instructions_file = open(scrapy_instructions_filename,'w')
missing_bilanci_args_file = open(missing_bilanci_args_filename,'w')
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing city parameter")
mapper = FLMapper()
cities = mapper.get_cities(cities_codes)
if cities_codes.lower() != 'all':
self.logger.info("Analyzing following cities: {0}".format(cities))
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year)+1)
else:
years = [int(y.strip()) for y in years.split(",") if 2001 < int(y.strip()) < 2013]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Analyzing years: {0}".format(years))
couchdb_server_name = options['couchdb_server']
if couchdb_server_name not in settings.COUCHDB_SERVERS:
raise Exception("Unknown couchdb server name.")
###
# Couchdb connections
###
couchdb_server_settings = settings.COUCHDB_SERVERS[couchdb_server_name]
# builds connection URL
server_connection_address = "http://"
if 'user' in couchdb_server_settings and 'password' in couchdb_server_settings:
server_connection_address += "{0}:{1}@".format(
couchdb_server_settings['user'],
couchdb_server_settings['password']
)
server_connection_address += "{0}:{1}".format(
couchdb_server_settings['host'],
couchdb_server_settings['port']
)
self.logger.info("Connecting to: {0} ...".format(server_connection_address))
# open connection to couchdb server and create instance
server = couchdb.Server(server_connection_address)
self.logger.info("Connected!")
# hook to source DB
source_db_name = options['source_db_name']
source_db = server[source_db_name]
self.logger.info("Hooked to source DB: {0}".format(source_db_name))
tipologie_bilancio = ['preventivo', 'consuntivo']
for city in cities:
# city: MILANO--1030491450 -> city_name: MILANO, city_code: 1030491450
city_name, city_code = city.split("--")
for year in years:
# need this for logging
self.city = city
self.year = year
self.logger.debug("Processing city of {0}, year {1}".format(
city, year
))
document_id = str(year)+"_"+city
source_document = source_db.get(document_id)
if source_document is None:
self.logger.error("Missing preventivo and consuntivo for Comune:{0}, yr:{1}".format(
city,year
))
missing_bilanci_tipo.append({'year':year, 'city_name': city_name, 'type': 'P'})
missing_bilanci_tipo.append({'year':year, 'city_name': city_name, 'type': 'C'})
if city_name not in missing_bilanci:
missing_bilanci[city_code]=[]
if year not in missing_bilanci[city_code]:
missing_bilanci[city_code].append(str(year))
else:
anno_is_missing = False
for tipologia in tipologie_bilancio:
tipologia_is_missing = False
if tipologia not in source_document.keys():
tipologia_is_missing=True
anno_is_missing = True
else:
if source_document[tipologia] == {}:
tipologia_is_missing=True
anno_is_missing = True
if tipologia_is_missing:
self.logger.error("Missing {0} for Comune:{1}, yr:{2}".format(
tipologia, city, year
))
if tipologia.lower() == "preventivo":
short_tipologia = 'P'
else:
short_tipologia = 'C'
missing_bilanci_tipo.append({'year':year, 'city_name': city_name, 'type': short_tipologia})
# in missing_bilanci ho un dizionario con chiave il nome citta' e come valori gli anni
# in cui manca almeno un tipo di bilancio.
# in questo modo se mancano prev. e cons. in missing_bilanci ho una sola riga per il comune e non 2
if anno_is_missing:
if city_code not in missing_bilanci:
missing_bilanci[city_code]=[]
if year not in missing_bilanci[city_code]:
missing_bilanci[city_code].append(str(year))
# scrapy instruction file (executable)
scrapy_instructions_file.write('cd ../scraper_project/\n')
for missing_bilancio in missing_bilanci_tipo:
scrapy_instructions_file.write(
"scrapy crawl bilanci_pages -a cities={0} -a years={1} -a type={2}\n".format(
missing_bilancio['city_name'],missing_bilancio['year'],missing_bilancio['type']
)
)
subprocess.call(["chmod", "a+x", scrapy_instructions_filename])
# missing bilanci arguments list
for city_code in missing_bilanci.keys():
missing_bilanci_args_file.write(
"--cities={0} --years={1}\n".format(
city_code, ",".join(missing_bilanci[city_code])
)
)
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
###
# dry run
###
dryrun = options['dryrun']
###
# cities
###
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing cities parameter")
mapper = FLMapper()
cities = mapper.get_cities(cities_codes)
if cities_codes.lower() != 'all':
self.logger.info("Processing cities: {0}".format(cities))
###
# years
###
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year) + 1)
else:
years = [
int(y.strip()) for y in years.split(",")
if 2001 < int(y.strip()) < 2013
]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Processing years: {0}".format(years))
self.years = years
# set contesto and filter out missing territories
missing_territories = []
for city in cities:
try:
territorio = Territorio.objects.get(cod_finloc=city)
except ObjectDoesNotExist:
self.logger.warning(
u"City {0} not found among territories in DB. Skipping.".
format(city))
for year in years:
popolazione = None
self.logger.info(
u"Calculating percapita for city: {0}, year:{1}".format(
territorio, year))
# read data from db
valore_yearset = ValoreBilancio.objects.filter(
territorio=territorio,
anno=year,
)
population_tuple = territorio.nearest_valid_population(year)
if population_tuple is not None:
popolazione = population_tuple[1]
self.logger.debug(
u"City:{0},year:{1},population:{2}".format(
territorio, year, popolazione))
else:
self.logger.error(
u"Cannot find valid population data for city:{0}, year:{1}"
.format(territorio, year))
missing_territories.append((city, year))
continue
for valore_obj in valore_yearset:
valore_obj.valore_procapite = valore_obj.valore * 1.0 / popolazione * 1.0
if not dryrun:
valore_obj.save()
if len(missing_territories) > 0:
self.logger.error(
u"Following context could not be found in DB and could not be processed:"
)
for missing_city, missing_yr in missing_territories:
self.logger.error("City:{0}, Year:{1}".format(
missing_city, missing_yr))
# To avoid repetitions in the testing code, as:
#
# class Roma2006TestCase(SimplifyBaseTestCaseMixin, TestCase):
# code = "2006_ROMA--3120700900"
#
# class Roma2008TestCase(SimplifyBaseTestCaseMixin, TestCase):
# code = "2008_ROMA--3120700900"
#
# ...
#
# and to allow for *parametric testing*, a bit of meta-programming is used.
#
# This loop generates TestCase subclasses, so that the python manage.py test
# can discover and launch the test suite for all of them.
#
# Invocation:
# python manage.py test bilanci --settings=bilanci.settings.testnodb [-v2]
mapper = FLMapper()
for year in (2004, 2008, 2010, 2012, 2013, 2014):
for city_name in ('Roma', 'Milano','Torino', 'Napoli'):
name = "{}{}TestCase".format(city_name, year)
city = mapper.get_city(city_name)
code = "{}_{}".format(year, city)
Class = type(name, (BilanciSimpleBaseTestCaseMixin, TestCase), dict(city=city, code=code))
globals()[name] = Class
# The Class variable contains a *TestCase type, at this point
# so we clear it, in order to avoid repeating an already
# performed TestCase
Class = None
def handle(self, *args, **options):
verbosity = options['verbosity']
if verbosity == '0':
self.logger.setLevel(logging.ERROR)
elif verbosity == '1':
self.logger.setLevel(logging.WARNING)
elif verbosity == '2':
self.logger.setLevel(logging.INFO)
elif verbosity == '3':
self.logger.setLevel(logging.DEBUG)
dryrun = options['dryrun']
# get the timestamp to ensure the document will be written in couchdb, this is a workaround for a bug,
# see later comment
timestamp = time.time()
if options['append'] is True:
self.logger = logging.getLogger('management_append')
force_google = options['force_google']
skip_existing = options['skip_existing']
cities_codes = options['cities']
if not cities_codes:
raise Exception("Missing city parameter")
mapper = FLMapper()
cities = mapper.get_cities(cities_codes)
if cities_codes.lower() != 'all':
self.logger.info(u"Processing cities: {0}".format(cities))
years = options['years']
if not years:
raise Exception("Missing years parameter")
if "-" in years:
(start_year, end_year) = years.split("-")
years = range(int(start_year), int(end_year) + 1)
else:
years = [
int(y.strip()) for y in years.split(",")
if 2001 < int(y.strip()) < 2014
]
if not years:
raise Exception("No suitable year found in {0}".format(years))
self.logger.info("Processing years: {0}".format(years))
couchdb_server_name = options['couchdb_server']
if couchdb_server_name not in settings.COUCHDB_SERVERS:
raise Exception("Unknown couchdb server name.")
###
# Couchdb connections
###
couchdb_server_alias = options['couchdb_server']
if couchdb_server_alias not in settings.COUCHDB_SERVERS:
raise Exception("Unknown couchdb server alias.")
self.logger.info("Connect to server: {0}".format(couchdb_server_alias))
# hook to source DB
source_db_name = options['source_db_name']
source_db = couch.connect(source_db_name,
couchdb_server_settings=settings.
COUCHDB_SERVERS[couchdb_server_alias])
self.logger.info("Hooked to source DB: {0}".format(source_db_name))
# hook to dest DB (creating it if non-existing)
couchdb_dest_name = settings.COUCHDB_SIMPLIFIED_NAME
couchdb_dest_settings = settings.COUCHDB_SERVERS[couchdb_server_alias]
couchdb_dest = couch.connect(
couchdb_dest_name, couchdb_server_settings=couchdb_dest_settings)
# create couch bulk writer
self.cbw = couch.CouchBulkWriter(logger=self.logger,
couchdb_dest=couchdb_dest)
self.logger.info(
"Hooked to destination DB: {0}".format(couchdb_dest_name))
###
# Mapping file and simplified leaves subtrees
###
# connect to google account and fetch tree mapping and simple tree structure
voci_map = gdocs.get_simple_map(n_header_lines=2,
force_google=force_google)
simplified_subtrees_leaves = gdocs.get_simplified_leaves(
force_google=force_google)
for city_id in cities:
dest_doc_id = city_id
if skip_existing:
if dest_doc_id in couchdb_dest:
self.logger.info(
u"Skipping city of {}, as already existing".format(
city_id))
continue
# create destination document, to REPLACE old one
# NB: the useless timestamps serves the only function to work around a bug in COUCHDB that
# if the written doc is exactly the same as the new doc then it will not be written
destination_document = {
'_id': city_id,
'useless_timestamp': timestamp
}
# if a doc with that id already exists on the destination document, gets the _rev value
# and insert it in the dest. document.
# this avoids document conflict on writing
# otherwise you should delete the old doc before writing the new one
old_destination_doc = couchdb_dest.get(city_id, None)
if old_destination_doc:
revision = old_destination_doc.get('_rev', None)
if revision:
destination_document['_rev'] = revision
self.logger.debug("Adds rev value to doc")
self.logger.info(u"Processing city of {0}".format(city_id, ))
for year in years:
# need this for logging
self.city = city_id
self.year = year
# get the source doc
doc_id = "{0}_{1}".format(year, city_id)
if doc_id not in source_db:
self.logger.warning(
u"Could not find {} in bilanci_voci couchdb instance. Skipping."
.format(doc_id))
continue
source_doc = source_db.get(doc_id)
# build the sub-trees, using the mapping and the source doc
# catch exceptions for non-existing sections in source doc
preventivo_tree = {}
consuntivo_tree = {}
try:
if 'preventivo' in source_doc and source_doc['preventivo']:
preventivo_entrate_tree = PreventivoEntrateBudgetTreeDict(
logger=self.logger).build_tree(
leaves=simplified_subtrees_leaves[
'preventivo-entrate'],
mapping=(voci_map['preventivo'], source_doc))
preventivo_tree.update(preventivo_entrate_tree)
preventivo_spese_tree = PreventivoSpeseBudgetTreeDict(
logger=self.logger).build_tree(
leaves=simplified_subtrees_leaves[
'preventivo-spese'],
mapping=(voci_map['preventivo'],
voci_map['interventi'], source_doc))
preventivo_tree.update(preventivo_spese_tree)
else:
self.logger.warning(
u"Could not find preventivo in source doc [{}]".
format(source_doc.get('_id')))
if 'consuntivo' in source_doc and source_doc['consuntivo']:
consuntivo_entrate_tree = ConsuntivoEntrateBudgetTreeDict(
logger=self.logger).build_tree(
leaves=simplified_subtrees_leaves[
'consuntivo-entrate'],
mapping=(voci_map['consuntivo'], source_doc))
consuntivo_tree.update(consuntivo_entrate_tree)
consuntivo_spese_tree = ConsuntivoSpeseBudgetTreeDict(
logger=self.logger).build_tree(
leaves=simplified_subtrees_leaves[
'consuntivo-spese'],
mapping=(voci_map['consuntivo'],
voci_map['interventi'], source_doc))
consuntivo_tree.update(consuntivo_spese_tree)
# creates branch RIASSUNTIVO
consuntivo_riassuntivo_tree = ConsuntivoRiassuntivoBudgetTreeDict(
logger=self.logger).build_tree(
leaves=simplified_subtrees_leaves[
'consuntivo-riassuntivo'],
mapping=(voci_map['consuntivo'], source_doc))
consuntivo_tree.update(consuntivo_riassuntivo_tree)
else:
self.logger.warning(
u"Could not find consuntivo in source doc [{}]".
format(source_doc.get('_id')))
except (SubtreeDoesNotExist, SubtreeIsEmpty) as e:
self.logger.error(e)
year_tree = {
'preventivo': preventivo_tree,
'consuntivo': consuntivo_tree,
}
destination_document[str(year)] = year_tree
if not dryrun:
# write doc to couchdb dest
ret = self.cbw.write(destination_document)
if ret is False:
email_utils.send_notification_email(
msg_string='Simplify has encountered problems')
self.logger.critical("Write critical problem. Quit")
exit()
if not dryrun:
# if the buffer in CBW is non-empty, flushes the docs to the db
ret = self.cbw.close()
if ret is False:
email_utils.send_notification_email(
msg_string='Simplify has encountered problems')
email_utils.send_notification_email(msg_string="Simplify has finished")