def test_pads_with_zeroes_when_string(self):
self.assertEqual("0400", municipality_code_from_county_code("04"))
def test_converts_from_string_to_string(self):
self.assertEqual("1200", municipality_code_from_county_code("12"))
def test_pads_with_zeroes_when_float(self):
self.assertEqual("0400", municipality_code_from_county_code(4.0))
def test_returns_none_for_none(self):
self.assertEqual(None, municipality_code_from_county_code(None))
def test_returns_none_for_empty_string(self):
self.assertEqual(None, municipality_code_from_county_code(""))
def test_converts_zero_float(self):
self.assertEqual("0000", municipality_code_from_county_code(0.0))
def _import_from_work_sheet(self, work_sheet, year, target_group):
def _parse_value(value):
if isinstance(value, (int, float, long)):
if value == 0:
value = None
elif variable.type == TYPE_BOOLEAN[0]:
value = (True if value == 1 else False)
elif variable.type == TYPE_INTEGER[0]:
value = int(value)
elif variable.type == TYPE_LONG[0]:
value = long(value)
if (isinstance(value, str) and value.strip() == ""):
value = None
return value
variable_keys = []
library_name_column = -1
for i in range(0, work_sheet.ncols):
key = work_sheet.cell_value(0, i)
variables = Variable.objects.filter(key=key)
if len(variables) > 0:
variable = variables[0]
if variable.sub_category in [u"Biblioteksnamn"]:
library_name_column = i
variable_keys.append((i, variable))
if library_name_column == -1:
raise CommandError(u"Library identifier variable not found, aborting!")
if not variable_keys:
raise CommandError(u"Failed to find any variables, aborting!")
municipality_code_column = -1
county_code_column = -1
for i in range(0, work_sheet.ncols):
if work_sheet.cell_value(1, i) == u"Kommunkod":
municipality_code_column = i
break
elif work_sheet.cell_value(1, i) == u"Länskod":
county_code_column = i
break
if county_code_column == -1 and municipality_code_column == -1:
raise CommandError("Could not find municipality or county code, aborting!")
num_imported_surveys = 0
for i in range(2, work_sheet.nrows):
row = work_sheet.row_values(i)
lib_col_value = row[library_name_column]
# Research libraries file and hospital libraries file has summary rows mixed with library response rows
if (lib_col_value and isinstance(lib_col_value, basestring)
and not lib_col_value.startswith(("Summa", "summa", "Riket",))):
library_name = lib_col_value.strip()
else:
continue
if Survey.objects.filter(library__name=library_name, sample_year=year):
continue
if municipality_code_column != -1:
municipality_code = municipality_code_from(row[municipality_code_column])
elif county_code_column != -1:
municipality_code = municipality_code_from_county_code(row[county_code_column])
if target_group == "specbib":
library_type = {
u"Nationalbibliotek": u"natbib",
u"Högskolebibliotek": u"univbib",
u"Specialbibliotek": u"specbib"
}[row[2]]
else:
library_type = target_group
library = Library(name=library_name, library_type=library_type)
if municipality_code is not None:
library.municipality_code = municipality_code
survey = Survey(sample_year=year, library=library, selected_libraries=[library.sigel])
for col, variable in variable_keys:
survey.observations.append(
SurveyObservation(variable=variable, value=_parse_value(row[col]), _is_public=variable.is_public))
survey.save().publish()
num_imported_surveys += 1
logger.info(u"...{} surveys imported".format(num_imported_surveys))
def _import_from_work_sheet(self, work_sheet, year, target_group):
def _parse_value(value):
if isinstance(value, (int, float, long)):
if value == 0:
value = None
elif variable.type == TYPE_BOOLEAN[0]:
value = (True if value == 1 else False)
elif variable.type == TYPE_INTEGER[0]:
value = int(value)
elif variable.type == TYPE_LONG[0]:
value = long(value)
if (isinstance(value, str) and value.strip() == ""):
value = None
return value
variable_keys = []
library_name_column = -1
for i in range(0, work_sheet.ncols):
key = work_sheet.cell_value(0, i)
variables = Variable.objects.filter(key=key)
if len(variables) > 0:
variable = variables[0]
if variable.sub_category in [u"Biblioteksnamn"]:
library_name_column = i
variable_keys.append((i, variable))
if library_name_column == -1:
raise CommandError(
u"Library identifier variable not found, aborting!")
if not variable_keys:
raise CommandError(u"Failed to find any variables, aborting!")
municipality_code_column = -1
county_code_column = -1
for i in range(0, work_sheet.ncols):
if work_sheet.cell_value(1, i) == u"Kommunkod":
municipality_code_column = i
break
elif work_sheet.cell_value(1, i) == u"Länskod":
county_code_column = i
break
if county_code_column == -1 and municipality_code_column == -1:
raise CommandError(
"Could not find municipality or county code, aborting!")
num_imported_surveys = 0
for i in range(2, work_sheet.nrows):
row = work_sheet.row_values(i)
lib_col_value = row[library_name_column]
# Research libraries file and hospital libraries file has summary rows mixed with library response rows
if (lib_col_value and isinstance(lib_col_value, basestring)
and not lib_col_value.startswith((
"Summa",
"summa",
"Riket",
))):
library_name = lib_col_value.strip()
else:
continue
if Survey.objects.filter(library__name=library_name,
sample_year=year):
continue
if municipality_code_column != -1:
municipality_code = municipality_code_from(
row[municipality_code_column])
elif county_code_column != -1:
municipality_code = municipality_code_from_county_code(
row[county_code_column])
if target_group == "specbib":
library_type = {
u"Nationalbibliotek": u"natbib",
u"Högskolebibliotek": u"univbib",
u"Specialbibliotek": u"specbib"
}[row[2]]
else:
library_type = target_group
library = Library(name=library_name, library_type=library_type)
if municipality_code is not None:
library.municipality_code = municipality_code
survey = Survey(sample_year=year,
library=library,
selected_libraries=[library.sigel])
for col, variable in variable_keys:
survey.observations.append(
SurveyObservation(variable=variable,
value=_parse_value(row[col]),
_is_public=variable.is_public))
survey.save().publish()
num_imported_surveys += 1
logger.info(u"...{} surveys imported".format(num_imported_surveys))