def load_tables(self, tables_source):
with file_adapter(tables_source) as tables_file:
reader = unicodecsv.DictReader(tables_file, encoding='utf-8')
slugs = set([fieldname.split(':')[0] for fieldname in reader.fieldnames])
for slug in slugs:
self.tables[slug] = []
for row in reader:
for slug in slugs:
table_item = {}
for language in self.languages:
value = row.get('%s:%s' % (slug, language), '')
if not value and language == self.native_language:
value = row.get(slug, '')
if value:
if value.find(';') > 0:
value = value.split(';')
table_item.setdefault('languages', {})[language] = value
elif table_item:
# some language already present but current is missing
raise exceptions.WrongCSVData(
'Missing language "%(language)s" for table "%(slug)s" with partial datum "%(table_item)s"',
language=language, slug=slug, table_item=table_item,
)
if table_item:
self.tables[slug].append(table_item)
return
def load_tables(self, tables_source):
with file_adapter(tables_source) as tables_file:
reader = unicodecsv.DictReader(tables_file, encoding='utf-8')
slugs = set(
[fieldname.split(':')[0] for fieldname in reader.fieldnames])
for slug in slugs:
self.tables[slug] = []
for row in reader:
for slug in slugs:
table_item = {}
for language in self.languages:
value = row.get('%s:%s' % (slug, language), '')
if not value and language == self.native_language:
value = row.get(slug, '')
if value:
if value.find(';') > 0:
value = value.split(';')
table_item.setdefault('languages',
{})[language] = value
elif table_item:
# some language already present but current is missing
raise exceptions.WrongCSVData(
'Missing language "%(language)s" for table "%(slug)s" with partial datum "%(table_item)s"',
language=language,
slug=slug,
table_item=table_item,
)
if table_item:
self.tables[slug].append(table_item)
return
def test_file_adapter(self):
if not DJANGO_INSTALLED:
return
root_dir = os.path.dirname(pynames.__file__)
test_file_path = os.path.join(root_dir, 'tests', 'fixtures', 'test_from_list_generator.json')
with open(test_file_path) as f:
target_content = f.read()
with file_adapter(test_file_path) as f:
self.assertEqual(f.read(), target_content)
django_file_object = ContentFile(target_content)
classic_file_object = open(test_file_path, 'r')
for tested_file_object in [django_file_object, classic_file_object]:
with file_adapter(tested_file_object) as f:
self.assertEqual(f.read(), target_content)
def test_file_adapter(self):
if not DJANGO_INSTALLED:
return
root_dir = os.path.dirname(pynames.__file__)
test_file_path = os.path.join(root_dir, 'tests', 'fixtures',
'test_from_list_generator.json')
with open(test_file_path, 'rb') as f:
target_content = f.read()
with file_adapter(test_file_path) as f:
self.assertEqual(f.read(), target_content)
django_file_object = ContentFile(target_content)
classic_file_object = open(test_file_path, 'rb')
for tested_file_object in [django_file_object, classic_file_object]:
with file_adapter(tested_file_object) as f:
self.assertEqual(f.read(), target_content)
def source_loader(self, source):
if source is None:
error_msg = 'FromTablesGenerator: you must make subclass of FromTablesGenerator and define attribute SOURCE in it.'
raise NotImplementedError(error_msg)
with file_adapter(source) as f:
data = json.loads(f.read().decode('utf-8'))
self.native_language = data['native_language']
self.languages = set(data['languages'])
self.full_forms_for_languages = set(data.get('full_forms_for_languages', set()))
self.templates = [
Template(template_name, self.native_language, self.languages, template_data)
for template_name, template_data in data['templates'].items()
]
self.tables = data['tables']
def load_settings(self, settings_source):
with file_adapter(settings_source) as settings_file:
reader = unicodecsv.DictReader(settings_file, encoding='utf-8')
for row in reader:
new_native_language = row.get('native_language', '').strip()
if new_native_language and not self.native_language:
self.native_language = new_native_language
elif self.native_language and new_native_language and self.native_language != new_native_language:
raise exceptions.WrongCSVData(
'Wrong settings csv file. Native language is already set to "%(native_language)s" but new value "%(new_value)s" is present on some row',
native_language=self.native_language,
new_value=new_native_language)
new_language = row.get('languages', '').strip()
if new_language:
self.languages.append(new_language)
self.languages = set(self.languages)
def load_templates(self, templates_source):
template_slugs = []
with file_adapter(templates_source) as templates_file:
reader = unicodecsv.DictReader(templates_file, encoding='utf-8')
for row in reader:
template_data = {
'probability': float(row['probability']),
'genders': row['genders'].replace(' ', '').split(';'),
'template': row['template'].split(';'),
}
self.templates.append(
Template(row['template_name'], self.native_language,
self.languages, template_data))
template_slugs.extend(template_data['template'])
template_slugs = set(template_slugs)
return template_slugs
def load_templates(self, templates_source):
template_slugs = []
with file_adapter(templates_source) as templates_file:
reader = unicodecsv.DictReader(templates_file, encoding='utf-8')
for row in reader:
template_data = {
'probability': float(row['probability']),
'genders': row['genders'].replace(' ', '').split(';'),
'template': row['template'].split(';'),
}
self.templates.append(
Template(row['template_name'], self.native_language, self.languages, template_data)
)
template_slugs.extend(template_data['template'])
template_slugs = set(template_slugs)
return template_slugs
def load_settings(self, settings_source):
with file_adapter(settings_source) as settings_file:
reader = unicodecsv.DictReader(settings_file, encoding='utf-8')
for row in reader:
new_native_language = row.get('native_language', '').strip()
if new_native_language and not self.native_language:
self.native_language = new_native_language
elif self.native_language and new_native_language and self.native_language != new_native_language:
raise exceptions.WrongCSVData(
'Wrong settings csv file. Native language is already set to "%(native_language)s" but new value "%(new_value)s" is present on some row',
native_language=self.native_language,
new_value=new_native_language
)
new_language = row.get('languages', '').strip()
if new_language:
self.languages.append(new_language)
self.languages = set(self.languages)
