def load_tables(self, tables_source): with file_adapter(tables_source) as tables_file: reader = unicodecsv.DictReader(tables_file, encoding='utf-8') slugs = set([fieldname.split(':')[0] for fieldname in reader.fieldnames]) for slug in slugs: self.tables[slug] = [] for row in reader: for slug in slugs: table_item = {} for language in self.languages: value = row.get('%s:%s' % (slug, language), '') if not value and language == self.native_language: value = row.get(slug, '') if value: if value.find(';') > 0: value = value.split(';') table_item.setdefault('languages', {})[language] = value elif table_item: # some language already present but current is missing raise exceptions.WrongCSVData( 'Missing language "%(language)s" for table "%(slug)s" with partial datum "%(table_item)s"', language=language, slug=slug, table_item=table_item, ) if table_item: self.tables[slug].append(table_item) return
def load_tables(self, tables_source): with file_adapter(tables_source) as tables_file: reader = unicodecsv.DictReader(tables_file, encoding='utf-8') slugs = set( [fieldname.split(':')[0] for fieldname in reader.fieldnames]) for slug in slugs: self.tables[slug] = [] for row in reader: for slug in slugs: table_item = {} for language in self.languages: value = row.get('%s:%s' % (slug, language), '') if not value and language == self.native_language: value = row.get(slug, '') if value: if value.find(';') > 0: value = value.split(';') table_item.setdefault('languages', {})[language] = value elif table_item: # some language already present but current is missing raise exceptions.WrongCSVData( 'Missing language "%(language)s" for table "%(slug)s" with partial datum "%(table_item)s"', language=language, slug=slug, table_item=table_item, ) if table_item: self.tables[slug].append(table_item) return
def test_file_adapter(self): if not DJANGO_INSTALLED: return root_dir = os.path.dirname(pynames.__file__) test_file_path = os.path.join(root_dir, 'tests', 'fixtures', 'test_from_list_generator.json') with open(test_file_path) as f: target_content = f.read() with file_adapter(test_file_path) as f: self.assertEqual(f.read(), target_content) django_file_object = ContentFile(target_content) classic_file_object = open(test_file_path, 'r') for tested_file_object in [django_file_object, classic_file_object]: with file_adapter(tested_file_object) as f: self.assertEqual(f.read(), target_content)
def test_file_adapter(self): if not DJANGO_INSTALLED: return root_dir = os.path.dirname(pynames.__file__) test_file_path = os.path.join(root_dir, 'tests', 'fixtures', 'test_from_list_generator.json') with open(test_file_path, 'rb') as f: target_content = f.read() with file_adapter(test_file_path) as f: self.assertEqual(f.read(), target_content) django_file_object = ContentFile(target_content) classic_file_object = open(test_file_path, 'rb') for tested_file_object in [django_file_object, classic_file_object]: with file_adapter(tested_file_object) as f: self.assertEqual(f.read(), target_content)
def source_loader(self, source): if source is None: error_msg = 'FromTablesGenerator: you must make subclass of FromTablesGenerator and define attribute SOURCE in it.' raise NotImplementedError(error_msg) with file_adapter(source) as f: data = json.loads(f.read().decode('utf-8')) self.native_language = data['native_language'] self.languages = set(data['languages']) self.full_forms_for_languages = set(data.get('full_forms_for_languages', set())) self.templates = [ Template(template_name, self.native_language, self.languages, template_data) for template_name, template_data in data['templates'].items() ] self.tables = data['tables']
def load_settings(self, settings_source): with file_adapter(settings_source) as settings_file: reader = unicodecsv.DictReader(settings_file, encoding='utf-8') for row in reader: new_native_language = row.get('native_language', '').strip() if new_native_language and not self.native_language: self.native_language = new_native_language elif self.native_language and new_native_language and self.native_language != new_native_language: raise exceptions.WrongCSVData( 'Wrong settings csv file. Native language is already set to "%(native_language)s" but new value "%(new_value)s" is present on some row', native_language=self.native_language, new_value=new_native_language) new_language = row.get('languages', '').strip() if new_language: self.languages.append(new_language) self.languages = set(self.languages)
def load_templates(self, templates_source): template_slugs = [] with file_adapter(templates_source) as templates_file: reader = unicodecsv.DictReader(templates_file, encoding='utf-8') for row in reader: template_data = { 'probability': float(row['probability']), 'genders': row['genders'].replace(' ', '').split(';'), 'template': row['template'].split(';'), } self.templates.append( Template(row['template_name'], self.native_language, self.languages, template_data)) template_slugs.extend(template_data['template']) template_slugs = set(template_slugs) return template_slugs
def load_templates(self, templates_source): template_slugs = [] with file_adapter(templates_source) as templates_file: reader = unicodecsv.DictReader(templates_file, encoding='utf-8') for row in reader: template_data = { 'probability': float(row['probability']), 'genders': row['genders'].replace(' ', '').split(';'), 'template': row['template'].split(';'), } self.templates.append( Template(row['template_name'], self.native_language, self.languages, template_data) ) template_slugs.extend(template_data['template']) template_slugs = set(template_slugs) return template_slugs
def load_settings(self, settings_source): with file_adapter(settings_source) as settings_file: reader = unicodecsv.DictReader(settings_file, encoding='utf-8') for row in reader: new_native_language = row.get('native_language', '').strip() if new_native_language and not self.native_language: self.native_language = new_native_language elif self.native_language and new_native_language and self.native_language != new_native_language: raise exceptions.WrongCSVData( 'Wrong settings csv file. Native language is already set to "%(native_language)s" but new value "%(new_value)s" is present on some row', native_language=self.native_language, new_value=new_native_language ) new_language = row.get('languages', '').strip() if new_language: self.languages.append(new_language) self.languages = set(self.languages)