def __init__(self, errors_dict=None): DoubleStringPatternSplitCollection.__init__(self) self._match = {} self._match_jp = {} if errors_dict is None: self._errors_dict = None else: errors_dict['denormals'] = [] self._errors_dict = errors_dict['denormals']
def test_double_strings_line_split(self): doubles = DoubleStringPatternSplitCollection () splits = doubles.split_line('"Key1","Val1"') self.assertIsNotNone(splits) self.assertEqual(2, len(splits)) self.assertEqual("Key1", splits[0]) self.assertEqual("Val1", splits[1]) splits = doubles.split_line('') self.assertIsNone(splits)
def test_double_strings_line_split(self): doubles = DoubleStringPatternSplitCollection() splits = doubles.split_line('"Key1","Val1"') self.assertIsNotNone(splits) self.assertEqual(2, len(splits)) self.assertEqual("Key1", splits[0]) self.assertEqual("Val1", splits[1]) splits = doubles.split_line('') self.assertIsNone(splits)
def test_double_strings_from_file(self): doubles = DoubleStringPatternSplitCollection () self.assertIsNotNone(doubles) count = doubles.load_from_filename(os.path.dirname(__file__) + os.sep + "test_files" + os.sep + "doubles_pattern.txt") self.assertEqual(count, 3) self.assertTrue(doubles.has_key("key1")) #self.assertEqual(doubles.value("key1"), "(^key1|key1|key1$)") self.assertTrue(doubles.has_key("key2")) #self.assertEqual(doubles.value("key2"), "(^key2|key2|key2$)") self.assertTrue(doubles.has_key("key3"))
def _load_file_contents(self, lookup_collection, filename): YLogger.debug(self, "Loading lookup [%s]", filename) try: line_no = 0 with open(filename, 'r', encoding='utf8') as my_file: for line in my_file: line_no += 1 line = line.strip() if line: if line[0] == '#': continue splits = DoubleStringPatternSplitCollection.split_line_by_pattern( line, DoubleStringPatternSplitCollection. RE_OF_SPLIT_PATTERN) if splits and len(splits) > 1: if type(lookup_collection ) is DoubleStringPatternSplitCollection: index, pattern = self.process_key_value( splits[0], splits[1]) lookup_collection.add_to_lookup(index, pattern) else: lookup_collection.add_to_lookup( splits[0], splits[1], filename, line_no) else: if type( lookup_collection ) is not DoubleStringPatternSplitCollection: error_info = "illegal format [%s]" % line lookup_collection.set_error_info( filename, line_no, error_info) except Exception as excep: YLogger.exception(self, "Failed to load lookup [%s]", excep, filename)
def load(self, lookup_collection): YLogger.debug(self, "Loading lookup from Mongo [%s]", self.collection_name()) collection = self.collection() lookups = collection.find() if lookups is not None: for lookup in lookups: key, value = DoubleStringPatternSplitCollection.process_key_value( lookup['key'], lookup['value']) lookup_collection.add_to_lookup(key, value)
def test_double_strings_from_file(self): doubles = DoubleStringPatternSplitCollection() self.assertIsNotNone(doubles) count = doubles.load_from_filename( os.path.dirname(__file__) + "/test_files/doubles_pattern.txt") self.assertEqual(count, 3) self.assertTrue(doubles.has_key("key1")) self.assertEqual(doubles.value("key1"), "(^key1|key1|key1$)") self.assertTrue(doubles.has_key("key2")) self.assertEqual(doubles.value("key2"), "(^key2|key2|key2$)") self.assertTrue(doubles.has_key("key3")) self.assertEqual(doubles.value("key3"), "(^key3|key3|key3$)")
def load(self, collector, name=None): YLogger.debug(self, "Loading lookup from Mongo [%s]", self.collection_name()) collection = self.collection() lookups = collection.find() for lookup in lookups: key, value = DoubleStringPatternSplitCollection.process_key_value( lookup['key'], lookup['value']) collector.add_to_lookup(key, value) return True
def _load_file_contents(self, lookup_collection, filename): YLogger.debug(self, "Loading lookup [%s]", filename) try: with open(filename, 'r', encoding='utf8') as my_file: for line in my_file: if line: splits = DoubleStringPatternSplitCollection.split_line_by_pattern(line, DoubleStringPatternSplitCollection.RE_OF_SPLIT_PATTERN) if splits and len(splits) > 1: if type(lookup_collection) is DoubleStringPatternSplitCollection: index, pattern = self.process_key_value(splits[0], splits[1]) lookup_collection.add_to_lookup(index, pattern) else: lookup_collection.add_to_lookup(splits[0], splits[1]) except Exception as excep: YLogger.exception(self, "Failed to load lookup [%s]", excep, filename)
def test_double_strings_from_text(self): doubles = DoubleStringPatternSplitCollection() self.assertIsNotNone(doubles) count = doubles.load_from_text(""" "key1","val1" "key2","val2" "key3","val3,val4" """) self.assertEqual(count, 3) self.assertTrue(doubles.has_key("key1")) self.assertTrue(doubles.has_key("key2")) self.assertTrue(doubles.has_key("key3")) self.assertFalse(doubles.has_key("key4")) doubles.empty() self.assertEqual(0, len(doubles.pairs))
def add_to_lookup(self, org_key, org_value): key = org_key value = org_value.strip() if JapaneseLanguage.is_CJKword(org_key) is True: key = key.strip() if key in self._pairs_jp: YLogger.error(self, "%s = %s already exists in jp_collection", key, value) return else: matchs = self._match_jp splits = key check_key = key[0] self._pairs_jp[key] = value else: if key[0] != ' ': key = key.strip() pattern_text = DoubleStringPatternSplitCollection.normalise_pattern( key) start = pattern_text.lstrip() middle = pattern_text end = pattern_text.rstrip() pattern = "(^%s|%s|%s$)" % (start, middle, end) replacement = value replaceInfo = [key, re.compile(pattern), replacement] self._replace.append(replaceInfo) return else: key = key.strip() if key in self._pairs: YLogger.error(self, "%s = %s already exists in en_collection", key, value) return else: matchs = self._match splits = key.split() check_key = splits[0] self._pairs[key] = value if check_key not in matchs: matchs[check_key] = [] matchs[check_key].append(splits)
def test_double_strings_from_text(self): doubles = DoubleStringPatternSplitCollection() self.assertIsNotNone(doubles) count = doubles.load_from_text(""" "key1","val1" "key2","val2" "key3","val3,val4" """) self.assertEqual(count, 3) self.assertTrue(doubles.has_key("key1")) #self.assertEqual(doubles.value("key1"), "(^key1|key1|key1$)") self.assertTrue(doubles.has_key("key2")) #self.assertEqual(doubles.value("key2"), "(^key2|key2|key2$)") self.assertTrue(doubles.has_key("key3")) #self.assertEqual(doubles.value("key3"), "(^key3|key3|key3$)") self.assertFalse(doubles.has_key("key4")) self.assertIsNone(doubles.value("keyX"))
def test_double_strings_from_text(self): doubles = DoubleStringPatternSplitCollection () self.assertIsNotNone(doubles) count = doubles.load_from_text(""" "key1","val1" "key2","val2" "key3","val3,val4" """) self.assertEqual(count, 3) self.assertTrue(doubles.has_key("key1")) #self.assertEqual(doubles.value("key1"), "(^key1|key1|key1$)") self.assertTrue(doubles.has_key("key2")) #self.assertEqual(doubles.value("key2"), "(^key2|key2|key2$)") self.assertTrue(doubles.has_key("key3")) #self.assertEqual(doubles.value("key3"), "(^key3|key3|key3$)") self.assertFalse(doubles.has_key("key4")) self.assertIsNone(doubles.value("keyX"))
def process_key_value(self, key, value, id=None): return DoubleStringPatternSplitCollection.process_key_value( key, value, id)
def __init__(self): DoubleStringPatternSplitCollection.__init__(self)
def __init__(self, process_splits_success=True): DoubleStringPatternSplitCollection.__init__(self) self._process_splits_success = process_splits_success
def add_to_lookup(self, org_key, org_value, filename=None, line=0): key = org_key value = org_value.strip() if key.strip() == '': error_info = "key is empty" self.set_error_info(filename, line, error_info) return if JapaneseLanguage.is_CJKword(org_key) is True: key = key.strip() if key in self._pairs_jp: YLogger.error(self, "%s = %s already exists in jp_collection", key, value) error_info = "duplicate key='%s' (value='%s' is invalid)" % ( key, value) self.set_error_info(filename, line, error_info) return else: matchs = self._match_jp splits = key check_key = key[0] self._pairs_jp[key] = value else: if key[0] != ' ': key = key.strip() if key in self._replace_key: YLogger.error( self, "%s = %s already exists in replace_collection", key, value) error_info = "duplicate replace_chars='%s' (value='%s' is invalid)" % ( key, value) self.set_error_info(filename, line, error_info) return pattern_text = DoubleStringPatternSplitCollection.normalise_pattern( key) start = pattern_text.lstrip() middle = pattern_text end = pattern_text.rstrip() pattern = "(^%s|%s|%s$)" % (start, middle, end) replacement = value replaceInfo = [key, re.compile(pattern), replacement] self._replace.append(replaceInfo) self._replace_key.append(key) return else: key = key.strip() if key in self._pairs: YLogger.error(self, "%s = %s already exists in en_collection", key, value) error_info = "duplicate key='%s' (value='%s' is invalid)" % ( key, value) self.set_error_info(filename, line, error_info) return else: matchs = self._match splits = key.split() check_key = splits[0] self._pairs[key] = value if check_key not in matchs: matchs[check_key] = [] matchs[check_key].append(splits)
def load(self, collector, name=None): db_lookups = self._get_all() for db_lookup in db_lookups: key, value = DoubleStringPatternSplitCollection.process_key_value( db_lookup.key, db_lookup.value) collector.add_to_lookup(key, value)
def split_into_fields(self, line): return DoubleStringPatternSplitCollection.split_line_by_pattern( line, DoubleStringPatternSplitCollection.RE_OF_SPLIT_PATTERN)
def __init__(self, process_splits_success=True): DoubleStringPatternSplitCollection.__init__(self) self._process_splits_success = process_splits_success
def __init__(self): DoubleStringPatternSplitCollection.__init__(self) self._match = {} self._match_jp = {}