def parse_4s_elem(s): def find_next_unescaped(ss, index): j = index + 1 while j < len(ss): if ss[j] == '\\' and j+2 < len(ss): j += 2 if ss[j] == ss[index]: return j j += 1 return -1 for gr in re_url.finditer(s): gr0 = gr.group(0) s = s.replace(gr0, gr0.replace('_', '\\_')) # for gr in re_scaps.finditer(s): # gr0 = gr.group(0) # s = s.replace(gr0, '(sc '+gr0.lower()+')') grs = sorted([match.group(0) for match in re_perc.finditer(s)], key=len, reverse=True) for gr in grs: try: s = s.replace(gr,urllib.unquote(gr.encode('utf8')).decode('utf8')) except: debug_print('error decoding on line {}: {}\n' .format(gr, traceback.format_exc())) s = list(s) i = 0 topart = [] while i < len(s): if s[i] == '_' and (i == 0 or s[i-1] != '\\'): debug_print('found _ at {} of line {}' .format(i, s)) topart.append(i) if find_next_unescaped(s, i) != -1: topart.append(find_next_unescaped(s, i)+1) i = find_next_unescaped(s, i) + 2 continue if (s[i] == '(' and i + len('(img') < len(s) and ''.join(s[i: i+len('(img')])=='(img'): debug_print('img candidate') topart.append(i) if not typotools.find_matching_closing_bracket(s, i) is None: topart.append( typotools.find_matching_closing_bracket(s, i)+1) i = typotools.find_matching_closing_bracket(s, i)+2 # if (s[i] == '(' and i + len('(sc') < len(s) and ''.join(s[i: # i+len('(sc')])=='(sc'): # debug_print('sc candidate') # topart.append(i) # if not typotools.find_matching_closing_bracket(s, i) is None: # topart.append( # typotools.find_matching_closing_bracket(s, i)+1) # i = typotools.find_matching_closing_bracket(s, i)+2 i += 1 topart = sorted(topart) parts = [['', ''.join(x)] for x in partition(s, topart)] debug_print(pprint.pformat(parts).decode('unicode_escape')) for part in parts: if part == ['', '']: continue try: if part[1][-1] == '_': part[1] = part[1][1:] part[0] = 'em' if part[1][-1] == '_': part[1] = part[1][:-1] part[0] = 'em' if len(part[1]) > 4 and part[1][:4] == '(img': if part[1][-1] != ')': part[1] = part[1] + ')' part[1] = typotools.remove_excessive_whitespace( part[1][4:-1]) part[0] = 'img' debug_print('found img at {}' .format(pprint.pformat(part[1]))) if len(part[1]) > 3 and part[1][:4] == '(sc': if part[1][-1] != ')': part[1] = part[1] + ')' part[1] = typotools.remove_excessive_whitespace( part[1][3:-1]) part[0] = 'sc' debug_print('found img at {}' .format(pprint.pformat(part[1]))) part[1] = part[1].replace('\\_', '_') except: sys.stderr.write('Error on part {}: {}' .format(pprint.pformat(part).decode('unicode_escape'), traceback.format_exc() )) return parts
def __init__(self, text): """ Parsing rationale: every Question has two required fields: 'question' and the immediately following 'answer'. All the rest are optional, as is the order of these fields. On the other hand, everything except the 'question' is obligatorily marked, while the 'question' is optionally marked. But IF the question is not marked, 'meta' comments between Questions will not be parsed as 'meta' but will be merged to 'question's. Parsing is done by regexes in the following steps: 1. Identify all the fields you can, mark them with their respective labels, mark all the others with '' 2. Merge fields inside Question with '' lines between them 3. Ensure every 'answer' has a 'question' 4. Mark all remaining '' fields as 'meta' 5. Prettify input 6. Pack Questions into dicts 7. Store the resulting structure as self.structure """ self.structure = [] # 1. for x in re.split(r'\r?\n',text): if x != '': self.structure.append(['',remove_excessive_whitespace(x)]) i = 0 st = self.structure while i < len(st): matching_regexes = { (regex, self.regexes[regex].search(st[i][1]).start(0)) for regex in self.regexes if self.regexes[regex].search(st[i][1])} # If more than one regex matches string, split it and # insert into structure separately. if len(matching_regexes) == 1: st[i][0] = matching_regexes.pop()[0] elif len(matching_regexes) > 1: sorted_r = sorted(matching_regexes, key=lambda x: x[1]) slices = [] for j in range(1, len(sorted_r)): slices.append( [sorted_r[j][0], st[i][1][ sorted_r[j][1] : sorted_r[j+1][1] if j+1 < len(sorted_r) else len(st[i][1])]]) for slice_ in slices: self.structure.insert( i+1, slice_) st[i][0] = sorted_r[0][0] st[i][1] = st[i][1][:sorted_r[1][1]] i += 1 self.structure = st i = 0 # 2. self.merge_y_to_x('question','answer') self.merge_to_x_until_nextfield('answer') self.merge_to_x_until_nextfield('comment') # 3. i = 0 while i < len(self.structure): if (self.structure[i][0] == 'answer' and self.structure[i-1][0] not in ('question', 'newquestion')): self.structure.insert(i,['newquestion','']) i = 0 i += 1 i = 0 while i < len(self.structure) - 1: if (self.structure[i][0] == '' and self.structure[i+1][0] == 'newquestion'): self.merge_to_next(i) if (re_number.search( remove_excessive_whitespace(self.structure[i][1])) and not re_number.search( remove_excessive_whitespace(self.structure[i-1][1]))): self.structure[i][0] = 'question' self.structure[i][1] = re_number.sub( '',remove_excessive_whitespace(self.structure[i][1])) try: self.structure.insert(i, ['number', int(re_number.search( remove_excessive_whitespace( self.structure[i][1]) ).group(0))]) except: pass # TODO: figure out what this means i = 0 i += 1 for element in self.structure: if element[0] == 'newquestion': element[0] = 'question' self.dirty_merge_to_x_until_nextfield('source') for id, element in enumerate(self.structure): if (element[0] == 'author' and re.search(r'^{}$'.format(re_author. pattern), remove_excessive_whitespace(element[1])) and id + 1 < len(self.structure)): merge_to_previous(id+1) self.merge_to_x_until_nextfield('accept') self.merge_to_x_until_nextfield('reject') # 4. self.structure = [x for x in self.structure if [x[0], remove_excessive_whitespace(x[1])] != ['', '']] if self.structure[0][0] == '' and re_number.search( remove_excessive_whitespace(self.structure[0][1])): self.merge_to_next(0) for id, element in enumerate(self.structure): if element[0] == '': element[0] = 'meta' if (element[0] in self.regexes and element[0] not in ['tour', 'tourrev']): if element[0] == 'question': try: num = re_question.search(element[1]).group(1) self.structure.insert(id, ['number', num]) except: pass element[1] = self.regexes[element[0]].sub('', element[1]) # 5. for id, element in enumerate(self.structure): # typogrify if element[0] != 'date': element[1] = recursive_typography(element[1]) # remove question numbers if element[0] == 'question': try: num = re_question.search(element[1]).group(1) self.structure.insert(id, ['number', num]) except: pass element[1] = re_number.sub('', element[1]) # detect inner lists mo = {m for m in re.finditer(r'(\s+|^)(\d+)[\.\)]\s*(?!\d)', element[1], re.U)} if len(mo) > 1: sorted_up = sorted(mo, key=lambda m: int(m.group(2))) j = 0 list_candidate = [] while j == int(sorted_up[j].group(2)) - 1: list_candidate.append((j+1, sorted_up[j].group(0), sorted_up[j].start())) if j+1 < len(sorted_up): j += 1 else: break if len(list_candidate) > 1: if (element[0] != 'question' or (element[0] == 'question' and 'дуплет' in element[1].lower() or 'блиц' in element[1].lower())): part = partition(element[1], [x[2] for x in list_candidate]) lc = 0 while lc < len(list_candidate): part[lc+1] = part[lc+1].replace( list_candidate[lc][1], '') lc += 1 element[1] = ([part[0], part[1:]] if part[0] != '' else part[1:]) # turn source into list if necessary if (element[0] == 'source' and isinstance(element[1], basestring) and len(re.split(r'\r?\n', element[1])) > 1): element[1] = [re_number.sub('', remove_excessive_whitespace(x)) for x in re.split(r'\r?\n', element[1])] # 6. final_structure = [] current_question = {} for element in self.structure: if element[0] in set(['tour', 'question', 'meta']): if current_question != {}: check_question(current_question) final_structure.append(Question(**current_question)) current_question = {} if element[0] in QUESTION_LABELS: if element[0] in current_question: try: current_question[element[0]] += SEP + element[1] except: print('{}'.format( current_question).decode('unicode_escape')) pdb.set_trace() else: current_question[element[0]] = element[1] else: final_structure.append([element[0], element[1]]) if current_question != {}: check_question(current_question) final_structure.append(Question(**current_question)) # 7. self.structure = final_structure