def gui_parse():
global __file__ # to fix stupid
__file__ = os.path.abspath(__file__) # __file__ handling
_file_ = os.path.basename(__file__) # in python 2
global debug
root = Tk()
root.withdraw()
sys.stderr = codecs.getwriter('utf8')(sys.stderr)
parser = argparse.ArgumentParser()
parser.add_argument('filename', nargs='?')
parser.add_argument('--debug', '-d', action='store_true')
args = parser.parse_args()
if args.debug:
debug = True
if args.filename is None:
args.filename = tkFileDialog.askopenfilename(
filetypes=[
('Word 2007+','*.docx'),
('Plain text','*.txt'),
])
os.chdir(os.path.dirname(os.path.abspath(args.filename)))
if os.path.splitext(args.filename)[1] == '.txt':
with codecs.open(args.filename, 'r', 'utf8') as input_file:
input_text = input_file.read()
input_text = input_text.replace('\r','')
final_structure = chgk_parse(input_text)
elif os.path.splitext(args.filename)[1] == '.docx':
from pydocx import PyDocX
from bs4 import BeautifulSoup
from parse import parse
import base64
import html2text
input_docx = PyDocX.to_html(args.filename)
bsoup = BeautifulSoup(input_docx)
if args.debug:
with codecs.open('debug.pydocx', 'w', 'utf8') as dbg:
dbg.write(input_docx)
def generate_imgname(ext):
imgcounter = 1
while os.path.isfile('{:03}.{}'
.format(imgcounter, ext)):
imgcounter += 1
return '{:03}.{}'.format(imgcounter, ext)
for tag in bsoup.find_all('style'):
tag.extract()
for tag in bsoup.find_all('p'):
if tag.string:
tag.string = tag.string + SEP
for tag in bsoup.find_all('b'):
tag.unwrap()
for tag in bsoup.find_all('strong'):
tag.unwrap()
for tag in bsoup.find_all('i'):
tag.string = '_' + tag.string + '_'
tag.unwrap()
for tag in bsoup.find_all('em'):
tag.string = '_' + tag.string + '_'
tag.unwrap()
for tag in bsoup.find_all('li'):
if tag.string:
tag.string = '- ' + tag.string
for tag in bsoup.find_all('img'):
imgparse = parse('data:image/{ext};base64,{b64}', tag['src'])
imgname = generate_imgname(imgparse['ext'])
tag.insert_before('(img {})'.format(imgname))
if not args.debug:
with open(imgname, 'wb') as f:
f.write(base64.b64decode(imgparse['b64']))
tag.extract()
for tag in bsoup.find_all('a'):
if rew(tag.string) == '':
tag.extract()
else:
tag.string = tag['href']
tag.unwrap()
h = html2text.HTML2Text()
h.body_width = 0
txt = (h.handle(bsoup.prettify())
.replace('\\-','')
.replace('\\.','.')
.replace('( ', '(')
.replace('[ ', '[')
.replace(' )', ')')
.replace(' ]', ']')
.replace(' :', ':')
)
if args.debug:
with codecs.open('debug.debug', 'w', 'utf8') as dbg:
dbg.write(txt)
final_structure = chgk_parse(txt)
else:
sys.stderr.write('Error: unsupported file format.' + SEP)
sys.exit()
os.chdir(os.path.dirname(os.path.abspath(__file__)))
with codecs.open(
make_filename(args.filename), 'w', 'utf8') as output_file:
output_file.write(
compose_4s(final_structure))
print('Please review the resulting file {}:'.format(
make_filename(args.filename)))
subprocess.call(shlex.split('{} "{}"'
.format(
TEXTEDITOR,
make_filename(args.filename)).encode('cp1251',errors='replace')))
def chgk_parse(text):
"""
Parsing rationale: every Question has two required fields: 'question' and
the immediately following 'answer'. All the rest are optional, as is
the order of these fields. On the other hand, everything
except the 'question' is obligatorily marked, while the 'question' is
optionally marked. But IF the question is not marked, 'meta' comments
between Questions will not be parsed as 'meta' but will be merged to
'question's.
Parsing is done by regexes in the following steps:
1. Identify all the fields you can, mark them with their respective
labels, mark all the others with ''
2. Merge fields inside Question with '' lines between them
3. Ensure every 'answer' has a 'question'
4. Mark all remaining '' fields as 'meta'
5. Prettify input
6. Pack Questions into dicts
7. Return the resulting structure
"""
BADNEXTFIELDS = set(['question', 'answer'])
WHITESPACE = set([' ', ' ', '\n', '\r'])
PUNCTUATION = set([',', '.', ':', ';', '?', '!'])
re_tour = re.compile(r'^ТУР ?([0-9IVXLCDM]*)([\.:])?$', re.I | re.U)
re_tourrev = re.compile(r'^([0-9]+) ТУР([\.:])?$', re.I | re.U)
re_question = re.compile(r'ВОПРОС ?[№N]?([0-9]*) ?[\.:]', re.I | re.U)
re_answer = re.compile(r'ОТВЕТЫ? ?[№N]?([0-9]+)? ?[:]', re.I | re.U)
re_zachet = re.compile(r'ЗАЧ[ЕЁ]Т ?[\.:]', re.I | re.U)
re_nezachet = re.compile(r'НЕЗАЧ[ЕЁ]Т ?[\.:]', re.I | re.U)
re_comment = re.compile(r'КОММЕНТАРИ[ИЙ] ?[№N]?([0-9]+)? ?[\.:]', re.I | re.U)
re_author = re.compile(r'АВТОР\(?Ы?\)? ?[\.:]', re.I | re.U)
re_source = re.compile(r'ИСТОЧНИК\(?И?\)? ?[\.:]', re.I | re.U)
re_editor = re.compile(r'РЕДАКТОР(Ы|СКАЯ ГРУППА)? ?[\.:]', re.I | re.U)
re_date = re.compile(r'ДАТА ?[\.:]', re.I | re.U)
re_handout = re.compile(r'РАЗДА(ЧА|ТКА|ТОЧНЫЙ МАТЕРИАЛ) ?[\.:]', re.I | re.U)
re_number = re.compile(r'^[0-9]+[\.\)] *')
regexes = {
'tour' : re_tour,
'tourrev' : re_tourrev,
'question' : re_question,
'answer' : re_answer,
'zachet' : re_zachet,
'nezachet' : re_nezachet,
'comment' : re_comment,
'author' : re_author,
'source' : re_source,
'editor' : re_editor,
'date' : re_date,
}
chgk_parse.structure = []
def merge_to_previous(index):
target = index - 1
chgk_parse.structure[target][1] = (
chgk_parse.structure[target][1] + SEP
+ chgk_parse.structure.pop(index)[1])
def merge_to_next(index):
target = chgk_parse.structure.pop(index)
chgk_parse.structure[index][1] = (target[1] + SEP
+ chgk_parse.structure[index][1])
def find_next_specific_field(index, fieldname):
target = index + 1
while chgk_parse.structure[target][0] != fieldname:
target += 1
return target
def find_next_fieldname(index):
target = index + 1
if target < len(chgk_parse.structure):
debug_print(pprint.pformat(
chgk_parse.structure[target]))
while (target < len(chgk_parse.structure)-1
and chgk_parse.structure[target][0] == ''):
target += 1
return chgk_parse.structure[target][0]
def merge_y_to_x(x, y):
i = 0
while i < len(chgk_parse.structure):
if chgk_parse.structure[i][0] == x:
while (i+1 < len(chgk_parse.structure)
and chgk_parse.structure[i+1][0] != y):
merge_to_previous(i+1)
i += 1
def merge_to_x_until_nextfield(x):
i = 0
while i < len(chgk_parse.structure):
if chgk_parse.structure[i][0] == x:
while (i+1 < len(chgk_parse.structure)
and chgk_parse.structure[i+1][0] == ''
and find_next_fieldname(i) not in BADNEXTFIELDS):
merge_to_previous(i+1)
i += 1
def dirty_merge_to_x_until_nextfield(x):
i = 0
while i < len(chgk_parse.structure):
if chgk_parse.structure[i][0] == x:
while (i+1 < len(chgk_parse.structure)
and chgk_parse.structure[i+1][0] == ''):
merge_to_previous(i+1)
i += 1
def swap_elements(x, y):
z = chgk_parse.structure[y]
chgk_parse.structure[y] = chgk_parse.structure[x]
chgk_parse.structure[x] = z
# 1.
for x in re.split(r'\r?\n',text):
if x != '':
chgk_parse.structure.append(['',rew(x)])
i = 0
st = chgk_parse.structure
while i < len(st):
matching_regexes = {(regex, regexes[regex].search(st[i][1]).start(0))
for regex in regexes if regexes[regex].search(st[i][1])}
# If more than one regex matches string, split it and
# insert into structure separately.
if len(matching_regexes) == 1:
st[i][0] = matching_regexes.pop()[0]
elif len(matching_regexes) > 1:
sorted_r = sorted(matching_regexes, key=lambda x: x[1])
slices = []
for j in range(1, len(sorted_r)):
slices.append(
[sorted_r[j][0], st[i][1][
sorted_r[j][1]
:
sorted_r[j+1][1] if j+1 < len(sorted_r)
else len(st[i][1])]])
for slice_ in slices:
chgk_parse.structure.insert(
i+1, slice_)
st[i][0] = sorted_r[0][0]
st[i][1] = st[i][1][:sorted_r[1][1]]
i += 1
chgk_parse.structure = st
i = 0
# 2.
merge_y_to_x('question','answer')
merge_to_x_until_nextfield('answer')
merge_to_x_until_nextfield('comment')
# 3.
i = 0
while i < len(chgk_parse.structure):
if (chgk_parse.structure[i][0] == 'answer'
and chgk_parse.structure[i-1][0] not in ('question',
'newquestion')):
chgk_parse.structure.insert(i,['newquestion',''])
i = 0
i += 1
i = 0
while i < len(chgk_parse.structure) - 1:
if (chgk_parse.structure[i][0] == ''
and chgk_parse.structure[i+1][0] == 'newquestion'):
merge_to_next(i)
if (re_number.search(
rew(chgk_parse.structure[i][1])) and
not re_number.search(
rew(chgk_parse.structure[i-1][1]))):
chgk_parse.structure[i][0] = 'question'
chgk_parse.structure[i][1] = re_number.sub('',rew(
chgk_parse.structure[i][1]))
try:
chgk_parse.structure.insert(i, ['number', int(re_number.search(
rew(chgk_parse.structure[i][1])).group(0))])
except:
pass
i = 0
i += 1
for element in chgk_parse.structure:
if element[0] == 'newquestion':
element[0] = 'question'
dirty_merge_to_x_until_nextfield('source')
for id, element in enumerate(chgk_parse.structure):
if (element[0] == 'author' and re.search(r'^{}$'.format(re_author.
pattern),
rew(element[1]))
and id + 1 < len(chgk_parse.structure)):
merge_to_previous(id+1)
merge_to_x_until_nextfield('zachet')
merge_to_x_until_nextfield('nezachet')
# 4.
chgk_parse.structure = [x for x in chgk_parse.structure if [x[0], rew(x[1])]
!= ['', '']]
if chgk_parse.structure[0][0] == '' and re_number.search(
rew(chgk_parse.structure[0][1])):
merge_to_next(0)
for id, element in enumerate(chgk_parse.structure):
if element[0] == '':
element[0] = 'meta'
if element[0] in regexes and element[0] not in ['tour', 'tourrev']:
if element[0] == 'question':
try:
num = re_question.search(element[1]).group(1)
chgk_parse.structure.insert(id, ['number', num])
except:
pass
element[1] = regexes[element[0]].sub('', element[1])
# 5.
for id, element in enumerate(chgk_parse.structure):
# typogrify
if element[0] != 'date':
element[1] = typotools.recursive_typography(element[1])
# remove question numbers
if element[0] == 'question':
try:
num = re_question.search(element[1]).group(1)
chgk_parse.structure.insert(id, ['number', num])
except:
pass
element[1] = re_number.sub('', element[1])
# detect inner lists
mo = {m for m
in re.finditer(r'(\s+|^)(\d+)[\.\)]\s*(?!\d)',element[1], re.U)}
if len(mo) > 1:
sorted_up = sorted(mo, key=lambda m: int(m.group(2)))
j = 0
list_candidate = []
while j == int(sorted_up[j].group(2)) - 1:
list_candidate.append((j+1, sorted_up[j].group(0),
sorted_up[j].start()))
if j+1 < len(sorted_up):
j += 1
else:
break
if len(list_candidate) > 1:
if (element[0] != 'question' or
(element[0] == 'question'
and 'дуплет' in element[1].lower()
or 'блиц' in element[1].lower())):
part = partition(element[1], [x[2] for x in
list_candidate])
lc = 0
while lc < len(list_candidate):
part[lc+1] = part[lc+1].replace(list_candidate[lc][1], '')
lc += 1
element[1] = ([part[0], part[1:]] if part[0] != ''
else part[1:])
# turn source into list if necessary
if (element[0] == 'source' and isinstance(element[1], basestring)
and len(re.split(r'\r?\n', element[1])) > 1):
element[1] = [re_number.sub('', rew(x))
for x in re.split(r'\r?\n', element[1])]
# 6.
final_structure = []
current_question = {}
for element in chgk_parse.structure:
if element[0] in set(['tour', 'question', 'meta']):
if current_question != {}:
check_question(current_question)
final_structure.append(['Question', current_question])
current_question = {}
if element[0] in QUESTION_LABELS:
if element[0] in current_question:
try:
current_question[element[0]] += SEP + element[1]
except:
print('{}'.format(current_question).decode('unicode_escape'))
pdb.set_trace()
else:
current_question[element[0]] = element[1]
else:
final_structure.append([element[0], element[1]])
if current_question != {}:
check_question(current_question)
final_structure.append(['Question', current_question])
# 7.
debug_print(pprint.pformat(final_structure).decode('unicode_escape'))
return final_structure
def parse_4s(s):
mapping = {
'#' : 'meta',
'##' : 'section',
'###' : 'heading',
'###LJ': 'ljheading',
'#EDITOR': 'editor',
'#DATE': 'date',
'?': 'question',
'№': 'number',
'№№': 'setcounter',
'!': 'answer',
'=': 'zachet',
'!=': 'nezachet',
'^': 'source',
'/': 'comment',
'@': 'author',
'>': 'handout',
}
structure = []
if s[0] == '\ufeff' and len(s) > 1:
s = s[1:]
with codecs.open('raw.debug', 'w', 'utf8') as debugf:
debugf.write(pprint.pformat(s.split('\n')).decode('unicode_escape'))
for line in s.split('\n'):
if rew(line) == '':
structure.append(['', ''])
else:
if line.split()[0] in mapping:
structure.append([mapping[line.split()[0]],
rew(line[
len(line.split()[0]):])])
else:
if len(structure) > 1:
structure[len(structure)-1][1] += '\n' + line
final_structure = []
current_question = {}
counter = 1
if debug:
with codecs.open('debug1st.debug', 'w', 'utf8') as debugf:
debugf.write(pprint.pformat(structure).decode('unicode_escape'))
for element in structure:
# find list in element
sp = element[1].split('\n')
if len(sp) > 1:
list_candidate = []
for line in sp:
if len(rew(line).split())>1 and rew(line).split()[0] == '-':
list_candidate.append(
rew(
rew(
line
)[1:]
))
sp = [spsp for spsp in sp if rew(rew(spsp)[1:]) not in list_candidate]
if len(sp) == 0 or len(sp) == 1 and sp[0] == '':
element[1] = list_candidate
else:
element[1] = (['\n'.join(sp), list_candidate]
if len(list_candidate)>1
else '\n'.join(element[1].split('\n')))
if element[0] in QUESTION_LABELS:
if element[0] in current_question:
if (isinstance(current_question[element[0]], basestring)
and isinstance(element[1], basestring)):
current_question[element[0]] += '\n' + element[1]
elif (isinstance(current_question[element[0]], list)
and isinstance(element[1], basestring)):
current_question[element[0]][0] += '\n' + element[1]
elif (isinstance(current_question[element[0]], basestring)
and isinstance(element[1], list)):
current_question[element[0]] = [element[1][0] + '\n'
+ current_question[element[0]], element[1][1]]
elif (isinstance(current_question[element[0]], list)
and isinstance(element[1], list)):
current_question[element[0]][0] += '\n' + element[1][0]
current_question[element[0]][1] += element[1][1]
else:
current_question[element[0]] = element[1]
elif element[0] == '':
if current_question != {}:
assert all(True for label in REQUIRED_LABELS
if label in current_question)
if 'setcounter' in current_question:
counter = int(current_question['setcounter'])
if not 'number' in current_question:
current_question['number'] = counter
counter += 1
final_structure.append(['Question', current_question])
current_question = {}
else:
final_structure.append([element[0], element[1]])
if current_question != {}:
assert all(True for label in REQUIRED_LABELS
if label in current_question)
if 'setcounter' in current_question:
counter = int(current_question['setcounter'])
if not 'number' in current_question:
current_question['number'] = counter
counter += 1
final_structure.append(['Question', current_question])
if debug:
with codecs.open('debug.debug', 'w', 'utf8') as debugf:
debugf.write(pprint.pformat(final_structure))
return final_structure
