def stop(exit_code, msg=None):
try:
cherrypy.engine.exit()
cherrypy.server.stop()
logger.info('Now shutting down test server on port {}'.format(
sc.config['test']['port']))
time.sleep(1)
except NameError:
pass
time.sleep(0.5)
if exit_code == 0:
logger.info(
_green_text('''\
Test Server returns HTTP 200 Responses. Production should be good to go.'''))
save_last_good()
else:
logger.error(
_red_text('''\
CRITICAL PROBLEM! [AKA Bad Dev, Don't break the server!]
A test instance of the server using the current branch code
{}. This is a critical problem causing
all or much of the site to generate error pages instead of the intended
pages.
Please attempt to reproduce this problem on a development server
or the staging server.
The production server has not been restarted and is still running the
pre-updated code. It should still be functioning.
However it may be in an inconsistent state or generating errors.
If you are unable to fix the problem, please inform the primary server
administrator that this problem has occurred.
This warning should not be bypassed as if the production server is
restarted now it will almost certainly be severely broken.'''.format(msg)))
os._exit(exit_code)
def start_server():
setup_logging()
app.setup()
sc.config['global']['server.socket_port'] = sc.config['test']['port']
logger.info('Now starting test server on port {}'.format(
sc.config['test']['port']))
app.mount()
def stop(exit_code, msg=None):
try:
cherrypy.engine.exit()
cherrypy.server.stop()
logger.info('Now shutting down test server on port {}'.format(sc.config['test']['port']))
time.sleep(1)
except NameError:
pass
time.sleep(0.5)
if exit_code == 0:
logger.info(_green_text('''\
Test Server returns HTTP 200 Responses. Production should be good to go.'''))
save_last_good()
else:
logger.error(_red_text('''\
CRITICAL PROBLEM! [AKA Bad Dev, Don't break the server!]
A test instance of the server using the current branch code
{}. This is a critical problem causing
all or much of the site to generate error pages instead of the intended
pages.
Please attempt to reproduce this problem on a development server
or the staging server.
The production server has not been restarted and is still running the
pre-updated code. It should still be functioning.
However it may be in an inconsistent state or generating errors.
If you are unable to fix the problem, please inform the primary server
administrator that this problem has occurred.
This warning should not be bypassed as if the production server is
restarted now it will almost certainly be severely broken.'''.format(msg)))
os._exit(exit_code)
def log_and_reyield_actions(self, actions):
for action in actions:
self.action = action
try:
logger.info('{}: {_type}/{_id}'.format(action.get('_op_type', 'index'),**action))
except:
print(action)
raise
yield action
def log_and_reyield_actions(self, actions):
for action in actions:
self.action = action
try:
logger.info('{}: {_type}/{_id}'.format(
action.get('_op_type', 'index'), **action))
except:
print(action)
raise
yield action
def roll_back():
import sc.scm
try:
with open(sc.config['test']['last_good_rev_file'], 'r') as f:
revs = json.load(f)
except FileNotFoundError:
logger.info('Unable to rollback')
return False
sc.scm.scm.checkout(revs['code'])
sc.scm.data_scm.checkout(revs['data'])
logger.info('Successfully rolled back to last known good state')
def roll_back():
import sc.scm
try:
with open(sc.config['test']['last_good_rev_file'], 'r') as f:
revs = json.load(f)
except FileNotFoundError:
logger.info('Unable to rollback')
return False
sc.scm.scm.checkout(revs['code'])
sc.scm.data_scm.checkout(revs['data'])
logger.info('Successfully rolled back to last known good state')
def update_data(self):
most_recent = None
params = {}
if most_recent:
params['since'] = most_recent
data = self.get_data(params)
logger.info('New categories: {}, new topics: {}, new posts: {}'.format(len(data['categories']),
len(data['topics']),
len(data['posts'])))
actions = self.yield_actions(data)
actions = self.log_and_reyield_actions(actions)
actions = self.update_time_stamp_and_reyield_actions(actions)
self.actions = actions
self.process_actions(actions)
def update_data(self):
most_recent = None
params = {}
if most_recent:
params['since'] = most_recent
data = self.get_data(params)
if not data:
return
logger.info('New categories: {}, new topics: {}, new posts: {}'.format(
len(data['categories']), len(data['topics']), len(data['posts'])))
actions = self.yield_actions(data)
actions = self.log_and_reyield_actions(actions)
actions = self.update_time_stamp_and_reyield_actions(actions)
self.actions = actions
self.process_actions(actions)
def test_server():
base_url = 'http://{host}:{port}'.format(**sc.config['test'])
attempts = 0
max_attempts = 2
time.sleep(5)
logger.info('Now loading test pages to confirm server stability')
while attempts < max_attempts:
if attempts != 0:
time.sleep(10)
attempts += 1
try:
for path in sc.config['test']['paths']:
url = urllib.parse.urljoin(base_url, path)
response = urllib.request.urlopen(url)
logger.info('{} [{}]'.format(url, response.code))
if response.code == 200:
continue
else:
break
# Success!
stop(0)
except urllib.error.URLError as e:
logger.info('{} : {!s}'.format(url, e))
logger.error(
_red_text('Failed to achieve a 200 response for {}'.format(url)))
stop(1, 'fails to generate HTTP 200 Responses')
def test_server():
base_url = 'http://{host}:{port}'.format(**sc.config['test'])
attempts = 0
max_attempts = 2
time.sleep(5)
logger.info('Now loading test pages to confirm server stability')
while attempts < max_attempts:
if attempts != 0:
time.sleep(10)
attempts += 1
try:
for path in sc.config['test']['paths']:
url = urllib.parse.urljoin(base_url, path)
response = urllib.request.urlopen(url)
logger.info('{} [{}]'.format(url, response.code))
if response.code == 200:
continue
else:
break
# Success!
stop(0)
except urllib.error.URLError as e:
logger.info('{} : {!s}'.format(url, e))
logger.error(_red_text('Failed to achieve a 200 response for {}'.format(url)))
stop(1, 'fails to generate HTTP 200 Responses')
def build(self, lang_dir, force=False):
# The pagenumbinator should be scoped because it uses
# a large chunk of memory which should be gc'd.
# But it shouldn't be created at all if we don't need it.
# So we use a getter, and delete it when we are done.
self._ppn = None
codepoints = set()
bold_codepoints = set()
italic_codepoints = set()
lang_uid = lang_dir.stem
all_files = sorted(lang_dir.glob('**/*.html'), key=lambda f: sc.util.numericsortkey(f.stem))
files = [f for f in all_files if f.stem == 'metadata'] + [f for f in all_files if f.stem != 'metadata']
for i, htmlfile in enumerate(files):
try:
if not self._should_process_file(htmlfile, force):
continue
logger.info('Adding file: {!s}'.format(htmlfile))
uid = htmlfile.stem
root = html.parse(str(htmlfile)).getroot()
#Set codepoint data
_stack = [root]
while _stack:
e = _stack.pop()
if self.is_bold(lang_uid, e):
bold_codepoints.update(e.text_content())
elif self.is_italic(lang_uid, e):
italic_codepoints.update(e.text_content())
else:
_stack.extend(e)
codepoints.update(root.text_content())
# Set the previous and next uids, using explicit data
# if available, otherwise making a safe guess.
# The safe guess relies on comparing uids, and will not
# capture relationships such as the order of patimokha
# rules.
prev_uid = root.get('data-prev')
next_uid = root.get('data-next')
if not (prev_uid or next_uid):
if i > 0:
prev_uid = files[i - 1].stem
if not self.uids_are_related(uid, prev_uid):
prev_uid = None
if i + 1 < len(files):
next_uid = files[i + 1].stem
if not self.uids_are_related(uid, next_uid):
next_uid = None
path = htmlfile.relative_to(sc.text_dir)
author = self._get_author(root, lang_uid, uid)
if uid == 'metadata':
if author is None:
raise ValueError('Metadata file {} does not define author'.format(path))
self.add_metadata(path, author, root)
continue
if author is None:
metadata = self.get_metadata(path)
if metadata:
author = metadata['author']
if author is None:
metadata = root.select_one('#metaarea')
if metadata:
metadata_text = metadata.text_content()
m = regex.match(r'.{,80}\.', metadata_text)
if not m:
m = regex.match(r'.{,80}(?=\s)', metadata_text)
if m:
author = m[0]
if author is None:
logger.warn('Could not determine author for {}/{}'.format(lang_uid, uid))
author = ''
name = self._get_name(root, lang_uid, uid)
volpage = self._get_volpage(root, lang_uid, uid)
embedded = self._get_embedded_uids(root, lang_uid, uid)
fstat = htmlfile.stat()
cdate = self.datestr(fstat.st_ctime)
mdate = self.datestr(fstat.st_mtime)
textinfo = TextInfo(uid=uid, lang=lang_uid, path=path,
name=name, author=author,
volpage=volpage, prev_uid=prev_uid,
next_uid=next_uid,
cdate=cdate,
mdate=mdate,
file_uid=uid)
self.add_text_info(lang_uid, uid, textinfo)
for child in embedded:
child.path = path
child.author = author
child.file_uid = uid
self.add_text_info(lang_uid, child.uid, child)
m = regex.match(r'(.*?)(\d+)-(\d+)$', uid)
if m:
range_textinfo = TextInfo(uid=uid+'#',
lang=lang_uid,
path=path,
name=name,
author=author,
volpage=volpage,
file_uid=uid)
start = int(m[2])
end = int(m[3]) + 1
for i in range(start, end):
iuid = m[1] + str(i)
if self.exists(iuid, lang_uid):
continue
self.add_text_info(lang_uid, iuid, range_textinfo)
except Exception as e:
print('An exception occured: {!s}'.format(htmlfile))
raise
self._codepoints[lang_uid] = {
'normal': codepoints,
'bold': bold_codepoints,
'italic': italic_codepoints
}
del self._ppn
def build(self, force=False):
# The pagenumbinator should be scoped because it uses
# a large chunk of memory which should be gc'd.
# But it shouldn't be created at all if we don't need it.
# So we use a getter, and delete it when we are done.
self._ppn = None
file_i = 0
file_of_total_i = 0
percent = 0
file_count = sum(1 for _ in sc.text_dir.glob('**/*.html'))
for lang_dir in sc.text_dir.glob('*'):
lang_uid = lang_dir.stem
files = sorted(lang_dir.glob('**/*.html'), key=lambda f: sc.util.numericsortkey(f.stem))
for i, htmlfile in enumerate(files):
try:
if not self._should_process_file(htmlfile, force):
continue
logger.info('Adding file: {!s}'.format(htmlfile))
uid = htmlfile.stem
root = html.parse(str(htmlfile)).getroot()
# Set the previous and next uids, using explicit data
# if available, otherwise making a safe guess.
# The safe guess relies on comparing uids, and will not
# capture relationships such as the order of patimokha
# rules.
prev_uid = root.get('data-prev')
next_uid = root.get('data-next')
if not (prev_uid or next_uid):
if i > 0:
prev_uid = files[i - 1].stem
if not self.uids_are_related(uid, prev_uid):
prev_uid = None
if i + 1 < len(files):
next_uid = files[i + 1].stem
if not self.uids_are_related(uid, next_uid):
next_uid = None
path = htmlfile.relative_to(sc.text_dir)
author = self._get_author(root, lang_uid, uid)
name = self._get_name(root, lang_uid, uid)
volpage = self._get_volpage(root, lang_uid, uid)
embedded = self._get_embedded_uids(root, lang_uid, uid)
fstat = htmlfile.stat()
cdate = self.datestr(fstat.st_ctime)
mdate = self.datestr(fstat.st_mtime)
textinfo = TextInfo(uid=uid, lang=lang_uid, path=path,
name=name, author=author,
volpage=volpage, prev_uid=prev_uid,
next_uid=next_uid,
cdate=cdate,
mdate=mdate)
self.add_text_info(lang_uid, uid, textinfo)
for child in embedded:
child.path = path
child.author = author
self.add_text_info(lang_uid, child.uid, child)
m = regex.match(r'(.*?)(\d+)-(\d+)$', uid)
if m:
range_textinfo = TextInfo(uid=uid+'#', lang=lang_uid, path=path, name=name, author=author, volpage=volpage)
start = int(m[2])
end = int(m[3]) + 1
for i in range(start, end):
iuid = m[1] + str(i)
if self.exists(iuid, lang_uid):
continue
self.add_text_info(lang_uid, iuid, range_textinfo)
file_i += 1
if (file_i % self.FILES_N) == 0:
self._on_n_files()
file_of_total_i += 1
new_percent = int(0.5 + 100 * file_of_total_i / file_count)
if new_percent > percent:
percent = new_percent
self.build_process(percent)
except Exception as e:
print('An exception occured: {!s}'.format(htmlfile))
raise
if (file_i % self.FILES_N) != 0:
self._on_n_files()
del self._ppn
def start_server():
setup_logging()
app.setup()
sc.config['global']['server.socket_port'] = sc.config['test']['port']
logger.info('Now starting test server on port {}'.format(sc.config['test']['port']))
app.mount()
def build(self, lang_dir, force=False):
# The pagenumbinator should be scoped because it uses
# a large chunk of memory which should be gc'd.
# But it shouldn't be created at all if we don't need it.
# So we use a getter, and delete it when we are done.
self._ppn = None
codepoints = set()
bold_codepoints = set()
italic_codepoints = set()
lang_uid = lang_dir.stem
all_files = sorted(lang_dir.glob('**/*.html'), key=lambda f: sc.util.numericsortkey(f.stem))
files = [f for f in all_files if f.stem == 'metadata'] + [f for f in all_files if f.stem != 'metadata']
for i, htmlfile in enumerate(files):
try:
if not self._should_process_file(htmlfile, force):
continue
logger.info('Adding file: {!s}'.format(htmlfile))
uid = htmlfile.stem
root = html.parse(str(htmlfile)).getroot()
#Set codepoint data
_stack = [root]
while _stack:
e = _stack.pop()
if self.is_bold(lang_uid, e):
bold_codepoints.update(e.text_content())
elif self.is_italic(lang_uid, e):
italic_codepoints.update(e.text_content())
else:
_stack.extend(e)
codepoints.update(root.text_content())
# Set the previous and next uids, using explicit data
# if available, otherwise making a safe guess.
# The safe guess relies on comparing uids, and will not
# capture relationships such as the order of patimokha
# rules.
prev_uid = root.get('data-prev')
next_uid = root.get('data-next')
if not (prev_uid or next_uid):
if i > 0:
prev_uid = files[i - 1].stem
if not self.uids_are_related(uid, prev_uid):
prev_uid = None
if i + 1 < len(files):
next_uid = files[i + 1].stem
if not self.uids_are_related(uid, next_uid):
next_uid = None
path = htmlfile.relative_to(sc.text_dir)
author = self._get_author(root, lang_uid, uid)
if uid == 'metadata':
if author is None:
raise ValueError('Metadata file {} does not define author'.format(path))
self.add_metadata(path, author, root)
continue
if author is None:
metadata = self.get_metadata(path)
if metadata:
author = metadata['author']
if author is None:
metadata = root.select_one('#metaarea')
if metadata:
metadata_text = metadata.text_content()
m = regex.match(r'.{,80}\.', metadata_text)
if not m:
m = regex.match(r'.{,80}(?=\s)', metadata_text)
if m:
author = m[0]
if author is None:
logger.warn('Could not determine author for {}/{}'.format(lang_uid, uid))
author = ''
name = self._get_name(root, lang_uid, uid)
volpage = self._get_volpage(root, lang_uid, uid)
embedded = self._get_embedded_uids(root, lang_uid, uid)
fstat = htmlfile.stat()
cdate = self.datestr(fstat.st_ctime)
mdate = self.datestr(fstat.st_mtime)
textinfo = TextInfo(uid=uid, lang=lang_uid, path=path,
name=name, author=author,
volpage=volpage, prev_uid=prev_uid,
next_uid=next_uid,
cdate=cdate,
mdate=mdate,
file_uid=uid)
self.add_text_info(lang_uid, uid, textinfo)
for child in embedded:
child.path = path
child.author = author
child.file_uid = uid
self.add_text_info(lang_uid, child.uid, child)
m = regex.match(r'(.*?)(\d+)-(\d+)$', uid)
if m:
range_textinfo = TextInfo(uid=uid+'#',
lang=lang_uid,
path=path,
name=name,
author=author,
volpage=volpage,
file_uid=uid)
start = int(m[2])
end = int(m[3]) + 1
for i in range(start, end):
iuid = m[1] + str(i)
if self.exists(iuid, lang_uid):
continue
self.add_text_info(lang_uid, iuid, range_textinfo)
except Exception as e:
print('An exception occured: {!s}'.format(htmlfile))
raise
self._codepoints[lang_uid] = {
'normal': codepoints,
'bold': bold_codepoints,
'italic': italic_codepoints
}
del self._ppn
