def stop(exit_code, msg=None): try: cherrypy.engine.exit() cherrypy.server.stop() logger.info('Now shutting down test server on port {}'.format( sc.config['test']['port'])) time.sleep(1) except NameError: pass time.sleep(0.5) if exit_code == 0: logger.info( _green_text('''\ Test Server returns HTTP 200 Responses. Production should be good to go.''')) save_last_good() else: logger.error( _red_text('''\ CRITICAL PROBLEM! [AKA Bad Dev, Don't break the server!] A test instance of the server using the current branch code {}. This is a critical problem causing all or much of the site to generate error pages instead of the intended pages. Please attempt to reproduce this problem on a development server or the staging server. The production server has not been restarted and is still running the pre-updated code. It should still be functioning. However it may be in an inconsistent state or generating errors. If you are unable to fix the problem, please inform the primary server administrator that this problem has occurred. This warning should not be bypassed as if the production server is restarted now it will almost certainly be severely broken.'''.format(msg))) os._exit(exit_code)
def start_server(): setup_logging() app.setup() sc.config['global']['server.socket_port'] = sc.config['test']['port'] logger.info('Now starting test server on port {}'.format( sc.config['test']['port'])) app.mount()
def stop(exit_code, msg=None): try: cherrypy.engine.exit() cherrypy.server.stop() logger.info('Now shutting down test server on port {}'.format(sc.config['test']['port'])) time.sleep(1) except NameError: pass time.sleep(0.5) if exit_code == 0: logger.info(_green_text('''\ Test Server returns HTTP 200 Responses. Production should be good to go.''')) save_last_good() else: logger.error(_red_text('''\ CRITICAL PROBLEM! [AKA Bad Dev, Don't break the server!] A test instance of the server using the current branch code {}. This is a critical problem causing all or much of the site to generate error pages instead of the intended pages. Please attempt to reproduce this problem on a development server or the staging server. The production server has not been restarted and is still running the pre-updated code. It should still be functioning. However it may be in an inconsistent state or generating errors. If you are unable to fix the problem, please inform the primary server administrator that this problem has occurred. This warning should not be bypassed as if the production server is restarted now it will almost certainly be severely broken.'''.format(msg))) os._exit(exit_code)
def log_and_reyield_actions(self, actions): for action in actions: self.action = action try: logger.info('{}: {_type}/{_id}'.format(action.get('_op_type', 'index'),**action)) except: print(action) raise yield action
def log_and_reyield_actions(self, actions): for action in actions: self.action = action try: logger.info('{}: {_type}/{_id}'.format( action.get('_op_type', 'index'), **action)) except: print(action) raise yield action
def roll_back(): import sc.scm try: with open(sc.config['test']['last_good_rev_file'], 'r') as f: revs = json.load(f) except FileNotFoundError: logger.info('Unable to rollback') return False sc.scm.scm.checkout(revs['code']) sc.scm.data_scm.checkout(revs['data']) logger.info('Successfully rolled back to last known good state')
def update_data(self): most_recent = None params = {} if most_recent: params['since'] = most_recent data = self.get_data(params) logger.info('New categories: {}, new topics: {}, new posts: {}'.format(len(data['categories']), len(data['topics']), len(data['posts']))) actions = self.yield_actions(data) actions = self.log_and_reyield_actions(actions) actions = self.update_time_stamp_and_reyield_actions(actions) self.actions = actions self.process_actions(actions)
def update_data(self): most_recent = None params = {} if most_recent: params['since'] = most_recent data = self.get_data(params) if not data: return logger.info('New categories: {}, new topics: {}, new posts: {}'.format( len(data['categories']), len(data['topics']), len(data['posts']))) actions = self.yield_actions(data) actions = self.log_and_reyield_actions(actions) actions = self.update_time_stamp_and_reyield_actions(actions) self.actions = actions self.process_actions(actions)
def test_server(): base_url = 'http://{host}:{port}'.format(**sc.config['test']) attempts = 0 max_attempts = 2 time.sleep(5) logger.info('Now loading test pages to confirm server stability') while attempts < max_attempts: if attempts != 0: time.sleep(10) attempts += 1 try: for path in sc.config['test']['paths']: url = urllib.parse.urljoin(base_url, path) response = urllib.request.urlopen(url) logger.info('{} [{}]'.format(url, response.code)) if response.code == 200: continue else: break # Success! stop(0) except urllib.error.URLError as e: logger.info('{} : {!s}'.format(url, e)) logger.error( _red_text('Failed to achieve a 200 response for {}'.format(url))) stop(1, 'fails to generate HTTP 200 Responses')
def test_server(): base_url = 'http://{host}:{port}'.format(**sc.config['test']) attempts = 0 max_attempts = 2 time.sleep(5) logger.info('Now loading test pages to confirm server stability') while attempts < max_attempts: if attempts != 0: time.sleep(10) attempts += 1 try: for path in sc.config['test']['paths']: url = urllib.parse.urljoin(base_url, path) response = urllib.request.urlopen(url) logger.info('{} [{}]'.format(url, response.code)) if response.code == 200: continue else: break # Success! stop(0) except urllib.error.URLError as e: logger.info('{} : {!s}'.format(url, e)) logger.error(_red_text('Failed to achieve a 200 response for {}'.format(url))) stop(1, 'fails to generate HTTP 200 Responses')
def build(self, lang_dir, force=False): # The pagenumbinator should be scoped because it uses # a large chunk of memory which should be gc'd. # But it shouldn't be created at all if we don't need it. # So we use a getter, and delete it when we are done. self._ppn = None codepoints = set() bold_codepoints = set() italic_codepoints = set() lang_uid = lang_dir.stem all_files = sorted(lang_dir.glob('**/*.html'), key=lambda f: sc.util.numericsortkey(f.stem)) files = [f for f in all_files if f.stem == 'metadata'] + [f for f in all_files if f.stem != 'metadata'] for i, htmlfile in enumerate(files): try: if not self._should_process_file(htmlfile, force): continue logger.info('Adding file: {!s}'.format(htmlfile)) uid = htmlfile.stem root = html.parse(str(htmlfile)).getroot() #Set codepoint data _stack = [root] while _stack: e = _stack.pop() if self.is_bold(lang_uid, e): bold_codepoints.update(e.text_content()) elif self.is_italic(lang_uid, e): italic_codepoints.update(e.text_content()) else: _stack.extend(e) codepoints.update(root.text_content()) # Set the previous and next uids, using explicit data # if available, otherwise making a safe guess. # The safe guess relies on comparing uids, and will not # capture relationships such as the order of patimokha # rules. prev_uid = root.get('data-prev') next_uid = root.get('data-next') if not (prev_uid or next_uid): if i > 0: prev_uid = files[i - 1].stem if not self.uids_are_related(uid, prev_uid): prev_uid = None if i + 1 < len(files): next_uid = files[i + 1].stem if not self.uids_are_related(uid, next_uid): next_uid = None path = htmlfile.relative_to(sc.text_dir) author = self._get_author(root, lang_uid, uid) if uid == 'metadata': if author is None: raise ValueError('Metadata file {} does not define author'.format(path)) self.add_metadata(path, author, root) continue if author is None: metadata = self.get_metadata(path) if metadata: author = metadata['author'] if author is None: metadata = root.select_one('#metaarea') if metadata: metadata_text = metadata.text_content() m = regex.match(r'.{,80}\.', metadata_text) if not m: m = regex.match(r'.{,80}(?=\s)', metadata_text) if m: author = m[0] if author is None: logger.warn('Could not determine author for {}/{}'.format(lang_uid, uid)) author = '' name = self._get_name(root, lang_uid, uid) volpage = self._get_volpage(root, lang_uid, uid) embedded = self._get_embedded_uids(root, lang_uid, uid) fstat = htmlfile.stat() cdate = self.datestr(fstat.st_ctime) mdate = self.datestr(fstat.st_mtime) textinfo = TextInfo(uid=uid, lang=lang_uid, path=path, name=name, author=author, volpage=volpage, prev_uid=prev_uid, next_uid=next_uid, cdate=cdate, mdate=mdate, file_uid=uid) self.add_text_info(lang_uid, uid, textinfo) for child in embedded: child.path = path child.author = author child.file_uid = uid self.add_text_info(lang_uid, child.uid, child) m = regex.match(r'(.*?)(\d+)-(\d+)$', uid) if m: range_textinfo = TextInfo(uid=uid+'#', lang=lang_uid, path=path, name=name, author=author, volpage=volpage, file_uid=uid) start = int(m[2]) end = int(m[3]) + 1 for i in range(start, end): iuid = m[1] + str(i) if self.exists(iuid, lang_uid): continue self.add_text_info(lang_uid, iuid, range_textinfo) except Exception as e: print('An exception occured: {!s}'.format(htmlfile)) raise self._codepoints[lang_uid] = { 'normal': codepoints, 'bold': bold_codepoints, 'italic': italic_codepoints } del self._ppn
def build(self, force=False): # The pagenumbinator should be scoped because it uses # a large chunk of memory which should be gc'd. # But it shouldn't be created at all if we don't need it. # So we use a getter, and delete it when we are done. self._ppn = None file_i = 0 file_of_total_i = 0 percent = 0 file_count = sum(1 for _ in sc.text_dir.glob('**/*.html')) for lang_dir in sc.text_dir.glob('*'): lang_uid = lang_dir.stem files = sorted(lang_dir.glob('**/*.html'), key=lambda f: sc.util.numericsortkey(f.stem)) for i, htmlfile in enumerate(files): try: if not self._should_process_file(htmlfile, force): continue logger.info('Adding file: {!s}'.format(htmlfile)) uid = htmlfile.stem root = html.parse(str(htmlfile)).getroot() # Set the previous and next uids, using explicit data # if available, otherwise making a safe guess. # The safe guess relies on comparing uids, and will not # capture relationships such as the order of patimokha # rules. prev_uid = root.get('data-prev') next_uid = root.get('data-next') if not (prev_uid or next_uid): if i > 0: prev_uid = files[i - 1].stem if not self.uids_are_related(uid, prev_uid): prev_uid = None if i + 1 < len(files): next_uid = files[i + 1].stem if not self.uids_are_related(uid, next_uid): next_uid = None path = htmlfile.relative_to(sc.text_dir) author = self._get_author(root, lang_uid, uid) name = self._get_name(root, lang_uid, uid) volpage = self._get_volpage(root, lang_uid, uid) embedded = self._get_embedded_uids(root, lang_uid, uid) fstat = htmlfile.stat() cdate = self.datestr(fstat.st_ctime) mdate = self.datestr(fstat.st_mtime) textinfo = TextInfo(uid=uid, lang=lang_uid, path=path, name=name, author=author, volpage=volpage, prev_uid=prev_uid, next_uid=next_uid, cdate=cdate, mdate=mdate) self.add_text_info(lang_uid, uid, textinfo) for child in embedded: child.path = path child.author = author self.add_text_info(lang_uid, child.uid, child) m = regex.match(r'(.*?)(\d+)-(\d+)$', uid) if m: range_textinfo = TextInfo(uid=uid+'#', lang=lang_uid, path=path, name=name, author=author, volpage=volpage) start = int(m[2]) end = int(m[3]) + 1 for i in range(start, end): iuid = m[1] + str(i) if self.exists(iuid, lang_uid): continue self.add_text_info(lang_uid, iuid, range_textinfo) file_i += 1 if (file_i % self.FILES_N) == 0: self._on_n_files() file_of_total_i += 1 new_percent = int(0.5 + 100 * file_of_total_i / file_count) if new_percent > percent: percent = new_percent self.build_process(percent) except Exception as e: print('An exception occured: {!s}'.format(htmlfile)) raise if (file_i % self.FILES_N) != 0: self._on_n_files() del self._ppn
def start_server(): setup_logging() app.setup() sc.config['global']['server.socket_port'] = sc.config['test']['port'] logger.info('Now starting test server on port {}'.format(sc.config['test']['port'])) app.mount()