def log_all(msg): for _ in range(0, 11): logger.debug(msg) logger.info(msg) logger.warning(msg) logger.error(msg) logger.critical(msg)
def parseHtml(cls, doc, highlight=None, parser='html.parser', source='unknown', target_source=None, item=None, id=None, title=None, link=None, abstract=None, details=None): records = [] try: if doc: soup = BeautifulSoup(doc, parser) log.debug(f"Got results from '{source}'.") elements = cls.__select(soup, item) for elt in elements: _source = source _target_source = target_source if target_source else source _id = cls.__select(elt, id) _title = cls.__select(elt, title) _link = cls.__select(elt, link) _abstract = cls.__select(elt, abstract) _details = cls.__select(elt, details) rec = cls.record(_source, _target_source, _id, _title, _link, _abstract, _details).to_dict() rec = Highlighter.highlight( rec, ['title', 'abstract'], highlight) log.info(f"Record: {rec}") records.append(rec) except Exception as ex: log.error(ex) traceback.print_exc(file=stdout) finally: return records
def destroy(cls): for hash in cls.trackers: log.info(f"Killing tracker {hash}.") cls.trackers[hash].kill() lockFile = join('temp', hash + '.lock') if Path(lockFile).is_file(): Path(lockFile).unlink()
def test_non_str_logging(self): logger.info(10) self.assertIn("10", self.last_line()) # Those should not throw any error. logger.debug([10, 20, 30]) logger.critical({}) logger.warning(set([-1, 4]))
def kill(self): self.running = False self.__status = 'dieing' log.info(f"[{self.id}] Waiting for flush...") self.join() self.__handleQ(True) log.info(f"[{self.id}] Final flush done.") self.__status = 'dead'
def test_message_is_logged(self): logger.debug("message 1") self.assertIn("message 1", self.last_line()) logger.info("message 2") self.assertIn("message 2", self.last_line()) logger.warning("message 3") self.assertIn("message 3", self.last_line()) logger.error("message 4") self.assertIn("message 4", self.last_line()) logger.critical("message 5") self.assertIn("message 5", self.last_line())
def test_filepath_is_logged(self): logger.debug("message 1") self.assertIn("test_colored_logger.py", self.last_line()) logger.info("message 2") self.assertIn("test_colored_logger.py", self.last_line()) logger.warning("message 3") self.assertIn("test_colored_logger.py", self.last_line()) logger.error("message 4") self.assertIn("test_colored_logger.py", self.last_line()) logger.critical("message 5") self.assertIn("test_colored_logger.py", self.last_line())
def test_level_is_logged(self): logger.debug("message 1") self.assertIn("DEBUG", self.last_line()) logger.info("message 2") self.assertIn("INFO", self.last_line()) logger.warning("message 3") self.assertIn("WARNING", self.last_line()) logger.error("message 4") self.assertIn("ERROR", self.last_line()) logger.critical("message 5") self.assertIn("CRITICAL", self.last_line())
def __handleQ(self, empty=False): # log.debug(f"[{self.id}] Handling change queue...") if not self.changesQ.empty(): changes = [] size = 0 while not self.changesQ.empty(): changes += self.changesQ.get() if not empty: if size > ChangeTracker.BULK_SIZE: break size += 1 self.__status = 'flushing' log.info(f"[{self.id}] Flushing {len(changes)} changes...") Change.objects.insert(changes) log.info(f"[{self.id}] Flushed {len(changes)} changes.") self.__status = 'idle'
def get(url: str, **kwargs): log.info(f"Request url={url} args={kwargs}") try: headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36' } res = requests.get(url, params=kwargs, timeout=5, headers=headers) log.debug(f"Response code={res.status_code}") if res.ok: with open(f"{url.replace('/', '_')}.html", 'w') as f: f.write(res.text) return res.text else: log.error(res.text) return None except requests.RequestException as ex: log.error(ex)
def __new__(cls, mongo_uri, collections, fields): hash = hashlib.md5( (mongo_uri + ''.join(collections)).encode('utf-8')).hexdigest() lockFile = join('temp', hash + '.lock') Path(lockFile).mkdir(parents=True, exist_ok=True) if not Path(lockFile).is_file(): Path(lockFile).touch() if hash not in cls.trackers: log.info( f"Start change tracker '{hash}' for {cls.__hide_pw(mongo_uri)}, collections=[{', '.join(collections)}] fields=[{', '.join(fields)}]" ) cls.trackers[hash] = cls.__ChangeTracker( mongo_uri, collections, fields) return cls.trackers[hash] else: log.warn( f"Change tracker '{hash}'' seams to be already started for {cls.__hide_pw(mongo_uri)}, collections=[{', '.join(collections)}] fields=[{', '.join(fields)}]" ) return cls.__ChangeTracker(None, None)
def log_all(): logger.debug("message 1") logger.info("message 2") logger.warning("message 3") logger.error("message 4") logger.critical("message 5")
def test_terminal_logging(self): logger.info("message to terminal device") self.assertIn("INFO", self.last_line()) # 118 (the length without colors) + 4 coloring characters. self.assertGreaterEqual(len(self.last_line()), 118 + 4)
def search(): # TODO: to be removed with open(r'./pwsp.yaml') as file: pwsp_config = yaml.load(file, Loader=yaml.FullLoader) ### must_terms, optional_terms = TermParser.parse(request.args.get('q')) must_terms, expanded_must_terms = Expander.expand(must_terms) optional_terms, expanded_optional_terms = Expander.expand(optional_terms) expand = request.args.get('expand', 'true') == 'true' max = request.args.get('max', 10) sources = request.args.get('source').split(',') if request.args.get( 'source') else None start = timer() items = [] for source in pwsp_config['sources']: if 'disabled' not in source or not source['disabled']: if sources is None or source['source'] in sources: log.info(f"Query source {source['source']}") queryBuilder = QueryBuilderFactory.create( source['query_builder'] if 'query_builder' in source else 'default') for query in queryBuilder.build( must_terms=expanded_must_terms if expand else must_terms, optional_terms=expanded_optional_terms if expand else optional_terms): # multistage if 'urls' in source: for url in source['urls']: __fix_parameters(url['params'], query, max) doc = MultistageRequester.get(source['urls']) else: __fix_parameters(source['params'], query, max) doc = Requester.get(source['url'], **source['params']) for parse in source['parse']: new_items = Parser.parseHtml( doc, highlight=expanded_must_terms + expanded_optional_terms if expand else must_terms + optional_terms, source=source['source'], target_source=source['target_source'] if 'target_source' in source else None, **parse) for item in new_items: item.update({'query': query}) items += new_items log.info("Finalize...") result = {} result['source_count_details'] = __get_statistics(items, 'source') items = __consolidate_items(items) result['count'] = len(items) result['source_count'] = __get_statistics(items) result['terms'] = must_terms + optional_terms result['expanded_terms'] = expanded_must_terms + expanded_optional_terms result['expand'] = expand end = timer() result['took'] = td(seconds=end - start).total_seconds() result['items'] = items return jsonify(result)
CORS(app) app.config['DEFAULT_RENDERERS'] = [ 'flask_api.renderers.JSONRenderer', 'flask_api.renderers.BrowsableAPIRenderer', ] app.secret_key = b'Q#G[DK.]uVs9qXW*hWvc32VW!wzL^2A?' # Add UTF-8 support app.config['JSON_AS_ASCII'] = False # disable sorting json keys app.config['JSON_SORT_KEYS'] = False log.info("Loading .env") env.read_envfile() with open(r'./pwsp.yaml') as file: pwsp_config = yaml.load(file, Loader=yaml.FullLoader) def __get_statistics(items, by='target_source'): source_count = {} for item in items: if item[by] not in source_count: source_count[item[by]] = 0 source_count[item[by]] += 1 return source_count
from flask_cors import CORS from flask_script import Manager, Server from friendlylog import colored_logger as log from mongoengine.queryset.visitor import Q from tendo import singleton from tendo.singleton import SingleInstanceException from modules.decorators import queryset_respose from flask_add_ons.logging import colorize_werkzeug # Logging logging.basicConfig(level=logging.DEBUG) colorize_werkzeug() __dev__ = environ['FLASK_ENV'] == 'development' or environ['DEBUG'] == 'True' log.info(f"Development mode is '{__dev__}'") # make sure only one instance is running if not __dev__: try: me = singleton.SingleInstance() except SingleInstanceException: log.error("MCT already running!") exit(1) app = Flask(__name__) CORS(app) app.config['DEFAULT_RENDERERS'] = [ 'flask_api.renderers.JSONRenderer', 'flask_api.renderers.BrowsableAPIRenderer',