def log_all(msg):
for _ in range(0, 11):
logger.debug(msg)
logger.info(msg)
logger.warning(msg)
logger.error(msg)
logger.critical(msg)
def parseHtml(cls, doc, highlight=None, parser='html.parser', source='unknown', target_source=None, item=None, id=None, title=None, link=None, abstract=None, details=None):
records = []
try:
if doc:
soup = BeautifulSoup(doc, parser)
log.debug(f"Got results from '{source}'.")
elements = cls.__select(soup, item)
for elt in elements:
_source = source
_target_source = target_source if target_source else source
_id = cls.__select(elt, id)
_title = cls.__select(elt, title)
_link = cls.__select(elt, link)
_abstract = cls.__select(elt, abstract)
_details = cls.__select(elt, details)
rec = cls.record(_source, _target_source, _id, _title,
_link, _abstract, _details).to_dict()
rec = Highlighter.highlight(
rec, ['title', 'abstract'], highlight)
log.info(f"Record: {rec}")
records.append(rec)
except Exception as ex:
log.error(ex)
traceback.print_exc(file=stdout)
finally:
return records
def test_non_str_logging(self):
logger.info(10)
self.assertIn("10", self.last_line())
# Those should not throw any error.
logger.debug([10, 20, 30])
logger.critical({})
logger.warning(set([-1, 4]))
def initChangeTracker():
log.debug("Initialze change tracker")
from modules.change_tracker import ChangeTracker
changeTracker = ChangeTracker(
__get_observe_database_uri(),
[coll.strip() for coll in environ.get('CT_COLLECTIONS').split(',')],
[field.strip() for field in environ.get('CT_FIELDS').split(',')])
changeTracker.start()
def test_message_is_logged(self):
logger.debug("message 1")
self.assertIn("message 1", self.last_line())
logger.info("message 2")
self.assertIn("message 2", self.last_line())
logger.warning("message 3")
self.assertIn("message 3", self.last_line())
logger.error("message 4")
self.assertIn("message 4", self.last_line())
logger.critical("message 5")
self.assertIn("message 5", self.last_line())
def test_filepath_is_logged(self):
logger.debug("message 1")
self.assertIn("test_colored_logger.py", self.last_line())
logger.info("message 2")
self.assertIn("test_colored_logger.py", self.last_line())
logger.warning("message 3")
self.assertIn("test_colored_logger.py", self.last_line())
logger.error("message 4")
self.assertIn("test_colored_logger.py", self.last_line())
logger.critical("message 5")
self.assertIn("test_colored_logger.py", self.last_line())
def test_level_is_logged(self):
logger.debug("message 1")
self.assertIn("DEBUG", self.last_line())
logger.info("message 2")
self.assertIn("INFO", self.last_line())
logger.warning("message 3")
self.assertIn("WARNING", self.last_line())
logger.error("message 4")
self.assertIn("ERROR", self.last_line())
logger.critical("message 5")
self.assertIn("CRITICAL", self.last_line())
def run(self):
try:
log.debug(f"[{self.id}] Flusher started.")
self.running = True
while (self.running):
self.__handleQ()
time.sleep(ChangeTracker.CHANGES_TIMEOUT)
log.debug(f"[{self.id}] Flusher stopped.")
except Exception as ex:
log.error(ex)
traceback.print_exc(file=sys.stdout)
def get(url: str, **kwargs):
log.info(f"Request url={url} args={kwargs}")
try:
headers = {
'User-Agent':
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/90.0.4430.93 Safari/537.36'
}
res = requests.get(url, params=kwargs, timeout=5, headers=headers)
log.debug(f"Response code={res.status_code}")
if res.ok:
with open(f"{url.replace('/', '_')}.html", 'w') as f:
f.write(res.text)
return res.text
else:
log.error(res.text)
return None
except requests.RequestException as ex:
log.error(ex)
def get(cls, urls: list):
if not isinstance(urls, list):
raise TypeError('urls is not of type list')
response = None
for url in urls:
params = url['params']
for param, value in params.items():
match = cls.RE_PARSE_PARAM.match(str(value))
if match:
action = match.group('action')
attributes = match.group('attributes')
content = match.group('content')
log.debug(
f"params: action={action}, attributes={attributes}, content={content}"
)
if action == 'eval':
if attributes == 'json':
if response:
try:
log.debug(f"response={response}")
json = loads(response)
log.debug(f"json={json}")
content = eval(content)
except Exception as ex:
log.warning(
f"Cannot evaluate {content}: {ex}")
traceback.print_exc(file=stdout)
content = ''
params[param] = content
response = Requester.get(url['url'], **params)
if not response:
break
return response
def __select(cls, elt, selector):
ret = None
if selector is not None:
log.debug(f"Try selector: '{selector}'")
parts = selector.split('|')
selector = parts[0]
filters = None
if len(parts) > 1:
filters = parts[1:]
match = cls.RE_PARSE_INSTRUCTION.match(selector)
if match:
action = match.group('action')
attributes = match.group('attributes')
selector = match.group('selector')
log.debug(
f"action={action} attributes={attributes} selector={selector}")
try:
if action == 'attr':
ret = elt.select(selector)[0][attributes]
elif action == 'first':
ret = elt.select_one(selector)
else:
ret = elt.select(selector)
if filters:
for filter in filters:
log.debug(f"Apply filter {filter}")
if not isinstance(ret, str):
ret = ret.text
ret = eval(f"{filter.replace('$','str(ret)')}")
except Exception as ex:
log.error(ex)
traceback.print_exc(file=stdout)
finally:
log.debug(f"Found {len(ret) if ret else 0} entries.")
log.debug("---")
if not ret:
ret = ''
return ret
def log_all():
logger.debug("message 1")
logger.info("message 2")
logger.warning("message 3")
logger.error("message 4")
logger.critical("message 5")
def run(self):
if self.mongo_uri is None:
return
db = pymongo.MongoClient(self.mongo_uri).get_database()
pipeline = [{
"$match": {
"ns.coll": {
"$in": self.collections
}
}
}, {
"$set": {
"timestamp": "$clusterTime",
"user": f"$fullDocument.{environ.get('CT_USER_FIELD')}",
"db": "$ns.db",
"coll": "$ns.coll",
"doc_id": "$fullDocument._id",
"type": "$operationType",
"updatedFields": "$updateDescription.updatedFields",
"removedFields": "$updateDescription.removedFields",
"fullDocument": "$fullDocument"
}
}, {
"$project": {
"timestamp": 1,
"user": 1,
"db": 1,
"coll": 1,
"type": 1,
"doc_id": 1,
"updatedFields": 1,
"removedFields": 1,
"fullDocument": 1
}
}]
resume_token = None
self.running = True
for flusher in self.flushers:
flusher.start()
self.__status = 'running'
while self.running:
try:
with db.watch(pipeline,
'updateLookup',
resume_after=resume_token) as stream:
if not self.running:
log.debug("Closeing stream...")
stream.close()
for change in stream:
if not self.running:
break
createDoc = False
ignoredFields = []
# General changes
change['timestamp'] = change[
'timestamp'].as_datetime().strftime(
'%Y-%m-%dT%H:%M:%S.%f')
if 'user' not in change:
change['user'] = '******'
else:
change['user'] = change['user']
# Type specific changes
if change['type'] == 'insert':
change['fullDocument'] = change['fullDocument']
createDoc = True
if environ.get('CT_DEBUG'):
log.debug(
"{timestamp}: user={user} db={db} coll={coll} type={type} doc_id={doc_id}"
.format(**change))
elif change['type'] == 'update':
updatedFields = {}
removedFields = []
for field, value in change[
'updatedFields'].items():
if self.__match(field):
# json_value = json.loads(value)
if isinstance(value, (dict, list)):
flat_value = flatten_json(value)
for _field, _value in flat_value.items(
):
updatedFields[
f"{field}.{_field}"] = _value
else:
updatedFields[field] = value
createDoc = True
else:
ignoredFields.append(field)
for field in change['removedFields']:
if self.__match(field):
removedFields.append(field)
createDoc = True
else:
ignoredFields.append(field)
change['updatedFields'] = updatedFields
change['removedFields'] = removedFields
del change['fullDocument']
if environ.get('CT_DEBUG'):
log_msg = "{timestamp}: user={user} db={db} coll={coll} type={type} doc_id={doc_id} updatedFields={updatedFields} removedFields={removedFields}".format(
**change)
log_msg = (
log_msg[:500] + '...'
) if len(log_msg) > 500 else log_msg
log.debug(log_msg)
# If we need to create a change entry
if createDoc:
self.__add(change)
else:
if change['type'] in ['insert', 'update']:
log.debug(
"Not tracking change for: {timestamp}: user={user} db={db} coll={coll} type={type} doc_id={doc_id} ignoredFields={ignoredFields}"
.format(**change,
ignoredFields=ignoredFields))
else:
log.warning(
"Not tracking change for: {0}".format(
change))
resume_token = stream.resume_token
except Exception as ex:
self.__status = 'error'
log.error(ex)
traceback.print_exc(file=sys.stdout)
pass