def test_smoke_test():
"""
This is just a smoke test, it exercises most of the logging functionality
it should just work.
There's no asserts in testing the logger
"""
logger = get_logger()
logger.setLevel(LEVELS.DEBUG)
logger.trace("trace")
logger.debug("debug")
logger.info("info")
logger.warning("warn")
logger.error("error")
logger.critical("critical")
@error_trap(propagate=False)
def i_fail():
return 0 / 0
@verbose_logger()
def i_pass(key="value"):
return key * 2
i_fail()
i_pass(key="invaluable")
def test_flow_builder_invalid_wrong_type():
"""
Test the flow builder doesn't succeed with an invalid Operator
"""
e = get_logger() # <- this should fail
n = NoOpOperator()
failed = False
try:
flow = n > e
except TypeError:
failed = True
assert failed
import shutil
import sys
import os
import time
sys.path.insert(1, os.path.join(sys.path[0], '..\..'))
from gva.data.writers import Writer, file_writer
from gva.logging import verbose_logger, get_logger
from gva.data.validator import Schema
from gva.data.formats import dictset
try:
import orjson as json
except ImportError:
import ujson as json
logger = get_logger()
logger.setLevel(5)
schema_definition = {
"fields": [
{"name":"userid","type":"numeric"},
{"name":"username","type":"string"},
{"name":"user_verified","type":"boolean"},
{"name":"followers","type":"numeric"},
{"name":"tweet","type":"string"},
{"name":"location","type":["string","nullable"]},
{"name":"sentiment","type":"numeric"},
{"name":"timestamp","type":"date"}
]
}
import os
import sys
import glob
sys.path.insert(1, os.path.join(sys.path[0], '..'))
from gva.data.writers import Writer, file_writer
from gva.data import Reader, FileReader
try:
from rich import traceback
traceback.install()
except ImportError:
pass
from gva.logging import get_logger
get_logger().setLevel(5)
def do_writer():
w = Writer(writer=file_writer,
to_path='_tests/year_%Y/test.jsonl',
date=datetime.date.today())
for i in range(int(1e5)):
w.append({"test": True})
w.append({"test": False})
w.finalize()
def do_writer_compressed():
w = Writer(writer=file_writer,
to_path='_tests/year_%Y/test.jsonl',
