from typing import Dict, List, Union import logging from datetime import date, datetime from collector.parser import Parser from collector.row_parser import RowParser from config import get_active_config conf = get_active_config() logger = logging.getLogger(__name__) Scalar = Union[int, float, str, None, datetime, date] Row = Dict[str, Scalar] class TransformationError(Exception): pass class Transformer(object): parser = RowParser.load_from_config(conf.PARSER_CONFIG) def __init__( self, aliases: Dict[str, str] = None, exclude: List[str] = None, normalize: bool = False, parser: Parser = None,
from datetime import datetime import pandas as pd from flask_sqlalchemy import Model import requests from api.models import * from collector.endpoint import Endpoint from collector.request import Request from collector.transformer import Transformer from config import get_active_config from collector.util import retry logger = logging.getLogger(__name__) config = get_active_config() class Collector(object): """ Acts as the conduit for transferring newly collected data into a backend data model """ _tf = None _endpoint = None _functions = None _model = None def __init__( self, endpoint: Endpoint, functions: Dict[Union[str, None], Union[str, None]] = None, ):