def main(): parser = argparse.ArgumentParser(description='ETL(es to odbc)') parser.add_argument('--conf', '--conf-dir', required=True) parser.add_argument('--profile', required=True) parser.add_argument('--body', required=True) parser.add_argument('--optimize', action='store_true', default=False) from settings import configure_logging configure_logging() config = Configuration(vars(parser.parse_args())) extractor = etl_es.ElasticsearchDataExtractor(config) transformer = etl.SimpleDataTransformer(config) loader = etl_odbc.ODBCDataLoader(config) etl.etl(config, extractor, transformer, loader)
def main(): parser = argparse.ArgumentParser(description='ETL(mongo to es)') parser.add_argument('--conf', '--conf-dir', required=True) parser.add_argument('--query', required=True) parser.add_argument('--profile', required=False) parser.add_argument('--optimize', action='store_true', default=False) parser.add_argument('--index-template-name', required=False) parser.add_argument('--index-settings', required=False) parser.add_argument('--settings', required=False, default='{}') from settings import configure_logging configure_logging() config = Configuration(vars(parser.parse_args())) extractor = etl_mongo.MongoDataExtractor(config) transformer = etl.SimpleDataTransformer(config) loader = etl_es.ElasticsearchDataLoader(config) etl.etl(config, extractor, transformer, loader)
def main(): parser = argparse.ArgumentParser(description='ETL(mongo to mongo)') parser.add_argument('--conf', '--conf-dir', required=True) parser.add_argument('--query', required=True) parser.add_argument('--profile', required=False) parser.add_argument('--optimize', action='store_true', default=False) parser.add_argument('--update', action='store_true', default=False) parser.add_argument('--settings', required=False, default='{}') from settings import configure_logging configure_logging() args = parser.parse_args() config = Configuration(vars(args)) extractor = etl_mongo.MongoDataExtractor(config) if args.update: transformer = etl_mongo.MongoUpdateDataTransformer(config) loader = etl_mongo.MongoUpdateDataLoader(config) else: transformer = etl.SimpleDataTransformer(config) loader = etl_mongo.MongoDataLoader(config) etl.etl(config, extractor, transformer, loader)
import logging import os import random from sqlalchemy.sql import text import db import settings from db.models import SQLAlchemyBase, User, GenereEnum, UserToken, RolEnum, PositionEnum, SmashEnum, TournamentTypeEnum, \ TournamentPrivacyTypeEnum, Facility, TournamentGenereEnum, AgeCategoriesTypeEnum, Category, Tournament, Couple, \ Round, Match from settings import DEFAULT_LANGUAGE # LOGGING mylogger = logging.getLogger(__name__) settings.configure_logging() def execute_sql_file(sql_file): sql_folder_path = os.path.join(os.path.dirname(__file__), "sql") sql_file_path = open(os.path.join(sql_folder_path, sql_file), encoding="utf-8") sql_command = text(sql_file_path.read()) db_session.execute(sql_command) db_session.commit() sql_file_path.close() if __name__ == "__main__": settings.configure_logging()
import json import logging from test import RandomMock, TimeMock from db import get_db from settings import configure_logging from .exceptions import FixtureError configure_logging() class DbFixture: def __init__(self, app): self.app = app self.client = app.test_client() def __enter__(self): self.commit() def __exit__(self, exc_type, exc_val, exc_tb): self.rollback() def check_reponse(self, r): if r.status_code != 200: logging.error('Response: %s', r.data) raise FixtureError() def commit(self): get_db().create_all(app=self.app) r = self.client.post('/api/v1/tag/',
__author__ = 'caninemwenja' from twisted.protocols.basic import LineReceiver from twisted.internet.protocol import Factory from twisted.internet import reactor from siafu import Siafu, SiafuError, SiafuSyntaxError import logging import settings settings.configure_logging() class SiafuProtocol(LineReceiver): def __init__(self, addr): self.addr = addr self.siafu = Siafu(settings.DATABASE) def connectionMade(self): logging.info("new connection: {0}".format(self.addr)) self.sendLine("Connected, Send it") def connectionLost(self, reason): logging.info("connection lost: {0}".format(self.addr)) def lineReceived(self, line): logging.info("Received line: {0} from {1}".format( line, self.addr))
result[key] = value if isinstance(value, list): _TOKEN_COUNTS[key].update(set(value)) yield result if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('config', help='JSON processing config') parser.add_argument('infile', help='raw JSON records, one per line') parser.add_argument('outfile') parser.add_argument('--vocabulary', help='target JSON file of token-count mappings') parser.add_argument('--logging', help='JSON logging config') args = parser.parse_args() settings.configure_logging(args.logging) config = load_json(args.config) preprocessors = list( filter(None, [get_preprocessor(item) for item in config])) with open(args.infile) as f_in: with open(args.outfile, 'w') as f_out: LOGGER.info('writing %s', args.outfile) for record in preprocess(preprocessors, f_in): f_out.write('%s\n' % json.dumps(record)) if args.vocabulary: dump_json(args.vocabulary, _TOKEN_COUNTS)