Ejemplo n.º 1
0
 def create_db(self):
     """
     Create database tables from models defined in schema and populate validation tables.
     """
     logger.info("Creating data models...")
     BaseSchema.metadata.create_all(self.connection)
     with self.create_session() as sess:
         create_seasons(sess, self.start_year, self.end_year)
         interior_path = pkg_resources.resource_filename('marcottimls', 'data')
         ingest_feeds(get_local_handles, interior_path, ('countries.csv',), CountryIngest(sess))
Ejemplo n.º 2
0
 def create_db(self):
     """
     Create database tables from models defined in schema and populate validation tables.
     """
     logger.info("Creating data models...")
     BaseSchema.metadata.create_all(self.connection)
     with self.create_session() as sess:
         create_seasons(sess, self.start_year, self.end_year)
         interior_path = pkg_resources.resource_filename(
             'marcottimls', 'data')
         ingest_feeds(get_local_handles, interior_path, ('countries.csv', ),
                      CountryIngest(sess))
Ejemplo n.º 3
0
    def create_db(self, base):
        """
        Create database tables from models defined in schema and populate validation tables.

        :param base: One of the declarative class objects (ClubSchema, NatlSchema).
        """
        print "Creating schemas..."
        base.metadata.create_all(self.connection)
        print "Ingesting validation files..."
        with self.create_session() as sess:
            create_seasons(sess, self.start_year, self.end_year)
            ingest_feeds(get_local_handles, "data", "countries.csv", validation.CountryIngest(sess))
            ingest_feeds(get_local_handles, "data", "timezones.csv", validation.TimezoneIngest(sess))
            ingest_feeds(get_local_handles, "data", "surfaces.csv", validation.SurfaceIngest(sess))
            ingest_feeds(get_local_handles, "data", "positions.csv", validation.PositionIngest(sess))
        print "Ingestion complete."
Ejemplo n.º 4
0
    def create_db(self, base):
        """
        Create database tables from models defined in schema and populate validation tables.

        :param base: One of the declarative class objects (ClubSchema, NatlSchema).
        """
        print "Creating schemas..."
        base.metadata.create_all(self.connection)
        print "Ingesting validation files..."
        with self.create_session() as sess:
            create_seasons(sess, self.start_year, self.end_year)
            ingest_feeds(get_local_handles, 'data', 'countries.csv',
                         validation.CountryIngest(sess))
            ingest_feeds(get_local_handles, 'data', 'timezones.csv',
                         validation.TimezoneIngest(sess))
            ingest_feeds(get_local_handles, 'data', 'surfaces.csv',
                         validation.SurfaceIngest(sess))
            ingest_feeds(get_local_handles, 'data', 'positions.csv',
                         validation.PositionIngest(sess))
        print "Ingestion complete."
Ejemplo n.º 5
0
from etl import get_local_handles, ingest_feeds
from etl.ecsv import CSV_ETL_CLASSES
from local import LocalConfig
from interface import Marcotti


if __name__ == "__main__":
    settings = LocalConfig()
    marcotti = Marcotti(settings)
    with marcotti.create_session() as sess:
        for group in ['Supplier', 'Overview', 'Personnel', 'Match']:
            for entity, datafile in settings.CSV_DATA.get(group, []):
                if group in ['Supplier', 'Overview', 'Personnel']:
                    if entity == 'Venues':
                        params = (sess, settings.VENUE_EFF_DATE)
                    elif entity in ['Competitions', 'Clubs', 'Players', 'Positions']:
                        params = (sess, settings.DATA_SUPPLIER)
                    else:
                        params = (sess,)
                else:
                    params = (sess, settings.COMPETITION_NAME, settings.SEASON_NAME)
                    if entity == 'PlayerStats':
                        params += (settings.DATA_SUPPLIER,)
                if type(CSV_ETL_CLASSES[group][entity]) is list:
                    for etl_class in CSV_ETL_CLASSES[group][entity]:
                        ingest_feeds(get_local_handles, settings.CSV_DATA_DIR, datafile, etl_class(*params))
                else:
                    ingest_feeds(get_local_handles, settings.CSV_DATA_DIR, datafile,
                                 CSV_ETL_CLASSES[group][entity](*params))