def run(className):

        os.environ['PYSPARK_PYTHON'] = '/usr/bin/python3.5'
        os.environ["PYSPARKDRIVER_PYTHON"]= "/usr/bin/python3.5"

        exec('from %s import %s' % (className, className))
        
        cfg = Configuration.configuration('config.yml')

        context = MyContext().context(cfg, '%s%s_%s' \
                        % (className, cfg['dates']['startDate'], cfg['dates']['endDate']))

        exec('item = %s(cfg, context)' % className)

        exec( 'item.process(cfg["dates"]["startDate"], cfg["dates"]["endDate"])' )
Пример #2
0
import os
from Configuration import Configuration
from Importer import Importer

os.environ['PYSPARK_PYTHON'] = '/usr/bin/python3.5'
os.environ["PYSPARKDRIVER_PYTHON"]= "/usr/bin/python3.5"
cfg = Configuration.configuration('config.yml')
importer = Importer(cfg)
importer.process(cfg['dates']['startDate'], cfg['dates']['endDate'])