def main(ENV): # read config file and set the application root path: project_id config = configparser.ConfigParser( {'APPLICATION_ROOT': os.path.abspath("../")}) config.read(PATH_CONF) # setup the logging system log_numeric_level = getattr(logging, config[ENV]['log_level'], None) logging.basicConfig(filename=config[ENV]['log_file'], format='%(levelname)s:%(asctime)s:%(message)s', level=log_numeric_level) # run data factory logging.info('Running all') importation.run(config[ENV]) clean.run(config[ENV]) transform.run(config[ENV]) print('Log file in : {} ; Log level is {}'.format( config[ENV]['log_file'], config[ENV]['log_level']))
def run(self): clean_data = clean.run(self.file_path[0], self.progressBarValue) # index = clean_data.columns.values.tolist() # data_dic = {} # for i in index: # data_dic[i] = clean_data[i].tolist() data_dic = clean_data.to_dict('list') data_json = json.dumps(data_dic) self.progressBarValue.emit('100') self.progressBarValue.emit('-1') # 关闭进度条 self.progressBarValue.emit(data_json) print("加载成功")
import os import sys import json from datetime import datetime, date import lux import clean import setup clean.run() assert lux.VERSION[3] == 'final' with open('CHANGELOG.rst', 'r') as f: changelog = f.read() top = changelog.split('\n')[0] version_date = top.split(' - ') assert len(version_date) == 2, 'Top of CHANGELOG.rst must be version and date' version, datestr = version_date dt = datetime.strptime(datestr, '%Y-%b-%d').date() assert dt == date.today() assert version == 'Ver. %s' % lux.__version__ with open('package.json', 'r') as f: pkg = json.loads(f.read()) pkg['version'] = lux.__version__ pkg['description'] = lux.__doc__
import clean import kmeans import merge import zoning exec(open('../pymongo_dm.py').read()) # connect to DBMS print("Connecting to the DBMS...") client = pymongo.MongoClient() repo = client.repo repo.authenticate('djmcc_jasper', 'djmcc_jasper') # execute scripts reset.run(repo) get.run(repo) clean.run(repo) merge.run(repo) kmeans.run(repo) zoning.run(repo) # disconnect from the DBMS print("Disconnecting from the DBMS...") repo.logout() # EOF