def main(ENV):
# read config file and set the application root path: project_id
config = configparser.ConfigParser(
{'APPLICATION_ROOT': os.path.abspath("../")})
config.read(PATH_CONF)
# setup the logging system
log_numeric_level = getattr(logging, config[ENV]['log_level'], None)
logging.basicConfig(filename=config[ENV]['log_file'],
format='%(levelname)s:%(asctime)s:%(message)s',
level=log_numeric_level)
# run data factory
logging.info('Running all')
importation.run(config[ENV])
clean.run(config[ENV])
transform.run(config[ENV])
print('Log file in : {} ; Log level is {}'.format(
config[ENV]['log_file'], config[ENV]['log_level']))
def run(self):
clean_data = clean.run(self.file_path[0], self.progressBarValue)
# index = clean_data.columns.values.tolist()
# data_dic = {}
# for i in index:
# data_dic[i] = clean_data[i].tolist()
data_dic = clean_data.to_dict('list')
data_json = json.dumps(data_dic)
self.progressBarValue.emit('100')
self.progressBarValue.emit('-1') # 关闭进度条
self.progressBarValue.emit(data_json)
print("加载成功")
import os
import sys
import json
from datetime import datetime, date
import lux
import clean
import setup
clean.run()
assert lux.VERSION[3] == 'final'
with open('CHANGELOG.rst', 'r') as f:
changelog = f.read()
top = changelog.split('\n')[0]
version_date = top.split(' - ')
assert len(version_date) == 2, 'Top of CHANGELOG.rst must be version and date'
version, datestr = version_date
dt = datetime.strptime(datestr, '%Y-%b-%d').date()
assert dt == date.today()
assert version == 'Ver. %s' % lux.__version__
with open('package.json', 'r') as f:
pkg = json.loads(f.read())
pkg['version'] = lux.__version__
pkg['description'] = lux.__doc__
import os
import sys
import json
from datetime import datetime, date
import lux
import clean
import setup
clean.run()
assert lux.VERSION[3] == 'final'
with open('CHANGELOG.rst', 'r') as f:
changelog = f.read()
top = changelog.split('\n')[0]
version_date = top.split(' - ')
assert len(version_date) == 2, 'Top of CHANGELOG.rst must be version and date'
version, datestr = version_date
dt = datetime.strptime(datestr, '%Y-%b-%d').date()
assert dt == date.today()
assert version == 'Ver. %s' % lux.__version__
with open('package.json', 'r') as f:
pkg = json.loads(f.read())
pkg['version'] = lux.__version__
pkg['description'] = lux.__doc__
import clean
import kmeans
import merge
import zoning
exec(open('../pymongo_dm.py').read())
# connect to DBMS
print("Connecting to the DBMS...")
client = pymongo.MongoClient()
repo = client.repo
repo.authenticate('djmcc_jasper', 'djmcc_jasper')
# execute scripts
reset.run(repo)
get.run(repo)
clean.run(repo)
merge.run(repo)
kmeans.run(repo)
zoning.run(repo)
# disconnect from the DBMS
print("Disconnecting from the DBMS...")
repo.logout()
# EOF
