Python cleanData示例

示例#1

文件： pw_transaction_data_prod.py 项目： rentapplication/etl_scripts

import re
import dataclean as dc
import sys
import traceback
import smtplib
import ConfigInfo as cf

try:
    #Transaction Data
    tr_url = cf.tr_url

    start = datetime.now()
    tr_data = dc.jsonToFrame(tr_url)

    #Cleaning Data
    tr_data = dc.cleanData(tr_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_transaction',tr_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

示例#2

文件： pw_workorder_data_prod.py 项目： rentapplication/etl_scripts

    
    columns = {}
    pattern = re.compile('[\W_]+')

    for items in frame['columns']:
        columns[items['index']]=items['label']
    for column in columns:
        columns[column] = pattern.sub('', columns[column])
        if columns[column][0].isdigit():
            columns[column] = '_' + columns[column]

    data = pd.DataFrame(frame['records'], index=None).rename(columns=columns)
    
    #Cleaning Data
    data = data.applymap(lambda x: x.replace('\n','').replace('\r','').replace(';','').replace("\\","") if isinstance(x,(str, unicode)) else x)
    data = dc.cleanData(data)
    
    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_workorder',data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

示例#3

文件： pw_payment_data_prod.py 项目： rentapplication/etl_scripts

import re
import dataclean as dc
import sys
import traceback
import smtplib
import ConfigInfo as cf

try:
    #Payment Data
    p_url = cf.p_url

    start = datetime.now()
    p_data = dc.jsonToFrame(p_url)

    #Cleaning Data
    p_data = dc.cleanData(p_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_payment',p_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

示例#4

文件： pw_billpayment_data_prod.py 项目： rentapplication/etl_scripts

import collections
import re
import dataclean as dc
import sys
import traceback
import smtplib
import ConfigInfo as cf

try:
    bp_url = cf.bp_url

    start = datetime.now()
    bp_data = dc.jsonToFrame(bp_url)

    #Cleaning Data
    bp_data = dc.cleanData(bp_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_billpayment',dp_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

示例#5

文件： pw_charge_data_prod.py 项目： rentapplication/etl_scripts

import re
import dataclean as dc
import sys
import traceback
import smtplib
import ConfigInfo as cf

try:
    #Charge Data
    ch_url = cf.ch_url

    start = datetime.now()
    ch_data = dc.jsonToFrame(ch_url)

    #Cleaning Data
    ch_data = dc.cleanData(ch_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_charge',ch_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

示例#6

文件： pw_building_data_prod.py 项目： rentapplication/etl_scripts

import re
import dataclean as dc
import sys
import traceback
import smtplib
import ConfigInfo as cf

try:
    #buildings Data
    b_url = cf.b_url

    start = datetime.now()
    b_data = dc.jsonToFrame(b_url)

    #Cleaning Data
    b_data = dc.cleanData(b_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_building',b_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

示例#7

文件： pw_disadjbd_data_prod.py 项目： rentapplication/etl_scripts

import re
import dataclean as dc
import sys
import traceback
import smtplib
import ConfigInfo as cf

try:
    #Transaction Data
    ds_url = cf.ds_url

    start = datetime.now()
    ds_data = dc.jsonToFrame(ds_url)

    #Cleaning Data
    ds_data = dc.cleanData(ds_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_disadjbd',ds_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)