Python writeFrame Exemples

Exemple #1

Fichier : pw_lease_data_prod.py Projet : rentapplication/etl_scripts

import sys
import traceback
import smtplib

try:
    url = cf.l_url
    
    start = datetime.now()
    data = dc.jsonToFrame(url)

    #Cleaning Data
    data = dc.cleanData(data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_lease',data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'
except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

    #sending email of error log
    sender = cf.gmail_id
    receivers = [cf.recip]
    message = """From: Linux Box <{0}>\nTo:<{1}>\nSubject: ERROR LOG-FAIL TO WRITE {2}\n\nError message: {3}.""".format(cf.gmail_id,cf.recip,'pw_lease', error_log)

Exemple #2

Fichier : pw_workorder_data_prod.py Projet : rentapplication/etl_scripts

    for items in frame['columns']:
        columns[items['index']]=items['label']
    for column in columns:
        columns[column] = pattern.sub('', columns[column])
        if columns[column][0].isdigit():
            columns[column] = '_' + columns[column]

    data = pd.DataFrame(frame['records'], index=None).rename(columns=columns)
    
    #Cleaning Data
    data = data.applymap(lambda x: x.replace('\n','').replace('\r','').replace(';','').replace("\\","") if isinstance(x,(str, unicode)) else x)
    data = dc.cleanData(data)
    
    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_workorder',data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

    #sending email of error log
    sender = cf.gmail_id
    receivers = [cf.recip]

Exemple #3

Fichier : pw_payment_data_prod.py Projet : rentapplication/etl_scripts

import smtplib
import ConfigInfo as cf

try:
    #Payment Data
    p_url = cf.p_url

    start = datetime.now()
    p_data = dc.jsonToFrame(p_url)

    #Cleaning Data
    p_data = dc.cleanData(p_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_payment',p_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

    #sending email of error log
    sender = cf.gmail_id
    receivers = [cf.recip]

Exemple #4

Fichier : pw_transaction_data_prod.py Projet : rentapplication/etl_scripts

import smtplib
import ConfigInfo as cf

try:
    #Transaction Data
    tr_url = cf.tr_url

    start = datetime.now()
    tr_data = dc.jsonToFrame(tr_url)

    #Cleaning Data
    tr_data = dc.cleanData(tr_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_transaction',tr_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

    #sending email of error log
    sender = cf.gmail_id
    receivers = [cf.recip]

Exemple #5

Fichier : pw_unit_data_prod.py Projet : rentapplication/etl_scripts

import sys
import traceback
import smtplib

try:
    url = cf.u_url
    
    start = datetime.now()
    data = dc.jsonToFrame(url)

    #Cleaning Data
    data = dc.cleanData(data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password={3} sslmode=allow".format(cf.chartio_host, cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_unit',data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'
except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

    #sending email of error log
    sender = cf.gmail_id
    receivers = [cf.recip]
    message = """From: Linux Box <{0}>\nTo: <{1}>\nSubject: ERROR LOG-FAIL TO WRITE {2}\n\nError message: {3}.""".format(cf.gmail_id,cf.recip,'pw_unit', error_log)

Exemple #6

Fichier : pw_billpayment_data_prod.py Projet : rentapplication/etl_scripts

import traceback
import smtplib
import ConfigInfo as cf

try:
    bp_url = cf.bp_url

    start = datetime.now()
    bp_data = dc.jsonToFrame(bp_url)

    #Cleaning Data
    bp_data = dc.cleanData(bp_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_billpayment',dp_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

    #sending email of error log
    sender = cf.gmail_id
    receivers = [cf.recip]

Exemple #7

Fichier : pw_charge_data_prod.py Projet : rentapplication/etl_scripts

import smtplib
import ConfigInfo as cf

try:
    #Charge Data
    ch_url = cf.ch_url

    start = datetime.now()
    ch_data = dc.jsonToFrame(ch_url)

    #Cleaning Data
    ch_data = dc.cleanData(ch_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_charge',ch_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

    #sending email of error log
    sender = cf.gmail_id
    receivers = [cf.recip]

Exemple #8

Fichier : pw_building_data_prod.py Projet : rentapplication/etl_scripts

import smtplib
import ConfigInfo as cf

try:
    #buildings Data
    b_url = cf.b_url

    start = datetime.now()
    b_data = dc.jsonToFrame(b_url)

    #Cleaning Data
    b_data = dc.cleanData(b_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_building',b_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

    #sending email of error log
    sender = cf.gmail_id
    receivers = [cf.recip]

Exemple #9

Fichier : pw_disadjbd_data_prod.py Projet : rentapplication/etl_scripts

import smtplib
import ConfigInfo as cf

try:
    #Transaction Data
    ds_url = cf.ds_url

    start = datetime.now()
    ds_data = dc.jsonToFrame(ds_url)

    #Cleaning Data
    ds_data = dc.cleanData(ds_data)

    conn = psycopg2.connect("host={0} dbname={1} user={2} password ={3} sslmode=allow".format(cf.chartio_host,cf.chartio_db,cf.chartio_id,cf.chartio_pwd))

    dc.writeFrame(conn,'pw_disadjbd',ds_data)

    conn.close()
    end = datetime.now()

    print end - start
    print 'Done'

except:
    exc_type, exc_value, exc_traceback = sys.exc_info()
    lines = traceback.format_exception(exc_type, exc_value, exc_traceback)
    error_log = ''.join('!! ' + line for line in lines)

    #sending email of error log
    sender = cf.gmail_id
    receivers = [cf.recip]