look(table3)
# filldown
table1 = (('foo', 'bar', 'baz'),
(1, 'a', None),
(1, None, .23),
(1, 'b', None),
(2, None, None),
(2, None, .56),
(2, 'c', None),
(None, 'c', .72))
from petl import filldown, look
look(table1)
table2 = filldown(table1)
look(table2)
table3 = filldown(table1, 'bar')
look(table3)
table4 = filldown(table1, 'bar', 'baz')
look(table4)
# fillright
table1 = (('foo', 'bar', 'baz'),
(1, 'a', None),
(1, None, .23),
(1, 'b', None),
(2, None, None),
(2, None, .56),
(2, 'c', None),
look(table3)
# filldown
table1 = (('foo', 'bar', 'baz'),
(1, 'a', None),
(1, None, .23),
(1, 'b', None),
(2, None, None),
(2, None, .56),
(2, 'c', None),
(None, 'c', .72))
from petl import filldown, look
look(table1)
table2 = filldown(table1)
look(table2)
table3 = filldown(table1, 'bar')
look(table3)
table4 = filldown(table1, 'bar', 'baz')
look(table4)
# fillright
table1 = (('foo', 'bar', 'baz'),
(1, 'a', None),
(1, None, .23),
(1, 'b', None),
(2, None, None),
(2, None, .56),
(2, 'c', None),
import pandas as pd import numpy as np import petl as pt from petl import filldown, fillright, fillleft, look from petl import unflatten input = ['A', 3, True, None, 7, False, 'B', 2, False, 'C', 9] table = unflatten(input, 3) print(table) a = filldown(table) print(a)
# filldown()
############
import petl as etl
table1 = [
["foo", "bar", "baz"],
[1, "a", None],
[1, None, 0.23],
[1, "b", None],
[2, None, None],
[2, None, 0.56],
[2, "c", None],
[None, "c", 0.72],
]
table2 = etl.filldown(table1)
table2.lookall()
table3 = etl.filldown(table1, "bar")
table3.lookall()
table4 = etl.filldown(table1, "bar", "baz")
table4.lookall()
# fillright()
#############
import petl as etl
table1 = [
["foo", "bar", "baz"],
[1, "a", None],
#add rules to clean the table - reversed for give the priority for top attributes
for x in reversed(range(length)):
attr = data['attibutes'][x]['attrName']
rules = data['attibutes'][x]['rules']
rulesListSize = len(rules)
for y in range(rulesListSize):
if rules[y] == "Remove Null Value Rows":
cleansedTable = etl.select(cleansedTable, attr, lambda v: v != '')
if rules[y] == "Remove Duplicates":
cleansedTable = etl.aggregate(cleansedTable, attr)
if rules[y] == "Sort":
cleansedTable = etl.mergesort(cleansedTable, key=attr)
if rules[y] == "Number Validation":
cleansedTable = etl.select(cleansedTable, attr)
if rules[y] == "Fill Missing Values":
cleansedTable = etl.filldown(cleansedTable, attr)
etl.tocsv(cleansedTable,'src/etl/outputs/cleansed.csv')
#Create rawData Table
dataTable = cleansedTable
rawDataTable = cleansedTable
reasonUniqueValues = etl.aggregate(dataTable,dataTable[0][20])
mappings = OrderedDict()
#mapping attributes
# go through each column (c = table)
for i in range(length):
#get unique values for each column
header=['date', 'rate'])
# print (exchangeRates)
# load expense document
try:
expenses = petl.io.xlsx.fromxlsx('Expenses.xlsx', sheet='Github')
except Exception as e:
print('could not open expenses.xlsx:' + str(e))
sys.exit()
# join tables
expenses = petl.outerjoin(exchangeRates, expenses, key='date')
# fill down missing values
expenses = petl.filldown(expenses, 'rate')
# remove dates with no expenses
expenses = petl.select(expenses, lambda rec: rec.USD != None)
# add CDN column
expenses = petl.addfield(expenses, 'CAD',
lambda rec: decimal.Decimal(rec.USD) * rec.rate)
# intialize database connection
try:
dbConnection = pymssql.connect(server=destServer,
database=destDatabase)
except Exception as e:
print('could not connect to database:' + str(e))
sys.exit()
from __future__ import division, print_function, absolute_import
# filldown()
############
import petl as etl
table1 = [['foo', 'bar', 'baz'], [1, 'a', None], [1, None, .23],
[1, 'b', None], [2, None, None], [2, None, .56], [2, 'c', None],
[None, 'c', .72]]
table2 = etl.filldown(table1)
table2.lookall()
table3 = etl.filldown(table1, 'bar')
table3.lookall()
table4 = etl.filldown(table1, 'bar', 'baz')
table4.lookall()
# fillright()
#############
import petl as etl
table1 = [['foo', 'bar', 'baz'], [1, 'a', None], [1, None, .23],
[1, 'b', None], [2, None, None], [2, None, .56], [2, 'c', None],
[None, 'c', .72]]
table2 = etl.fillright(table1)
table2.lookall()
# fillleft()
############
import petl as etl
table1 = [['foo', 'bar', 'baz'], [1, 'a', None], [1, None, .23],