#!/usr/bin/python
import psycopg2
from pymongo import MongoClient
import database
import sys
from mongo_batch import MongoBatch
client = MongoClient("localhost:27017")
db = client.etlpro
orders = db.orders
batch = MongoBatch(orders, int(sys.argv[1])) if (len(sys.argv) > 1 and sys.argv[1].isdigit()) else MongoBatch(orders)
cnx = psycopg2.connect(database.dsn())
cursor = cnx.cursor()
join_query = """
SELECT o.*, i.items, t.trackings
FROM orders AS o
INNER JOIN (
SELECT order_id, array_agg(ARRAY[id, qty, description, price]::text[]) AS items
FROM items
GROUP BY order_id
) AS i
ON (i.order_id = o.id)
INNER JOIN (
SELECT order_id, array_agg(ARRAY[status, tmstmp]::text[]) AS trackings
FROM tracking
GROUP BY order_id
) AS t
ON (t.order_id = o.id)
#!/usr/bin/python
import psycopg2
import database
from pymongo import MongoClient
from collections import defaultdict
import sys
from mongo_batch import MongoBatch
client = MongoClient("localhost:27017")
db = client.etlpro
orders = db.orders
batch = MongoBatch(orders, int(sys.argv[1])) if (
len(sys.argv) > 1 and sys.argv[1].isdigit()) else MongoBatch(orders)
order_cnx = psycopg2.connect(database.dsn())
#order_cnx.time_zone = 'UTC'
order_cursor = order_cnx.cursor()
item_cnx = psycopg2.connect(database.dsn())
#item_cnx.time_zone = 'UTC'
item_cursor = item_cnx.cursor()
tracking_cnx = psycopg2.connect(database.dsn())
#tracking_cnx.time_zone = 'UTC'
tracking_cursor = tracking_cnx.cursor()
order_cursor.execute("""
select id as order_id, first_name, last_name, shipping_address
from orders
order by order_id
#!/usr/bin/python
import psycopg2
from pymongo import MongoClient
import database
import sys
from mongo_batch import MongoBatch
client = MongoClient("localhost:27017")
db = client.etlpro
orders = db.orders
batch = MongoBatch(orders, int(sys.argv[1])) if (len(sys.argv) > 1 and sys.argv[1].isdigit()) else MongoBatch(orders)
cnx = psycopg2.connect(database.dsn())
#cnx.time_zone = 'UTC'
cursor = cnx.cursor()
item_cnx = psycopg2.connect(database.dsn())
#item_cnx.time_zone = 'UTC'
item_cursor = item_cnx.cursor()
tracking_cnx = psycopg2.connect(database.dsn())
#tracking_cnx.time_zone = 'UTC'
tracking_cursor = tracking_cnx.cursor()
order_query_stmt = ("select id as order_id, first_name, last_name, shipping_address "
"from orders")
cursor.execute(order_query_stmt)