def test_freeze_csv(self):
freeze(self.tbl.all(), format='csv',
filename='weather.csv', prefix=self.d)
path = os.path.join(self.d, 'weather.csv')
if PY3:
fh = open(path, 'rt', encoding='utf8', newline='')
else:
fh = open(path, 'rU')
try:
rows = list(reader(fh))
keys = rows[0]
for i, d1 in enumerate(TEST_DATA):
d2 = dict(zip(keys, rows[i + 1]))
for k in d1.keys():
v2 = d2[k]
if not PY3:
v2 = v2.decode('utf8')
v1 = value_to_str(d1[k])
if not isinstance(v1, text_type):
if isinstance(v1, binary_type):
v1 = text_type(v1, encoding='utf8')
else:
v1 = '%s' % v1
self.assertEqual(v2, v1)
finally:
fh.close()
def test_freeze(self):
freeze(self.tbl.all(), format='csv',
filename=u'wäther.csv'.encode('utf8'), prefix=self.d)
self.assertTrue(os.path.exists(os.path.join(self.d, u'wäther.csv')))
freeze(self.tbl.all(), format='csv',
filename=u'wäther.csv', prefix=self.d)
self.assertTrue(os.path.exists(os.path.join(self.d, u'wäther.csv')))
def export_tables(params):
"""Dump tables to csv files and upload to s3 bucket"""
tempdir = tempfile.mkdtemp()
for entry in params:
table = datastore[entry["table"]]
filename = entry.get("csv_filename", entry["table"] + ".csv")
bucket = entry.get("bucket")
freeze(table.all(), filename=filename, prefix=tempdir)
file_path = os.path.join(tempdir, filename)
s3_upload(bucket, file_path)
shutil.rmtree(tempdir)
def test_memory_streams(self):
if PY3:
from io import StringIO
else:
from io import BytesIO as StringIO
for fmt in ('csv', 'json', 'tabson'):
with StringIO() as fd:
freeze(self.tbl.all(), format=fmt, fileobj=fd)
self.assertFalse(fd.closed, 'fileobj was closed for format %s' % fmt)
fd.getvalue() # should not throw
def test_freeze_json_no_wrap(self):
freeze(self.tbl.all(), format='json',
filename='weather.csv', prefix=self.d, wrap=False)
path = os.path.join(self.d, 'weather.csv')
if PY3:
fh = open(path, 'rt', encoding='utf8', newline='')
else:
fh = open(path, 'rU')
try:
import json
data = json.load(fh)
self.assertIsInstance(data, list,
'Without wrapping, returned JSON should be a list')
finally:
fh.close()
def test_freeze_json_wrap(self):
freeze(self.tbl.all(), format='json',
filename='weather.csv', prefix=self.d, wrap=True)
path = os.path.join(self.d, 'weather.csv')
if PY3:
fh = open(path, 'rt', encoding='utf8', newline='')
else:
fh = open(path, 'rU')
try:
import json
data = json.load(fh)
self.assertIsInstance(data, dict,
'With wrapping, returned JSON should be a dict')
self.assertIn('results', data.keys())
self.assertIn('count', data.keys())
self.assertIn('meta', data.keys())
finally:
fh.close()
import settings
import tweepy
import dataset
from textblob import TextBlob
from datafreeze.app import freeze
import csv, json
db = dataset.connect(settings.CONNECTION_STRING)
result = db[settings.TABLE_NAME].all()
freeze(result, format='csv', filename=settings.CSV_NAME)
cities = {}
with open('us_cities.csv') as csvFile:
csvReader = csv.DictReader(csvFile)
for i, rows in enumerate(csvReader):
city = rows['CITY']
state = rows['STATE_NAME']
cities[city] = state
data = {}
results = {}
with open('tweets.csv') as csvFile:
csvReader = csv.DictReader(csvFile)
for i, rows in enumerate(csvReader):
result = {}
if not rows['user_location']:
continue
else:
loc = rows['user_location'].split(',')