def test_freeze_csv(self): freeze(self.tbl.all(), format='csv', filename='weather.csv', prefix=self.d) path = os.path.join(self.d, 'weather.csv') if PY3: fh = open(path, 'rt', encoding='utf8', newline='') else: fh = open(path, 'rU') try: rows = list(reader(fh)) keys = rows[0] for i, d1 in enumerate(TEST_DATA): d2 = dict(zip(keys, rows[i + 1])) for k in d1.keys(): v2 = d2[k] if not PY3: v2 = v2.decode('utf8') v1 = value_to_str(d1[k]) if not isinstance(v1, text_type): if isinstance(v1, binary_type): v1 = text_type(v1, encoding='utf8') else: v1 = '%s' % v1 self.assertEqual(v2, v1) finally: fh.close()
def test_freeze(self): freeze(self.tbl.all(), format='csv', filename=u'wäther.csv'.encode('utf8'), prefix=self.d) self.assertTrue(os.path.exists(os.path.join(self.d, u'wäther.csv'))) freeze(self.tbl.all(), format='csv', filename=u'wäther.csv', prefix=self.d) self.assertTrue(os.path.exists(os.path.join(self.d, u'wäther.csv')))
def export_tables(params): """Dump tables to csv files and upload to s3 bucket""" tempdir = tempfile.mkdtemp() for entry in params: table = datastore[entry["table"]] filename = entry.get("csv_filename", entry["table"] + ".csv") bucket = entry.get("bucket") freeze(table.all(), filename=filename, prefix=tempdir) file_path = os.path.join(tempdir, filename) s3_upload(bucket, file_path) shutil.rmtree(tempdir)
def test_memory_streams(self): if PY3: from io import StringIO else: from io import BytesIO as StringIO for fmt in ('csv', 'json', 'tabson'): with StringIO() as fd: freeze(self.tbl.all(), format=fmt, fileobj=fd) self.assertFalse(fd.closed, 'fileobj was closed for format %s' % fmt) fd.getvalue() # should not throw
def test_freeze_json_no_wrap(self): freeze(self.tbl.all(), format='json', filename='weather.csv', prefix=self.d, wrap=False) path = os.path.join(self.d, 'weather.csv') if PY3: fh = open(path, 'rt', encoding='utf8', newline='') else: fh = open(path, 'rU') try: import json data = json.load(fh) self.assertIsInstance(data, list, 'Without wrapping, returned JSON should be a list') finally: fh.close()
def test_freeze_json_wrap(self): freeze(self.tbl.all(), format='json', filename='weather.csv', prefix=self.d, wrap=True) path = os.path.join(self.d, 'weather.csv') if PY3: fh = open(path, 'rt', encoding='utf8', newline='') else: fh = open(path, 'rU') try: import json data = json.load(fh) self.assertIsInstance(data, dict, 'With wrapping, returned JSON should be a dict') self.assertIn('results', data.keys()) self.assertIn('count', data.keys()) self.assertIn('meta', data.keys()) finally: fh.close()
import settings import tweepy import dataset from textblob import TextBlob from datafreeze.app import freeze import csv, json db = dataset.connect(settings.CONNECTION_STRING) result = db[settings.TABLE_NAME].all() freeze(result, format='csv', filename=settings.CSV_NAME) cities = {} with open('us_cities.csv') as csvFile: csvReader = csv.DictReader(csvFile) for i, rows in enumerate(csvReader): city = rows['CITY'] state = rows['STATE_NAME'] cities[city] = state data = {} results = {} with open('tweets.csv') as csvFile: csvReader = csv.DictReader(csvFile) for i, rows in enumerate(csvReader): result = {} if not rows['user_location']: continue else: loc = rows['user_location'].split(',')