def export(self):
"""
Export records to one of the following formats:
- Print: Just print to console
- JSON: Write json to file
- CSV: write to csv file
- Mongo: Load into MongoDB
:return:
"""
if self.__records:
if self.formats['print']:
pp = pprint.PrettyPrinter(indent=4)
pp.pprint(self.__records)
if self.formats['json']:
self.export_to_json()
if self.formats['csv']:
self.export_to_csv()
if self.formats['mongo']:
self.load_into_mongo()
else:
u.make_message('\t\tNo Records Exist!', 3)
def export_to_json(self):
"""
Exports the records to a JSON formatted file. Output file is in the output location + input file name + '.json'
:return:
"""
import json
u.make_message('\t\tExporting to JSON %s' % os.path.basename(self.file))
output = os.path.join(self.output_location, self.file_name + '.json')
with open(output, 'w') as w:
w.write(json.dumps(self.__records, default=json_util.default))
del output
def unzip(self):
"""
Unzips and loads records into a temp list
:return:
"""
u.make_message('\t\tUnzipping %s' % os.path.basename(self.file), 2)
with gzip.open(self.file, 'rb') as f:
f.next() # Skip first row
# Load all records into a list
for row in f:
wr = weather_reading.weatherReading(row)
self.__records.append(wr.parsed)
def export_to_csv(self):
"""
Exports the records to a CSV formatted file. Output file is in the output location + input file name + '.csv'
:return:
"""
import csv
u.make_message('\t\tExporting to JSON %s' % os.path.basename(self.file))
output = os.path.join(self.output_location, self.file_name + '.csv')
with open(output, 'w') as w:
c = csv.DictWriter(w, self.__records[0].keys())
c.writeheader()
c.writerows(self.__records)
del c
del output
def chunks(l, n):
return [l[i:i+n] for i in range(0, len(l), n)]
if __name__ == '__main__':
log_file = settings.log_info['location']
if settings.log_info['do_timestamp']:
timestamp = datetime.datetime.now().strftime('%Y-%m-%d')
log_file = join(dirname(log_file), "%s_%s" % (timestamp, basename(log_file)))
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s',
filename=log_file,
filemode='w')
# Chunk this by 5 so that we do not totally fill up the HD with large downloads!
year_groups = chunks(settings.years, 5)
for years in year_groups:
u.make_message('Starting...')
if settings.do_download:
whole_years = u.download(years)
unzipped_years = u.expand(whole_years)
else:
unzipped_years = [join(settings.file_dir, str(year)) for year in years]
u.parse_years(unzipped_years)
u.make_message('Finished...')
if __name__ == '__main__':
log_file = settings.log_info['location']
if settings.log_info['do_timestamp']:
timestamp = datetime.datetime.now().strftime('%Y-%m-%d')
log_file = join(dirname(log_file),
"%s_%s" % (timestamp, basename(log_file)))
logging.basicConfig(level=logging.DEBUG,
format='%(asctime)s %(levelname)s %(message)s',
filename=log_file,
filemode='w')
# Chunk this by 5 so that we do not totally fill up the HD with large downloads!
year_groups = chunks(settings.years, 5)
for years in year_groups:
u.make_message('Starting...')
if settings.do_download:
whole_years = u.download(years)
unzipped_years = u.expand(whole_years)
else:
unzipped_years = [
join(settings.file_dir, str(year)) for year in years
]
u.parse_years(unzipped_years)
u.make_message('Finished...')
