def handle(self, *args, **options):
if options['flush']:
old = Story.objects.all()
confirm = raw_input(
'This will delete all %d existing stories. Are you sure? [y/N] '
% old.count())
if confirm == 'y':
old.delete()
# get input_file from stdin
input_file = fileinput.input(args)
temp_file = tempfile.TemporaryFile()
# save to temp storage for json parsing
for line in input_file:
temp_file.write(line)
temp_file.seek(0)
with temp_file as jsonfile:
stories = json.load(jsonfile)
n_s, n_a = (0, 0)
for data in stories['data']:
story = Story(content=data.get('Content'))
author, new_author = Author.objects.get_or_create_user(
user__name=data.get('UserName'))
if new_author:
n_a = n_a + 1
author.part_time = bool(data.get('PartTime'))
author.employed = bool(data.get('Employed'))
author.employer = data.get('Employer')
author.occupation = data.get('Occupation')
if author.user.last_name.lower() == "anonymous":
author.anonymous = True
author.save()
story.author = author
if data.get('Truncated'):
story.truncated = True
if data.get('Latitude') and data.get('Longitude'):
location, new_location = Location.objects.get_or_create(
city=data.get('City'), state=data.get('State'))
if new_location and data.get('Latitude') and data.get(
'Longitude'):
location.lat = data.get('Latitude')
location.lon = data.get('Longitude')
location.geocoded = True
location.save()
story.location = location
story.save()
if data.get('Timestamp'):
story.created_at = data['Timestamp']
else:
# old, put it before anything else
story.created_at = datetime(2013, 7, 1, 0, 0)
story.updated_at = datetime.now()
story.save()
n_s = n_s + 1
self.stdout.write("imported %d stories by %d authors" % (n_s, n_a))
def handle(self, *args, **options):
if options['flush']:
old = Story.objects.filter(employer__startswith="Walmart")
confirm = raw_input(
'This will delete all %d existing Walmart stories. Are you sure? [y/N] '
% old.count())
if confirm == 'y':
old.delete()
# get input_file from stdin
input_file = fileinput.input(args)
temp_file = tempfile.TemporaryFile()
# save to temp storage for json parsing
for line in input_file:
temp_file.write(line)
temp_file.seek(0)
with temp_file as jsonfile:
stories = json.load(jsonfile)
n_s, n_a, n_l = (0, 0, 0)
for data in stories:
try:
story, new_story = Story.objects.get_or_create(
content=data.get('story'))
except Story.MultipleObjectsReturned:
duplicates = Story.objects.filter(
content=data.get('story'))
duplicates.delete()
story = Story(content=data.get('story'))
first_name = unidecode(data.get('fname'))
last_name = unidecode(data.get('lname'))
author, new_author = Author.objects.get_or_create_user(
first_name=first_name, last_name=last_name)
if new_author:
n_a = n_a + 1
if data.get('email'):
author.user.email = data.get('email')
author.user.active = False
author.user.save()
if data.get('store'):
author.employer = "Walmart #" + data.get('store')
else:
author.employer = "Walmart"
if data.get('associate'):
author.occupation = "Associate"
if data.get('anonymous'):
author.anonymous = True
author.save()
story.author = author
if data.get('zip'):
zipcode = data.get('zip')
# do zip -> city lookup inline
zip_lookup = requests.get("http://api.zippopotam.us/us/" +
zipcode)
print "lookup", zipcode
place = zip_lookup.json().get('places', [{}])[0]
city = place.get('place name')
state = place.get('state abbreviation')
lat, lon = place.get('latitude'), place.get('longitude')
try:
location, new_location = Location.objects.get_or_create(
city__iexact=city, state=state)
except Location.MultipleObjectsReturned:
duplicate_locations = Location.objects.filter(
city__iexact=city, state=state)
stories_at_location = duplicate_locations.values_list(
'story', flat=True)
duplicate_locations.delete()
location = Location(city=city, state=state)
location.save()
for reset_id in stories_at_location:
try:
reset_story = Story.objects.get(id=reset_id)
reset_story.location = location
reset_story.save()
except Story.DoesNotExist:
pass
new_location = True
if new_location and lat and lon:
location.city = city
location.state = state
location.lat = lat
location.lon = lon
location.geocoded = True
location.save()
n_l = n_l + 1
story.location = location
story.save()
# export date from OurWalmart
story.created_at = datetime(2013, 9, 4, 0, 0)
story.updated_at = datetime.now()
story.save()
if new_story:
n_s = n_s + 1
self.stdout.write(
"imported %d stories by %d authors in %d locations" %
(n_s, n_a, n_l))