def save_and_return_story_id(self, user):
url = self.cleaned_data["url"]
story_title = self.cleaned_data["story_title"]
try:
story = Story.objects.get(title=story_title, created_by=user)
article = Article(url=url, story=story, created_by=user)
except Story.DoesNotExist:
story = Story(title=story_title, created_by=user)
story.save()
article = Article(url=url, story=story, created_by=user)
#Get title and picture using the Open Graph meta tags in the html
#file of the article
response = requests.get(article.url)
html_soup = BeautifulSoup(response.text, "html.parser")
title = html_soup.find("meta", property="og:title")["content"]
image_url = html_soup.find("meta", property="og:image")["content"]
article.title = title
article.image_url = image_url
article.save()
return story.id
def new_story(request, twilio_request, response=twiml.Response()):
try:
author = Author.objects.get(sms_number=twilio_request.from_)
except Author.DoesNotExist:
first_name = "Anonymous"
last_anon = User.objects.filter(first_name=first_name).count()
last_name = "SMS#%d" % (last_anon + 1, )
username = "******".format(first_name, last_name).lower()
user, new_user = User.objects.get_or_create(first_name=first_name,
last_name=last_name,
username=username)
author = Author(user=user, sms_number=twilio_request.from_)
author.save()
story = Story(author=author, content=twilio_request.body)
story.save()
# save storyId for continuation
request.set_cookie('storyId', story.id)
response.message(
"Thank you for sharing with StoriesOfSolidarity.org" +
" Now put your story on the map! Reply with your zipcode for accurate posting."
)
return response
def submit(request):
PhotoFormSet = formset_factory(PhotoForm, extra=1)
if request.method == 'POST':
storyform = StoryForm(request.POST, request.FILES)
photoformset = PhotoFormSet(request.POST, request.FILES)
if storyform.is_valid() and photoformset.is_valid():
s = Story(
title = storyform.cleaned_data['title'],
moderator = request.user
)
s.save();
for form in photoformset.cleaned_data:
image = form['image']
p = Photo(
name=s.title,
story=s,
image=image
)
p.save()
else:
storyform = StoryForm()
photoformset = PhotoFormSet()
return render(request, 'stories/submit.html', {'storyfrom': storyform, 'photoformset': photoformset})
def save_story(self, story):
s = Story(
headline=story['title']['#cdata-section'],
byline=story['author'],
pubdate=parser.parse(story['pubDate']),
description=story['description']['#cdata-section'],
full_text=story['body']['#cdata-section'],
small_image_url=story['smallimage'],
large_image_url=story['largeimage'],
video_url=story['video']
)
s.save()
entities = []
if story['metadata'] is not None:
for k, v in story['metadata'].items():
entity_type, entity_type_created = EntityType.objects.get_or_create(entity_type=k)
if isinstance(v, dict):
entity_obj, entity_created = Entity.objects.get_or_create(
entity_name=v['@value'],
entity_type=entity_type
)
entities.append(entity_obj)
else:
for e in v:
entity, entity_created = Entity.objects.get_or_create(
entity_name=e['@value'],
entity_type=entity_type
)
entities.append(entity)
s.entities.add(*entities)
class StoryStatsModelTest(TestCase):
"""Tests for story stats (one to one model relationship with story)"""
def setUp(self):
"""Give 'em a nice story"""
self.story = Story(title="My Story title2")
self.story.save()
self.stats = None
def test_storystats_save(self):
"""test the maths adds up after stats added"""
self.stats = StoryStats(
story=self.story,
minidome_public_wins=1,
minidome_public_losses=2,
minidome_logged_in_wins=3,
minidome_logged_in_losses=4,
)
self.stats.save()
self.assertEqual(self.stats.get_minidome_total_logged_in_tests(), 7)
self.assertEqual(self.stats.get_minidome_total_public_tests(), 3)
self.assertEqual(self.stats.get_minidome_total_wins(), 4)
self.assertEqual(self.stats.get_minidome_total_losses(), 6)
def test_storystats_string_representation(self):
"""Test storystats returns title as string"""
self.stats = StoryStats(
story=self.story,
minidome_public_wins=1,
minidome_public_losses=2,
minidome_logged_in_wins=3,
minidome_logged_in_losses=4,
)
self.stats.save()
self.assertEqual(str(self.stats), f"stats for {self.story.title}")
def create_story(self):
story = Story()
story.headline = "This Is A Test Story"
#normally slugify will be handled by the UI
story.slug = slugify(story.headline)
summary = "Like I said in the headline, this is a test story."
#TODO add an author(s)
story.save()
return story
def newStory(request):
title1 = request.POST['Title']
firstLine = request.POST['firstLine']
numStory = request.POST['numLine']
u = request.user
new = Story(title = title1, user = u, maxNum = numStory, lineNum = 1)
new.save()
Line(content = firstLine, user = u, story = new).save()
return HttpResponseRedirect('/story/' + str(new.id))
class EnterContestOldStoryFormTest(TestCase):
"""Enter contest with an old story form test"""
def setUp(self):
"""set up"""
self.story = Story(
title="Story1", content="This is a <b>story</b>", access=Story.PRIVATE
)
self.story.save()
self.contest = Contest(
max_wordcount=10,
expiry_date=timezone.now() + timezone.timedelta(7),
start_date=timezone.now(),
title="Contest1",
content="This is a <b>contest</b>",
)
self.contest.save()
self.form = EnterContestOldStoryForm(
data={"story": self.story},
story_wordcount=html_wordcount(self.story.content),
contest_max_wordcount=self.contest.max_wordcount,
contest_expiry_date=self.contest.expiry_date,
)
def test_enter_contest_old_story_is_valid(self):
"""test form is valid"""
self.assertTrue(self.form.is_valid())
def test_enter_contest_old_story_form_exceeds_wordcount(self):
"""test wordcount limit"""
self.contest.max_wordcount = 1
self.verify_wordcount_for_old_stories(
"More words than wordcount. Kill your darlings!"
)
def test_enter_contest_form_contest_expired(self):
"""test expiry date not passed"""
self.contest.expiry_date = timezone.now() - timezone.timedelta(7)
self.verify_wordcount_for_old_stories(
"The deadline for that contest is over."
)
def verify_wordcount_for_old_stories(self, error_message):
"""repeated code to verify old stories meet criteria"""
self.contest.save()
self.form = EnterContestOldStoryForm(
data={"story": self.story},
story_wordcount=html_wordcount(self.story.content),
contest_max_wordcount=self.contest.max_wordcount,
contest_expiry_date=self.contest.expiry_date,
)
self.assertFalse(self.form.is_valid())
self.assertTrue(error_message in self.form.non_field_errors())
def post(self, request):
# Get user and unique identifier
user = request.user
# Create new story
new_story = Story(author=user)
new_story.save()
# Serialize the story
serializer = StorySerializer(new_story)
# Return the story
return Response(serializer.data, status=status.HTTP_201_CREATED)
def add_story(request, emp_id):
if request.method == 'POST':
try:
story_data = json.loads(request.body)
story = Story(project_name=story_data['name'],
project_desc=story_data['desc'],
skills_required=[
Skill.objects.get(tag=s)
for s in story_data['skills']
],
deadline=datetime.strptime(story_data['deadline']))
story.save()
return JsonResponse({}, status=200)
except:
return JsonResponse({}, status=400)
return JsonResponse({}, status=400)
def create_multiple_stories(self, num_stories=100):
"""creates range of stories and authors and privacy levels
modulo 3 = public
3+1 = logged in
3+2 = private
"""
self.users = []
self.stories = []
for story_num in range(num_stories):
if story_num % 3 == 0:
access = Story.PUBLIC
elif story_num % 3 == 1:
access = Story.LOGGED_IN
else:
access = Story.PRIVATE
self.users.append(
self.User.objects.create_user(
username=f"djangotestuser{story_num}",
password=f"{story_num}2345abcde",
)
)
# users[-1] = last in list
self.users[-1].save()
self.stories.append(
Story(
title=f"My Story{story_num}",
content="This is a story all about how...",
author=self.users[-1],
access=access,
)
)
self.stories[-1].save()
def create_stories(self):
import datetime
from django.template.defaultfilters import slugify
from django.contrib.sites.models import Site
from django_ext import markov
from stories.models import Story
from categories.models import Category
from staff.models import StaffMember
chain = markov.MarkovChain()
staff = StaffMember.objects.all()[:]
categories = Category.objects.all()[:]
# Load the dictionary into the markov chain
sentences = os.path.abspath(
os.path.join(fixtures_path, 'demo_story_data.txt'))
for line in open(sentences):
words = line.strip().split()
chain.add(words)
# Let's generate 10 random stories
for i in range(10):
story_data = {'headline': " ".join(chain.random_output(15))}
story_data['slug'] = slugify(story_data['headline'])[:30]
# 25% chance of a subhead
if random.randint(1, 4) == 1:
story_data['subhead'] = " ".join(chain.random_output(20))
story_data['teaser'] = " ".join(chain.random_output())
story_data['publish_date'] = datetime.datetime.now().date()
story_data['publish_time'] = datetime.datetime.now().time()
story_data['site'] = Site.objects.get_current()
story_data['primary_category'] = random.choice(categories)
# Create a story with 4-10 paragraphs with 1-4 sentences each
body = []
for j in range(0, random.randint(4, 10)):
p = []
for i in range(0, random.randint(1, 4)):
p.append(" ".join(chain.random_output()))
body.append("<p>%s</p>" % " ".join(p))
story_data['body'] = "\n".join(body)
story = Story(**story_data)
story.save()
story.authors.add(random.choice(staff))
def create_story(
self, access=UNIVERSAL_PRIVATE, author=None, title=None, wordcount=600
):
"""Creates many types of stories"""
fake = Faker()
if not author:
author = self.user
if not title:
title = fake.sentence()
content = fake.sentence(nb_words=wordcount, variable_nb_words=False)
self.story = Story(
title=title,
content=content,
author=author,
access=access,
)
self.story.save()
def create_stories(self):
import datetime
from django.template.defaultfilters import slugify
from django.contrib.sites.models import Site
from django_ext import markov
from stories.models import Story
from categories.models import Category
from staff.models import StaffMember
chain = markov.MarkovChain()
staff = StaffMember.objects.all()[:]
categories = Category.objects.all()[:]
# Load the dictionary into the markov chain
sentences = os.path.abspath(os.path.join(fixtures_path, "demo_story_data.txt"))
for line in open(sentences):
words = line.strip().split()
chain.add(words)
# Let's generate 10 random stories
for i in range(10):
story_data = {"headline": " ".join(chain.random_output(15))}
story_data["slug"] = slugify(story_data["headline"])[:30]
# 25% chance of a subhead
if random.randint(1, 4) == 1:
story_data["subhead"] = " ".join(chain.random_output(20))
story_data["teaser"] = " ".join(chain.random_output())
story_data["publish_date"] = datetime.datetime.now().date()
story_data["publish_time"] = datetime.datetime.now().time()
story_data["site"] = Site.objects.get_current()
story_data["primary_category"] = random.choice(categories)
# Create a story with 4-10 paragraphs with 1-4 sentences each
body = []
for j in range(0, random.randint(4, 10)):
p = []
for i in range(0, random.randint(1, 4)):
p.append(" ".join(chain.random_output()))
body.append("<p>%s</p>" % " ".join(p))
story_data["body"] = "\n".join(body)
story = Story(**story_data)
story.save()
story.authors.add(random.choice(staff))
def add_story(request):
if request.method == 'POST':
project_id = request.POST.get('project', None)
story = Story(accepted=True)
story.order = Story.objects.filter(project=project_id).count()+1
form = StoryForm(request.POST, instance=story)
if form.is_valid():
form.save()
return redirect('project_page', project_id=project_id )
else:
request.session['error'] = {
'ref':'add_error',
'name':request.POST.get('name', ''),
'time':request.POST.get('time', ''),
'user':request.POST.get('user', ''),
}
return redirect('project_page', project_id=project_id)
else:
raise Http404
def setUp(self):
"""set up"""
self.story = Story(
title="Story1", content="This is a <b>story</b>", access=Story.PRIVATE
)
self.story.save()
self.contest = Contest(
max_wordcount=10,
expiry_date=timezone.now() + timezone.timedelta(7),
start_date=timezone.now(),
title="Contest1",
content="This is a <b>contest</b>",
)
self.contest.save()
self.form = EnterContestOldStoryForm(
data={"story": self.story},
story_wordcount=html_wordcount(self.story.content),
contest_max_wordcount=self.contest.max_wordcount,
contest_expiry_date=self.contest.expiry_date,
)
def NewStory(request):
user = request.user
file_objs = []
if request.method == "POST":
form = NewStoryForm(request.POST, request.FILES)
if form.is_valid():
file = request.FILES.get('content')
caption = form.cleaned_data.get('caption')
story = Story(user=user, content=file, caption=caption)
story.save()
return redirect('index')
else:
form = NewStoryForm()
context = {
'form': form,
}
return render(request, 'newstory.html', context)
def create_stories(self): for x in xrange(0,2): story = Story() story.title = story_titles[x] story.article_body = story_articles[x] story.image = story_images[x] story.save()
def main():
# Attempt to retrieve and process the data from the Unoffical Hacker News API
for i in range(RETRY_ATTEMPTS + 1):
try:
response = urllib2.urlopen(HACKER_NEWS_API_URL)
status_code = response.code
except urllib2.HTTPError as e:
status_code = e.code
# If the service errored, hit it again
if status_code != 200:
if i <= RETRY_ATTEMPTS:
print("An error occured while retrieving the data, retrying (%d)..." % (i + 1), file=sys.stderr)
continue
# If everything went ok, try to load the data
try:
items = json.load(response)["items"]
break
except ValueError as e:
if i <= RETRY_ATTEMPTS:
print("An error occurred while loading the data, retrying (%d)..." % i + 1, file=sys.stderr)
continue
else:
sys.exit("Too many errors occurred while attempting to retrieve the data")
# Add the stories to the database
moderator = User.objects.get(username=USERNAME)
for item in items:
story = Story(title=item["title"], url=item["url"], points=item["points"], moderator=moderator)
story.save()
story.created_at = created_at(item)
story.save()
def submit(request):
"""Submit new post view"""
topic_list = Topic.objects.get_tree()
selected_topic = ''
if request.method == 'POST':
form = StoryForm(request.POST)
if form.is_valid():
s = Story(title = form.cleaned_data['title'], text = form.cleaned_data['text'], user = request.user, topic = form.cleaned_data['topic'], url = form.cleaned_data['url'])
s.slug = slugify(form.cleaned_data['title'])
s.save()
ds = DuggedStory(user = request.user, story = s)
ds.save()
urlredirect = reverse('stories_slug', args=[s.topic.urlname, s.slug])
return redirect_to(request, urlredirect)
else:
if form.data['topic']:
try:
selected_topic = int(form.data['topic'])
except:
pass
else:
form = StoryForm()
return render_to_response('stories/submit.html', {'form': form, 'user': request.user, 'topic_list': topic_list, 'selected_topic': selected_topic})
def create_common_components(self, num_entrants):
"""Sets up a range of users, stories and entries"""
self.users = []
self.stories = []
self.entries = []
for user_num in range(num_entrants):
self.users.append(
self.User.objects.create_user(
username=f"djangotestuser{user_num}",
password=f"{user_num}2345abcde",
)
)
self.users[-1].save()
for user in self.users:
self.stories.append(Story(author=user, access=Story.PRIVATE))
self.stories[-1].save()
def story_create(request):
# create a new story
story = Story(title="New story")
# make our story author set
story.author = request.user
# Generate a random path for this story
prefix = STORY_PATH + str(request.user)
story.path = str(request.user) + '/' + str(uuid.uuid4()) + ".txt"
# Make the path if it's not already there
if not os.path.exists(prefix):
os.makedirs(prefix)
# Save story details
story.save()
# Initial commit -- create story and mark it
story.commit("", "Story created")
# Direct the user to the edit interface for their new story
return redirect(story_edit, story_id=story.pk)
class StoryModelTest(TestCase):
"""Test A Story"""
def setUp(self):
self.story = Story(title="My Story title")
def test_story_string_representation(self):
"""Test story returns title as string"""
self.assertEqual(str(self.story), self.story.title)
def test_story_verbose_name_plural(self):
"""Test story has verbose name"""
self.assertEqual(str(Story._meta.verbose_name_plural), "stories")
def test_story_save_function(self):
"""Test Story can save into database"""
self.story.public = False
self.story.save()
self.assertTrue(self.story.slug == "my-story-title")
def test_story_save_title_again_function(self):
"""Test story can update title"""
self.story.public = False
self.story.title = "my changed title"
self.story.save()
self.assertTrue(self.story.slug == "my-changed-title")
def test_story_wordcount(self):
"""Test wordcount function"""
self.story.public = False
self.story.content = "1 2 3 4 5 6 7 8 9 10"
self.story.save()
self.assertTrue(self.story.wordcount == 10)
def test_story_creates_storystats(self):
"""Tests the creation of stats by listener after story gets saved"""
self.story.save()
self.assertTrue(self.story.stats.minidome_public_wins == 0)
def saveItemIfPositionChanged(pos, item, feed, update):
url = item['link']
newItem = False
try:
# try to find this item in Django
s = Story.objects.get(url=url)
except Story.DoesNotExist:
# story does not exist, so create one now
title = item['title']
# active stories will have null time_closed values: this will make it easy to filter for open stories.
time_closed = None
s = Story(url=url, title=title, current_position=pos, time_closed=time_closed, feed=feed)
s.save()
newItem = True
except Story.MultipleObjectsReturned:
# todo_sam: this should never happen
# error("MultipleObjectsReturned for url='%s'" % url)
return False
# at this point s will be a valid Story object, either new or old
currPos = s.current_position
story_closed = bool(s.time_closed)
if newItem or currPos != pos:
# the position changed,
# create a story history...
sh = StoryHistory(position = pos, story = s, update = update)
sh.save()
# if the position doesn't match or the story has been closed we will need to save the story
if currPos != pos or story_closed:
# update the cached pos
s.current_position = pos
s.time_closed = None
s.save()
# True: we succeeded in saving the pos, return the story's id
return s.id
def main():
# Attempt to retrieve and process the data from the Unoffical Hacker News API
for i in range(RETRY_ATTEMPTS + 1):
try:
response = urlopen(HACKER_NEWS_API_URL)
status_code = response.code
except HTTPError as e:
status_code = e.code
print (e.code)
# If the service errored, hit it again
if status_code != 200:
if i <= RETRY_ATTEMPTS:
print("An error occured while retrieving the data, retrying (%d)..." % (i+1), file=sys.stderr)
print (status_code.__str__())
continue
# If everything went ok, try to load the data
try:
items = json.loads(response.read().decode('utf-8'))['items']
break
except ValueError as e:
if i <= RETRY_ATTEMPTS:
print("An error occurred while loading the data, retrying (%d)..." % i+1, file=sys.stderr)
continue
else:
sys.exit("Too many errors occurred while attempting to retrieve the data")
# Add the stories to the database
moderator = User.objects.get(username=USERNAME)
for item in items:
story = Story(
title=item['title'],
url=item['url'],
points=item['points'],
moderator=moderator)
story.save()
story.created_at = created_at(item)
story.save()
def save_item(json_item):
global STORY_COUNT
# Add the stories to the database
moderator = User.objects.get(username='******')
for item in json_item:
STORY_COUNT += 1
print item['data']['title'].encode('utf-8')
print "http://www.reddit.com" + item['data']['permalink'].encode('utf-8')
print datetime.fromtimestamp(item['data']['created_utc']).strftime('%Y-%m-%d %H:%M:%S.000000+00:00'), '\n'
story = Story(title=item['data']['title'],
url="http://www.reddit.com" + item['data']['permalink'],
points=1,
moderator=moderator)
story.save()
story.created_at = datetime.fromtimestamp(item['data']['created_utc']).strftime('%Y-%m-%d %H:%M:%S.000000+00:00')
story.save()
def main():
# Get the Top 100 Stories id's as list of int's
top_stories_ids = get_data(HACKER_NEWS_API_TOP_STORIES_URL)
# Get & load the json data of the Top 30 Stories by their id's
for id in top_stories_ids[:30]:
story = get_data(HACKER_NEWS_API_STORY_URL % id)
story_created_at = created_at(story['time'])
# Add the stories to the database
moderator = User.objects.get(username=USERNAME)
story = Story(
title=story['title'],
url=story['url'],
points=story['score'],
moderator=moderator,)
story.save()
story.created_at = story_created_at
story.save()
# If everything went ok, try to load the data
try:
items = json.loads(response.content)['items']
break
except ValueError, e:
if i <= RETRY_ATTEMPTS:
print(
"An error occurred while loading the data, retrying (%d)..."
% i + 1,
file=sys.stderr)
continue
else:
sys.exit(
"Too many errors occurred while attempting to retrieve the data")
# Add the stories to the database
moderator = User.objects.get(username=USERNAME)
for item in items:
story = Story(title=item['title'],
url=item['url'],
points=item['points'],
moderator=moderator)
story.save()
story.created_at = created_at(item)
story.save()
if __name__ == '__main__':
main()
# If the service errored, hit it again
if status_code != 200:
if i <= RETRY_ATTEMPTS:
print("An error occured while retrieving the data, retrying (%d)..." % (i+1), file=sys.stderr)
continue
# If everything went ok, try to load the data
try:
items = json.load(response)['items']
break
except ValueError, e:
if i <= RETRY_ATTEMPTS:
print("An error occurred while loading the data, retrying (%d)..." % i+1, file=sys.stderr)
continue
else:
sys.exit("Too many errors occurred while attempting to retrieve the data")
# Add the stories to the database
moderator = User.objects.get(username=USERNAME)
for item in items:
story = Story(
title=item['title'],
url=item['url'],
points=item['points'],
moderator=moderator)
story.save()
story.created_at = created_at(item)
story.save()
if __name__ == '__main__':
main()
def handle(self, *args, **options):
if options['flush']:
old = Story.objects.filter(employer__startswith="Walmart")
confirm = raw_input(
'This will delete all %d existing Walmart stories. Are you sure? [y/N] '
% old.count())
if confirm == 'y':
old.delete()
# get input_file from stdin
input_file = fileinput.input(args)
temp_file = tempfile.TemporaryFile()
# save to temp storage for json parsing
for line in input_file:
temp_file.write(line)
temp_file.seek(0)
with temp_file as jsonfile:
stories = json.load(jsonfile)
n_s, n_a, n_l = (0, 0, 0)
for data in stories:
try:
story, new_story = Story.objects.get_or_create(
content=data.get('story'))
except Story.MultipleObjectsReturned:
duplicates = Story.objects.filter(
content=data.get('story'))
duplicates.delete()
story = Story(content=data.get('story'))
first_name = unidecode(data.get('fname'))
last_name = unidecode(data.get('lname'))
author, new_author = Author.objects.get_or_create_user(
first_name=first_name, last_name=last_name)
if new_author:
n_a = n_a + 1
if data.get('email'):
author.user.email = data.get('email')
author.user.active = False
author.user.save()
if data.get('store'):
author.employer = "Walmart #" + data.get('store')
else:
author.employer = "Walmart"
if data.get('associate'):
author.occupation = "Associate"
if data.get('anonymous'):
author.anonymous = True
author.save()
story.author = author
if data.get('zip'):
zipcode = data.get('zip')
# do zip -> city lookup inline
zip_lookup = requests.get("http://api.zippopotam.us/us/" +
zipcode)
print "lookup", zipcode
place = zip_lookup.json().get('places', [{}])[0]
city = place.get('place name')
state = place.get('state abbreviation')
lat, lon = place.get('latitude'), place.get('longitude')
try:
location, new_location = Location.objects.get_or_create(
city__iexact=city, state=state)
except Location.MultipleObjectsReturned:
duplicate_locations = Location.objects.filter(
city__iexact=city, state=state)
stories_at_location = duplicate_locations.values_list(
'story', flat=True)
duplicate_locations.delete()
location = Location(city=city, state=state)
location.save()
for reset_id in stories_at_location:
try:
reset_story = Story.objects.get(id=reset_id)
reset_story.location = location
reset_story.save()
except Story.DoesNotExist:
pass
new_location = True
if new_location and lat and lon:
location.city = city
location.state = state
location.lat = lat
location.lon = lon
location.geocoded = True
location.save()
n_l = n_l + 1
story.location = location
story.save()
# export date from OurWalmart
story.created_at = datetime(2013, 9, 4, 0, 0)
story.updated_at = datetime.now()
story.save()
if new_story:
n_s = n_s + 1
self.stdout.write(
"imported %d stories by %d authors in %d locations" %
(n_s, n_a, n_l))
def publish_story(story_id):
def process_shared_words(querySet, halfOfArticles):
# Building a dictionary with words as indexes so that I can see how
# many articles Re-Use the same word, if at least half of them re-use
# a word that word will be Added to the "agreed upon" words for
# the story and filtered out of the final Analysis.
shared_dict = {}
for detail in potentialWords:
try:
shared_dict[detail.potentialword.word].append(detail)
except:
shared_dict[detail.potentialword.word] = [detail]
shared_words = ""
for key, value in shared_dict.iteritems():
try:
if len(value) > halfOfArticles:
print ("Deleting: " + key)
for detail in value:
detail.delete()
shared_words = shared_words + key + "|"
except Exception as err:
print "Couldn't create dict: " + str(err)
return shared_words
potentialStory = PotentialStory.objects.get(id=story_id)
if potentialStory:
print (u"Publishing " + potentialStory.title)
try:
title_slug = slugify(potentialStory.title)
story = Story(title=potentialStory.title, article_slug=title_slug,
tag=potentialStory.tag)
story.save()
potentialArticles = PotentialArticle.objects.filter(
potentialstory=potentialStory).select_related()
potentialWords = WordDetail.objects.filter(
potentialarticle__potentialstory=potentialStory).select_related()
# If at least half of the articles re-use the same words remove
# them from the final analysis
half_of_total_articles = len(potentialArticles) / 2
sharedWords = process_shared_words(potentialWords,
half_of_total_articles)
story.shared_words = sharedWords
story.save()
bulk_insert_details = []
for article in potentialArticles:
# Process article
try:
newArticle = Article(story=story, source=article.source,
title=article.title, url=article.url)
newArticle.save()
# Process words and word details
try:
for word_detail in WordDetail.objects.filter(
potentialarticle=article):
word_to_use, exists = Word.objects.get_or_create(
word=word_detail.potentialword.word)
bulk_insert_details.append(
Detail(word=word_to_use,
article=newArticle,
usage=word_detail.usage))
except Exception as err:
print err
except Exception as err:
print err
Detail.objects.bulk_create(bulk_insert_details)
except Exception as err:
print ("Unable to publish story: " + str(story_id))
print err
def handle(self, *args, **options):
if options['flush']:
old = Story.objects.all()
confirm = raw_input(
'This will delete all %d existing stories. Are you sure? [y/N] '
% old.count())
if confirm == 'y':
old.delete()
# get input_file from stdin
input_file = fileinput.input(args)
temp_file = tempfile.TemporaryFile()
# save to temp storage for json parsing
for line in input_file:
temp_file.write(line)
temp_file.seek(0)
with temp_file as jsonfile:
stories = json.load(jsonfile)
n_s, n_a = (0, 0)
for data in stories['data']:
story = Story(content=data.get('Content'))
author, new_author = Author.objects.get_or_create_user(
user__name=data.get('UserName'))
if new_author:
n_a = n_a + 1
author.part_time = bool(data.get('PartTime'))
author.employed = bool(data.get('Employed'))
author.employer = data.get('Employer')
author.occupation = data.get('Occupation')
if author.user.last_name.lower() == "anonymous":
author.anonymous = True
author.save()
story.author = author
if data.get('Truncated'):
story.truncated = True
if data.get('Latitude') and data.get('Longitude'):
location, new_location = Location.objects.get_or_create(
city=data.get('City'), state=data.get('State'))
if new_location and data.get('Latitude') and data.get(
'Longitude'):
location.lat = data.get('Latitude')
location.lon = data.get('Longitude')
location.geocoded = True
location.save()
story.location = location
story.save()
if data.get('Timestamp'):
story.created_at = data['Timestamp']
else:
# old, put it before anything else
story.created_at = datetime(2013, 7, 1, 0, 0)
story.updated_at = datetime.now()
story.save()
n_s = n_s + 1
self.stdout.write("imported %d stories by %d authors" % (n_s, n_a))
def setUp(self):
"""Give 'em a nice story"""
self.story = Story(title="My Story title2")
self.story.save()
self.stats = None
def setUp(self):
self.story = Story(title="My Story title")
#okay got everything imported i need, so lets try pull our stories in from RSS
d = feedparser.parse( "http://www.news-leader.com/feeds/RSS16.xml")
print "%d Stories scraped from Crime RSS" % len( d[ "entries" ] )
#loop through our stories
for entry in d[ "entries" ]:
try:
#seeing if the story is in the db already or not.
story = Story.objects.get( id= entry.link.split("/")[-3] )
print "Story %s already in the db" % entry.link.split("/")[-3]
except:
story = Story()
print "Story %s not found. inserting into db" % entry.link.split("/")[-3]
#now we need to get our story id from the url
#we take the link from the entry. split it by / and then grab the 3rd item from the end
story.id = entry.link.split("/")[-3]
story.short_url = "http://www.news-leader.com/article/2010%s" % story.id
story.long_url = entry.link
story.active = True
story.category = entry.link.split("/")[-4]
story.headline = entry.title
story.summary = entry.summary
story.date_published = datetime.datetime( entry.updated_parsed[0], entry.updated_parsed[1], entry.updated_parsed[2], entry.updated_parsed[3], entry.updated_parsed[4], entry.updated_parsed[5] )
print "saving story"
story.save()
class UniversalAppTestCase(TestCase):
"""Handy TestClass Extension that can also perform functions to create useful entities"""
@classmethod
#reminder: setUpTestData is set up once at the beginning - setup is created once per test
def setUpTestData(cls):
cls.User = get_user_model()
#Generic test user for when you don't care about roles
cls.user = cls.User.objects.create_user(
username="******",
password="******",
email="*****@*****.**",
)
cls.user.timezone = "Africa/Abidjan"
cls.user.save()
#role-based test users
cls.contest_creator = cls.User.objects.create_user(
username="******",
password="******",
email="*****@*****.**",
)
cls.story_author = cls.User.objects.create_user(
username="******",
password="******",
email="*****@*****.**",
)
cls.staff_member = cls.User.objects.create_user(
username="******",
password="******",
email="*****@*****.**",
)
cls.staff_member.is_staff = True
cls.staff_member.save()
def login_testuser(self, username="******"):
"""re-usable code to login class user"""
self.login = self.client.login(username=username, password="******")
def create_brawl_participants(self):
"""creates participants in brawl"""
self.challenger = self.User.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
)
self.defender = self.User.objects.create_user(
username="******",
email="*****@*****.**",
password="******",
)
self.brawl_judge = self.User.objects.create_user(
username="******", email="*****@*****.**", password="******"
)
def create_brawl(self, challenger="", defender="", judge="", title=""):
"""creates contest and entries for brawl"""
if challenger == "":
challenger = self.challenger
if defender == "":
defender = self.defender
if judge == "":
judge = self.brawl_judge
if title == "":
title = "My brawl"
self.create_contest(Brawl, creator=judge, title=title)
self.contest.save()
self.challenge_entry = Entry.objects.create(
title="Challenge",
content="challenging content",
author=challenger,
contest=self.contest,
)
self.defend_entry = Entry.objects.create(
title="Defend",
content="defending content",
author=defender,
contest=self.contest,
)
def create_story(
self, access=UNIVERSAL_PRIVATE, author=None, title=None, wordcount=600
):
"""Creates many types of stories"""
fake = Faker()
if not author:
author = self.user
if not title:
title = fake.sentence()
content = fake.sentence(nb_words=wordcount, variable_nb_words=False)
self.story = Story(
title=title,
content=content,
author=author,
access=access,
)
self.story.save()
def create_multiple_stories(self, num_stories=100):
"""creates range of stories and authors and privacy levels
modulo 3 = public
3+1 = logged in
3+2 = private
"""
self.users = []
self.stories = []
for story_num in range(num_stories):
if story_num % 3 == 0:
access = Story.PUBLIC
elif story_num % 3 == 1:
access = Story.LOGGED_IN
else:
access = Story.PRIVATE
self.users.append(
self.User.objects.create_user(
username=f"djangotestuser{story_num}",
password=f"{story_num}2345abcde",
)
)
# users[-1] = last in list
self.users[-1].save()
self.stories.append(
Story(
title=f"My Story{story_num}",
content="This is a story all about how...",
author=self.users[-1],
access=access,
)
)
self.stories[-1].save()
def create_minidome_public(self):
"""creates a minidome public contest
requires create_multiple_stories
"""
self.minidome = MiniDome(
winner=self.stories[0], loser=self.stories[3], category=MiniDome.PUBLIC
)
self.minidome.save()
def create_minidome_logged_in(self):
"""creates a minidome logged in contest
requires create_multiple_stories
"""
self.minidome = MiniDome(
winner=self.stories[1], loser=self.stories[4], category=MiniDome.LOGGED_IN
)
self.minidome.save()
def create_contest(
self,
mode,
creator=None,
title="My Contest Title",
start_date=timezone.now(),
expiry_date=None,
hellrules=0,
):
"""Re-usable routine to set up contest object"""
if not creator and mode != Contest.BRAWL:
creator = self.user
if not expiry_date:
expiry_date = start_date + timezone.timedelta(7)
self.contest = mode(
title=title,
start_date=start_date,
expiry_date=expiry_date,
# status=Contest.OPEN,
max_wordcount=1000,
creator=creator,
content="PROOOOOMPT",
hellrules=hellrules,
)
def create_multiple_contests(self, mode, total):
"""Re-usable routine to set up contest object"""
self.contests = []
self.contests.extend (
mode(
title=f"Contest{num}",
start_date=timezone.now(),
expiry_date=timezone.now() + timezone.timedelta(7),
status=Contest.OPEN,
) for num in range(total))
def create_contest_components(self, num_entrants=7, hellrules=Contest.NO_HELLRULE):
"""reusable routine to set up contest associated objects
sets up:
5 users
5 stories
5 entries
"""
self.create_common_components(num_entrants)
if hellrules:
request_hellrule = True if hellrules > 1 else choice([True, False])
else:
request_hellrule = False
for story in self.stories:
self.entries.append(
Entry(
contest=self.contest,
author=story.author,
story=story,
title=f"good title{story.id}",
content="good content",
request_hellrule=request_hellrule,
)
)
self.entries[-1].save()
#add two failures, one toxxed if we have enough entrants
if num_entrants > 5:
self.entries[-1].content = ""
self.entries[-1].toxx = True
self.entries[-1].save()
self.entries[-2].content = ""
self.entries[-2].save()
def create_multiple_contest_components(self):
"""reusable routine to set up contest associated objects
sets up:
5 users
5 stories
5 entries
"""
self.create_common_components(5)
for contest, story in product(self.contests, self.stories):
self.entries.append(
Entry(
contest=contest,
author=story.author,
story=story,
title=f"bad title{story.id}",
)
)
self.entries[-1].save()
def create_common_components(self, num_entrants):
"""Sets up a range of users, stories and entries"""
self.users = []
self.stories = []
self.entries = []
for user_num in range(num_entrants):
self.users.append(
self.User.objects.create_user(
username=f"djangotestuser{user_num}",
password=f"{user_num}2345abcde",
)
)
self.users[-1].save()
for user in self.users:
self.stories.append(Story(author=user, access=Story.PRIVATE))
self.stories[-1].save()
def create_unscored_judge_table(self, contest_id, num_judges=3):
"""creates judges for judgetable"""
fake = Faker()
self.judges = []
ContestJudges.objects.filter(contest_id=contest_id).delete()
scores = [3, 5, 7, 11, 13, 17, 19]
self.judges.extend(
self.User.objects.create_user(
username=f"contest_judge{judge_num}",
password="******",
)
for judge_num in range(1, num_judges + 1))
for judge in self.judges:
ContestJudges.objects.create(
contest_id=contest_id, judge=judge, category=ContestJudges.ACCEPTED
)
for entry in self.entries:
Crit.objects.create(
reviewer=judge,
content=fake.sentence(nb_words=101),
score=choice(scores),
entry=entry,
final=True,
)
self.contest.creator = self.judges[0]
chief_judge = ContestJudges.objects.get(
contest_id=contest_id, judge=self.judges[0]
)
chief_judge.category = ContestJudges.CHIEF
self.contest.save()
chief_judge.save()
def create_judge_table(self, contest_id, num_judges=3):
"""creates judges for judgetable"""
fake = Faker()
self.judges = []
ContestJudges.objects.filter(contest_id=contest_id).delete()
scores = [3, 5, 7, 11, 13, 17, 19]
self.judges.extend(
self.User.objects.create_user(
username=f"contest_judge{judge_num}",
password="******",
) for judge_num in range(1, num_judges + 1))
for judge in self.judges:
ContestJudges.objects.create(
contest_id=contest_id, judge=judge, category=ContestJudges.ACCEPTED
)
for entry in self.entries:
Crit.objects.create(
reviewer=judge,
content=fake.sentence(nb_words=101),
score=choice(scores),
entry=entry,
final=True,
)
self.contest.creator = self.judges[0]
chief_judge = ContestJudges.objects.get(
contest_id=contest_id, judge=self.judges[0]
)
chief_judge.category = ContestJudges.CHIEF
self.contest.save()
chief_judge.save()
for entry in self.entries:
entry.score = Crit.objects.filter(entry=entry).aggregate(
avg_score=Avg("score")
)["avg_score"]
entry.final = True
entry.save()
self.sorted_entries = Entry.objects.filter(contest_id=contest_id).order_by(
"-score"
)
ent = self.sorted_entries[0]
ent.distinction = Entry.WIN
ent.save()
ent = self.sorted_entries[1]
ent.distinction = Entry.HM
# this story has previously been entered
ent.story.stats.final_score = ent.score
ent.story.stats.save()
ent.save()
self.sorted_entries = Entry.objects.filter(contest_id=contest_id).order_by(
"score"
)
ent = self.sorted_entries[0]
ent.distinction = Entry.LOSS
ent.save()
ent = self.sorted_entries[1]
ent.distinction = Entry.DM
ent.save()
ent = self.sorted_entries[2]
ent.distinction = Entry.DQ
ent.save()
def score_contest(self):
"""reuable routine to assign scores to contest entries"""
for user in self.users:
scores = [3, 5, 7, 11, 13, 17, 19]
self.crits = Crit.objects.filter(entry__contest=self.contest, reviewer=user)
for crit in self.crits:
shuffle(scores)
crit.score = scores.pop()
crit.final = True
crit.save()
def sign_up_for_contest(self, contest, user):
"""Signs up user for a particular contest
Args:
contest (Contest): Contest to assign scores
user (CustomUser): user account
"""
self.entry = Entry(contest=contest, author=user)
self.entry.save()
def get_messages(self, response, message_type=""):
"""returns messages for a given response"""
# Sourcery
# type_messages = []
# messages = list(response.context['messages'])
# for message in messages:
# if message_type:
# if message_type in message.tags:
# type_messages.append(message)
# else:
# type_messages.append(message)
# return type_messages
messages = list(response.context["messages"])
return [
message
for message in messages
if message_type and message_type in message.tags or not message_type
]
def create_brawl_challenge(self, challenger, defender):
"""set up brawl challenge"""
self.challenge = BrawlChallenge(
challenge="Fight Me", challenger=challenger, defender=defender
)
self.challenge.save()
def make(request):
st = Story(title=request.POST['title'], text=request.POST['block'])
st.save()
return redirect('/')
def setUpClass(cls):
cls.u = User(name="test", email="*****@*****.**")
cls.u.save()
cls.new_title = Story(moderator=cls.u, title="hello world")
cls.new_title.save()
if timeName == 'hour' or timeName == 'hours':
createTime = timeNow - datetime.timedelta(hours=timeValue)
elif timeName == 'minute' or timeName == 'minutes':
createTime = timeNow - datetime.timedelta(minutes=timeValue)
else:
createTime = timeNow - datetime.timedelta(days=timeValue)
existingUser = User.objects.filter(username=user)[:1]
if existingUser:
existingUser = existingUser[0]
else:
existingUser = User(username=user, password='******')
existingUser.save()
# Check if story already present
existingStory = Story.objects.filter(url=url)[:1]
if not existingStory:
existingStory = Story()
else:
existingStory = existingStory[0]
existingStory.title = title
existingStory.url = url
existingStory.points = points
existingStory.created_at = createTime
existingStory.moderator = existingUser
existingStory.save()
print title, '::', existingUser.username, '::', points