def history(request, id = None):
""" Show history list or display diff between two versions """
snippet = get_object_or_404(Snippet, pk=id)
if request.GET.get('v'):
version = int(request.GET['v'])
if version == 0:
body = snippet.highlight(snippet.body, get_lexer_by_name(snippet.lexer))
else:
ver = get_object_or_404(SnippetVersion, snippet = snippet, version=version)
body = snippet.highlight(ver.body, get_lexer_by_name(snippet.lexer))
return render_to_response('django_snippify/version.html',
{'snippet': snippet, 'version': version, 'body': body, 'lines': range(1, body.count('\n'))},
context_instance=build_context(request))
elif request.GET.get('v1') and request.GET.get('v2'):
version1 = int(request.GET['v1'])
version2 = int(request.GET['v2'])
if version1 == 0:
version1_label = 'current'
body_1 = snippet.body
else:
version1_label = 'v' + str(version1)
body_1 = get_object_or_404(SnippetVersion, snippet = snippet, version=version1).body
if version2 == 0:
version2_label = 'current'
body_2 = snippet.body
else:
version2_label = 'v' + str(version2)
body_2 = get_object_or_404(SnippetVersion, snippet = snippet, version=version2).body
fromlines = str(body_1).splitlines(True)
tolines = str(body_2).splitlines(True)
if len(fromlines) >= len(tolines):
no = len(fromlines)
else:
no = len(tolines)
diffbody = ''
for line in difflib.unified_diff(fromlines, tolines, fromfile=version1_label, tofile=version2_label):
diffbody = diffbody + str(line)
diffbody = snippet.highlight(diffbody, get_lexer_by_name('diff'))
return render_to_response('django_snippify/diff.html', {
'snippet': snippet,
'version1': version1,
'version2': version2,
'diffbody': diffbody,
'lines': range(1, diffbody.count('\n'))
},
context_instance=build_context(request))
else:
snippet_versions = SnippetVersion.objects.filter(snippet = snippet).all()
return render_to_response('django_snippify/history_index.html', {'snippet': snippet, 'snippet_versions': snippet_versions},
context_instance=build_context(request))
def archive_list(racedate):
racedate_db = PostgresqlExtDatabase('elex_%s' % racedate,
user=os.environ.get('ELEX_ADMIN_USER', 'elex'),
host=os.environ.get('ELEX_ADMIN_HOST', '127.0.0.1')
)
models.database_proxy.initialize(racedate_db)
context = utils.build_context(racedate)
context['files'] = sorted(
[
{
"name": f.split('/')[-1],
"date": datetime.datetime.fromtimestamp(float(f.split('/')[-1].split('-')[-1].split('.json')[0]))
}
for f in glob.glob('/tmp/%s/*.json' % racedate)
],
key=lambda x:x,
reverse=True
)[:750]
context['states'] = []
state_list = sorted(list(Set([race.statepostal for race in models.ElexRace.select()])), key=lambda x: x)
for state in state_list:
race = models.ElexRace.select().where(models.ElexRace.statepostal == state)[0]
state_dict = {}
state_dict['statepostal'] = state
state_dict['report'] = race.report
state_dict['report_description'] = race.report_description
context['states'].append(state_dict)
return render_template('archive_list.html', **context)
def index():
"""
Will match directories named like the following:
2015-09-10
09-10-2015
"""
context = utils.build_context()
context['elections'] = []
elections = sorted([a.split('/')[-1] for a in glob.glob('%s/*' % DATA_DIR) if re.match('(\d{2,4}[-]\d{2,4}[-]\d{2,4})', a.split('/')[-1])], key=lambda x:x)
for e in elections:
for level in ['local', 'national']:
national = False
if level == 'national':
national = True
e_dict = {}
election_key = 'AP_DEJAVU_%s' % e
e_dict['election_date'] = e
e_dict['national'] = national
e_dict['level'] = level
e_dict['title'] = "%s [%s]" % (e, level)
e_dict['position'] = int(r_conn.get(election_key + '_POSITION') or 0)
e_dict['total_positions'] = len(glob.glob('%s%s/%s/*' % (DATA_DIR, e, level)))
e_dict['playback'] = int(r_conn.get(election_key + '_PLAYBACK') or 1)
e_dict['errormode'] = utils.to_bool(r_conn.get(election_key + '_ERRORMODE') or 'False')
e_dict['ratelimited'] = utils.to_bool(r_conn.get(election_key + '_RATELIMITED') or 'False')
context['elections'].append(e_dict)
return render_template('index.html', **context)
def archive_list(racedate, raceyear):
racedate_db = PostgresqlExtDatabase(
'elex_%s' % racedate,
user=os.environ.get('ELEX_ADMIN_USER', 'elex'),
host=os.environ.get('ELEX_ADMIN_HOST', '127.0.0.1'))
models.database_proxy.initialize(racedate_db)
context = utils.build_context(racedate, raceyear)
context['files'] = sorted([{
"name":
f.split('/')[-1],
"date":
datetime.datetime.fromtimestamp(
float(f.split('/')[-1].split('-')[-1].split('.json')[0]))
} for f in glob.glob('/tmp/%s/*.json' % racedate)],
key=lambda x: x,
reverse=True)[:750]
context['states'] = []
state_list = sorted(list(
set([race.statepostal for race in models.ElexRace.select()])),
key=lambda x: x)
for state in state_list:
race = models.ElexRace.select().where(
models.ElexRace.statepostal == state)[0]
state_dict = {}
state_dict['statepostal'] = state
state_dict['report'] = None
state_dict['report_description'] = None
context['states'].append(state_dict)
return render_template('archive_list.html', **context)
def page_index(request):
""" First page of the app. Shows latest 10 snippets. """
snippets = Snippet.objects.all()[0:5]
if snippets == None:
snippets = []
return render_to_response('pages/index.html',
{'snippets': snippets, 'home_page': True },
context_instance=build_context(request))
def search(request):
""" Fulltext search view. Uses whoosh """
data = {}
data['query'] = request.GET.get('q', '')
paginator = Paginator(Snippet.indexer.search(data['query']).prefetch(), 25)
data['results'] = paginator.page(int(request.GET.get('page', 1)))
return render_to_response('django_snippify/search.html', data,
context_instance=build_context(request))
def tag_user(request, tag = None, username = None):
""" View all snippets of a tag of a user """
try:
tag_object = Tag.objects.get(name=tag)
snippets = TaggedItem.objects.get_by_model(Snippet, tag_object)
except:
snippets = None
return render_to_response('tags/view.html',
{'tag': tag, 'snippets': snippets},
context_instance=build_context(request))
def index():
if request.method == 'GET':
context = utils.build_context(request)
page = utils.get_page(request)
all_entities = models.Entity.select()\
.where(models.Entity.active == True)\
.where(models.Entity.canonical_entity >> None)
context['entities'] = all_entities\
.order_by(models.Entity.created.desc())\
.paginate(page, settings.ITEMS_PER_PAGE)
context = utils.paginate(request, all_entities.count(), page, context)
return render_template('entity_list.html', **context)
if request.method == 'POST':
payload = utils.clean_payload(dict(request.form))
if not payload.get('name', None):
return Response('bad request', 400)
lookup = dict([(e.name, e.id) for e in models.Entity.select()])
entity_list = list(lookup.keys())
name = payload['name']
score = 0
if len(entity_list) > 0:
name, score = process.extractOne(payload['name'], entity_list)
response = {}
response['request'] = {}
response['request']['name'] = payload['name']
response['request']['create_if_below'] = payload.get(
'create_if_below', settings.MINIMUM_SCORE)
response['response'] = {}
response['response']['score'] = score
if payload.get('create_if_below', None):
if score < int(payload['create_if_below']):
response = create_entity(response)
return jsonify(response)
if score < settings.MINIMUM_SCORE:
response = create_entity(response)
return jsonify(response)
response['response']['created'] = False
response['response']['name'] = name
response['response']['uuid'] = lookup[name]
return jsonify(response)
def race_list(racedate):
context = utils.build_context(racedate)
context['presidential_races'] = []
context['national_races'] = []
context['other_races'] = []
context['states'] = []
try:
racedate_db = PostgresqlExtDatabase('elex_%s' % racedate,
user=os.environ.get('ELEX_ADMIN_USER', 'elex'),
host=os.environ.get('ELEX_ADMIN_HOST', '127.0.0.1')
)
models.database_proxy.initialize(racedate_db)
context['presidential_races'] = models.ElexRace\
.select()\
.where(
models.ElexRace.national == True,
models.ElexRace.officeid == "P"
)\
.order_by(+models.ElexRace.statepostal)
context['national_races'] = models.ElexRace\
.select()\
.where(
models.ElexRace.national == True,
models.ElexRace.officeid << ["G","S","H"]
)\
.order_by(+models.ElexRace.statepostal)
context['other_races'] = models.ElexRace\
.select()\
.where(
~(models.ElexRace.id << context['national_races']),
~(models.ElexRace.id << context['presidential_races']),
)\
.order_by(+models.ElexRace.statepostal)
state_list = sorted(list(Set([race.statepostal for race in models.ElexRace.select()])), key=lambda x: x)
for state in state_list:
race = models.ElexRace.select().where(models.ElexRace.statepostal == state)[0]
state_dict = {}
state_dict['statepostal'] = state
state_dict['report'] = race.report
state_dict['report_description'] = race.report_description
context['states'].append(state_dict)
return render_template('race_list.html', **context)
except peewee.OperationalError, e:
context['error'] = e
return render_template('error.html', **context)
def generate_json_file(release_excel_path):
""" Generate the machine-readable release file.
Args:
release_excel_path: str
Returns:
None
"""
release_json_file = "DataSchema_spec.json"
context = build_context(release_excel_path, "DataSchema Table")
with open(release_json_file, "w") as json_file:
DataSchema_keys = [
value.get_content()
for key, value in context.items()
if value.datatype != "group"
and key != "gt"
and value.datatype != "group-thirdparty"
]
output_DataSchema_keys = []
# post process the & reserved key
for DataSchema_key in DataSchema_keys:
if '&' in DataSchema_key['title']:
for slot_index in range(len(DataSchema_key['title'].split('|'))):
if '&' in DataSchema_key['title'].split('|')[slot_index]:
sub_group_keys = DataSchema_key['title'].split('|')[slot_index].split('&')
for sub_group_key_index in range(len(sub_group_keys)):
sub_group_key_name = '|'.join(DataSchema_key['title'].split('|')[:slot_index]) + '|' + sub_group_keys[sub_group_key_index].strip() + '|' + '|'.join(DataSchema_key['title'].split('|')[slot_index + 1:])
if sub_group_key_name.endswith('|'): # handle the slot_index is the end index number
sub_group_key_name = sub_group_key_name[:-1]
sub_DataSchema_key = copy.deepcopy(DataSchema_key)
sub_DataSchema_key['title'] = sub_group_key_name
output_DataSchema_keys.append(sub_DataSchema_key)
else:
output_DataSchema_keys.append(DataSchema_key)
json.dump(output_DataSchema_keys, json_file, indent=2)
def index():
"""
Will match directories named like the following:
2015-09-10
09-10-2015
"""
context = utils.build_context()
context['elections'] = []
elections = sorted([a.split('/')[-1] for a in glob.glob('%s/*' % DATA_DIR) if re.match('(\d{2,4}[-]\d{2,4}[-]\d{2,4})', a.split('/')[-1])], key=lambda x:x)
for e in elections:
e_dict = {}
election_key = 'AP_DEJAVU_%s' % e
e_dict['election_date'] = e
e_dict['position'] = int(os.environ.get(election_key + '_POSITION', '0'))
e_dict['total_positions'] = len(glob.glob('%s%s/*' % (DATA_DIR, e)))
e_dict['playback'] = int(os.environ.get(election_key + '_PLAYBACK', '1'))
e_dict['errormode'] = utils.to_bool(os.environ.get(election_key + '_ERRORMODE', 'False'))
e_dict['ratelimited'] = utils.to_bool(os.environ.get(election_key + '_RATELIMITED', 'False'))
context['elections'].append(e_dict)
return render_template('index.html', **context)
def generate_markdown_file(release_excel_path: str):
""" Generate the human-readable release file.
Args:
release_excel_path: str
Returns:
None
"""
context = build_context(release_excel_path,
"Ground Truth DataSchema Table")
release_markdown_file = "DataSchema_spec.md"
available_keys = [e for e in context.values()]
# this list used to generate the numbering format
level_index_list = [0] * (max([e.level_number
for e in available_keys]) + 1)
with open(f"{release_markdown_file}", "w", newline="\n") as md_file:
foreword = extract_foreword(
release_excel_path,
["Version", "Specification", "Reserved Keywords"])
# add the version control information to markdown file
construct_version_section(md_file, foreword)
# add specification statement information to markdown file
construct_specification_section(md_file, foreword)
# add reserved keyword information to markdown file
construct_reserved_keyword_section(md_file, foreword)
# add table of content to makedown file
construct_table_of_content_section(md_file, foreword, available_keys)
# add main body to markdown file
construct_main_body_section(md_file, available_keys, level_index_list)
def read(request, id=None):
""" View some snippet, a snippet can be private so check that too """
snippet = get_object_or_404(Snippet, pk=id)
if SnippetVersion.objects.filter(snippet = snippet).all():
versions = True
else:
versions = False
comments_paginator = Paginator(SnippetComment.objects.filter(snippet=snippet).all(), 2)
try:
comments = comments_paginator.page(int(request.GET.get('page', 1)))
except EmptyPage:
comments = None
snippet.highlight_body = snippet.highlight(snippet.body, get_lexer_by_name(snippet.lexer))
return render_to_response('django_snippify/read.html', {
'snippet': snippet,
'comments': comments,
'versions': versions,
'lines': range(1, snippet.body.count('\n')+2),
},
context_instance=build_context(request)
)
def race_detail(racedate,raceid):
if request.method == 'GET':
try:
racedate_db = PostgresqlExtDatabase('elex_%s' % racedate,
user=os.environ.get('ELEX_ADMIN_USER', 'elex'),
host=os.environ.get('ELEX_ADMIN_HOST', '127.0.0.1')
)
models.database_proxy.initialize(racedate_db)
context = utils.build_context(racedate)
context['race'] = models.ElexRace.get(models.ElexRace.raceid == raceid.split('-')[1], models.ElexRace.statepostal == raceid.split('-')[0])
context['candidates'] = sorted(models.ElexCandidate.select().where(models.ElexCandidate.nyt_races.contains(raceid)), key=lambda x:x.nyt_display_order)
context['ap_winner'] = None
ap_winner = models.ElexResult.select().where(
models.ElexResult.raceid == raceid.split('-')[1],
models.ElexResult.statepostal == raceid.split('-')[0],
models.ElexResult.winner == True
)
if len(ap_winner) > 0:
context['ap_winner'] = ap_winner[0]
context['states'] = []
state_list = sorted(list(Set([race.statepostal for race in models.ElexRace.select()])), key=lambda x: x)
for state in state_list:
race = models.ElexRace.select().where(models.ElexRace.statepostal == state)[0]
state_dict = {}
state_dict['statepostal'] = state
state_dict['report'] = race.report
state_dict['report_description'] = race.report_description
context['states'].append(state_dict)
return render_template('race_detail.html', **context)
except peewee.OperationalError, e:
context['error'] = e
return render_template('error.html', **context)
def race_list(racedate, raceyear):
context = utils.build_context(racedate, raceyear)
context['races'] = []
context['states'] = []
try:
racedate_db = PostgresqlExtDatabase(
'elex_%s' % racedate,
user=os.environ.get('ELEX_ADMIN_USER', 'elex'),
host=os.environ.get('ELEX_ADMIN_HOST', '127.0.0.1'))
models.database_proxy.initialize(racedate_db)
context['races'] = [
r for r in models.ElexResult.raw(
"""select officename, seatname, race_unique_id, raceid, statepostal, accept_ap_calls from elex_results group by officename, seatname, race_unique_id, raceid, statepostal, accept_ap_calls"""
)
]
state_list = sorted(list(
set([race.statepostal for race in models.ElexRace.select()])),
key=lambda x: x)
for state in state_list:
race = models.OverrideRace.select().where(
models.OverrideRace.statepostal == state)[0]
state_dict = {}
state_dict['statepostal'] = state
state_dict['report'] = race.report
context['states'].append(state_dict)
return render_template('race_list.html', **context)
except peewee.OperationalError as e:
context['error'] = e
return render_template('error.html', **context)
except peewee.ProgrammingError as e:
context['error'] = e
return render_template('error.html', **context)
def index(year):
"""
Will match directories named like the following:
2015-09-10
09-10-2015
"""
context = utils.build_context()
context['elections'] = []
context['year'] = year
elections = sorted([
a.split('/')[-1] for a in glob.glob('%s/*' % DATA_DIR)
if re.match('(\d{2,4}[-]\d{2,4}[-]\d{2,4})',
a.split('/')[-1])
],
key=lambda x: x)
for e in elections:
for level in ['national']:
national = True
e_dict = {}
election_key = 'AP_DEJAVU_%s' % e
e_dict['election_date'] = e
e_dict['national'] = national
e_dict['level'] = level
e_dict['title'] = "%s [%s]" % (e, level)
e_dict['position'] = int(
r_conn.get(election_key + '_POSITION') or 0)
e_dict['total_positions'] = len(
glob.glob('%s%s/%s/*' % (DATA_DIR, e, level)))
e_dict['playback'] = int(
r_conn.get(election_key + '_PLAYBACK') or 1)
e_dict['errormode'] = utils.to_bool(
r_conn.get(election_key + '_ERRORMODE') or 'False')
e_dict['ratelimited'] = utils.to_bool(
r_conn.get(election_key + '_RATELIMITED') or 'False')
context['elections'].append(e_dict)
return render_template('index.html', **context)
def _process(request, id=None):
""" Create/Update snippet """
if id is not None:#Update
snippet = get_object_or_404(Snippet, pk=id)
form = SnippetForm(instance=snippet)
if request.user.is_staff or request.user.id != snippet.author_id:
request.session['flash'] = ['Access denied', 'error'];
return HttpResponseRedirect('/accounts/profile/')
if 'delete' in request.POST:
snippet.delete()
request.session['flash'] = ['#%s deleted successfuly' % id, 'sucess']
return HttpResponseRedirect('/accounts/profile/')
else:#Create
snippet = None
form = SnippetForm()
if request.method == 'POST':
form = SnippetForm(request.POST)#Bounding form to the POST data
if not form.is_valid(): # redirect to form with errors
return render_to_response('django_snippify/process.html', {'form': form }, context_instance=build_context(request))
formData = form.save(commit = False)
formData.pk = id
if 'preview' in request.POST:
data = {}
data['title'] = formData.title;
data['preview_body'] = highlight(formData.body, get_lexer_by_name(formData.lexer), HtmlFormatter(cssclass = 'source'))
data['lines'] = range(1, formData.body.count('\n') + 2)
data['form'] = form
data['snippet'] = formData
return render_to_response('djnago_snippify/process.html', data, context_instance=build_context(request))
else:#save
formData.author = request.user
if not formData.lexer:
try:
lexer = guess_lexer(formData.body)
for lex in LEXERS.itervalues():
if lexer.name == lex[1]:
formData.lexer = lex[2][0].lower()
except ClassNotFound:
formData.lexer = 'text'
formData.save()
if snippet is not None and snippet.body != formData.body:
try:
last_version = SnippetVersion.objects.order_by('-version').filter(snippet = snippet).all()[0]
new_version = SnippetVersion(snippet = snippet, version = last_version.version + 1, body = snippet.body)
new_version.save()
except:
create_version = SnippetVersion(snippet = snippet, version = 1, body = snippet.body)
create_version.save()
request.session['flash'] = ['#%s %s successfuly' % (formData.pk, 'update' if id is not None else 'created'), 'sucess'];
return HttpResponseRedirect('/accounts/profile/')
else:
return render_to_response('django_snippify/process.html', {'form': form, 'snippet': snippet }, context_instance=build_context(request))
def race_detail(racedate, raceid, raceyear):
if request.method == 'GET':
try:
racedate_db = PostgresqlExtDatabase(
'elex_%s' % racedate,
user=os.environ.get('ELEX_ADMIN_USER', 'elex'),
host=os.environ.get('ELEX_ADMIN_HOST', '127.0.0.1'))
models.database_proxy.initialize(racedate_db)
context = utils.build_context(racedate, raceyear)
context['race'] = [
r for r in models.ElexResult.raw(
"""select officename, seatname, race_unique_id, raceid, statepostal, accept_ap_calls from elex_results where race_unique_id = '%s' group by officename, seatname, race_unique_id, raceid, statepostal, accept_ap_calls"""
% raceid)
][0]
context['candidates'] = models.ElexResult.raw(
"""select nyt_runoff, party, nyt_winner, candidate_unique_id, first, last from elex_results where race_unique_id = '%s' group by nyt_runoff, party, nyt_winner, candidate_unique_id, first, last order by last, first DESC;"""
% raceid)
context['ap_winner'] = None
ap_winner = [
m for m in models.ElexResult.raw(
"""select candidate_unique_id, first, last, winner, nyt_winner, nyt_called from elex_results where race_unique_id = '%s' and winner = 'true' group by candidate_unique_id, first, last, winner, nyt_winner, nyt_called order by last, first DESC;"""
% raceid)
]
if len(ap_winner) > 0:
context['ap_winner'] = ap_winner[0]
context['states'] = []
state_list = sorted(list(
set([race.statepostal for race in models.ElexRace.select()])),
key=lambda x: x)
for state in state_list:
race = models.ElexRace.select().where(
models.ElexRace.statepostal == state)[0]
state_dict = {}
state_dict['statepostal'] = state
state_dict['report'] = None
state_dict['report_description'] = None
context['states'].append(state_dict)
return render_template('race_detail.html', **context)
except peewee.OperationalError as e:
context['error'] = e
return render_template('error.html', **context)
if request.method == 'POST':
racedate_db = PostgresqlExtDatabase(
'elex_%s' % racedate,
user=os.environ.get('ELEX_ADMIN_USER', 'elex'),
host=os.environ.get('ELEX_ADMIN_HOST', '127.0.0.1'))
models.database_proxy.initialize(racedate_db)
payload = utils.clean_payload(dict(request.form))
try:
r = models.OverrideRace.get(
models.OverrideRace.race_unique_id == raceid)
except models.OverrideRace.DoesNotExist:
r = models.OverrideRace.create(race_unique_id=raceid,
raceid=raceid.split('-')[1],
statepostal=raceid.split('-')[0])
# nyt_winner is a single ID, there can only be one winner.
utils.set_winner(payload['nyt_winner'], raceid)
print(payload)
# nyt_runoff is a list of ids, there can be 2 or more advancing.
runoff_cands = []
if payload.get('nyt_runoff', None):
runoff_cands = [
x.strip() for x in payload['nyt_runoff'].split(',')
]
utils.set_runoff(runoff_cands, raceid)
utils.update_model(r, payload)
utils.update_views(models.database_proxy)
return json.dumps({"message": "success"})
def index(request):
""" My snippets """
snippets = Snippet.objects.filter(author=request.user)
return render_to_response('django_snippify/index.html', {'snippets': snippets},
context_instance=build_context(request))
def tag_index(request):
""" View all tags into a tag cloud """
tags = Tag.objects.all()
return render_to_response('tags/index.html', {},
context_instance=build_context(request))
def index(raceyear):
context = utils.build_context()
context['racedate'] = None
return render_template('index.html', **context)
def racedate(raceyear, racedate):
context = utils.build_context()
context['racedate'] = racedate
return render_template('index.html', **context)
