def api_streets():
start_time = time.time()
jsonp_callback = request.args.get("callback", None)
lon = request.args.get("lon", "")
lat = request.args.get("lat", "")
radius = request.args.get("radius", "1000")
if lat == "" or lon == "":
abort(400)
lon = float(lon)
lat = float(lat)
radius = int(radius)
radius = min(radius, 500)
result = db.get_locations(lon, lat, radius)
ret = {
"status": 0,
"duration": round((time.time() - start_time) * 1000),
"request": {"lon": lon, "lat": lat, "radius": radius},
"response": result,
}
json_output = json.dumps(ret, cls=util.MyEncoder, sort_keys=True)
if jsonp_callback is not None:
json_output = jsonp_callback + "(" + json_output + ")"
response = make_response(json_output, 200)
response.mimetype = "application/json"
response.headers["Expires"] = util.expires_date(hours=24)
response.headers["Cache-Control"] = util.cache_max_age(hours=24)
return response
def api_streets():
start_time = time.time()
jsonp_callback = request.args.get('callback', None)
lon = request.args.get('lon', '')
lat = request.args.get('lat', '')
radius = request.args.get('radius', '1000')
if lat == '' or lon == '':
abort(400)
lon = float(lon)
lat = float(lat)
radius = int(radius)
radius = min(radius, 500)
result = db.get_locations(lon, lat, radius)
ret = {
'status': 0,
'duration': round((time.time() - start_time) * 1000),
'request': {
'lon': lon,
'lat': lat,
'radius': radius
},
'response': result
}
json_output = json.dumps(ret, cls=util.MyEncoder, sort_keys=True)
if jsonp_callback is not None:
json_output = jsonp_callback + '(' + json_output + ')'
response = make_response(json_output, 200)
response.mimetype = 'application/json'
response.headers['Expires'] = util.expires_date(hours=24)
response.headers['Cache-Control'] = util.cache_max_age(hours=24)
return response
def api_streets():
start_time = time.time()
jsonp_callback = request.args.get('callback', None)
lon = request.args.get('lon', '')
lat = request.args.get('lat', '')
region = request.args.get('region', app.config['region_default'])
radius = request.args.get('radius', '1000')
if lat == '' or lon == '':
abort(400)
lon = float(lon)
lat = float(lat)
radius = int(radius)
radius = min(radius, 500)
streets = db.get_locations(lon, lat, radius)
result = {}
# TODO: use msearch for getting paper num
for street in streets:
nodes = []
for point in street['nodes']:
nodes.append(point['location']['coordinates'])
if street['name'] in result:
result[street['name']]['nodes'].append(nodes)
else:
search_result = db.query_paper_num(region, street['name'])
result[street['name']] = {
'name': street['name'],
'nodes': [nodes],
'paper_count': search_result['num']
}
if 'name' in search_result:
result[street['name']]['paper_name'] = search_result['name']
if 'name' in search_result:
result[street['name']]['paper_publishedDate'] = search_result[
'publishedDate']
ret = {
'status': 0,
'duration': round((time.time() - start_time) * 1000),
'request': {
'lon': lon,
'lat': lat,
'radius': radius,
'region': region
},
'response': result
}
try:
json_output = json.dumps(ret, cls=util.MyEncoder, sort_keys=True)
except AttributeError:
return null
if jsonp_callback is not None:
json_output = jsonp_callback + '(' + json_output + ')'
response = make_response(json_output, 200)
response.mimetype = 'application/json'
response.headers['Expires'] = util.expires_date(hours=24)
response.headers['Cache-Control'] = util.cache_max_age(hours=24)
return response
def api_streets():
start_time = time.time()
jsonp_callback = request.args.get('callback', None)
lon = request.args.get('lon', '')
lat = request.args.get('lat', '')
region = request.args.get('region', app.config['region_default'])
radius = request.args.get('radius', '1000')
if lat == '' or lon == '':
abort(400)
lon = float(lon)
lat = float(lat)
radius = int(radius)
radius = min(radius, 500)
streets = db.get_locations(lon, lat, radius)
result = {}
# TODO: use msearch for getting paper num
for street in streets:
nodes = []
for point in street['nodes']:
nodes.append(point['location']['coordinates'])
if street['name'] in result:
result[street['name']]['nodes'].append(nodes)
else:
search_result = db.query_paper_num(region, street['name'])
result[street['name']] = {
'name': street['name'],
'nodes': [ nodes ],
'paper_count': search_result['num']
}
if 'name' in search_result:
result[street['name']]['paper_name'] = search_result['name']
if 'name' in search_result:
result[street['name']]['paper_publishedDate'] = search_result['publishedDate']
ret = {
'status': 0,
'duration': round((time.time() - start_time) * 1000),
'request': {
'lon': lon,
'lat': lat,
'radius': radius,
'region': region
},
'response': result
}
try:
json_output = json.dumps(ret, cls=util.MyEncoder, sort_keys=True)
except AttributeError:
return null
if jsonp_callback is not None:
json_output = jsonp_callback + '(' + json_output + ')'
response = make_response(json_output, 200)
response.mimetype = 'application/json'
response.headers['Expires'] = util.expires_date(hours=24)
response.headers['Cache-Control'] = util.cache_max_age(hours=24)
return response
def render_new_procedure_form():
procedures = db.get_procedures()
locations = db.get_locations()
timeframes = db.get_timeframes()
doctors = db.get_doctors()
demo_mode2 = config.demo_mode
print(str.format("Demo mode is: {0}", demo_mode2))
return flask.render_template(
"new_procedure.html",
procedures=procedures,
locations=locations,
timeframes=timeframes,
doctors=doctors,
demo_mode=demo_mode2,
)
def api_streets():
start_time = time.time()
jsonp_callback = request.args.get("callback", None)
lon = request.args.get("lon", "")
lat = request.args.get("lat", "")
region = request.args.get("region", app.config["region_default"])
radius = request.args.get("radius", "1000")
if lat == "" or lon == "":
abort(400)
lon = float(lon)
lat = float(lat)
radius = int(radius)
radius = min(radius, 500)
streets = db.get_locations(lon, lat, radius)
result = {}
# TODO: use msearch for getting paper num
for street in streets:
nodes = []
for point in street["nodes"]:
nodes.append(point["location"]["coordinates"])
if street["name"] in result:
result[street["name"]]["nodes"].append(nodes)
else:
search_result = db.query_paper_num(region, street["name"])
result[street["name"]] = {"name": street["name"], "nodes": [nodes], "paper_count": search_result["num"]}
if "name" in search_result:
result[street["name"]]["paper_name"] = search_result["name"]
if "name" in search_result:
result[street["name"]]["paper_publishedDate"] = search_result["publishedDate"]
ret = {
"status": 0,
"duration": round((time.time() - start_time) * 1000),
"request": {"lon": lon, "lat": lat, "radius": radius, "region": region},
"response": result,
}
try:
json_output = json.dumps(ret, cls=util.MyEncoder, sort_keys=True)
except AttributeError:
return null
if jsonp_callback is not None:
json_output = jsonp_callback + "(" + json_output + ")"
response = make_response(json_output, 200)
response.mimetype = "application/json"
response.headers["Expires"] = util.expires_date(hours=24)
response.headers["Cache-Control"] = util.cache_max_age(hours=24)
return response
def on_fetch_places(self, response):
""" Callback invoked after places fetched """
places = json.loads(response.body)
location_ids = [location["id"] for location in places["data"]]
data = db.get_locations(location_ids)
# Add additional metadata we've stored locally
for location in places["data"]:
if location["id"] in data.keys():
# micro JS template doesn't like conditionals
location["data"] = data[location["id"]]
# if loc_data["owner"] is not None:
# location["owner_name"] = loc_data["owner"]["name"]
# json.dumps doesn't serialize datetime.datetime instances
# see http://twitter.com/#!/ikai/status/78364766720098304
if location["data"]["last_extort_time"] is not None:
# We can fix this nonsense later
location["data"]["last_extort_time"] = str(location["data"]["last_extort_time"])
self.write(json.dumps(places))
self.finish()
def api_streets():
start_time = time.time()
jsonp_callback = request.args.get('callback', None)
lon = request.args.get('lon', '')
lat = request.args.get('lat', '')
radius = request.args.get('radius', '1000')
if lat == '' or lon == '':
abort(400)
lon = float(lon)
lat = float(lat)
radius = int(radius)
radius = min(radius, 500)
result = db.get_locations(lon, lat, radius)
ret = {
'status': 0,
'duration': round((time.time() - start_time) * 1000),
'request': {
'lon': lon,
'lat': lat,
'radius': radius
},
'response': result
}
try:
json_output = json.dumps(ret, cls=util.MyEncoder, sort_keys=True)
except AttributeError:
print >> sys.stderr, ret
return null
if jsonp_callback is not None:
json_output = jsonp_callback + '(' + json_output + ')'
response = make_response(json_output, 200)
response.mimetype = 'application/json'
response.headers['Expires'] = util.expires_date(hours=24)
response.headers['Cache-Control'] = util.cache_max_age(hours=24)
return response
def on_fetch_places(self, response):
""" Callback invoked after places fetched """
places = json.loads(response.body)
location_ids = [location["id"] for location in places["data"]]
data = db.get_locations(location_ids)
# Add additional metadata we've stored locally
for location in places["data"]:
if location["id"] in data.keys():
# micro JS template doesn't like conditionals
location["data"] = data[location["id"]]
# if loc_data["owner"] is not None:
# location["owner_name"] = loc_data["owner"]["name"]
# json.dumps doesn't serialize datetime.datetime instances
# see http://twitter.com/#!/ikai/status/78364766720098304
if location["data"]["last_extort_time"] is not None:
# We can fix this nonsense later
location["data"]["last_extort_time"] = str(
location["data"]["last_extort_time"])
self.write(json.dumps(places))
self.finish()
from nltk.sentiment.vader import SentimentIntensityAnalyzer
from dotenv import load_dotenv, find_dotenv
from db import get_locations, get_provinces, get_one, insert_article, insert_log, get_uuid, get_rand_sources, get_sources, insert_item
from utils import PH_TIMEZONE, search_locations, search_authors, get_publish_date, sleep, get_popularity, get_proxy, clean_url, categorize
# from aylien import categorize, get_rate_limits
from nlp import get_entities, summarize2
from nlp.keywords import parse_topics
from fake_useragent import UserAgent
import rethinkdb as r
import re
from random import shuffle
import os
load_dotenv(find_dotenv(), override=True)
locations = get_locations()
provinces = get_provinces()
# news_sources = get_news_sources('timestamp')
# shuffle(news_sources)
# if not news_sources:
# if PY_ENV == 'development':
# print('EMPTY NEWS SOURCES')
PY_ENV = os.environ.get('PY_ENV')
count = 0
slp_time = 0
last_proxy = ''
while True:
news_sources = get_rand_sources()