def view_master(request, urlid):
data = []
logging.debug('View (Master)Url %s' % (urlid))
try:
urlid = int(urlid)
except:
pass
url = Url.get_by_id(urlid)
if url:
channelurls = ChannelUrl.query(ChannelUrl.url == url.key)
for channelurl in channelurls:
channel = channelurl.channel.get()
if channel.private == False:
extras = Extra.query(Extra.channelurl == channelurl.key)
rates = Rate.query(Rate.channelurl == channelurl.key)
rating = channelurl.rating()
# data.append({'channel':channel,'post':post,'url':url,'extras': extras})
data.append({
'channel': channel,
'channelurl': channelurl,
'post': post,
'url': url,
'extras': extras,
'rates': rates,
'rating': rating
})
template_values = {
'data': data,
'user': users.get_current_user(),
}
return render('masterurl.html', template_values)
def view(request, urlid):
data = []
logging.debug('View ChannelUrl %s' % (urlid))
try:
urlid = int(urlid)
except:
pass
channelurl = ChannelUrl.get_by_id(urlid)
if channelurl:
channel = channelurl.channel.get()
url = channelurl.url.get()
if channel.private == False:
extras = Extra.query(Extra.channelurl == channelurl.key)
rates = Rate.query(Rate.channelurl == channelurl.key)
rating = channelurl.rating()
# data.append({'channel':channel,'post':post,'url':url,'extras': extras})
data = {
'channel': channel,
'post': post,
'url': url,
'extras': extras,
'rates': rates,
'rating': rating
}
template_values = {
'data': data,
}
return render('url.html', template_values)
def create_lifo_tracker_id():
"""
This is the internal endpoint for creating lifo tracker id, and requires auth from Influencers.
The caller is currently from campaign side, where queries are sent from api_nodejs during influencer
signing up for brand initiated campaigns.
The request payload will include campaign_data as defined in campaign.js under api_nodejs.
"""
contract_data = flask.request.json
logging.debug(f'{request.path} receiving contract data {contract_data}')
if not contract_data.get('brand_campaign_id'):
response = flask.jsonify(
{'Status': 'A valid brand_campaign_id is required'})
response.status_code = 422
return response
if not contract_data.get('account_id'):
response = flask.jsonify({'Status': 'A valid account_id is required'})
response.status_code = 422
return response
tracking_url = gen_lifo_tracker_id(contract_data.get('account_id'),
contract_data)
if type(tracking_url) is flask.Response:
return tracking_url
response = flask.jsonify({'tracking_url': tracking_url})
response.status_code = 200
return response
def create_lifo_tracker_id():
"""
This is the internal endpoint for creating lifo tracker id, and requires auth from Influencers.
The caller is currently from campaign side, where queries are sent from api_nodejs during influencer
signing up for brand initiated campaigns.
The request payload will include campaign_data as defined in campaign.js under api_nodejs.
"""
# influencer uid, NOT shop
uid = flask.session['uid']
# a flag to differentiate Shopify accounts, which use store domain as uid
from_shopify = flask.session['from_shopify']
campaign_data = flask.request.json
logging.debug(f'{request.path} receiving campaign data {campaign_data}')
if not campaign_data.get('brand_campaign_id'):
response = flask.jsonify(
{'Status': 'A valid brand_campaign_id is required'})
response.status_code = 422
return response
tracking_url = gen_lifo_tracker_id(uid, campaign_data)
if type(tracking_url) is flask.Response:
return tracking_url
response = flask.jsonify({'tracking_url': tracking_url})
response.status_code = 200
return response
def post(self):
id = self.request.get('id')
if id:
url = Url.get_by_id(int(id))
if url:
result = None
try:
result = urlfetch.fetch(url.url, allow_truncated=True)
except urlfetch.DownloadError:
url.status = 'DE'
logging.info('DownloadError, url: %s' % (url.url))
except urlfetch.ResponseTooLargeError:
url.status = 'RTL'
logging.info('ResponseTooLargeError, url: %s' % (url.url))
except urlfetch.InvalidURLError:
url.status = 'IUE'
logging.info('InvalidURLError, url: %s' % (url.url))
except:
url.status = 'UE'
logging.error('"Unexpected error: %s, url: %s' %
(sys.exc_info()[0], url.url))
if result:
if result.content_was_truncated:
logging.debug('truncated')
if result.status_code:
url.status = str(result.status_code)
if result and result.status_code and result.status_code == 200:
url.valid = 2
else:
if url.valid > -5:
url.valid = url.valid - 1
else:
logging.info('Broken url: %s' % (url.url))
url.last_check = datetime.datetime.now()
url.put()
def rate(request, post_channel, post_user, type, post_url):
logging.debug('rate: C=%s U=%s T=%s P=%s' %
(post_channel, post_user, type, post_url))
types = ['up', 'down', 'wtf', 'nsfw']
if type in types:
name = ''.join(
re.findall(
'[a-zA-Z0-9_-]', post_channel + '_' + post_user + '_' + type +
'_' + post_url))[:500]
try:
taskqueue.add(name=name,
queue_name='default',
url='/tasks/rate',
params={
'channel': post_channel,
'post_user': post_user,
'type': type,
'post_url': post_url
})
except taskqueue.TombstonedTaskError:
logging.warning('Duplicate task name %s' % name)
else:
logging.warning('Wrong type (%s)! %s %s %s' %
(type, post_channel, post_user, post_url))
return HttpResponseRedirect('/url/')
def track():
"""
public endpoint (no auth)
Note: Shopify client side is using the following code snippet to send tracking events:
# req.send(JSON.stringify({
# lifo_tracker_id: lifo_tracker_id,
# shop: getShop(),
# location: document.location,
# navigator: navigator.userAgent,
# referrer: document.referrer,
# discount_code,
# })),
"""
data = flask.request.json
logging.info(f'Receiving /track request {data}')
if not data.get('shop'):
logging.warning(f'Invalid shop data received {data}')
elif not data.get('lifo_tracker_id'):
logging.debug(f'Skip none lifo event {data}')
else:
try:
res = sql_handler.save_track_visit(data)
if res.status_code == 200:
logging.info('Data saved to cloud SQL')
except Exception as e:
logging.error(f'Saving events error: {e}')
response = flask.jsonify({'status': 'OK'})
response.status_code = 200
return response
def post(self):
type = self.request.get('type', '')
if type == 'stats':
pass
elif type == 'cleanup':
last_year = datetime.datetime.now() - datetime.timedelta(days=365)
last_quarter = datetime.datetime.now() - datetime.timedelta(
days=92)
last_month = datetime.datetime.now() - datetime.timedelta(days=31)
# Old news
old_news = News.query(News.date < last_quarter).order(
News.date).fetch(500, keys_only=True)
# logging.info('Cleaning up old news %s' % News.query().order(News.date).count(100,keys_only=True))
ndb.delete_multi(old_news)
elif type == 'tag_cloud':
channel_urls = []
tags = {}
extras = Extra.query(Extra.tag != None)
for extra in extras:
if extra.channelurl not in channel_urls:
channel_urls.append(extra.channelurl)
tag = extra.tag
if tag in tags:
tags[tag] += 1
else:
tags[tag] = 1
tags_sorted = sorted(tags.iteritems(),
key=operator.itemgetter(1),
reverse=True)
memcache.set("tag_cloud", tags_sorted)
logging.debug('Tags: %s' % (tags_sorted))
elif type == 'fix':
test_channel = '#kanava'
channel = Channel.query(Channel.name == test_channel).get()
channelurls = ChannelUrl.query(
ChannelUrl.channel == channel.key).fetch(50)
for channelurl in channelurls:
url = channelurl.url.get()
logging.debug('Channel: %s, channelurl: %s (id %s)' %
(test_channel, url, channelurl))
posts = Post.query(Post.channelurl == channelurl.key)
for post in posts:
logging.debug(' * posted by %s' % (post.user))
post.key.delete()
rates = Rate.query(Rate.channelurl == channelurl.key)
for rate in rates:
logging.debug(' * rate %s' % (rate))
rate.key.delete()
extras = Extra.query(Extra.channelurl == channelurl.key)
for extra in extras:
logging.debug(' * extra %s, by %s' % (extra, extra.user))
extra.key.delete()
channelurl.key.delete()
def info(request):
today = datetime.date.today()
now = datetime.datetime.now()
l = list(string.ascii_uppercase)
l.append('Z')
DEFAULT_COUNTER_NAME = chr(now.isocalendar()[0] - 2010 +
65) + l[(now.isocalendar()[1] - 1) / 2]
id = None
url = None
if request.method == "POST":
try:
data = simplejson.loads(request.raw_post_data)
id = str(data.get('id', '')).upper()
url = data.get('url', '')
logging.debug('id: %s/url: %s' % (id, url))
except Exception as e:
logging.warning('Error %s' % (e))
try:
id = int(id)
if id < 1000:
id = DEFAULT_COUNTER_NAME + str(id)
# logging.debug('%s' % id)
except:
pattern1 = r'^[A-Z]{1}$'
pattern2 = r'^[A-Z]{2}$'
if re.match(pattern1, id):
id = DEFAULT_COUNTER_NAME[0] + str(id)
# logging.debug('%s' % id)
id = id + str(counter.get_count(id))
# logging.debug('%s' % id)
elif re.match(pattern2, id):
id = id + str(counter.get_count(id))
channelurl = ChannelUrl.get_by_id(id)
if channelurl:
url = channelurl.url.get().url
url_title = channelurl.url.get().title
rate = channelurl.rating()
extra = channelurl.extras(plain='True')
posts = channelurl.posts()
channel = channelurl.channel.get().name
retval = simplejson.dumps({
'id': channelurl.key.id(),
'url': url,
'title': url_title,
'rate': rate,
'extra': extra,
'posts': posts
})
else:
retval = simplejson.dumps({'id': 0})
return HttpResponse(retval, mimetype="application/json")
def download_file(storage_bucket_name, file_path):
# extract file name
_file_path = file_path.split('/')
file_name = _file_path[-1]
logging.debug("Downloading file {} from blob {}".format(
file_name, storage_bucket_name))
storage_client = storage.Client()
bucket = storage_client.get_bucket(storage_bucket_name)
blob = bucket.blob(file_name)
blob.download_to_filename(upload_folder + '/' + file_name)
return (upload_folder + '/' + file_name), file_name, storage_bucket_name
def schedule():
"""
Checks if it's time to run a schedule.
Returns:
"""
logging.debug("From Cron start /tasks/schedule")
with client.context():
keys = PolicyModel.query().fetch(keys_only=True)
for key in keys:
logging.debug("Creating deferred task for %s", key.id())
policy_tasks.policy_checker(key.id())
return "ok", 200
def track_visits():
shop = flask.session['uid']
if not shop:
response = flask.jsonify({'Status': 'Failed'})
response.status_code = 422
return response
try:
# schema: COUNT(*) as visits, shop
sql_results = sql_handler.counts_visits_per_shop(shop)
visit_results = count_visits_daily(sql_results)
except Exception as e:
logging.error(f'Getting track_visit error: {e}')
logging.debug(f'Obtained tracking report {visit_results}')
response = flask.jsonify(visit_results)
response.status_code = 200
return response
def get_schedule():
"""
Get a schedule.
Returns: schedule json
"""
name = request.args.get("schedule")
schedule = {}
with client.context():
res = SchedulesModel.query(SchedulesModel.Name == name).get()
if not res:
return "not found", 404
schedule.update({"name": res.Name})
schedule.update(res.Schedule)
schedule.update({"timezone": res.Timezone})
logging.debug(json.dumps(res.Schedule))
return json.dumps(schedule)
def change_state():
"""
Initiate change state.
Returns:
"""
payload = json.loads(request.get_data(as_text=False) or "(empty payload)")
logging.debug(
"Starting change_state action %s project %s tagkey %s tagvalue %s",
payload["action"],
payload["project"],
payload["tagkey"],
payload["tagvalue"],
)
schedule_tasks.change_state(payload["tagkey"], payload["tagvalue"],
payload["action"], payload["project"])
return "ok", 200
def get_policy():
"""
Get policy.
Returns: policy json
"""
policy = {}
name = request.args.get("policy")
with client.context():
res = PolicyModel.query(PolicyModel.Name == name).get()
logging.debug(res)
if not res:
return "not found", 404
policy.update({"name": res.Name})
policy.update({"schedulename": res.Schedule})
policy.update({"tags": res.Tags})
policy.update({"projects": res.Projects})
return json.dumps(policy)
def hook():
if request.method == "OPTIONS": # CORS preflight
return _build_cors_prelight_response()
if request.path.startswith('/brand') or request.path.startswith(
'/am') or request.path.startswith('/influencer'):
id_token = flask.request.headers.get(
'Authorization') or flask.request.args.get('id_token')
if not id_token:
logging.error('Valid id_token required')
response = flask.jsonify('Valid id_token required')
response.status_code = 401
return response
decoded_token = token_verification(id_token)
uid = decoded_token['uid']
if not uid:
logging.error('id_token verification failed')
response = flask.jsonify('id_token verification failed')
response.status_code = 401
return response
logging.info(
f'request path is: {request.path} with decoded token {decoded_token}'
)
if decoded_token.get(ACCOUNT_MANAGER_FLAG):
logging.info('AM account has admin access')
elif not decoded_token.get(
ACCOUNT_MANAGER_FLAG) and request.path.startswith('/am'):
response = flask.jsonify({"status": "not authorized"})
response.status_code = 403
return response
elif (request.path.startswith('/brand') and not decoded_token.get(STORE_ACCOUNT))\
or (request.path.startswith('/influencer') and decoded_token.get(STORE_ACCOUNT)):
response = flask.jsonify({"status": "not authorized"})
response.status_code = 403
return response
flask.session['uid'] = uid
flask.session[FROM_SHOPIFY] = decoded_token.get(FROM_SHOPIFY)
flask.session[STORE_ACCOUNT] = decoded_token.get(STORE_ACCOUNT)
flask.session[ACCOUNT_MANAGER_FLAG] = decoded_token.get(
ACCOUNT_MANAGER_FLAG)
flask.session['name'] = decoded_token.get('name')
flask.session['email'] = decoded_token.get('email')
else:
logging.debug(f'By passing auth for request {request.path}')
def post(self):
id = self.request.get('id')
if id:
url = Url.get_by_id(int(id))
if url:
# TODO: fetch title
# try:
req = urllib2.Request(url.url)
# logging.debug('req %s' % (req))
# req.add_header('User-agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11')
req.add_header(
'User-agent',
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
)
req.add_header(
'Accept',
'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8'
)
req.add_header('Accept-Charset',
'ISO-8859-1,utf-8;q=0.7,*;q=0.3')
req.add_header('Accept-Encoding', 'none')
req.add_header('Accept-Language', 'en-US,en;q=0.8')
req.add_header('Connection', 'keep-alive')
res = urllib2.urlopen(req)
# logging.debug('res %s' % (res))
doc = res.read()
# logging.debug('doc %s' % (doc))
encoding = res.headers.getparam('charset')
logging.debug('encoding %s' % (encoding))
try:
tree = etree.fromstring(
doc, etree.HTMLParser(encoding=encoding))
except LookupError:
tree = etree.fromstring(doc,
etree.HTMLParser(encoding='utf-8'))
title = tree.find(".//title").text
logging.debug('title %s' % (title))
url.title = smart_text(re.sub(r'\s+', ' ', title).strip())
# except:
# logging.debug('TitleTask: title not fetched %s' % (post_url))
# url.title = post_url
url.put()
def extra(request, post_channel, post_user, extra, post_url):
logging.debug('rate: C=%s U=%s E=%s P=%s' %
(post_channel, post_user, extra, post_url))
name = ''.join(
re.findall(
'[a-zA-Z0-9_-]', post_channel + '_' + post_user + '_' + extra +
'_' + post_url))[:500]
try:
taskqueue.add(name=name,
queue_name='default',
url='/tasks/extra',
params={
'channel': post_channel,
'post_user': post_user,
'extra': extra,
'post_url': post_url
})
except taskqueue.TombstonedTaskError:
logging.warning('Duplicate task name %s' % name)
return HttpResponseRedirect('/url/')
def post(request, post_url, post_channel=None, post_user=None):
if not post_user:
post_user = '******'
if not post_channel:
post_channel = '!' + post_user
logging.debug('post: C=%s U=%s P=%s' % (post_channel, post_user, post_url))
name = ''.join(
re.findall('[a-zA-Z0-9_-]',
post_channel + '_' + post_user + '_' + post_url))[:500]
try:
taskqueue.add(name=name,
queue_name='default',
url='/tasks/post',
params={
'post_channel': post_channel,
'post_user': post_user,
'post_url': post_url
})
except taskqueue.TombstonedTaskError:
logging.warning('Duplicate task name %s' % name)
return HttpResponseRedirect('/url/')
def pubsub_push():
if (request.args.get('token', '') != token):
return 'Invalid request', 400
envelope = json.loads(request.data.decode('utf-8'))
payload = base64.b64decode(envelope['message']['data'])
MESSAGES.append(payload)
# download file from blob
logging.debug("Downloading file {} from storage {}".format(
payload, storage_bucket_name))
_payload = payload.decode('utf-8')
_file_name = download_file(storage_bucket_name, _payload)
file_name = _file_name[0]
file_name_wo_folder = _file_name[1]
ai_score = analyze_picture(file_name)[0]
other_objects = analyze_picture(file_name)[1]
if ai_score > float(detection_score):
message = "Object: {} identified with probability {} on picture {}".format(
label_to_detect, round(ai_score, 2), file_name)
send_message_to_pubsub(message, file_name_wo_folder,
storage_bucket_name)
else:
if notify_about_all_checks == "yes":
message = "Object: {} not detected. Other objects are {}. Picture {}".format(
label_to_detect, other_objects, file_name)
send_message_to_pubsub(message, file_name_wo_folder,
storage_bucket_name)
# delete local copy of file
logging.debug("Deleting local copy of file {}".format(file_name))
os.remove(file_name)
# Returning any 2xx status indicates successful receipt of the message.
return 'OK', 200
def order_complete():
"""
Public endpoint (no auth)
Don't confuse this one with Shopify webhook.
Note: Shopify client side is using the following code snippet to send order_complete events:
# n.send(JSON.stringify({
# lifo_tracker_id: lifo_tracker_id,
# shop: getShop(),
# location: document.location,
# navigator: navigator.userAgent,
# referrer: document.referrer,
# discount_code,
# order_id,
# customer_id: data.customer_id,
# order_data,
# }));
"""
data = flask.request.json
logging.info(f'Receiving order_complete request {data}')
if not data.get('shop') or not data.get('order_id') or not data.get(
'customer_id'):
logging.warning(f'Invalid shop/customer data received {data}')
elif not data.get('lifo_tracker_id'):
logging.debug(f'Skip none lifo event {data}')
else:
try:
order_data = data.get('order_data')
# we use subtotal_price for calculation, before tax and shipping
subtotal_price = float(order_data.get('subtotal_price'))
logging.debug(f'Received order with revenue of {subtotal_price}')
res = sql_handler.save_order_complete(data, subtotal_price)
if res.status_code == 200:
logging.info('Data saved to cloud SQL')
except Exception as e:
logging.error(f'Saving events error: {e}')
response = flask.jsonify({'status': 'OK'})
response.status_code = 200
return response
def get(self):
type = self.request.get('type')
logging.debug('Maintenance task, %s' % (type))
if type == 'stats':
taskqueue.add(queue_name='default',
url='/tasks/maintenance',
params={'type': 'stats'})
elif type == 'tag_cloud':
taskqueue.add(queue_name='default',
url='/tasks/maintenance',
params={'type': 'tag_cloud'})
elif type == 'fix':
taskqueue.add(queue_name='default',
url='/tasks/maintenance',
params={'type': 'fix'})
elif type == 'cleanup':
taskqueue.add(queue_name='maintenance',
url='/tasks/maintenance',
params={'type': 'cleanup'})
elif type == 'url_test_clean':
channel = self.request.get('channel')
nick = self.request.get('nick')
logging.debug('url_test_clean, channel: %s, nick: %s' %
(channel, nick))
def add_policy():
"""
Add policy.
Returns:
"""
logging.debug(json.dumps(request.json))
name = request.json["name"]
tags = request.json["tags"]
projects = request.json["projects"]
schedule_name = request.json["schedulename"]
with client.context():
res = SchedulesModel.query(SchedulesModel.Name == schedule_name).get()
if not res:
return "not found", 404
policy_model = PolicyModel()
policy_model.Name = name
policy_model.Tags = tags
policy_model.Projects = projects
policy_model.Schedule = schedule_name
policy_model.key = ndb.Key("PolicyModel", name)
policy_model.put()
return "ok", 200
def post(request):
# today=datetime.date.today()
topic = None
nick = None
channel = None
date = None
retval = {}
if request.method == "POST":
logging.debug('raw_post_data %s' % (request.raw_post_data))
try:
data = simplejson.loads(request.raw_post_data)
logging.debug('data %s' % (data))
topic = data.get('topic')
channel = data.get('channel')
nick = data.get('nick')
date = datetime.datetime.strptime(data.get('date'), '%Y%m%d%H%M')
logging.debug('date: %s, nick: %s, channel, %s, topic: %s' %
(date, nick, channel, topic))
except Exception as e:
logging.warning('TOPIC/POST: Somewhere near %s' % (e))
retval = {'id': 0, 'topic': e}
else:
try:
t = Topic(channel=channel, nick=nick, topic=topic, date=date)
t.put()
except Exception as e:
logging.warning('TOPIC/POST/NEW: Somewhere near %s' % (e))
retval = {'id': 0, 'topic': e}
else:
retval = {'id': t.key.id(), 'topic': topic}
# TODO private channel?
# News(content='Topic: %s @ %s' % (topic,channel),link='/topic/').put()
News(content='Topic',
link_text='%s @ %s' % (topic, channel),
link='/topic/').put()
logging.debug('Return: %s' % (retval))
return HttpResponse(simplejson.dumps(retval), mimetype="application/json")
def post(request):
today = datetime.date.today()
now = datetime.datetime.now()
url = None
url_title = None
channel = None
user = None
line = None
date = None
comment = None
tags = None
old_url = False
old_user = None
old_date = None
if request.method == "POST":
try:
data = simplejson.loads(request.raw_post_data)
url = data.get('url')
channel = data.get('channel').lower()
user = data.get('user')
line = data.get('line')
date = datetime.datetime.strptime(data.get('date'), '%Y%m%d%H%M')
logging.debug(
'date: %s, user: %s, channel: %s, url: %s, line: %s' %
(date, user, channel, url, line))
except Exception as e:
logging.warning('Error %s' % (e))
if not url:
retval = simplejson.dumps({'id': 0, 'title': ''})
return HttpResponse(retval, mimetype="application/json")
orig_url = url
# Add http:// when needed
if not url.lower().startswith('http'):
url = 'http://' + url
# logging.info('Url/API/Post: Channel=%s User=%s Url=%s' % (channel,user,url))
# Fetch url (async):
# a) check statuscode LATER
# b) get title LATER
rpc = urlfetch.create_rpc()
urlfetch.make_fetch_call(rpc, url, allow_truncated=True)
# Get url from DB:
# a) already exists
# b) ChannelCheck
# 1. tarkista onko olemassa jo ko. Url, lisää jos ei, muuten päivitä (udate, valid?): valid-juttu joo ehkä jos tarpeen, ei muuten
urlquery = Url.query(Url.url == url)
urlinstance = urlquery.get()
if not urlinstance:
urlinstance = Url(url=url)
urlinstance.put()
# logging.debug('New url %s' % (url))
else:
logging.info('Old url %s' % (url))
# 2. tarkista onko olemassa jo ko. Channel, lisää jos ei
channelquery = Channel.query(Channel.name == channel)
channelinstance = channelquery.get()
if not channelinstance:
if channel.startswith('#'):
private = False
else:
private = True
channelinstance = Channel(name=channel, private=private)
channelinstance.put()
logging.info('New channel %s' % (channel))
# 3. tarkista onko url jo olemassa channel-tasolla
channelurlquery = ChannelUrl.query(
ChannelUrl.url == urlinstance.key,
ChannelUrl.channel == channelinstance.key)
channelurlinstance = channelurlquery.get()
if not channelurlinstance:
l = list(string.ascii_uppercase)
l.append('Z')
DEFAULT_COUNTER_NAME = chr(now.isocalendar()[0] - 2010 +
65) + l[(now.isocalendar()[1] - 1) / 2]
# logging.debug('DEFAULT_COUNTER_NAME: %s' % (DEFAULT_COUNTER_NAME))
counter.increment(DEFAULT_COUNTER_NAME)
key_name = DEFAULT_COUNTER_NAME + str(
counter.get_count(DEFAULT_COUNTER_NAME))
# logging.debug('key_name %s' % (key_name))
channelurlinstance = ChannelUrl(id=key_name,
channel=channelinstance.key,
url=urlinstance.key)
channelurlinstance.put()
# logging.debug('New channelurl %s/%s' % (channel,url))
else:
# logging.info('OLDIE! %s %s' % (channelurlinstance.channel.name,channelurlinstance.url.url))
logging.info('Old channelurl %s %s' % (channel, url))
old_url = True
old_post = Post.query(Post.channelurl == channelurlinstance.key).order(
Post.date).get()
try:
old_date = old_post.date.strftime("%d.%m.%y %H:%M")
except:
try:
old_date = old_post.idate.strftime("%d.%m.%y %H:%M")
except:
old_date = ''
old_user = old_post.user
# 4. Lisätään postaus (tarkistetaan ettei ole jo)
postquery = Post.query(Post.channelurl == channelurlinstance.key,
Post.user == user, Post.date == date)
postinstance = postquery.get()
if postinstance:
logging.info('Old post; channel: %s, url: %s, user: %s' %
(channel, url, user))
else:
postinstance = Post(channelurl=channelurlinstance.key,
user=user,
date=date)
postinstance.put()
# 5. extrat
# Comment
if orig_url != line:
comment = line.replace(orig_url, '<url>')
# logging.debug('Line: %s, url: %s, comment: %s' % (line,orig_url,comment))
# TODO:
# <url>/
# Tyhjät kommentit: ''
if comment not in ['<url>/', '']:
Extra(user=user,
comment=comment,
channelurl=channelurlinstance.key).put()
else:
comment = None
# Resolve async calls
# Urlfetch
try:
result = rpc.get_result()
if result and result.content_was_truncated:
logging.debug('Truncated')
tags = 'truncated'
Extra(user='******',
tag=tags,
channelurl=channelurlinstance.key).put()
if result and result.status_code:
urlinstance.status = str(result.status_code)
urlinstance.last_check = datetime.datetime.now()
if result.status_code == 200:
urlinstance.valid = 2
try:
# Fetch title
req = urllib2.Request(url)
# logging.debug('req %s' % (req))
req.add_header(
'User-agent',
'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)'
)
res = urllib2.urlopen(req)
# logging.debug('res %s' % (res))
doc = res.read()
# logging.debug('doc %s' % (doc))
encoding = res.headers.getparam('charset')
# logging.debug('encoding %s' % (encoding))
tree = etree.fromstring(doc,
etree.HTMLParser(encoding=encoding))
title = tree.find(".//title").text
url_title = smart_text(re.sub(r'\s+', ' ', title).strip())
except:
logging.warning('Title not fetched %s' % (url))
else:
# logging.debug('Url: %s, title %s' % (url,title))
urlinstance.title = url_title
# except:
# logging.debug('TitleTask: title not fetched %s' % (url))
# urlinstance.title = url
else:
if urlinstance.valid > -5:
urlinstance.valid = urlinstance.valid - 1
else:
logging.info('Broken url: %s' % (url))
urlinstance.put()
# logging.debug('URL %s saved (status code %s).' % (url,str(result.status_code)))
except (urlfetch.DownloadError, urlfetch.DeadlineExceededError,
urlfetch.InternalTransientError):
# Request timed out or failed.
# ...
urlinstance.valid = urlinstance.valid - 1
urlinstance.put()
logging.warning('Urlfetch \'%s\' failed.' % (url))
# Update News
if channelinstance.private == False and date.date() >= today:
try:
news = News(content='Link')
news.link = url
news.link_text = url_title or url
news.put()
# logging.debug('News updated')
except:
logging.warning('News update failed')
else:
logging.info('News not updated, private channel/old url')
if not url_title:
url_title = ''.join(url.split('/')[-1:])
# logging.debug('Title: %s' % (url_title))
# Create Document (FullTextSearch)
doc_id = str(urlinstance.key.id())
try:
doc = search.Document(doc_id=doc_id,
fields=[
search.TextField(name='channel',
value=channel),
search.TextField(name='user', value=user),
search.TextField(name='url', value=url),
search.DateField(name='date', value=date),
search.TextField(name='title',
value=url_title),
search.TextField(name='comment',
value=comment,
language='fi'),
search.TextField(name='tag',
value=tags,
language='fi'),
search.NumberField(name='rate', value=0)
],
language='en')
except Exception as e:
logging.error('Error %s' % (e))
# logging.debug('Document fields updated')
if urlinstance.document_date:
try:
taskqueue.add(name=doc_id + '_post',
queue_name='document',
url='/tasks/update_document',
params={'doc_id': doc_id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s_post' % (doc_id))
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s_post' % (doc_id))
except:
logging.critical('Something weird happened')
try:
search.Index(name='url').put(doc)
urlinstance.document_date = datetime.datetime.now()
urlinstance.put()
except search.Error:
logging.warning('Create Document failed.')
try:
taskqueue.add(name=doc_id + '_retry',
queue_name='document',
url='/tasks/update_document',
params={'doc_id': doc_id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s_retry' % (doc_id))
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s_retry' % (doc_id))
except:
logging.critical('Something weird happened, again?')
# Finally: return status and/or title (+something)
logging.info('Returning id: %s, title: %s, old: %s' %
(channelurlinstance.key.id(), url_title, old_url))
retval = simplejson.dumps({
'id': channelurlinstance.key.id(),
'title': url_title,
'old': old_url,
'old_user': old_user,
'old_date': old_date
})
return HttpResponse(retval, mimetype="application/json")
def post(self):
doc_id = self.request.get('doc_id', '')
if doc_id:
# logging.debug('doc_id: %s' % (doc_id))
urlinstance = Url.get_by_id(int(doc_id))
if urlinstance:
# If not valid url, delete from index
if urlinstance.valid < 0:
doc_index = search.Index(name='url')
logging.info(
'Delete invalid (%s) url (ID %s) from document index \'url\' (%s)'
% (str(urlinstance.valid), doc_id, doc_index))
doc_index.delete(doc_id)
else:
url = urlinstance.url
title = urlinstance.title
# logging.debug('url: %s, title: %s' % (url, title))
channels = []
channel = None
users = []
user = None
date = datetime.datetime.fromtimestamp(0)
comments = []
comment = None
tags = []
tag = None
rate = 0
channelurlquery = ChannelUrl.query(
ChannelUrl.url == urlinstance.key)
for channelurlinstance in channelurlquery:
channelinstance = channelurlinstance.channel.get()
if channelinstance.name not in channels:
channels.append(channelinstance.name)
# logging.info('Adding channel %s' % (channelinstance.name))
postquery = Post.query(
Post.channelurl == channelurlinstance.key)
for postinstance in postquery:
if postinstance.user not in users:
users.append(postinstance.user)
if date:
if date < postinstance.date:
date = postinstance.date
else:
date = postinstance.date
extraquery = Extra.query(
Extra.channelurl == channelurlinstance.key)
for extrainstance in extraquery:
if extrainstance.tag:
if extrainstance.tag not in tags:
tags.append(extrainstance.tag)
# logging.info('Adding tag %s' % (extrainstance.tag))
if extrainstance.comment:
if extrainstance.comment not in comments:
comments.append(extrainstance.comment)
# logging.info('Adding comment %s' % (extrainstance.comment))
ratequery = Rate.query(
Rate.channelurl == channelurlinstance.key)
for rateinstance in ratequery:
rate += rateinstance.value
# logging.debug('rate %s' % (rate))
if not date:
date = datetime.datetime.fromtimestamp(0)
# lists to strings
channel = ' '.join(channels)
user = '******'.join(users)
tag = ' '.join(tags)
if not tag:
tag = None
comment = ' '.join(comments)
if not comment:
comment = None
logging.debug(
'doc; channel=%s, user=%s, url=%s, date=%s, title=%s, comment=%s, tag=%s, rate=%s'
%
(channel, user, url, date, title, comment, tag, rate))
try:
doc = search.Document(
doc_id=str(doc_id),
fields=[
search.TextField(name='channel',
value=channel),
search.TextField(name='user', value=user),
search.TextField(name='url', value=url),
search.DateField(name='date', value=date),
search.TextField(name='title', value=title),
search.TextField(name='comment',
value=comment,
language='fi'),
search.TextField(name='tag',
value=tag,
language='fi'),
search.NumberField(name='rate', value=rate)
],
language='en')
except Exception, e:
logging.error('doc_id: %s, error %s' %
(str(doc_id), e))
doc = None
try:
if doc:
search.Index(name='url').put(doc)
urlinstance.document_date = datetime.datetime.now()
urlinstance.put()
else:
logging.error('Doc missing.')
except search.Error:
logging.error('Create Document failed.')
else:
logging.debug('No urlinstance for doc_id: %s' % (doc_id))
def extra(request):
id = '0'
user = None
type = None
value = None
new_doc = None
if request.method == "POST":
try:
data = simplejson.loads(request.raw_post_data)
id = data.get('id').upper()
user = data.get('user')
type = data.get('type')
value = data.get('value')
logging.debug('user: %s, id: %s, type: %s, value: %s' %
(user, id, type, value))
except Exception as e:
logging.warning('Error %s' % (e))
try:
id = int(id)
except:
pass
channelurl = ChannelUrl.get_by_id(id)
if not channelurl:
retval = simplejson.dumps({'id': 0, 'extra': ''})
return HttpResponse(retval, mimetype="application/json")
else:
if type == 'comment':
Extra(user=user, comment=value, channelurl=channelurl.key).put()
if type == 'tag':
Extra(user=user, tag=value, channelurl=channelurl.key).put()
if type == 'related':
Extra(user=user, related=value, channelurl=channelurl.key).put()
# Update Document (FullTextSearch)
url = channelurl.url.get()
doc_id = str(url.key.id())
try:
doc = search.Index(name='url').get(doc_id)
if not doc:
logging.warning('Document not found.')
try:
taskqueue.add(name=str(doc_id) + '_extra',
queue_name='document',
url='/tasks/update_document',
params={'doc_id': doc_id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s_extra' %
(str(doc_id)))
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s_extra' %
(str(doc_id)))
else:
new_fields = []
for field in doc.fields:
if type == 'tag' and field.name == 'tag':
if field.value:
new_value = field.value + ' ' + value
else:
new_value = value
logging.debug('Updating tags: %s + %s = %s' %
(field.value, value, new_value))
new_fields.append(
search.TextField(name=field.name, value=new_value))
if type == 'comment' and field.name == 'comment':
if field.value:
new_value = field.value + ' ' + value
else:
new_value = value
logging.debug('Updating comments: %s + %s = %s' %
(field.value, value, new_value))
new_fields.append(
search.TextField(name=field.name, value=new_value))
elif field.name == 'rate':
new_fields.append(
search.NumberField(name=field.name,
value=field.value))
elif field.name == 'date':
new_fields.append(
search.DateField(name=field.name,
value=field.value))
else:
new_fields.append(
search.TextField(name=field.name,
value=field.value))
new_doc = search.Document(doc_id=doc_id,
fields=new_fields,
language='en')
except Exception as e:
logging.warning('Error %s' % (e))
try:
if new_doc:
search.Index(name='url').put(new_doc)
else:
logging.warning('New document (new_doc) missing?')
except search.Error:
logging.exception('Create/Update Document failed.')
retval = simplejson.dumps({'id': id, type: value})
return HttpResponse(retval, mimetype="application/json")
def rate(request):
id = 0
user = None
value = 0
if request.method == "POST":
try:
data = simplejson.loads(request.raw_post_data)
id = data.get('id').upper()
user = data.get('user')
value = int(data.get('value', 0))
logging.debug('user: %s, id: %s, value: %s' % (user, id, value))
except Exception as e:
logging.warning('Error %s' % (e))
try:
id = int(id)
except:
pass
channelurl = ChannelUrl.get_by_id(id)
if not channelurl:
retval = simplejson.dumps({'id': 0, 'rate': ''})
return HttpResponse(retval, mimetype="application/json")
else:
rate = Rate(user=user, value=value, channelurl=channelurl.key)
rate.put()
# Update Document (FullTextSearch)
url = channelurl.url.get()
doc_id = str(url.key.id())
try:
doc = search.Index(name='url').get(doc_id)
if not doc:
logging.warning('Document not found.')
try:
taskqueue.add(name=str(doc_id) + '_update',
queue_name='document',
url='/tasks/update_document',
params={'doc_id': doc_id})
except taskqueue.TombstonedTaskError:
logging.warning('TombstonedTaskError %s_update' %
(str(doc_id)))
except taskqueue.TaskAlreadyExistsError:
logging.warning('TaskAlreadyExistsError %s_update' %
(str(doc_id)))
else:
new_fields = []
for field in doc.fields:
if field.name == 'rate':
new_value = float(field.value) + float(value)
logging.debug('Updating rate: %s + %s = %s' %
(field.value, value, new_value))
new_fields.append(
search.NumberField(name='rate', value=new_value))
elif field.name == 'date':
new_fields.append(
search.DateField(name=field.name,
value=field.value))
else:
new_fields.append(
search.TextField(name=field.name,
value=field.value))
new_doc = search.Document(doc_id=doc_id,
fields=new_fields,
language='en')
except Exception as e:
logging.warning('Error %s' % (e))
try:
search.Index(name='url').put(new_doc)
except search.Error:
logging.exception('Create/Update Document failed.')
retval = simplejson.dumps({'id': id, 'rate': channelurl.rating()})
return HttpResponse(retval, mimetype="application/json")
def find(request):
idx = ''
channel = '*'
content = ''
retval = []
limit = 5
offset = 0
if request.method == "POST":
try:
data = simplejson.loads(request.raw_post_data)
channel = data.get('channel', '*').lower()
content = data.get('content', '')
idx = data.get('index', 'url')
try:
limit = int(data.get('limit', 5))
except:
limit = 5
try:
offset = int(data.get('offset', 0))
except:
offset = 0
logging.debug('channel: %s, content: %s' % (channel, content))
except Exception as e:
logging.warning('Error %s' % (e))
# if not content:
# retval=simplejson.dumps([{'id':0,'title': ''}])
# return HttpResponse(retval, mimetype="application/json")
try:
# Set query options
# date_desc = search.SortExpression(
# expression='_score',
# direction=search.SortExpression.DESCENDING,
# default_value='')
# Sort up to 1000 matching results by subject in descending order
# sort = search.SortOptions(expressions=[date_desc], limit=10)
options = search.QueryOptions(
limit=limit, # the number of results to return
offset=offset,
# cursor=cursor,
# sort_options=sort,
# returned_fields=['author', 'subject', 'summary'],
# snippeted_fields=['content']
)
if channel and channel != '*':
content = 'channel:' + channel + ' ' + content
query = search.Query(query_string=content, options=options)
index = search.Index(name=idx)
results = index.search(query)
for scored_document in results:
# process scored_document
doc_id = scored_document.doc_id
doc_url = None
doc_user = None
doc_channel = None
doc_date = None
for field in scored_document.fields:
if field.name == 'url':
doc_url = field.value
if field.name == 'user':
doc_user = field.value
if field.name == 'channel':
doc_channel = field.value
if field.name == 'date':
doc_date = field.value
# logging.debug('Search result: %s' % (scored_document))
urlinstance = Url.get_by_id(int(doc_id))
if channel == '*':
channelurlquery = ChannelUrl.query(
ChannelUrl.url == urlinstance.key)
else:
channelinstance = Channel.query(Channel.name == channel).get()
channelurlquery = ChannelUrl.query(
ChannelUrl.url == urlinstance.key,
ChannelUrl.channel == channelinstance.key)
channelurls = channelurlquery.fetch(3)
for channelurl in channelurls:
retval.append({
'id': channelurl.key.id(),
'url': urlinstance.url,
'posts': channelurl.posts()
})
except search.Error:
logging.exception('Search failed')
# logging.debug('retval %s' % (retval))
retvaljson = simplejson.dumps(retval)
return HttpResponse(retvaljson, mimetype="application/json")
def post(request):
today = datetime.date.today()
retval = {}
error = 10
channel = None
title = None
lines = []
for key in request.POST:
error = 0
data = json.loads(key)
logging.debug('Data: %s' % (str(data)))
# Parse and check data
if 'channel' in data:
channel = data['channel']
else:
error += 1
if 'title' in data:
title = data['title']
else:
title = today
if 'date' in data:
date = datetime.datetime.strptime(data['date'], '%y%m%d%H%M%S')
else:
date = today
if 'lines' in data:
lines = data['lines']
else:
error += 2
if not error:
prevnick = ''
rage = Rage(channel=channel, title=title, date=date)
rage.put()
i = 1
for line in lines:
# logging.debug('Line: %s, nick: %s, msg: %s' % (line,line['nick'],line['msg']))
nick = line['nick']
# HTML safe
msg = cgi.escape(line['msg'])
if msg.endswith(':)'):
faces = ['smile', 'happy', 'hehehe']
elif msg.endswith(':('):
faces = ['unhappy', 'herp']
elif msg.endswith(';)'):
faces = ['pfftch']
elif msg.endswith(':D'):
faces = ['epicwin', 'grin']
elif msg.endswith('!'):
faces = ['loool']
elif msg.endswith('.'):
faces = ['monocole']
else:
faces = [
'beh', 'dude-come-on', 'epicwin', 'french', 'grin',
'happy', 'hehehe', 'herp', 'horror', 'loool', 'monocole',
'pfftch', 'rage', 'redeyes', 'smile', 'suspicious',
'unhappy', 'wait', 'concentrated', 'kittehsmile'
]
face = faces[random.randint(0, (len(faces)) - 1)]
# logging.debug('Face: %s' % (face))
if nick != prevnick:
panel = Panel(rage=rage.key,
count=i,
nick=nick,
face=face,
msg=msg)
panel.put()
i += 1
else:
panel.face = face
panel.msg = panel.msg + '<br /><b> |</b><br />' + msg
panel.put()
prevnick = nick
retval = {'id': rage.key.id(), 'title': title}
News(content='<a href="/rage/%s/">%s</a>' %
(rage.key.id(), title)).put()
else:
retval = {'id': 0, 'title': 'error: %s' % (error)}
logging.debug('Return: %s' % (retval))
return HttpResponse(json.dumps(retval), mimetype="application/json")
