def view_master(request, urlid): data = [] logging.debug('View (Master)Url %s' % (urlid)) try: urlid = int(urlid) except: pass url = Url.get_by_id(urlid) if url: channelurls = ChannelUrl.query(ChannelUrl.url == url.key) for channelurl in channelurls: channel = channelurl.channel.get() if channel.private == False: extras = Extra.query(Extra.channelurl == channelurl.key) rates = Rate.query(Rate.channelurl == channelurl.key) rating = channelurl.rating() # data.append({'channel':channel,'post':post,'url':url,'extras': extras}) data.append({ 'channel': channel, 'channelurl': channelurl, 'post': post, 'url': url, 'extras': extras, 'rates': rates, 'rating': rating }) template_values = { 'data': data, 'user': users.get_current_user(), } return render('masterurl.html', template_values)
def view(request, urlid): data = [] logging.debug('View ChannelUrl %s' % (urlid)) try: urlid = int(urlid) except: pass channelurl = ChannelUrl.get_by_id(urlid) if channelurl: channel = channelurl.channel.get() url = channelurl.url.get() if channel.private == False: extras = Extra.query(Extra.channelurl == channelurl.key) rates = Rate.query(Rate.channelurl == channelurl.key) rating = channelurl.rating() # data.append({'channel':channel,'post':post,'url':url,'extras': extras}) data = { 'channel': channel, 'post': post, 'url': url, 'extras': extras, 'rates': rates, 'rating': rating } template_values = { 'data': data, } return render('url.html', template_values)
def create_lifo_tracker_id(): """ This is the internal endpoint for creating lifo tracker id, and requires auth from Influencers. The caller is currently from campaign side, where queries are sent from api_nodejs during influencer signing up for brand initiated campaigns. The request payload will include campaign_data as defined in campaign.js under api_nodejs. """ contract_data = flask.request.json logging.debug(f'{request.path} receiving contract data {contract_data}') if not contract_data.get('brand_campaign_id'): response = flask.jsonify( {'Status': 'A valid brand_campaign_id is required'}) response.status_code = 422 return response if not contract_data.get('account_id'): response = flask.jsonify({'Status': 'A valid account_id is required'}) response.status_code = 422 return response tracking_url = gen_lifo_tracker_id(contract_data.get('account_id'), contract_data) if type(tracking_url) is flask.Response: return tracking_url response = flask.jsonify({'tracking_url': tracking_url}) response.status_code = 200 return response
def create_lifo_tracker_id(): """ This is the internal endpoint for creating lifo tracker id, and requires auth from Influencers. The caller is currently from campaign side, where queries are sent from api_nodejs during influencer signing up for brand initiated campaigns. The request payload will include campaign_data as defined in campaign.js under api_nodejs. """ # influencer uid, NOT shop uid = flask.session['uid'] # a flag to differentiate Shopify accounts, which use store domain as uid from_shopify = flask.session['from_shopify'] campaign_data = flask.request.json logging.debug(f'{request.path} receiving campaign data {campaign_data}') if not campaign_data.get('brand_campaign_id'): response = flask.jsonify( {'Status': 'A valid brand_campaign_id is required'}) response.status_code = 422 return response tracking_url = gen_lifo_tracker_id(uid, campaign_data) if type(tracking_url) is flask.Response: return tracking_url response = flask.jsonify({'tracking_url': tracking_url}) response.status_code = 200 return response
def post(self): id = self.request.get('id') if id: url = Url.get_by_id(int(id)) if url: result = None try: result = urlfetch.fetch(url.url, allow_truncated=True) except urlfetch.DownloadError: url.status = 'DE' logging.info('DownloadError, url: %s' % (url.url)) except urlfetch.ResponseTooLargeError: url.status = 'RTL' logging.info('ResponseTooLargeError, url: %s' % (url.url)) except urlfetch.InvalidURLError: url.status = 'IUE' logging.info('InvalidURLError, url: %s' % (url.url)) except: url.status = 'UE' logging.error('"Unexpected error: %s, url: %s' % (sys.exc_info()[0], url.url)) if result: if result.content_was_truncated: logging.debug('truncated') if result.status_code: url.status = str(result.status_code) if result and result.status_code and result.status_code == 200: url.valid = 2 else: if url.valid > -5: url.valid = url.valid - 1 else: logging.info('Broken url: %s' % (url.url)) url.last_check = datetime.datetime.now() url.put()
def rate(request, post_channel, post_user, type, post_url): logging.debug('rate: C=%s U=%s T=%s P=%s' % (post_channel, post_user, type, post_url)) types = ['up', 'down', 'wtf', 'nsfw'] if type in types: name = ''.join( re.findall( '[a-zA-Z0-9_-]', post_channel + '_' + post_user + '_' + type + '_' + post_url))[:500] try: taskqueue.add(name=name, queue_name='default', url='/tasks/rate', params={ 'channel': post_channel, 'post_user': post_user, 'type': type, 'post_url': post_url }) except taskqueue.TombstonedTaskError: logging.warning('Duplicate task name %s' % name) else: logging.warning('Wrong type (%s)! %s %s %s' % (type, post_channel, post_user, post_url)) return HttpResponseRedirect('/url/')
def track(): """ public endpoint (no auth) Note: Shopify client side is using the following code snippet to send tracking events: # req.send(JSON.stringify({ # lifo_tracker_id: lifo_tracker_id, # shop: getShop(), # location: document.location, # navigator: navigator.userAgent, # referrer: document.referrer, # discount_code, # })), """ data = flask.request.json logging.info(f'Receiving /track request {data}') if not data.get('shop'): logging.warning(f'Invalid shop data received {data}') elif not data.get('lifo_tracker_id'): logging.debug(f'Skip none lifo event {data}') else: try: res = sql_handler.save_track_visit(data) if res.status_code == 200: logging.info('Data saved to cloud SQL') except Exception as e: logging.error(f'Saving events error: {e}') response = flask.jsonify({'status': 'OK'}) response.status_code = 200 return response
def post(self): type = self.request.get('type', '') if type == 'stats': pass elif type == 'cleanup': last_year = datetime.datetime.now() - datetime.timedelta(days=365) last_quarter = datetime.datetime.now() - datetime.timedelta( days=92) last_month = datetime.datetime.now() - datetime.timedelta(days=31) # Old news old_news = News.query(News.date < last_quarter).order( News.date).fetch(500, keys_only=True) # logging.info('Cleaning up old news %s' % News.query().order(News.date).count(100,keys_only=True)) ndb.delete_multi(old_news) elif type == 'tag_cloud': channel_urls = [] tags = {} extras = Extra.query(Extra.tag != None) for extra in extras: if extra.channelurl not in channel_urls: channel_urls.append(extra.channelurl) tag = extra.tag if tag in tags: tags[tag] += 1 else: tags[tag] = 1 tags_sorted = sorted(tags.iteritems(), key=operator.itemgetter(1), reverse=True) memcache.set("tag_cloud", tags_sorted) logging.debug('Tags: %s' % (tags_sorted)) elif type == 'fix': test_channel = '#kanava' channel = Channel.query(Channel.name == test_channel).get() channelurls = ChannelUrl.query( ChannelUrl.channel == channel.key).fetch(50) for channelurl in channelurls: url = channelurl.url.get() logging.debug('Channel: %s, channelurl: %s (id %s)' % (test_channel, url, channelurl)) posts = Post.query(Post.channelurl == channelurl.key) for post in posts: logging.debug(' * posted by %s' % (post.user)) post.key.delete() rates = Rate.query(Rate.channelurl == channelurl.key) for rate in rates: logging.debug(' * rate %s' % (rate)) rate.key.delete() extras = Extra.query(Extra.channelurl == channelurl.key) for extra in extras: logging.debug(' * extra %s, by %s' % (extra, extra.user)) extra.key.delete() channelurl.key.delete()
def info(request): today = datetime.date.today() now = datetime.datetime.now() l = list(string.ascii_uppercase) l.append('Z') DEFAULT_COUNTER_NAME = chr(now.isocalendar()[0] - 2010 + 65) + l[(now.isocalendar()[1] - 1) / 2] id = None url = None if request.method == "POST": try: data = simplejson.loads(request.raw_post_data) id = str(data.get('id', '')).upper() url = data.get('url', '') logging.debug('id: %s/url: %s' % (id, url)) except Exception as e: logging.warning('Error %s' % (e)) try: id = int(id) if id < 1000: id = DEFAULT_COUNTER_NAME + str(id) # logging.debug('%s' % id) except: pattern1 = r'^[A-Z]{1}$' pattern2 = r'^[A-Z]{2}$' if re.match(pattern1, id): id = DEFAULT_COUNTER_NAME[0] + str(id) # logging.debug('%s' % id) id = id + str(counter.get_count(id)) # logging.debug('%s' % id) elif re.match(pattern2, id): id = id + str(counter.get_count(id)) channelurl = ChannelUrl.get_by_id(id) if channelurl: url = channelurl.url.get().url url_title = channelurl.url.get().title rate = channelurl.rating() extra = channelurl.extras(plain='True') posts = channelurl.posts() channel = channelurl.channel.get().name retval = simplejson.dumps({ 'id': channelurl.key.id(), 'url': url, 'title': url_title, 'rate': rate, 'extra': extra, 'posts': posts }) else: retval = simplejson.dumps({'id': 0}) return HttpResponse(retval, mimetype="application/json")
def download_file(storage_bucket_name, file_path): # extract file name _file_path = file_path.split('/') file_name = _file_path[-1] logging.debug("Downloading file {} from blob {}".format( file_name, storage_bucket_name)) storage_client = storage.Client() bucket = storage_client.get_bucket(storage_bucket_name) blob = bucket.blob(file_name) blob.download_to_filename(upload_folder + '/' + file_name) return (upload_folder + '/' + file_name), file_name, storage_bucket_name
def schedule(): """ Checks if it's time to run a schedule. Returns: """ logging.debug("From Cron start /tasks/schedule") with client.context(): keys = PolicyModel.query().fetch(keys_only=True) for key in keys: logging.debug("Creating deferred task for %s", key.id()) policy_tasks.policy_checker(key.id()) return "ok", 200
def track_visits(): shop = flask.session['uid'] if not shop: response = flask.jsonify({'Status': 'Failed'}) response.status_code = 422 return response try: # schema: COUNT(*) as visits, shop sql_results = sql_handler.counts_visits_per_shop(shop) visit_results = count_visits_daily(sql_results) except Exception as e: logging.error(f'Getting track_visit error: {e}') logging.debug(f'Obtained tracking report {visit_results}') response = flask.jsonify(visit_results) response.status_code = 200 return response
def get_schedule(): """ Get a schedule. Returns: schedule json """ name = request.args.get("schedule") schedule = {} with client.context(): res = SchedulesModel.query(SchedulesModel.Name == name).get() if not res: return "not found", 404 schedule.update({"name": res.Name}) schedule.update(res.Schedule) schedule.update({"timezone": res.Timezone}) logging.debug(json.dumps(res.Schedule)) return json.dumps(schedule)
def change_state(): """ Initiate change state. Returns: """ payload = json.loads(request.get_data(as_text=False) or "(empty payload)") logging.debug( "Starting change_state action %s project %s tagkey %s tagvalue %s", payload["action"], payload["project"], payload["tagkey"], payload["tagvalue"], ) schedule_tasks.change_state(payload["tagkey"], payload["tagvalue"], payload["action"], payload["project"]) return "ok", 200
def get_policy(): """ Get policy. Returns: policy json """ policy = {} name = request.args.get("policy") with client.context(): res = PolicyModel.query(PolicyModel.Name == name).get() logging.debug(res) if not res: return "not found", 404 policy.update({"name": res.Name}) policy.update({"schedulename": res.Schedule}) policy.update({"tags": res.Tags}) policy.update({"projects": res.Projects}) return json.dumps(policy)
def hook(): if request.method == "OPTIONS": # CORS preflight return _build_cors_prelight_response() if request.path.startswith('/brand') or request.path.startswith( '/am') or request.path.startswith('/influencer'): id_token = flask.request.headers.get( 'Authorization') or flask.request.args.get('id_token') if not id_token: logging.error('Valid id_token required') response = flask.jsonify('Valid id_token required') response.status_code = 401 return response decoded_token = token_verification(id_token) uid = decoded_token['uid'] if not uid: logging.error('id_token verification failed') response = flask.jsonify('id_token verification failed') response.status_code = 401 return response logging.info( f'request path is: {request.path} with decoded token {decoded_token}' ) if decoded_token.get(ACCOUNT_MANAGER_FLAG): logging.info('AM account has admin access') elif not decoded_token.get( ACCOUNT_MANAGER_FLAG) and request.path.startswith('/am'): response = flask.jsonify({"status": "not authorized"}) response.status_code = 403 return response elif (request.path.startswith('/brand') and not decoded_token.get(STORE_ACCOUNT))\ or (request.path.startswith('/influencer') and decoded_token.get(STORE_ACCOUNT)): response = flask.jsonify({"status": "not authorized"}) response.status_code = 403 return response flask.session['uid'] = uid flask.session[FROM_SHOPIFY] = decoded_token.get(FROM_SHOPIFY) flask.session[STORE_ACCOUNT] = decoded_token.get(STORE_ACCOUNT) flask.session[ACCOUNT_MANAGER_FLAG] = decoded_token.get( ACCOUNT_MANAGER_FLAG) flask.session['name'] = decoded_token.get('name') flask.session['email'] = decoded_token.get('email') else: logging.debug(f'By passing auth for request {request.path}')
def post(self): id = self.request.get('id') if id: url = Url.get_by_id(int(id)) if url: # TODO: fetch title # try: req = urllib2.Request(url.url) # logging.debug('req %s' % (req)) # req.add_header('User-agent', 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.11 (KHTML, like Gecko) Chrome/23.0.1271.64 Safari/537.11') req.add_header( 'User-agent', 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' ) req.add_header( 'Accept', 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8' ) req.add_header('Accept-Charset', 'ISO-8859-1,utf-8;q=0.7,*;q=0.3') req.add_header('Accept-Encoding', 'none') req.add_header('Accept-Language', 'en-US,en;q=0.8') req.add_header('Connection', 'keep-alive') res = urllib2.urlopen(req) # logging.debug('res %s' % (res)) doc = res.read() # logging.debug('doc %s' % (doc)) encoding = res.headers.getparam('charset') logging.debug('encoding %s' % (encoding)) try: tree = etree.fromstring( doc, etree.HTMLParser(encoding=encoding)) except LookupError: tree = etree.fromstring(doc, etree.HTMLParser(encoding='utf-8')) title = tree.find(".//title").text logging.debug('title %s' % (title)) url.title = smart_text(re.sub(r'\s+', ' ', title).strip()) # except: # logging.debug('TitleTask: title not fetched %s' % (post_url)) # url.title = post_url url.put()
def extra(request, post_channel, post_user, extra, post_url): logging.debug('rate: C=%s U=%s E=%s P=%s' % (post_channel, post_user, extra, post_url)) name = ''.join( re.findall( '[a-zA-Z0-9_-]', post_channel + '_' + post_user + '_' + extra + '_' + post_url))[:500] try: taskqueue.add(name=name, queue_name='default', url='/tasks/extra', params={ 'channel': post_channel, 'post_user': post_user, 'extra': extra, 'post_url': post_url }) except taskqueue.TombstonedTaskError: logging.warning('Duplicate task name %s' % name) return HttpResponseRedirect('/url/')
def post(request, post_url, post_channel=None, post_user=None): if not post_user: post_user = '******' if not post_channel: post_channel = '!' + post_user logging.debug('post: C=%s U=%s P=%s' % (post_channel, post_user, post_url)) name = ''.join( re.findall('[a-zA-Z0-9_-]', post_channel + '_' + post_user + '_' + post_url))[:500] try: taskqueue.add(name=name, queue_name='default', url='/tasks/post', params={ 'post_channel': post_channel, 'post_user': post_user, 'post_url': post_url }) except taskqueue.TombstonedTaskError: logging.warning('Duplicate task name %s' % name) return HttpResponseRedirect('/url/')
def pubsub_push(): if (request.args.get('token', '') != token): return 'Invalid request', 400 envelope = json.loads(request.data.decode('utf-8')) payload = base64.b64decode(envelope['message']['data']) MESSAGES.append(payload) # download file from blob logging.debug("Downloading file {} from storage {}".format( payload, storage_bucket_name)) _payload = payload.decode('utf-8') _file_name = download_file(storage_bucket_name, _payload) file_name = _file_name[0] file_name_wo_folder = _file_name[1] ai_score = analyze_picture(file_name)[0] other_objects = analyze_picture(file_name)[1] if ai_score > float(detection_score): message = "Object: {} identified with probability {} on picture {}".format( label_to_detect, round(ai_score, 2), file_name) send_message_to_pubsub(message, file_name_wo_folder, storage_bucket_name) else: if notify_about_all_checks == "yes": message = "Object: {} not detected. Other objects are {}. Picture {}".format( label_to_detect, other_objects, file_name) send_message_to_pubsub(message, file_name_wo_folder, storage_bucket_name) # delete local copy of file logging.debug("Deleting local copy of file {}".format(file_name)) os.remove(file_name) # Returning any 2xx status indicates successful receipt of the message. return 'OK', 200
def order_complete(): """ Public endpoint (no auth) Don't confuse this one with Shopify webhook. Note: Shopify client side is using the following code snippet to send order_complete events: # n.send(JSON.stringify({ # lifo_tracker_id: lifo_tracker_id, # shop: getShop(), # location: document.location, # navigator: navigator.userAgent, # referrer: document.referrer, # discount_code, # order_id, # customer_id: data.customer_id, # order_data, # })); """ data = flask.request.json logging.info(f'Receiving order_complete request {data}') if not data.get('shop') or not data.get('order_id') or not data.get( 'customer_id'): logging.warning(f'Invalid shop/customer data received {data}') elif not data.get('lifo_tracker_id'): logging.debug(f'Skip none lifo event {data}') else: try: order_data = data.get('order_data') # we use subtotal_price for calculation, before tax and shipping subtotal_price = float(order_data.get('subtotal_price')) logging.debug(f'Received order with revenue of {subtotal_price}') res = sql_handler.save_order_complete(data, subtotal_price) if res.status_code == 200: logging.info('Data saved to cloud SQL') except Exception as e: logging.error(f'Saving events error: {e}') response = flask.jsonify({'status': 'OK'}) response.status_code = 200 return response
def get(self): type = self.request.get('type') logging.debug('Maintenance task, %s' % (type)) if type == 'stats': taskqueue.add(queue_name='default', url='/tasks/maintenance', params={'type': 'stats'}) elif type == 'tag_cloud': taskqueue.add(queue_name='default', url='/tasks/maintenance', params={'type': 'tag_cloud'}) elif type == 'fix': taskqueue.add(queue_name='default', url='/tasks/maintenance', params={'type': 'fix'}) elif type == 'cleanup': taskqueue.add(queue_name='maintenance', url='/tasks/maintenance', params={'type': 'cleanup'}) elif type == 'url_test_clean': channel = self.request.get('channel') nick = self.request.get('nick') logging.debug('url_test_clean, channel: %s, nick: %s' % (channel, nick))
def add_policy(): """ Add policy. Returns: """ logging.debug(json.dumps(request.json)) name = request.json["name"] tags = request.json["tags"] projects = request.json["projects"] schedule_name = request.json["schedulename"] with client.context(): res = SchedulesModel.query(SchedulesModel.Name == schedule_name).get() if not res: return "not found", 404 policy_model = PolicyModel() policy_model.Name = name policy_model.Tags = tags policy_model.Projects = projects policy_model.Schedule = schedule_name policy_model.key = ndb.Key("PolicyModel", name) policy_model.put() return "ok", 200
def post(request): # today=datetime.date.today() topic = None nick = None channel = None date = None retval = {} if request.method == "POST": logging.debug('raw_post_data %s' % (request.raw_post_data)) try: data = simplejson.loads(request.raw_post_data) logging.debug('data %s' % (data)) topic = data.get('topic') channel = data.get('channel') nick = data.get('nick') date = datetime.datetime.strptime(data.get('date'), '%Y%m%d%H%M') logging.debug('date: %s, nick: %s, channel, %s, topic: %s' % (date, nick, channel, topic)) except Exception as e: logging.warning('TOPIC/POST: Somewhere near %s' % (e)) retval = {'id': 0, 'topic': e} else: try: t = Topic(channel=channel, nick=nick, topic=topic, date=date) t.put() except Exception as e: logging.warning('TOPIC/POST/NEW: Somewhere near %s' % (e)) retval = {'id': 0, 'topic': e} else: retval = {'id': t.key.id(), 'topic': topic} # TODO private channel? # News(content='Topic: %s @ %s' % (topic,channel),link='/topic/').put() News(content='Topic', link_text='%s @ %s' % (topic, channel), link='/topic/').put() logging.debug('Return: %s' % (retval)) return HttpResponse(simplejson.dumps(retval), mimetype="application/json")
def post(request): today = datetime.date.today() now = datetime.datetime.now() url = None url_title = None channel = None user = None line = None date = None comment = None tags = None old_url = False old_user = None old_date = None if request.method == "POST": try: data = simplejson.loads(request.raw_post_data) url = data.get('url') channel = data.get('channel').lower() user = data.get('user') line = data.get('line') date = datetime.datetime.strptime(data.get('date'), '%Y%m%d%H%M') logging.debug( 'date: %s, user: %s, channel: %s, url: %s, line: %s' % (date, user, channel, url, line)) except Exception as e: logging.warning('Error %s' % (e)) if not url: retval = simplejson.dumps({'id': 0, 'title': ''}) return HttpResponse(retval, mimetype="application/json") orig_url = url # Add http:// when needed if not url.lower().startswith('http'): url = 'http://' + url # logging.info('Url/API/Post: Channel=%s User=%s Url=%s' % (channel,user,url)) # Fetch url (async): # a) check statuscode LATER # b) get title LATER rpc = urlfetch.create_rpc() urlfetch.make_fetch_call(rpc, url, allow_truncated=True) # Get url from DB: # a) already exists # b) ChannelCheck # 1. tarkista onko olemassa jo ko. Url, lisää jos ei, muuten päivitä (udate, valid?): valid-juttu joo ehkä jos tarpeen, ei muuten urlquery = Url.query(Url.url == url) urlinstance = urlquery.get() if not urlinstance: urlinstance = Url(url=url) urlinstance.put() # logging.debug('New url %s' % (url)) else: logging.info('Old url %s' % (url)) # 2. tarkista onko olemassa jo ko. Channel, lisää jos ei channelquery = Channel.query(Channel.name == channel) channelinstance = channelquery.get() if not channelinstance: if channel.startswith('#'): private = False else: private = True channelinstance = Channel(name=channel, private=private) channelinstance.put() logging.info('New channel %s' % (channel)) # 3. tarkista onko url jo olemassa channel-tasolla channelurlquery = ChannelUrl.query( ChannelUrl.url == urlinstance.key, ChannelUrl.channel == channelinstance.key) channelurlinstance = channelurlquery.get() if not channelurlinstance: l = list(string.ascii_uppercase) l.append('Z') DEFAULT_COUNTER_NAME = chr(now.isocalendar()[0] - 2010 + 65) + l[(now.isocalendar()[1] - 1) / 2] # logging.debug('DEFAULT_COUNTER_NAME: %s' % (DEFAULT_COUNTER_NAME)) counter.increment(DEFAULT_COUNTER_NAME) key_name = DEFAULT_COUNTER_NAME + str( counter.get_count(DEFAULT_COUNTER_NAME)) # logging.debug('key_name %s' % (key_name)) channelurlinstance = ChannelUrl(id=key_name, channel=channelinstance.key, url=urlinstance.key) channelurlinstance.put() # logging.debug('New channelurl %s/%s' % (channel,url)) else: # logging.info('OLDIE! %s %s' % (channelurlinstance.channel.name,channelurlinstance.url.url)) logging.info('Old channelurl %s %s' % (channel, url)) old_url = True old_post = Post.query(Post.channelurl == channelurlinstance.key).order( Post.date).get() try: old_date = old_post.date.strftime("%d.%m.%y %H:%M") except: try: old_date = old_post.idate.strftime("%d.%m.%y %H:%M") except: old_date = '' old_user = old_post.user # 4. Lisätään postaus (tarkistetaan ettei ole jo) postquery = Post.query(Post.channelurl == channelurlinstance.key, Post.user == user, Post.date == date) postinstance = postquery.get() if postinstance: logging.info('Old post; channel: %s, url: %s, user: %s' % (channel, url, user)) else: postinstance = Post(channelurl=channelurlinstance.key, user=user, date=date) postinstance.put() # 5. extrat # Comment if orig_url != line: comment = line.replace(orig_url, '<url>') # logging.debug('Line: %s, url: %s, comment: %s' % (line,orig_url,comment)) # TODO: # <url>/ # Tyhjät kommentit: '' if comment not in ['<url>/', '']: Extra(user=user, comment=comment, channelurl=channelurlinstance.key).put() else: comment = None # Resolve async calls # Urlfetch try: result = rpc.get_result() if result and result.content_was_truncated: logging.debug('Truncated') tags = 'truncated' Extra(user='******', tag=tags, channelurl=channelurlinstance.key).put() if result and result.status_code: urlinstance.status = str(result.status_code) urlinstance.last_check = datetime.datetime.now() if result.status_code == 200: urlinstance.valid = 2 try: # Fetch title req = urllib2.Request(url) # logging.debug('req %s' % (req)) req.add_header( 'User-agent', 'Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)' ) res = urllib2.urlopen(req) # logging.debug('res %s' % (res)) doc = res.read() # logging.debug('doc %s' % (doc)) encoding = res.headers.getparam('charset') # logging.debug('encoding %s' % (encoding)) tree = etree.fromstring(doc, etree.HTMLParser(encoding=encoding)) title = tree.find(".//title").text url_title = smart_text(re.sub(r'\s+', ' ', title).strip()) except: logging.warning('Title not fetched %s' % (url)) else: # logging.debug('Url: %s, title %s' % (url,title)) urlinstance.title = url_title # except: # logging.debug('TitleTask: title not fetched %s' % (url)) # urlinstance.title = url else: if urlinstance.valid > -5: urlinstance.valid = urlinstance.valid - 1 else: logging.info('Broken url: %s' % (url)) urlinstance.put() # logging.debug('URL %s saved (status code %s).' % (url,str(result.status_code))) except (urlfetch.DownloadError, urlfetch.DeadlineExceededError, urlfetch.InternalTransientError): # Request timed out or failed. # ... urlinstance.valid = urlinstance.valid - 1 urlinstance.put() logging.warning('Urlfetch \'%s\' failed.' % (url)) # Update News if channelinstance.private == False and date.date() >= today: try: news = News(content='Link') news.link = url news.link_text = url_title or url news.put() # logging.debug('News updated') except: logging.warning('News update failed') else: logging.info('News not updated, private channel/old url') if not url_title: url_title = ''.join(url.split('/')[-1:]) # logging.debug('Title: %s' % (url_title)) # Create Document (FullTextSearch) doc_id = str(urlinstance.key.id()) try: doc = search.Document(doc_id=doc_id, fields=[ search.TextField(name='channel', value=channel), search.TextField(name='user', value=user), search.TextField(name='url', value=url), search.DateField(name='date', value=date), search.TextField(name='title', value=url_title), search.TextField(name='comment', value=comment, language='fi'), search.TextField(name='tag', value=tags, language='fi'), search.NumberField(name='rate', value=0) ], language='en') except Exception as e: logging.error('Error %s' % (e)) # logging.debug('Document fields updated') if urlinstance.document_date: try: taskqueue.add(name=doc_id + '_post', queue_name='document', url='/tasks/update_document', params={'doc_id': doc_id}) except taskqueue.TombstonedTaskError: logging.warning('TombstonedTaskError %s_post' % (doc_id)) except taskqueue.TaskAlreadyExistsError: logging.warning('TaskAlreadyExistsError %s_post' % (doc_id)) except: logging.critical('Something weird happened') try: search.Index(name='url').put(doc) urlinstance.document_date = datetime.datetime.now() urlinstance.put() except search.Error: logging.warning('Create Document failed.') try: taskqueue.add(name=doc_id + '_retry', queue_name='document', url='/tasks/update_document', params={'doc_id': doc_id}) except taskqueue.TombstonedTaskError: logging.warning('TombstonedTaskError %s_retry' % (doc_id)) except taskqueue.TaskAlreadyExistsError: logging.warning('TaskAlreadyExistsError %s_retry' % (doc_id)) except: logging.critical('Something weird happened, again?') # Finally: return status and/or title (+something) logging.info('Returning id: %s, title: %s, old: %s' % (channelurlinstance.key.id(), url_title, old_url)) retval = simplejson.dumps({ 'id': channelurlinstance.key.id(), 'title': url_title, 'old': old_url, 'old_user': old_user, 'old_date': old_date }) return HttpResponse(retval, mimetype="application/json")
def post(self): doc_id = self.request.get('doc_id', '') if doc_id: # logging.debug('doc_id: %s' % (doc_id)) urlinstance = Url.get_by_id(int(doc_id)) if urlinstance: # If not valid url, delete from index if urlinstance.valid < 0: doc_index = search.Index(name='url') logging.info( 'Delete invalid (%s) url (ID %s) from document index \'url\' (%s)' % (str(urlinstance.valid), doc_id, doc_index)) doc_index.delete(doc_id) else: url = urlinstance.url title = urlinstance.title # logging.debug('url: %s, title: %s' % (url, title)) channels = [] channel = None users = [] user = None date = datetime.datetime.fromtimestamp(0) comments = [] comment = None tags = [] tag = None rate = 0 channelurlquery = ChannelUrl.query( ChannelUrl.url == urlinstance.key) for channelurlinstance in channelurlquery: channelinstance = channelurlinstance.channel.get() if channelinstance.name not in channels: channels.append(channelinstance.name) # logging.info('Adding channel %s' % (channelinstance.name)) postquery = Post.query( Post.channelurl == channelurlinstance.key) for postinstance in postquery: if postinstance.user not in users: users.append(postinstance.user) if date: if date < postinstance.date: date = postinstance.date else: date = postinstance.date extraquery = Extra.query( Extra.channelurl == channelurlinstance.key) for extrainstance in extraquery: if extrainstance.tag: if extrainstance.tag not in tags: tags.append(extrainstance.tag) # logging.info('Adding tag %s' % (extrainstance.tag)) if extrainstance.comment: if extrainstance.comment not in comments: comments.append(extrainstance.comment) # logging.info('Adding comment %s' % (extrainstance.comment)) ratequery = Rate.query( Rate.channelurl == channelurlinstance.key) for rateinstance in ratequery: rate += rateinstance.value # logging.debug('rate %s' % (rate)) if not date: date = datetime.datetime.fromtimestamp(0) # lists to strings channel = ' '.join(channels) user = '******'.join(users) tag = ' '.join(tags) if not tag: tag = None comment = ' '.join(comments) if not comment: comment = None logging.debug( 'doc; channel=%s, user=%s, url=%s, date=%s, title=%s, comment=%s, tag=%s, rate=%s' % (channel, user, url, date, title, comment, tag, rate)) try: doc = search.Document( doc_id=str(doc_id), fields=[ search.TextField(name='channel', value=channel), search.TextField(name='user', value=user), search.TextField(name='url', value=url), search.DateField(name='date', value=date), search.TextField(name='title', value=title), search.TextField(name='comment', value=comment, language='fi'), search.TextField(name='tag', value=tag, language='fi'), search.NumberField(name='rate', value=rate) ], language='en') except Exception, e: logging.error('doc_id: %s, error %s' % (str(doc_id), e)) doc = None try: if doc: search.Index(name='url').put(doc) urlinstance.document_date = datetime.datetime.now() urlinstance.put() else: logging.error('Doc missing.') except search.Error: logging.error('Create Document failed.') else: logging.debug('No urlinstance for doc_id: %s' % (doc_id))
def extra(request): id = '0' user = None type = None value = None new_doc = None if request.method == "POST": try: data = simplejson.loads(request.raw_post_data) id = data.get('id').upper() user = data.get('user') type = data.get('type') value = data.get('value') logging.debug('user: %s, id: %s, type: %s, value: %s' % (user, id, type, value)) except Exception as e: logging.warning('Error %s' % (e)) try: id = int(id) except: pass channelurl = ChannelUrl.get_by_id(id) if not channelurl: retval = simplejson.dumps({'id': 0, 'extra': ''}) return HttpResponse(retval, mimetype="application/json") else: if type == 'comment': Extra(user=user, comment=value, channelurl=channelurl.key).put() if type == 'tag': Extra(user=user, tag=value, channelurl=channelurl.key).put() if type == 'related': Extra(user=user, related=value, channelurl=channelurl.key).put() # Update Document (FullTextSearch) url = channelurl.url.get() doc_id = str(url.key.id()) try: doc = search.Index(name='url').get(doc_id) if not doc: logging.warning('Document not found.') try: taskqueue.add(name=str(doc_id) + '_extra', queue_name='document', url='/tasks/update_document', params={'doc_id': doc_id}) except taskqueue.TombstonedTaskError: logging.warning('TombstonedTaskError %s_extra' % (str(doc_id))) except taskqueue.TaskAlreadyExistsError: logging.warning('TaskAlreadyExistsError %s_extra' % (str(doc_id))) else: new_fields = [] for field in doc.fields: if type == 'tag' and field.name == 'tag': if field.value: new_value = field.value + ' ' + value else: new_value = value logging.debug('Updating tags: %s + %s = %s' % (field.value, value, new_value)) new_fields.append( search.TextField(name=field.name, value=new_value)) if type == 'comment' and field.name == 'comment': if field.value: new_value = field.value + ' ' + value else: new_value = value logging.debug('Updating comments: %s + %s = %s' % (field.value, value, new_value)) new_fields.append( search.TextField(name=field.name, value=new_value)) elif field.name == 'rate': new_fields.append( search.NumberField(name=field.name, value=field.value)) elif field.name == 'date': new_fields.append( search.DateField(name=field.name, value=field.value)) else: new_fields.append( search.TextField(name=field.name, value=field.value)) new_doc = search.Document(doc_id=doc_id, fields=new_fields, language='en') except Exception as e: logging.warning('Error %s' % (e)) try: if new_doc: search.Index(name='url').put(new_doc) else: logging.warning('New document (new_doc) missing?') except search.Error: logging.exception('Create/Update Document failed.') retval = simplejson.dumps({'id': id, type: value}) return HttpResponse(retval, mimetype="application/json")
def rate(request): id = 0 user = None value = 0 if request.method == "POST": try: data = simplejson.loads(request.raw_post_data) id = data.get('id').upper() user = data.get('user') value = int(data.get('value', 0)) logging.debug('user: %s, id: %s, value: %s' % (user, id, value)) except Exception as e: logging.warning('Error %s' % (e)) try: id = int(id) except: pass channelurl = ChannelUrl.get_by_id(id) if not channelurl: retval = simplejson.dumps({'id': 0, 'rate': ''}) return HttpResponse(retval, mimetype="application/json") else: rate = Rate(user=user, value=value, channelurl=channelurl.key) rate.put() # Update Document (FullTextSearch) url = channelurl.url.get() doc_id = str(url.key.id()) try: doc = search.Index(name='url').get(doc_id) if not doc: logging.warning('Document not found.') try: taskqueue.add(name=str(doc_id) + '_update', queue_name='document', url='/tasks/update_document', params={'doc_id': doc_id}) except taskqueue.TombstonedTaskError: logging.warning('TombstonedTaskError %s_update' % (str(doc_id))) except taskqueue.TaskAlreadyExistsError: logging.warning('TaskAlreadyExistsError %s_update' % (str(doc_id))) else: new_fields = [] for field in doc.fields: if field.name == 'rate': new_value = float(field.value) + float(value) logging.debug('Updating rate: %s + %s = %s' % (field.value, value, new_value)) new_fields.append( search.NumberField(name='rate', value=new_value)) elif field.name == 'date': new_fields.append( search.DateField(name=field.name, value=field.value)) else: new_fields.append( search.TextField(name=field.name, value=field.value)) new_doc = search.Document(doc_id=doc_id, fields=new_fields, language='en') except Exception as e: logging.warning('Error %s' % (e)) try: search.Index(name='url').put(new_doc) except search.Error: logging.exception('Create/Update Document failed.') retval = simplejson.dumps({'id': id, 'rate': channelurl.rating()}) return HttpResponse(retval, mimetype="application/json")
def find(request): idx = '' channel = '*' content = '' retval = [] limit = 5 offset = 0 if request.method == "POST": try: data = simplejson.loads(request.raw_post_data) channel = data.get('channel', '*').lower() content = data.get('content', '') idx = data.get('index', 'url') try: limit = int(data.get('limit', 5)) except: limit = 5 try: offset = int(data.get('offset', 0)) except: offset = 0 logging.debug('channel: %s, content: %s' % (channel, content)) except Exception as e: logging.warning('Error %s' % (e)) # if not content: # retval=simplejson.dumps([{'id':0,'title': ''}]) # return HttpResponse(retval, mimetype="application/json") try: # Set query options # date_desc = search.SortExpression( # expression='_score', # direction=search.SortExpression.DESCENDING, # default_value='') # Sort up to 1000 matching results by subject in descending order # sort = search.SortOptions(expressions=[date_desc], limit=10) options = search.QueryOptions( limit=limit, # the number of results to return offset=offset, # cursor=cursor, # sort_options=sort, # returned_fields=['author', 'subject', 'summary'], # snippeted_fields=['content'] ) if channel and channel != '*': content = 'channel:' + channel + ' ' + content query = search.Query(query_string=content, options=options) index = search.Index(name=idx) results = index.search(query) for scored_document in results: # process scored_document doc_id = scored_document.doc_id doc_url = None doc_user = None doc_channel = None doc_date = None for field in scored_document.fields: if field.name == 'url': doc_url = field.value if field.name == 'user': doc_user = field.value if field.name == 'channel': doc_channel = field.value if field.name == 'date': doc_date = field.value # logging.debug('Search result: %s' % (scored_document)) urlinstance = Url.get_by_id(int(doc_id)) if channel == '*': channelurlquery = ChannelUrl.query( ChannelUrl.url == urlinstance.key) else: channelinstance = Channel.query(Channel.name == channel).get() channelurlquery = ChannelUrl.query( ChannelUrl.url == urlinstance.key, ChannelUrl.channel == channelinstance.key) channelurls = channelurlquery.fetch(3) for channelurl in channelurls: retval.append({ 'id': channelurl.key.id(), 'url': urlinstance.url, 'posts': channelurl.posts() }) except search.Error: logging.exception('Search failed') # logging.debug('retval %s' % (retval)) retvaljson = simplejson.dumps(retval) return HttpResponse(retvaljson, mimetype="application/json")
def post(request): today = datetime.date.today() retval = {} error = 10 channel = None title = None lines = [] for key in request.POST: error = 0 data = json.loads(key) logging.debug('Data: %s' % (str(data))) # Parse and check data if 'channel' in data: channel = data['channel'] else: error += 1 if 'title' in data: title = data['title'] else: title = today if 'date' in data: date = datetime.datetime.strptime(data['date'], '%y%m%d%H%M%S') else: date = today if 'lines' in data: lines = data['lines'] else: error += 2 if not error: prevnick = '' rage = Rage(channel=channel, title=title, date=date) rage.put() i = 1 for line in lines: # logging.debug('Line: %s, nick: %s, msg: %s' % (line,line['nick'],line['msg'])) nick = line['nick'] # HTML safe msg = cgi.escape(line['msg']) if msg.endswith(':)'): faces = ['smile', 'happy', 'hehehe'] elif msg.endswith(':('): faces = ['unhappy', 'herp'] elif msg.endswith(';)'): faces = ['pfftch'] elif msg.endswith(':D'): faces = ['epicwin', 'grin'] elif msg.endswith('!'): faces = ['loool'] elif msg.endswith('.'): faces = ['monocole'] else: faces = [ 'beh', 'dude-come-on', 'epicwin', 'french', 'grin', 'happy', 'hehehe', 'herp', 'horror', 'loool', 'monocole', 'pfftch', 'rage', 'redeyes', 'smile', 'suspicious', 'unhappy', 'wait', 'concentrated', 'kittehsmile' ] face = faces[random.randint(0, (len(faces)) - 1)] # logging.debug('Face: %s' % (face)) if nick != prevnick: panel = Panel(rage=rage.key, count=i, nick=nick, face=face, msg=msg) panel.put() i += 1 else: panel.face = face panel.msg = panel.msg + '<br /><b> |</b><br />' + msg panel.put() prevnick = nick retval = {'id': rage.key.id(), 'title': title} News(content='<a href="/rage/%s/">%s</a>' % (rage.key.id(), title)).put() else: retval = {'id': 0, 'title': 'error: %s' % (error)} logging.debug('Return: %s' % (retval)) return HttpResponse(json.dumps(retval), mimetype="application/json")