def ex_search(): if request.method == "POST": q = request.form.get("q") result = reddit_search(q) if len(result): return render_template("ex_search.html", **{"heads": result[0].keys(), "posts": result, "content_present": True, "count": len(result)}) return render_template("ex_search.html", **{"content_present": False})
def search_load(): params = {} params['name'] = name = request.form.get("name") if not name: return jsonify(**{"ok": False, "detail": "name required"}) params['rate_min'] = int(request.form.get("rate_min") or 0) params['rate_max'] = int(request.form.get("rate_max") or 99999) params['reposts_max'] = int(request.form.get("reposts_max") or 10) params['time_min'] = request.form.get("time_min") or default_time_min before_raw = request.form.get("before") if before_raw and len(before_raw): before = datetime.strptime(before_raw, "%d/%m/%Y") else: before = datetime.utcnow() - timedelta(days=30) words_raw = str(request.form.get("words")) params['before'] = before params['words'] = words = re.split("[;,:\.]\s?", words_raw) db.add_search_params(name, params, {}) log.info("will search for %s before %s \nwith params:%s" % (name, before, params)) video_ids = set() all_posts = [] for word in words: query = "site:youtube.com title:%s subreddit:%s" % (word, name) log.info("Start search: %s" % query) posts = reddit_search(query) posts = filter( lambda x: (before - x.get("created_dt")).total_seconds() > 0 and x. get("video_id") not in video_ids, posts) cur_v_ids = set(map(lambda x: x.get("video_id"), posts)) difference = cur_v_ids.difference(video_ids) difference = filter(lambda x: not db.is_post_video_id_present(x), difference) log.info("New posts: %s" % len(difference)) if difference: map(lambda x: video_ids.add(x), difference) all_posts.extend( [el for el in posts if el['video_id'] in difference]) elif len(video_ids) > 0: break log.info("will process %s posts..." % len(all_posts)) rtrv = Retriever() for post in rtrv.process_subreddit(all_posts, params): db.save_post(post, SRC_SEARCH) db.add_search_params(name, params, rtrv.statistic) return jsonify(**{"ok": True, "name": name})
def search_load(): params = {} params['name'] = name = request.form.get("name") if not name: return jsonify(**{"ok": False, "detail": "name required"}) params['rate_min'] = int(request.form.get("rate_min") or 0) params['rate_max'] = int(request.form.get("rate_max") or 99999) params['reposts_max'] = int(request.form.get("reposts_max") or 10) params['time_min'] = request.form.get("time_min") or default_time_min before_raw = request.form.get("before") if before_raw and len(before_raw): before = datetime.strptime(before_raw, "%d/%m/%Y") else: before = datetime.utcnow() - timedelta(days=30) words_raw = str(request.form.get("words")) params['before'] = before params['words'] = words = re.split("[;,:\.]\s?", words_raw) db.add_search_params(name, params, {}) log.info("will search for %s before %s \nwith params:%s" % (name, before, params)) video_ids = set() all_posts = [] for word in words: query = "site:youtube.com title:%s subreddit:%s" % (word, name) log.info("Start search: %s" % query) posts = reddit_search(query) posts = filter( lambda x: (before - x.get("created_dt")).total_seconds() > 0 and x.get("video_id") not in video_ids, posts) cur_v_ids = set(map(lambda x: x.get("video_id"), posts)) difference = cur_v_ids.difference(video_ids) difference = filter( lambda x: not db.is_post_video_id_present(x), difference ) log.info("New posts: %s" % len(difference)) if difference: map(lambda x: video_ids.add(x), difference) all_posts.extend([el for el in posts if el['video_id'] in difference]) elif len(video_ids) > 0: break log.info("will process %s posts..." % len(all_posts)) rtrv = Retriever() for post in rtrv.process_subreddit(all_posts, params): db.save_post(post, SRC_SEARCH) db.add_search_params(name, params, rtrv.statistic) return jsonify(**{"ok": True, "name": name})
def ex_search(): if request.method == "POST": q = request.form.get("q") result = reddit_search(q) if len(result): return render_template( "ex_search.html", **{ "heads": result[0].keys(), "posts": result, "content_present": True, "count": len(result) }) return render_template("ex_search.html", **{"content_present": False})