def searcha(): ''' Responde las peticiones de busqueda por ajax ''' data=request.form.get("filters",None) #puede venir query/filtros:... o solo query query,filters=(None,None) if not data else data.split("/",1) if "/" in data else (data,None) if not query: form_info = request.form logging.error("Invalid data for AJAX search request.") return jsonify({}) #para que funcionen la busqueda cuando vienen varias palabras query=query.replace("_"," ") last_items = None try: last_items = b64decode(str(request.form["last_items"]) if "last_items" in request.form else "", "-_") if last_items: last_items = unpack("%dh"%(len(last_items)/2), last_items) except BaseException as e: logging.error("Error parsing last_items information from request.") dict_filters, has_changed = url2filters(filters) prepare_args(query, dict_filters) search_results = search_files(query, dict_filters, min_results=request.args.get("min_results",0), last_items=last_items or [], extra_wait_time=300) search_results["files"] = render_template('files/file.html',files=search_results["files"]) return jsonify(search_results)
def current_user(cls, userid=None): anonymous = False user_changed = False if userid is None: # Usuario anónimo data = session.get("user", None) anonymous = True else: # Usuario logeado data = session.get("user", None) if data is None or data.get("anonymous", False): # Usuario sin sesión data = usersdb.find_userid(userid) user_changed = True if data is None: # Sin datos de usuario, datos de usuario anónimo data = {"anonymous": True} anonymous = True user_changed = True elif not anonymous: # Si tengo datos y no se si soy anónimo, compruebo anonymous = data.get("anonymous", False) if user_changed: # Los datos de usuario han cambiado data["session_id"] = cls.generate_session_id() data["session_ip"] = md5(request.remote_addr or "127.0.0.1").hexdigest() session["user"] = data # Guardamos en sesión if "_id" in data: del session["user"]["_id"] a = AnonymousUser(data) if anonymous else User(userid, data) # TODO(felipe): borrar con error solucionado if a.id < 0 and not anonymous: logging.error("Inconsistencia de usuario logeado id negativo.", extra=locals()) a = AnonymousUser(data) return a
def set_file_vote(self, file_id, user, lang, vote): ''' Guarda el voto en la colección y actualiza el archivo correspondiente con los nuevos datos ''' data = { "u": user.id, "k": 1 if vote == 1 else -1, "d": datetime.utcnow(), "l": lang, } # TODO(felipe): borrar con error solucionado if user.id < 0 and user.is_authenticated(): logging.error("Inconsistencia de usuario votando logeado id negativo.", extra=locals()) else: if user.is_authenticated(): data["_id"] = "%s_%s" % (mid2hex(file_id), user.id) self.user_conn.users.vote.update( {"_id": data["_id"]}, data, upsert=True) else: data["_id"] = "%s:%s" % (mid2hex(file_id), user.session_ip) self.user_conn.users.vote.update( {"_id": data["_id"], "u": data["u"]}, data, upsert=True) # Para cada idioma guarda el karma, la cuenta total y la suma map_function = Code(''' function() { emit(this.l,{ k:this.k, c:new Array((this.k>0)?1:0,(this.k<0)?1:0), s:new Array((this.k>0)?this.k:0,(this.k<0)?this.k:0) }) }''') # Suma todo y aplica la funcion 1/1+E^(-X) para que el valor este entre 0 y 1 reduce_function = Code(''' function(lang, vals) { c=new Array(0,0); s=new Array(0,0); for (var i in vals) { c[0]+=vals[i].c[0]; c[1]+=vals[i].c[1]; s[0]+=vals[i].s[0]; s[1]+=vals[i].s[1]; } return {t:1/(1+Math.exp(-((s[0]*c[0]+s[1]*c[1])/(c[0]+c[1])))), c:c, s:s}; }''') # Tercer parametro para devolverlo en vez de generar una coleccion nueva votes = self.user_conn.users.vote.map_reduce( map_function, reduce_function, {"inline": 1}, query = {"_id": {"$regex": "^%s" % mid2hex(file_id)}} ) # Devolver un diccionario de la forma idioma:valores data = {values["_id"]: values["value"] for values in votes["results"]} self.user_conn.end_request() return data
def vote(): ''' Gestiona las votaciones de archivos ''' ok = False try: file_id = url2mid(request.form.get("file_id", None)) server = int(request.form.get("server", 0)) vote = int(request.form.get("vote", 0)) if server > 1 and file_id and vote in (0, 1): file_info = usersdb.set_file_vote(file_id, current_user, g.lang, vote) filesdb.update_file({ "_id": file_id, "vs": file_info, "s": server }, direct_connection=True) ok = True except BaseException as e: logging.error("Error on vote.") response = make_response("true" if ok else "false") response.mimetype = "application/json" return response
def vote(vtype): g.must_cache = 0 result = {} # don't allow bots to access this feature if g.search_bot: logging.warn("Bot is trying to access to file information.") return abort(404) # check referrer's domain matches this request's domain referrer = urllib2.unquote(request.referrer).decode("utf-8") if not referrer.startswith(request.url_root): logging.warn("Don't allows votes from %s."%referrer) return abort(404) # get data from request try: # get file id from referrer url filemid = fileurl2mid(referrer) # get user's ip and calculates a unique id for this file and user ip = (request.headers.getlist("X-Forwarded-For") or [request.remote_addr])[0] userid = hashlib.sha1(str(filemid)+"_"+ip).hexdigest()[:10] except BaseException as e: logging.warn("Error parsing information from request.") return jsonify(result) if not vtype in VOTES: logging.warn("Wrong vote type: %s."%unicode(vtype)) return jsonify(result) try: # save user vote updated_votes = torrentsdb.save_vote(filemid, userid, vtype) filesdb.update_file({"_id":filemid, "vs.u":Counter(updated_votes.itervalues())}) result["user"] = vtype result["ret"] = ["report", _("Your report has been registered."), "info"] except BaseException as e: logging.warn("Error registering vote.") return jsonify(result) try: f = filesdb.get_file(filemid, "1") rate = rate_torrent(f) result["votes"] = (rate["votes"].get(VERIFIED_VOTE,0), sum(value for vtype, value in rate["votes"].iteritems() if vtype!=VERIFIED_VOTE)) if "flag" in rate: result["flag"] = rate["flag"] result['flag'][1] = _(result['flag'][1]) # translate flag text result["rating"] = int(round(rate["rating"]*5)) except BaseException as e: logging.error("Error retrieving file information: %s."%str(filemid)) return jsonify(result)
def get_last_items(): last_items = [] try: last_items = urlsafe_b64decode(str(request.args.get("p", ""))) if last_items: last_items = unpack("%dh" % (len(last_items) / 2), last_items) except BaseException as e: last_items = [] logging.error("Error parsing last_items information from request.") return last_items
def init_g(app): g.license_name = "torrents" g.tos_link = "http://torrents.com/legal#tos" g.privacy_link = "http://torrents.com/legal#privacy" g.analytics_code = """<script type="text/javascript"> var _gaq = _gaq || []; _gaq.push(['_setAccount', 'UA-38333996-2']); _gaq.push(['_trackPageview']); (function() { var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true; ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js'; var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s); })(); </script>""" # caracteristicas del cliente g.search_bot = is_search_bot() # peticiones en modo preproduccion g.beta_request = request.url_root[request.url_root.index("//")+2:].startswith("beta.") # endpoint de entrada (usado en la página de error) if app.config["APPWEB_MODE"] == "search": g.home_route = "files.home" elif app.config["APPWEB_MODE"] == "extras": g.home_route = "extras.home" else: logging.error(u"APPWEB_MODE no especificado en la configuración") g.home_route = "files.home" # prefijo para los contenidos estáticos if g.beta_request: app_static_prefix = app.static_url_path else: app_static_prefix = app.config["STATIC_PREFIX"] or app.static_url_path g.static_prefix = app.assets.url = app_static_prefix g.keywords = set() g.args = {} g.page_description=g.title="" g.categories = (('movies', {"q": "movie"}), ('games', {"q": "game"}), ('tv', {"q": "series"}), ('music', {"q": "audio"}), ('anime', {"q": "anime"}), ('books', {"q": "ebook"}), ('adult', {"q": "p**n"}), ('software', {"q": "software"}), ('mobile', {"q": "mobile"}), ('pictures', {"q": "image"}) )
def save_visited(files): ''' Recibe una lista de resultados de fill_data y guarda las url que sean de web ''' if not g.search_bot and feedbackdb.initialized: result=[{"_id": data['urls'][0], "type":src} for afile in files for src, data in afile['view']['sources'].iteritems() if data['urls'] and data['icon'] in ['web','torrent']] if result: try: feedbackdb.visited_links(result) except: logging.error("Error saving visited links.")
def searcha(): ''' Responde las peticiones de busqueda por ajax ''' query=request.form.get("query",None) filetype=request.form.get("filetype",None) if not query: #si no se ha buscado nada se manda al inicio logging.error("Invalid data for AJAX search request.") return jsonify({}) query = query.replace("_"," ") if query is not None else None #para que funcionen la busqueda cuando vienen varias palabras filters = {"src":"torrent"} ori_query = query _in = [i for i, v in enumerate(g.categories) if v[0] == filetype] if filetype and _in : _id = _in[0] if "t" in g.categories[_id][1]: filters["type"] = g.categories[_id][1]['t'] if "q" in g.categories[_id][1]: _type = g.categories[_id][1]['q'] query = "%s (%s)" % (query, _type) args = filters.copy() args["q"] = ori_query if ori_query != query: args['type'] = _type g.args=args last_items = [] try: last_items = b64decode(str(request.form["last_items"]) if "last_items" in request.form else "", "-_") if last_items: last_items = unpack("%dh"%(len(last_items)/2), last_items) except BaseException as e: logging.error("Error parsing last_items information from request.") sure = False total_found=0 search_results = search_files(query, filters, min_results=0, last_items=last_items, non_group=True, order=("@weight*(r+10)", "e DESC, ok DESC, r2 DESC, fs DESC, uri1 DESC", "@weight*(r+10)"), weight_processor=weight_processor, tree_visitor=tree_visitor) response = make_response(render_template('file.html',files=[torrents_data(afile) for afile in search_results["files"]])) del search_results["files"] response.headers["X-JSON"]=json.dumps(search_results) return response
def update_file(self, data, remove=None, direct_connection=False, update_sphinx=True): ''' Actualiza los datos del fichero dado. @type data: dict @param data: Diccionario con los datos del fichero a guardar. Se debe incluir '_id', y es recomendable incluir 's' por motivos de rendimiento. @type remove: iterable @param remove: Lista de campos a quitar del registro. @type direct_connection: bool @param direct_connection: Especifica si se crea una conexión directa, ineficiente e independiente al foo primario. @type update_sphinx: bool @param update_sphinx: si se especifica bl, de este parámetro depende el si se conecta al sphinx para actualizarlo ''' update = {"$set":data.copy()} if remove is not None: update["$unset"] = {} for rem in remove: if rem in update["$set"]: del update["$set"][rem] update["$unset"][rem] = 1 fid = hex2mid(data["_id"]) _indir = self.server_conn.foofind.indir.find_one({"_id": fid}) if _indir and "t" in _indir: fid = hex2mid(_indir['t']) if "s" in data: server = str(data["s"]) else: try: server = str(self.get_file(fid, bl=None)["s"]) except (TypeError, KeyError) as e: logging.error("Se ha intentado actualizar un fichero que no se encuentra en la base de datos", extra=data) raise if update_sphinx and "bl" in data: try: file_name = i["fn"].itervalues().next()["n"] except: file_name = None block_files(mongo_ids=((i["_id"],file_name),), block=data["bl"]) for i in ("_id", "s"): if i in update["$set"]: del update["$set"][i] # TODO: update cache self.servers_conn[server].foofind.foo.update({"_id":fid}, update)
def init_g(app): if not g.lang in current_app.config["APPWEB_AVAILABLE_LANGS"]: g.lang = "en" g.design = bool(request.form.get("wklk", False)) g.license_name = license_name = "foofind" if "foofind" in request.url_root else "blubster" g.version_code = request.form.get("version",None) g.analytics_code = current_app.config["ANALYTICS_CODE"][license_name] % {"client_version":g.version_code or ""} # caracteristicas del cliente g.search_bot = is_search_bot() # peticiones en modo preproduccion g.beta_request = request.url_root[request.url_root.index("//")+2:].startswith("beta.") # endpoint de entrada (usado en la página de error) if current_app.config["APPWEB_MODE"] == "search": g.home_route = "files.home" elif current_app.config["APPWEB_MODE"] == "extras": g.home_route = "extras.home" else: logging.error(u"APPWEB_MODE no especificado en la configuración") g.home_route = "files.home" # prefijo para los contenidos estáticos if g.beta_request: app_static_prefix = current_app.static_url_path else: app_static_prefix = current_app.config["STATIC_PREFIX"] or current_app.static_url_path g.static_prefix = current_app.assets.url = app_static_prefix g.keywords = set() g.args = {} g.page_description=g.title="" g.tos_link = current_app.config["TOS_LINK"][license_name] g.privacy_link = current_app.config["PRIVACY_LINK"][license_name] g.categories = (('video',{"q":["video"]}), ('audio',{"q":["audio"]}), ('document',{"q":["document"]}), ('image',{"q":["image"]}), ('software',{"q":["software"]}) ) g.downloader_properties = local_cache["downloader_properties"]
def get_downloader_properties(base_path, downloader_files_builds): # source code information properties = {"common":{"base_path": base_path}} # builds information for build, info in downloader_files_builds.iteritems(): try: with open(os.path.join(base_path, info["metadata"]), "r") as f: metadata = json.load(f) properties[build] = info.copy() properties[build].update(metadata) properties[build]["length"] = os.path.getsize(os.path.join(base_path, properties[build]["main"])) except: logging.error("Error checking downloader files.") return properties
def complaint(): ''' Procesa los datos del formulario para reportar enlaces. ''' try: form = ReportLinkForm(request.form) if form.validate(): urlreported = "/download/"+form.file_id.data+"/" pagesdb.create_complaint(dict([("linkreported","-"),("urlreported",urlreported),("ip",request.remote_addr)]+[(field.name,field.data) for field in form])) response = make_response("true") else: response = make_response(repr(form.errors.keys())) except BaseException as e: logging.error("Error on file complaint.") response = make_response("false") response.mimetype = "application/json" return response
def init_g(app): g.wakalaka = bool(request.form.get("wklk", False)) g.license_name = "foofind" if "foofind" in request.url_root else "blubster" # caracteristicas del cliente g.search_bot = is_search_bot() # peticiones en modo preproduccion g.beta_request = request.url_root[request.url_root.index("//") + 2:].startswith("beta.") # endpoint de entrada (usado en la página de error) if app.config["APPWEB_MODE"] == "search": g.home_route = "files.home" elif app.config["APPWEB_MODE"] == "extras": g.home_route = "extras.home" else: logging.error(u"APPWEB_MODE no especificado en la configuración") g.home_route = "files.home" # prefijo para los contenidos estáticos if g.beta_request: app_static_prefix = app.static_url_path else: app_static_prefix = app.config["STATIC_PREFIX"] or app.static_url_path g.static_prefix = app.assets.url = app_static_prefix g.keywords = set() g.args = {} g.page_description = g.title = "" g.categories = (('video', { "t": "video" }), ('audio', { "t": "audio" }), ('document', { "t": "document" }), ('image', { "t": "image" }), ('software', { "t": "software" }))
def vote(): ''' Gestiona las votaciones de archivos ''' ok = False try: file_id=url2mid(request.form.get("file_id",None)) server=int(request.form.get("server",0)) vote=int(request.form.get("vote",0)) if server>1 and file_id and vote in (0,1): file_info = usersdb.set_file_vote(file_id, current_user, g.lang, vote) filesdb.update_file({"_id":file_id, "vs":file_info, "s":server}, direct_connection=True) ok = True except BaseException as e: logging.error("Error on vote.") response = make_response("true" if ok else "false") response.mimetype = "application/json" return response
def sitemap(): urlformat = current_app.config["FILES_SITEMAP_URL"].get(g.domain, None) rules = [] if urlformat: servers = filesdb.get_servers() threads = [ threading.Thread( target = lambda x: rules.append( gensitemap(x, urlformat ) ), args = ( server, ) ) for server in servers ] for t in threads: t.start() for t in threads: if t.is_alive(): t.join() if None in rules: rules = [rule for rule in rules if rule!=None] logging.error("Hay sitemaps no disponibles", extra={"servers":servers, "rules":rules}) return render_template('sitemap.xml', rules=rules)
def init_g(app): g.design = bool(request.form.get("wklk", False)) g.license_name = "foofind" if "foofind" in request.url_root else "blubster" # caracteristicas del cliente g.search_bot = is_search_bot() # peticiones en modo preproduccion g.beta_request = request.url_root[request.url_root.index("//")+2:].startswith("beta.") # endpoint de entrada (usado en la página de error) if app.config["APPWEB_MODE"] == "search": g.home_route = "files.home" elif app.config["APPWEB_MODE"] == "extras": g.home_route = "extras.home" else: logging.error(u"APPWEB_MODE no especificado en la configuración") g.home_route = "files.home" # prefijo para los contenidos estáticos if g.beta_request: app_static_prefix = app.static_url_path else: app_static_prefix = app.config["STATIC_PREFIX"] or app.static_url_path g.static_prefix = app.assets.url = app_static_prefix g.keywords = set() g.args = {} g.page_description=g.title="" g.tos_link = app.config["TOS_LINK"] g.privacy_link = app.config["PRIVACY_LINK"] g.categories = (('video',{"t":"video"}), ('audio',{"t":"audio"}), ('document',{"t":"document"}), ('image',{"t":"image"}), ('software',{"t":"software"}) )
def get_last_files(self, n=25): ''' Obtiene los últimos n ficheros indexados. ''' p = Parallel(( (self._get_last_files_indir_offset, (), {}), (self._get_last_files_foo_offset, (), {}), (self._get_last_files_from_foo, (n+self._last_files_max_offset, self._last_files_initial_skip), {}), )) p.join(1) # Parallel.join_and_terminate(1) evitaría los end_request: hay que # dejar que los hilos de parallel terminen if p.is_alive(): raise MongoTimeout, "Timeout in get_last_files" elif p.failed(): logging.error( "Some error found in Parallel tasks on get_last_files", extra={ "output": p.output, "exceptions": p.exceptions }) cache.throw_fallback() inoff, fooff, lf = p.output fooff -= self._last_files_initial_skip/self._last_files_per_second if inoff < fooff: # Si indir está más desactualizado que el foo offset = (fooff-inoff)*self._last_files_per_second if offset > self._last_files_max_offset: logging.error( "Indir and last foo's replicas are too desynchronized, get_last_files will fallback", extra={ "file_offset":offset, "file_offset_maximum":self._last_files_max_offset, "replication_time_indir":inoff, "replication_time_foo":fooff, "replication_time_offset":fooff-inoff, "replication_time_maximum_offset (calculated)": self._last_files_max_offset/float(self._last_files_per_second) }) cache.throw_fallback() return lf[offset:offset+n] return lf[:n]
def complaint(): ''' Procesa los datos del formulario para reportar enlaces. ''' try: form = ReportLinkForm(request.form) if form.validate(): urlreported = "/download/" + form.file_id.data + "/" pagesdb.create_complaint( dict([("linkreported", "-"), ("urlreported", urlreported), ("ip", request.remote_addr)] + [(field.name, field.data) for field in form])) response = make_response("true") else: response = make_response(repr(form.errors.keys())) except BaseException as e: logging.error("Error on file complaint.") response = make_response("false") response.mimetype = "application/json" return response
def searcha(): ''' Responde las peticiones de busqueda por ajax ''' query = request.form.get("query", None) filetype = request.form.get("filetype", None) if not query: #si no se ha buscado nada se manda al inicio logging.error("Invalid data for AJAX search request.") return jsonify({}) query = query.replace( "_", " " ) if query is not None else None #para que funcionen la busqueda cuando vienen varias palabras filters = {"src": "torrent"} ori_query = query _in = [i for i, v in enumerate(g.categories) if v[0] == filetype] if filetype and _in: _id = _in[0] if "t" in g.categories[_id][1]: filters["type"] = g.categories[_id][1]['t'] if "q" in g.categories[_id][1]: _type = g.categories[_id][1]['q'] query = "%s (%s)" % (query, _type) args = filters.copy() args["q"] = ori_query if ori_query != query: args['type'] = _type g.args = args last_items = [] try: last_items = b64decode( str(request.form["last_items"]) if "last_items" in request.form else "", "-_") if last_items: last_items = unpack("%dh" % (len(last_items) / 2), last_items) except BaseException as e: logging.error("Error parsing last_items information from request.") sure = False total_found = 0 search_results = search_files( query, filters, min_results=0, last_items=last_items, non_group=True, order=("@weight*(r+10)", "e DESC, ok DESC, r2 DESC, fs DESC, uri1 DESC", "@weight*(r+10)"), weight_processor=weight_processor, tree_visitor=tree_visitor) response = make_response( render_template( 'file.html', files=[torrents_data(afile) for afile in search_results["files"]])) del search_results["files"] response.headers["X-JSON"] = json.dumps(search_results) return response
def prepare_data(f, text=None, ntts=[], details=False, current_category=None): try: return torrents_data(secure_fill_data(f,text,ntts), details, current_category) except BaseException as e: logging.error("Error retrieving torrent data.") return None
def create_app(config=None, debug=False): ''' Inicializa la aplicación Flask. Carga los siguientes módulos: - index: página de inicio - page: páginas estáticas - user: gestión del usuario - files: búsqueda y obtención de ficheros - status: servicio de monitorización de la aplicación Y además, inicializa los siguientes servicios: - Configuración: carga valores por defecto y modifica con el @param config - Web Assets: compilación y compresión de recursos estáticos - i18n: detección de idioma en la URL y catálogos de mensajes - Cache y auth: Declarados en el módulo services - Files: Clases para acceso a datos ''' app = Flask(__name__) app.config.from_object(defaults) app.debug = debug # Configuración if config: app.config.from_object(config) # Modo de appweb appmode = app.config["APPWEB_MODE"] # Gestión centralizada de errores if app.config["SENTRY_DSN"]: sentry.init_app(app) logging.getLogger().setLevel(logging.DEBUG if debug else logging.INFO) # Configuración dependiente de la versión del código revision_filename_path = os.path.join(os.path.dirname(app.root_path), "revision") if os.path.exists(revision_filename_path): f = open(revision_filename_path, "r") data = f.read() f.close() revisions = tuple( tuple(i.strip() for i in line.split("#")[0].split()) for line in data.strip().split("\n") if line.strip() and not line.strip().startswith("#")) revision_hash = md5(data).hexdigest() app.config.update( CACHE_KEY_PREFIX = "%s%s/" % ( app.config["CACHE_KEY_PREFIX"] if "CACHE_KEY_PREFIX" in app.config else "", revision_hash ), REVISION_HASH = revision_hash, REVISION = revisions ) else: app.config.update( REVISION_HASH = None, REVISION = () ) # Registra filtros de plantillas register_filters(app) # Registra valores/funciones para plantillas app.jinja_env.globals["u"] = u # proteccion CSRF csrf.init_app(app) # Blueprints if appmode == "search": app.register_blueprint(files) elif appmode == "extras": app.register_blueprint(extras) else: logging.error("No se ha especificado modo en la configuración. Blueprints sin cargar.") # Web Assets dir_static = app.static_folder # shortcut scss.config.LOAD_PATHS = [os.path.abspath('%s/../..' % dir_static)] for subdir in ['%s/%s' % (dir_static, x) for x in ['gen', 'torrents/gen']]: if not os.path.isdir(subdir): os.makedirs(subdir) app.assets = Environment(app) app.assets.debug = app.debug app.assets.versions = "hash" register_filter(JsSlimmer) register_filter(CssSlimmer) app.assets.register( 'css_torrents', Bundle('torrents/css/torrents.scss', filters='pyscss', output='torrents/gen/torrents.css', debug=False), filters='css_slimmer', output='torrents/gen/torrents.css') if appmode == "search": app.assets.register( 'js_appweb', Bundle('prototype.js', 'event.simulate.js', 'chosen.proto.min.js','appweb.js', filters='rjsmin', output='gen/appweb.js')) else: app.assets.register( 'js_appweb', Bundle('prototype.js', filters='rjsmin', output='gen/appweb.js')) # Traducciones babel.init_app(app) @babel.localeselector def get_locale(): return "en" # Autenticación auth.setup_app(app) auth.user_loader(User.current_user) auth.anonymous_user = User.current_user # Cache cache.init_app(app) # Acceso a bases de datos filesdb.init_app(app) pagesdb.init_app(app) feedbackdb.init_app(app) entitiesdb.init_app(app) usersdb.init_app(app) plugindb.init_app(app) # Servicio de búsqueda @app.before_first_request def init_process(): if not eventmanager.is_alive(): # Fallback inicio del eventManager eventmanager.start() # Profiler profiler.init_app(app, feedbackdb) eventmanager.once(searchd.init_app, hargs=(app, filesdb, entitiesdb, profiler)) # Refresco de conexiones eventmanager.once(filesdb.load_servers_conn) eventmanager.interval(app.config["FOOCONN_UPDATE_INTERVAL"], filesdb.load_servers_conn) eventmanager.interval(app.config["FOOCONN_UPDATE_INTERVAL"], entitiesdb.connect) @app.url_value_preprocessor def pull_lang_code(endpoint, values): if values is None: g.lang = "en" else: g.lang = values.pop('lang', "en") @app.url_defaults def add_language_code(endpoint, values): if not 'lang' in values and app.url_map.is_endpoint_expecting(endpoint, 'lang'): values['lang'] = g.lang @app.before_request def before_request(): # No preprocesamos la peticiones a static if request.path.startswith("/static/"): return init_g(app) # ignora peticiones sin blueprint if request.blueprint is None and len(request.path)>1 and request.path.endswith("/"): if "?" in request.url: root = request.url_root[:-1] path = request.path.rstrip("/") query = request.url.decode("utf-8") query = query[query.find(u"?"):] return redirect(root+path+query, 301) return redirect(request.url.rstrip("/"), 301) @app.after_request def after_request(response): response.headers["X-UA-Compatible"] = "IE=edge" return response # Páginas de error errors = { 404: ("Page not found", "The requested address does not exists."), 410: ("Page not available", "The requested address is no longer available."), 500: ("An error happened", "We had some problems displaying this page. Maybe later we can show it to you."), 503: ("Service unavailable", "This page is temporarily unavailable. Please try again later.") } @allerrors(app, 400, 401, 403, 404, 405, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 500, 501, 502, 503) def all_errors(e): error = e.code if hasattr(e,"code") else 500 title, description = errors[error if error in errors else 500] init_g(app) return render_template('error.html', code=str(error), title=title, description=description), error return app
def build_source_links(f): ''' Construye los enlaces correctamente ''' def get_domain(src): ''' Devuelve el dominio de una URL ''' url_parts=urlparse(src).netloc.split('.') i=len(url_parts)-1 if len(url_parts[i])<=2 and len(url_parts[i-1])<=3: return url_parts[i-2]+'.'+url_parts[i-1]+'.'+url_parts[i] else: return url_parts[i-1]+'.'+url_parts[i]; f['view']['action']='download' f['view']['sources']={} max_weight=0 icon="" # agrupación de origenes source_groups = {} file_sources = f['file']['src'].items() file_sources.sort(key=lambda x:x[1]["t"]) for hexuri,src in file_sources: if not src.get('bl',None) in (0, None): continue url_pattern=downloader=join=False count=0 part=url="" source_data=g.sources[src["t"]] if "t" in src and src["t"] in g.sources else None if source_data is None: #si no existe el origen del archivo logging.error("El fichero contiene un origen inexistente en la tabla \"sources\": %s" % src["t"], extra={"file":f}) if feedbackdb.initialized: feedbackdb.notify_source_error(f['file']["_id"], f['file']["s"]) continue elif "crbl" in source_data and int(source_data["crbl"])==1: #si el origen esta bloqueado continue elif "w" in source_data["g"] or "f" in source_data["g"] or "s" in source_data["g"]: #si es descarga directa o streaming link_weight=1 tip=source_data["d"] icon="web" source_groups[icon] = tip source=get_domain(src['url']) if "f" in source_data["g"] else source_data["d"] url=src['url'] if "url_pattern" in source_data and not url.startswith(("https://","http://","ftp://")): url_pattern=True #en caso de duda se prefiere streaming if "s" in source_data["g"]: f['view']['action']="listen" if f['view']['ct']==CONTENT_AUDIO else 'watch' link_weight*=2 #torrenthash antes de torrent porque es un caso especifico elif source_data["d"]=="BitTorrentHash": downloader=True link_weight=0.7 if 'torrent:tracker' in f['file']['md'] or 'torrent:trackers' in f['file']['md'] else 0.1 tip="Torrent MagnetLink" source="tmagnet" icon="torrent" if not icon in source_groups: source_groups[icon] = tip # magnet link tiene menos prioridad para el texto join=True count=int(src['m']) part="xt=urn:btih:"+src['url'] if 'torrent:tracker' in f['file']['md']: part += unicode('&tr=' + urllib.quote_plus(u(f['file']['md']['torrent:tracker']).encode("UTF-8")), "UTF-8") elif 'torrent:trackers' in f['file']['md']: trackers = f['file']['md']['torrent:trackers'] if isinstance(trackers, basestring): part += unicode("".join('&tr='+urllib.quote_plus(tr) for tr in u(trackers).encode("UTF-8").split(" ")), "UTF-8") elif "t" in source_data["g"]: downloader=True link_weight=0.8 url=src['url'] if "url_pattern" in source_data and not url.startswith(("https://","http://","ftp://")): url_pattern=True tip=source=get_domain(source_data["url_pattern"]%url) else: tip=source=get_domain(src['url']) icon="torrent" source_groups[icon] = tip elif source_data["d"]=="Gnutella": link_weight=0.2 tip="Gnutella" source=icon="gnutella" part="xt=urn:sha1:"+src['url'] join=True count=int(src['m']) source_groups[icon] = tip elif source_data["d"]=="eD2k": downloader=True link_weight=0.1 tip="eD2k" source=icon="ed2k" url="ed2k://|file|"+f['view']['pfn']+"|"+str(f['file']['z'] if "z" in f["file"] else 1)+"|"+src['url']+"|/" count=int(src['m']) source_groups[icon] = tip elif source_data["d"]=="Tiger": link_weight=0 tip="Gnutella" source=icon="gnutella" part="xt=urn:tiger:"+src['url'] join=True elif source_data["d"]=="MD5": link_weight=0 tip="Gnutella" source=icon="gnutella" part="xt=urn:md5:"+src['url'] source_groups[icon] = tip join=True else: continue if source in f['view']['sources']: view_source = f['view']['sources'][source] else: view_source = f['view']['sources'][source] = {} view_source.update(source_data) if 'downloader' in view_source: if downloader: view_source['downloader']=1 else: view_source['downloader']=1 if downloader else 0 view_source['tip']=tip view_source['icon']=icon view_source['icons']=source_data.get("icons",False) view_source['join']=join view_source['source']="streaming" if "s" in source_data["g"] else "direct_download" if "w" in source_data["g"] else "P2P" if "p" in source_data["g"] else "" #para no machacar el numero si hay varios archivos del mismo source if not 'count' in view_source or count>0: view_source['count']=count if not "parts" in view_source: view_source['parts']=[] if not 'urls' in view_source: view_source['urls']=[] if part: view_source['parts'].append(part) if url: if url_pattern: view_source['urls']=[source_data["url_pattern"]%url] f['view']['source_id']=url view_source["pattern_used"]=True elif not "pattern_used" in view_source: view_source['urls'].append(url) if source_data["d"]!="eD2k": view_source['count']+=1 if link_weight>max_weight: max_weight = link_weight f['view']['source'] = source f['view']['source_groups'] = sorted(source_groups.items()) if "source" not in f["view"]: raise FileNoSources if icon!="web": for src,info in f['view']['sources'].items(): if info['join']: f['view']['sources'][src]['urls'].append("magnet:?"+"&".join(info['parts'])+"&dn="+f['view']['pfn']+("&xl="+str(f['file']['z']) if 'z' in f['file'] else "")) elif not 'urls' in info: del(f['view']['sources'][src])
def create_app(config=None, debug=False): ''' Inicializa la aplicación Flask. Carga los siguientes módulos: - index: página de inicio - page: páginas estáticas - user: gestión del usuario - files: búsqueda y obtención de ficheros - status: servicio de monitorización de la aplicación Y además, inicializa los siguientes servicios: - Configuración: carga valores por defecto y modifica con el @param config - Web Assets: compilación y compresión de recursos estáticos - i18n: detección de idioma en la URL y catálogos de mensajes - Cache y auth: Declarados en el módulo services - Files: Clases para acceso a datos ''' app = Flask(__name__) app.config.from_object(defaults) app.debug = debug # Configuración if config: app.config.from_object(config) # Modo de appweb appmode = app.config["APPWEB_MODE"] # Gestión centralizada de errores if app.config["SENTRY_DSN"]: sentry.init_app(app) logging.getLogger().setLevel(logging.DEBUG if debug else logging.INFO) # Configuración dependiente de la versión del código revision_filename_path = os.path.join(os.path.dirname(app.root_path), "revision") if os.path.exists(revision_filename_path): f = open(revision_filename_path, "r") data = f.read() f.close() revisions = tuple( tuple(i.strip() for i in line.split("#")[0].split()) for line in data.strip().split("\n") if line.strip() and not line.strip().startswith("#")) revision_hash = md5(data).hexdigest() app.config.update(CACHE_KEY_PREFIX="%s%s/" % (app.config["CACHE_KEY_PREFIX"] if "CACHE_KEY_PREFIX" in app.config else "", revision_hash), REVISION_HASH=revision_hash, REVISION=revisions) else: app.config.update(REVISION_HASH=None, REVISION=()) # Registra filtros de plantillas register_filters(app) # Registra valores/funciones para plantillas app.jinja_env.globals["u"] = u # proteccion CSRF csrf.init_app(app) # Blueprints if appmode == "search": app.register_blueprint(files) elif appmode == "extras": app.register_blueprint(extras) else: logging.error( "No se ha especificado modo en la configuración. Blueprints sin cargar." ) # Web Assets scss.config.LOAD_PATHS = [ os.path.dirname(os.path.dirname(app.static_folder)) ] if not os.path.isdir(app.static_folder + "/gen"): os.mkdir(app.static_folder + "/gen") if not os.path.isdir(app.static_folder + "/blubster/gen"): os.mkdir(app.static_folder + "/blubster/gen") if not os.path.isdir(app.static_folder + "/foofind/gen"): os.mkdir(app.static_folder + "/foofind/gen") app.assets = assets = Environment(app) assets.debug = app.debug assets.versions = "hash" register_filter(JsSlimmer) register_filter(CssSlimmer) assets.register('css_blubster', Bundle('blubster/css/blubster.scss', filters='pyscss', output='blubster/gen/blubster.css', debug=False, depends='appweb.scss'), filters='css_slimmer', output='blubster/gen/blubster.css') assets.register('css_foofind', Bundle('foofind/css/foofind.scss', filters='pyscss', output='foofind/gen/foofind.css', debug=False), filters='css_slimmer', output='foofind/gen/foofind.css') # Traducciones babel.init_app(app) @babel.localeselector def get_locale(): return "en" # Autenticación auth.setup_app(app) auth.user_loader(User.current_user) auth.anonymous_user = User.current_user # Cache cache.init_app(app) # Acceso a bases de datos filesdb.init_app(app) pagesdb.init_app(app) feedbackdb.init_app(app) entitiesdb.init_app(app) usersdb.init_app(app) plugindb.init_app(app) # Servicio de búsqueda @app.before_first_request def init_process(): if not eventmanager.is_alive(): # Fallback inicio del eventManager eventmanager.start() # Profiler profiler.init_app(app, feedbackdb) eventmanager.once(searchd.init_app, hargs=(app, filesdb, entitiesdb, profiler)) # Refresco de conexiones eventmanager.once(filesdb.load_servers_conn) eventmanager.interval(app.config["FOOCONN_UPDATE_INTERVAL"], filesdb.load_servers_conn) eventmanager.interval(app.config["FOOCONN_UPDATE_INTERVAL"], entitiesdb.connect) @app.url_value_preprocessor def pull_lang_code(endpoint, values): if values is None: g.lang = "en" else: g.lang = values.pop('lang', "en") @app.url_defaults def add_language_code(endpoint, values): if not 'lang' in values and app.url_map.is_endpoint_expecting( endpoint, 'lang'): values['lang'] = g.lang @app.before_request def before_request(): # No preprocesamos la peticiones a static if request.path.startswith("/static/"): return init_g(app) # ignora peticiones sin blueprint if request.blueprint is None and len( request.path) > 1 and request.path.endswith("/"): if "?" in request.url: root = request.url_root[:-1] path = request.path.rstrip("/") query = request.url.decode("utf-8") query = query[query.find(u"?"):] return redirect(root + path + query, 301) return redirect(request.url.rstrip("/"), 301) @app.after_request def after_request(response): response.headers["X-UA-Compatible"] = "IE=edge" return response # Páginas de error errors = { 404: ("Page not found", "The requested address does not exists."), 410: ("Page not available", "The requested address is no longer available."), 500: ("An error happened", "We had some problems displaying this page. Maybe later we can show it to you." ), 503: ("Service unavailable", "This page is temporarily unavailable. Please try again later.") } @allerrors(app, 400, 401, 403, 404, 405, 408, 409, 410, 411, 412, 413, 414, 415, 416, 417, 418, 500, 501, 502, 503) def all_errors(e): error = e.code if hasattr(e, "code") else 500 title, description = errors[error if error in errors else 500] init_g(app) return render_template('error.html', code=str(error), title=title, description=description), error return app
def get_files(self, ids, servers_known = False, bl = 0): ''' Devuelve los datos de los ficheros correspondientes a los ids dados en formato hexadecimal. @param ids: Lista de identificadores de los ficheros a recuperar. Si server_known es False, es una lista de cadenas. Si server_known es True, es una lista de tuplas, que incluyen el identificador del fichero y el número de servidor. @param servers_known: Indica si la lista de identificadores incluye el servidor donde se encuentra el fichero. @type bl: int o None @param bl: valor de bl para buscar, None para no restringir @rtype generator @return Generador con los documentos de ficheros ''' if not ids: return () sids = defaultdict(list) # si conoce los servidores en los que están los ficheros, # se analiza ids como un iterable (id, servidor, ...) # y evitamos buscar los servidores if servers_known: for x in ids: sids[x[1]].append(hex2mid(x[0])) else: # averigua en qué servidor está cada fichero nindir = self.server_conn.foofind.indir.find({"_id": {"$in": [hex2mid(fid) for fid in ids]}, "s": {"$exists": 1}}) for ind in nindir: indserver = str(int(ind["s"])) # Bug en indir: 's' como float if indserver in self.servers_conn: if "t" in ind: # si apunta a otro id, lo busca en vez del id dado sids[indserver].append(ind["t"]) else: sids[indserver].append(ind["_id"]) self.server_conn.end_request() lsids = len(sids) if lsids == 0: # Si no hay servidores, no hay ficheros return () elif lsids == 1: k, v = sids.iteritems().next() return self._get_server_files((k, v, bl)) else: # crea el pool de hilos si no existe if not self.thread_pool: self.thread_pool = ThreadPool(processes=self.thread_pool_size) # obtiene la información de los ficheros de cada servidor results = [] chunks = self.thread_pool.imap_unordered(self._get_server_files, ((k, v, bl) for k, v in sids.iteritems())) end = time.time()+self.get_files_timeout try: for i in xrange(len(sids)): now = time.time() if now>end: break for r in chunks.next(end-now): results.append(r) except TimeoutError: pass except BaseException as e: logging.error("Error on get_files.") return results