def searcha():
'''
Responde las peticiones de busqueda por ajax
'''
data=request.form.get("filters",None) #puede venir query/filtros:... o solo query
query,filters=(None,None) if not data else data.split("/",1) if "/" in data else (data,None)
if not query:
form_info = request.form
logging.error("Invalid data for AJAX search request.")
return jsonify({})
#para que funcionen la busqueda cuando vienen varias palabras
query=query.replace("_"," ")
last_items = None
try:
last_items = b64decode(str(request.form["last_items"]) if "last_items" in request.form else "", "-_")
if last_items:
last_items = unpack("%dh"%(len(last_items)/2), last_items)
except BaseException as e:
logging.error("Error parsing last_items information from request.")
dict_filters, has_changed = url2filters(filters)
prepare_args(query, dict_filters)
search_results = search_files(query, dict_filters, min_results=request.args.get("min_results",0), last_items=last_items or [], extra_wait_time=300)
search_results["files"] = render_template('files/file.html',files=search_results["files"])
return jsonify(search_results)
def current_user(cls, userid=None):
anonymous = False
user_changed = False
if userid is None: # Usuario anónimo
data = session.get("user", None)
anonymous = True
else: # Usuario logeado
data = session.get("user", None)
if data is None or data.get("anonymous", False): # Usuario sin sesión
data = usersdb.find_userid(userid)
user_changed = True
if data is None: # Sin datos de usuario, datos de usuario anónimo
data = {"anonymous": True}
anonymous = True
user_changed = True
elif not anonymous: # Si tengo datos y no se si soy anónimo, compruebo
anonymous = data.get("anonymous", False)
if user_changed: # Los datos de usuario han cambiado
data["session_id"] = cls.generate_session_id()
data["session_ip"] = md5(request.remote_addr or "127.0.0.1").hexdigest()
session["user"] = data # Guardamos en sesión
if "_id" in data:
del session["user"]["_id"]
a = AnonymousUser(data) if anonymous else User(userid, data)
# TODO(felipe): borrar con error solucionado
if a.id < 0 and not anonymous:
logging.error("Inconsistencia de usuario logeado id negativo.", extra=locals())
a = AnonymousUser(data)
return a
def set_file_vote(self, file_id, user, lang, vote):
'''
Guarda el voto en la colección y actualiza el archivo correspondiente con los nuevos datos
'''
data = {
"u": user.id,
"k": 1 if vote == 1 else -1,
"d": datetime.utcnow(),
"l": lang,
}
# TODO(felipe): borrar con error solucionado
if user.id < 0 and user.is_authenticated():
logging.error("Inconsistencia de usuario votando logeado id negativo.", extra=locals())
else:
if user.is_authenticated():
data["_id"] = "%s_%s" % (mid2hex(file_id), user.id)
self.user_conn.users.vote.update(
{"_id": data["_id"]}, data, upsert=True)
else:
data["_id"] = "%s:%s" % (mid2hex(file_id), user.session_ip)
self.user_conn.users.vote.update(
{"_id": data["_id"], "u": data["u"]},
data, upsert=True)
# Para cada idioma guarda el karma, la cuenta total y la suma
map_function = Code('''
function()
{
emit(this.l,{
k:this.k,
c:new Array((this.k>0)?1:0,(this.k<0)?1:0),
s:new Array((this.k>0)?this.k:0,(this.k<0)?this.k:0)
})
}''')
# Suma todo y aplica la funcion 1/1+E^(-X) para que el valor este entre 0 y 1
reduce_function = Code('''
function(lang, vals)
{
c=new Array(0,0);
s=new Array(0,0);
for (var i in vals)
{
c[0]+=vals[i].c[0];
c[1]+=vals[i].c[1];
s[0]+=vals[i].s[0];
s[1]+=vals[i].s[1];
}
return {t:1/(1+Math.exp(-((s[0]*c[0]+s[1]*c[1])/(c[0]+c[1])))), c:c, s:s};
}''')
# Tercer parametro para devolverlo en vez de generar una coleccion nueva
votes = self.user_conn.users.vote.map_reduce(
map_function,
reduce_function,
{"inline": 1},
query = {"_id": {"$regex": "^%s" % mid2hex(file_id)}}
)
# Devolver un diccionario de la forma idioma:valores
data = {values["_id"]: values["value"] for values in votes["results"]}
self.user_conn.end_request()
return data
def vote():
'''
Gestiona las votaciones de archivos
'''
ok = False
try:
file_id = url2mid(request.form.get("file_id", None))
server = int(request.form.get("server", 0))
vote = int(request.form.get("vote", 0))
if server > 1 and file_id and vote in (0, 1):
file_info = usersdb.set_file_vote(file_id, current_user, g.lang,
vote)
filesdb.update_file({
"_id": file_id,
"vs": file_info,
"s": server
},
direct_connection=True)
ok = True
except BaseException as e:
logging.error("Error on vote.")
response = make_response("true" if ok else "false")
response.mimetype = "application/json"
return response
def vote(vtype):
g.must_cache = 0
result = {}
# don't allow bots to access this feature
if g.search_bot:
logging.warn("Bot is trying to access to file information.")
return abort(404)
# check referrer's domain matches this request's domain
referrer = urllib2.unquote(request.referrer).decode("utf-8")
if not referrer.startswith(request.url_root):
logging.warn("Don't allows votes from %s."%referrer)
return abort(404)
# get data from request
try:
# get file id from referrer url
filemid = fileurl2mid(referrer)
# get user's ip and calculates a unique id for this file and user
ip = (request.headers.getlist("X-Forwarded-For") or [request.remote_addr])[0]
userid = hashlib.sha1(str(filemid)+"_"+ip).hexdigest()[:10]
except BaseException as e:
logging.warn("Error parsing information from request.")
return jsonify(result)
if not vtype in VOTES:
logging.warn("Wrong vote type: %s."%unicode(vtype))
return jsonify(result)
try:
# save user vote
updated_votes = torrentsdb.save_vote(filemid, userid, vtype)
filesdb.update_file({"_id":filemid, "vs.u":Counter(updated_votes.itervalues())})
result["user"] = vtype
result["ret"] = ["report", _("Your report has been registered."), "info"]
except BaseException as e:
logging.warn("Error registering vote.")
return jsonify(result)
try:
f = filesdb.get_file(filemid, "1")
rate = rate_torrent(f)
result["votes"] = (rate["votes"].get(VERIFIED_VOTE,0), sum(value for vtype, value in rate["votes"].iteritems() if vtype!=VERIFIED_VOTE))
if "flag" in rate:
result["flag"] = rate["flag"]
result['flag'][1] = _(result['flag'][1]) # translate flag text
result["rating"] = int(round(rate["rating"]*5))
except BaseException as e:
logging.error("Error retrieving file information: %s."%str(filemid))
return jsonify(result)
def get_last_items():
last_items = []
try:
last_items = urlsafe_b64decode(str(request.args.get("p", "")))
if last_items:
last_items = unpack("%dh" % (len(last_items) / 2), last_items)
except BaseException as e:
last_items = []
logging.error("Error parsing last_items information from request.")
return last_items
def init_g(app):
g.license_name = "torrents"
g.tos_link = "http://torrents.com/legal#tos"
g.privacy_link = "http://torrents.com/legal#privacy"
g.analytics_code = """<script type="text/javascript">
var _gaq = _gaq || [];
_gaq.push(['_setAccount', 'UA-38333996-2']); _gaq.push(['_trackPageview']);
(function() {
var ga = document.createElement('script'); ga.type = 'text/javascript'; ga.async = true;
ga.src = ('https:' == document.location.protocol ? 'https://ssl' : 'http://www') + '.google-analytics.com/ga.js';
var s = document.getElementsByTagName('script')[0]; s.parentNode.insertBefore(ga, s);
})();
</script>"""
# caracteristicas del cliente
g.search_bot = is_search_bot()
# peticiones en modo preproduccion
g.beta_request = request.url_root[request.url_root.index("//")+2:].startswith("beta.")
# endpoint de entrada (usado en la página de error)
if app.config["APPWEB_MODE"] == "search":
g.home_route = "files.home"
elif app.config["APPWEB_MODE"] == "extras":
g.home_route = "extras.home"
else:
logging.error(u"APPWEB_MODE no especificado en la configuración")
g.home_route = "files.home"
# prefijo para los contenidos estáticos
if g.beta_request:
app_static_prefix = app.static_url_path
else:
app_static_prefix = app.config["STATIC_PREFIX"] or app.static_url_path
g.static_prefix = app.assets.url = app_static_prefix
g.keywords = set()
g.args = {}
g.page_description=g.title=""
g.categories = (('movies', {"q": "movie"}),
('games', {"q": "game"}),
('tv', {"q": "series"}),
('music', {"q": "audio"}),
('anime', {"q": "anime"}),
('books', {"q": "ebook"}),
('adult', {"q": "p**n"}),
('software', {"q": "software"}),
('mobile', {"q": "mobile"}),
('pictures', {"q": "image"})
)
def save_visited(files):
'''
Recibe una lista de resultados de fill_data y guarda las url que sean de web
'''
if not g.search_bot and feedbackdb.initialized:
result=[{"_id": data['urls'][0], "type":src} for afile in files for src, data in afile['view']['sources'].iteritems() if data['urls'] and data['icon'] in ['web','torrent']]
if result:
try:
feedbackdb.visited_links(result)
except:
logging.error("Error saving visited links.")
def searcha():
'''
Responde las peticiones de busqueda por ajax
'''
query=request.form.get("query",None)
filetype=request.form.get("filetype",None)
if not query: #si no se ha buscado nada se manda al inicio
logging.error("Invalid data for AJAX search request.")
return jsonify({})
query = query.replace("_"," ") if query is not None else None #para que funcionen la busqueda cuando vienen varias palabras
filters = {"src":"torrent"}
ori_query = query
_in = [i for i, v in enumerate(g.categories) if v[0] == filetype]
if filetype and _in :
_id = _in[0]
if "t" in g.categories[_id][1]:
filters["type"] = g.categories[_id][1]['t']
if "q" in g.categories[_id][1]:
_type = g.categories[_id][1]['q']
query = "%s (%s)" % (query, _type)
args = filters.copy()
args["q"] = ori_query
if ori_query != query:
args['type'] = _type
g.args=args
last_items = []
try:
last_items = b64decode(str(request.form["last_items"]) if "last_items" in request.form else "", "-_")
if last_items:
last_items = unpack("%dh"%(len(last_items)/2), last_items)
except BaseException as e:
logging.error("Error parsing last_items information from request.")
sure = False
total_found=0
search_results = search_files(query, filters, min_results=0, last_items=last_items, non_group=True, order=("@weight*(r+10)", "e DESC, ok DESC, r2 DESC, fs DESC, uri1 DESC", "@weight*(r+10)"), weight_processor=weight_processor, tree_visitor=tree_visitor)
response = make_response(render_template('file.html',files=[torrents_data(afile) for afile in search_results["files"]]))
del search_results["files"]
response.headers["X-JSON"]=json.dumps(search_results)
return response
def update_file(self, data, remove=None, direct_connection=False, update_sphinx=True):
'''
Actualiza los datos del fichero dado.
@type data: dict
@param data: Diccionario con los datos del fichero a guardar. Se debe incluir '_id', y es recomendable incluir 's' por motivos de rendimiento.
@type remove: iterable
@param remove: Lista de campos a quitar del registro.
@type direct_connection: bool
@param direct_connection: Especifica si se crea una conexión directa, ineficiente e independiente al foo primario.
@type update_sphinx: bool
@param update_sphinx: si se especifica bl, de este parámetro depende el si se conecta al sphinx para actualizarlo
'''
update = {"$set":data.copy()}
if remove is not None:
update["$unset"] = {}
for rem in remove:
if rem in update["$set"]:
del update["$set"][rem]
update["$unset"][rem] = 1
fid = hex2mid(data["_id"])
_indir = self.server_conn.foofind.indir.find_one({"_id": fid})
if _indir and "t" in _indir:
fid = hex2mid(_indir['t'])
if "s" in data:
server = str(data["s"])
else:
try:
server = str(self.get_file(fid, bl=None)["s"])
except (TypeError, KeyError) as e:
logging.error("Se ha intentado actualizar un fichero que no se encuentra en la base de datos", extra=data)
raise
if update_sphinx and "bl" in data:
try:
file_name = i["fn"].itervalues().next()["n"]
except:
file_name = None
block_files(mongo_ids=((i["_id"],file_name),), block=data["bl"])
for i in ("_id", "s"):
if i in update["$set"]:
del update["$set"][i]
# TODO: update cache
self.servers_conn[server].foofind.foo.update({"_id":fid}, update)
def init_g(app):
if not g.lang in current_app.config["APPWEB_AVAILABLE_LANGS"]:
g.lang = "en"
g.design = bool(request.form.get("wklk", False))
g.license_name = license_name = "foofind" if "foofind" in request.url_root else "blubster"
g.version_code = request.form.get("version",None)
g.analytics_code = current_app.config["ANALYTICS_CODE"][license_name] % {"client_version":g.version_code or ""}
# caracteristicas del cliente
g.search_bot = is_search_bot()
# peticiones en modo preproduccion
g.beta_request = request.url_root[request.url_root.index("//")+2:].startswith("beta.")
# endpoint de entrada (usado en la página de error)
if current_app.config["APPWEB_MODE"] == "search":
g.home_route = "files.home"
elif current_app.config["APPWEB_MODE"] == "extras":
g.home_route = "extras.home"
else:
logging.error(u"APPWEB_MODE no especificado en la configuración")
g.home_route = "files.home"
# prefijo para los contenidos estáticos
if g.beta_request:
app_static_prefix = current_app.static_url_path
else:
app_static_prefix = current_app.config["STATIC_PREFIX"] or current_app.static_url_path
g.static_prefix = current_app.assets.url = app_static_prefix
g.keywords = set()
g.args = {}
g.page_description=g.title=""
g.tos_link = current_app.config["TOS_LINK"][license_name]
g.privacy_link = current_app.config["PRIVACY_LINK"][license_name]
g.categories = (('video',{"q":["video"]}),
('audio',{"q":["audio"]}),
('document',{"q":["document"]}),
('image',{"q":["image"]}),
('software',{"q":["software"]})
)
g.downloader_properties = local_cache["downloader_properties"]
def get_downloader_properties(base_path, downloader_files_builds):
# source code information
properties = {"common":{"base_path": base_path}}
# builds information
for build, info in downloader_files_builds.iteritems():
try:
with open(os.path.join(base_path, info["metadata"]), "r") as f:
metadata = json.load(f)
properties[build] = info.copy()
properties[build].update(metadata)
properties[build]["length"] = os.path.getsize(os.path.join(base_path, properties[build]["main"]))
except:
logging.error("Error checking downloader files.")
return properties
def complaint():
'''
Procesa los datos del formulario para reportar enlaces.
'''
try:
form = ReportLinkForm(request.form)
if form.validate():
urlreported = "/download/"+form.file_id.data+"/"
pagesdb.create_complaint(dict([("linkreported","-"),("urlreported",urlreported),("ip",request.remote_addr)]+[(field.name,field.data) for field in form]))
response = make_response("true")
else:
response = make_response(repr(form.errors.keys()))
except BaseException as e:
logging.error("Error on file complaint.")
response = make_response("false")
response.mimetype = "application/json"
return response
def init_g(app):
g.wakalaka = bool(request.form.get("wklk", False))
g.license_name = "foofind" if "foofind" in request.url_root else "blubster"
# caracteristicas del cliente
g.search_bot = is_search_bot()
# peticiones en modo preproduccion
g.beta_request = request.url_root[request.url_root.index("//") +
2:].startswith("beta.")
# endpoint de entrada (usado en la página de error)
if app.config["APPWEB_MODE"] == "search":
g.home_route = "files.home"
elif app.config["APPWEB_MODE"] == "extras":
g.home_route = "extras.home"
else:
logging.error(u"APPWEB_MODE no especificado en la configuración")
g.home_route = "files.home"
# prefijo para los contenidos estáticos
if g.beta_request:
app_static_prefix = app.static_url_path
else:
app_static_prefix = app.config["STATIC_PREFIX"] or app.static_url_path
g.static_prefix = app.assets.url = app_static_prefix
g.keywords = set()
g.args = {}
g.page_description = g.title = ""
g.categories = (('video', {
"t": "video"
}), ('audio', {
"t": "audio"
}), ('document', {
"t": "document"
}), ('image', {
"t": "image"
}), ('software', {
"t": "software"
}))
def vote():
'''
Gestiona las votaciones de archivos
'''
ok = False
try:
file_id=url2mid(request.form.get("file_id",None))
server=int(request.form.get("server",0))
vote=int(request.form.get("vote",0))
if server>1 and file_id and vote in (0,1):
file_info = usersdb.set_file_vote(file_id, current_user, g.lang, vote)
filesdb.update_file({"_id":file_id, "vs":file_info, "s":server}, direct_connection=True)
ok = True
except BaseException as e:
logging.error("Error on vote.")
response = make_response("true" if ok else "false")
response.mimetype = "application/json"
return response
def sitemap():
urlformat = current_app.config["FILES_SITEMAP_URL"].get(g.domain, None)
rules = []
if urlformat:
servers = filesdb.get_servers()
threads = [
threading.Thread(
target = lambda x: rules.append( gensitemap(x, urlformat ) ),
args = ( server, ) )
for server in servers ]
for t in threads:
t.start()
for t in threads:
if t.is_alive():
t.join()
if None in rules:
rules = [rule for rule in rules if rule!=None]
logging.error("Hay sitemaps no disponibles", extra={"servers":servers, "rules":rules})
return render_template('sitemap.xml', rules=rules)
def init_g(app):
g.design = bool(request.form.get("wklk", False))
g.license_name = "foofind" if "foofind" in request.url_root else "blubster"
# caracteristicas del cliente
g.search_bot = is_search_bot()
# peticiones en modo preproduccion
g.beta_request = request.url_root[request.url_root.index("//")+2:].startswith("beta.")
# endpoint de entrada (usado en la página de error)
if app.config["APPWEB_MODE"] == "search":
g.home_route = "files.home"
elif app.config["APPWEB_MODE"] == "extras":
g.home_route = "extras.home"
else:
logging.error(u"APPWEB_MODE no especificado en la configuración")
g.home_route = "files.home"
# prefijo para los contenidos estáticos
if g.beta_request:
app_static_prefix = app.static_url_path
else:
app_static_prefix = app.config["STATIC_PREFIX"] or app.static_url_path
g.static_prefix = app.assets.url = app_static_prefix
g.keywords = set()
g.args = {}
g.page_description=g.title=""
g.tos_link = app.config["TOS_LINK"]
g.privacy_link = app.config["PRIVACY_LINK"]
g.categories = (('video',{"t":"video"}),
('audio',{"t":"audio"}),
('document',{"t":"document"}),
('image',{"t":"image"}),
('software',{"t":"software"})
)
def get_last_files(self, n=25):
'''
Obtiene los últimos n ficheros indexados.
'''
p = Parallel((
(self._get_last_files_indir_offset, (), {}),
(self._get_last_files_foo_offset, (), {}),
(self._get_last_files_from_foo, (n+self._last_files_max_offset, self._last_files_initial_skip), {}),
))
p.join(1)
# Parallel.join_and_terminate(1) evitaría los end_request: hay que
# dejar que los hilos de parallel terminen
if p.is_alive():
raise MongoTimeout, "Timeout in get_last_files"
elif p.failed():
logging.error(
"Some error found in Parallel tasks on get_last_files",
extra={
"output": p.output,
"exceptions": p.exceptions
})
cache.throw_fallback()
inoff, fooff, lf = p.output
fooff -= self._last_files_initial_skip/self._last_files_per_second
if inoff < fooff: # Si indir está más desactualizado que el foo
offset = (fooff-inoff)*self._last_files_per_second
if offset > self._last_files_max_offset:
logging.error(
"Indir and last foo's replicas are too desynchronized, get_last_files will fallback",
extra={
"file_offset":offset,
"file_offset_maximum":self._last_files_max_offset,
"replication_time_indir":inoff,
"replication_time_foo":fooff,
"replication_time_offset":fooff-inoff,
"replication_time_maximum_offset (calculated)": self._last_files_max_offset/float(self._last_files_per_second)
})
cache.throw_fallback()
return lf[offset:offset+n]
return lf[:n]
def complaint():
'''
Procesa los datos del formulario para reportar enlaces.
'''
try:
form = ReportLinkForm(request.form)
if form.validate():
urlreported = "/download/" + form.file_id.data + "/"
pagesdb.create_complaint(
dict([("linkreported", "-"), ("urlreported", urlreported),
("ip", request.remote_addr)] + [(field.name, field.data)
for field in form]))
response = make_response("true")
else:
response = make_response(repr(form.errors.keys()))
except BaseException as e:
logging.error("Error on file complaint.")
response = make_response("false")
response.mimetype = "application/json"
return response
def searcha():
'''
Responde las peticiones de busqueda por ajax
'''
query = request.form.get("query", None)
filetype = request.form.get("filetype", None)
if not query: #si no se ha buscado nada se manda al inicio
logging.error("Invalid data for AJAX search request.")
return jsonify({})
query = query.replace(
"_", " "
) if query is not None else None #para que funcionen la busqueda cuando vienen varias palabras
filters = {"src": "torrent"}
ori_query = query
_in = [i for i, v in enumerate(g.categories) if v[0] == filetype]
if filetype and _in:
_id = _in[0]
if "t" in g.categories[_id][1]:
filters["type"] = g.categories[_id][1]['t']
if "q" in g.categories[_id][1]:
_type = g.categories[_id][1]['q']
query = "%s (%s)" % (query, _type)
args = filters.copy()
args["q"] = ori_query
if ori_query != query:
args['type'] = _type
g.args = args
last_items = []
try:
last_items = b64decode(
str(request.form["last_items"])
if "last_items" in request.form else "", "-_")
if last_items:
last_items = unpack("%dh" % (len(last_items) / 2), last_items)
except BaseException as e:
logging.error("Error parsing last_items information from request.")
sure = False
total_found = 0
search_results = search_files(
query,
filters,
min_results=0,
last_items=last_items,
non_group=True,
order=("@weight*(r+10)",
"e DESC, ok DESC, r2 DESC, fs DESC, uri1 DESC",
"@weight*(r+10)"),
weight_processor=weight_processor,
tree_visitor=tree_visitor)
response = make_response(
render_template(
'file.html',
files=[torrents_data(afile) for afile in search_results["files"]]))
del search_results["files"]
response.headers["X-JSON"] = json.dumps(search_results)
return response
def prepare_data(f, text=None, ntts=[], details=False, current_category=None):
try:
return torrents_data(secure_fill_data(f,text,ntts), details, current_category)
except BaseException as e:
logging.error("Error retrieving torrent data.")
return None
def create_app(config=None, debug=False):
'''
Inicializa la aplicación Flask. Carga los siguientes módulos:
- index: página de inicio
- page: páginas estáticas
- user: gestión del usuario
- files: búsqueda y obtención de ficheros
- status: servicio de monitorización de la aplicación
Y además, inicializa los siguientes servicios:
- Configuración: carga valores por defecto y modifica con el @param config
- Web Assets: compilación y compresión de recursos estáticos
- i18n: detección de idioma en la URL y catálogos de mensajes
- Cache y auth: Declarados en el módulo services
- Files: Clases para acceso a datos
'''
app = Flask(__name__)
app.config.from_object(defaults)
app.debug = debug
# Configuración
if config:
app.config.from_object(config)
# Modo de appweb
appmode = app.config["APPWEB_MODE"]
# Gestión centralizada de errores
if app.config["SENTRY_DSN"]:
sentry.init_app(app)
logging.getLogger().setLevel(logging.DEBUG if debug else logging.INFO)
# Configuración dependiente de la versión del código
revision_filename_path = os.path.join(os.path.dirname(app.root_path), "revision")
if os.path.exists(revision_filename_path):
f = open(revision_filename_path, "r")
data = f.read()
f.close()
revisions = tuple(
tuple(i.strip() for i in line.split("#")[0].split())
for line in data.strip().split("\n")
if line.strip() and not line.strip().startswith("#"))
revision_hash = md5(data).hexdigest()
app.config.update(
CACHE_KEY_PREFIX = "%s%s/" % (
app.config["CACHE_KEY_PREFIX"] if "CACHE_KEY_PREFIX" in app.config else "",
revision_hash
),
REVISION_HASH = revision_hash,
REVISION = revisions
)
else:
app.config.update(
REVISION_HASH = None,
REVISION = ()
)
# Registra filtros de plantillas
register_filters(app)
# Registra valores/funciones para plantillas
app.jinja_env.globals["u"] = u
# proteccion CSRF
csrf.init_app(app)
# Blueprints
if appmode == "search":
app.register_blueprint(files)
elif appmode == "extras":
app.register_blueprint(extras)
else:
logging.error("No se ha especificado modo en la configuración. Blueprints sin cargar.")
# Web Assets
dir_static = app.static_folder # shortcut
scss.config.LOAD_PATHS = [os.path.abspath('%s/../..' % dir_static)]
for subdir in ['%s/%s' % (dir_static, x) for x in ['gen', 'torrents/gen']]:
if not os.path.isdir(subdir):
os.makedirs(subdir)
app.assets = Environment(app)
app.assets.debug = app.debug
app.assets.versions = "hash"
register_filter(JsSlimmer)
register_filter(CssSlimmer)
app.assets.register(
'css_torrents',
Bundle('torrents/css/torrents.scss',
filters='pyscss', output='torrents/gen/torrents.css', debug=False),
filters='css_slimmer', output='torrents/gen/torrents.css')
if appmode == "search":
app.assets.register(
'js_appweb',
Bundle('prototype.js', 'event.simulate.js', 'chosen.proto.min.js','appweb.js',
filters='rjsmin', output='gen/appweb.js'))
else:
app.assets.register(
'js_appweb',
Bundle('prototype.js', filters='rjsmin', output='gen/appweb.js'))
# Traducciones
babel.init_app(app)
@babel.localeselector
def get_locale():
return "en"
# Autenticación
auth.setup_app(app)
auth.user_loader(User.current_user)
auth.anonymous_user = User.current_user
# Cache
cache.init_app(app)
# Acceso a bases de datos
filesdb.init_app(app)
pagesdb.init_app(app)
feedbackdb.init_app(app)
entitiesdb.init_app(app)
usersdb.init_app(app)
plugindb.init_app(app)
# Servicio de búsqueda
@app.before_first_request
def init_process():
if not eventmanager.is_alive():
# Fallback inicio del eventManager
eventmanager.start()
# Profiler
profiler.init_app(app, feedbackdb)
eventmanager.once(searchd.init_app, hargs=(app, filesdb, entitiesdb, profiler))
# Refresco de conexiones
eventmanager.once(filesdb.load_servers_conn)
eventmanager.interval(app.config["FOOCONN_UPDATE_INTERVAL"], filesdb.load_servers_conn)
eventmanager.interval(app.config["FOOCONN_UPDATE_INTERVAL"], entitiesdb.connect)
@app.url_value_preprocessor
def pull_lang_code(endpoint, values):
if values is None:
g.lang = "en"
else:
g.lang = values.pop('lang', "en")
@app.url_defaults
def add_language_code(endpoint, values):
if not 'lang' in values and app.url_map.is_endpoint_expecting(endpoint, 'lang'):
values['lang'] = g.lang
@app.before_request
def before_request():
# No preprocesamos la peticiones a static
if request.path.startswith("/static/"):
return
init_g(app)
# ignora peticiones sin blueprint
if request.blueprint is None and len(request.path)>1 and request.path.endswith("/"):
if "?" in request.url:
root = request.url_root[:-1]
path = request.path.rstrip("/")
query = request.url.decode("utf-8")
query = query[query.find(u"?"):]
return redirect(root+path+query, 301)
return redirect(request.url.rstrip("/"), 301)
@app.after_request
def after_request(response):
response.headers["X-UA-Compatible"] = "IE=edge"
return response
# Páginas de error
errors = {
404: ("Page not found", "The requested address does not exists."),
410: ("Page not available", "The requested address is no longer available."),
500: ("An error happened", "We had some problems displaying this page. Maybe later we can show it to you."),
503: ("Service unavailable", "This page is temporarily unavailable. Please try again later.")
}
@allerrors(app, 400, 401, 403, 404, 405, 408, 409, 410, 411, 412, 413,
414, 415, 416, 417, 418, 500, 501, 502, 503)
def all_errors(e):
error = e.code if hasattr(e,"code") else 500
title, description = errors[error if error in errors else 500]
init_g(app)
return render_template('error.html', code=str(error), title=title,
description=description), error
return app
def build_source_links(f):
'''
Construye los enlaces correctamente
'''
def get_domain(src):
'''
Devuelve el dominio de una URL
'''
url_parts=urlparse(src).netloc.split('.')
i=len(url_parts)-1
if len(url_parts[i])<=2 and len(url_parts[i-1])<=3:
return url_parts[i-2]+'.'+url_parts[i-1]+'.'+url_parts[i]
else:
return url_parts[i-1]+'.'+url_parts[i];
f['view']['action']='download'
f['view']['sources']={}
max_weight=0
icon=""
# agrupación de origenes
source_groups = {}
file_sources = f['file']['src'].items()
file_sources.sort(key=lambda x:x[1]["t"])
for hexuri,src in file_sources:
if not src.get('bl',None) in (0, None):
continue
url_pattern=downloader=join=False
count=0
part=url=""
source_data=g.sources[src["t"]] if "t" in src and src["t"] in g.sources else None
if source_data is None: #si no existe el origen del archivo
logging.error("El fichero contiene un origen inexistente en la tabla \"sources\": %s" % src["t"], extra={"file":f})
if feedbackdb.initialized:
feedbackdb.notify_source_error(f['file']["_id"], f['file']["s"])
continue
elif "crbl" in source_data and int(source_data["crbl"])==1: #si el origen esta bloqueado
continue
elif "w" in source_data["g"] or "f" in source_data["g"] or "s" in source_data["g"]: #si es descarga directa o streaming
link_weight=1
tip=source_data["d"]
icon="web"
source_groups[icon] = tip
source=get_domain(src['url']) if "f" in source_data["g"] else source_data["d"]
url=src['url']
if "url_pattern" in source_data and not url.startswith(("https://","http://","ftp://")):
url_pattern=True
#en caso de duda se prefiere streaming
if "s" in source_data["g"]:
f['view']['action']="listen" if f['view']['ct']==CONTENT_AUDIO else 'watch'
link_weight*=2
#torrenthash antes de torrent porque es un caso especifico
elif source_data["d"]=="BitTorrentHash":
downloader=True
link_weight=0.7 if 'torrent:tracker' in f['file']['md'] or 'torrent:trackers' in f['file']['md'] else 0.1
tip="Torrent MagnetLink"
source="tmagnet"
icon="torrent"
if not icon in source_groups:
source_groups[icon] = tip # magnet link tiene menos prioridad para el texto
join=True
count=int(src['m'])
part="xt=urn:btih:"+src['url']
if 'torrent:tracker' in f['file']['md']:
part += unicode('&tr=' + urllib.quote_plus(u(f['file']['md']['torrent:tracker']).encode("UTF-8")), "UTF-8")
elif 'torrent:trackers' in f['file']['md']:
trackers = f['file']['md']['torrent:trackers']
if isinstance(trackers, basestring):
part += unicode("".join('&tr='+urllib.quote_plus(tr) for tr in u(trackers).encode("UTF-8").split(" ")), "UTF-8")
elif "t" in source_data["g"]:
downloader=True
link_weight=0.8
url=src['url']
if "url_pattern" in source_data and not url.startswith(("https://","http://","ftp://")):
url_pattern=True
tip=source=get_domain(source_data["url_pattern"]%url)
else:
tip=source=get_domain(src['url'])
icon="torrent"
source_groups[icon] = tip
elif source_data["d"]=="Gnutella":
link_weight=0.2
tip="Gnutella"
source=icon="gnutella"
part="xt=urn:sha1:"+src['url']
join=True
count=int(src['m'])
source_groups[icon] = tip
elif source_data["d"]=="eD2k":
downloader=True
link_weight=0.1
tip="eD2k"
source=icon="ed2k"
url="ed2k://|file|"+f['view']['pfn']+"|"+str(f['file']['z'] if "z" in f["file"] else 1)+"|"+src['url']+"|/"
count=int(src['m'])
source_groups[icon] = tip
elif source_data["d"]=="Tiger":
link_weight=0
tip="Gnutella"
source=icon="gnutella"
part="xt=urn:tiger:"+src['url']
join=True
elif source_data["d"]=="MD5":
link_weight=0
tip="Gnutella"
source=icon="gnutella"
part="xt=urn:md5:"+src['url']
source_groups[icon] = tip
join=True
else:
continue
if source in f['view']['sources']:
view_source = f['view']['sources'][source]
else:
view_source = f['view']['sources'][source] = {}
view_source.update(source_data)
if 'downloader' in view_source:
if downloader:
view_source['downloader']=1
else:
view_source['downloader']=1 if downloader else 0
view_source['tip']=tip
view_source['icon']=icon
view_source['icons']=source_data.get("icons",False)
view_source['join']=join
view_source['source']="streaming" if "s" in source_data["g"] else "direct_download" if "w" in source_data["g"] else "P2P" if "p" in source_data["g"] else ""
#para no machacar el numero si hay varios archivos del mismo source
if not 'count' in view_source or count>0:
view_source['count']=count
if not "parts" in view_source:
view_source['parts']=[]
if not 'urls' in view_source:
view_source['urls']=[]
if part:
view_source['parts'].append(part)
if url:
if url_pattern:
view_source['urls']=[source_data["url_pattern"]%url]
f['view']['source_id']=url
view_source["pattern_used"]=True
elif not "pattern_used" in view_source:
view_source['urls'].append(url)
if source_data["d"]!="eD2k":
view_source['count']+=1
if link_weight>max_weight:
max_weight = link_weight
f['view']['source'] = source
f['view']['source_groups'] = sorted(source_groups.items())
if "source" not in f["view"]:
raise FileNoSources
if icon!="web":
for src,info in f['view']['sources'].items():
if info['join']:
f['view']['sources'][src]['urls'].append("magnet:?"+"&".join(info['parts'])+"&dn="+f['view']['pfn']+("&xl="+str(f['file']['z']) if 'z' in f['file'] else ""))
elif not 'urls' in info:
del(f['view']['sources'][src])
def create_app(config=None, debug=False):
'''
Inicializa la aplicación Flask. Carga los siguientes módulos:
- index: página de inicio
- page: páginas estáticas
- user: gestión del usuario
- files: búsqueda y obtención de ficheros
- status: servicio de monitorización de la aplicación
Y además, inicializa los siguientes servicios:
- Configuración: carga valores por defecto y modifica con el @param config
- Web Assets: compilación y compresión de recursos estáticos
- i18n: detección de idioma en la URL y catálogos de mensajes
- Cache y auth: Declarados en el módulo services
- Files: Clases para acceso a datos
'''
app = Flask(__name__)
app.config.from_object(defaults)
app.debug = debug
# Configuración
if config:
app.config.from_object(config)
# Modo de appweb
appmode = app.config["APPWEB_MODE"]
# Gestión centralizada de errores
if app.config["SENTRY_DSN"]:
sentry.init_app(app)
logging.getLogger().setLevel(logging.DEBUG if debug else logging.INFO)
# Configuración dependiente de la versión del código
revision_filename_path = os.path.join(os.path.dirname(app.root_path),
"revision")
if os.path.exists(revision_filename_path):
f = open(revision_filename_path, "r")
data = f.read()
f.close()
revisions = tuple(
tuple(i.strip() for i in line.split("#")[0].split())
for line in data.strip().split("\n")
if line.strip() and not line.strip().startswith("#"))
revision_hash = md5(data).hexdigest()
app.config.update(CACHE_KEY_PREFIX="%s%s/" %
(app.config["CACHE_KEY_PREFIX"] if "CACHE_KEY_PREFIX"
in app.config else "", revision_hash),
REVISION_HASH=revision_hash,
REVISION=revisions)
else:
app.config.update(REVISION_HASH=None, REVISION=())
# Registra filtros de plantillas
register_filters(app)
# Registra valores/funciones para plantillas
app.jinja_env.globals["u"] = u
# proteccion CSRF
csrf.init_app(app)
# Blueprints
if appmode == "search":
app.register_blueprint(files)
elif appmode == "extras":
app.register_blueprint(extras)
else:
logging.error(
"No se ha especificado modo en la configuración. Blueprints sin cargar."
)
# Web Assets
scss.config.LOAD_PATHS = [
os.path.dirname(os.path.dirname(app.static_folder))
]
if not os.path.isdir(app.static_folder + "/gen"):
os.mkdir(app.static_folder + "/gen")
if not os.path.isdir(app.static_folder + "/blubster/gen"):
os.mkdir(app.static_folder + "/blubster/gen")
if not os.path.isdir(app.static_folder + "/foofind/gen"):
os.mkdir(app.static_folder + "/foofind/gen")
app.assets = assets = Environment(app)
assets.debug = app.debug
assets.versions = "hash"
register_filter(JsSlimmer)
register_filter(CssSlimmer)
assets.register('css_blubster',
Bundle('blubster/css/blubster.scss',
filters='pyscss',
output='blubster/gen/blubster.css',
debug=False,
depends='appweb.scss'),
filters='css_slimmer',
output='blubster/gen/blubster.css')
assets.register('css_foofind',
Bundle('foofind/css/foofind.scss',
filters='pyscss',
output='foofind/gen/foofind.css',
debug=False),
filters='css_slimmer',
output='foofind/gen/foofind.css')
# Traducciones
babel.init_app(app)
@babel.localeselector
def get_locale():
return "en"
# Autenticación
auth.setup_app(app)
auth.user_loader(User.current_user)
auth.anonymous_user = User.current_user
# Cache
cache.init_app(app)
# Acceso a bases de datos
filesdb.init_app(app)
pagesdb.init_app(app)
feedbackdb.init_app(app)
entitiesdb.init_app(app)
usersdb.init_app(app)
plugindb.init_app(app)
# Servicio de búsqueda
@app.before_first_request
def init_process():
if not eventmanager.is_alive():
# Fallback inicio del eventManager
eventmanager.start()
# Profiler
profiler.init_app(app, feedbackdb)
eventmanager.once(searchd.init_app,
hargs=(app, filesdb, entitiesdb, profiler))
# Refresco de conexiones
eventmanager.once(filesdb.load_servers_conn)
eventmanager.interval(app.config["FOOCONN_UPDATE_INTERVAL"],
filesdb.load_servers_conn)
eventmanager.interval(app.config["FOOCONN_UPDATE_INTERVAL"],
entitiesdb.connect)
@app.url_value_preprocessor
def pull_lang_code(endpoint, values):
if values is None:
g.lang = "en"
else:
g.lang = values.pop('lang', "en")
@app.url_defaults
def add_language_code(endpoint, values):
if not 'lang' in values and app.url_map.is_endpoint_expecting(
endpoint, 'lang'):
values['lang'] = g.lang
@app.before_request
def before_request():
# No preprocesamos la peticiones a static
if request.path.startswith("/static/"):
return
init_g(app)
# ignora peticiones sin blueprint
if request.blueprint is None and len(
request.path) > 1 and request.path.endswith("/"):
if "?" in request.url:
root = request.url_root[:-1]
path = request.path.rstrip("/")
query = request.url.decode("utf-8")
query = query[query.find(u"?"):]
return redirect(root + path + query, 301)
return redirect(request.url.rstrip("/"), 301)
@app.after_request
def after_request(response):
response.headers["X-UA-Compatible"] = "IE=edge"
return response
# Páginas de error
errors = {
404: ("Page not found", "The requested address does not exists."),
410: ("Page not available",
"The requested address is no longer available."),
500:
("An error happened",
"We had some problems displaying this page. Maybe later we can show it to you."
),
503: ("Service unavailable",
"This page is temporarily unavailable. Please try again later.")
}
@allerrors(app, 400, 401, 403, 404, 405, 408, 409, 410, 411, 412, 413, 414,
415, 416, 417, 418, 500, 501, 502, 503)
def all_errors(e):
error = e.code if hasattr(e, "code") else 500
title, description = errors[error if error in errors else 500]
init_g(app)
return render_template('error.html',
code=str(error),
title=title,
description=description), error
return app
def get_files(self, ids, servers_known = False, bl = 0):
'''
Devuelve los datos de los ficheros correspondientes a los ids
dados en formato hexadecimal.
@param ids: Lista de identificadores de los ficheros a recuperar. Si server_known es False, es una lista de cadenas. Si server_known es True, es una lista de tuplas, que incluyen el identificador del fichero y el número de servidor.
@param servers_known: Indica si la lista de identificadores incluye el servidor donde se encuentra el fichero.
@type bl: int o None
@param bl: valor de bl para buscar, None para no restringir
@rtype generator
@return Generador con los documentos de ficheros
'''
if not ids: return ()
sids = defaultdict(list)
# si conoce los servidores en los que están los ficheros,
# se analiza ids como un iterable (id, servidor, ...)
# y evitamos buscar los servidores
if servers_known:
for x in ids:
sids[x[1]].append(hex2mid(x[0]))
else:
# averigua en qué servidor está cada fichero
nindir = self.server_conn.foofind.indir.find({"_id": {"$in": [hex2mid(fid) for fid in ids]}, "s": {"$exists": 1}})
for ind in nindir:
indserver = str(int(ind["s"])) # Bug en indir: 's' como float
if indserver in self.servers_conn:
if "t" in ind: # si apunta a otro id, lo busca en vez del id dado
sids[indserver].append(ind["t"])
else:
sids[indserver].append(ind["_id"])
self.server_conn.end_request()
lsids = len(sids)
if lsids == 0:
# Si no hay servidores, no hay ficheros
return ()
elif lsids == 1:
k, v = sids.iteritems().next()
return self._get_server_files((k, v, bl))
else:
# crea el pool de hilos si no existe
if not self.thread_pool:
self.thread_pool = ThreadPool(processes=self.thread_pool_size)
# obtiene la información de los ficheros de cada servidor
results = []
chunks = self.thread_pool.imap_unordered(self._get_server_files, ((k, v, bl) for k, v in sids.iteritems()))
end = time.time()+self.get_files_timeout
try:
for i in xrange(len(sids)):
now = time.time()
if now>end:
break
for r in chunks.next(end-now):
results.append(r)
except TimeoutError:
pass
except BaseException as e:
logging.error("Error on get_files.")
return results
