def get_entities(self, entities_ids=None, entities_keys=None, schemas=None):
'''
Obtiene la información de una entidad por identificador
@type entity_id: long
@param entity_id: id de la entidad
@rtype: MongoDB document or None
@return: resultado
'''
if self.enabled:
try:
query = {}
if schemas:
if len(schemas[1])==1:
query["s"] = schemas[1][0] if schemas[0] else {"$ne":schemas[1][0]}
else:
query["s"] = {("$in" if schemas[0] else "$nin"):schemas[1]}
if entities_ids and entities_keys:
query["$or"] = [{"_id":{"$in":entities_ids}}, {"k":{"$in":entities_keys}}]
elif entities_ids:
query["_id"] = {"$in":entities_ids}
elif entities_keys:
query["k"] = {"$in":entities_keys}
data = tuple(self.entities_conn.ontology.ontology.find(query))
self.entities_conn.end_request()
return data
except BaseException as e:
logging.warn("Can't access to entities database. Entities disabled.")
self.enabled = False
return ()
def download(instfile):
g.cache_code += "D"
downloader_files = current_app.config["DOWNLOADER_FILES"]
downloader_files_aliases = current_app.config["DOWNLOADER_FILES_ALIASES"]
if instfile in downloader_files_aliases:
instfile = downloader_files_aliases[instfile]
if not instfile in downloader_files:
abort(404)
version = request.args.get("version", "")
lang = request.args.get("lang", "en")
platform = request.args.get("platform", None)
path = downloader_files[instfile]
# Redirect downloads on static directories to static_download endpoint
# hoping server will handle request and serve it directly
prefix = os.path.abspath(os.path.join(current_app.root_path,"../downloads")) + os.sep
if path.startswith(prefix):
relative_path = path[len(prefix):]
return redirect(url_for('.static_download', instfile=relative_path))
# All downloads should be inside downloads static dir
logging.warn("Download %r served dinamically because not in static directory %r" % (path, prefix))
return send_file(path, mimetypes.guess_type(path)[0])
def pixel():
pixel_response = make_response(PIXEL)
pixel_response.mimetype="image/gif"
g.must_cache = 0
if not g.search_bot and request.referrer:
try:
parts = urllib2.unquote(request.referrer).decode("utf-8").split("?")[0].split("/")
get_query_info(parts[-1], parts[-2] if parts[-2]!="search" else None, check_qs=False)
if g.query and g.safe_query:
# no registra busquedas muy largas
if len(g.safe_query)>=current_app.config["MAX_LENGTH_SAVE"]:
return pixel_response
# no registra busquedas con palabras no permitidas
if blacklists.prepare_phrase(g.safe_query) in (blacklists_adult if g.is_adult_content else blacklists):
return pixel_response
# si toca registrar y hay resultados, registra busqueda para nubes de tags
torrentsdb.save_search(g.query, hashlib.md5((g.safe_query+"_"+request.remote_addr).encode("utf-8")).digest(), g.category.cat_id if g.category else 0)
except BaseException as e:
logging.warn("Error registering search.")
return pixel_response
def run(self):
gconn = None
not_found_count = 0
with open("nf_ntts.csv", "w") as not_found_ntts:
while True:
# obtiene peticiones de buscar entidades
afile = self.requests.get(True)
if afile is None:
self.requests.task_done()
break
if not gconn:
gconn = pymongo.Connection(self.server, slave_okay=True)
try:
# busca la entidad principal
main_ntt_id = int(afile["se"]["_id"])
ntt = gconn.ontology.ontology.find_one({"_id":main_ntt_id})
ntts1_info = set()
ntts2_info = set()
if ntt:
afile["se"]["info"] = ntt
# busca entidades de primer y segundo nivel
if "r" in ntt and ntt["r"]:
# genera la lista de entidades y tipos de relacion de primer nivel
ntts1_info = {(ntt_id, relation[:3])
for relation, relation_ids in ntt["r"].iteritems()
for ntt_id in relation_ids if ntt_id!=main_ntt_id}
# si hay entidades de primer nivel...
if ntts1_info:
# obtiene entidades de primer nivel
ntts1_ids = [ntt_id for ntt_id, relation in ntts1_info]
ntts1 = list(gconn.ontology.ontology.find({"_id":{"$in":ntts1_ids}}))
# genera la lista de entidades y tipos de relacion de segundo nivel
ntts1_ids.append(main_ntt_id) # añade el id de la relacion para usar la lista como filtro
ntts2_info = {(ntt_id, relation[:3])
for ntt2 in ntts1 if "r" in ntt2
for relation, relation_ids in ntt2["r"].iteritems()
for ntt_id in relation_ids if ntt_id not in ntts1_ids}
afile["se"]["rel"] = (ntts1_info, ntts2_info)
else:
not_found_ntts.write(str(afile["_id"])+"\n")
not_found_count += 1
del afile["se"]["_id"]
except BaseException:
ntt_id = str(afile["se"]["_id"]) if "_id" in afile["se"] else "???"
del afile["se"]["_id"]
gconn.close()
gconn = None
logging.exception("Error obtaining entities for file %s: %s."%(str(afile["_id"]), ntt_id))
self.results.put(afile)
self.requests.task_done()
if not_found_count:
logging.warn("Entities not found for some files. Check file nf_ntts.csv.")
def generate_id(afile, part):
binary_id = afile["_id"].binary[:5]
counter, = h_struct.unpack(binary_id[:2])
counters[counter]+=1
if counters[counter]>0x10000:
logging.warn("Counter overflow: %s" % hex(counter))
counters[counter] = 1
return id_struct.unpack(binary_id+"\0\0\0")[0]+part+counters[counter]-1, afile
def end_guess_categories_with_results(s):
# averigua si ha encontrado resultados para otras categorias
count_results = s.get_group_count(lambda x:(long(x)>>28)&0xF)
if count_results and count_results[0]:
count_results[0] -= sum(count_results.get(cat.cat_id,0) for cat in g.categories if cat.adult_content)
if count_results[0]<0:
count_results[0]=0
logging.warn("Count results for home lower than zero for search '%s' in category '%s'"%(g.query, g.category.title if g.category else "-"))
return count_results
def all_errors(e):
error_code, error_title, error_description = get_error_code_information(e)
try:
init_g()
g.page_description = g.title = "%d %s" % (error_code, error_title)
return render_template('error.html', zone="error", error=error_code, description=error_description, search_form=SearchForm()), error_code
except BaseException as ex: #si el error ha llegado sin contexto se encarga el servidor de él
logging.warn(ex)
return make_response("", error_code)
def vote(vtype):
g.must_cache = 0
result = {}
# don't allow bots to access this feature
if g.search_bot:
logging.warn("Bot is trying to access to file information.")
return abort(404)
# check referrer's domain matches this request's domain
referrer = urllib2.unquote(request.referrer).decode("utf-8")
if not referrer.startswith(request.url_root):
logging.warn("Don't allows votes from %s."%referrer)
return abort(404)
# get data from request
try:
# get file id from referrer url
filemid = fileurl2mid(referrer)
# get user's ip and calculates a unique id for this file and user
ip = (request.headers.getlist("X-Forwarded-For") or [request.remote_addr])[0]
userid = hashlib.sha1(str(filemid)+"_"+ip).hexdigest()[:10]
except BaseException as e:
logging.warn("Error parsing information from request.")
return jsonify(result)
if not vtype in VOTES:
logging.warn("Wrong vote type: %s."%unicode(vtype))
return jsonify(result)
try:
# save user vote
updated_votes = torrentsdb.save_vote(filemid, userid, vtype)
filesdb.update_file({"_id":filemid, "vs.u":Counter(updated_votes.itervalues())})
result["user"] = vtype
result["ret"] = ["report", _("Your report has been registered."), "info"]
except BaseException as e:
logging.warn("Error registering vote.")
return jsonify(result)
try:
f = filesdb.get_file(filemid, "1")
rate = rate_torrent(f)
result["votes"] = (rate["votes"].get(VERIFIED_VOTE,0), sum(value for vtype, value in rate["votes"].iteritems() if vtype!=VERIFIED_VOTE))
if "flag" in rate:
result["flag"] = rate["flag"]
result['flag'][1] = _(result['flag'][1]) # translate flag text
result["rating"] = int(round(rate["rating"]*5))
except BaseException as e:
logging.error("Error retrieving file information: %s."%str(filemid))
return jsonify(result)
def _get_closed_file(self, filename, version=None):
try:
if not version is None:
f = self.download_fs.get_last_version(filename=filename, version_code=version)
else:
f = self.download_fs.get_last_version(filename=filename)
f.close()
return f
except gridfs.errors.NoFile as e:
logging.warn("Requested download not found: %s" % filename)
return None
def twitter():
'''
Acceso a traves de twitter
'''
try:
logout_oauth()
return o_twitter.authorize(url_for('.twitter_authorized',next=oauth_redirect()))
except BaseException as e:
logging.warn(e)
flash(_("technical_problems", service="twitter"))
return redirect(url_for('.login'))
def facebook():
'''
Acceso a traves de facebook
'''
try:
logout_oauth()
return o_facebook.authorize(url_for('.facebook_authorized',next=oauth_redirect(),_external=True))
except BaseException as e:
logging.warn(e)
flash(_("technical_problems", service="facebook"))
return redirect(url_for('.login'))
def send_instfile(instfile, build):
g.cache_code += "D"
downloader_files = g.downloader_properties.get(build, None)
if not downloader_files:
abort(404)
downloader_files_aliases = downloader_files.get("aliases",{})
if instfile in downloader_files_aliases:
path = downloader_files[downloader_files_aliases[instfile]]
else:
# check that can be downloaded
for downloadable in downloader_files["downloadables"]:
if downloader_files.get(downloadable, None)==instfile:
path = instfile
break
else:
abort(404)
return send_file(os.path.join(g.downloader_properties["common"]["base_path"], path), mimetypes.guess_type(path)[0])
downloader_files = current_app.config["DOWNLOADER_FILES"]
downloader_files_aliases = current_app.config["DOWNLOADER_FILES_ALIASES"]
if instfile in downloader_files_aliases:
instfile = downloader_files_aliases[instfile]
if not instfile in downloader_files:
abort(404)
version = request.args.get("version", "")
lang = request.args.get("lang", "en")
platform = request.args.get("platform", None)
path = downloader_files[instfile]
# Redirect downloads on static directories to static_download endpoint
# hoping server will handle request and serve it directly
prefix = os.path.abspath(os.path.join(current_app.root_path,"../downloads")) + os.sep
if path.startswith(prefix):
relative_path = path[len(prefix):]
return redirect(url_for('.static_download', instfile=relative_path))
# All downloads should be inside downloads static dir
logging.warn("Download %r served dinamically because not in static directory %r" % (path, prefix))
return send_file(path, mimetypes.guess_type(path)[0])
def send_mail(subject,to,template=None,attachment=None,**kwargs):
'''
Envia un correo y trata y loguea los errores
'''
try:
msg=Message(_(subject),to if isinstance(to,list) else [to],html=render_template('email/'+(template if template else subject)+'.html',**kwargs))
if attachment is not None:
msg.attach(attachment[0],attachment[1],attachment[2])
mail.send(msg)
return True
except SMTPRecipientsRefused as e:
# se extrae el código y el mensaje de error
(code,message)=e[0].values()[0]
logging.warn("%d: %s"%(code,message))
flash("error_mail_send")
return False
def get_entity(self, entity_id):
'''
Obtiene la información de una entidad por identificador
@type entity_id: long
@param entity_id: id de la entidad
@rtype: MongoDB document or None
@return: resultado
'''
if self.enabled:
try:
data = self.entities_conn.ontology.ontology.find_one({"_id":entity_id})
self.entities_conn.end_request()
return data
except BaseException as e:
logging.warn("Can't access to entities database. Entities disabled.")
self.enabled = False
return {}
def init_app(self, app):
'''
Apply entities database access configuration.
@param app: Flask application.
'''
if app.config["DATA_SOURCE_ENTITIES"]:
try:
if "DATA_SOURCE_ENTITIES_RS" in app.config:
self.entities_conn = pymongo.MongoReplicaSetClient(app.config["DATA_SOURCE_ENTITIES"],
max_pool_size=app.config["DATA_SOURCE_MAX_POOL_SIZE"],
replicaSet = app.config["DATA_SOURCE_ENTITIES_RS"],
read_preference = pymongo.read_preferences.ReadPreference.SECONDARY_PREFERRED,
tag_sets = app.config.get("DATA_SOURCE_ENTITIES_RS_TAG_SETS",[{}]),
secondary_acceptable_latency_ms = app.config.get("SECONDARY_ACCEPTABLE_LATENCY_MS", 15))
else:
self.entities_conn = pymongo.MongoClient(app.config["DATA_SOURCE_ENTITIES"], max_pool_size=app.config["DATA_SOURCE_MAX_POOL_SIZE"], slave_okay=True)
self.enabled = True
except BaseException as e:
logging.warn("Can't connect to entities database. Entities disabled.")
def find_entities(self, keys, exact = False):
'''
Obtiene la información de una o varias entidades por claves
@type keys: array de diccionarios
@param keys: claves a buscar
@type exact: boolean
@param exact: indica si la clave debe ser exacta o puede ser un subconjunto
@rtype: MongoDB documents or None
@return: resultado
'''
if self.enabled:
try:
data = tuple(self.entities_conn.ontology.ontology.find({"k":{"$all":keys} if exact else keys}))
self.entities_conn.end_request()
return data
except BaseException as e:
logging.warn("Can't access to entities database. Entities disabled.")
self.enabled = False
return ()
def pull_lang_code(endpoint, values):
'''
Carga el código de idioma en la variable global.
'''
all_langs = current_app.config["ALL_LANGS"]
# obtiene el idioma de la URL
g.url_lang = None
if values is not None:
g.url_lang = values.pop('lang', None)
# si esta lista de idiomas permitidos
if g.url_lang and g.url_lang in all_langs:
g.lang = g.url_lang
# si el usuario esta logueado y tiene establecido el idioma se asigna ese
elif "user" in session and "lang" in session["user"]:
g.lang = session["user"]["lang"]
# si no esta logueado y ha elegido un idioma
elif "lang" in session:
g.lang = session["lang"]
else:
accept = request.accept_languages.values()
# si viene, se coge el que mas convenga dependiendo del que tenga establecido en su navegador o el idioma por defecto
locale = Locale.negotiate((option.replace("-","_") for option in accept), all_langs) if accept else None
if locale:
g.lang = locale.language
else:
g.lang = app.config["LANGS"][0] # valor por defecto si todo falla
if g.lang not in all_langs:
logging.warn("Wrong language choosen.")
g.lang = current_app.config["LANGS"][0]
# se carga la lista de idiomas como se dice en cada idioma
g.languages = OrderedDict((code, (localedata.load(code)["languages"], code in current_app.config["BETA_LANGS"])) for code in all_langs)
g.beta_lang = g.lang in current_app.config["BETA_LANGS"]
def handler():
try:
if request.method == "POST":
data = request.form.to_dict()
elif request.method == "GET":
data = request.args.to_dict()
else:
abort(404)
rdata = zlib.decompress(base64.b64decode(str(data["records"]), "-_"))
data["records"] = json.loads(rdata)
data["remote_addr"] = request.remote_addr
logging.warn("Downloader error received", extra=data)
response = make_response("OK")
response.status_code = 202
except BaseException as e:
logging.exception(e)
response = make_response("ERROR")
response.status_code = 500
response.mimetype = "text/plain"
return response
def generate(self, server, entities_server, part, afilter, batch_size, stop_set=None, stop_set_len=0, last_count=None, headers=True):
ff = FilesFetcher(server, entities_server, afilter, batch_size, stop_set, stop_set_len, last_count, self.processes)
ff.start()
if headers: self.generate_header()
count = error_count = 0
logging.warn("Comienza indexado en servidor %s."%server)
if self.pool:
for doc, extra in self.pool.imap(generate_file, (generate_id(afile, part) for afile in ff)):
count+=1
if doc:
outwrite(doc)
stats_file(extra, self.stats)
elif extra:
error_count += 1
if error_count>100: raise extra # ante mas de 100 errores, detiene la indexacion con error
if count%1000000==0:
outwrite("\n")
logging.warn("Progreso de indexado del servidor %s."%(server), extra={"count":count, "error_count":error_count})
else:
for afile in ff:
doc, extra = generate_file(generate_id(afile, part))
count+=1
if doc:
outwrite(doc+"\n")
stats_file(extra, self.stats)
elif extra:
error_count += 1
if error_count>100: raise extra # ante mas de 100 errores, detiene la indexacion con error
if count%1000000==0:
logging.warn("Progreso de indexado del servidor %s."%(server), extra={"count":count, "error_count":error_count})
if headers: self.generate_footer()
logging.warn("Finaliza indexado en servidor %s."%server)
self.total_count = ff.total_count
self.count = count
return ff.stop_set
def download_file(file_id,file_name=None):
'''
Devuelve el archivo a descargar, votos, comentarios y archivos relacionados
'''
error=(None,"") #guarda el id y el texto de un error
file_data=None
if file_id is not None: #si viene un id se comprueba que sea correcto
if is_valid_url_fileid(file_id):
try: #intentar convertir el id que viene de la url a uno interno
file_id=url2mid(file_id)
except (bson.objectid.InvalidId, TypeError) as e:
try: #comprueba si se trate de un ID antiguo
possible_file_id = filesdb.get_newid(file_id)
if possible_file_id is None:
logging.warn("Identificadores numericos antiguos sin resolver: %s."%e, extra={"fileid":file_id})
error=(404,"link_not_exist")
else:
logging.warn("Identificadores numericos antiguos encontrados: %s."%e, extra={"fileid":file_id})
return {"html": redirect(url_for(".download", file_id=mid2url(possible_file_id), file_name=file_name), 301),"error":(301,"")}
except BaseException as e:
logging.exception(e)
error=(503,"")
file_id=None
else:
abort(404)
if file_id:
try:
file_data=get_file_metadata(file_id, file_name)
except DatabaseError:
error=(503,"")
except FileNotExist:
error=(404,"link_not_exist")
except (FileRemoved, FileFoofindRemoved, FileNoSources):
error=(410,"error_link_removed")
except FileUnknownBlock:
error=(404,"")
if error[0] is None and not file_data: #si no ha habido errores ni hay datos, es porque existe y no se ha podido recuperar
error=(503,"")
if file_id is None or error[0] is not None:
html=""
if error[0] is not None: #si hay algun error se devuelve renderizado
message_msgid="error_%s_message" % error[0]
message_msgstr=_(message_msgid)
g.title="%s %s" % (error[0], message_msgstr if message_msgstr!=message_msgid else _("error_500_message"))
html=render_template('error.html',error=error,full_screen=True)
return {"html": html,"play":None,"file_data":file_data,"error":error}
else:
save_visited([file_data])
title = u(file_data['view']['fn'])
g.title = u"%s \"%s\" - %s" % (
_(file_data['view']['action']).capitalize(),
title[:100],
g.title)
g.page_description = u"%s %s"%(_(file_data['view']['action']).capitalize(), seoize_text(title," ",True))
#si el usuario esta logueado se comprueba si ha votado el archivo para el idioma activo y si ha marcado el archivo como favorito
vote=None
favorite = False
if current_user.is_authenticated():
vote=usersdb.get_file_vote(file_id,current_user,g.lang)
favorite=any(file_id==favorite["id"] for favorite in usersdb.get_fav_files(current_user))
#formulario para enviar comentarios
form = CommentForm(request.form)
if request.method=='POST' and current_user.is_authenticated() and (current_user.type is None or current_user.type==0) and form.validate():
usersdb.set_file_comment(file_id,current_user,g.lang,form.t.data)
form.t.data=""
flash("comment_published_succesfully")
#actualizar el fichero con la suma de los comentarios por idioma
filesdb.update_file({"_id":file_id,"cs":usersdb.get_file_comments_sum(file_id),"s":file_data["file"]["s"]},direct_connection=True)
#si tiene comentarios se guarda el número del comentario, el usuario que lo ha escrito, el comentario en si y los votos que tiene
comments=[]
if "cs" in file_data["file"]:
comments=[(i,usersdb.find_userid(comment["_id"].split("_")[0]),comment,comment_votes(file_id,comment)) for i,comment in enumerate(usersdb.get_file_comments(file_id,g.lang),1)]
# en la pagina de download se intentan obtener palabras para buscar si no las hay
if g.args.get("q", None) is None:
query = download_search(file_data, file_name, "foofind")
if query:
g.args["q"] = query.replace(":","")
return {
"html":render_template('files/download.html',file=file_data,vote={"k":0} if vote is None else vote,favorite=favorite,form=form,comments=comments),
"play":file_data["view"]["play"] if "play" in file_data["view"] else "",
"file_data":file_data,
}
def signal_handler(signal, frame):
logging.warn("Process killed processing file %s." % current_id)
sys.exit(1)
def format_metadata(f,text_cache, search_text_shown=False):
'''
Formatea los metadatos de los archivos
'''
text = text_cache[2] if text_cache else None
view_md = f['view']['md'] = {}
view_searches = f["view"]["searches"]={}
file_type = f['view']['file_type'] if 'file_type' in f['view'] else None
if 'md' in f['file']:
#si viene con el formato tipo:metadato se le quita el tipo
file_md = {(meta.split(":")[-1] if ":" in meta else meta): value for meta, value in f['file']['md'].iteritems()}
# Duración para vídeo e imágenes
seconds = get_float(file_md, "seconds")
minutes = get_float(file_md, "minutes")
hours = get_float(file_md, "hours")
# Si no he recibido duracion de otra forma, pruebo con length y duration
if seconds==minutes==hours==None:
seconds = get_float(file_md, "length") or get_float(file_md, "duration")
duration = [hours or 0, minutes or 0, seconds or 0] # h, m, s
if any(duration):
carry = 0
for i in xrange(len(duration)-1,-1,-1):
unit = long(duration[i]) + carry
duration[i] = unit%60
carry = unit/60
view_md["length"] = "%d:%02d:%02d" % tuple(duration) if duration[0] > 0 else "%02d:%02d" % tuple(duration[1:])
# Tamaño para vídeos e imágenes
width = get_int(file_md, "width")
height = get_int(file_md, "height")
if width and height:
view_md["size"] = "%dx%dpx" % (width, height)
# Metadatos que no cambian
try:
view_md.update(
(meta, file_md[meta]) for meta in
(
"folders","description","fileversion","os","files","pages","format",
"seeds","leechs","composer","publisher","encoding","director","writer","starring","producer","released"
) if meta in file_md
)
view_searches.update(
(meta, seoize_text(file_md[meta],"_",False)) for meta in
(
"folders","os","composer","publisher","director","writer","starring","producer"
) if meta in file_md
)
except BaseException as e:
logging.warn(e)
# thumbnail
if "thumbnail" in file_md:
f["view"]["thumbnail"] = file_md["thumbnail"]
#metadatos que tienen otros nombres
try:
view_md.update(("tags", file_md[meta]) for meta in ("keywords", "tags", "tag") if meta in file_md)
if "tags" in view_md and isinstance(view_md["tags"], basestring):
view_searches["tags"] = []
view_md.update(("comments", file_md[meta]) for meta in ("comments", "comment") if meta in file_md)
view_md.update(("track", file_md[meta]) for meta in ("track", "track_number") if meta in file_md)
view_md.update(("created_by", file_md[meta]) for meta in ("created_by", "encodedby","encoder") if meta in file_md)
view_md.update(("language", file_md[meta]) for meta in ("language", "lang") if meta in file_md)
view_md.update(("date", file_md[meta]) for meta in ("published", "creationdate") if meta in file_md)
view_md.update(("trackers", "\n".join(file_md[meta].split(" "))) for meta in ("trackers", "tracker") if meta in file_md and isinstance(file_md[meta], basestring))
view_md.update(("hash", file_md[meta]) for meta in ("hash", "infohash") if meta in file_md)
view_md.update(("visualizations", file_md[meta]) for meta in ("count", "viewCount") if meta in file_md)
if "unpackedsize" in file_md:
view_md["unpacked_size"]=file_md["unpackedsize"]
if "privateflag" in file_md:
view_md["private_file"]=file_md["privateflag"]
except BaseException as e:
logging.warn(e)
#torrents -> filedir filesizes filepaths
if "filepaths" in file_md:
filepaths = {}
for path, size in izip_longest(u(file_md["filepaths"]).split("///"), u(file_md.get("filesizes","")).split(" "), fillvalue=None):
# no permite tamaños sin fichero
if not path: break
parts = path.strip("/").split("/")
# crea subdirectorios
relative_path = filepaths
for part in parts[:-1]:
if "/"+part not in relative_path:
relative_path["/"+part] = {}
relative_path = relative_path["/"+part]
# si ya existe el directorio no hace nada
if "/"+parts[-1] in relative_path:
pass
# si el ultimo nivel se repite es un directorio (fallo de contenido)
elif parts[-1] in relative_path:
relative_path["/"+parts[-1]] = {}
del relative_path[parts[-1]]
else:
relative_path[parts[-1]] = size
if "filedir" in file_md:
filepaths = {"/"+u(file_md["filedir"]).strip("/"):filepaths}
if filepaths:
view_md["filepaths"] = filepaths
view_searches["filepaths"] = {}
# Metadatos multimedia
try:
#extraccion del codec de video y/o audio
if "video_codec" in file_md: #si hay video_codec se concatena el audio_codec detras si es necesario
view_md["codec"]=file_md["video_codec"]+" "+file_md["audio_codec"] if "audio_codec" in file_md else file_md["video_codec"]
else: #sino se meten directamente
view_md.update(("codec", file_md[meta]) for meta in ("audio_codec", "codec") if meta in file_md)
if file_type in ("audio", "video", "image"):
view_md.update((meta, file_md[meta]) for meta in ("genre", "track", "artist", "author", "colors") if meta in file_md)
view_searches.update((meta, seoize_text(file_md[meta], "_", False)) for meta in ("artist", "author") if meta in file_md)
except BaseException as e:
logging.warn(e)
# No muestra titulo si es igual al nombre del fichero
if "name" in file_md:
title = u(file_md["name"])
elif "title" in file_md:
title = u(file_md["title"])
else:
title = f['view']['nfn']
if title:
show_title = True
text_longer = title
text_shorter = f["view"]["fn"]
if len(text_shorter)>len(text_longer):
text_longer, text_shorter = text_shorter, text_longer
if text_longer.startswith(text_shorter):
text_longer = text_longer[len(text_shorter):]
if len(text_longer)==0 or (len(text_longer)>0 and text_longer.startswith(".") and text_longer[1:] in EXTENSIONS):
show_title = False
if show_title:
view_md["title"] = title
view_searches["title"] = seoize_text(title, "_", False)
# Los que cambian o son especificos de un tipo
try:
if "date" in view_md: #intentar obtener una fecha válida
try:
view_md["date"]=format_datetime(datetime.fromtimestamp(strtotime(view_md["date"])))
except:
del view_md["date"]
if file_type == 'audio': #album, year, bitrate, seconds, track, genre, length
if 'album' in file_md:
album = u(file_md["album"])
year = get_int(file_md, "year")
if album:
view_md["album"] = album + (" (%d)"%year if year and 1900<year<2100 else "")
view_searches["album"] = seoize_text(album, "_", False)
if 'bitrate' in file_md: # bitrate o bitrate - soundtype o bitrate - soundtype - channels
bitrate = get_int(file_md, "bitrate")
if bitrate:
soundtype=" - %s" % file_md["soundtype"] if "soundtype" in file_md else ""
channels = get_float(file_md, "channels")
channels=" (%g %s)" % (round(channels,1),_("channels")) if channels else ""
view_md["quality"] = "%g kbps %s%s" % (bitrate,soundtype,channels)
elif file_type == 'document': #title, author, pages, format, version
if "format" in file_md:
view_md["format"] = "%s%s" % (file_md["format"]," %s" % file_md["formatversion"] if "formatversion" in file_md else "")
version = []
if "formatVersion" in file_md:
version.append(u(file_md["formatVersion"]))
elif "version" in file_md:
version.append(u(file_md["version"]))
if "revision" in file_md:
version.append(u(file_md["revision"]))
if version:
view_md["version"] = " ".join(version)
elif file_type == 'image': #title, artist, description, width, height, colors
pass
elif file_type == 'software': #title, version, fileversion, os
if "title" in view_md and "version" in file_md:
view_md["title"] += " %s" % file_md["version"]
view_searches["title"] += " %s" % seoize_text(file_md["version"], "_", False)
elif file_type == 'video':
quality = []
framerate = get_int(file_md, "framerate")
if framerate:
quality.append("%d fps" % framerate)
if 'codec' in view_md: #si ya venia codec se muestra ahora en quality solamente
quality.append(u(view_md["codec"]))
del view_md["codec"]
if quality:
view_md["quality"] = " - ".join(quality)
if "series" in file_md:
series = u(file_md["series"])
if series:
safe_series = seoize_text(series, "_", False)
view_md["series"] = series
view_searches["series"]="%s_%s"%(safe_series,"(series)")
season = get_int(file_md, "season")
if season:
view_md["season"] = season
view_searches["season"]="%s_(s%d)"%(safe_series,season)
episode = get_int(file_md, "episode")
if episode:
view_md["episode"] = episode
view_searches["episode"]="%s_(s%de%d)"%(safe_series,season,episode)
except BaseException as e:
logging.exception("Error obteniendo metadatos especificos del tipo de contenido.")
view_mdh=f['view']['mdh']={}
for metadata,value in view_md.items():
if isinstance(value, basestring):
value = clean_html(value)
if not value:
del view_md[metadata]
continue
view_md[metadata]=value
# resaltar contenidos que coinciden con la busqueda, para textos no muy largos
if len(value)<500:
view_mdh[metadata]=highlight(text,value) if text and len(text)<100 else value
elif isinstance(value, float): #no hay ningun metadato tipo float
view_md[metadata]=str(int(value))
else:
view_md[metadata]=value
def download(file_id, file_name=""):
g.page_type = FILE_PAGE_TYPE
if request.referrer:
try:
posibles_queries = referrer_parser.match(request.referrer)
if posibles_queries:
query = posibles_queries.group(1) or posibles_queries.group(2) or ""
if query:
get_query_info(u(urllib2.unquote_plus(query).decode("utf-8")))
except:
pass
error = None
file_data=None
if file_id is not None: #si viene un id se comprueba que sea correcto
try: #intentar convertir el id que viene de la url a uno interno
file_id=url2mid(file_id)
except TypeError as e:
try: #comprueba si se trate de un ID antiguo
possible_file_id = filesdb.get_newid(file_id)
if possible_file_id is None:
logging.warn("Identificadores numericos antiguos sin resolver: %s."%e, extra={"fileid":file_id})
error=404
else:
logging.warn("Identificadores numericos antiguos encontrados: %s."%e, extra={"fileid":file_id})
return {"html": empty_redirect(url_for(".download", file_id=mid2url(possible_file_id), file_name=file_name), 301),"error":301}
except BaseException as e:
logging.exception(e)
error=503
file_id=None
if file_id:
try:
file_data=get_file_metadata(file_id, file_name.replace("-"," "))
except DatabaseError:
error=503
except FileNotExist:
error=404
except (FileRemoved, FileFoofindRemoved, FileNoSources):
error=410
except FileUnknownBlock:
error=404
if error is None and not file_data: #si no ha habido errores ni hay datos, es porque existe y no se ha podido recuperar
error=503
if error:
abort(error)
# completa datos de torrent
file_data = torrents_data(file_data, True, g.category)
if not file_data:
abort(404)
if file_data["view"]["category"]:
g.category = file_data["view"]["category"]
if file_data["view"]["category"].tag=="p**n":
g.is_adult_content = True
else:
g.category = file_data["view"]["category_type"]
# no permite acceder ficheros que deberian ser bloqueados
prepared_phrase = blacklists.prepare_phrase(file_data['view']['nfn'])
if prepared_phrase in blacklists["forbidden"] or (prepared_phrase in blacklists["misconduct"] and prepared_phrase in blacklists["underage"]):
g.blacklisted_content = "File"
if not g.show_blacklisted_content:
abort(404)
query = download_search(file_data, file_name, "torrent").replace("-"," ")
related = single_search(query, category=None, not_category=(None if g.is_adult_content else "p**n"), title=("Related torrents",3,None), zone="File / Related", last_items=[], limit=30, max_limit=15, ignore_ids=[mid2hex(file_id)], show_order=None)
# elige el titulo de la página
title = file_data['view']['fn']
# recorta el titulo hasta el proximo separador
if len(title)>101:
for pos in xrange(101, 30, -1):
if title[pos] in SEPPER:
title = title[:pos].strip()
break
else:
title = title[:101]
g.title = [title]
page_description = ""
if "description" in file_data["view"]["md"]:
page_description = file_data["view"]["md"]["description"].replace("\n", " ")
if not page_description:
if g.category:
page_description = _("download_category_desc", category=singular_filter(g.category.title).lower(), categorys=g.category.title.lower()).capitalize()
else:
page_description = _("download_desc")
if len(page_description)<50:
if page_description:
page_description += ". "
page_description += " ".join(text.capitalize()+"." for text in related[1]["files_text"])
if len(page_description)>180:
last_stop = page_description[:180].rindex(".") if "." in page_description[:180] else 0
if last_stop<100:
last_stop = page_description[:180].rindex(" ") if " " in page_description[:180] else 0
if last_stop<100:
last_stop = 180
page_description = page_description[:last_stop]+"."
g.page_description = page_description
is_canonical_filename = file_data["view"]["seo-fn"]==file_name
# registra visita al fichero
if g.search_bot:
searchd.log_bot_event(g.search_bot, True)
else:
save_visited([file_data])
if related[0]:
g.must_cache = 3600
# last-modified
g.last_modified = file_data["file"]["ls"]
return render_template('file.html', related_query = query, file_data=file_data, related_files=related, is_canonical_filename=is_canonical_filename, featured=get_featured(related[1]["count"]+len(file_data["view"]["md"]), 1))
def get_file_metadata(file_id, file_name=None):
'''
Obtiene el fichero de base de datos y rellena sus metadatos.
@type file_id: mongoid
@param file_id: id de mongo del fichero
@type file_name: basestring
@param file_name: nombre del fichero
@rtype dict
@return Diccionario de datos del fichero con metadatos
@raise DatabaseError: si falla la conexión con la base de datos
@raise FileNotExist: si el fichero no existe o ha sido bloqueado
@raise FileRemoved: si el fichero ha sido eliminado de su origen
@raise FileFoofindRemoved: si el fichero ha sido bloqueado por foofind
@raise FileUnknownBlock: si el fichero está bloqueado pero se desconoce el porqué
@raise FileNoSources: si el fichero no tiene orígenes
'''
try:
data = filesdb.get_file(file_id, bl = None)
except BaseException as e:
logging.exception(e)
raise DatabaseError
# intenta sacar el id del servidor de sphinx,
# resuelve inconsistencias de los datos
if not data:
sid = searchd.get_id_server_from_search(file_id, file_name)
if sid:
try:
data = filesdb.get_file(file_id, sid = sid, bl = None)
if feedbackdb.initialized:
feedbackdb.notify_indir(file_id, sid)
except BaseException as e:
logging.exception(e)
raise DatabaseError
if data:
bl = data.get("bl",None)
if bl and isinstance(bl, (str, unicode)) and bl.isdigit():
bl = int(bl)
if bl:
if bl == 1: raise FileFoofindRemoved
elif bl == 3: raise FileRemoved
logging.warn(
"File with an unknown 'bl' value found: %s" % repr(bl),
extra=data)
raise FileUnknownBlock
file_se = data["se"] if "se" in data else None
file_ntt = entitiesdb.get_entity(file_se["_id"]) if file_se and "_id" in file_se else None
ntts = {file_se["_id"]:file_ntt} if file_ntt else {}
'''
# trae entidades relacionadas
if file_ntt and "r" in file_ntt:
rel_ids = list(set(eid for eids in file_ntt["r"].itervalues() for eid in eids))
ntts.update({int(ntt["_id"]):ntt for ntt in entitiesdb.get_entities(rel_ids, None, (False, [u"episode"]))})
'''
else:
raise FileNotExist
#obtener los datos
return fill_data(data, file_name, ntts)
