def download_previously_not_included():
print "searching includes that were previously skipped"
connection = get_mongo_client()
security_id = General.login(session_name())
torrents = connection.descargas.torrents
"""
we go through includes checking if something that was not included neither
excluded can be now marked as included
"""
for include in includes:
# print "checking include ", include['cadena'], " to see if we left something..."
documento = {"titulo": re.compile(include["cadena"], re.IGNORECASE), "incluido": False, "excluido": False}
for doc in torrents.find(documento):
# we check if it has been downloaded by exact title
titulo = doc["titulo"]
url = doc["url"]
if excluded(titulo):
print "se encontro ", titulo, " que esta incluido por ", include[
"cadena"
], " pero no se descarga por estar tambien excluido"
else:
encontrado = torrents.find_one({"titulo": titulo})
if "descargado" not in encontrado or encontrado["descargado"] is False:
print "downloading ", titulo, " at url ", url
html = download_file.download_url_html(url)
magnets = download_file.download_magnet_in_html_regex(html)
for magnet in magnets:
DownloadStation.add_task(magnet, security_id)
encontrado["descargado"] = True
encontrado["incluido"] = True
torrents.save(encontrado)
print "updated document to remember that it was already downloaded, waiting 10 seg..."
time.sleep(10) # wait 10 seconds trying not to trigger server DDOS counter measures
General.logout(session_name())
connection.close()
def treat_entry(entry, security_id, torrents):
""" treats each of the rss entries """
esperar = False
titulo = entry["title_detail"]["value"]
url = entry["link"]
fecha = entry["published_parsed"]
documento = {"titulo": titulo, "url": url, "fecha": from_datetime_struct_to_timestamp(fecha)}
document = torrents.find_one(documento)
if document:
print "'", unicode_functions.printable_string(titulo), "' already processed, skipping"
else:
is_included = included(titulo)
is_excluded = excluded(titulo)
documento["incluido"] = is_included
documento["excluido"] = is_excluded
if is_excluded:
print "filter discards '", unicode_functions.printable_string(titulo), "'"
else:
if is_included:
print "downloading ", unicode_functions.printable_string(titulo), " at url ", url
html = download_file.download_url_html(url)
esperar = True
magnets = download_file.download_magnet_in_html_regex(html)
for magnet in magnets:
DownloadStation.add_task(magnet, security_id)
documento["descargado"] = True
else:
print "filter does not include '", unicode_functions.printable_string(titulo), "'"
# notify_mobile_phone(titulo + " not included", url) # lets you download it manually
notify_mobile_phone(titulo + " not included", "")
documento["notificado"] = True
torrents.insert(documento)
if esperar:
print "waiting 10 seg..."
time.sleep(10) # wait 10 seconds trying not to trigger server DOS counter measures
def treat_entry(entry, security_id, torrents):
""" treats each of the rss entries """
esperar = False
titulo = entry['title_detail']['value']
url = entry['link']
fecha = entry['published_parsed']
documento = {
"titulo": titulo,
"url": url,
"fecha": from_datetime_struct_to_timestamp(fecha)
}
document = torrents.find_one(documento)
if document:
print "'", unicode_functions.printable_string(
titulo), "' already processed, skipping"
else:
is_included = included(titulo)
is_excluded = excluded(titulo)
documento['incluido'] = is_included
documento['excluido'] = is_excluded
if is_excluded:
print "filter discards '", unicode_functions.printable_string(
titulo), "'"
else:
if is_included:
print "downloading ", unicode_functions.printable_string(
titulo), " at url ", url
html = download_file.download_url_html(url)
esperar = True
magnets = download_file.download_magnet_in_html_regex(html)
for magnet in magnets:
DownloadStation.add_task(magnet, security_id)
documento['descargado'] = True
else:
print "filter does not include '", unicode_functions.printable_string(
titulo), "'"
#notify_mobile_phone(titulo + " not included", url) # lets you download it manually
notify_mobile_phone(titulo + " not included", "")
documento['notificado'] = True
torrents.insert(documento)
if esperar:
print "waiting 10 seg..."
time.sleep(
10
) # wait 10 seconds trying not to trigger server DOS counter measures
def download_previously_not_included():
print "searching includes that were previously skipped"
connection = get_mongo_client()
security_id = General.login(session_name())
torrents = connection.descargas.torrents
"""
we go through includes checking if something that was not included neither
excluded can be now marked as included
"""
for include in includes:
# print "checking include ", include['cadena'], " to see if we left something..."
documento = {
"titulo": re.compile(include['cadena'], re.IGNORECASE),
"incluido": False,
"excluido": False
}
for doc in torrents.find(documento):
# we check if it has been downloaded by exact title
titulo = doc['titulo']
url = doc['url']
if excluded(titulo):
print "se encontro ", titulo, " que esta incluido por ", include['cadena'], \
" pero no se descarga por estar tambien excluido"
else:
encontrado = torrents.find_one({'titulo': titulo})
if "descargado" not in encontrado or encontrado[
'descargado'] is False:
print "downloading ", titulo, " at url ", url
html = download_file.download_url_html(url)
magnets = download_file.download_magnet_in_html_regex(html)
for magnet in magnets:
DownloadStation.add_task(magnet, security_id)
encontrado['descargado'] = True
encontrado['incluido'] = True
torrents.save(encontrado)
print "updated document to remember that it was already downloaded, waiting 10 seg..."
time.sleep(
10
) # wait 10 seconds trying not to trigger server DDOS counter measures
General.logout(session_name())
connection.close()