def download_previously_not_included(): print "searching includes that were previously skipped" connection = get_mongo_client() security_id = General.login(session_name()) torrents = connection.descargas.torrents """ we go through includes checking if something that was not included neither excluded can be now marked as included """ for include in includes: # print "checking include ", include['cadena'], " to see if we left something..." documento = {"titulo": re.compile(include["cadena"], re.IGNORECASE), "incluido": False, "excluido": False} for doc in torrents.find(documento): # we check if it has been downloaded by exact title titulo = doc["titulo"] url = doc["url"] if excluded(titulo): print "se encontro ", titulo, " que esta incluido por ", include[ "cadena" ], " pero no se descarga por estar tambien excluido" else: encontrado = torrents.find_one({"titulo": titulo}) if "descargado" not in encontrado or encontrado["descargado"] is False: print "downloading ", titulo, " at url ", url html = download_file.download_url_html(url) magnets = download_file.download_magnet_in_html_regex(html) for magnet in magnets: DownloadStation.add_task(magnet, security_id) encontrado["descargado"] = True encontrado["incluido"] = True torrents.save(encontrado) print "updated document to remember that it was already downloaded, waiting 10 seg..." time.sleep(10) # wait 10 seconds trying not to trigger server DDOS counter measures General.logout(session_name()) connection.close()
def treat_entry(entry, security_id, torrents): """ treats each of the rss entries """ esperar = False titulo = entry["title_detail"]["value"] url = entry["link"] fecha = entry["published_parsed"] documento = {"titulo": titulo, "url": url, "fecha": from_datetime_struct_to_timestamp(fecha)} document = torrents.find_one(documento) if document: print "'", unicode_functions.printable_string(titulo), "' already processed, skipping" else: is_included = included(titulo) is_excluded = excluded(titulo) documento["incluido"] = is_included documento["excluido"] = is_excluded if is_excluded: print "filter discards '", unicode_functions.printable_string(titulo), "'" else: if is_included: print "downloading ", unicode_functions.printable_string(titulo), " at url ", url html = download_file.download_url_html(url) esperar = True magnets = download_file.download_magnet_in_html_regex(html) for magnet in magnets: DownloadStation.add_task(magnet, security_id) documento["descargado"] = True else: print "filter does not include '", unicode_functions.printable_string(titulo), "'" # notify_mobile_phone(titulo + " not included", url) # lets you download it manually notify_mobile_phone(titulo + " not included", "") documento["notificado"] = True torrents.insert(documento) if esperar: print "waiting 10 seg..." time.sleep(10) # wait 10 seconds trying not to trigger server DOS counter measures
def treat_entry(entry, security_id, torrents): """ treats each of the rss entries """ esperar = False titulo = entry['title_detail']['value'] url = entry['link'] fecha = entry['published_parsed'] documento = { "titulo": titulo, "url": url, "fecha": from_datetime_struct_to_timestamp(fecha) } document = torrents.find_one(documento) if document: print "'", unicode_functions.printable_string( titulo), "' already processed, skipping" else: is_included = included(titulo) is_excluded = excluded(titulo) documento['incluido'] = is_included documento['excluido'] = is_excluded if is_excluded: print "filter discards '", unicode_functions.printable_string( titulo), "'" else: if is_included: print "downloading ", unicode_functions.printable_string( titulo), " at url ", url html = download_file.download_url_html(url) esperar = True magnets = download_file.download_magnet_in_html_regex(html) for magnet in magnets: DownloadStation.add_task(magnet, security_id) documento['descargado'] = True else: print "filter does not include '", unicode_functions.printable_string( titulo), "'" #notify_mobile_phone(titulo + " not included", url) # lets you download it manually notify_mobile_phone(titulo + " not included", "") documento['notificado'] = True torrents.insert(documento) if esperar: print "waiting 10 seg..." time.sleep( 10 ) # wait 10 seconds trying not to trigger server DOS counter measures
def download_previously_not_included(): print "searching includes that were previously skipped" connection = get_mongo_client() security_id = General.login(session_name()) torrents = connection.descargas.torrents """ we go through includes checking if something that was not included neither excluded can be now marked as included """ for include in includes: # print "checking include ", include['cadena'], " to see if we left something..." documento = { "titulo": re.compile(include['cadena'], re.IGNORECASE), "incluido": False, "excluido": False } for doc in torrents.find(documento): # we check if it has been downloaded by exact title titulo = doc['titulo'] url = doc['url'] if excluded(titulo): print "se encontro ", titulo, " que esta incluido por ", include['cadena'], \ " pero no se descarga por estar tambien excluido" else: encontrado = torrents.find_one({'titulo': titulo}) if "descargado" not in encontrado or encontrado[ 'descargado'] is False: print "downloading ", titulo, " at url ", url html = download_file.download_url_html(url) magnets = download_file.download_magnet_in_html_regex(html) for magnet in magnets: DownloadStation.add_task(magnet, security_id) encontrado['descargado'] = True encontrado['incluido'] = True torrents.save(encontrado) print "updated document to remember that it was already downloaded, waiting 10 seg..." time.sleep( 10 ) # wait 10 seconds trying not to trigger server DDOS counter measures General.logout(session_name()) connection.close()