Ejemplo n.º 1
0
def download_previously_not_included():
    print "searching includes that were previously skipped"
    connection = get_mongo_client()
    security_id = General.login(session_name())
    torrents = connection.descargas.torrents
    """
    we go through includes checking if something that was not included neither
    excluded can be now marked as included
    """
    for include in includes:
        # print "checking include ", include['cadena'], " to see if we left something..."
        documento = {"titulo": re.compile(include["cadena"], re.IGNORECASE), "incluido": False, "excluido": False}
        for doc in torrents.find(documento):
            # we check if it has been downloaded by exact title
            titulo = doc["titulo"]
            url = doc["url"]
            if excluded(titulo):
                print "se encontro ", titulo, " que esta incluido por ", include[
                    "cadena"
                ], " pero no se descarga por estar tambien excluido"
            else:
                encontrado = torrents.find_one({"titulo": titulo})
                if "descargado" not in encontrado or encontrado["descargado"] is False:
                    print "downloading ", titulo, " at url ", url
                    html = download_file.download_url_html(url)
                    magnets = download_file.download_magnet_in_html_regex(html)
                    for magnet in magnets:
                        DownloadStation.add_task(magnet, security_id)
                    encontrado["descargado"] = True
                    encontrado["incluido"] = True
                    torrents.save(encontrado)
                    print "updated document to remember that it was already downloaded, waiting 10 seg..."
                    time.sleep(10)  # wait 10 seconds trying not to trigger server DDOS counter measures
    General.logout(session_name())
    connection.close()
Ejemplo n.º 2
0
def treat_entry(entry, security_id, torrents):
    """ treats each of the rss entries """
    esperar = False
    titulo = entry["title_detail"]["value"]
    url = entry["link"]
    fecha = entry["published_parsed"]
    documento = {"titulo": titulo, "url": url, "fecha": from_datetime_struct_to_timestamp(fecha)}
    document = torrents.find_one(documento)
    if document:
        print "'", unicode_functions.printable_string(titulo), "' already processed, skipping"
    else:
        is_included = included(titulo)
        is_excluded = excluded(titulo)
        documento["incluido"] = is_included
        documento["excluido"] = is_excluded
        if is_excluded:
            print "filter discards '", unicode_functions.printable_string(titulo), "'"
        else:
            if is_included:
                print "downloading ", unicode_functions.printable_string(titulo), " at url ", url
                html = download_file.download_url_html(url)
                esperar = True
                magnets = download_file.download_magnet_in_html_regex(html)
                for magnet in magnets:
                    DownloadStation.add_task(magnet, security_id)
                documento["descargado"] = True
            else:
                print "filter does not include '", unicode_functions.printable_string(titulo), "'"
                # notify_mobile_phone(titulo + " not included", url)  # lets you download it manually
                notify_mobile_phone(titulo + " not included", "")
                documento["notificado"] = True
        torrents.insert(documento)
    if esperar:
        print "waiting 10 seg..."
        time.sleep(10)  # wait 10 seconds trying not to trigger server DOS counter measures
Ejemplo n.º 3
0
def treat_entry(entry, security_id, torrents):
    """ treats each of the rss entries """
    esperar = False
    titulo = entry['title_detail']['value']
    url = entry['link']
    fecha = entry['published_parsed']
    documento = {
        "titulo": titulo,
        "url": url,
        "fecha": from_datetime_struct_to_timestamp(fecha)
    }
    document = torrents.find_one(documento)
    if document:
        print "'", unicode_functions.printable_string(
            titulo), "' already processed, skipping"
    else:
        is_included = included(titulo)
        is_excluded = excluded(titulo)
        documento['incluido'] = is_included
        documento['excluido'] = is_excluded
        if is_excluded:
            print "filter discards '", unicode_functions.printable_string(
                titulo), "'"
        else:
            if is_included:
                print "downloading ", unicode_functions.printable_string(
                    titulo), " at url ", url
                html = download_file.download_url_html(url)
                esperar = True
                magnets = download_file.download_magnet_in_html_regex(html)
                for magnet in magnets:
                    DownloadStation.add_task(magnet, security_id)
                documento['descargado'] = True
            else:
                print "filter does not include '", unicode_functions.printable_string(
                    titulo), "'"
                #notify_mobile_phone(titulo + " not included", url)  # lets you download it manually
                notify_mobile_phone(titulo + " not included", "")
                documento['notificado'] = True
        torrents.insert(documento)
    if esperar:
        print "waiting 10 seg..."
        time.sleep(
            10
        )  # wait 10 seconds trying not to trigger server DOS counter measures
Ejemplo n.º 4
0
def download_previously_not_included():
    print "searching includes that were previously skipped"
    connection = get_mongo_client()
    security_id = General.login(session_name())
    torrents = connection.descargas.torrents
    """
    we go through includes checking if something that was not included neither
    excluded can be now marked as included
    """
    for include in includes:
        # print "checking include ", include['cadena'], " to see if we left something..."
        documento = {
            "titulo": re.compile(include['cadena'], re.IGNORECASE),
            "incluido": False,
            "excluido": False
        }
        for doc in torrents.find(documento):
            # we check if it has been downloaded by exact title
            titulo = doc['titulo']
            url = doc['url']
            if excluded(titulo):
                print "se encontro ", titulo, " que esta incluido por ", include['cadena'], \
                    " pero no se descarga por estar tambien excluido"
            else:
                encontrado = torrents.find_one({'titulo': titulo})
                if "descargado" not in encontrado or encontrado[
                        'descargado'] is False:
                    print "downloading ", titulo, " at url ", url
                    html = download_file.download_url_html(url)
                    magnets = download_file.download_magnet_in_html_regex(html)
                    for magnet in magnets:
                        DownloadStation.add_task(magnet, security_id)
                    encontrado['descargado'] = True
                    encontrado['incluido'] = True
                    torrents.save(encontrado)
                    print "updated document to remember that it was already downloaded, waiting 10 seg..."
                    time.sleep(
                        10
                    )  # wait 10 seconds trying not to trigger server DDOS counter measures
    General.logout(session_name())
    connection.close()