Beispiel #1
0
def test_logger_test():
    """Test logger"""
    sys.path.insert(0, "../test")
    try:
        os.remove("log/test.log")
    except OSError:
        print "file don't exist"

    logger = Logger("test", "DEBUG").get()
    logger.error("Error")
    data = open("log/test.log", "r").read()

    assert " ERROR:log_namespace.test Error" in data
Beispiel #2
0
class CleanFlights(object):
    """clean Flights"""

    mongodbaccess = None
    logger = None

    def __init__(self, mongo_db_access, level_log):
        self.logger = Logger(self.__class__.__name__, level_log).get()
        self.mongodbaccess = mongo_db_access
        self.logger.info("Inicio: %s", datetime.datetime.now())


    def clean(self):
        """ clean Process """
        self.logger.info("++INFO-- CLEAN FASE I")
        result = {"total":0}
        for vuelo in self.mongodbaccess.find("vuelos", {}):
            result = self.analize_each_flight(result, vuelo)
        return result

    def analize_each_flight(self, result, vuelo):
        """each flight analyze each rule"""
        apply(lambda rule: accumulate_dic(result, rule(vuelo)), self.create_all_rules())
        result["total"] += 1
        return result

    def create_all_rules(self):
        """ insert all rules created for run all"""
        return [self.rule_older_than_15days]

    def rule_older_than_15days(self, elemento):
        """First Rule: move all flights from vuelos to vuelosOld older than 15 days """
        date15 = datetime.datetime.now()-datetime.timedelta(days=15)
        deleted = 0
        inserted_old = 0
        if elemento.get("dateDirect", datetime.datetime) < date15:
            if self.mongodbaccess.insert("vuelosOld", elemento) is not None:
                inserted_old = 1
                self.logger.error("Error vuelo not insert backup but delete %s", elemento)
            self.mongodbaccess.delete_one("vuelos", {"_id":elemento.get("_id")})
            deleted = 1
        return {"deleted":deleted, "inserted_old":inserted_old}
Beispiel #3
0
class MongoDBAccess(object):
    """Class to access to MongoDB allow access and review connections"""

    db_access = None
    _client = None

    def __init__(self, config, levelLog):
        """Need a file where has got all parameters and level of Loggin"""
        self.logger = Logger(self.__class__.__name__, levelLog).get()
        self.logger.setLevel('INFO')

        try:
            self.logger.debug(config.get("url", ""))
            self._client = MongoClient(config.get("url", ""))
            self.db_access = self._client[config.get("nameDB")]
            self.logger.info("-- INFO -- DATA BASE CONECT OK")
        except ConfigurationError:
            self.logger.error("ConfigurationErr")
        except ConnectionFailure:
            self.logger.error("ConnectionFailure")
        except OperationFailure:
            self.logger.error("Authentication failure")

    def status(self):
        """Determinate True is connect or False if is not connect"""
        if self._client is None:
            return False
        try:
            self.logger.debug(self._client.server_info())
            return True
        except ConnectionFailure:
            self.logger.error("ConnectionFailure")
            return False
        except OperationFailure:
            self.logger.error("Authentication failure")
            return False

    def find_one(self, collection, query, sort=None):
        """Find one element only return a json element"""
        if self.status():
            sort = None if sort is None else sort.items()
            self.logger.info("Access to collection: %s, query %s", collection,
                             query)
            return self.db_access[collection].find_one(query, sort=sort)
        else:
            self.logger.error("Database Not INIT Find_one")
            return None

    def find(self, collection, query, sort=None, limite=None):
        """Find several elements is a cursor, atention for line in cursor is better"""
        if self.status():
            self.logger.info("Access to collection Multi: %s, query: %s, sort: %s, limit: %s",\
                collection, query, sort, limite)
            limite = 0 if limite is None else limite
            sort = None if sort is None else sort.items()
            return self.db_access[collection].find(query,
                                                   sort=sort,
                                                   limit=limite)
        else:
            self.logger.error("Database Not INIT Find")
            return None

    def update_one(self, collection, query, change, is_set="set"):
        """Update One return status of update"""
        if self.status():
            self.logger.info("Modify collection: %s, query: %s, modify: %s, set: %s",\
                collection, query, change, is_set)
            setdollar = "$" + is_set
            return self.db_access[collection].update_one(
                query, {setdollar: change})
        else:
            self.logger.error("Database Not INIT Update_one")
            return None

    def update_many(self, collection, query, change, is_set="set"):
        """Update Many return status of update"""
        if self.status():
            self.logger.info("Modify Many collection: %s, query: %s, modify: %s, set: %s",\
                collection, query, change, is_set)
            setdollar = "$" + is_set
            return self.db_access[collection].update_many(
                query, {setdollar: change})
        else:
            self.logger.error("Database Not INIT Update_one")
            return None

    def insert(self, collection, element):
        """Insert return status of insert"""
        if self.status():
            self.logger.debug("Insert collection: %s, data: %s", collection,
                              element)
            #control duplicated
            try:
                return self.db_access[collection].insert(element)
            except DuplicateKeyError:
                return None
        else:
            self.logger.error("Database Not INIT Find")
            return None

    def delete_one(self, collection, element):
        """delete One return status of delete"""
        if self.status():
            self.logger.info("Remove collection: %s, data: %s", collection,
                             element)
            return self.db_access[collection].delete_one(element)
        else:
            self.logger.error("Database Not INIT Find")
            return None

    def delete_many(self, collection, element):
        """delete return status of delete"""
        if self.status():
            self.logger.info("Remove collection: %s, data: %s", collection,
                             element)
            return self.db_access[collection].delete_many(element)
        else:
            self.logger.error("Database Not INIT Find")
            return None

    def aggregate(self, collection, element):
        """delete return status of delete"""
        if self.status():
            self.logger.info("Aggregate collection: %s, data: %s", collection,
                             element)
            return self.db_access[collection].aggregate(element)
        else:
            self.logger.error("Database Not INIT Find")
            return None

    def drop(self, collection):
        """Drop a collection return status of drop"""
        if self.status():
            self.logger.info("Drop collection: %s", collection)
            return self.db_access[collection].drop()
        else:
            self.logger.error("Database Not INIT Find")
            return None
Beispiel #4
0
class Vuelos(object):
    """find Flight"""

    level_log = None
    config = None
    mongodbaccess = None
    logger = None

    def __init__(self, file_config, level_log):
        self.level_log = level_log
        self.logger = Logger(self.__class__.__name__, level_log).get()
        try:
            self.config = json.loads(open(file_config, "r").read())
            self.mongodbaccess = MongoDBAccess(self.config, level_log)
        except IOError:
            self.logger.error("File Error: %s", file_config)
            self.config = {}
            self.mongodbaccess = MongoDBAccess({}, level_log)
        self.logger.info("Inicio: %s", datetime.datetime.now())

    def ejecutar(self, nivel):
        """ run load process """
        print "++ INFO ++ MODULO PRINCIPAL MODO DE EJECUCION: {0}".format(
            nivel)
        if nivel == "1":
            print "-- INFO -- MODO 1 duro ejecuta y limpia los datos del dia"
            #proceso duro vaciamos informacion y empezamos
            print "++ INFO ++ Vaciamos informacion del dia"
            print "-- INFO -- dia: {0}".format(today())
            borrados = self.vaciar_dia()
            print "-- INFO -- vaciamos informacion -- Vuelos borrados del dia: {0}"\
                  .format(borrados.deleted_count)
            urls = BuildUrls(self.mongodbaccess, self.level_log).build_urls()
            print "-- INFO -- construir urls -- numero de URLS: {0}".format(
                urls)
        else:
            print "-- INFO -- MODO 0 suave solo si hay datos que ejecutar"
            #proceso soft miramos si hay algo que procesar
            #si no hay nada que procesar o el dia no se ha ejecutado.
            if self.return_urls().count() == 0:
                #no hay nada que ejecutar
                if self.find_last_day() < today():
                    # ultimo dia es anterior a hoy a las 12... no se ha procesado
                    print "++ WARN ++  1.1 PRIMERA VEZ DEL DIA creamos las URLS y seguimos"
                    urls = BuildUrls(self.mongodbaccess,
                                     self.level_log).build_urls()
                    print "-- INFO -- construir urls -- numero de URLS: {0}".format(
                        urls)
                else:
                    # ultimo dia posterior hoy a las 12... esta todo Ok
                    print "++ WARN ++  1.2 SE HA PROCESADO TODO Y NO HAY NADA QUE HACER"
            else:
                if self.find_last_day() < today():
                    # prblemas en el paraiso ayer la cosa no fue bien. Reiniciamos y procesamos
                    print "** ERROR **  2.1 AYER NO SE EJECUTARON TODOS LOS VUELOS"
                    print "** ERROR **  vuelos pendientes {0}".format(
                        self.return_urls().count())
                    self.logger.error("AYER no se ejecutaron todos los vuelos")
                    urls = BuildUrls(self.mongodbaccess,
                                     self.level_log).build_urls()
                    print "-- INFO -- construir urls -- numero de URLS: {0}".format(
                        urls)

                else:
                    #hay cosas que ejecutar
                    print "++ WARN ++  2.2 HA HABIDO UNA CANCELACION y el "\
                          +"SISTEMA SIGUE DESDE ESE PUNTO"
                    print "++ WARN ++  vuelos pendientes {0}".format(
                        self.return_urls().count())
                    self.logger.error(
                        "Ha habido una cancelacion y se sigue desde ese punto")
        result = FindFlights(self.config, self.mongodbaccess, self.level_log)\
                   .get_flights(self.return_urls())
        print "++ INFO -- TOTAL PROCESO, Save: {0}".format(
            result.get("save", 0))
        print "++ INFO -- TOTAL PROCESO, errores sin Informacion: {0}".format(
            result.get("warn", 0))
        print "++ INFO -- TOTAL PROCESO, errores NO ENCONTRADO: {0}".format(
            result.get("error", 0))

    def vaciar_dia(self):
        """ delete all info of day """
        return self.mongodbaccess.delete_many("vuelos",
                                              {"dBusqueda": {
                                                  "$gt": today()
                                              }})

    def return_urls(self):
        """ doc to explain """
        return self.mongodbaccess.find("urls", {})

    def find_last_day(self):
        """ doc to explain """
        print "++ INFO ++ find_last_day"
        if self.mongodbaccess.find_one("vuelos", {}, sort={"dBusqueda": -1
                                                           }) is None:
            return datetime.datetime(2000, 01, 01)
        else:
            return self.mongodbaccess.find_one("vuelos", {}, sort={"dBusqueda":-1})\
                                     .get("dBusqueda", "")