def test_update_many_ok():
"""test_update_many_ok"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
mongo_db_access.insert("testUpdateMany_ok", {
"clave": "IPUpdateMany",
"value": 0
})
mongo_db_access.insert("testUpdateMany_ok", {
"clave": "IPUpdateMany",
"value": 0
})
res_update = mongo_db_access.update_many("testUpdateMany_ok",\
{"clave":"IPUpdateMany"}, {'value':1})
res_find = mongo_db_access.find("testUpdateMany_ok",
{"clave": "IPUpdateMany"})
count_elements = 0
for element_finding in res_find:
count_elements += 1
assert element_finding['value'] == 1
assert res_update.modified_count == 2
mongo_db_access.drop("testUpdateMany_ok")
assert count_elements == 2
def test_update_many_error_db():
"""test_update_many_error_db"""
mongo_db_access = MongoDBAccess(CONFIG_ERROR, "DEBUG")
res = mongo_db_access.update_many("testUpdateManyErrorDB", {"clave": "IP"},
{'valorX': "1234"})
assert res is None
def test_find_error_collection():
"""test_find_error_collection"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
res = mongo_db_access.find("correoX", {})
count_elements = 0
for element_finding in res:
if element_finding != None:
count_elements += 1
assert count_elements == 0
def __init__(self, file_config, level_log):
self.level_log = level_log
self.logger = Logger(self.__class__.__name__, level_log).get()
try:
self.config = json.loads(open(file_config, "r").read())
self.mongodbaccess = MongoDBAccess(self.config, level_log)
except IOError:
self.logger.error("File Error: %s", file_config)
self.config = {}
self.mongodbaccess = MongoDBAccess({}, level_log)
self.logger.info("Inicio: %s", datetime.datetime.now())
def test_update_many_error_collect():
"""test_update_many_error_collection"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
mongo_db_access.insert("testUpdateManyErrorColection",\
{"clave":"IPUpdateMany", "value":0})
mongo_db_access.insert("testUpdateManyErrorColection",\
{"clave":"IPUpdateMany", "value":0})
res = mongo_db_access.update_many("testUpdateManyErrorColectionXX",\
{"clave":"IPManyUpdateError"}, {'valorX':"1234"})
mongo_db_access.drop("testUpdateManyErrorColection")
assert res.modified_count == 0
def test_update_one_ok():
"""test_update_one_ok"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
mongo_db_access.insert("testUpdateOne", {
"clave": "IPUpdateOne",
"value": 0
})
res_update1 = mongo_db_access.update_one("testUpdateOne",
{"clave": "IPUpdateOne"},
{'value': 1})
res_find1 = mongo_db_access.find_one("testUpdateOne",
{"clave": "IPUpdateOne"})
mongo_db_access.delete_one("testUpdateOne", {"clave": "IPUpdateOne"})
mongo_db_access.drop("testUpdateOne")
assert res_update1 != None
assert res_find1['value'] == 1
def test_find_one_error_collection():
"""test_find_one_error_collection"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
mongo_db_access.insert("testFindone_errorCollection", {"clave": "IPFind"})
res = mongo_db_access.find_one("testFindone_errorCollectionXX",
{"clave": "IP"})
mongo_db_access.drop("testFindone_errorCollection")
assert res is None
def test_find_one_ok():
"""test_find_one_ok"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
mongo_db_access.insert("testFindOne", {"clave": "IPFind", "value": 0})
res = mongo_db_access.find_one("testFindOne", {"clave": "IPFind"})
mongo_db_access.drop("testFindOne")
assert res != None
def test_update_many_error_find():
"""test_update_many_error_find"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
mongo_db_access.insert("testUpdateManyErrorFind",\
{"clave":"IPUpdateManyErrorF", "value":0})
res = mongo_db_access.update_many("testUpdateManyErrorFind",\
{"clave":"IPUpdateManyErrorFXX"}, {'value':1})
mongo_db_access.drop("testUpdateManyErrorFind")
assert res.modified_count == 0
def test_update_one_error_find():
"""test_update_one_error_find"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
mongo_db_access.insert("testUpdateone_errorFind",\
{"clave":"IPUpdateOne", "value":0})
res = mongo_db_access.update_one("testUpdateone_errorFind",\
{"clave":"IPFind"}, {'valorX':"1234"})
mongo_db_access.drop("testUpdateone_errorFind")
assert res.modified_count == 0
def test_find_one_error_db():
"""test_find_one_error_db"""
mongo_db_access = MongoDBAccess(CONFIG_ERROR, "DEBUG")
mongo_db_access.insert("testFindone_error_db", {
"clave": "IPFind",
"value": 0
})
res = mongo_db_access.find_one("testFindone_error_db", {"clave": "IPFind"})
mongo_db_access.drop("testFindone_error_db")
assert res is None
def test_find_ok():
"""test_find_ok"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
mongo_db_access.insert("testFind_ok", {"clave": "IPFind", "value": 0})
mongo_db_access.insert("testFind_ok", {"clave": "IPFind", "value": 1})
res = mongo_db_access.find("testFind_ok", {})
count_elements = 0
for element_finding in res:
if element_finding != None:
print "XXXXX"
count_elements = +1
mongo_db_access.drop("testFind_ok")
assert count_elements > 0
def test_insert_ok():
"""test_insert_ok"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
res_insert = mongo_db_access.insert("testInsert_ok", {"clave": "IPInsert"})
res_find1 = mongo_db_access.find_one("testInsert_ok",
{"clave": "IPInsert"})
assert res_insert != None
assert res_find1['clave'] == "IPInsert"
res_remove = mongo_db_access.delete_one("testInsert_ok",
{"clave": "IPInsert"})
res_find2 = mongo_db_access.find_one("testInsert_ok",
{"clave": "IPInsert"})
mongo_db_access.drop("testInsert_ok")
assert res_remove.deleted_count == 1
assert res_find2 is None
def test_find_error_filter():
"""test_find_error_filter"""
mongo_db_access = MongoDBAccess(CONFIG, "DEBUG")
mongo_db_access.insert("testFindErrorFilter", {
"clave": "IPFind",
"value": 0
})
res = mongo_db_access.find("testFindErrorFilter", {"xxx": "xxxx"})
count_elements = 0
for element_finding in res:
if element_finding != None:
count_elements += 1
mongo_db_access.drop("testFindErrorFilter")
assert count_elements == 0
def test_error():
"""test_error"""
mongo_db_access_error = MongoDBAccess(CONFIG_ERROR, "DEBUG")
assert not mongo_db_access_error.status()
def test_insert_error_db():
"""test_insertError_db"""
mongo_db_access = MongoDBAccess(CONFIG_ERROR, "DEBUG")
res = mongo_db_access.insert("testInsertError", {"clave": "IPInsertError"})
assert res is None
def test_file_error():
"""test file error"""
mongo_db_access = MongoDBAccess({}, "DEBUG")
assert not mongo_db_access.status()
class Vuelos(object):
"""find Flight"""
level_log = None
config = None
mongodbaccess = None
logger = None
def __init__(self, file_config, level_log):
self.level_log = level_log
self.logger = Logger(self.__class__.__name__, level_log).get()
try:
self.config = json.loads(open(file_config, "r").read())
self.mongodbaccess = MongoDBAccess(self.config, level_log)
except IOError:
self.logger.error("File Error: %s", file_config)
self.config = {}
self.mongodbaccess = MongoDBAccess({}, level_log)
self.logger.info("Inicio: %s", datetime.datetime.now())
def ejecutar(self, nivel):
""" run load process """
print "++ INFO ++ MODULO PRINCIPAL MODO DE EJECUCION: {0}".format(
nivel)
if nivel == "1":
print "-- INFO -- MODO 1 duro ejecuta y limpia los datos del dia"
#proceso duro vaciamos informacion y empezamos
print "++ INFO ++ Vaciamos informacion del dia"
print "-- INFO -- dia: {0}".format(today())
borrados = self.vaciar_dia()
print "-- INFO -- vaciamos informacion -- Vuelos borrados del dia: {0}"\
.format(borrados.deleted_count)
urls = BuildUrls(self.mongodbaccess, self.level_log).build_urls()
print "-- INFO -- construir urls -- numero de URLS: {0}".format(
urls)
else:
print "-- INFO -- MODO 0 suave solo si hay datos que ejecutar"
#proceso soft miramos si hay algo que procesar
#si no hay nada que procesar o el dia no se ha ejecutado.
if self.return_urls().count() == 0:
#no hay nada que ejecutar
if self.find_last_day() < today():
# ultimo dia es anterior a hoy a las 12... no se ha procesado
print "++ WARN ++ 1.1 PRIMERA VEZ DEL DIA creamos las URLS y seguimos"
urls = BuildUrls(self.mongodbaccess,
self.level_log).build_urls()
print "-- INFO -- construir urls -- numero de URLS: {0}".format(
urls)
else:
# ultimo dia posterior hoy a las 12... esta todo Ok
print "++ WARN ++ 1.2 SE HA PROCESADO TODO Y NO HAY NADA QUE HACER"
else:
if self.find_last_day() < today():
# prblemas en el paraiso ayer la cosa no fue bien. Reiniciamos y procesamos
print "** ERROR ** 2.1 AYER NO SE EJECUTARON TODOS LOS VUELOS"
print "** ERROR ** vuelos pendientes {0}".format(
self.return_urls().count())
self.logger.error("AYER no se ejecutaron todos los vuelos")
urls = BuildUrls(self.mongodbaccess,
self.level_log).build_urls()
print "-- INFO -- construir urls -- numero de URLS: {0}".format(
urls)
else:
#hay cosas que ejecutar
print "++ WARN ++ 2.2 HA HABIDO UNA CANCELACION y el "\
+"SISTEMA SIGUE DESDE ESE PUNTO"
print "++ WARN ++ vuelos pendientes {0}".format(
self.return_urls().count())
self.logger.error(
"Ha habido una cancelacion y se sigue desde ese punto")
result = FindFlights(self.config, self.mongodbaccess, self.level_log)\
.get_flights(self.return_urls())
print "++ INFO -- TOTAL PROCESO, Save: {0}".format(
result.get("save", 0))
print "++ INFO -- TOTAL PROCESO, errores sin Informacion: {0}".format(
result.get("warn", 0))
print "++ INFO -- TOTAL PROCESO, errores NO ENCONTRADO: {0}".format(
result.get("error", 0))
def vaciar_dia(self):
""" delete all info of day """
return self.mongodbaccess.delete_many("vuelos",
{"dBusqueda": {
"$gt": today()
}})
def return_urls(self):
""" doc to explain """
return self.mongodbaccess.find("urls", {})
def find_last_day(self):
""" doc to explain """
print "++ INFO ++ find_last_day"
if self.mongodbaccess.find_one("vuelos", {}, sort={"dBusqueda": -1
}) is None:
return datetime.datetime(2000, 01, 01)
else:
return self.mongodbaccess.find_one("vuelos", {}, sort={"dBusqueda":-1})\
.get("dBusqueda", "")
#!/usr/bin/env python
# -*- coding: utf-8 -*-
"""Test vuelos"""
import sys
import json
sys.path.insert(0, "..")
try:
from pyproj.vuelos import Vuelos
from pyproj.mongodbaccess import MongoDBAccess
except ImportError:
print 'No Import'
FILE_CONFIG = "../test/config/configOk.json"
CONFIG4 = json.loads(open(FILE_CONFIG, "r").read())
MONGO_DB_ACCESS4 = MongoDBAccess(CONFIG4, "DEBUG")
def test_vuelos_ok_buld():
"""test vuelos run ok"""
vuelos = Vuelos("../test/config/configOk.json", "DEBUG")
assert vuelos.mongodbaccess.status()
def test_vuelos_error_buld():
"""test vuelos run bad with file wrong"""
vuelos = Vuelos("../test/config/configXX.json", "DEBUG")
assert not vuelos.mongodbaccess.status()
def test_find_error_db():
"""test_find_error_db"""
mongo_db_access = MongoDBAccess(CONFIG_ERROR, "DEBUG")
res = mongo_db_access.find("correo", {})
assert res is None
def test_ok():
"""test_ok"""
mongo_db_access_ok = MongoDBAccess(CONFIG, "DEBUG")
assert mongo_db_access_ok.status()
"""Test vuelos"""
import sys
import datetime
import json
sys.path.insert(0, "..")
try:
from pyproj.findflights import FindFlights
from pyproj.vuelos import Vuelos
from pyproj.mongodbaccess import MongoDBAccess
except ImportError:
print 'No Import'
FILE_CONFIG = "../test/config/configOk.json"
CONFIG5 = json.loads(open(FILE_CONFIG, "r").read())
MONGO_DB_ACCESS5 = MongoDBAccess(CONFIG5, "DEBUG")
FIND_FLIGHTS = FindFlights(CONFIG5, MONGO_DB_ACCESS5, "DEBUG")
VUELOS = Vuelos(FILE_CONFIG, "DEBUG")
def test_find_flights_url_onetrip():
"""test vuelos run bad with file wrong"""
MONGO_DB_ACCESS5.delete_many("urls", {})
MONGO_DB_ACCESS5.delete_many("vuelos", {})
date_direct = datetime.datetime.now() + datetime.timedelta(days=10)
from_data = "MAD"
to_data = "DUB"
url = "https://www.google.es/flights/#flt="+from_data+"."+to_data+"."\
+ date_direct.strftime("%Y-%m-%d")\
+ ";c:EUR;e:1;sd:1;t:f;tt:o"
# -*- coding: utf-8 -*-
"""Test holidays"""
import json
import datetime
import sys
sys.path.insert(0, "..")
try:
from pyproj.cleanflights import CleanFlights
from pyproj.mongodbaccess import MongoDBAccess
except ImportError:
print 'No Import'
FILE_CONFIG = "../test/config/configOk.json"
CONFIG3 = json.loads(open(FILE_CONFIG, "r").read())
MONGO_DB_ACCESS3 = MongoDBAccess(CONFIG3, "DEBUG")
CLEAN_FLIGHTS = CleanFlights(MONGO_DB_ACCESS3, "DEBUG")
def test_clean_flights_plus_15_days():
"""test clean flights"""
MONGO_DB_ACCESS3.delete_many("vuelos", {})
MONGO_DB_ACCESS3.delete_many("vuelosOld", {})
date = datetime.datetime.now() - datetime.timedelta(days=(15 + 1))
insert = {"dBusqueda":"d_busqueda", "precio":100, "type":"x", "horaS":"11:22", \
"horaLl":"", "company":"ONE", "duracion":"5h 30m", "escalas":"NO", \
"from":"HOME", "to":"HOLIDAYS", "dateDirect": date, \
"dateReturn": date, "holidays": 0}
MONGO_DB_ACCESS3.insert("vuelos", insert)
sum_urls = CLEAN_FLIGHTS.clean()