def post(self):
for sd in SearchData.all():
db.delete(sd)
appid = APPID
if APPID == 'ebfmex-pub.appspot.com' or APPID == 'ebfmxorg.appspot.com':
appid = 'movil.' + APPID
url = 'http://' + appid + '/backend/generatesearch?kind=Oferta&field=Oferta'
#result = urlfetch.fetch(url)
url = 'http://' + appid + '/backend/generatesearch?kind=Oferta&field=Descripcion'
#result = urlfetch.fetch(url)
#self.redirect('/backend/generatesearch?kind=Oferta&field=Descripcion')
#self.redirect('/backend/generatesearch?kind=Oferta&field=Oferta')
for oferta in Oferta.all():
for palabra in OfertaPalabra.all().filter("IdOft =", oferta.IdOft):
newsd = SearchData()
newsd.Enlinea = oferta.Enlinea
newsd.FechaHora = datetime.now() - timedelta(hours = H)
newsd.Field = 'OfertaPalabra'
newsd.IdCat = oferta.IdCat
newsd.Kind = 'Oferta'
newsd.Sid = str(oferta.key())
newsd.Value = palabra.Palabra.lower()
newsd.put()
def gensearch_tr():
nbofertas = Oferta.all().count()
batchsize = 10
batchnumber = 0
while nbofertas >= 0:
taskqueue.add(url='/backend/gensearch', params={'kind': 'Oferta', 'field': 'Descripcion', 'batchsize': batchsize, 'batchnumber': batchnumber})
taskqueue.add(url='/backend/gensearch', params={'kind': 'Oferta', 'field': 'Oferta', 'batchsize': batchsize, 'batchnumber': batchnumber})
nbofertas -= batchsize
batchnumber += 1
taskqueue.add(url='/backend/searchinit')
logging.info(str(nbofertas) + ' ofertas. Batch size: ' + str(batchsize) + '. Done queueing.')
def get(self): """cats = Categoria.all() for cat in cats: db.delete(cat)""" ofs = Oferta.all() for of in ofs: db.delete(of) ofsucs = OfertaSucursal.all() for ofsuc in ofsucs: db.delete(ofsuc) ofpals = OfertaPalabra.all() for ofpal in ofpals: db.delete(ofpal)
def get(self):
#db.run_in_transaction(gensearch_tr)
#gensearch_tr
nbofertas = Oferta.all().count()
batchsize = 30
batchnumber = 0
logging.info(str(nbofertas) + ' ofertas. Batch size: ' + str(batchsize) + '. Queueing.')
while nbofertas >= 0:
taskqueue.add(url='/backend/gensearch', params={'split': 1, 'kind': 'Oferta', 'field': 'Descripcion', 'batchsize': batchsize, 'batchnumber': batchnumber})
taskqueue.add(url='/backend/gensearch', params={'split': 1, 'kind': 'Oferta', 'field': 'Oferta', 'batchsize': batchsize, 'batchnumber': batchnumber})
nbofertas -= batchsize
batchnumber += 1
taskqueue.add(url='/backend/searchinit')
def banOferta(oid):
key = ''
idoft = ''
cat = 0
edos = []
future = datetime.strptime('9999-12-31 23:59:59', '%Y-%m-%d %H:%M:%S')
ofertas = Oferta.all().filter("IdOft =", oid).run(limit=1)
for oferta in ofertas:
key = str(oferta.key())
cat = oferta.IdCat
idoft = oferta.IdOft
oferta.FechaHoraPub = future
oferta.put()
for estado in db.GqlQuery("SELECT IdEnt FROM OfertaEstado WHERE IdOft = :1", idoft).run():
edos.append(estado.IdEnt)
for edo in edos:
edocache = memcache.get('cacheEstado' + edo)
if edocache is not None:
edocache = json.loads(edocache)
indice = 0
for oferta in edocache:
if oferta['IdOft'] == idoft:
edocache.pop(indice)
indice += 1
memcache.add('cacheEstado' + edo, json.dumps(edocache), 3600)
catcache = memcache.get('cacheCategoria' + str(cat))
if catcache is not None:
catcache = json.loads(catcache)
indice = 0
for oferta in catcache:
if oferta['IdOft'] == idoft:
ofertacache.pop(indice)
indice += 1
memcache.add('cacheCategoria' + str(cat), json.dumps(catcache), 3600)
cachegen = memcache.get('cacheGeneral')
if cachegen is not None:
cachegen = json.loads(cachegen)
indice = 0
for oferta in cachegen:
if oferta['IdOft'] == idoft:
cachegen.pop(indice)
indice += 1
memcache.add('cacheGeneral', json.dumps(cachegen), 3600)
banSD(key)
def get(self):
timestamp = self.request.get('timestamp')
horas = self.request.get('horas')
self.response.headers['Content-Type'] = 'text/plain'
if not timestamp or not horas or timestamp == None or horas == None or timestamp == '' or horas == '':
errordict = {'error': -1, 'message': 'Must specify variables in GET method (i.e. /db?timestamp=<YYYYMMDDHH24>&horas=<int>)'}
self.response.out.write(json.dumps(errordict))
elif len(timestamp) != 10:
errordict = {'error': -2, 'message': 'timestamp must be 10 chars long: YYYYMMDDHH24'}
self.response.out.write(json.dumps(errordict))
else:
try:
fechastr = timestamp[0:8]
timestamp = datetime.strptime(timestamp,'%Y%m%d%H')
timestampdia = datetime.strptime(fechastr, '%Y%m%d')
horas = int(horas)
timestampend = timestamp + timedelta(hours = horas)
except ValueError:
errordict = {'error': -2, 'message': 'Value Error. Timestamp must be YYYYMMDDHH24 and horas is an integer'}
self.response.out.write(json.dumps(errordict))
if horas > 24:
errordict = {'error': -2, 'message': 'Horas must be <= 24'}
self.response.out.write(json.dumps(errordict))
else:
self.response.headers['Content-Type'] = 'text/plain'
suclist = memcache.get('wssucursales-' + fechastr)
if suclist is None:
outputlist = []
sucursales = Sucursal.all().filter("FechaHora >=", timestampdia).filter("FechaHora <", timestampdia + timedelta(days = 1))
for sucursal in sucursales:
sucdict = {}
sucdict['id'] = sucursal.IdSuc
sucdict['nombre'] = sucursal.Nombre
ent = None
entidades = Entidad.all().filter("CveEnt =", sucursal.DirEnt)
for entidad in entidades:
ent = entidad.Entidad
mun = None
municipios = Municipio.all().filter("CveEnt =", sucursal.DirEnt).filter("CveMun =", sucursal.DirMun)
for municipio in municipios:
mun = municipio.Municipio
sucdict['direccion'] = {'calle': sucursal.DirCalle, 'colonia': sucursal.DirCol, 'cp': sucursal.DirCp,'entidad_id': sucursal.DirEnt, 'entidad': ent,'municipio_id': sucursal.DirMun, 'municipio': mun}
sucdict['logo'] = None
sucdict['lat'] = sucursal.Geo1
sucdict['long'] = sucursal.Geo2
empresaQ = db.GqlQuery("SELECT * FROM Empresa WHERE IdEmp = :1", sucursal.IdEmp)
empresas = empresaQ.fetch(1)
empresadict = {}
for empresa in empresas:
empresadict = {'id': empresa.IdEmp, 'nombre': empresa.Nombre, 'url': empresa.Url, 'url_logo': 'http://' + APPID + '/spic?IdEmp=' + empresa.IdEmp}
sucdict['empresa'] = empresadict
ofertas = OfertaSucursal.all().filter("IdSuc =", sucursal.IdSuc)
ofertaslist = []
for oferta in ofertas.run(batch_size=10000):
ofs = Oferta.all().filter("IdOft =", oferta.IdOft)
of = []
for ofinst in ofs.run(limit=1):
of = ofinst
ofertadict = {}
ofertadict['id'] = oferta.IdOft
ofertadict['oferta'] = oferta.Oferta
ofertadict['descripcion'] = oferta.Descripcion
ofertadict['descuento'] = oferta.Descuento
ofertadict['promocion'] = oferta.Promocion
ofertadict['enlinea'] = oferta.Enlinea
#ofertadict['categoria'] = oferta.IdCat
ofertadict['precio'] = oferta.Precio
ofertadict['url'] = oferta.Url
if of.BlobKey and of.BlobKey is not None:
url = 'http://' + APPID + '/ofimg?id=' + str(of.BlobKey.key())
else:
url = ''
ofertadict['url_logo'] = url
palabraslist = []
palabras = OfertaPalabra.all().filter("IdOft =", oferta.IdOft)
for palabra in palabras:
palabraslist.append(palabra.Palabra)
ofertadict['palabras'] = palabraslist
cat = None
# CATEGORIA -temporal-
idcat = None
ofts = Oferta.all().filter("IdOft =", oferta.IdOft)
for oft in ofts:
idcat = oft.IdCat
categorias = Categoria.all().filter("IdCat =", idcat)
for categoria in categorias:
cat = categoria.Categoria
ofertadict['categoria_id'] = idcat
ofertadict['categoria'] = cat
ofertaslist.append(ofertadict)
sucdict['ofertas'] = ofertaslist
sucdict['timestamp'] = str(sucursal.FechaHora)
outputlist.append(sucdict)
memcache.add('wssucursales-' + fechastr, outputlist, 3600)
suclist = outputlist
if suclist is None:
self.response.out.write(json.dumps([]))
else:
outputlist = []
for suc in suclist:
valid = False
suctimestamp = datetime.strptime(suc['timestamp'].split('.')[0], '%Y-%m-%d %H:%M:%S')
if timestamp <= suctimestamp and suctimestamp <= timestampend:
valid = True
if valid == True:
outputlist.append(suc)
self.response.out.write(json.dumps(outputlist))
def post(self):
self.response.headers['Content-Type'] = 'text/plain'
try:
batchsize = int(self.request.get('batchsize'))
batchnumber = int(self.request.get('batchnumber'))
if batchsize is None:
batchsize = 10
if batchnumber is None:
batchnumber = 0
except ValueError:
batchsize = 10
batchnumber = 0
offset = batchnumber * batchsize
sucs = Sucursal.all()
for b in MvBlob.all().order ("-FechaHora").run(limit=1):
sucs.filter("FechaHora >", b.FechaHora)
sucs.order("FechaHora")#[offset:offset + batchsize]
logging.info('MvBlob generation, batchsize: ' + str(batchsize) + ',batchnumber: ' + str(batchnumber) + '. [' + str(offset) + ':' + str(offset + batchsize) + ']')
for suc in sucs.run(offset=offset, limit=batchsize):
HasOferta = False
olist = []
OSs = OfertaSucursal.all().filter("IdSuc =", suc.IdSuc)
for OS in OSs:
HasOferta = True
olist.append(OS.IdOft)
if HasOferta:
als = MvBlob.all().filter("IdSuc =", suc.IdSuc)
for al in als:
db.delete(al)
sucdict = {'id': suc.IdSuc, 'nombre': suc.Nombre, 'lat': suc.Geo1, 'long': suc.Geo2, 'fechamod': str(suc.FechaHora)}
ent = None
entidades = Entidad.all().filter("CveEnt =", suc.DirEnt)
for entidad in entidades:
ent = entidad.Entidad
mun = None
municipios = Municipio.all().filter("CveEnt =", suc.DirEnt).filter("CveMun =", suc.DirMun)
for municipio in municipios:
mun = municipio.Municipio
sucdict['direccion'] = {'calle': suc.DirCalle, 'colonia': suc.DirCol, 'cp': suc.DirCp,'entidad_id': suc.DirEnt, 'entidad': ent,'municipio_id': suc.DirMun, 'municipio': mun}
empresas = Empresa.all().filter("IdEmp = ", suc.IdEmp)
for empresa in empresas.run(limit=1):
empresadict = {'id': empresa.IdEmp, 'nombre': empresa.Nombre, 'url': empresa.Url, 'url_logo': ''}
sucdict['empresa'] = empresadict
urllogo = 'http://www.elbuenfin.org/imgs/imageDefault.png'
ofertaslist = []
for o in olist:
ofertas = Oferta.all().filter("IdOft =", o).run()
for oferta in ofertas:
url = 'http://www.elbuenfin.org/imgs/imageDefault.png'
try:
if oferta.Codigo and oferta.Codigo.replace('https://','http://')[0:7] == 'http://':
url = oferta.Codigo
except AttributeError:
err = 'logourl'
try:
if oferta.Codigo is None and oferta.BlobKey and oferta.BlobKey != None and oferta.BlobKey.key() != 'none':
url = 'http://' + APPID + '/ofimg?id=' + str(oferta.BlobKey.key())
except AttributeError:
err = 'logourl'
if url == 'http://www.elbuenfin.org/imgs/imageDefault.png' and oferta.Promocion is not None and oferta.Promocion != '':
url = oferta.Promocion
if oferta.Promocion is not None and oferta.Promocion != '':
urllogo = oferta.Promocion
ofertadict = {'id': oferta.IdOft, 'oferta': oferta.Oferta, 'descripcion': oferta.Descripcion, 'descuento': oferta.Descuento, 'promocion': oferta.Promocion, 'enlinea': oferta.Enlinea, 'precio': oferta.Precio, 'url': oferta.Url, 'url_logo': url, 'fechapub': str(oferta.FechaHoraPub.strftime('%Y-%m-%d'))}
palabraslist = []
palabras = OfertaPalabra.all().filter("IdOft =", oferta.IdOft)
for palabra in palabras:
palabraslist.append(palabra.Palabra)
ofertadict['palabras'] = palabraslist
cat = None
categorias = Categoria.all().filter("IdCat =", oferta.IdCat)
for categoria in categorias:
cat = categoria.Categoria
ofertadict['categoria_id'] = oferta.IdCat
ofertadict['categoria'] = cat
ofertaslist.append(ofertadict)
sucdict['ofertas'] = ofertaslist
sucdict['empresa']['url_logo'] = urllogo
mvblob = MvBlob()
mvblob.FechaHora = suc.FechaHora
mvblob.IdSuc = suc.IdSuc
mvblob.Blob = json.dumps(sucdict)
mvblob.put()
else:
pass
def get(self):
olist = []
count = 0
for o in Oferta.all():
count += 1
self.response.out.write(count)
def token(self):
token = self.request.get('token')
if token and str(token) == 'ZWJmbWV4LXB1YnIeCxISX0FoQWRtaW5Yc3JmVG9rZW5fIgZfWFNSRl8M':
try:
gminutes = self.request.get('minutes')
ghours = self.request.get('hours')
gdays = self.request.get('days')
if not gminutes:
gminutes = 0
else:
gminutes = int(gminutes)
if not ghours:
ghours = 0
else:
ghours = int(ghours)
if not gdays:
gdays = 0
else:
gdays = int(gdays)
except ValueError:
gminutes = 30
ghours = 0
gdays = 0
time = datetime.now() - timedelta(days = gdays, hours = ghours, minutes = gminutes)
self.response.headers['Content-Type'] = 'application/json'
#self.response.out.write (str(time))
changecontrol = ChangeControl.all().filter("FechaHora >=", time).filter("Kind =", 'Oferta').filter("Status IN", ["A","M"])
nbadded = 0
nbremoved = 0
for cc in changecontrol:
#self.response.out.write(cc.Id + '\n')
if cc.Status != 'B':
ofertas = Oferta.all().filter("IdOft =", cc.Id)
for oferta in ofertas:
if cc.Status == 'M':
searchdata = SearchData.all().filter("Sid =", str(oferta.key()))
for sd in searchdata:
db.delete(sd)
nbremoved += 1
desc = oferta.Descripcion.replace('\n',' ').replace('\r',' ').replace('.',' ').replace(',',' ').split(' ')
nombre = oferta.Oferta.replace('.',' ').replace(',',' ').split(' ')
for palabra in desc:
if len(palabra) > 3:
newsd = SearchData()
newsd.Enlinea = oferta.Enlinea
newsd.FechaHora = datetime.now() - timedelta(hours = H)
newsd.Field = 'Descripcion'
newsd.IdCat = oferta.IdCat
newsd.Kind = 'Oferta'
newsd.Sid = str(oferta.key())
newsd.Value = palabra.lower()
newsd.put()
nbadded += 1
for palabra in nombre:
if len(palabra) > 3:
newsd = SearchData()
newsd.Enlinea = oferta.Enlinea
newsd.FechaHora = datetime.now() - timedelta(hours = H)
newsd.Field = 'Oferta'
newsd.IdCat = oferta.IdCat
newsd.Kind = 'Oferta'
newsd.Sid = str(oferta.key())
newsd.Value = palabra.lower()
newsd.put()
nbadded += 1
palabraclave = OfertaPalabra.all().filter("IdOft =", oferta.IdOft)
for palabra in palabraclave.Palabra:
if len(palabra) > 3:
newsd = SearchData()
newsd.Enlinea = oferta.Enlinea
newsd.FechaHora = datetime.now() - timedelta(hours = H)
newsd.Field = 'OfertaPalabra'
newsd.IdCat = oferta.IdCat
newsd.Kind = 'Oferta'
newsd.Sid = str(oferta.key())
newsd.Value = palabra.Palabra.lower()
newsd.put()
nbadded += 1
logging.info("Finished updating. Added: %s. Removed: %s.", str(nbadded), str(nbremoved))
else:
logging.error('Wrong token given.')
