def post(self): for sd in SearchData.all(): db.delete(sd) appid = APPID if APPID == 'ebfmex-pub.appspot.com' or APPID == 'ebfmxorg.appspot.com': appid = 'movil.' + APPID url = 'http://' + appid + '/backend/generatesearch?kind=Oferta&field=Oferta' #result = urlfetch.fetch(url) url = 'http://' + appid + '/backend/generatesearch?kind=Oferta&field=Descripcion' #result = urlfetch.fetch(url) #self.redirect('/backend/generatesearch?kind=Oferta&field=Descripcion') #self.redirect('/backend/generatesearch?kind=Oferta&field=Oferta') for oferta in Oferta.all(): for palabra in OfertaPalabra.all().filter("IdOft =", oferta.IdOft): newsd = SearchData() newsd.Enlinea = oferta.Enlinea newsd.FechaHora = datetime.now() - timedelta(hours = H) newsd.Field = 'OfertaPalabra' newsd.IdCat = oferta.IdCat newsd.Kind = 'Oferta' newsd.Sid = str(oferta.key()) newsd.Value = palabra.Palabra.lower() newsd.put()
def gensearch_tr(): nbofertas = Oferta.all().count() batchsize = 10 batchnumber = 0 while nbofertas >= 0: taskqueue.add(url='/backend/gensearch', params={'kind': 'Oferta', 'field': 'Descripcion', 'batchsize': batchsize, 'batchnumber': batchnumber}) taskqueue.add(url='/backend/gensearch', params={'kind': 'Oferta', 'field': 'Oferta', 'batchsize': batchsize, 'batchnumber': batchnumber}) nbofertas -= batchsize batchnumber += 1 taskqueue.add(url='/backend/searchinit') logging.info(str(nbofertas) + ' ofertas. Batch size: ' + str(batchsize) + '. Done queueing.')
def get(self): """cats = Categoria.all() for cat in cats: db.delete(cat)""" ofs = Oferta.all() for of in ofs: db.delete(of) ofsucs = OfertaSucursal.all() for ofsuc in ofsucs: db.delete(ofsuc) ofpals = OfertaPalabra.all() for ofpal in ofpals: db.delete(ofpal)
def get(self): #db.run_in_transaction(gensearch_tr) #gensearch_tr nbofertas = Oferta.all().count() batchsize = 30 batchnumber = 0 logging.info(str(nbofertas) + ' ofertas. Batch size: ' + str(batchsize) + '. Queueing.') while nbofertas >= 0: taskqueue.add(url='/backend/gensearch', params={'split': 1, 'kind': 'Oferta', 'field': 'Descripcion', 'batchsize': batchsize, 'batchnumber': batchnumber}) taskqueue.add(url='/backend/gensearch', params={'split': 1, 'kind': 'Oferta', 'field': 'Oferta', 'batchsize': batchsize, 'batchnumber': batchnumber}) nbofertas -= batchsize batchnumber += 1 taskqueue.add(url='/backend/searchinit')
def banOferta(oid): key = '' idoft = '' cat = 0 edos = [] future = datetime.strptime('9999-12-31 23:59:59', '%Y-%m-%d %H:%M:%S') ofertas = Oferta.all().filter("IdOft =", oid).run(limit=1) for oferta in ofertas: key = str(oferta.key()) cat = oferta.IdCat idoft = oferta.IdOft oferta.FechaHoraPub = future oferta.put() for estado in db.GqlQuery("SELECT IdEnt FROM OfertaEstado WHERE IdOft = :1", idoft).run(): edos.append(estado.IdEnt) for edo in edos: edocache = memcache.get('cacheEstado' + edo) if edocache is not None: edocache = json.loads(edocache) indice = 0 for oferta in edocache: if oferta['IdOft'] == idoft: edocache.pop(indice) indice += 1 memcache.add('cacheEstado' + edo, json.dumps(edocache), 3600) catcache = memcache.get('cacheCategoria' + str(cat)) if catcache is not None: catcache = json.loads(catcache) indice = 0 for oferta in catcache: if oferta['IdOft'] == idoft: ofertacache.pop(indice) indice += 1 memcache.add('cacheCategoria' + str(cat), json.dumps(catcache), 3600) cachegen = memcache.get('cacheGeneral') if cachegen is not None: cachegen = json.loads(cachegen) indice = 0 for oferta in cachegen: if oferta['IdOft'] == idoft: cachegen.pop(indice) indice += 1 memcache.add('cacheGeneral', json.dumps(cachegen), 3600) banSD(key)
def get(self): timestamp = self.request.get('timestamp') horas = self.request.get('horas') self.response.headers['Content-Type'] = 'text/plain' if not timestamp or not horas or timestamp == None or horas == None or timestamp == '' or horas == '': errordict = {'error': -1, 'message': 'Must specify variables in GET method (i.e. /db?timestamp=<YYYYMMDDHH24>&horas=<int>)'} self.response.out.write(json.dumps(errordict)) elif len(timestamp) != 10: errordict = {'error': -2, 'message': 'timestamp must be 10 chars long: YYYYMMDDHH24'} self.response.out.write(json.dumps(errordict)) else: try: fechastr = timestamp[0:8] timestamp = datetime.strptime(timestamp,'%Y%m%d%H') timestampdia = datetime.strptime(fechastr, '%Y%m%d') horas = int(horas) timestampend = timestamp + timedelta(hours = horas) except ValueError: errordict = {'error': -2, 'message': 'Value Error. Timestamp must be YYYYMMDDHH24 and horas is an integer'} self.response.out.write(json.dumps(errordict)) if horas > 24: errordict = {'error': -2, 'message': 'Horas must be <= 24'} self.response.out.write(json.dumps(errordict)) else: self.response.headers['Content-Type'] = 'text/plain' suclist = memcache.get('wssucursales-' + fechastr) if suclist is None: outputlist = [] sucursales = Sucursal.all().filter("FechaHora >=", timestampdia).filter("FechaHora <", timestampdia + timedelta(days = 1)) for sucursal in sucursales: sucdict = {} sucdict['id'] = sucursal.IdSuc sucdict['nombre'] = sucursal.Nombre ent = None entidades = Entidad.all().filter("CveEnt =", sucursal.DirEnt) for entidad in entidades: ent = entidad.Entidad mun = None municipios = Municipio.all().filter("CveEnt =", sucursal.DirEnt).filter("CveMun =", sucursal.DirMun) for municipio in municipios: mun = municipio.Municipio sucdict['direccion'] = {'calle': sucursal.DirCalle, 'colonia': sucursal.DirCol, 'cp': sucursal.DirCp,'entidad_id': sucursal.DirEnt, 'entidad': ent,'municipio_id': sucursal.DirMun, 'municipio': mun} sucdict['logo'] = None sucdict['lat'] = sucursal.Geo1 sucdict['long'] = sucursal.Geo2 empresaQ = db.GqlQuery("SELECT * FROM Empresa WHERE IdEmp = :1", sucursal.IdEmp) empresas = empresaQ.fetch(1) empresadict = {} for empresa in empresas: empresadict = {'id': empresa.IdEmp, 'nombre': empresa.Nombre, 'url': empresa.Url, 'url_logo': 'http://' + APPID + '/spic?IdEmp=' + empresa.IdEmp} sucdict['empresa'] = empresadict ofertas = OfertaSucursal.all().filter("IdSuc =", sucursal.IdSuc) ofertaslist = [] for oferta in ofertas.run(batch_size=10000): ofs = Oferta.all().filter("IdOft =", oferta.IdOft) of = [] for ofinst in ofs.run(limit=1): of = ofinst ofertadict = {} ofertadict['id'] = oferta.IdOft ofertadict['oferta'] = oferta.Oferta ofertadict['descripcion'] = oferta.Descripcion ofertadict['descuento'] = oferta.Descuento ofertadict['promocion'] = oferta.Promocion ofertadict['enlinea'] = oferta.Enlinea #ofertadict['categoria'] = oferta.IdCat ofertadict['precio'] = oferta.Precio ofertadict['url'] = oferta.Url if of.BlobKey and of.BlobKey is not None: url = 'http://' + APPID + '/ofimg?id=' + str(of.BlobKey.key()) else: url = '' ofertadict['url_logo'] = url palabraslist = [] palabras = OfertaPalabra.all().filter("IdOft =", oferta.IdOft) for palabra in palabras: palabraslist.append(palabra.Palabra) ofertadict['palabras'] = palabraslist cat = None # CATEGORIA -temporal- idcat = None ofts = Oferta.all().filter("IdOft =", oferta.IdOft) for oft in ofts: idcat = oft.IdCat categorias = Categoria.all().filter("IdCat =", idcat) for categoria in categorias: cat = categoria.Categoria ofertadict['categoria_id'] = idcat ofertadict['categoria'] = cat ofertaslist.append(ofertadict) sucdict['ofertas'] = ofertaslist sucdict['timestamp'] = str(sucursal.FechaHora) outputlist.append(sucdict) memcache.add('wssucursales-' + fechastr, outputlist, 3600) suclist = outputlist if suclist is None: self.response.out.write(json.dumps([])) else: outputlist = [] for suc in suclist: valid = False suctimestamp = datetime.strptime(suc['timestamp'].split('.')[0], '%Y-%m-%d %H:%M:%S') if timestamp <= suctimestamp and suctimestamp <= timestampend: valid = True if valid == True: outputlist.append(suc) self.response.out.write(json.dumps(outputlist))
def post(self): self.response.headers['Content-Type'] = 'text/plain' try: batchsize = int(self.request.get('batchsize')) batchnumber = int(self.request.get('batchnumber')) if batchsize is None: batchsize = 10 if batchnumber is None: batchnumber = 0 except ValueError: batchsize = 10 batchnumber = 0 offset = batchnumber * batchsize sucs = Sucursal.all() for b in MvBlob.all().order ("-FechaHora").run(limit=1): sucs.filter("FechaHora >", b.FechaHora) sucs.order("FechaHora")#[offset:offset + batchsize] logging.info('MvBlob generation, batchsize: ' + str(batchsize) + ',batchnumber: ' + str(batchnumber) + '. [' + str(offset) + ':' + str(offset + batchsize) + ']') for suc in sucs.run(offset=offset, limit=batchsize): HasOferta = False olist = [] OSs = OfertaSucursal.all().filter("IdSuc =", suc.IdSuc) for OS in OSs: HasOferta = True olist.append(OS.IdOft) if HasOferta: als = MvBlob.all().filter("IdSuc =", suc.IdSuc) for al in als: db.delete(al) sucdict = {'id': suc.IdSuc, 'nombre': suc.Nombre, 'lat': suc.Geo1, 'long': suc.Geo2, 'fechamod': str(suc.FechaHora)} ent = None entidades = Entidad.all().filter("CveEnt =", suc.DirEnt) for entidad in entidades: ent = entidad.Entidad mun = None municipios = Municipio.all().filter("CveEnt =", suc.DirEnt).filter("CveMun =", suc.DirMun) for municipio in municipios: mun = municipio.Municipio sucdict['direccion'] = {'calle': suc.DirCalle, 'colonia': suc.DirCol, 'cp': suc.DirCp,'entidad_id': suc.DirEnt, 'entidad': ent,'municipio_id': suc.DirMun, 'municipio': mun} empresas = Empresa.all().filter("IdEmp = ", suc.IdEmp) for empresa in empresas.run(limit=1): empresadict = {'id': empresa.IdEmp, 'nombre': empresa.Nombre, 'url': empresa.Url, 'url_logo': ''} sucdict['empresa'] = empresadict urllogo = 'http://www.elbuenfin.org/imgs/imageDefault.png' ofertaslist = [] for o in olist: ofertas = Oferta.all().filter("IdOft =", o).run() for oferta in ofertas: url = 'http://www.elbuenfin.org/imgs/imageDefault.png' try: if oferta.Codigo and oferta.Codigo.replace('https://','http://')[0:7] == 'http://': url = oferta.Codigo except AttributeError: err = 'logourl' try: if oferta.Codigo is None and oferta.BlobKey and oferta.BlobKey != None and oferta.BlobKey.key() != 'none': url = 'http://' + APPID + '/ofimg?id=' + str(oferta.BlobKey.key()) except AttributeError: err = 'logourl' if url == 'http://www.elbuenfin.org/imgs/imageDefault.png' and oferta.Promocion is not None and oferta.Promocion != '': url = oferta.Promocion if oferta.Promocion is not None and oferta.Promocion != '': urllogo = oferta.Promocion ofertadict = {'id': oferta.IdOft, 'oferta': oferta.Oferta, 'descripcion': oferta.Descripcion, 'descuento': oferta.Descuento, 'promocion': oferta.Promocion, 'enlinea': oferta.Enlinea, 'precio': oferta.Precio, 'url': oferta.Url, 'url_logo': url, 'fechapub': str(oferta.FechaHoraPub.strftime('%Y-%m-%d'))} palabraslist = [] palabras = OfertaPalabra.all().filter("IdOft =", oferta.IdOft) for palabra in palabras: palabraslist.append(palabra.Palabra) ofertadict['palabras'] = palabraslist cat = None categorias = Categoria.all().filter("IdCat =", oferta.IdCat) for categoria in categorias: cat = categoria.Categoria ofertadict['categoria_id'] = oferta.IdCat ofertadict['categoria'] = cat ofertaslist.append(ofertadict) sucdict['ofertas'] = ofertaslist sucdict['empresa']['url_logo'] = urllogo mvblob = MvBlob() mvblob.FechaHora = suc.FechaHora mvblob.IdSuc = suc.IdSuc mvblob.Blob = json.dumps(sucdict) mvblob.put() else: pass
def get(self): olist = [] count = 0 for o in Oferta.all(): count += 1 self.response.out.write(count)
def token(self): token = self.request.get('token') if token and str(token) == 'ZWJmbWV4LXB1YnIeCxISX0FoQWRtaW5Yc3JmVG9rZW5fIgZfWFNSRl8M': try: gminutes = self.request.get('minutes') ghours = self.request.get('hours') gdays = self.request.get('days') if not gminutes: gminutes = 0 else: gminutes = int(gminutes) if not ghours: ghours = 0 else: ghours = int(ghours) if not gdays: gdays = 0 else: gdays = int(gdays) except ValueError: gminutes = 30 ghours = 0 gdays = 0 time = datetime.now() - timedelta(days = gdays, hours = ghours, minutes = gminutes) self.response.headers['Content-Type'] = 'application/json' #self.response.out.write (str(time)) changecontrol = ChangeControl.all().filter("FechaHora >=", time).filter("Kind =", 'Oferta').filter("Status IN", ["A","M"]) nbadded = 0 nbremoved = 0 for cc in changecontrol: #self.response.out.write(cc.Id + '\n') if cc.Status != 'B': ofertas = Oferta.all().filter("IdOft =", cc.Id) for oferta in ofertas: if cc.Status == 'M': searchdata = SearchData.all().filter("Sid =", str(oferta.key())) for sd in searchdata: db.delete(sd) nbremoved += 1 desc = oferta.Descripcion.replace('\n',' ').replace('\r',' ').replace('.',' ').replace(',',' ').split(' ') nombre = oferta.Oferta.replace('.',' ').replace(',',' ').split(' ') for palabra in desc: if len(palabra) > 3: newsd = SearchData() newsd.Enlinea = oferta.Enlinea newsd.FechaHora = datetime.now() - timedelta(hours = H) newsd.Field = 'Descripcion' newsd.IdCat = oferta.IdCat newsd.Kind = 'Oferta' newsd.Sid = str(oferta.key()) newsd.Value = palabra.lower() newsd.put() nbadded += 1 for palabra in nombre: if len(palabra) > 3: newsd = SearchData() newsd.Enlinea = oferta.Enlinea newsd.FechaHora = datetime.now() - timedelta(hours = H) newsd.Field = 'Oferta' newsd.IdCat = oferta.IdCat newsd.Kind = 'Oferta' newsd.Sid = str(oferta.key()) newsd.Value = palabra.lower() newsd.put() nbadded += 1 palabraclave = OfertaPalabra.all().filter("IdOft =", oferta.IdOft) for palabra in palabraclave.Palabra: if len(palabra) > 3: newsd = SearchData() newsd.Enlinea = oferta.Enlinea newsd.FechaHora = datetime.now() - timedelta(hours = H) newsd.Field = 'OfertaPalabra' newsd.IdCat = oferta.IdCat newsd.Kind = 'Oferta' newsd.Sid = str(oferta.key()) newsd.Value = palabra.Palabra.lower() newsd.put() nbadded += 1 logging.info("Finished updating. Added: %s. Removed: %s.", str(nbadded), str(nbremoved)) else: logging.error('Wrong token given.')