def get(self): groupidList=[] result = urlfetch.fetch( url =setting.WEBURL+'/NeedSyncGuPiao', # payload = login_data, method = urlfetch.GET, headers = {'Content-Type':'application/x-www-form-urlencoded', 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6'}, follow_redirects = False,deadline=20) if result.status_code == 200 : logging.info(result.content) groupidList.extend(result.content.split(',')) if groupidList: needGuPiao=memcache.get('syncgupiao') if not needGuPiao: needGuPiao=NeedSyncGuPiao.get_by_key_name('syncgupiao') if needGuPiao : for groupid in groupidList: gupiaoGroup=memcache.get('group'+groupid) if not gupiaoGroup: gupiaoGroup=GuPiaoGroup.get_by_key_name('g'+groupid) if gupiaoGroup: memcache.set('group'+groupid,gupiaoGroup,36000) if gupiaoGroup.realNo not in needGuPiao.gpcode: needGuPiao.gpcode.append(gupiaoGroup.realNo) needGuPiao.memcachegroupid.append("needsyncgupiao_groupid%s"%gupiaoGroup.key().name()) date=datetime.datetime.utcnow()+datetime.timedelta(hours =8) needGuPiao.date=date.strftime('%Y-%m-%d') needGuPiao.put() memcache.set('syncgupiao',needGuPiao,36000)
def get(self): groupid=self.request.get('groupid') dm=self.request.get('dm') type=self.request.get('type') realNo=self.request.get('realNo') gupiaoGroup=memcache.get('group'+groupid) if not gupiaoGroup: gupiaoGroup=GuPiaoGroup.get_by_key_name('g'+groupid) if gupiaoGroup: memcache.set('group'+groupid,gupiaoGroup,36000) if not gupiaoGroup: if groupid and dm and type and realNo: gupiaoGroup=GuPiaoGroup(key_name='g'+groupid) gupiaoGroup.gpcode=dm gupiaoGroup.realNo=realNo gupiaoGroup.type=type gupiaoGroup.put() memcache.set('group'+groupid,gupiaoGroup,36000) else: self.error(500) return needGuPiao=memcache.get('syncgupiao') if not needGuPiao: needGuPiao=NeedSyncGuPiao.get_by_key_name('syncgupiao') if needGuPiao and gupiaoGroup.realNo not in needGuPiao.gpcode: needGuPiao.gpcode.append(gupiaoGroup.realNo) needGuPiao.memcachegroupid.append("needsyncgupiao_groupid%s"%gupiaoGroup.key().name()) needGuPiao.put() memcache.set('syncgupiao',needGuPiao,36000) baseurl='http://hq.sinajs.cn/list=' result = urlfetch.fetch( url =baseurl+realNo, method = urlfetch.GET, headers = {'Content-Type':'application/x-www-form-urlencoded', 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6'}, follow_redirects = False,deadline=20) if result.status_code == 200 : for gupiaostr in result.content.decode('gbk').split('\n'): if gupiaostr: gupiao_data_arr=gupiaostr[11:].split('=') # gupiao_group=memcache.get(gupiao_data_arr[0]) # if not gupiao_group: # gupiao_group=GuPiaoGroup.all().filter('realNo =',gupiao_data_arr[0]).fetch(1) # if len(gupiao_group)==1: # gupiao_group=gupiao_group[0] # memcache.set(gupiao_group.realNo,gupiao_group,3600*24*3) # else: # gupiao_group=None # if gupiao_group: # groupid=gupiao_group.key().name() # gupiaolist.append(groupid) # post_data['flag'+groupid]=gupiao_data_arr[-25:] self.response.out.write("{'groupid':'%s','realNo':'%s','type':'%s','min':'[*sys/min_%s_%s/a777_1*]','daily':'[*sys/daily_%s_%s/a777_1*]','data':'%s'}"%(groupid,realNo,type,realNo,type,realNo,type,gupiao_data_arr[1][1:-2]))
def get(self): needGuPiao=NeedSyncGuPiao.get_by_key_name('syncgupiao') if needGuPiao: date=datetime.datetime.utcnow()+datetime.timedelta(hours =8) if date.strftime('%Y-%m-%d')!=needGuPiao.date: pam={} pam['needdelgroupid']=','.join(needGuPiao.memcachegroupid) login_data=urllib.urlencode(pam) result = urlfetch.fetch( url =setting.WEBURL+'/DeleteNeedSyncGuPiao', payload = login_data, method = urlfetch.POST, headers = {'Content-Type':'application/x-www-form-urlencoded', 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6'}, follow_redirects = False,deadline=20) if result.status_code == 200 : needGuPiao.memcachegroupid=[] needGuPiao.gpcode=[] needGuPiao.put() memcache.delete('syncgupiao')
def get(self): memcacheFlag=self.request.get('memcache','') if memcacheFlag: urllist=memcache.get('needsyncgupiao') if not urllist: urllist=[] else: needGuPiao=NeedSyncGuPiao.get_by_key_name('syncgupiao') if not needGuPiao: needGuPiao=NeedSyncGuPiao(key_name='syncgupiao') needGuPiao.put() baseurl='http://hq.sinajs.cn/list=' # for gp in needGuPiao.gpcode: tempurl=baseurl urllist=[] for gp in needGuPiao.gpcode: if len(tempurl)+len(gp)>2047: urllist.append(tempurl) tempurl=baseurl tempurl+='%s,'%gp if tempurl==baseurl and 0==len(urllist): return if tempurl!=baseurl: urllist.append(tempurl) memcache.set('needsyncgupiao',urllist[10:],3600) resultlist=[] rpcs=[] for url in urllist[:10]: rpc = urlfetch.create_rpc(deadline=30) rpc.callback = self.create_callback(rpc,resultlist) urlfetch.make_fetch_call(rpc, url,headers = {'Content-Type':'application/x-www-form-urlencoded', 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6'}) rpcs.append(rpc) for rpc in rpcs: rpc.wait() gpstrlist=[] for gupiaoArr in resultlist: gpstrlist.extend(gupiaoArr.split('\n')) #self.response.out.write(u'%s'%gupiaoArr) gupiaolist=[] post_data={} tmpgupiaostr=None for gupiaostr in gpstrlist: if not gupiaostr: continue if gupiaostr.find('var hq_str_s')==0: tmpgupiaostr=gupiaostr.split('=')[0]+gupiaostr[-15:-8] else: tmpgupiaostr=gupiaostr flag=memcache.get(tmpgupiaostr) if flag: continue memcache.set(tmpgupiaostr,'flag',360000) gupiao_data_arr=gupiaostr[11:].split('=') gupiao_group=memcache.get(gupiao_data_arr[0]) if not gupiao_group: gupiao_group=GuPiaoGroup.all().filter('realNo =',gupiao_data_arr[0]).fetch(1) if len(gupiao_group)==1: gupiao_group=gupiao_group[0] memcache.set(gupiao_group.realNo,gupiao_group,3600*24*3) else: gupiao_group=None if gupiao_group: groupid=gupiao_group.key().name() gupiaolist.append(groupid) # post_data['flag'+groupid]=gupiao_data_arr[-25:] post_data[groupid]="{'groupid':'%s','realNo':'%s','type':'%s','min':'[*sys/min_%s_%s/a777_1*]','daily':'[*sys/daily_%s_%s/a777_1*]','data':'%s'}"%(groupid[1:],gupiao_group.realNo,gupiao_group.type,gupiao_group.realNo,gupiao_group.type,gupiao_group.realNo,gupiao_group.type,gupiao_data_arr[1][1:-2]) post_data[groupid]=post_data[groupid].encode('gbk').decode('gbk').encode('utf-8') #json.dumps({'groupid': groupid[1:], 'realNo': gupiao_group.realNo, # 'type': gupiao_group.type, 'data': gupiao_data_arr[1][1:-2]}) post_data['groupids']=','.join(gupiaolist) result = urlfetch.fetch( url =setting.WEBURL+'/SyncGuPiao', payload = urllib.urlencode(post_data), method = urlfetch.POST, headers = {'Content-Type':'application/x-www-form-urlencoded', 'User-Agent':'Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.9.2) Gecko/20100115 Firefox/3.6'}, follow_redirects = False,deadline=30) if result.status_code == 200 : return