def transferData(self,end,file_path) : FILE = file_path.split('/')[-1] with open(file_path,'rb') as f : data = f.read() time_Stamp = helper.md5(time.time())[:6] + helper.md5(data[:100])[:6] client = helper.rpc_client() reply = json.loads(client.call('GET ROUTE' + json.dumps({'start':self.id,'end':end}))) packets_seq = [ '!_$@$_!' + FILE + '!_$@$_!' + str(i) + '!_$@$_!' + time_Stamp + '!_$@$_!' for i in range((len(data)/MAX_DATA_SIZE)+1)] client = socket.socket(socket.AF_INET,socket.SOCK_DGRAM) count = 00 for item in packets_seq : client.sendto('FWRD DATA' + json.dumps({'next':reply['next'],'start':self.id,'end':end}) + '!_$@damei@$_!' + FILE + '!_$@damei@$_!' + str(count) + '!_$@damei@$_!' + time_Stamp + '!_$@damei@$_!' + data[(count)*MAX_DATA_SIZE:(count+1)*MAX_DATA_SIZE],(reply['server_ip'],reply['server_port'])) ack_data , ack_addr = client.recvfrom(MAX_DATA_SIZE) count += 1 client.sendto('FWRD DATA' + json.dumps({'next':reply['next'],'start':self.id,'end':end}) + '!_$@damei@$_!' + FILE + '!_$@damei@$_!ENDENDEND!_$@damei@$_!' + time_Stamp + '!_$@damei@$_!',(reply['server_ip'],reply['server_port'])) ack_data , ack_addr = client.recvfrom(MAX_DATA_SIZE)
def transferData(self, end, file_path): FILE = file_path.split("/")[-1] with open(file_path, "rb") as f: data = f.read() time_Stamp = helper.md5(time.time())[:6] + helper.md5(data[:100])[:6] client = helper.rpc_client() reply = json.loads(client.call("GET ROUTE" + json.dumps({"start": self.id, "end": end}))) packets_seq = [ "!_$@$_!" + FILE + "!_$@$_!" + str(i) + "!_$@$_!" + time_Stamp + "!_$@$_!" for i in range((len(data) / MAX_DATA_SIZE) + 1) ] client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) count = 00 for item in packets_seq: client.sendto( "FWRD DATA" + json.dumps({"next": reply["next"], "start": self.id, "end": end}) + "!_$@damei@$_!" + FILE + "!_$@damei@$_!" + str(count) + "!_$@damei@$_!" + time_Stamp + "!_$@damei@$_!" + data[(count) * MAX_DATA_SIZE : (count + 1) * MAX_DATA_SIZE], (reply["server_ip"], reply["server_port"]), ) ack_data, ack_addr = client.recvfrom(MAX_DATA_SIZE) count += 1 client.sendto( "FWRD DATA" + json.dumps({"next": reply["next"], "start": self.id, "end": end}) + "!_$@damei@$_!" + FILE + "!_$@damei@$_!ENDENDEND!_$@damei@$_!" + time_Stamp + "!_$@damei@$_!", (reply["server_ip"], reply["server_port"]), ) ack_data, ack_addr = client.recvfrom(MAX_DATA_SIZE)
def get_topic_information(self, topic): base_url = 'http://weibo.com/p/100808' topic = topic.replace('#', '') topic = topic.replace('\r', '') topic = topic.replace('\t', '') complete_url = base_url + md5(topic) #+'?k='+topic print('Now get html for topic information') html = self.get_html(complete_url, self.all_headers['huati_headers']) print('Got html for topic information') if html == '': print('Error when getting topic information') print(complete_url) return None information = dict() information['topic_name'] = topic.replace('#', '').decode('utf8') counts = get_counts(html) print counts if (counts == None): self.output_keke(html) return None information['readers_count'] = counts[0] information['discuss_count'] = counts[1] information['followers_count'] = counts[2] attributes = get_category_location_tag(html) print attributes if (attributes == None): self.output_keke(html) return None information['category'] = ';'.join(attributes[0]) information['location'] = ';'.join(attributes[1]) information['tags'] = ';'.join(attributes[2]) emcee = get_emcee(html) print emcee if (emcee == None): self.output_keke(html) return None information['host'] = emcee description = get_description(html) print description if (description == None): self.output_keke(html) return None information['description'] = description print 'Success to get topic information' return information
def get_topic_information(self, topic): base_url='http://weibo.com/p/100808' topic=topic.replace('#','') topic=topic.replace('\r','') topic=topic.replace('\t','') complete_url=base_url+md5(topic)#+'?k='+topic print('Now get html for topic information') html=self.get_html(complete_url, self.all_headers['huati_headers']) print('Got html for topic information') if html=='': print('Error when getting topic information') print(complete_url) return None information=dict() information['topic_name']=topic.replace('#','').decode('utf8') counts=get_counts(html) print counts if(counts==None): self.output_keke(html) return None information['readers_count']=counts[0] information['discuss_count']=counts[1] information['followers_count']=counts[2] attributes=get_category_location_tag(html) print attributes if(attributes==None): self.output_keke(html) return None information['category']=';'.join(attributes[0]) information['location']=';'.join(attributes[1]) information['tags']=';'.join(attributes[2]) emcee=get_emcee(html) print emcee if(emcee==None): self.output_keke(html) return None information['host']=emcee description=get_description(html) print description if(description==None): self.output_keke(html) return None information['description']=description print 'Success to get topic information' return information
def parseJs(filePath): # remove all file md5filename=helper.md5(filePath) saveDir=os.path.join(SAVE_DIR,md5filename) deleteFiles(saveDir,saveDir) # create dir if not os.path.exists(saveDir): os.makedirs(saveDir) # add filepath to filepath.txt for debug filepath=os.path.join(saveDir,"filepath.txt") helper.writeFile(filepath,filePath) f=codecs.open(filePath,"r","utf-8") lineNum=0 while True: line=f.readline() if line: lineNum+=1 # function m=re.match("^\s*function\s*(\w+)\s*\((.*)\)",line) m2=re.match("^\s*(\w+)\:\s*function\s*\((.*)\)",line) m3=re.match("^\s*(\w+\.\w+)\s*=\s*function\s*\((.*)\)",line) if m2: m=m2 elif m3: m=m3 if m: handleFunction(saveDir,"",m.group(1),m.group(2)) handleDefinition(m.group(1),filePath,lineNum) continue m=re.match("^\s*(\w+)\.prototype\.(\w+)\s*=\s*function\s*\((.*)\)",line) if m: handleFunction(saveDir,m.group(1),m.group(2),m.group(3)) handleDefinition(m.group(2),filePath,lineNum) continue # vars m=re.match("^\s*var\s+(\w+)",line) m2=re.match("^\s*(\w+\.\w+)\s*=",line) if m2: m=m2 if m: handleVar(saveDir,m.group(1)) handleDefinition(m.group(1),filePath,lineNum) continue else: break f.close()
def parseJs(file): # remove all file md5filename = helper.md5(file) saveDir = os.path.join(SAVE_DIR, md5filename) deleteFiles(saveDir, saveDir) # create dir if not os.path.exists(saveDir): os.makedirs(saveDir) filepath = os.path.join(saveDir, "filepath.txt") helper.writeFile(filepath, file) end_index = file.rfind("/") className = file[end_index + 1:-3] completionsList = [] with open(file, "r", encoding='utf-8') as f: for line in f: #sample: onLoad: function () { m = re.match(' +(\w+): *function *\((.*?)\).*', line) if m: saveFunction(saveDir, className, m.group(1), m.group(2)) continue #sample: ComFun.dump = function(arr, maxLevel) { m = re.match(' *([a-zA-Z0-9\.]*) = function *\((.*?)\).*', line) if m: saveFunction(saveDir, className, m.group(1), m.group(2)) continue #sample var Constant = {} m = re.match('^var (.*) *=.*', line) if m: completionsList.append(m.group(1).strip()) continue #sample Constant.Enums = something m = re.match('^([\w\.]*) *=.*', line) if m: completionsList.append(m.group(1).strip()) if "module.exports" in completionsList: completionsList.remove("module.exports") saveCompletions(completionsList, saveDir, "c")
def parseLua(file): # remove all file md5filename = helper.md5(file) saveDir = os.path.join(SAVE_DIR, md5filename) deleteFiles(saveDir, saveDir) # create dir if not os.path.exists(saveDir): os.makedirs(saveDir) # add filepath to filepath.txt for debug filepath = os.path.join(saveDir, "filepath.txt") helper.writeFile(filepath, file) completionsList = [] f = codecs.open(file, "r", "utf-8") lineNum = 0 while True: line = f.readline() if line: lineNum += 1 # class m = re.match("^local\s+(\w+)\s*=\s*\{\}", line) if m: completionsList.append(m.group(1)) handleDefinition(m.group(1), None, file, lineNum) continue m = re.match("^local\s+(\w+)\s*=\s*class\(", line) if m: completionsList.append(m.group(1)) handleDefinition(m.group(1), None, file, lineNum) continue m = re.match("^(\w+)\s*=\s*class\(", line) if m: completionsList.append(m.group(1)) handleDefinition(m.group(1), None, file, lineNum) continue # function m = re.match("^function\s+(\w+\.*\w*)\s*\((.*)\)", line) if m: saveFunction(saveDir, "", m.group(1), m.group(2)) handleDefinition(m.group(1), m.group(2), file, lineNum) continue # class function m = re.match("^function\s+(\w+)\:(\w+)\s*\((.*)\)", line) if m: method = m.group(2) if method == "ctor": continue saveFunction(saveDir, m.group(1), method, m.group(3)) handleDefinition(m.group(2), m.group(3), file, lineNum, m.group(1) + ":" + m.group(2)) continue # local property m = re.match("^\s*local\s+(\w+)\s*", line) if m: completionsList.append(m.group(1)) continue m = re.match("^\s*(self\.\w+)\s*=", line) if m: completionsList.append(m.group(1)) continue # global property m = re.match("^(\w+\.?\w*)\s*=", line) if m: completionsList.append(m.group(1)) handleDefinition(m.group(1), None, file, lineNum) continue else: break f.close() saveCompletions(completionsList, saveDir, "c")
def build_mysign(prestr, key=None, sign_type='MD5'): if sign_type == 'MD5': prestr += '&key=%s' % str(key) return md5(prestr).hexdigest().upper() return ''
def parseLua(file): # remove all file md5filename=helper.md5(file) saveDir=os.path.join(SAVE_DIR,md5filename) deleteFiles(saveDir,saveDir) # create dir if not os.path.exists(saveDir): os.makedirs(saveDir) # add filepath to filepath.txt for debug filepath=os.path.join(saveDir,"filepath.txt") helper.writeFile(filepath,file) completionsList=[] f=codecs.open(file,"r","utf-8") lineNum=0 while True: line=f.readline() if line: lineNum+=1 # class m=re.match("^local\s+(\w+)\s*=\s*\{\}",line) if m: completionsList.append(m.group(1)) handleDefinition(m.group(1),None,file,lineNum) continue m=re.match("^local\s+(\w+)\s*=\s*class\(",line) if m: completionsList.append(m.group(1)) handleDefinition(m.group(1),None,file,lineNum) continue m=re.match("^(\w+)\s*=\s*class\(",line) if m: completionsList.append(m.group(1)) handleDefinition(m.group(1),None,file,lineNum) continue # function m=re.match("^function\s+(\w+\.*\w*)\s*\((.*)\)",line) if m: saveFunction(saveDir,"",m.group(1),m.group(2)) handleDefinition(m.group(1),m.group(2),file,lineNum) continue # class function m=re.match("^function\s+(\w+)\:(\w+)\s*\((.*)\)",line) if m: method=m.group(2) if method=="ctor": continue saveFunction(saveDir,m.group(1),method,m.group(3)) handleDefinition(m.group(2),m.group(3),file,lineNum,m.group(1)+":"+m.group(2)) continue # local property m=re.match("^\s*local\s+(\w+)\s*",line) if m: completionsList.append(m.group(1)) continue m=re.match("^\s*(self\.\w+)\s*=",line) if m: completionsList.append(m.group(1)) continue # global property m=re.match("^(\w+\.?\w*)\s*=",line) if m: completionsList.append(m.group(1)) handleDefinition(m.group(1),None,file,lineNum) continue else: break f.close() saveCompletions(completionsList,saveDir,"c")