def transferData(self,end,file_path) :
FILE = file_path.split('/')[-1]
with open(file_path,'rb') as f :
data = f.read()
time_Stamp = helper.md5(time.time())[:6] + helper.md5(data[:100])[:6]
client = helper.rpc_client()
reply = json.loads(client.call('GET ROUTE' + json.dumps({'start':self.id,'end':end})))
packets_seq = [ '!_$@$_!' + FILE + '!_$@$_!' + str(i) + '!_$@$_!' + time_Stamp + '!_$@$_!' for i in range((len(data)/MAX_DATA_SIZE)+1)]
client = socket.socket(socket.AF_INET,socket.SOCK_DGRAM)
count = 00
for item in packets_seq :
client.sendto('FWRD DATA' + json.dumps({'next':reply['next'],'start':self.id,'end':end}) + '!_$@damei@$_!' + FILE + '!_$@damei@$_!' + str(count) + '!_$@damei@$_!' + time_Stamp + '!_$@damei@$_!' + data[(count)*MAX_DATA_SIZE:(count+1)*MAX_DATA_SIZE],(reply['server_ip'],reply['server_port']))
ack_data , ack_addr = client.recvfrom(MAX_DATA_SIZE)
count += 1
client.sendto('FWRD DATA' + json.dumps({'next':reply['next'],'start':self.id,'end':end}) + '!_$@damei@$_!' + FILE + '!_$@damei@$_!ENDENDEND!_$@damei@$_!' + time_Stamp + '!_$@damei@$_!',(reply['server_ip'],reply['server_port']))
ack_data , ack_addr = client.recvfrom(MAX_DATA_SIZE)
def transferData(self, end, file_path):
FILE = file_path.split("/")[-1]
with open(file_path, "rb") as f:
data = f.read()
time_Stamp = helper.md5(time.time())[:6] + helper.md5(data[:100])[:6]
client = helper.rpc_client()
reply = json.loads(client.call("GET ROUTE" + json.dumps({"start": self.id, "end": end})))
packets_seq = [
"!_$@$_!" + FILE + "!_$@$_!" + str(i) + "!_$@$_!" + time_Stamp + "!_$@$_!"
for i in range((len(data) / MAX_DATA_SIZE) + 1)
]
client = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
count = 00
for item in packets_seq:
client.sendto(
"FWRD DATA"
+ json.dumps({"next": reply["next"], "start": self.id, "end": end})
+ "!_$@damei@$_!"
+ FILE
+ "!_$@damei@$_!"
+ str(count)
+ "!_$@damei@$_!"
+ time_Stamp
+ "!_$@damei@$_!"
+ data[(count) * MAX_DATA_SIZE : (count + 1) * MAX_DATA_SIZE],
(reply["server_ip"], reply["server_port"]),
)
ack_data, ack_addr = client.recvfrom(MAX_DATA_SIZE)
count += 1
client.sendto(
"FWRD DATA"
+ json.dumps({"next": reply["next"], "start": self.id, "end": end})
+ "!_$@damei@$_!"
+ FILE
+ "!_$@damei@$_!ENDENDEND!_$@damei@$_!"
+ time_Stamp
+ "!_$@damei@$_!",
(reply["server_ip"], reply["server_port"]),
)
ack_data, ack_addr = client.recvfrom(MAX_DATA_SIZE)
def get_topic_information(self, topic):
base_url = 'http://weibo.com/p/100808'
topic = topic.replace('#', '')
topic = topic.replace('\r', '')
topic = topic.replace('\t', '')
complete_url = base_url + md5(topic) #+'?k='+topic
print('Now get html for topic information')
html = self.get_html(complete_url, self.all_headers['huati_headers'])
print('Got html for topic information')
if html == '':
print('Error when getting topic information')
print(complete_url)
return None
information = dict()
information['topic_name'] = topic.replace('#', '').decode('utf8')
counts = get_counts(html)
print counts
if (counts == None):
self.output_keke(html)
return None
information['readers_count'] = counts[0]
information['discuss_count'] = counts[1]
information['followers_count'] = counts[2]
attributes = get_category_location_tag(html)
print attributes
if (attributes == None):
self.output_keke(html)
return None
information['category'] = ';'.join(attributes[0])
information['location'] = ';'.join(attributes[1])
information['tags'] = ';'.join(attributes[2])
emcee = get_emcee(html)
print emcee
if (emcee == None):
self.output_keke(html)
return None
information['host'] = emcee
description = get_description(html)
print description
if (description == None):
self.output_keke(html)
return None
information['description'] = description
print 'Success to get topic information'
return information
def get_topic_information(self, topic):
base_url='http://weibo.com/p/100808'
topic=topic.replace('#','')
topic=topic.replace('\r','')
topic=topic.replace('\t','')
complete_url=base_url+md5(topic)#+'?k='+topic
print('Now get html for topic information')
html=self.get_html(complete_url, self.all_headers['huati_headers'])
print('Got html for topic information')
if html=='':
print('Error when getting topic information')
print(complete_url)
return None
information=dict()
information['topic_name']=topic.replace('#','').decode('utf8')
counts=get_counts(html)
print counts
if(counts==None):
self.output_keke(html)
return None
information['readers_count']=counts[0]
information['discuss_count']=counts[1]
information['followers_count']=counts[2]
attributes=get_category_location_tag(html)
print attributes
if(attributes==None):
self.output_keke(html)
return None
information['category']=';'.join(attributes[0])
information['location']=';'.join(attributes[1])
information['tags']=';'.join(attributes[2])
emcee=get_emcee(html)
print emcee
if(emcee==None):
self.output_keke(html)
return None
information['host']=emcee
description=get_description(html)
print description
if(description==None):
self.output_keke(html)
return None
information['description']=description
print 'Success to get topic information'
return information
def parseJs(filePath):
# remove all file
md5filename=helper.md5(filePath)
saveDir=os.path.join(SAVE_DIR,md5filename)
deleteFiles(saveDir,saveDir)
# create dir
if not os.path.exists(saveDir):
os.makedirs(saveDir)
# add filepath to filepath.txt for debug
filepath=os.path.join(saveDir,"filepath.txt")
helper.writeFile(filepath,filePath)
f=codecs.open(filePath,"r","utf-8")
lineNum=0
while True:
line=f.readline()
if line:
lineNum+=1
# function
m=re.match("^\s*function\s*(\w+)\s*\((.*)\)",line)
m2=re.match("^\s*(\w+)\:\s*function\s*\((.*)\)",line)
m3=re.match("^\s*(\w+\.\w+)\s*=\s*function\s*\((.*)\)",line)
if m2:
m=m2
elif m3:
m=m3
if m:
handleFunction(saveDir,"",m.group(1),m.group(2))
handleDefinition(m.group(1),filePath,lineNum)
continue
m=re.match("^\s*(\w+)\.prototype\.(\w+)\s*=\s*function\s*\((.*)\)",line)
if m:
handleFunction(saveDir,m.group(1),m.group(2),m.group(3))
handleDefinition(m.group(2),filePath,lineNum)
continue
# vars
m=re.match("^\s*var\s+(\w+)",line)
m2=re.match("^\s*(\w+\.\w+)\s*=",line)
if m2:
m=m2
if m:
handleVar(saveDir,m.group(1))
handleDefinition(m.group(1),filePath,lineNum)
continue
else:
break
f.close()
def parseJs(file):
# remove all file
md5filename = helper.md5(file)
saveDir = os.path.join(SAVE_DIR, md5filename)
deleteFiles(saveDir, saveDir)
# create dir
if not os.path.exists(saveDir):
os.makedirs(saveDir)
filepath = os.path.join(saveDir, "filepath.txt")
helper.writeFile(filepath, file)
end_index = file.rfind("/")
className = file[end_index + 1:-3]
completionsList = []
with open(file, "r", encoding='utf-8') as f:
for line in f:
#sample: onLoad: function () {
m = re.match(' +(\w+): *function *\((.*?)\).*', line)
if m:
saveFunction(saveDir, className, m.group(1), m.group(2))
continue
#sample: ComFun.dump = function(arr, maxLevel) {
m = re.match(' *([a-zA-Z0-9\.]*) = function *\((.*?)\).*', line)
if m:
saveFunction(saveDir, className, m.group(1), m.group(2))
continue
#sample var Constant = {}
m = re.match('^var (.*) *=.*', line)
if m:
completionsList.append(m.group(1).strip())
continue
#sample Constant.Enums = something
m = re.match('^([\w\.]*) *=.*', line)
if m:
completionsList.append(m.group(1).strip())
if "module.exports" in completionsList:
completionsList.remove("module.exports")
saveCompletions(completionsList, saveDir, "c")
def parseLua(file):
# remove all file
md5filename = helper.md5(file)
saveDir = os.path.join(SAVE_DIR, md5filename)
deleteFiles(saveDir, saveDir)
# create dir
if not os.path.exists(saveDir):
os.makedirs(saveDir)
# add filepath to filepath.txt for debug
filepath = os.path.join(saveDir, "filepath.txt")
helper.writeFile(filepath, file)
completionsList = []
f = codecs.open(file, "r", "utf-8")
lineNum = 0
while True:
line = f.readline()
if line:
lineNum += 1
# class
m = re.match("^local\s+(\w+)\s*=\s*\{\}", line)
if m:
completionsList.append(m.group(1))
handleDefinition(m.group(1), None, file, lineNum)
continue
m = re.match("^local\s+(\w+)\s*=\s*class\(", line)
if m:
completionsList.append(m.group(1))
handleDefinition(m.group(1), None, file, lineNum)
continue
m = re.match("^(\w+)\s*=\s*class\(", line)
if m:
completionsList.append(m.group(1))
handleDefinition(m.group(1), None, file, lineNum)
continue
# function
m = re.match("^function\s+(\w+\.*\w*)\s*\((.*)\)", line)
if m:
saveFunction(saveDir, "", m.group(1), m.group(2))
handleDefinition(m.group(1), m.group(2), file, lineNum)
continue
# class function
m = re.match("^function\s+(\w+)\:(\w+)\s*\((.*)\)", line)
if m:
method = m.group(2)
if method == "ctor":
continue
saveFunction(saveDir, m.group(1), method, m.group(3))
handleDefinition(m.group(2), m.group(3), file, lineNum, m.group(1) + ":" + m.group(2))
continue
# local property
m = re.match("^\s*local\s+(\w+)\s*", line)
if m:
completionsList.append(m.group(1))
continue
m = re.match("^\s*(self\.\w+)\s*=", line)
if m:
completionsList.append(m.group(1))
continue
# global property
m = re.match("^(\w+\.?\w*)\s*=", line)
if m:
completionsList.append(m.group(1))
handleDefinition(m.group(1), None, file, lineNum)
continue
else:
break
f.close()
saveCompletions(completionsList, saveDir, "c")
def build_mysign(prestr, key=None, sign_type='MD5'):
if sign_type == 'MD5':
prestr += '&key=%s' % str(key)
return md5(prestr).hexdigest().upper()
return ''
def parseLua(file):
# remove all file
md5filename=helper.md5(file)
saveDir=os.path.join(SAVE_DIR,md5filename)
deleteFiles(saveDir,saveDir)
# create dir
if not os.path.exists(saveDir):
os.makedirs(saveDir)
# add filepath to filepath.txt for debug
filepath=os.path.join(saveDir,"filepath.txt")
helper.writeFile(filepath,file)
completionsList=[]
f=codecs.open(file,"r","utf-8")
lineNum=0
while True:
line=f.readline()
if line:
lineNum+=1
# class
m=re.match("^local\s+(\w+)\s*=\s*\{\}",line)
if m:
completionsList.append(m.group(1))
handleDefinition(m.group(1),None,file,lineNum)
continue
m=re.match("^local\s+(\w+)\s*=\s*class\(",line)
if m:
completionsList.append(m.group(1))
handleDefinition(m.group(1),None,file,lineNum)
continue
m=re.match("^(\w+)\s*=\s*class\(",line)
if m:
completionsList.append(m.group(1))
handleDefinition(m.group(1),None,file,lineNum)
continue
# function
m=re.match("^function\s+(\w+\.*\w*)\s*\((.*)\)",line)
if m:
saveFunction(saveDir,"",m.group(1),m.group(2))
handleDefinition(m.group(1),m.group(2),file,lineNum)
continue
# class function
m=re.match("^function\s+(\w+)\:(\w+)\s*\((.*)\)",line)
if m:
method=m.group(2)
if method=="ctor":
continue
saveFunction(saveDir,m.group(1),method,m.group(3))
handleDefinition(m.group(2),m.group(3),file,lineNum,m.group(1)+":"+m.group(2))
continue
# local property
m=re.match("^\s*local\s+(\w+)\s*",line)
if m:
completionsList.append(m.group(1))
continue
m=re.match("^\s*(self\.\w+)\s*=",line)
if m:
completionsList.append(m.group(1))
continue
# global property
m=re.match("^(\w+\.?\w*)\s*=",line)
if m:
completionsList.append(m.group(1))
handleDefinition(m.group(1),None,file,lineNum)
continue
else:
break
f.close()
saveCompletions(completionsList,saveDir,"c")
