def get_module(): modindex = Modindex.objects.all() modindex.delete() salt_modindex = get_modindex('http://docs.saltstack.com/en/latest/salt-modindex.html') print salt_modindex for mod in salt_modindex: url = 'http://docs.saltstack.com/en/latest/ref/modules/all/' + mod + '.html#module-' + mod print url request = urllib2.Request(url, None, req_header) try: response = urllib2.urlopen(request) except: time.sleep(10) response = urllib2.urlopen(request) index = response.read().decode('utf-8') pattern_name = re.compile('<code class="descname">(.*)</code>') #pattern_info = re.compile('<dd><p>(.*)</p>') #pattern_example = re.compile('<pre>(.*)') items_name = re.findall(pattern_name, index) #items_info = re.findall(pattern_info,index) #items_example = re.findall(pattern_example,index) mod_fun = mod.split('.')[2] print mod_fun print items_name #print items_info #print items_example for i in items_name: modindex = Modindex() if mod_fun == 'sysmod': mod_fun = 'sys' name = mod_fun + '.' + i modindex.module_name = name modindex.save() time.sleep(1) print name
def getfun(): salt_modindex = getPage('http://docs.saltstack.com/en/latest/salt-modindex.html') for i in salt_modindex[307:]: url = 'http://docs.saltstack.com/en/latest/ref/modules/all/'+ i + '.html#module-' + i print url request = urllib2.Request(url,None,req_header) response = urllib2.urlopen(request) index = response.read().decode('utf-8') #print index pattern_name = re.compile('<code class="descname">(.*)</code>') pattern_info = re.compile('<dd><p>(.*)</p>') pattern_example = re.compile('<pre>(.*)') items_name = re.findall(pattern_name,index) #items_info = re.findall(pattern_info,index) #items_example = re.findall(pattern_example,index) i_fun = i.split('.')[2] print i_fun print items_name #print items_info #print items_example for i in items_name: modindex = Modindex() name = i_fun + '.' + i modindex.module_name = name modindex.save() time.sleep(1) print i_fun + '.' + i ''' modindex = Modindex() modindex.module_name = i_fun modindex.module_fun = items_name modindex.module_des = items_info modindex.module_exa = items_example modindex.save() time.sleep(2) #fun_dict = {i_fun:items} #all_fun.append(fun_dict) #return items #print salt_modindex #print len(salt_modindex) #salt_fun = getfun() #print salt_fun ''' #salt_fun = getfun() #salt_modindex = getPage('http://docs.saltstack.com/en/latest/salt-modindex.html') #print salt_modindex[307:]