def get_module():
modindex = Modindex.objects.all()
modindex.delete()
salt_modindex = get_modindex('http://docs.saltstack.com/en/latest/salt-modindex.html')
print salt_modindex
for mod in salt_modindex:
url = 'http://docs.saltstack.com/en/latest/ref/modules/all/' + mod + '.html#module-' + mod
print url
request = urllib2.Request(url, None, req_header)
try:
response = urllib2.urlopen(request)
except:
time.sleep(10)
response = urllib2.urlopen(request)
index = response.read().decode('utf-8')
pattern_name = re.compile('<code class="descname">(.*)</code>')
#pattern_info = re.compile('<dd><p>(.*)</p>')
#pattern_example = re.compile('<pre>(.*)')
items_name = re.findall(pattern_name, index)
#items_info = re.findall(pattern_info,index)
#items_example = re.findall(pattern_example,index)
mod_fun = mod.split('.')[2]
print mod_fun
print items_name
#print items_info
#print items_example
for i in items_name:
modindex = Modindex()
if mod_fun == 'sysmod':
mod_fun = 'sys'
name = mod_fun + '.' + i
modindex.module_name = name
modindex.save()
time.sleep(1)
print name
def getfun():
salt_modindex = getPage('http://docs.saltstack.com/en/latest/salt-modindex.html')
for i in salt_modindex[307:]:
url = 'http://docs.saltstack.com/en/latest/ref/modules/all/'+ i + '.html#module-' + i
print url
request = urllib2.Request(url,None,req_header)
response = urllib2.urlopen(request)
index = response.read().decode('utf-8')
#print index
pattern_name = re.compile('<code class="descname">(.*)</code>')
pattern_info = re.compile('<dd><p>(.*)</p>')
pattern_example = re.compile('<pre>(.*)')
items_name = re.findall(pattern_name,index)
#items_info = re.findall(pattern_info,index)
#items_example = re.findall(pattern_example,index)
i_fun = i.split('.')[2]
print i_fun
print items_name
#print items_info
#print items_example
for i in items_name:
modindex = Modindex()
name = i_fun + '.' + i
modindex.module_name = name
modindex.save()
time.sleep(1)
print i_fun + '.' + i
'''
modindex = Modindex()
modindex.module_name = i_fun
modindex.module_fun = items_name
modindex.module_des = items_info
modindex.module_exa = items_example
modindex.save()
time.sleep(2)
#fun_dict = {i_fun:items}
#all_fun.append(fun_dict)
#return items
#print salt_modindex
#print len(salt_modindex)
#salt_fun = getfun()
#print salt_fun
'''
#salt_fun = getfun()
#salt_modindex = getPage('http://docs.saltstack.com/en/latest/salt-modindex.html')
#print salt_modindex[307:]