Ejemplo n.º 1
0
def get_module():
    modindex = Modindex.objects.all()
    modindex.delete()
    salt_modindex = get_modindex('http://docs.saltstack.com/en/latest/salt-modindex.html')
    print salt_modindex
    for mod in salt_modindex:
        url = 'http://docs.saltstack.com/en/latest/ref/modules/all/' + mod + '.html#module-' + mod
        print url
        request = urllib2.Request(url, None, req_header)
        try:
            response = urllib2.urlopen(request)
        except:
            time.sleep(10)
            response = urllib2.urlopen(request)
        index = response.read().decode('utf-8')
        pattern_name = re.compile('<code class="descname">(.*)</code>')
        #pattern_info = re.compile('<dd><p>(.*)</p>')
        #pattern_example = re.compile('<pre>(.*)')
        items_name = re.findall(pattern_name, index)
        #items_info = re.findall(pattern_info,index)
        #items_example = re.findall(pattern_example,index)
        mod_fun = mod.split('.')[2]
        print mod_fun
        print items_name
        #print items_info
        #print items_example
        for i in items_name:
            modindex = Modindex()
            if mod_fun == 'sysmod':
                mod_fun = 'sys'
            name = mod_fun + '.' + i
            modindex.module_name = name
            modindex.save()
            time.sleep(1)
            print name
Ejemplo n.º 2
0
def getfun():
    salt_modindex = getPage('http://docs.saltstack.com/en/latest/salt-modindex.html')
    for i in salt_modindex[307:]:
        url = 'http://docs.saltstack.com/en/latest/ref/modules/all/'+ i + '.html#module-' + i
        print url
        request = urllib2.Request(url,None,req_header)
        response = urllib2.urlopen(request)
        index = response.read().decode('utf-8')
        #print index
        pattern_name = re.compile('<code class="descname">(.*)</code>')
        pattern_info = re.compile('<dd><p>(.*)</p>')
        pattern_example = re.compile('<pre>(.*)')
        items_name = re.findall(pattern_name,index)
        #items_info = re.findall(pattern_info,index)
        #items_example = re.findall(pattern_example,index)
        i_fun = i.split('.')[2]
        print i_fun
        print items_name
        #print items_info
        #print items_example
        for i in items_name:
            modindex = Modindex()
            name = i_fun + '.' + i
            modindex.module_name = name
            modindex.save()
            time.sleep(1)
            print i_fun + '.' + i
        '''
        modindex = Modindex()
        modindex.module_name = i_fun
        modindex.module_fun = items_name
        modindex.module_des = items_info
        modindex.module_exa = items_example
        modindex.save()
        time.sleep(2)
        #fun_dict = {i_fun:items}
        #all_fun.append(fun_dict)
        #return items


#print salt_modindex
#print len(salt_modindex)
#salt_fun = getfun()
#print salt_fun
        '''
#salt_fun = getfun()
#salt_modindex = getPage('http://docs.saltstack.com/en/latest/salt-modindex.html')
#print salt_modindex[307:]