def insert(title, url, breadcrumbs=set()):
"""Insert a page into the current tree
Set the current language using the 'set_current_language()' function."""
global __structure__
# convert url
page_link = urltoname(url)
# convert breadcrumbs
page_breadcrumbs = list()
for x in breadcrumbs:
page_breadcrumbs.append(urltoname(x))
if __lang__ != "":
__structure__[__lang__].insert(title, page_link, page_breadcrumbs)
# print >> sys.stderr, "**** Dump after insert:"
# __structure__[__lang__].dump()
else:
print >>sys.stderr, "*** Error in wikidot.structure.insert: set the language beforehand"
def __is_link_local__(self, link):
"""Private - Tell if a link match a local file.
Output:
True if a local file match the link
False otherwise"""
if urltoname(link) in self.local_links:
return True
else:
return False
def __is_link_local__(self, link):
"""Private - Tell if a link match a local file.
Output:
True if a local file match the link
False otherwise"""
if urltoname(link) in self.local_links:
return True
else:
return False
def insert(title, url, breadcrumbs=set()):
"""Insert a page into the current tree
Set the current language using the 'set_current_language()' function."""
global __structure__
# convert url
page_link = urltoname(url)
# convert breadcrumbs
page_breadcrumbs = list()
for x in breadcrumbs:
page_breadcrumbs.append(urltoname(x))
if __lang__ != '':
__structure__[__lang__].insert(title, page_link, page_breadcrumbs)
# print >> sys.stderr, "**** Dump after insert:"
# __structure__[__lang__].dump()
else:
print >> sys.stderr, "*** Error in wikidot.structure.insert: set the language beforehand"
def __fix_link__(self, link):
"""Private - Take a link and convert it,
either as a local link, either as a link pointing
to the remote host."""
if self.__is_link_toc__(link) == True:
# don't touch it!
return link
if self.__is_link_local__(link) == True:
# Convert link
new_link = urltoname(link)
self.local_set.add(new_link)
else:
# Remote link
new_link = urlparse.urljoin(self.remote_host, link)
self.remote_set.add(new_link)
return new_link
def __fix_link__(self, link):
"""Private - Take a link and convert it,
either as a local link, either as a link pointing
to the remote host."""
if self.__is_link_toc__(link) == True:
# don't touch it!
return link
if self.__is_link_local__(link) == True:
# Convert link
new_link = urltoname(link)
self.local_set.add(new_link)
else:
# Remote link
new_link = urlparse.urljoin(self.remote_host, link)
self.remote_set.add(new_link)
return new_link
from wikidot.tools import tidy
from wikidot.tools import fix_latex
from wikidot.urltoname import urltoname
from wikidot.orderedset import OrderedSet
def fetchwikidot(starturl, outputdir):
# Create the output directory, if needed
try:
os.stat(outputdir)
except OSError, e:
print >>sys.stderr, "Creating output directory; ", outputdir
os.mkdir(outputdir)
# Fetch root page
output = os.path.join(outputdir, urltoname(starturl))
retval = fetchurl(starturl, output)
newlinks = retval["links"]
breadcrumbs = retval["breadcrumbs"]
# get the last element of the list
if len(breadcrumbs) > 0:
breadcrumbs = breadcrumbs[len(breadcrumbs) - 1]
else:
breadcrumbs = ""
# Create a set with fetched links (avoid loops...)
links = OrderedSet(starturl)
# Iterate on the links, and recursively download / convert
fetchlinks = newlinks
while len(fetchlinks) > 0:
#print "Replacing ", server, " with ", base_server, "..."
return re.sub(server, base_server, url)
# no match
return url
def fetchwikidot(starturl, outputdir):
# Create the output directory, if needed
try:
os.stat(outputdir)
except OSError, e:
print >> sys.stderr, "Creating output directory; ", outputdir
os.mkdir(outputdir)
# Fetch root page
output = os.path.join(outputdir, urltoname(starturl))
retval = fetchurl(starturl, output)
newlinks = retval['links']
breadcrumbs = retval['breadcrumbs']
# get the last element of the list
if len(breadcrumbs) > 0:
breadcrumbs = breadcrumbs[len(breadcrumbs)-1]
else:
breadcrumbs = ''
# Create a set with fetched links (avoid loops...)
links = OrderedSet(starturl)
# Iterate on the links, and recursively download / convert
fetchlinks = newlinks
while len(fetchlinks) > 0:
