def h_type_all_categories(self, m): return '<ul>' + "\n".join( '<li><a href="' + getName(i["category"], self.params["ext"]) + '">' + i["category"] + '</a></li>' for i in ItemsData.getCategoriesForType(self.params["type"]) ) + '</ul>'
def h_category_items_link(self, m): return '<ul>' + "\n".join( '<li><a href="' + getName(i["name"], self.params["ext"]) + '">' + i["name"] + '</a><br><span>' + i["descr"] + '</span></li>' for i in ItemsData.getItems(self.params["category"]) ) + '</ul>'
def h_type_all_link(self, m): return '<ul>' + "\n".join( '<li><a href="' + getName(i["type"], self.params["ext"]) + '">' + i["type"] + '</a></li>' for i in ItemsData.getTypes() ) + '</ul>'
def processSite(rec):
params = dict(map(None, ["name", "num", "output", "map", "ext"], rec.split("|")))
params["num"] = int(params["num"])
params["topitems"] = 250
if not os.path.isdir(params["output"]):
os.mkdir(params["output"])
# Copy template data
X.File.copytree(TEMPLATES_DIR + params["name"], params["output"])
templater = PageTemplater()
# Index
print "Generate index"
fname = TEMPLATES_DIR + params["name"] + "/index.tpl"
X.File.save(params["output"] + "/" + os.path.splitext(os.path.basename(fname))[0] + "." + params["ext"],
templater.process(X.File.load(fname), params))
# Categories
print "Generate categories"
fname = TEMPLATES_DIR + params["name"] + "/categories.tpl"
X.File.save(params["output"] + "/" + os.path.splitext(os.path.basename(fname))[0] + "." + params["ext"],
templater.process(X.File.load(fname), params))
tparams = copy.copy(params)
for i in ItemsData.getCategories():
tparams["category"] = i["category"]
fname = TEMPLATES_DIR + params["name"] + "/category.tpl"
X.File.save(params["output"] + "/" + getName(i["category"], params["ext"]),
templater.process(X.File.load(fname), tparams))
# Types
print "Generate types"
fname = TEMPLATES_DIR + params["name"] + "/types.tpl"
X.File.save(params["output"] + "/" + os.path.splitext(os.path.basename(fname))[0] + "." + params["ext"],
templater.process(X.File.load(fname), params))
tparams = copy.copy(params)
for i in ItemsData.getTypes():
if i["type"] == None:
tparams["type"] = DEFAULT_TYPE
else:
tparams["type"] = i["type"]
fname = TEMPLATES_DIR + params["name"] + "/type.tpl"
X.File.save(params["output"] + "/" + getName(tparams["type"], params["ext"]),
templater.process(X.File.load(fname), tparams))
# Items
print "Generate top downloads"
fname = TEMPLATES_DIR + params["name"] + "/top-downloads.tpl"
X.File.save(params["output"] + "/" + os.path.splitext(os.path.basename(fname))[0] + "." + params["ext"],
templater.process(X.File.load(fname), params))
print "Generate letters"
tparams = copy.copy(params)
fname = TEMPLATES_DIR + params["name"] + "/letter.tpl"
templateCode = X.File.load(fname)
for letter in CurrentLetters:
print "Generate for '" + letter + "'"
itemsNum = ItemsData.getItemsForLetterNum(letter)
pagesNum = int(math.ceil(float(itemsNum) / tparams["num"]))
print "Pages count: " + str(pagesNum) + "\tItems count: " + str(itemsNum)
tparams["letter"] = letter
tparams["pages"] = pagesNum
for i in xrange(pagesNum):
tparams["page"] = i
if i == 0:
s = ""
else:
s = str(i)
X.File.save(params["output"] + "/" + letter + s +"." + params["ext"],
templater.process(templateCode, tparams))
print "Generate items"
tparams = copy.copy(params)
for i in ItemsData.getItems():
tparams["item"] = i
fname = TEMPLATES_DIR + params["name"] + "/item.tpl"
X.File.save(params["output"] + "/" + getName(i["name"], params["ext"]),
templater.process(X.File.load(fname), tparams))
#Partners
print "Generate partners"
fname = TEMPLATES_DIR + params["name"] + "/partners.tpl"
X.File.save(params["output"] + "/" + os.path.splitext(os.path.basename(fname))[0] + "." + params["ext"],
templater.process(X.File.load(fname), params))
def h_letter_all_items(self, m): return '<ul>' + "\n".join( '<li><a href="' + getName(i["name"], self.params["ext"]) + '">' + i["name"] + '</a> <span>' + i["type"] + '</span></li>' for i in ItemsData.getItemsForLetter(self.params["letter"], int(self.params["page"]), self.params["num"]) ) + '</ul>'
def h_best_downloads(self, m): return '<ul>' + "\n".join( '<li><a href="' + getName(i["name"], self.params["ext"]) + '">' + i["name"] + '</a><br><span>' + i["descr"] + '</span></li>' for i in ItemsData.getTopItems(int(m[1]), self.params["category"]) ) + '</ul>'
def h_topdownloads_all_link(self, m): return '<ul>' + "\n".join( '<li><a href="' + getName(i["name"], self.params["ext"]) + '">' + i["name"] + '</a></li>' for i in ItemsData.getTopItems(self.params["topitems"]) ) + '</ul>'
import sys, re, csv, os, os.path, subprocess, string, math, threading, Queue, urllib, urlparse, socket, time, random, datetime, copy
import X.File, X.Templater
import ItemsData
THREADS_NUM = 10
CONFIG_NAME = "data/config.txt"
ALLURLS_NAME = "data/log.txt"
TEMPLATES_DIR = "data/templates/"
SITEMAP_LINKS_NUM = 4
DEFAULT_TYPE = "None Type"
SpacedChars = re.compile("\W+")
CurrentLetters = ItemsData.getLetters()
def getName(name, ext):
return SpacedChars.sub("-", name.strip().lower()) + "." + ext
class PageTemplater(X.Templater.Templater):
tagspattern = re.compile("|".join([
"\[(domen)\]",
"\[(alphabet)\]",
"\[(category)\]",
"\[(category-link)\]",
"\[(category-items-link)\]",
"\[(category-all-link)\]",
"\[(category-all-types)\]",
"\[(type)\]",
"\[(type-link)\]",