def worker(self, job):
job.status = "downloading"
feed = getDataFrom(job.url, job.logon, job.password)
writeToFile(feed, job.name, "download", extension="xml", timestamp=True)
job.last_update = time.time()
job.status = "idle"
job.save()
def saveSchema(self, feed_item, schema, data):
map_file = feed_item.name.replace(" ","_")
filepath = os.path.join("schemas",schema,map_file)
writeToFile(content = data, filename = "mapping", folder = filepath , extension = ".xml", timestamp = False, overwrite = True)
feed_item()
def saveSchema(self, feed_item, schema, data):
map_file = feed_item.name.replace(" ", "_")
filepath = os.path.join("schemas", schema, map_file)
writeToFile(content=data,
filename="mapping",
folder=filepath,
extension=".xml",
timestamp=False,
overwrite=True)
feed_item()
def worker(self, job):
try:
job.status = "downloading"
feed = getDataFrom(job.url, job.logon, job.password)
if job.backup:
writeToFile(feed, job.name, backup_folder, extension='xml', timestamp=True)
modulename = CodeGenerator.getClassName(job)
m = __import__(modulename)
output = m.Downloader().parseFeed(job, feed)
if output:
writeToFile(output, job.name, download_folder, extension="xml", timestamp=True)
job.status = "idle"
except Exception, e:
cherrypy.log("error when parsing feed id %s name %s"%(str(job.id),job.name))
cherrypy.log(str(e))
def savefeed(self, item_id, schema, data, testdownload="false"):
feed_item = Feed.findById(item_id)
map_file = str(feed_item.id) + "_" + feed_item.name
rx = re.compile('[^\d\w]')
map_file = rx.sub("_", map_file)
filepath = os.path.join("schemas", schema, map_file)
try:
yield "saving xml file..."
writeToFile(content=data,
filename="mapping",
folder=filepath,
extension=".xml",
timestamp=False,
overwrite=True)
yield "OK" + "<br/>"
feed_item.output_scheme = schema
feed_item.map_rules = map_file
yield "generating python code..."
CodeGenerator().generateCode(feed_item)
yield "OK" + "<br/>"
classname = CodeGenerator.getClassName(feed_item)
#folder = os.path.join("webrss","downloaders")
yield "compile test..."
py_compile.compile(os.path.join(
os.path.join("webrss", "downloaders"), classname + ".py"),
doraise=True)
yield "OK" + "<br/>"
feed_item.save()
if testdownload == "true":
yield "Testing download..."
feed_item.cache_data = feed_item.cache_data - (8 * 24 * 60 *
60)
m = __import__(classname)
m = reload(m)
out = m.Downloader().parseFeed(feed_item)
p = re.compile("^([\s]+)", re.MULTILINE)
yield "OK" + "<br/>"
out = cgi.escape(out)
out = p.sub(lambda x: (x.end() - x.start()) * "  ",
out)
yield out.replace("\n", "<br>")
except PyCompileError, e:
yield "Error in compiling:" + "<br/>"
yield e.exc_type_name + "<br/>"
yield e.exc_value[0] + " on line, with text: " + e.exc_value[1][
3] + "<br/>"
def generateCode(self, feed):
#feed = Feed.findById(feed_id)
classname = CodeGenerator.getClassName(feed)
mapper = Mapper(feed)
mapper.loadMappingAsDict()
self.begin(tab=" ")
self.writeImports()
self.writeHead("class Downloader(BaseDownloader):")
self.indent()
if mapper.func_update:
self.writeFunc("update_feed", mapper.func_update,
["self", "raw_feed"])
for tag, body in mapper.funcs.items():
if body["text"]:
if body["text"]["functype"] == "userfunc":
fname = ("get_" + tag + "_text").replace("/", "_")
self.writeFunc(fname, body["text"]["func_name"])
body["text"]["func_name"] = fname
for a in body["attr"]:
if a["functype"] == "userfunc":
fname = ("get_" + tag + "_attr_" + a["name"]).replace(
"/", "_")
self.writeFunc(fname, a["func_name"])
a["func_name"] = fname
pp = pprint.PrettyPrinter(indent=4)
self.writeHead("prules = " + pp.pformat(mapper.funcs))
writeToFile(content=self.end(),
filename=classname,
folder=os.path.join("webrss", "downloaders"),
extension=".py",
timestamp=False,
overwrite=True)
def savefeed(self, item_id, schema, data, testdownload = "false"):
feed_item = Feed.findById(item_id)
map_file = str(feed_item.id) + "_" + feed_item.name
rx = re.compile('[^\d\w]')
map_file = rx.sub("_", map_file)
filepath = os.path.join("schemas",schema, map_file)
try:
yield "saving xml file..."
writeToFile(content = data, filename = "mapping", folder = filepath , extension = ".xml", timestamp = False, overwrite = True)
yield "OK"+ "<br/>"
feed_item.output_scheme = schema
feed_item.map_rules = map_file
yield "generating python code..."
CodeGenerator().generateCode(feed_item)
yield "OK"+ "<br/>"
classname = CodeGenerator.getClassName(feed_item)
#folder = os.path.join("webrss","downloaders")
yield "compile test..."
py_compile.compile(os.path.join(os.path.join("webrss","downloaders"),classname+".py"), doraise = True)
yield "OK"+ "<br/>"
feed_item.save()
if testdownload == "true":
yield "Testing download..."
feed_item.cache_data = feed_item.cache_data - (8*24*60*60)
m = __import__(classname)
m = reload(m)
out = m.Downloader().parseFeed(feed_item)
p = re.compile("^([\s]+)", re.MULTILINE)
yield "OK"+ "<br/>"
out = cgi.escape(out)
out = p.sub(lambda x: (x.end()-x.start())*"  ",out)
yield out.replace("\n","<br>")
except PyCompileError, e:
yield "Error in compiling:"+ "<br/>"
yield e.exc_type_name + "<br/>"
yield e.exc_value[0] + " on line, with text: " + e.exc_value[1][3]+ "<br/>"
def generateCode(self, feed):
#feed = Feed.findById(feed_id)
classname = CodeGenerator.getClassName(feed)
mapper = Mapper(feed)
mapper.loadMappingAsDict()
self.begin(tab=" ")
self.writeImports()
self.writeHead("class Downloader(BaseDownloader):")
self.indent()
if mapper.func_update:
self.writeFunc("update_feed",mapper.func_update, ["self","raw_feed"])
for tag,body in mapper.funcs.items():
if body["text"]:
if body["text"]["functype"] == "userfunc":
fname = ("get_"+tag+"_text").replace("/","_")
self.writeFunc(fname, body["text"]["func_name"])
body["text"]["func_name"] = fname
for a in body["attr"]:
if a["functype"] == "userfunc":
fname = ("get_"+tag+"_attr_"+a["name"]).replace("/","_")
self.writeFunc(fname, a["func_name"])
a["func_name"] = fname
pp = pprint.PrettyPrinter(indent = 4)
self.writeHead("prules = " + pp.pformat(mapper.funcs))
writeToFile(content = self.end(),
filename = classname,
folder = os.path.join("webrss","downloaders"),
extension = ".py",
timestamp = False,
overwrite = True)