def savefeed(self, item_id, schema, data, testdownload="false"):
feed_item = Feed.findById(item_id)
map_file = str(feed_item.id) + "_" + feed_item.name
rx = re.compile('[^\d\w]')
map_file = rx.sub("_", map_file)
filepath = os.path.join("schemas", schema, map_file)
try:
yield "saving xml file..."
writeToFile(content=data,
filename="mapping",
folder=filepath,
extension=".xml",
timestamp=False,
overwrite=True)
yield "OK" + "<br/>"
feed_item.output_scheme = schema
feed_item.map_rules = map_file
yield "generating python code..."
CodeGenerator().generateCode(feed_item)
yield "OK" + "<br/>"
classname = CodeGenerator.getClassName(feed_item)
#folder = os.path.join("webrss","downloaders")
yield "compile test..."
py_compile.compile(os.path.join(
os.path.join("webrss", "downloaders"), classname + ".py"),
doraise=True)
yield "OK" + "<br/>"
feed_item.save()
if testdownload == "true":
yield "Testing download..."
feed_item.cache_data = feed_item.cache_data - (8 * 24 * 60 *
60)
m = __import__(classname)
m = reload(m)
out = m.Downloader().parseFeed(feed_item)
p = re.compile("^([\s]+)", re.MULTILINE)
yield "OK" + "<br/>"
out = cgi.escape(out)
out = p.sub(lambda x: (x.end() - x.start()) * "  ",
out)
yield out.replace("\n", "<br>")
except PyCompileError, e:
yield "Error in compiling:" + "<br/>"
yield e.exc_type_name + "<br/>"
yield e.exc_value[0] + " on line, with text: " + e.exc_value[1][
3] + "<br/>"
def worker(self, job):
try:
job.status = "downloading"
feed = getDataFrom(job.url, job.logon, job.password)
if job.backup:
writeToFile(feed, job.name, backup_folder, extension='xml', timestamp=True)
modulename = CodeGenerator.getClassName(job)
m = __import__(modulename)
output = m.Downloader().parseFeed(job, feed)
if output:
writeToFile(output, job.name, download_folder, extension="xml", timestamp=True)
job.status = "idle"
except Exception, e:
cherrypy.log("error when parsing feed id %s name %s"%(str(job.id),job.name))
cherrypy.log(str(e))
def savefeed(self, item_id, schema, data, testdownload = "false"):
feed_item = Feed.findById(item_id)
map_file = str(feed_item.id) + "_" + feed_item.name
rx = re.compile('[^\d\w]')
map_file = rx.sub("_", map_file)
filepath = os.path.join("schemas",schema, map_file)
try:
yield "saving xml file..."
writeToFile(content = data, filename = "mapping", folder = filepath , extension = ".xml", timestamp = False, overwrite = True)
yield "OK"+ "<br/>"
feed_item.output_scheme = schema
feed_item.map_rules = map_file
yield "generating python code..."
CodeGenerator().generateCode(feed_item)
yield "OK"+ "<br/>"
classname = CodeGenerator.getClassName(feed_item)
#folder = os.path.join("webrss","downloaders")
yield "compile test..."
py_compile.compile(os.path.join(os.path.join("webrss","downloaders"),classname+".py"), doraise = True)
yield "OK"+ "<br/>"
feed_item.save()
if testdownload == "true":
yield "Testing download..."
feed_item.cache_data = feed_item.cache_data - (8*24*60*60)
m = __import__(classname)
m = reload(m)
out = m.Downloader().parseFeed(feed_item)
p = re.compile("^([\s]+)", re.MULTILINE)
yield "OK"+ "<br/>"
out = cgi.escape(out)
out = p.sub(lambda x: (x.end()-x.start())*"  ",out)
yield out.replace("\n","<br>")
except PyCompileError, e:
yield "Error in compiling:"+ "<br/>"
yield e.exc_type_name + "<br/>"
yield e.exc_value[0] + " on line, with text: " + e.exc_value[1][3]+ "<br/>"