def savefeed(self, item_id, schema, data, testdownload="false"): feed_item = Feed.findById(item_id) map_file = str(feed_item.id) + "_" + feed_item.name rx = re.compile('[^\d\w]') map_file = rx.sub("_", map_file) filepath = os.path.join("schemas", schema, map_file) try: yield "saving xml file..." writeToFile(content=data, filename="mapping", folder=filepath, extension=".xml", timestamp=False, overwrite=True) yield "OK" + "<br/>" feed_item.output_scheme = schema feed_item.map_rules = map_file yield "generating python code..." CodeGenerator().generateCode(feed_item) yield "OK" + "<br/>" classname = CodeGenerator.getClassName(feed_item) #folder = os.path.join("webrss","downloaders") yield "compile test..." py_compile.compile(os.path.join( os.path.join("webrss", "downloaders"), classname + ".py"), doraise=True) yield "OK" + "<br/>" feed_item.save() if testdownload == "true": yield "Testing download..." feed_item.cache_data = feed_item.cache_data - (8 * 24 * 60 * 60) m = __import__(classname) m = reload(m) out = m.Downloader().parseFeed(feed_item) p = re.compile("^([\s]+)", re.MULTILINE) yield "OK" + "<br/>" out = cgi.escape(out) out = p.sub(lambda x: (x.end() - x.start()) * "  ", out) yield out.replace("\n", "<br>") except PyCompileError, e: yield "Error in compiling:" + "<br/>" yield e.exc_type_name + "<br/>" yield e.exc_value[0] + " on line, with text: " + e.exc_value[1][ 3] + "<br/>"
def worker(self, job): try: job.status = "downloading" feed = getDataFrom(job.url, job.logon, job.password) if job.backup: writeToFile(feed, job.name, backup_folder, extension='xml', timestamp=True) modulename = CodeGenerator.getClassName(job) m = __import__(modulename) output = m.Downloader().parseFeed(job, feed) if output: writeToFile(output, job.name, download_folder, extension="xml", timestamp=True) job.status = "idle" except Exception, e: cherrypy.log("error when parsing feed id %s name %s"%(str(job.id),job.name)) cherrypy.log(str(e))
def savefeed(self, item_id, schema, data, testdownload = "false"): feed_item = Feed.findById(item_id) map_file = str(feed_item.id) + "_" + feed_item.name rx = re.compile('[^\d\w]') map_file = rx.sub("_", map_file) filepath = os.path.join("schemas",schema, map_file) try: yield "saving xml file..." writeToFile(content = data, filename = "mapping", folder = filepath , extension = ".xml", timestamp = False, overwrite = True) yield "OK"+ "<br/>" feed_item.output_scheme = schema feed_item.map_rules = map_file yield "generating python code..." CodeGenerator().generateCode(feed_item) yield "OK"+ "<br/>" classname = CodeGenerator.getClassName(feed_item) #folder = os.path.join("webrss","downloaders") yield "compile test..." py_compile.compile(os.path.join(os.path.join("webrss","downloaders"),classname+".py"), doraise = True) yield "OK"+ "<br/>" feed_item.save() if testdownload == "true": yield "Testing download..." feed_item.cache_data = feed_item.cache_data - (8*24*60*60) m = __import__(classname) m = reload(m) out = m.Downloader().parseFeed(feed_item) p = re.compile("^([\s]+)", re.MULTILINE) yield "OK"+ "<br/>" out = cgi.escape(out) out = p.sub(lambda x: (x.end()-x.start())*"  ",out) yield out.replace("\n","<br>") except PyCompileError, e: yield "Error in compiling:"+ "<br/>" yield e.exc_type_name + "<br/>" yield e.exc_value[0] + " on line, with text: " + e.exc_value[1][3]+ "<br/>"