def worker(self, job): job.status = "downloading" feed = getDataFrom(job.url, job.logon, job.password) writeToFile(feed, job.name, "download", extension="xml", timestamp=True) job.last_update = time.time() job.status = "idle" job.save()
def saveSchema(self, feed_item, schema, data): map_file = feed_item.name.replace(" ","_") filepath = os.path.join("schemas",schema,map_file) writeToFile(content = data, filename = "mapping", folder = filepath , extension = ".xml", timestamp = False, overwrite = True) feed_item()
def saveSchema(self, feed_item, schema, data): map_file = feed_item.name.replace(" ", "_") filepath = os.path.join("schemas", schema, map_file) writeToFile(content=data, filename="mapping", folder=filepath, extension=".xml", timestamp=False, overwrite=True) feed_item()
def worker(self, job): try: job.status = "downloading" feed = getDataFrom(job.url, job.logon, job.password) if job.backup: writeToFile(feed, job.name, backup_folder, extension='xml', timestamp=True) modulename = CodeGenerator.getClassName(job) m = __import__(modulename) output = m.Downloader().parseFeed(job, feed) if output: writeToFile(output, job.name, download_folder, extension="xml", timestamp=True) job.status = "idle" except Exception, e: cherrypy.log("error when parsing feed id %s name %s"%(str(job.id),job.name)) cherrypy.log(str(e))
def savefeed(self, item_id, schema, data, testdownload="false"): feed_item = Feed.findById(item_id) map_file = str(feed_item.id) + "_" + feed_item.name rx = re.compile('[^\d\w]') map_file = rx.sub("_", map_file) filepath = os.path.join("schemas", schema, map_file) try: yield "saving xml file..." writeToFile(content=data, filename="mapping", folder=filepath, extension=".xml", timestamp=False, overwrite=True) yield "OK" + "<br/>" feed_item.output_scheme = schema feed_item.map_rules = map_file yield "generating python code..." CodeGenerator().generateCode(feed_item) yield "OK" + "<br/>" classname = CodeGenerator.getClassName(feed_item) #folder = os.path.join("webrss","downloaders") yield "compile test..." py_compile.compile(os.path.join( os.path.join("webrss", "downloaders"), classname + ".py"), doraise=True) yield "OK" + "<br/>" feed_item.save() if testdownload == "true": yield "Testing download..." feed_item.cache_data = feed_item.cache_data - (8 * 24 * 60 * 60) m = __import__(classname) m = reload(m) out = m.Downloader().parseFeed(feed_item) p = re.compile("^([\s]+)", re.MULTILINE) yield "OK" + "<br/>" out = cgi.escape(out) out = p.sub(lambda x: (x.end() - x.start()) * "  ", out) yield out.replace("\n", "<br>") except PyCompileError, e: yield "Error in compiling:" + "<br/>" yield e.exc_type_name + "<br/>" yield e.exc_value[0] + " on line, with text: " + e.exc_value[1][ 3] + "<br/>"
def generateCode(self, feed): #feed = Feed.findById(feed_id) classname = CodeGenerator.getClassName(feed) mapper = Mapper(feed) mapper.loadMappingAsDict() self.begin(tab=" ") self.writeImports() self.writeHead("class Downloader(BaseDownloader):") self.indent() if mapper.func_update: self.writeFunc("update_feed", mapper.func_update, ["self", "raw_feed"]) for tag, body in mapper.funcs.items(): if body["text"]: if body["text"]["functype"] == "userfunc": fname = ("get_" + tag + "_text").replace("/", "_") self.writeFunc(fname, body["text"]["func_name"]) body["text"]["func_name"] = fname for a in body["attr"]: if a["functype"] == "userfunc": fname = ("get_" + tag + "_attr_" + a["name"]).replace( "/", "_") self.writeFunc(fname, a["func_name"]) a["func_name"] = fname pp = pprint.PrettyPrinter(indent=4) self.writeHead("prules = " + pp.pformat(mapper.funcs)) writeToFile(content=self.end(), filename=classname, folder=os.path.join("webrss", "downloaders"), extension=".py", timestamp=False, overwrite=True)
def savefeed(self, item_id, schema, data, testdownload = "false"): feed_item = Feed.findById(item_id) map_file = str(feed_item.id) + "_" + feed_item.name rx = re.compile('[^\d\w]') map_file = rx.sub("_", map_file) filepath = os.path.join("schemas",schema, map_file) try: yield "saving xml file..." writeToFile(content = data, filename = "mapping", folder = filepath , extension = ".xml", timestamp = False, overwrite = True) yield "OK"+ "<br/>" feed_item.output_scheme = schema feed_item.map_rules = map_file yield "generating python code..." CodeGenerator().generateCode(feed_item) yield "OK"+ "<br/>" classname = CodeGenerator.getClassName(feed_item) #folder = os.path.join("webrss","downloaders") yield "compile test..." py_compile.compile(os.path.join(os.path.join("webrss","downloaders"),classname+".py"), doraise = True) yield "OK"+ "<br/>" feed_item.save() if testdownload == "true": yield "Testing download..." feed_item.cache_data = feed_item.cache_data - (8*24*60*60) m = __import__(classname) m = reload(m) out = m.Downloader().parseFeed(feed_item) p = re.compile("^([\s]+)", re.MULTILINE) yield "OK"+ "<br/>" out = cgi.escape(out) out = p.sub(lambda x: (x.end()-x.start())*"  ",out) yield out.replace("\n","<br>") except PyCompileError, e: yield "Error in compiling:"+ "<br/>" yield e.exc_type_name + "<br/>" yield e.exc_value[0] + " on line, with text: " + e.exc_value[1][3]+ "<br/>"
def generateCode(self, feed): #feed = Feed.findById(feed_id) classname = CodeGenerator.getClassName(feed) mapper = Mapper(feed) mapper.loadMappingAsDict() self.begin(tab=" ") self.writeImports() self.writeHead("class Downloader(BaseDownloader):") self.indent() if mapper.func_update: self.writeFunc("update_feed",mapper.func_update, ["self","raw_feed"]) for tag,body in mapper.funcs.items(): if body["text"]: if body["text"]["functype"] == "userfunc": fname = ("get_"+tag+"_text").replace("/","_") self.writeFunc(fname, body["text"]["func_name"]) body["text"]["func_name"] = fname for a in body["attr"]: if a["functype"] == "userfunc": fname = ("get_"+tag+"_attr_"+a["name"]).replace("/","_") self.writeFunc(fname, a["func_name"]) a["func_name"] = fname pp = pprint.PrettyPrinter(indent = 4) self.writeHead("prules = " + pp.pformat(mapper.funcs)) writeToFile(content = self.end(), filename = classname, folder = os.path.join("webrss","downloaders"), extension = ".py", timestamp = False, overwrite = True)