Ejemplo n.º 1
0
 def worker(self, job):
     job.status = "downloading"
     feed = getDataFrom(job.url, job.logon, job.password)
     writeToFile(feed, job.name, "download", extension="xml", timestamp=True)
     job.last_update = time.time()
     job.status = "idle"
     job.save()
Ejemplo n.º 2
0
 def saveSchema(self, feed_item, schema, data):
     map_file = feed_item.name.replace(" ","_")
     filepath = os.path.join("schemas",schema,map_file)
     writeToFile(content = data, filename = "mapping", folder = filepath , extension = ".xml", timestamp = False, overwrite = True)
     feed_item()
     
     
     
     
Ejemplo n.º 3
0
 def saveSchema(self, feed_item, schema, data):
     map_file = feed_item.name.replace(" ", "_")
     filepath = os.path.join("schemas", schema, map_file)
     writeToFile(content=data,
                 filename="mapping",
                 folder=filepath,
                 extension=".xml",
                 timestamp=False,
                 overwrite=True)
     feed_item()
Ejemplo n.º 4
0
 def worker(self, job):
     try:
         job.status = "downloading"
         feed = getDataFrom(job.url, job.logon, job.password)
         if job.backup:
             writeToFile(feed, job.name, backup_folder, extension='xml', timestamp=True)
         modulename = CodeGenerator.getClassName(job)
         m = __import__(modulename)
         output = m.Downloader().parseFeed(job, feed)
         if output:
             writeToFile(output, job.name, download_folder, extension="xml", timestamp=True)
         job.status = "idle"
     except Exception, e:
         cherrypy.log("error when parsing feed id %s name %s"%(str(job.id),job.name))
         cherrypy.log(str(e))
Ejemplo n.º 5
0
    def savefeed(self, item_id, schema, data, testdownload="false"):
        feed_item = Feed.findById(item_id)
        map_file = str(feed_item.id) + "_" + feed_item.name
        rx = re.compile('[^\d\w]')
        map_file = rx.sub("_", map_file)
        filepath = os.path.join("schemas", schema, map_file)
        try:
            yield "saving xml file..."
            writeToFile(content=data,
                        filename="mapping",
                        folder=filepath,
                        extension=".xml",
                        timestamp=False,
                        overwrite=True)
            yield "OK" + "<br/>"
            feed_item.output_scheme = schema
            feed_item.map_rules = map_file
            yield "generating python code..."
            CodeGenerator().generateCode(feed_item)
            yield "OK" + "<br/>"
            classname = CodeGenerator.getClassName(feed_item)
            #folder = os.path.join("webrss","downloaders")
            yield "compile test..."
            py_compile.compile(os.path.join(
                os.path.join("webrss", "downloaders"), classname + ".py"),
                               doraise=True)
            yield "OK" + "<br/>"
            feed_item.save()
            if testdownload == "true":
                yield "Testing download..."
                feed_item.cache_data = feed_item.cache_data - (8 * 24 * 60 *
                                                               60)
                m = __import__(classname)
                m = reload(m)
                out = m.Downloader().parseFeed(feed_item)
                p = re.compile("^([\s]+)", re.MULTILINE)

                yield "OK" + "<br/>"
                out = cgi.escape(out)
                out = p.sub(lambda x: (x.end() - x.start()) * "&nbsp&nbsp",
                            out)
                yield out.replace("\n", "<br>")
        except PyCompileError, e:
            yield "Error in compiling:" + "<br/>"
            yield e.exc_type_name + "<br/>"
            yield e.exc_value[0] + " on line, with text: " + e.exc_value[1][
                3] + "<br/>"
Ejemplo n.º 6
0
    def generateCode(self, feed):

        #feed = Feed.findById(feed_id)

        classname = CodeGenerator.getClassName(feed)
        mapper = Mapper(feed)
        mapper.loadMappingAsDict()
        self.begin(tab="    ")
        self.writeImports()
        self.writeHead("class Downloader(BaseDownloader):")
        self.indent()

        if mapper.func_update:
            self.writeFunc("update_feed", mapper.func_update,
                           ["self", "raw_feed"])

        for tag, body in mapper.funcs.items():
            if body["text"]:
                if body["text"]["functype"] == "userfunc":
                    fname = ("get_" + tag + "_text").replace("/", "_")
                    self.writeFunc(fname, body["text"]["func_name"])
                    body["text"]["func_name"] = fname

            for a in body["attr"]:
                if a["functype"] == "userfunc":
                    fname = ("get_" + tag + "_attr_" + a["name"]).replace(
                        "/", "_")
                    self.writeFunc(fname, a["func_name"])
                    a["func_name"] = fname

        pp = pprint.PrettyPrinter(indent=4)
        self.writeHead("prules = " + pp.pformat(mapper.funcs))
        writeToFile(content=self.end(),
                    filename=classname,
                    folder=os.path.join("webrss", "downloaders"),
                    extension=".py",
                    timestamp=False,
                    overwrite=True)
Ejemplo n.º 7
0
 def savefeed(self, item_id, schema, data, testdownload = "false"):
     feed_item = Feed.findById(item_id)
     map_file = str(feed_item.id) + "_" + feed_item.name
     rx = re.compile('[^\d\w]')
     map_file = rx.sub("_", map_file)
     filepath = os.path.join("schemas",schema, map_file)
     try:
         yield "saving xml file..."
         writeToFile(content = data, filename = "mapping", folder = filepath , extension = ".xml", timestamp = False, overwrite = True)
         yield "OK"+ "<br/>"
         feed_item.output_scheme = schema
         feed_item.map_rules = map_file
         yield "generating python code..."
         CodeGenerator().generateCode(feed_item)
         yield "OK"+ "<br/>"
         classname = CodeGenerator.getClassName(feed_item)
         #folder = os.path.join("webrss","downloaders")
         yield "compile test..."
         py_compile.compile(os.path.join(os.path.join("webrss","downloaders"),classname+".py"), doraise = True)
         yield "OK"+ "<br/>"
         feed_item.save()
         if testdownload == "true":
             yield "Testing download..."
             feed_item.cache_data = feed_item.cache_data - (8*24*60*60)
             m = __import__(classname)
             m = reload(m)
             out = m.Downloader().parseFeed(feed_item)
             p = re.compile("^([\s]+)", re.MULTILINE)
 
             yield "OK"+ "<br/>"
             out =  cgi.escape(out)
             out = p.sub(lambda x: (x.end()-x.start())*"&nbsp&nbsp",out)
             yield out.replace("\n","<br>")
     except PyCompileError, e:
         yield "Error in compiling:"+ "<br/>"
         yield e.exc_type_name + "<br/>"
         yield e.exc_value[0] + " on line, with text: " + e.exc_value[1][3]+ "<br/>"
Ejemplo n.º 8
0
    def generateCode(self, feed):
        
        #feed = Feed.findById(feed_id)
            
        classname = CodeGenerator.getClassName(feed)
        mapper = Mapper(feed)
        mapper.loadMappingAsDict()
        self.begin(tab="    ")
        self.writeImports()
        self.writeHead("class Downloader(BaseDownloader):")
        self.indent()

        if mapper.func_update:
            self.writeFunc("update_feed",mapper.func_update, ["self","raw_feed"])
        
        for tag,body in mapper.funcs.items():
            if body["text"]:
                if body["text"]["functype"] == "userfunc":
                    fname = ("get_"+tag+"_text").replace("/","_")
                    self.writeFunc(fname, body["text"]["func_name"])
                    body["text"]["func_name"] = fname
                
            for a in body["attr"]:
                if a["functype"] == "userfunc":
                    fname = ("get_"+tag+"_attr_"+a["name"]).replace("/","_")
                    self.writeFunc(fname, a["func_name"])
                    a["func_name"] = fname
        
        pp = pprint.PrettyPrinter(indent = 4)
        self.writeHead("prules = " + pp.pformat(mapper.funcs))
        writeToFile(content = self.end(),
                    filename = classname, 
                    folder = os.path.join("webrss","downloaders"), 
                    extension = ".py", 
                    timestamp = False,
                    overwrite = True)