def edit_feed_map(self, item_id, schema=None):
item = Feed.findById(item_id)
if schema:
item.output_scheme = schema
item.map_rules = ''
m = Mapper(item)
m.loadScheme()
m.loadMapping()
f = None
if item.output_scheme:
f = FeedAnalyzer()
f.analyze(item, m)
tmpl = lookup.get_template("edit_feed_map.html")
return tmpl.render(item = item, mapper = m,pgname="edit_feed_map", flz = f)
def generateCode(self, feed):
#feed = Feed.findById(feed_id)
classname = CodeGenerator.getClassName(feed)
mapper = Mapper(feed)
mapper.loadMappingAsDict()
self.begin(tab=" ")
self.writeImports()
self.writeHead("class Downloader(BaseDownloader):")
self.indent()
if mapper.func_update:
self.writeFunc("update_feed",mapper.func_update, ["self","raw_feed"])
for tag,body in mapper.funcs.items():
if body["text"]:
if body["text"]["functype"] == "userfunc":
fname = ("get_"+tag+"_text").replace("/","_")
self.writeFunc(fname, body["text"]["func_name"])
body["text"]["func_name"] = fname
for a in body["attr"]:
if a["functype"] == "userfunc":
fname = ("get_"+tag+"_attr_"+a["name"]).replace("/","_")
self.writeFunc(fname, a["func_name"])
a["func_name"] = fname
pp = pprint.PrettyPrinter(indent = 4)
self.writeHead("prules = " + pp.pformat(mapper.funcs))
writeToFile(content = self.end(),
filename = classname,
folder = os.path.join("webrss","downloaders"),
extension = ".py",
timestamp = False,
overwrite = True)
def edit_feed_map(self, item_id, schema=None):
item = Feed.findById(item_id)
if schema:
item.output_scheme = schema
item.map_rules = ''
m = Mapper(item)
m.loadScheme()
m.loadMapping()
f = None
if item.output_scheme:
f = FeedAnalyzer()
f.analyze(item, m)
tmpl = lookup.get_template("edit_feed_map.html")
return tmpl.render(item=item, mapper=m, pgname="edit_feed_map", flz=f)
def parseFeed(self, dbfeeditem, feed_data = None):
#feed_data = getDataFrom(self.dbfeed.url, self.dbfeed.logon, self.dbfeed.password)
self.dbfeed = dbfeeditem
if not feed_data:
#feed_data = getDataFrom(self.dbfeed.url, self.dbfeed.logon, self.dbfeed.password)
return None
raw_feed = feedparser.parse(feed_data)
self.update_feed(raw_feed)
self.feed = raw_feed.feed
maper = Mapper(self.dbfeed)
maper.loadMapping()
maper.loadScheme()
root = ElementTree.Element(maper.schema_root.tag)
old_timestamp = self.dbfeed.cache_data
timestamp = old_timestamp
new_timestamp = old_timestamp
cntr = 0
for self.item in raw_feed.entries:
#self.item = item
if self.item.has_key("updated_parsed"):
timestamp = time.mktime(self.item.updated_parsed)
if new_timestamp < timestamp:
new_timestamp = timestamp
if old_timestamp >= timestamp:
continue
item = ElementTree.Element(maper.schema_container.tag)
for t in maper.schema:
text = None
node_args = {}
if self.__class__.prules.has_key(t.xpath):
i = self.__class__.prules[t.xpath]
if i["text"]:
fname = i["text"]["func_name"]
args = i["text"]["args"]
ftype = i["text"]["functype"]
if ftype == "static":
text = args
else:
text = getattr(self, fname)(args) if len(args) > 0 else getattr(self, fname)()
for at in i["attr"]:
fnc = getattr(self,at["func_name"])
if len(at["args"]) > 0:
node_args[at["name"]] = fnc(at["args"])
else:
node_args[at["name"]] = fnc()
self.build_xpath(item, t.xpath, text = text, args = node_args)
root.append(item)
cntr = cntr + 1
#self.indent(root)
#return ElementTree.tostring(root, 'utf-8')
if cntr > 0:
self.dbfeed.cache_data = new_timestamp
rough_string = ElementTree.tostring(root, 'utf-8')
reparsed = minidom.parseString(rough_string)
return reparsed.toprettyxml(indent=" ", encoding="UTF-8" )
else:
return None
