def main(): try: opts, args = getopt.getopt(sys.argv[1:], "c:d:eho:t:v", ["config=", "dir=", "erase", "help", "output=", "time=", "quiet", "box2poly"]) except getopt.GetoptError: # print help information and exit: usage() sys.exit(2) # Set initial values... output = None dirMode = False eraseItems = False quiet = False showUsage = False box2poly = False interItemTime = 47 configFile = "./config.cfg" # Process opts for o, a in opts: if o in ("-c", "--config"): configFile = a if o in ("-d", "--dir"): dirMode = True dir = a if o in ("-h", "--help"): usage() sys.exit() if o in ("-o", "--output"): output = a if o in ("-e", "--erase"): eraseItems = True if o in ("-t", "--time"): interItemTime = int(a) if o in ("-q", "--quiet"): quiet = True if o == "--box2poly": box2poly = True # Did we get called with proper agruments? if (not dirMode and len(args) == 0): usage() sys.exit() # Read in config file information if not os.path.lexists(configFile): print "ERROR: Can't find config file, exiting in shame. No items ingested." sys.exit() cp = DatacastingGen.DatacastingConfigParser() cp.read(configFile) # Extract necessary variables from config file # Get feed level info; needed to deal with custom elements rss = DatacastingGen.Feed.FromConfig(cp) # itemSource: if present it needs a Source() object isTuple = eval(cp.get('Item','itemsource', dne = 'None')) if isTuple is not None: itemSource = PyRSS2Gen.Source(isTuple[0], isTuple[1]) else: itemSource = None itemLink = eval(cp.get('Item','itemlink', dne = 'None')) itemPathRoot = eval(cp.get('Generation', 'itempathroot')) queueDirectory = eval(cp.get('Generation', 'queuedirectory')) if dirMode: fileList = [ os.path.abspath(dir) + "/" + x for x in os.listdir(dir) ] else: fileList = [ os.path.abspath(x) for x in args ] # DEBUG # print "Item files processed: %d" % len(fileList) while 1: for file in fileList: try: item = DatacastingGen.Item(txtFileName = file, source = itemSource, link = itemLink, parentFeed = rss) # Set the pubDate to now since we're looping pd = datetime.datetime.utcnow() item.pubDate = pd # If box2poly, use bounding box to make a polygon if (box2poly and (item.extents.__class__.__name__ == "Extents")): ext = item.extents item.extents = ext.asPoly() # Twiddle the guid to match the enclosure + a filename.... enc = None if item.enclosure is not None: enc = item.enclosure elif item.enclosureList is not None : enc = item.enclosureList[0] if enc is not None: genGuid = (enc.url + "?file=" + file + "&time=" + ("%04d-%02d-%02d-%02d-%02d-%02d" % (pd.year, pd.month, pd.day, pd.hour, pd.minute, pd.second))) else: genGuid = ("about:blank" + "?file=" + file + "&time=" + ("%04d-%02d-%02d-%02d-%02d-%02d" % (pd.year, pd.month, pd.day, pd.hour, pd.minute, pd.second))) if not quiet: print "Generated GUID:\n %s" % genGuid item.guid = PyRSS2Gen.Guid(genGuid) basename = os.path.basename(file) (filebase, ext) = os.path.splitext(basename) # Store the item in a hierarchical directory structure. If needed # subdirectories don't exist, create them. itemOutputPath = (itemPathRoot + "/" + "%04d/" % pd.year + ("%02d-%02d" % (pd.month, pd.day))) if not os.path.lexists(itemOutputPath): os.makedirs(itemOutputPath) # Save off the XML snippet outXMLName = os.path.abspath(itemOutputPath + ("/%02d-%02d-%02d-" % (pd.hour, pd.minute, pd.second)) + basename + ".xml") if (not quiet and os.path.lexists(outXMLName)): print "WARNING: Overwriting %s" % outXMLName item.SaveXML(outXMLName) # DEBUG # print "outXMLName: %s" % outXMLName # Link the created item into the queuing directory fullQueuedPath = os.path.abspath(queueDirectory + ("/%04d-%02d-%02d-%02d-%02d-%02d-" % (pd.year, pd.month, pd.day, pd.hour, pd.minute, pd.second)) + basename + ".xml") if not os.path.lexists(queueDirectory): os.makedirs(queueDirectory) if os.path.lexists(fullQueuedPath): if not quiet: print "WARNING: Requeuing %s" % fullQueuedPath os.unlink(fullQueuedPath) os.symlink(outXMLName, fullQueuedPath) # Printed queued item name for status... if not quiet: print ("STATUS: Queued item: %s" % item.title) print (" at %04d-%02d-%02d %02d:%02d:%02d" % (pd.year, pd.month, pd.day, pd.hour, pd.minute, pd.second)) # Generate the feed command = ("GenerateFeed.py -c %s" % configFile) os.system(command) # Pause after creating item time.sleep(interItemTime) except Exception, inst: print "WARNING: Item text file ingestion failed..." print " Item file name: %s" % file print " Exception: %s" % inst print " Continuing processing." raise
textInput = PyRSS2Gen.TextInput("Qwerty", "Shrdlu", "Etaoin", "http://some/link"), skipDays = PyRSS2Gen.SkipDays(["Monday", "Thursday"]), skipHours = PyRSS2Gen.SkipHours([0, 5, 22]), items = [PyRSS2Gen.RSSItem( "Chapter 1", "http://xe.com/", "How to convert money.", author = "x@xe", # categories comments = "http://slashdot.org", # enclosure guid = "http://asdfghjk/", pubDate = datetime.datetime(1999, 1, 30), source = PyRSS2Gen.Source("ftp://x.y/", "Quotes of the Day"), ), PyRSS2Gen.RSSItem("Chapter 2", "http://acm.org/", "AT&T is <Ma Bell>.", guid = PyRSS2Gen.Guid(guid = "12345", isPermaLink = False), ), ]) def _convert_to_liberal(obj): if isinstance(obj, basestring): return obj elif isinstance(obj, int): return str(obj) elif isinstance(obj, datetime.datetime): return PyRSS2Gen._format_date(obj)
def main(): try: opts, args = getopt.getopt(sys.argv[1:], "c:d:eho:v", ["config=", "dir=", "erase", "help", "output=", "verbose"]) except getopt.GetoptError: # print help information and exit: usage() sys.exit(2) # Set initial values... output = None dirMode = False eraseItems = False verbose = False showUsage = False configFile = "./config.cfg" # Process opts for o, a in opts: if o in ("-c", "--config"): configFile = a if o in ("-d", "--dir"): dirMode = True dir = a if o in ("-h", "--help"): usage() sys.exit() if o in ("-o", "--output"): output = a if o in ("-e", "--erase"): eraseItems = True if o in ("-v", "--verboes"): verbose = True # Did we get called with proper agruments? if (not dirMode and len(args) == 0): usage() sys.exit() # Read in config file information if not os.path.lexists(configFile): print "ERROR: Can't find config file, exiting in shame. No items ingested." sys.exit() cp = DatacastingGen.DatacastingConfigParser() cp.read(configFile) # Extract necessary variables from config file # Get feed level info; needed to deal with custom elements rss = DatacastingGen.Feed.FromConfig(cp) # itemSource: if present it needs a Source() object isTuple = eval(cp.get('Item','itemsource', dne = 'None')) if isTuple is not None: itemSource = PyRSS2Gen.Source(isTuple[0], isTuple[1]) else: itemSource = None itemLink = eval(cp.get('Item','itemlink', dne = 'None')) itemPathRoot = eval(cp.get('Generation', 'itempathroot')) queueDirectory = eval(cp.get('Generation', 'queuedirectory')) configDict = {} configDict['simplifyPolygons'] = eval(cp.get('Item/Extents', 'simplifyPolygons', dne = 'False')) configDict['polyMaxPoints'] = eval(cp.get('Item/Extents', 'polyMaxPoints', dne = '256')) configDict['polyDistThresh'] = eval(cp.get('Item/Extents', 'polyDistThresh', dne = '5.0')) if dirMode: fileList = [ os.path.abspath(dir) + "/" + x for x in os.listdir(dir) ] else: fileList = [ os.path.abspath(x) for x in args ] # DEBUG # print "Item files processed: %d" % len(fileList) for file in fileList: try: item = DatacastingGen.Item(txtFileName = file, source = itemSource, link = itemLink, parentFeed = rss, addlConfig = configDict) # Twiddle the guid to match the enclosure + a filename.... enc = None if item.enclosure is not None: if item.enclosureList is not [] : enc = item.enclosureList[0] else: enc = item.enclosure if enc is not None: item.guid = PyRSS2Gen.Guid(enc.url + "?file=" + file) basename = os.path.basename(file) (filebase, ext) = os.path.splitext(basename) # Store the item in a hierarchical directory structure. If needed # subdirectories don't exist, create them. if item.pubDate is not None: pd = item.pubDate else: pd = datetime.datetime.utcnow() itemOutputPath = (itemPathRoot + "/" + "%04d/" % pd.year + ("%02d-%02d" % (pd.month, pd.day))) if not os.path.lexists(itemOutputPath): os.makedirs(itemOutputPath) # Save off the XML snippet outXMLName = os.path.abspath(itemOutputPath + ("/%02d-%02d-%02d-" % (pd.hour, pd.minute, pd.second)) + basename + ".xml") if (verbose and os.path.lexists(outXMLName)): print "WARNING: Overwriting %s" % outXMLName item.SaveXML(outXMLName) # DEBUG # print "outXMLName: %s" % outXMLName # Link the created item into the queuing directory fullQueuedPath = os.path.abspath(queueDirectory + ("/%04d-%02d-%02d-%02d-%02d-%02d-" % (pd.year, pd.month, pd.day, pd.hour, pd.minute, pd.second)) + basename + ".xml") if not os.path.lexists(queueDirectory): os.makedirs(queueDirectory) if os.path.lexists(fullQueuedPath): if verbose: print "WARNING: Requeuing %s" % fullQueuedPath os.unlink(fullQueuedPath) os.symlink(outXMLName, fullQueuedPath) except Exception, inst: print "WARNING: Item text file ingestion failed..." print " Item file name: %s" % file print " Exception: %s" % inst print " Continuing processing." raise