def main():
try:
opts, args = getopt.getopt(sys.argv[1:],
"c:d:eho:t:v",
["config=", "dir=", "erase", "help", "output=",
"time=", "quiet", "box2poly"])
except getopt.GetoptError:
# print help information and exit:
usage()
sys.exit(2)
# Set initial values...
output = None
dirMode = False
eraseItems = False
quiet = False
showUsage = False
box2poly = False
interItemTime = 47
configFile = "./config.cfg"
# Process opts
for o, a in opts:
if o in ("-c", "--config"):
configFile = a
if o in ("-d", "--dir"):
dirMode = True
dir = a
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-o", "--output"):
output = a
if o in ("-e", "--erase"):
eraseItems = True
if o in ("-t", "--time"):
interItemTime = int(a)
if o in ("-q", "--quiet"):
quiet = True
if o == "--box2poly":
box2poly = True
# Did we get called with proper agruments?
if (not dirMode and len(args) == 0):
usage()
sys.exit()
# Read in config file information
if not os.path.lexists(configFile):
print "ERROR: Can't find config file, exiting in shame. No items ingested."
sys.exit()
cp = DatacastingGen.DatacastingConfigParser()
cp.read(configFile)
# Extract necessary variables from config file
# Get feed level info; needed to deal with custom elements
rss = DatacastingGen.Feed.FromConfig(cp)
# itemSource: if present it needs a Source() object
isTuple = eval(cp.get('Item','itemsource',
dne = 'None'))
if isTuple is not None:
itemSource = PyRSS2Gen.Source(isTuple[0], isTuple[1])
else:
itemSource = None
itemLink = eval(cp.get('Item','itemlink',
dne = 'None'))
itemPathRoot = eval(cp.get('Generation', 'itempathroot'))
queueDirectory = eval(cp.get('Generation', 'queuedirectory'))
if dirMode:
fileList = [ os.path.abspath(dir) + "/" + x
for x in os.listdir(dir) ]
else:
fileList = [ os.path.abspath(x) for x in args ]
# DEBUG
# print "Item files processed: %d" % len(fileList)
while 1:
for file in fileList:
try:
item = DatacastingGen.Item(txtFileName = file,
source = itemSource,
link = itemLink,
parentFeed = rss)
# Set the pubDate to now since we're looping
pd = datetime.datetime.utcnow()
item.pubDate = pd
# If box2poly, use bounding box to make a polygon
if (box2poly and (item.extents.__class__.__name__ == "Extents")):
ext = item.extents
item.extents = ext.asPoly()
# Twiddle the guid to match the enclosure + a filename....
enc = None
if item.enclosure is not None:
enc = item.enclosure
elif item.enclosureList is not None :
enc = item.enclosureList[0]
if enc is not None:
genGuid = (enc.url +
"?file=" + file +
"&time=" +
("%04d-%02d-%02d-%02d-%02d-%02d" %
(pd.year, pd.month, pd.day,
pd.hour, pd.minute, pd.second)))
else:
genGuid = ("about:blank" +
"?file=" + file +
"&time=" +
("%04d-%02d-%02d-%02d-%02d-%02d" %
(pd.year, pd.month, pd.day,
pd.hour, pd.minute, pd.second)))
if not quiet:
print "Generated GUID:\n %s" % genGuid
item.guid = PyRSS2Gen.Guid(genGuid)
basename = os.path.basename(file)
(filebase, ext) = os.path.splitext(basename)
# Store the item in a hierarchical directory structure. If needed
# subdirectories don't exist, create them.
itemOutputPath = (itemPathRoot + "/" +
"%04d/" % pd.year +
("%02d-%02d" % (pd.month, pd.day)))
if not os.path.lexists(itemOutputPath):
os.makedirs(itemOutputPath)
# Save off the XML snippet
outXMLName = os.path.abspath(itemOutputPath +
("/%02d-%02d-%02d-" %
(pd.hour, pd.minute, pd.second)) +
basename + ".xml")
if (not quiet and os.path.lexists(outXMLName)):
print "WARNING: Overwriting %s" % outXMLName
item.SaveXML(outXMLName)
# DEBUG
# print "outXMLName: %s" % outXMLName
# Link the created item into the queuing directory
fullQueuedPath = os.path.abspath(queueDirectory +
("/%04d-%02d-%02d-%02d-%02d-%02d-" %
(pd.year, pd.month, pd.day,
pd.hour, pd.minute, pd.second)) +
basename + ".xml")
if not os.path.lexists(queueDirectory):
os.makedirs(queueDirectory)
if os.path.lexists(fullQueuedPath):
if not quiet:
print "WARNING: Requeuing %s" % fullQueuedPath
os.unlink(fullQueuedPath)
os.symlink(outXMLName, fullQueuedPath)
# Printed queued item name for status...
if not quiet:
print ("STATUS: Queued item: %s" % item.title)
print (" at %04d-%02d-%02d %02d:%02d:%02d" %
(pd.year, pd.month, pd.day, pd.hour, pd.minute, pd.second))
# Generate the feed
command = ("GenerateFeed.py -c %s" % configFile)
os.system(command)
# Pause after creating item
time.sleep(interItemTime)
except Exception, inst:
print "WARNING: Item text file ingestion failed..."
print " Item file name: %s" % file
print " Exception: %s" % inst
print " Continuing processing."
raise
textInput = PyRSS2Gen.TextInput("Qwerty", "Shrdlu", "Etaoin",
"http://some/link"),
skipDays = PyRSS2Gen.SkipDays(["Monday", "Thursday"]),
skipHours = PyRSS2Gen.SkipHours([0, 5, 22]),
items = [PyRSS2Gen.RSSItem(
"Chapter 1", "http://xe.com/",
"How to convert money.",
author = "x@xe",
# categories
comments = "http://slashdot.org",
# enclosure
guid = "http://asdfghjk/",
pubDate = datetime.datetime(1999, 1, 30),
source = PyRSS2Gen.Source("ftp://x.y/",
"Quotes of the Day"),
),
PyRSS2Gen.RSSItem("Chapter 2", "http://acm.org/",
"AT&T is <Ma Bell>.",
guid = PyRSS2Gen.Guid(guid = "12345", isPermaLink = False),
),
])
def _convert_to_liberal(obj):
if isinstance(obj, basestring):
return obj
elif isinstance(obj, int):
return str(obj)
elif isinstance(obj, datetime.datetime):
return PyRSS2Gen._format_date(obj)
def main():
try:
opts, args = getopt.getopt(sys.argv[1:],
"c:d:eho:v",
["config=", "dir=", "erase", "help", "output=",
"verbose"])
except getopt.GetoptError:
# print help information and exit:
usage()
sys.exit(2)
# Set initial values...
output = None
dirMode = False
eraseItems = False
verbose = False
showUsage = False
configFile = "./config.cfg"
# Process opts
for o, a in opts:
if o in ("-c", "--config"):
configFile = a
if o in ("-d", "--dir"):
dirMode = True
dir = a
if o in ("-h", "--help"):
usage()
sys.exit()
if o in ("-o", "--output"):
output = a
if o in ("-e", "--erase"):
eraseItems = True
if o in ("-v", "--verboes"):
verbose = True
# Did we get called with proper agruments?
if (not dirMode and len(args) == 0):
usage()
sys.exit()
# Read in config file information
if not os.path.lexists(configFile):
print "ERROR: Can't find config file, exiting in shame. No items ingested."
sys.exit()
cp = DatacastingGen.DatacastingConfigParser()
cp.read(configFile)
# Extract necessary variables from config file
# Get feed level info; needed to deal with custom elements
rss = DatacastingGen.Feed.FromConfig(cp)
# itemSource: if present it needs a Source() object
isTuple = eval(cp.get('Item','itemsource',
dne = 'None'))
if isTuple is not None:
itemSource = PyRSS2Gen.Source(isTuple[0], isTuple[1])
else:
itemSource = None
itemLink = eval(cp.get('Item','itemlink',
dne = 'None'))
itemPathRoot = eval(cp.get('Generation', 'itempathroot'))
queueDirectory = eval(cp.get('Generation', 'queuedirectory'))
configDict = {}
configDict['simplifyPolygons'] = eval(cp.get('Item/Extents',
'simplifyPolygons',
dne = 'False'))
configDict['polyMaxPoints'] = eval(cp.get('Item/Extents',
'polyMaxPoints',
dne = '256'))
configDict['polyDistThresh'] = eval(cp.get('Item/Extents',
'polyDistThresh',
dne = '5.0'))
if dirMode:
fileList = [ os.path.abspath(dir) + "/" + x
for x in os.listdir(dir) ]
else:
fileList = [ os.path.abspath(x) for x in args ]
# DEBUG
# print "Item files processed: %d" % len(fileList)
for file in fileList:
try:
item = DatacastingGen.Item(txtFileName = file,
source = itemSource,
link = itemLink,
parentFeed = rss,
addlConfig = configDict)
# Twiddle the guid to match the enclosure + a filename....
enc = None
if item.enclosure is not None:
if item.enclosureList is not [] :
enc = item.enclosureList[0]
else:
enc = item.enclosure
if enc is not None:
item.guid = PyRSS2Gen.Guid(enc.url + "?file=" + file)
basename = os.path.basename(file)
(filebase, ext) = os.path.splitext(basename)
# Store the item in a hierarchical directory structure. If needed
# subdirectories don't exist, create them.
if item.pubDate is not None:
pd = item.pubDate
else:
pd = datetime.datetime.utcnow()
itemOutputPath = (itemPathRoot + "/" +
"%04d/" % pd.year +
("%02d-%02d" % (pd.month, pd.day)))
if not os.path.lexists(itemOutputPath):
os.makedirs(itemOutputPath)
# Save off the XML snippet
outXMLName = os.path.abspath(itemOutputPath +
("/%02d-%02d-%02d-" %
(pd.hour, pd.minute, pd.second)) +
basename + ".xml")
if (verbose and os.path.lexists(outXMLName)):
print "WARNING: Overwriting %s" % outXMLName
item.SaveXML(outXMLName)
# DEBUG
# print "outXMLName: %s" % outXMLName
# Link the created item into the queuing directory
fullQueuedPath = os.path.abspath(queueDirectory +
("/%04d-%02d-%02d-%02d-%02d-%02d-" %
(pd.year, pd.month, pd.day,
pd.hour, pd.minute, pd.second)) +
basename + ".xml")
if not os.path.lexists(queueDirectory):
os.makedirs(queueDirectory)
if os.path.lexists(fullQueuedPath):
if verbose:
print "WARNING: Requeuing %s" % fullQueuedPath
os.unlink(fullQueuedPath)
os.symlink(outXMLName, fullQueuedPath)
except Exception, inst:
print "WARNING: Item text file ingestion failed..."
print " Item file name: %s" % file
print " Exception: %s" % inst
print " Continuing processing."
raise
