class PubsItemsMgr(ProvItemsMgr):
"""
build mapping from provCode to provItem: self[provCode] = provItem
also
- contry code to vocabValue
- provTypeName to provItems
"""
def __init__(self, xsd):
"""
- xsd is a StateCountryVocabXSD instance which provides the items
- provItemsMdr calls load()
"""
self.countryCodes = UserDict()
self.provTypes = UserDict()
self.xsd = xsd
ProvItemsMgr.__init__(self)
def load(self):
"""
obtain items from the pubs xsd file, and
populate this class's data structures
"""
for vocab in self.xsd.getCountriesType().getValues():
self.countryCodes[vocab.value] = vocab
for vocab in self.xsd.getProvTypes():
prov = ProvItems(vocab)
self.provTypes[prov.name] = prov
for provItem in prov.values():
self[provItem.value] = provItem
def getProvItems(self, countryTypeName):
"""
gets all items having supplied countryTypeName (e.g., 'austrailiaType'),
which is how the pubs iems for a country are organized.
note: it would be nice to never deal with the typeName - very ugly
"""
return self.provTypes[provTypeName].values()
def getProvItem(self, provCode):
"""
obtain the provItem for the specified provCode (e.g., 'US-CO')
"""
return self[provCode]
def report(self):
print "%d items were read" % len(self)
if 0:
print "\nCountryCodes"
for code in self.countryCodes.keys():
print self.countryCodes[code]
for provTypeName in self.provTypes.keys():
self.provTypes[provTypeName].report()
def test_UserDict_as_prototype(self):
d = UserDict()
e = derive(d)
e = derive(d, bind=dict(keys=d))
d['a'] = 1
self.assertRaises(KeyError, lambda :e['a'])
self.assertEqual(e.keys(), d.keys())
class CollectionInfoSearcher(RepositorySearcher):
"""
Searches the collection of collection records and exposes
"collections" as a mapping from collection id to collectionInfo
"""
batchSize = 500
verbose = True
collection_info_constructor = DleseCollectionInfo
def __init__(self, collection, xmlFormat, baseUrl):
self.collections = UserDict()
RepositorySearcher.__init__(self,
collection=collection,
xmlFormat=xmlFormat,
baseUrl=baseUrl)
def get_paramsOFF(self, collection, xmlFormat):
params = RepositorySearcher.get_params(self, collection, xmlFormat)
for p in params:
print '- %s: %s' % (p, params[p])
return params
def processResults(self):
"""
concrete classes should override this method to do some real processing
"""
for result in self:
# print result
# sys.exit()
info = self.collection_info_constructor(result)
self.collections[info.key] = info
# print '- %s (%s)' % (info.title, info.key)
# sys.exit()
def getCollectionInfo(self, key):
"""
return collection info for specified collection
"""
print 'lookinfo for key:"%s"' % key
if not self.collections.has_key(key):
for key in self.collections.keys():
print '- ', key
return None
return self.collections[key]
def writeActivityLog(self, trans):
"""Write an entry to the activity log.
Writes an entry to the script log file. Uses settings
``ActivityLogFilename`` and ``ActivityLogColumns``.
"""
filename = self.serverSidePath(
self.setting('ActivityLogFilename'))
if os.path.exists(filename):
f = open(filename, 'a')
else:
f = open(filename, 'w')
f.write(','.join(self.setting('ActivityLogColumns')) + '\n')
values = []
# We use UserDict on the next line because we know it inherits
# NamedValueAccess and reponds to valueForName()
objects = UserDict({
'application': self,
'transaction': trans,
'request': trans.request(),
'response': trans.response(),
'servlet': trans.servlet(),
'session': trans._session, # don't cause creation of session
})
for column in self.setting('ActivityLogColumns'):
try:
value = objects.valueForName(column)
except Exception:
value = '(unknown)'
if type(value) is FloatType:
# probably need more flexibility in the future
value = '%0.2f' % value
else:
value = str(value)
values.append(value)
f.write(','.join(values) + '\n')
f.close()
for i in objects.keys():
objects[i] = None
# Test copy()
u2a = u2.copy()
assert u2a == u2
class MyUserDict(UserDict):
def display(self): print self
m2 = MyUserDict(u2)
m2a = m2.copy()
assert m2a == m2
# Test keys, items, values
assert u2.keys() == d2.keys()
assert u2.items() == d2.items()
assert u2.values() == d2.values()
# Test has_key
for i in u2.keys():
assert u2.has_key(i) == 1
assert u1.has_key(i) == d1.has_key(i)
assert u0.has_key(i) == d0.has_key(i)
# Test update
t = UserDict()
t.update(u2)
assert t == u2
def keys(self):
self.__populate()
return UserDict.keys( self )
def keys (self):
if self.finalized:
return range(1, self.worksheet.GetMaxColumn() + 1)
else:
return UserDict.keys(self)
def keys(self):
self.__populate()
return UserDict.keys( self )
class VdxRecord(MetaDataRecord):
verbose = 0
xpath_delimiter = '/'
# default_vdx_template = 'VDX-TEMPLATE.xml'
default_vdx_template = util.getTemplate('VDX-TEMPLATE')
def __init__(self, template=None):
template = template or self.default_vdx_template
MetaDataRecord.__init__(self, path=template)
self.shapes = UserDict()
self.edges = UserDict()
self.nodes = UserDict()
def getShapeNodes(self):
"""
return the shape elements from DOM
"""
return self.selectNodes(self.dom,
'VisioDocument/Pages/Page/Shapes/Shape')
def getShapeId(self):
return str(len(self.getShapeNodes()) + 1)
def _shapeGetter(self, shapeId):
return self.getShape(shapeId)
def getShapeByName(self, name):
# print 'getShapeByName (%s)' % name
for node in self.shapes.values():
if node.Name == name:
return node
def getShape(self, shapeId):
"""
return the Shape instance for provided shapeId or None if not defined
"""
return self.shapes.has_key(shapeId) and self.shapes[shapeId] or None
def makeEdgeShape(self, source, target, relation, id=None):
id = id or self.getShapeId()
# return makeConnectorShape(source, target, relation, self.getShapeId())
line_args = {
'name': 'connector',
'label': {
'text': relation
},
# 'x' : avg (start[0], end[0]),
# 'y' : avg (start[1], end[1]),
# 'x' : start[0],
# 'y' : start[1],
# 'height' : diff (end[1], start[1]) or util.pt2in(2),
# 'width' : diff (end[0],start[0]),
# 'begin_x' : start[0],
# 'begin_y' : start[1],
# 'end_x' : end[0],
# 'end_y' : end[1]
}
return Line(id, line_args)
def addEdge(self, sourceId, targetId, relation):
try:
# source = self.getShape(sourceId)
#source = self.getShapeByName(sourceId)
source = self._shapeGetter(sourceId)
if not source:
raise Exception, "sourceId '%s'" % sourceId
# target = self.getShape(targetId)
# target = self.getShapeByName(targetId)
target = self._shapeGetter(targetId)
if not target:
raise Exception, "targetId '%s'" % targetId
except Exception, msg:
print "addEdge Error: could not find a shape (%s)" % msg
# print self
print "SHAPE KEYS: %s" % self.shapes.keys()
for key in self.shapes.keys():
# print "%s: %s" % (key, self.shapes[key])
print " - ", key
print "HALTInG ..."
sys.exit()
# edge = makeConnectorShape(source, target, relation, self.getShapeId())
edge = self.makeEdgeShape(source, target, relation, self.getShapeId())
# parent = self.selectSingleNode (self.dom, 'VisioDocument/Pages/Page/Shapes')
# if not parent:
# raise xml.dom.NotFoundErr, 'Shapes node not found'
# parent.appendChild(edge.getElement());
self.edges[edge.ID] = edge
self.addShapeObj(edge)
# add the connect element
self.addConnect(edge, source, target)
return edge.ID
def keys(self):
if self.finalized:
return list(range(1, self.worksheet.GetMaxColumn() + 1))
else:
return UserDict.keys(self)
class MetadataScanner (SimpleScanner):
outpath = 'METADATA_SCAN_URLS.txt'
def __init__ (self, baseDir):
self.unique_protected_urls = []
self.protected_url_count = 0
self.unique_asset_filenames = []
self.metadata_info = UserDict()
self.recordsToUpdate = UserDict() # recordId -> RecordInfo
SimpleScanner.__init__(self, baseDir)
def getRecordInfo (self, recordId):
if self.recordsToUpdate.has_key(recordId):
return self.recordsToUpdate[recordId]
return None
def getRecordInfoForPath (self, path):
"""
return existing recordInfo if possible, otherwise
create and return new recordInfo
"""
tmpInfo = RecordInfo(path)
recordId = tmpInfo.recordId
if self.recordsToUpdate.has_key(recordId):
return self.recordsToUpdate[recordId]
else:
self.recordsToUpdate[recordId] = tmpInfo
return tmpInfo
def acceptFileToScan (self, path):
"""
bolean determining whether this file is scanned
e.g., typically look for xml files when processing metadata
"""
return os.path.isfile(path) and os.path.basename(path).endswith('.xml')
def processPath (self, path):
SimpleScanner.processPath(self, path)
content = open(path, 'r').read()
urlPattern= 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
for m in re.finditer(urlPattern, content):
url = m.group()
# urlPattern doesn't catch the tag ending at the tail of urls
for ending in [
'</primaryURL>',
'</url>',
'</urlEntry>',
'</standardURL>'
]:
if url.endswith(ending):
url = url.replace(ending,'')
# print '-m: ' + m.group()
if bscs.protected.isAnyProtectedUrl (url):
filename = os.path.basename(url)
if not url in self.unique_protected_urls:
self.unique_protected_urls.append(url)
if not filename in self.unique_asset_filenames:
self.unique_asset_filenames.append(filename)
self.protected_url_count = self.protected_url_count + 1
infos = self.metadata_info.has_key(url) and self.getMetadataInfo(url) or []
info = self.getRecordInfoForPath(path)
info.protectedUrls.append(url)
infos.append(info)
self.metadata_info[url] = infos
# print 'just assigned
def report (self):
SimpleScanner.report(self)
print '\nunique protected URLs: %d (out of %d total)' % (len(self.unique_protected_urls), self.protected_url_count)
print 'unique asset filenames: %d' % len(self.unique_asset_filenames)
def reportMetadataInfo (self):
urls = self.metadata_info.keys()
urls.sort()
print "metadata info report (%d records)" % len(urls)
for url in urls:
print '- %s' % url
for info in self.metadata_info[url]:
print ' -', info.recordId
def getRecordMap (self):
"""
OBSOLETE - Use self.recordsToUpdate
"""
recordMap = {}
for url in self.metadata_info.keys():
for info in self.metadata_info[url]:
recId = info.recordId
if not recordMap.has_key(recId):
values = []
else:
values = recordMap[recId]
values.append(url)
recordMap[recId] = values
return recordMap
def reportRecordMap (self):
"""
OBSOLETE - see self.recordsToUpdate()
"""
recordMap = self.getRecordMap()
keys = recordMap.keys()
keys.sort()
print '\n%d records containing protectedUrls' % len(keys)
for recId in keys:
print "- %s" % recId
for url in recordMap[recId]:
print ' - %s' % url
def getMetadataInfo (self, url):
"""
returns a LIST of metadata infos for given url
"""
if not self.metadata_info.has_key(url):
return None
return self.metadata_info[url]
def writeProtectedUrls (self, outpath=None):
if outpath is None:
outpath = self.outpath
self.unique_protected_urls.sort()
fp = open(outpath, 'w')
fp.write ('\n'.join(self.unique_protected_urls))
fp.close()
print 'wrote unique protected URLs to ', outpath
# Test copy()
u2a = u2.copy()
verify(u2a == u2)
class MyUserDict(UserDict):
def display(self): print self
m2 = MyUserDict(u2)
m2a = m2.copy()
verify(m2a == m2)
# Test keys, items, values
verify(u2.keys() == d2.keys())
verify(u2.items() == d2.items())
verify(u2.values() == d2.values())
# Test has_key
for i in u2.keys():
verify(u2.has_key(i) == 1)
verify(u1.has_key(i) == d1.has_key(i))
verify(u0.has_key(i) == d0.has_key(i))
# Test update
t = UserDict()
t.update(u2)
verify(t == u2)
