def test_format(val,format):
if format == 'numeric' and not is_num_like(val):
if is_string_like(val):
return 'string'
else:
return 'other'
elif format == 'string' and not is_string_like(val):
return 'other'
return format
def checkMetadata(iterator):
"""
Sanity check on metadata.
"""
assert hasattr(iterator,'metadata'), 'Has no metadata attribute.'
metadata = iterator.metadata
assert isinstance(metadata,dict), 'Metadata isnt a dictionary.'
assert all(map(is_string_like,metadata.keys())), 'Metadata keys must be strings'
assert '' in metadata.keys(), 'Metadata must contain "" key.'
assert all(map(lambda x : isinstance(x,dict),metadata.values())), 'Metadata values must be dictionaries.'
assert all(map(lambda x : all(map(is_string_like,x.keys())),metadata.values())), 'metadata values\' keys must be strings.'
assert all(['title' in metadata[k].keys() for k in metadata.keys() if k]), 'The subcollections following have no "title" metadata:' + str([k for k in metadata.keys() if 'title' not in metadata[k].keys() and k])
M = metadata['']
assert isinstance(M.get('keywords'),list) and all(map(is_string_like,M['keywords'])), 'Metadata must contain "keywords" entry, which must be a python list.'
assert is_string_like(M.get('description')), 'Metadata must contain description.'
S = M.get('source')
assert isinstance(S,list), 'Metadata must contain source list.'
try:
S = OrderedDict(S)
except:
print 'Metadata source list in wrong format for making SON object.'
else:
pass
required = ['agency','subagency','dataset']
assert all([r in S.keys() for r in required]), 'Source dictionary must contain agency, subagency, and dataset keys.'
assert all(map(lambda x : isinstance(x,dict),S.values())), 'All source dictionary keys must be dictionaries containing "name" and possibly "shortName" keys.'
assert all(map(lambda x : 'name' in x.keys(),S.values()))
assert all(['shortName' in S[r].keys() for r in required]), 'shortName entry required for agency, subagency, and dataset source.'
p = re.compile('[\w]*')
assert all([p.match(y.get('shortName','')) for y in S.values()]), 'shortName entry can only contain alphanumeric and "_" characters.'
SC = M.get('sliceCols')
assert isinstance(SC,list) and all([isinstance(x,list) and all([is_string_like(y) or isinstance(y,tuple) for y in x]) for x in SC]), 'sliceCol metadata not present or improperly formed.'
CG = M.get('columnGroups')
assert isinstance(CG,dict) and 'labelColumns' in CG.keys(), 'columnGroup metadata not present or improperly formed.'
def decode_obj(x):
if is_string_like(x):
return decode(x)
elif hasattr(x,'keys'):
return pm.son.SON([(k,decode_obj(x[k])) for k in x.keys()])
elif is_instance(x,list):
return [decode_obj(y) for y in x]
else:
return x
def makeHierarchy(V,E):
"""Convert hierarchy into a network graph and creates a dictionary of ordered list to describe the hiearchy"""
G = nx.DiGraph()
for v in V:
G.add_node(v)
for e in E:
if is_string_like(e[0]):
s = e[0]
else:
s = e[0][0]
if is_string_like(e[1]):
t = e[1]
else:
t = e[1][0]
G.add_edge(s,t)
H = dict([(o,v.keys()) for (o,v) in nx.all_pairs_shortest_path(G).items()])
T = nx.topological_sort(G)
for k in H.keys():
H[k] = [j for j in T if j in H[k]]
return [G,H]
def processArg(arg,collection):
"""Translates the arg to human readable to collections"""
V = collection.columns
C = collection.columnGroups
if is_string_like(arg):
argsplit = arg.split('.')
if argsplit[0] in V:
argsplit[0] = str(V.index(argsplit[0]))
return '.'.join(argsplit)
elif arg in C.keys():
return [str(V.index(c)) for c in C[arg]]
else:
return arg
elif isinstance(arg, list):
T = [processArg(d,collection) for d in arg]
Tr = []
for t in T:
if is_string_like(t):
Tr.append(t)
else:
Tr += t
return Tr
elif isinstance(arg,tuple):
return tuple(processArg(list(arg),collection))
elif isinstance(arg,dict):
T = [(processArg(k,collection), v) for (k,v) in arg.items() if k != '$where' ]
S = dict([(k,v) for (k,v) in T if not (isinstance(k,list) or isinstance(k,tuple))])
for (k,v) in T:
if isinstance(k,list) or isinstance(k,tuple):
S["$or"] = [{kk:v} for kk in k]
if '$where' in arg:
S['$where'] = arg['$where']
return S
else:
return arg
def applyHierarchy(H,bdict):
"""Helper for getFunc"""
if isinstance(H,list):
L = [applyHierarchy(h,bdict) for h in H]
return son.SON([l for l in L if l[1]])
elif isinstance(H,tuple):
assert 1 <= len(H) <= 2
assert is_string_like(H[0])
F = {}
if not 'X' in bdict[H[0]]:
F[''] = int(bdict[H[0]])
if len(H) == 2:
assert isinstance(H[1],list)
F.update(applyHierarchy(H[1],bdict))
return (H[0],F)
def integrate(l1,l2):
"""Combines two space objects"""
if not l1:
return l2
elif not l2:
return l1
else:
D = l1.copy()
for (k,v) in l2.items():
if is_string_like(v):
D[k] = v
else:
assert is_instance(v,dict)
D[k].update(v)
return D
def intersect(l1,l2):
"""Intersects two space objects"""
if l1 and l2 :
I = dict([(k,l1[k]) for k in set(l1.keys()).intersection(l2.keys())])
for (k,v) in l2.items():
if k in I.keys() and v != I[k]:
if is_string_like(v):
for j in SPACE_HIERARCHY[k]:
if j in I.keys():
I.pop(j)
else:
D = intersect(I[k],v)
if D:
I[k] = D
else:
I.pop(k)
return I
def expand(r):
L = [k for (k,v) in r if isinstance(v,list)]
I = itertools.product(*tuple([v for (k,v) in r if isinstance(v,list)]))
return [tuple([(k,v) for (k,v) in r if is_string_like(v)] + zip(L,x)) for x in I]
def geodbGuts(g,level_code):
"""General geodb lookup function Helper"""
level = loc.SPACE_DB_TABLES[level_code]
methodstr = g['method']
method = getattr(getattr(models,level).objects,methodstr)
if 'querylist' in g:
querylist = g['querylist']
if is_string_like(querylist):
querylist = json.loads(querylist)
else:
keylist =['field','query','type','pattern','radius','units']
querylist = [dict( [(k,g[k]) for k in keylist if k in g])]
argdict = {}
for d in querylist:
field = d['field']
type = d['type'] if 'type' in d.keys() else ''
key = field + ('__' + type if type else '')
if type == 'relate':
geom = d['query']
pattern = d['pattern']
results = (geom,pattern)
elif type in ['distance_lt','distance_lte','distance_gt','distance_gte','dwithin']:
geom = d['query']
radius = float(d['radius'])
if 'units' in d.keys():
units = d['units']
from django.contrib.gis.measure import D
dobj = D(**{units:radius})
else:
dobj = radius
args = (geom,dobj)
else:
args = d['query']
argdict[str(key)] = args
results = method(**argdict)
if 'return' in g:
returnVals = g['return'].split(',')
if 'name' in returnVals:
returnVals[returnVals.index('name')] = loc.LEVEL_NAMES[level_code]
if 'code' in returnVals:
returnVals[returnVals.index('code')] = loc.LEVEL_CODES[level_code]
else:
name = loc.LEVEL_NAMES[level_code]
code = loc.LEVEL_CODES[level_code]
returnVals = [name,code]
if g.get('stringify',True):
results = [dict([(a,str(Rgetattr(x,a.split('.')))) for a in returnVals if Rhasattr(x,a.split('.'))]) for x in results]
else:
results = [dict([(a,Rgetattr(x,a.split('.'))) for a in returnVals if Rhasattr(x,a.split('.'))]) for x in results]
return results
def processSolrArg(base,key,value):
"""
helper for solrURL
"""
return base + ('.' if key and base else '') + key + '=' + urllib.quote(value) if is_string_like(value) else '&'.join([base + ('.' if key and base else '') + key + '=' + urllib.quote(v) for v in value])