def test_format(val,format): if format == 'numeric' and not is_num_like(val): if is_string_like(val): return 'string' else: return 'other' elif format == 'string' and not is_string_like(val): return 'other' return format
def checkMetadata(iterator): """ Sanity check on metadata. """ assert hasattr(iterator,'metadata'), 'Has no metadata attribute.' metadata = iterator.metadata assert isinstance(metadata,dict), 'Metadata isnt a dictionary.' assert all(map(is_string_like,metadata.keys())), 'Metadata keys must be strings' assert '' in metadata.keys(), 'Metadata must contain "" key.' assert all(map(lambda x : isinstance(x,dict),metadata.values())), 'Metadata values must be dictionaries.' assert all(map(lambda x : all(map(is_string_like,x.keys())),metadata.values())), 'metadata values\' keys must be strings.' assert all(['title' in metadata[k].keys() for k in metadata.keys() if k]), 'The subcollections following have no "title" metadata:' + str([k for k in metadata.keys() if 'title' not in metadata[k].keys() and k]) M = metadata[''] assert isinstance(M.get('keywords'),list) and all(map(is_string_like,M['keywords'])), 'Metadata must contain "keywords" entry, which must be a python list.' assert is_string_like(M.get('description')), 'Metadata must contain description.' S = M.get('source') assert isinstance(S,list), 'Metadata must contain source list.' try: S = OrderedDict(S) except: print 'Metadata source list in wrong format for making SON object.' else: pass required = ['agency','subagency','dataset'] assert all([r in S.keys() for r in required]), 'Source dictionary must contain agency, subagency, and dataset keys.' assert all(map(lambda x : isinstance(x,dict),S.values())), 'All source dictionary keys must be dictionaries containing "name" and possibly "shortName" keys.' assert all(map(lambda x : 'name' in x.keys(),S.values())) assert all(['shortName' in S[r].keys() for r in required]), 'shortName entry required for agency, subagency, and dataset source.' p = re.compile('[\w]*') assert all([p.match(y.get('shortName','')) for y in S.values()]), 'shortName entry can only contain alphanumeric and "_" characters.' SC = M.get('sliceCols') assert isinstance(SC,list) and all([isinstance(x,list) and all([is_string_like(y) or isinstance(y,tuple) for y in x]) for x in SC]), 'sliceCol metadata not present or improperly formed.' CG = M.get('columnGroups') assert isinstance(CG,dict) and 'labelColumns' in CG.keys(), 'columnGroup metadata not present or improperly formed.'
def decode_obj(x): if is_string_like(x): return decode(x) elif hasattr(x,'keys'): return pm.son.SON([(k,decode_obj(x[k])) for k in x.keys()]) elif is_instance(x,list): return [decode_obj(y) for y in x] else: return x
def makeHierarchy(V,E): """Convert hierarchy into a network graph and creates a dictionary of ordered list to describe the hiearchy""" G = nx.DiGraph() for v in V: G.add_node(v) for e in E: if is_string_like(e[0]): s = e[0] else: s = e[0][0] if is_string_like(e[1]): t = e[1] else: t = e[1][0] G.add_edge(s,t) H = dict([(o,v.keys()) for (o,v) in nx.all_pairs_shortest_path(G).items()]) T = nx.topological_sort(G) for k in H.keys(): H[k] = [j for j in T if j in H[k]] return [G,H]
def processArg(arg,collection): """Translates the arg to human readable to collections""" V = collection.columns C = collection.columnGroups if is_string_like(arg): argsplit = arg.split('.') if argsplit[0] in V: argsplit[0] = str(V.index(argsplit[0])) return '.'.join(argsplit) elif arg in C.keys(): return [str(V.index(c)) for c in C[arg]] else: return arg elif isinstance(arg, list): T = [processArg(d,collection) for d in arg] Tr = [] for t in T: if is_string_like(t): Tr.append(t) else: Tr += t return Tr elif isinstance(arg,tuple): return tuple(processArg(list(arg),collection)) elif isinstance(arg,dict): T = [(processArg(k,collection), v) for (k,v) in arg.items() if k != '$where' ] S = dict([(k,v) for (k,v) in T if not (isinstance(k,list) or isinstance(k,tuple))]) for (k,v) in T: if isinstance(k,list) or isinstance(k,tuple): S["$or"] = [{kk:v} for kk in k] if '$where' in arg: S['$where'] = arg['$where'] return S else: return arg
def applyHierarchy(H,bdict): """Helper for getFunc""" if isinstance(H,list): L = [applyHierarchy(h,bdict) for h in H] return son.SON([l for l in L if l[1]]) elif isinstance(H,tuple): assert 1 <= len(H) <= 2 assert is_string_like(H[0]) F = {} if not 'X' in bdict[H[0]]: F[''] = int(bdict[H[0]]) if len(H) == 2: assert isinstance(H[1],list) F.update(applyHierarchy(H[1],bdict)) return (H[0],F)
def integrate(l1,l2): """Combines two space objects""" if not l1: return l2 elif not l2: return l1 else: D = l1.copy() for (k,v) in l2.items(): if is_string_like(v): D[k] = v else: assert is_instance(v,dict) D[k].update(v) return D
def intersect(l1,l2): """Intersects two space objects""" if l1 and l2 : I = dict([(k,l1[k]) for k in set(l1.keys()).intersection(l2.keys())]) for (k,v) in l2.items(): if k in I.keys() and v != I[k]: if is_string_like(v): for j in SPACE_HIERARCHY[k]: if j in I.keys(): I.pop(j) else: D = intersect(I[k],v) if D: I[k] = D else: I.pop(k) return I
def expand(r): L = [k for (k,v) in r if isinstance(v,list)] I = itertools.product(*tuple([v for (k,v) in r if isinstance(v,list)])) return [tuple([(k,v) for (k,v) in r if is_string_like(v)] + zip(L,x)) for x in I]
def geodbGuts(g,level_code): """General geodb lookup function Helper""" level = loc.SPACE_DB_TABLES[level_code] methodstr = g['method'] method = getattr(getattr(models,level).objects,methodstr) if 'querylist' in g: querylist = g['querylist'] if is_string_like(querylist): querylist = json.loads(querylist) else: keylist =['field','query','type','pattern','radius','units'] querylist = [dict( [(k,g[k]) for k in keylist if k in g])] argdict = {} for d in querylist: field = d['field'] type = d['type'] if 'type' in d.keys() else '' key = field + ('__' + type if type else '') if type == 'relate': geom = d['query'] pattern = d['pattern'] results = (geom,pattern) elif type in ['distance_lt','distance_lte','distance_gt','distance_gte','dwithin']: geom = d['query'] radius = float(d['radius']) if 'units' in d.keys(): units = d['units'] from django.contrib.gis.measure import D dobj = D(**{units:radius}) else: dobj = radius args = (geom,dobj) else: args = d['query'] argdict[str(key)] = args results = method(**argdict) if 'return' in g: returnVals = g['return'].split(',') if 'name' in returnVals: returnVals[returnVals.index('name')] = loc.LEVEL_NAMES[level_code] if 'code' in returnVals: returnVals[returnVals.index('code')] = loc.LEVEL_CODES[level_code] else: name = loc.LEVEL_NAMES[level_code] code = loc.LEVEL_CODES[level_code] returnVals = [name,code] if g.get('stringify',True): results = [dict([(a,str(Rgetattr(x,a.split('.')))) for a in returnVals if Rhasattr(x,a.split('.'))]) for x in results] else: results = [dict([(a,Rgetattr(x,a.split('.'))) for a in returnVals if Rhasattr(x,a.split('.'))]) for x in results] return results
def processSolrArg(base,key,value): """ helper for solrURL """ return base + ('.' if key and base else '') + key + '=' + urllib.quote(value) if is_string_like(value) else '&'.join([base + ('.' if key and base else '') + key + '=' + urllib.quote(v) for v in value])