def run_server(path_cartograph_cfg, path_tilestache_cfg):
Config.initConf(path_cartograph_cfg)
path_tilestache_cfg = os.path.abspath(path_tilestache_cfg)
path_cache = json.load(open(path_tilestache_cfg, 'r'))['cache']['path']
static_files = {'/static': os.path.join(os.path.abspath('./web'))}
if os.path.isdir(path_cache):
assert (len(path_cache) > 5)
shutil.rmtree(path_cache)
app = CartographServer(path_tilestache_cfg, Config.get())
run_simple('0.0.0.0', 8080, app, static_files=static_files)
def run_server(path_cartograph_cfg, path_tilestache_cfg):
"""Server run function, you probably don't care about this part if all you care about is implementing search"""
Config.initConf(path_cartograph_cfg)
path_tilestache_cfg = os.path.abspath(path_tilestache_cfg)
path_cache = json.load(open(path_tilestache_cfg, 'r'))['cache']['path']
static_files = {'/static': os.path.join(os.path.abspath('./web'))}
if os.path.isdir(path_cache):
assert (len(path_cache) > 5)
shutil.rmtree(path_cache)
app = CartographServer(path_tilestache_cfg, Config.get())
run_simple('0.0.0.0', 8080, app, static_files=static_files)
def run_server(path_cartograph_cfg, path_tilestache_cfg):
Config.initConf(path_cartograph_cfg)
path_tilestache_cfg = os.path.abspath(path_tilestache_cfg)
path_cache = json.load(open(path_tilestache_cfg, 'r'))['cache']['path']
static_files = { '/static': os.path.join(os.path.abspath('./web')) }
if os.path.isdir(path_cache):
assert(len(path_cache) > 5)
shutil.rmtree(path_cache)
app = CartographServer(path_tilestache_cfg, Config.get())
run_simple('0.0.0.0', 8080, app, static_files=static_files)
def run_server(path_cartograph_cfg, path_tilestache_cfg):
"""Server run function, you probably don't care about this part if all you care about is implementing search"""
Config.initConf(path_cartograph_cfg)
path_tilestache_cfg = os.path.abspath(path_tilestache_cfg)
path_cache = json.load(open(path_tilestache_cfg, 'r'))['cache']['path']
static_files = { '/static': os.path.join(os.path.abspath('./web')) }
if os.path.isdir(path_cache):
assert(len(path_cache) > 5)
shutil.rmtree(path_cache)
app = CartographServer(path_tilestache_cfg, Config.get())
run_simple('0.0.0.0', 8080, app, static_files=static_files)
def _renderBackground(self, metric, z, x, y):
(polys, points) = self.countryService.getPolys(z, x, y)
clusterIds = set()
polysByName = {}
for pinfo in polys:
layer, shp, props = pinfo
if layer == 'countries':
polysByName[props['clusterId']] = shp
clusterIds.add(props['clusterId'])
else:
assert(layer == 'centroid_contours')
polysByName[props['clusterId'], int(props['contourNum'])] = shp
numContours = self.conf.getint('PreprocessingConstants', 'num_contours')
colors = Config.getFullColorWheel()
surface = cairo.ImageSurface(cairo.FORMAT_ARGB32, self.size, self.size)
context = cairo.Context(surface)
# First draw clusters
for i in clusterIds:
shp = polysByName[i]
c = metric.adjustCountryColor(colors[i][numContours], 0)
self._drawPoly(z, x, y, context, shp, c, (0.7, 0.7, 0.7))
for j in range(numContours):
if (i, j) in polysByName:
shp = polysByName[i, j]
c = metric.adjustCountryColor(colors[i][j], j + 1)
self._drawPoly(z, x, y, context, shp, c)
return surface
def run(self):
config = Config.get()
points = read_features(config.get('GeneratedFiles',
'article_coordinates'),
config.get('ExternalFiles', 'external_ids'),
required=('x', 'y', 'externalId'))
externalData = read_features(self.inPath)
records = []
stringFields = set()
for i, (id, p) in enumerate(points.items()):
if i % 100000 == 0:
logger.info('insert point %d of %d' % (i, len(points)))
extId = p['externalId']
if extId not in externalData: continue
pinfo = {'id': id}
for (k, v) in externalData[extId].items():
try:
v = float(v)
except ValueError:
stringFields.add(k)
pinfo[k] = v
records.append(pinfo)
for r in records:
for sf in stringFields:
r[sf] = str(r[sf])
with open(self.outPath, "w") as f:
for r in records:
json.dump(r, f)
f.write('\n')
def run(self):
conf = Config.get()
w2vPath = conf.get('ExternalFiles', 'w2v')
idMapping = {}
for id, idInfo in read_features(
conf.get('ExternalFiles', 'external_ids')).items():
idMapping[idInfo['externalId']] = id
ft = FreeText(w2vPath, idMapping)
ft.rebuildIfNecessary()
def __init__(self, conf):
self.fields = []
nc = conf.getint('PreprocessingConstants', 'num_contours')
self.colors = {}
self.colorCountries = True
for cid, ccolors in Config.getFullColorWheel().items():
self.colors[cid] = colour.Color(ccolors[nc - 1]).rgb
self.neutralColor = colour.Color("#777").rgb
def __init__(self):
config = Config.get()
self._host = config.get('PG', 'host')
self._database = config.get('PG', 'database')
self._user = config.get('PG', 'user') or None
self._password = config.get('PG', 'password') or None
self._table = 'edges'
self._dataFile = config.get('GeneratedFiles', 'edge_bundles')
self.columns = ['bundle', 'weights', 'numPoints', 'endPoints']
logger.info('loading %s.%s from %s' %
(self._database, self._table, self._dataFile))
super(LoadCoordinateEdges, self).__init__()
print self.table
def requires(self):
config = Config.get()
result = []
metricDir = config.get('DEFAULT', 'metricDir')
for name in config.get('Metrics', 'active').split():
metricConf = json.loads(config.get('Metrics', name))
path = metricConf['path']
args = {
'name': name,
'inPath': path,
'outPath': os.path.join(metricDir, name + '.json'),
}
result.append(MetricData(**args))
return result
def configData(self):
result = {}
sections = [
'PreprocessingConstants', 'MapConstants', 'Server', 'Metrics'
]
for sec in sections:
result[sec] = {}
for (key, value) in self.config.items(sec):
result[sec][key] = value
for name in result['Metrics']['active'].split():
result['Metrics'][name] = json.loads(result['Metrics'][name])
result['ColorWheel'] = Config.getColorWheel()
return result
def run(self):
config = Config.get()
coords = read_features(
config.get("GeneratedFiles", "article_coordinates"))
pathIn = config.get("ExternalFiles", "links")
pathOut = config.get("GeneratedFiles", "edges_with_coords")
with open(pathIn) as fin, open(pathOut, 'w') as fout:
logging.getLogger('counting edges')
# Count num of valid edges and vertices
numEdges = 0
numVertices = 0
for line in fin:
vertices = line.split()
src = vertices[0]
if src not in coords: continue
numVertices += 1
for dest in vertices[1:]:
if dest in coords: numEdges += 1
logger.info(
'found %d edges containing %d vertices (out of %d total vertices)'
% (numEdges, numVertices, len(coords)))
fin.seek(0)
fout.write(str(numEdges) + '\n')
for line in fin:
vertices = line.split()
src = vertices[0]
if src not in coords: continue
numVertices += 1
for dest in vertices[1:]:
if dest in coords:
fields = (coords[src]['x'], coords[src]['y'],
coords[dest]['x'], coords[dest]['y'])
fout.write(' '.join(fields) + '\n')
fin.close()
fout.close()
def run(self):
config = Config.get()
featureDict = Utils.read_features(config.get("ExternalFiles",
"names_with_id"))
idList = list(featureDict.keys())
nameDict = {}
with open(config.get("ExternalFiles", "popularity")) as popularity:
lines = popularity.readlines()
for line in lines:
lineAr = line.split("\t")
name = lineAr[0]
pop = lineAr[1][:-1]
nameDict[name] = pop
popularityList = []
for featureID in idList:
name = featureDict[featureID]["name"]
popularityList.append(nameDict[name])
Utils.write_tsv(config.get('GeneratedFiles', 'popularity_with_id'),
("id", "popularity"),
idList, popularityList)
def output(self):
w2vPath = Config.get().get('ExternalFiles', 'w2v')
return [TimestampedLocalTarget(p) for p in FreeText.paths(w2vPath)]
def output(self):
config = Config.get()
return TimestampedLocalTarget(
config.get("GeneratedFiles", "edge_bundles"))
from cartograph.server.TemplateService import TemplateService
from cartograph.server.TileService import TileService
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
if __name__ == '__main__' and len(sys.argv) > 1:
confPath = sys.argv[1]
else:
confPath = os.getenv('CARTOGRAPH_CONFIG')
if not confPath:
raise Exception, 'CARTOGRAPH_CONFIG environment variable not set!'
if not os.path.isfile(confPath):
raise Exception, 'Cartograph Config Path %s does not exist' % `confPath`
conf = Config.initConf(confPath)
if os.getenv('CLEAR_CACHE'):
logging.info('clearing cache directory %s' % conf.get('DEFAULT', 'webCacheDir'))
shutil.rmtree(conf.get('DEFAULT', 'webCacheDir'), ignore_errors=True)
logging.info('intitializing services')
loggingService = LoggingService(conf)
pointService = PointService(conf)
countryService = CountryService(conf)
tileService = TileService(conf, pointService, countryService)
mapnikService = RasterService(conf, pointService, countryService)
templateService = TemplateService(conf)
staticService = StaticService(conf)
# freeText = FreeText(conf.get('ExternalFiles', 'w2v'))
def output(self):
config = Config.get()
return (TimestampedLocalTarget(config.get("ExternalFiles",
"vecs_with_id")),
TimestampedLocalTarget(config.get("ExternalFiles",
"names_with_id")))
def run(self):
config = Config.get()
assert (False)
def run(self):
config = Config.get()
for k in self.configKeys:
fn = config.get('DEFAULT', k)
if not os.path.isdir(fn): os.makedirs(fn)
def __init__(self, config):
self.config = config
def on_get(self, req, resp):
js = self.configData()
resp.status = falcon.HTTP_200
resp.body = 'var CG = CG || {}; CG.config = ' + json.dumps(js) + ';'
resp.content_type = 'application/javascript'
def configData(self):
result = {}
sections = [
'PreprocessingConstants', 'MapConstants', 'Server', 'Metrics'
]
for sec in sections:
result[sec] = {}
for (key, value) in self.config.items(sec):
result[sec][key] = value
for name in result['Metrics']['active'].split():
result['Metrics'][name] = json.loads(result['Metrics'][name])
result['ColorWheel'] = Config.getColorWheel()
return result
if __name__ == '__main__':
conf = Config.initConf('./data/conf/simple.txt')
svc = ConfigService(conf)
print svc.configData()
def output(self):
config = Config.get()
return (TimestampedLocalTarget(
config.get("ExternalFiles", "region_names")))
def output(self):
config = Config.get()
return (TimestampedLocalTarget(
config.get("ExternalFiles", "vecs_with_id")),
TimestampedLocalTarget(
config.get("ExternalFiles", "names_with_id")))
def output(self):
config = Config.get()
return TimestampedLocalTarget(
config.get("GeneratedFiles", "edges_with_coords"))
'external_ids')).items():
external2Internal[record['externalId']] = id
qualities = {}
if pathIn.lower().endswith('bz2'):
f = bz2.BZ2File(pathIn, 'r')
else:
f = open(pathIn, 'r')
for i, line in enumerate(f):
if i % 500000 == 0:
print 'processing line %d, found %d' % (i, len(qualities))
if i == 0: continue # skip header
tokens = line.split('\t')
externalId = tokens[0]
grade = tokens[4]
score = float(tokens[5])
if externalId in external2Internal:
qualities[externalId] = grade, score
ids = sorted(qualities.keys())
grades = [qualities[id][0] for id in ids]
scores = [qualities[id][1] for id in ids]
metricDir = conf.get('DEFAULT', 'metricDir')
if not os.path.isdir(metricDir):
os.makedirs(metricDir)
write_tsv(metricDir + '/quality.tsv', ['id', 'grade', 'score'], ids,
grades, scores)
main(Config.initConf(sys.argv[1]), sys.argv[2])
def requires(self):
conf = Config.get()
return (ExternalFile(self.inPath),
ExternalFile(conf.get('ExternalFiles', 'external_ids')),
CreateFullCoordinates(), MetricsCode())
def output(self):
config = Config.get()
return (TimestampedLocalTarget(config.get("ExternalFiles", "region_names")))
def f(x): return int(255 * x)
color = 'rgba(%d,%d,%d,%.3f)' % (f(r), f(g), f(b), a)
props = { 'id' : p['id'],
'zpop' : p['zpop'],
'color' : color,
'zoff' : (z - p['zpop'])
}
for f in metric.fields:
props[f] = p.get(f, 0.0)
builder.addPoint('cities', p['name'],
shapely.geometry.Point(p['x'], p['y']), props)
if __name__ == '__main__':
from cartograph import Config
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
Config.initConf('data/conf/simple.txt')
pd = PointService(Config.get())
for p in pd.getTilePoints(6, 26, 36, 10):
print p['id'], p['zpop']
# for p in pd.getTilePoints(6, 27, 37, 1000):
# if p['id'] == '13616':
# print p['id']
# builder = TopoJsonBuilder()
# pd.addLayers(builder, 'gender', 6, 27, 36)
# pd.addLayers(builder, 'gender', 6, 27, 37)
# print builder.toJson()
def run(self):
config = Config.get()
for k in self.configKeys:
fn = config.get('DEFAULT', k)
if not os.path.isdir(fn): os.makedirs(fn)
def output(self):
config = Config.get()
return list(
luigi.LocalTarget(config.get('DEFAULT', n)) for n in self.configKeys
)
def output(self):
config = Config.get()
return (TimestampedLocalTarget(
config.get("ExternalFiles", "popularity")))
def on_get(self, req, resp):
resp.status = falcon.HTTP_200
builder = TopoJsonBuilder()
mc = 100
builder.addPolygon('related-overlay',
shapely.geometry.box(-mc, -mc, +mc, +mc))
n = int(req.params['n'])
results = self.related(req.params['q'], n)
for i, (p, score) in enumerate(results):
percentile = (1.0 - 1.0 * i / len(results))
props = {
'score': score,
'name': p['name'],
'percentile': percentile,
'percentile2': percentile**2,
'fontsize': int(percentile**2 * 20)
}
builder.addPoint('related', p['name'],
shapely.geometry.Point(p['x'], p['y']), props)
resp.body = builder.toJson()
resp.content_type = 'application/json'
if __name__ == '__main__':
logging.basicConfig(stream=sys.stderr, level=logging.INFO)
conf = Config.initConf(sys.argv[1])
ps = PointService(conf)
rs = RelatednessService(conf, ps)
for p, score in rs.related('dog'):
print p, score
def output(self):
config = Config.get()
return list(
luigi.LocalTarget(config.get('DEFAULT', n))
for n in self.configKeys)
def output(self):
config = Config.get()
return (TimestampedLocalTarget(config.get("GeneratedFiles",
"popularity_with_id")))
confPaths = os.getenv('CARTOGRAPH_CONFIGS')
if not confPaths:
raise Exception, 'CARTOGRAPH_CONFIGS environment variable not set!'
configs = {}
logging.info('configuring falcon')
# falcon.API instances are callable WSGI apps
app = falcon.API()
for path in confPaths.split(':'):
if not os.path.isfile(path):
raise Exception, 'Cartograph Config Path %s does not exist' % ` path `
conf = Config.initConf(path)
name = conf.get('DEFAULT', 'dataset')
if os.getenv('CLEAR_CACHE'):
logging.info('clearing cache directory %s' %
conf.get('DEFAULT', 'webCacheDir'))
shutil.rmtree(conf.get('DEFAULT', 'webCacheDir'), ignore_errors=True)
if os.getenv('BASE_URL'):
conf.set('Server', 'base_url', os.getenv('BASE_URL'))
logging.info('intitializing services for ' + name)
loggingService = LoggingService(conf)
pointService = PointService(conf)
countryService = CountryService(conf)
def samplePath(self):
config = Config.get()
n = config.getint('PreprocessingConstants', 'sample_size')
return Config.samplePath(self.path, n)
def output(self):
config = Config.get()
return TimestampedLocalTarget(config.get("ExternalFiles", "links"))
def output(self):
config = Config.get()
return (TimestampedLocalTarget(config.get("ExternalFiles", "popularity")))