def getTimelineValues():
global items
yearCol = "year"
if yearCol not in items[0]:
print("Could not find column %s in items, please add this column to metadata cols with 'type' = 'int'" % yearCol)
sys.exit()
years = [item[yearCol] for item in items]
minYear = min(years)
maxYear = max(years)
totalYears = maxYear - minYear + 1
groups = lu.groupList(items, yearCol) # group by year
yearDataLookup = lu.createLookup(groups, yearCol)
timelineItems = []
for i in range(totalYears):
year = i + minYear
yearKey = str(year)
count = 0
if yearKey in yearDataLookup:
count = yearDataLookup[yearKey]["count"]
timelineItems.append({
"year": year,
"value": count
})
return timelineItems
def getTimelineTunnelLayout(userOptions={}):
global items
options = {
"thickness": 0.2
}
options.update(userOptions)
cfg = {}
yearCol = "year"
dimensions = 3
if yearCol not in items[0]:
print("`dateColumn` needs to be set in config yml to support timelineTunnel layout")
return (False, False)
years = [item[yearCol] for item in items]
minYear = min(years)
maxYear = max(years) + 1
nUnit = 1.0 / (maxYear-minYear)
groups = lu.groupList(items, yearCol) # group by year
groups = sorted(groups, key=lambda group: group[yearCol])
nThickness = options["thickness"]
minDistance = 0.5-nThickness
maxDistance = 0.5
count = 0
values = np.zeros(len(items) * dimensions)
for i, group in enumerate(groups):
minZ = mu.norm(group[yearCol], (minYear, maxYear))
maxZ = minZ + nUnit
for j, item in enumerate(group["items"]):
index = item["index"]
x = y = 0.5
z = mu.randomUniform(minZ, maxZ, seed=count+5)
# angle = mu.randomUniform(0, 360, seed=count+7)
angle = mu.randomUniform(-240, 60, seed=count+7)
distance = mu.randomUniform(minDistance, maxDistance, seed=count+9)
# ensure story items are visible
if itemHasStory(item):
distance = minDistance * 0.8
x, y = mu.translatePoint(x, y, distance, angle)
values[index*dimensions] = round(x, PRECISION)
values[index*dimensions+1] = round(y, PRECISION)
values[index*dimensions+2] = round(z, PRECISION)
count += 1
values = values.tolist()
return (cfg, values)
def getCountryLabels(userOptions={}):
global items
cfg = {}
options = {"y": 0.5}
options.update(userOptions)
countryCol = "country"
latCol = "lat"
lonCol = "lon"
if countryCol not in items[0]:
print(
"`countryColumn` needs to be set in config yml to support country labels; they will not show otherwise."
)
return (None, None)
if latCol not in items[0] or lonCol not in items[0]:
# print("`latitudeColumn` and `latitudeColumn` need to be set in config yml to support country labels; they will not show otherwise.")
return (None, None)
latRange = (90.0, -90.0)
lonRange = (-180.0, 180.0)
groups = lu.groupList(items, countryCol) # group by country
counts = [group["count"] for group in groups]
minCount, maxCount = (min(counts), max(counts))
labels = []
for group in groups:
firstItem = group["items"][0]
label = firstItem[countryCol]
lon = firstItem[lonCol]
lat = firstItem[latCol]
y = options["y"]
x = 1.0 - mu.norm(lon, lonRange)
z = 1.0 - mu.norm(lat, latRange)
# HACK: offset z slightly to acommodate size of bar
w = mu.norm(group["count"], (minCount, maxCount))
w = mu.lerp((0.01, 1.0), w)
# assume height is half the depth; divide by 6 for radius calculation (see geometry.js)
radius = 0.5 / 6.0 * w + 0.005
z = z - radius
labels += [
round(x, PRECISION),
round(y, PRECISION),
round(z, PRECISION), label
]
return (cfg, labels)
def getTimelineTunnelLayout(userOptions={}):
global items
options = {"thickness": 0.2}
options.update(userOptions)
cfg = {}
yearCol = "year"
dimensions = 3
if yearCol not in items[0]:
print(
"Could not find column %s in items, please add this column to metadata cols with 'type' = 'int'"
% yearCol)
sys.exit()
years = [item[yearCol] for item in items]
minYear = min(years)
maxYear = max(years) + 1
nUnit = 1.0 / (maxYear - minYear)
groups = lu.groupList(items, yearCol) # group by year
groups = sorted(groups, key=lambda group: group[yearCol])
nThickness = options["thickness"]
count = 0
values = np.zeros(len(items) * dimensions)
for i, group in enumerate(groups):
minZ = mu.norm(group[yearCol], (minYear, maxYear))
maxZ = minZ + nUnit
for j, item in enumerate(group["items"]):
index = item["index"]
x = y = 0.5
z = mu.randomUniform(minZ, maxZ, seed=count + 5)
# angle = mu.randomUniform(0, 360, seed=count+7)
angle = mu.randomUniform(-240, 60, seed=count + 7)
distance = mu.randomUniform(0.5 - nThickness, 0.5, seed=count + 9)
x, y = mu.translatePoint(x, y, distance, angle)
values[index * dimensions] = round(x, PRECISION)
values[index * dimensions + 1] = round(y, PRECISION)
values[index * dimensions + 2] = round(z, PRECISION)
count += 1
values = values.tolist()
return (cfg, values)
def getGeographyBarsLayout(userOptions={}):
global items
cfg = {
"layout": "bars"
}
latCol = "lat"
lonCol = "lon"
if latCol not in items[0] or lonCol not in items[0]:
print("`latitudeColumn` and `latitudeColumn` need to be set in config yml to support geographyBars layout")
return (False, False)
# create unique key for lat lon
for i, item in enumerate(items):
items[i]["lonLatKey"] = (mu.roundInt(item[lonCol]*PRECISION), mu.roundInt(item[latCol]*PRECISION))
latRange = (90.0, -90.0)
lonRange = (-180.0, 180.0)
dimensions = 3
groups = lu.groupList(items, "lonLatKey") # group by lat lon
counts = [group["count"] for group in groups]
minCount, maxCount = (min(counts), max(counts))
# assign position values
values = np.zeros(len(items) * dimensions)
for group in groups:
y = mu.norm(group["count"], (minCount, maxCount))
y = mu.lerp((0.01, 1.0), y)
for item in group["items"]:
itemIndex = item["index"]
x = 1.0 - mu.norm(item[lonCol], lonRange)
z = 1.0 - mu.norm(item[latCol], latRange)
itemY = y
# a bit of a hack to ensure highighted items are visible
if itemHasStory(item):
itemY = y + 1.05
values[itemIndex*dimensions] = round(x, PRECISION)
values[itemIndex*dimensions+1] = round(itemY, PRECISION)
values[itemIndex*dimensions+2] = round(z, PRECISION)
values = values.tolist()
return (cfg, values)
def getGeographyBarsLayout(userOptions={}):
global items
cfg = {"layout": "bars"}
latCol = "lat"
lonCol = "lon"
if latCol not in items[0] or lonCol not in items[0]:
print(
"Could not find column (%s, %s) in items, please add these columns to metadata cols with 'type' = 'float'"
% (lonCol, latCol))
sys.exit()
# create unique key for lat lon
for i, item in enumerate(items):
items[i]["lonLatKey"] = (mu.roundInt(item[lonCol] * PRECISION),
mu.roundInt(item[latCol] * PRECISION))
latRange = (90.0, -90.0)
lonRange = (-180.0, 180.0)
dimensions = 3
groups = lu.groupList(items, "lonLatKey") # group by lat lon
counts = [group["count"] for group in groups]
minCount, maxCount = (min(counts), max(counts))
# assign position values
values = np.zeros(len(items) * dimensions)
for group in groups:
y = mu.norm(group["count"], (minCount, maxCount))
y = mu.lerp((0.01, 1.0), y)
for item in group["items"]:
itemIndex = item["index"]
x = 1.0 - mu.norm(item[lonCol], lonRange)
z = 1.0 - mu.norm(item[latCol], latRange)
values[itemIndex * dimensions] = round(x, PRECISION)
values[itemIndex * dimensions + 1] = round(y, PRECISION)
values[itemIndex * dimensions + 2] = round(z, PRECISION)
values = values.tolist()
return (cfg, values)
if item["Acquisition Year"] >= 9999:
items[i]["Acquisition Year"] = None
items = mu.addNormalizedValues(items, "Acquisition Year", "nalpha")
for i, item in enumerate(items):
if item["Acquisition Year"] is None:
items[i]["Acquisition Year"] = 9999
# Process data into sections and groups within each section
for i, col in enumerate(collections):
# Break items into sections
itemsBySection = []
groupBy = None
if "groupKey" in col:
groupBy = col["groupKey"]
itemsBySection = lu.groupList([
item for item in items
if groupBy in item and len(str(item[groupBy])) > 0
], groupBy)
itemsBySection = sorted(itemsBySection, key=lambda k: k[groupBy])
else:
itemsBySection = [{"items": items, "count": itemCount}]
# Break groups down further into groups
sections = []
for sectionItems in itemsBySection:
sectionItemsByGroup = lu.groupList(sectionItems["items"],
col["key"],
sort=True)
# group by year if year
if "Year" in col["key"]:
sectionItemsByGroup = sorted(sectionItemsByGroup,
key=lambda k: k[col["key"]])
def getSphereCategoryTimelineLayout(userOptions={}):
global items
global sets
cfg = {"layout": "spheres"}
categoryCol = "category"
yearCol = "year"
if yearCol not in items[0]:
print(
"Could not find column %s in items, please add this column to metadata cols with 'type' = 'int'"
% yearCol)
sys.exit()
if categoryCol not in sets:
print(
"Could not find column %s in sets, please add this column to metadata cols with 'asIndex' = true"
% categoryCol)
sys.exit()
categorySet = sets[categoryCol]
categoryCount = len(categorySet)
dimensions = 3
groups = lu.groupList(items, yearCol) # group by year
groups = sorted(groups, key=lambda group: group[yearCol])
years = [item[yearCol] for item in items]
minYear = min(years)
maxYear = max(years) + 1
nUnit = 1.0 / (maxYear - minYear)
# determine category sphere count range
minCount = 9999999999
maxCount = 0
for i, group in enumerate(groups):
subgroups = lu.groupList(group["items"],
categoryCol) # group by category
for subgroup in subgroups:
minCount = min(minCount, subgroup["count"])
maxCount = max(maxCount, subgroup["count"])
groups[i]["categoryGroups"] = subgroups
# assign position values
values = np.zeros(len(items) * dimensions)
for i, group in enumerate(groups):
z = mu.norm(
group[yearCol],
(minYear,
maxYear)) + nUnit * 0.5 # place spheres in the center of the year
subgroups = group["categoryGroups"]
subgroupLookup = lu.createLookup(subgroups, categoryCol)
for j, category in enumerate(categorySet):
x = 1.0 - 1.0 * j / (categoryCount - 1)
categoryKey = str(j)
if categoryKey in subgroupLookup:
subgroup = subgroupLookup[categoryKey]
y = mu.norm(subgroup["count"], (minCount, maxCount))
y = mu.lerp((0.01, 1.0), y)
for catItem in subgroup["items"]:
itemIndex = catItem["index"]
values[itemIndex * dimensions] = round(x, PRECISION)
values[itemIndex * dimensions + 1] = round(y, PRECISION)
values[itemIndex * dimensions + 2] = round(z, PRECISION)
values = values.tolist()
return (cfg, values)
def getSphereCategoryTimelineLayout(userOptions={}):
global items
global categories
cfg = {
"layout": "spheres"
}
categoryCol = "category"
yearCol = "year"
if yearCol not in items[0]:
print("`dateColumn` needs to be set in config yml to support timelineTracks layout")
return (False, False)
if categoryCol not in items[0]:
print("`groupByColumn` needs to be set in config yml to support timelineTracks layout")
return (False, False)
categoryCount = len(categories)
dimensions = 3
groups = lu.groupList(items, yearCol) # group by year
groups = sorted(groups, key=lambda group: group[yearCol])
years = [item[yearCol] for item in items]
minYear = min(years)
maxYear = max(years) + 1
nUnit = 1.0 / (maxYear-minYear)
# determine category sphere count range
minCount = 9999999999
maxCount = 0
for i, group in enumerate(groups):
subgroups = lu.groupList(group["items"], categoryCol) # group by category
for subgroup in subgroups:
minCount = min(minCount, subgroup["count"])
maxCount = max(maxCount, subgroup["count"])
groups[i]["categoryGroups"] = subgroups
# assign position values
values = np.zeros(len(items) * dimensions)
for i, group in enumerate(groups):
z = mu.norm(group[yearCol], (minYear, maxYear)) + nUnit*0.5 # place spheres in the center of the year
subgroups = group["categoryGroups"]
subgroupLookup = lu.createLookup(subgroups, categoryCol)
for j, category in enumerate(categories):
x = 1.0 - 1.0 * j / (categoryCount-1)
categoryKey = category["text"]
if categoryKey in subgroupLookup:
subgroup = subgroupLookup[categoryKey]
y = mu.norm(subgroup["count"], (minCount, maxCount))
y = mu.lerp((0.01, 1.0), y)
for catItem in subgroup["items"]:
itemIndex = catItem["index"]
cy = y
# a bit of a hack to ensure highighted items are visible
if itemHasStory(catItem):
cy = y + 1.25
values[itemIndex*dimensions] = round(x, PRECISION)
values[itemIndex*dimensions+1] = round(cy, PRECISION)
values[itemIndex*dimensions+2] = round(z, PRECISION)
values = values.tolist()
return (cfg, values)
def getItems(config):
inputFile = config["metadataFile"]
idCol = config["identifierColumn"] if "identifierColumn" in config else None
fieldnames, items = io.readCsv(inputFile, parseNumbers=False)
if "metadataFilterQuery" in config:
items = lu.filterByQueryString(items, config["metadataFilterQuery"])
print("%s items after filtering" % len(items))
# map year, lat/lon, and category
columnMap = [("dateColumn", "year"), ("latitudeColumn", "lat"),
("longitudeColumn", "lon"), ("countryColumn", "country"),
("groupByColumn", "category")]
minimumYear = config["minimumYear"] if "minimumYear" in config else None
maximumYear = config["maximumYear"] if "maximumYear" in config else None
validItems = []
for i, item in enumerate(items):
validatedItem = item.copy()
isValid = True
for configKey, toColumn in columnMap:
if configKey not in config:
continue
value = item[config[configKey]]
if toColumn == "year":
value = su.validateYear(value, minimumYear, maximumYear)
elif toColumn == "lat":
value = su.validateLat(value)
elif toColumn == "lon":
value = su.validateLon(value)
if value is None:
isValid = False
break
validatedItem[toColumn] = value
if isValid:
validItems.append(validatedItem)
diff = len(items) - len(validItems)
print(f'Found {diff} invalid items.')
# Sort so that index corresponds to ID
if idCol is not None:
for i, item in enumerate(validItems):
validItems[i]["_id"] = str(item[idCol])
validItems = sorted(validItems, key=lambda item: item["_id"])
validItems = lu.addIndices(validItems)
if idCol is None:
for i, item in enumerate(validItems):
validItems[i]["_id"] = str(i)
# Retrieve categories
categories = []
itemsByCategory = lu.groupList(validItems,
"category",
sort=True,
desc=True)
if "groupLimit" in config and len(itemsByCategory) > config["groupLimit"]:
limit = config["groupLimit"] - 1
otherItems = itemsByCategory[limit:]
otherLabel = config["otherLabel"] if "otherLabel" in config else "Other"
otherCount = 0
for group in otherItems:
for item in group["items"]:
validItems[item["index"]]["category"] = otherLabel
otherCount += 1
itemsByCategory = itemsByCategory[:limit] + [{
"category": otherLabel,
"count": otherCount
}]
categoryColors = config["groupColors"]
colorCount = len(categoryColors)
for i, category in enumerate(itemsByCategory):
color = categoryColors[i % colorCount]
categories.append({
"text": category["category"],
"color": color,
"count": category["count"]
})
return (validItems, categories)
a = parser.parse_args()
COLORS = ["#612323", "#204f1c", "#4d1e59", "#112e6b", "#4b5713", "#571330"]
colorCount = len(COLORS)
# Make sure output dirs exist
io.makeDirectories([a.OUTPUT_FILE, a.CACHE_FILE])
font = ImageFont.truetype(font="fonts/Open_Sans/OpenSans-Regular.ttf", size=a.FONT_SIZE)
print("Reading data...")
fieldNames, items = io.readCsv(a.INPUT_FILE)
yLabels = lu.unique([item[a.Y_AXIS] for item in items])
if a.Y_AXIS == "Region":
items = [item for item in items if item["Region"] != "Europe"]
itemsByRegion = lu.groupList(items, "Region")
for i, region in enumerate(itemsByRegion):
itemsByRegion[i]["lat"] = np.mean([item["Latitude"] for item in region["items"] if -90 <= item["Latitude"] <= 90])
itemsByRegion = sorted(itemsByRegion, key=lambda region: -region["lat"])
yLabels = [region["Region"] for region in itemsByRegion]
else:
yLabels = sorted(yLabels)
yLabelCount = len(yLabels)
xLabels = []
yearStart = yearEnd = None
if "Year" in a.X_AXIS:
items = [item for item in items if item[a.X_AXIS] < 9999]
items = sorted(items, key=lambda item: item[a.X_AXIS])
yearStart = items[0][a.X_AXIS]
yearEnd = items[-1][a.X_AXIS]