def process(self):
"""
This takes a 4CAT results file as input, and outputs a plain text file
containing all post bodies as one continuous string, sanitized.
"""
# OrderedDict because dates and headers should have order
intervals = {}
timeframe = self.parameters.get("timeframe",
self.options["timeframe"]["default"])
first_interval = "9999"
last_interval = "0000"
self.dataset.update_status("Processing posts")
with self.dataset.get_results_path().open("w") as results:
counter = 0
for post in self.iterate_csv_items(self.source_file):
date = get_interval_descriptor(post, timeframe)
# Add a count for the respective timeframe
if date not in intervals:
intervals[date] = 1
else:
intervals[date] += 1
first_interval = min(first_interval, date)
last_interval = max(last_interval, date)
counter += 1
if counter % 2500 == 0:
self.dataset.update_status("Counted through " +
str(counter) + " posts.")
# pad interval if needed, this is useful if the result is to be
# visualised as a histogram, for example
if self.parameters.get("pad", self.options["pad"].get(
"default", True)) and timeframe != "all":
missing, intervals = pad_interval(intervals, first_interval,
last_interval)
# Write to csv
csv_writer = csv.DictWriter(results,
fieldnames=("date", "item",
"frequency"))
csv_writer.writeheader()
for interval in intervals:
csv_writer.writerow({
"date": interval,
"item": "activity",
"frequency": intervals[interval]
})
self.dataset.finish(len(intervals))
def process(self):
"""
This takes a 4CAT results file as input, and outputs a plain text file
containing all post bodies as one continuous string, sanitized.
"""
# OrderedDict because dates and headers should have order
intervals = {}
timeframe = self.parameters.get("timeframe",
self.options["timeframe"]["default"])
first_interval = "9999"
last_interval = "0000"
self.dataset.update_status("Processing posts")
with self.dataset.get_results_path().open("w") as results:
counter = 0
for post in self.iterate_csv_items(self.source_file):
# Add a count for the respective timeframe
if timeframe == "all":
date = "overall"
else:
try:
timestamp = int(
datetime.datetime.strptime(
post["timestamp"],
"%Y-%m-%d %H:%M:%S").timestamp())
except ValueError:
self.dataset.update_status(
"Invalid date found in dataset; cannot count posts per interval."
)
self.dataset.finish(0)
return
date = datetime.datetime.utcfromtimestamp(timestamp)
if timeframe == "year":
date = str(date.year)
elif timeframe == "month":
date = str(date.year) + "-" + str(date.month).zfill(2)
elif timeframe == "week":
date = str(date.isocalendar()[0]) + "-" + str(
date.isocalendar()[1]).zfill(2)
else:
date = str(date.year) + "-" + str(
date.month).zfill(2) + "-" + str(date.day).zfill(2)
if date not in intervals:
intervals[date] = 1
else:
intervals[date] += 1
first_interval = min(first_interval, date)
last_interval = max(last_interval, date)
counter += 1
if counter % 2500 == 0:
self.dataset.update_status("Counted through " +
str(counter) + " posts.")
# pad interval if needed, this is useful if the result is to be
# visualised as a histogram, for example
if self.parameters.get("pad", self.options["pad"].get(
"default", True)) and timeframe != "all":
missing, intervals = pad_interval(intervals, first_interval,
last_interval)
# Write to csv
csv_writer = csv.DictWriter(results,
fieldnames=("date", "item",
"frequency"))
csv_writer.writeheader()
for interval in intervals:
csv_writer.writerow({
"date": interval,
"item": "activity",
"frequency": intervals[interval]
})
self.dataset.finish(len(intervals))
def process(self):
"""
Render an SVG histogram/bar chart using a previous frequency analysis
as input.
"""
self.dataset.update_status("Reading source file")
header = self.parameters.get("header", self.options["header"]["default"])
max_posts = 0
# collect post numbers per month
intervals = {}
for post in self.iterate_csv_items(self.source_file):
intervals[post["date"]] = int(post["frequency"])
max_posts = max(max_posts, int(post["frequency"]))
if len(intervals) <= 1:
self.dataset.update_status("Not enough data available for a histogram; need more than one time series.")
self.dataset.finish(0)
return
self.dataset.update_status("Cleaning up data")
(missing, intervals) = pad_interval(intervals)
# create histogram
self.dataset.update_status("Drawing histogram")
# you may change the following four variables to adjust the graph dimensions
width = 1024
height = 786
y_margin = 75
x_margin = 50
x_margin_left = x_margin * 2
tick_width = 5
fontsize_normal = int(height / 40)
fontsize_small = int(height / 75)
# better don't touch the following
line_width = round(width / 512)
y_margin_top = 150 if header else 50
y_height = height - (y_margin + y_margin_top)
x_width = width - (x_margin + x_margin_left)
canvas = Drawing(filename=str(self.dataset.get_results_path()), size=(width, height),
style="font-family:monospace;font-size:%ipx" % fontsize_normal)
# normalize the Y axis to a multiple of a power of 10
magnitude = pow(10, len(str(max_posts)) - 1) # ew
max_neat = math.ceil(max_posts / magnitude) * magnitude
self.dataset.update_status("Max (normalized): %i (%i) (magnitude: %i)" % (max_posts, max_neat, magnitude))
# draw border
canvas.add(Rect(
insert=(0, 0),
size=(width, height),
stroke="#000",
stroke_width=line_width,
fill="#FFF"
))
# draw header on a black background if needed
if header:
if len(header) > 40:
header = header[:37] + "..."
header_rect_height = (y_margin_top / 1.5)
header_fontsize = (width / len(header))
header_container = SVG(insert=(0, 0), size=(width, header_rect_height))
header_container.add(Rect(
insert=(0, 0),
size=(width, header_rect_height),
fill="#000"
))
header_container.add(Text(
insert=("50%", "50%"),
text=header,
dominant_baseline="middle",
text_anchor="middle",
fill="#FFF",
style="font-size:%i" % header_fontsize
))
canvas.add(header_container)
# horizontal grid lines
for i in range(0, 10):
offset = (y_height / 10) * i
canvas.add(Line(
start=(x_margin_left, y_margin_top + offset),
end=(width - x_margin, y_margin_top + offset),
stroke="#EEE",
stroke_width=line_width
))
# draw bars
item_width = (width - (x_margin + x_margin_left)) / len(intervals)
item_height = (height - y_margin - y_margin_top)
bar_width = item_width * 0.9
x = x_margin_left + (item_width / 2) - (bar_width / 2)
if bar_width >= 8:
arc_adjust = max(8, int(item_width / 5)) / 2
else:
arc_adjust = 0
for interval in intervals:
posts = int(intervals[interval])
bar_height = ((posts / max_neat) * item_height)
self.dataset.update_status("%s: %i posts" % (interval, posts))
bar_top = height - y_margin - bar_height
bar_bottom = height - y_margin
if bar_height == 0:
x += item_width
continue
bar = Path(fill="#000")
bar.push("M %f %f" % (x, bar_bottom))
bar.push("L %f %f" % (x, bar_top + (arc_adjust if bar_height > arc_adjust else 0)))
if bar_height > arc_adjust > 0:
control = (x, bar_top)
bar.push("C %f %f %f %f %f %f" % (*control, *control, x + arc_adjust, bar_top))
bar.push("L %f %f" % (x + bar_width - arc_adjust, height - y_margin - bar_height))
if bar_height > arc_adjust > 0:
control = (x + bar_width, bar_top)
bar.push("C %f %f %f %f %f %f" % (*control, *control, x + bar_width, bar_top + arc_adjust))
bar.push("L %f %f" % (x + bar_width, height - y_margin))
bar.push("Z")
canvas.add(bar)
x += item_width
# draw X and Y axis
canvas.add(Line(
start=(x_margin_left, height - y_margin),
end=(width - x_margin, height - y_margin),
stroke="#000",
stroke_width=2
))
canvas.add(Line(
start=(x_margin_left, y_margin_top),
end=(x_margin_left, height - y_margin),
stroke="#000",
stroke_width=2
))
# draw ticks on Y axis
for i in range(0, 10):
offset = (y_height / 10) * i
canvas.add(Line(
start=(x_margin_left - tick_width, y_margin_top + offset),
end=(x_margin_left, y_margin_top + offset),
stroke="#000",
stroke_width=line_width
))
# draw ticks on X axis
for i in range(0, len(intervals)):
offset = (x_width / len(intervals)) * (i + 0.5)
canvas.add(Line(
start=(x_margin_left + offset, height - y_margin),
end=(x_margin_left + offset, height - y_margin + tick_width),
stroke="#000",
stroke_width=line_width
))
# prettify
# y labels
origin = (x_margin_left / 2)
step = y_height / 10
for i in range(0, 11):
label = str(int((max_neat / 10) * i))
labelsize = (len(label) * fontsize_normal * 1.25, fontsize_normal)
label_x = origin - (tick_width * 2)
label_y = height - y_margin - (i * step) - (labelsize[1] / 2)
label_container = SVG(
insert=(label_x, label_y),
size=(x_margin_left / 2, x_margin_left / 5)
)
label_container.add(Text(
insert=("100%", "50%"),
text=label,
dominant_baseline="middle",
text_anchor="end"
))
canvas.add(label_container)
# x labels
label_width = max(fontsize_small * 6, item_width)
label_x = x_margin_left
label_y = height - y_margin + (tick_width * 2)
next = 0
for interval in intervals:
if len(interval) == 7:
label = month_abbr[int(interval[5:7])] + "\n" + interval[0:4]
elif len(interval) == 10:
label = str(int(interval[8:10])) + month_abbr[int(interval[5:7])] + "\n" + interval[0:4]
else:
label = interval.replace("-", "\n")
if label_x > next:
shift = 0
for line in label.split("\n"):
label_container = SVG(
insert=(label_x + (item_width / 2) - (label_width / 2), label_y + (tick_width * 2)),
size=(label_width, y_margin), overflow="visible")
label_container.add(Text(
insert=("50%", "0%"),
text=line,
dominant_baseline="middle",
text_anchor="middle",
baseline_shift=-shift
))
shift += fontsize_small * 2
canvas.add(label_container)
next = label_x + (label_width * 0.9)
label_x += item_width
# 4cat logo
label = "made with 4cat - 4cat.oilab.nl"
footersize = (fontsize_small * len(label) * 0.7, fontsize_small * 2)
footer = SVG(insert=(width - footersize[0], height - footersize[1]), size=footersize)
footer.add(Rect(insert=(0, 0), size=("100%", "100%"), fill="#000"))
footer.add(Text(
insert=("50%", "50%"),
text=label,
dominant_baseline="middle",
text_anchor="middle",
fill="#FFF",
style="font-size:%i" % fontsize_small
))
canvas.add(footer)
canvas.save(pretty=True)
self.dataset.update_status("Finished")
self.dataset.finish(len(intervals))
def process(self):
graphs = {}
intervals = []
smooth = self.parameters.get("smooth",
self.options["smooth"]["default"])
normalise_values = self.parameters.get(
"normalise", self.options["normalise"]["default"])
completeness = convert_to_int(
self.parameters.get("complete",
self.options["complete"]["default"]), 0)
graph_label = self.parameters.get("label",
self.options["label"]["default"])
top = convert_to_int(
self.parameters.get("top", self.options["top"]["default"]), 10)
# first gather graph data: each distinct item gets its own graph and
# for each graph we have a sequence of intervals, each interval with
# its own value
first_date = "9999-99-99"
last_date = "0000-00-00"
with self.source_file.open() as input:
reader = csv.DictReader(input)
item_key = "text" if "text" in reader.fieldnames else "item"
date_key = "time" if "time" in reader.fieldnames else "date"
value_key = "value" if "value" in reader.fieldnames else "frequency"
for row in self.iterate_csv_items(self.source_file):
if row[item_key] not in graphs:
graphs[row[item_key]] = {}
# make sure the months and days are zero-padded
interval = row.get(date_key, "")
interval = "-".join([
str(bit).zfill(2 if len(bit) != 4 else 4)
for bit in interval.split("-")
])
first_date = min(first_date, interval)
last_date = max(last_date, interval)
if interval not in intervals:
intervals.append(interval)
if interval not in graphs[row[item_key]]:
graphs[row[item_key]][interval] = 0
graphs[row[item_key]][interval] += float(row.get(value_key, 0))
# first make sure we actually have something to render
intervals = sorted(intervals)
if len(intervals) <= 1:
self.dataset.update_status(
"Not enough data for a side-by-side over-time visualisation.")
self.dataset.finish(0)
return
# only retain most-occurring series - sort by sum of all frequencies
if len(graphs) > top:
selected_graphs = {
graph: graphs[graph]
for graph in sorted(
graphs,
key=lambda x: sum(
[graphs[x][interval] for interval in graphs[x]]),
reverse=True)[0:top]
}
graphs = selected_graphs
# there may be items that do not have values for all intervals
# this will distort the graph, so the next step is to make sure all
# graphs consist of the same continuous interval list
missing = {graph: 0 for graph in graphs}
for graph in graphs:
missing[graph], graphs[graph] = pad_interval(
graphs[graph],
first_interval=first_date,
last_interval=last_date)
# now that's done, make sure the graph datapoints are in order
intervals = sorted(list(graphs[list(graphs)[0]].keys()))
# delete graphs that do not have the required amount of intervals
# this is useful to get rid of outliers and items that only occur
# very few times over the full interval
if completeness > 0:
intervals_required = len(intervals) * (completeness / 100)
disqualified = []
for graph in graphs:
if len(intervals) - missing[graph] < intervals_required:
disqualified.append(graph)
graphs = {
graph: graphs[graph]
for graph in graphs if graph not in disqualified
}
# determine max value per item, so we can normalize them later
limits = {}
max_limit = 0
for graph in graphs:
for interval in graphs[graph]:
limits[graph] = max(limits.get(graph, 0),
abs(graphs[graph][interval]))
max_limit = max(max_limit, abs(graphs[graph][interval]))
# order graphs by highest (or lowest) value)
limits = {
limit: limits[limit]
for limit in sorted(limits, key=lambda l: limits[l])
}
graphs = {graph: graphs[graph] for graph in limits}
if not graphs:
# maybe nothing is actually there to be graphed
self.dataset.update_status(
"No items match the selection criteria - nothing to visualise."
)
self.dataset.finish(0)
return None
# how many vertical grid lines (and labels) are to be included at most
# 12 is a sensible default because it allows one label per month for a full
# year's data
max_gridlines = 12
# If True, label is put at the lower left bottom of the graph rather than
# outside it. Automatically set to True if one of the labels is long, as
# else the label would fall off the screen
label_in_graph = max([len(item) for item in graphs]) > 30
# determine how wide each interval should be
# the graph has a minimum width - but the graph's width will be
# extended if at this minimum width each item does not have the
# minimum per-item width
min_full_width = 600
min_item_width = 50
item_width = max(min_item_width, min_full_width / len(intervals))
# determine how much space each graph should get
# same trade-off as for the interval width
min_full_height = 300
min_item_height = 100
item_height = max(min_item_height, min_full_height / len(graphs))
# margin - this should be enough for the text labels to fit in
margin_base = 50
margin_right = margin_base * 4
margin_top = margin_base * 3
# this determines the "flatness" of the isometric projection and an be
# tweaked for different looks - basically corresponds to how far the
# camera is above the horizon
plane_angle = 120
# don't change these
plane_obverse = radians((180 - plane_angle) / 2)
plane_angle = radians(plane_angle)
# okay, now determine the full graphic size with these dimensions projected
# semi-isometrically. We can also use these values later for drawing for
# drawing grid lines, et cetera. The axis widths and heights here are the
# dimensions of the bounding box wrapping the isometrically projected axes.
x_axis_length = (item_width * (len(intervals) - 1))
y_axis_length = (item_height * len(graphs))
x_axis_width = (sin(plane_angle / 2) * x_axis_length)
y_axis_width = (sin(plane_angle / 2) * y_axis_length)
canvas_width = x_axis_width + y_axis_width
# leave room for graph header
if graph_label:
margin_top += (2 * (canvas_width / 50))
x_axis_height = (cos(plane_angle / 2) * x_axis_length)
y_axis_height = (cos(plane_angle / 2) * y_axis_length)
canvas_height = x_axis_height + y_axis_height
# now we have the dimensions, the canvas can be instantiated
canvas = get_4cat_canvas(
self.dataset.get_results_path(),
width=(canvas_width + margin_base + margin_right),
height=(canvas_height + margin_base + margin_top),
header=graph_label)
# draw gridlines - vertical
gridline_x = y_axis_width + margin_base
gridline_y = margin_top + canvas_height
step_x_horizontal = sin(plane_angle / 2) * item_width
step_y_horizontal = cos(plane_angle / 2) * item_width
step_x_vertical = sin(plane_angle / 2) * item_height
step_y_vertical = cos(plane_angle / 2) * item_height
# labels for x axis
skip = max(1, int(len(intervals) / max_gridlines))
for i in range(0, len(intervals)):
if i % skip == 0:
canvas.add(
Line(start=(gridline_x, gridline_y),
end=(gridline_x - y_axis_width,
gridline_y - y_axis_height),
stroke="grey",
stroke_width=0.25))
# to properly position the rotated and skewed text a container
# element is needed
label1 = str(intervals[i])[0:4]
center = (gridline_x, gridline_y)
container = SVG(x=center[0] - 25,
y=center[1],
width="50",
height="1.5em",
overflow="visible",
style="font-size:0.8em;")
container.add(
Text(insert=("25%", "100%"),
text=label1,
transform="rotate(%f) skewX(%f)" %
(-degrees(plane_obverse), degrees(plane_obverse)),
text_anchor="middle",
baseline_shift="-0.5em",
style="font-weight:bold;"))
if re.match(r"^[0-9]{4}-[0-9]{2}", intervals[i]):
label2 = month_abbr[int(str(intervals[i])[5:7])]
if re.match(r"^[0-9]{4}-[0-9]{2}-[0-9]{2}", intervals[i]):
label2 += " %i" % int(intervals[i][8:10])
container.add(
Text(insert=("25%", "150%"),
text=label2,
transform="rotate(%f) skewX(%f)" %
(-degrees(plane_obverse), degrees(plane_obverse)),
text_anchor="middle",
baseline_shift="-0.5em"))
canvas.add(container)
gridline_x += step_x_horizontal
gridline_y -= step_y_horizontal
# draw graphs as filled beziers
top = step_y_vertical * 1.5
graph_start_x = y_axis_width + margin_base
graph_start_y = margin_top + canvas_height
# draw graphs in reverse order, so the bottom one is most in the
# foreground (in case of overlap)
for graph in reversed(list(graphs)):
self.dataset.update_status("Rendering graph for '%s'" % graph)
# path starting at lower left corner of graph
area_graph = Path(fill=self.colours[self.colour_index])
area_graph.push("M %f %f" % (graph_start_x, graph_start_y))
previous_value = None
graph_x = graph_start_x
graph_y = graph_start_y
for interval in graphs[graph]:
# normalise value
value = graphs[graph][interval]
try:
limit = limits[graph] if normalise_values else max_limit
value = top * copysign(abs(value) / limit, value)
except ZeroDivisionError:
value = 0
if previous_value is None:
# vertical line upwards to starting value of graph
area_graph.push("L %f %f" %
(graph_start_x, graph_start_y - value))
elif not smooth:
area_graph.push("L %f %f" % (graph_x, graph_y - value))
else:
# quadratic bezier from previous value to current value
control_left = (graph_x - (step_x_horizontal / 2),
graph_y + step_y_horizontal -
previous_value - (step_y_horizontal / 2))
control_right = (graph_x - (step_x_horizontal / 2),
graph_y - value + (step_y_horizontal / 2))
area_graph.push("C %f %f %f %f %f %f" %
(*control_left, *control_right, graph_x,
graph_y - value))
previous_value = value
graph_x += step_x_horizontal
graph_y -= step_y_horizontal
# line to the bottom of the graph at the current Y position
area_graph.push(
"L %f %f" %
(graph_x - step_x_horizontal, graph_y + step_y_horizontal))
area_graph.push("Z") # then close the Path
canvas.add(area_graph)
# add text labels - skewing is a bit complicated and we need a
# "center" to translate the origins properly.
if label_in_graph:
insert = (graph_start_x + 5, graph_start_y - 10)
else:
insert = (graph_x - (step_x_horizontal) + 5,
graph_y + step_y_horizontal - 10)
# we need to take the skewing into account for the translation
offset_y = tan(plane_obverse) * insert[0]
canvas.add(
Text(insert=(0, 0),
text=graph,
transform="skewY(%f) translate(%f %f)" %
(-degrees(plane_obverse), insert[0],
insert[1] + offset_y)))
# cycle colours, back to the beginning if all have been used
self.colour_index += 1
if self.colour_index >= len(self.colours):
self.colour_index = 0
graph_start_x -= step_x_vertical
graph_start_y -= step_y_vertical
# draw gridlines - horizontal
gridline_x = margin_base
gridline_y = margin_top + canvas_height - y_axis_height
for graph in graphs:
gridline_x += step_x_vertical
gridline_y += step_y_vertical
canvas.add(
Line(start=(gridline_x, gridline_y),
end=(gridline_x + x_axis_width,
gridline_y - x_axis_height),
stroke="black",
stroke_width=1))
# x axis
canvas.add(
Line(start=(margin_base + y_axis_width,
margin_top + canvas_height),
end=(margin_base + canvas_width,
margin_top + canvas_height - x_axis_height),
stroke="black",
stroke_width=2))
# and finally save the SVG
canvas.save(pretty=True)
self.dataset.finish(len(graphs))