def __init__(self, MMVVectorial, context, skia_object):
self.vectorial = MMVVectorial
self.context = context
self.skia = skia_object
self.config = self.vectorial.config
self.polar = PolarCoordinates()
self.functions = Functions()
self.center_x = self.vectorial.context.width / 2
self.center_y = self.vectorial.context.height / 2
def __init__(self, MMVVectorial, context, skia_object, midi):
self.vectorial = MMVVectorial
self.context = context
self.skia = skia_object
self.midi = midi
self.config = self.vectorial.config
self.functions = Functions()
self.datautils = DataUtils()
self.piano_keys = {}
self.keys_centers = {}
self.background_color = 22 / 255
def __init__(self, mmv, MMVSkiaPianoRollVectorial):
self.mmvskia_main = mmv
self.vectorial = MMVSkiaPianoRollVectorial
self.config = self.vectorial.config
self.functions = Functions()
self.datautils = DataUtils()
self.piano_keys = {}
self.keys_centers = {}
"""
When converting the colors we divide by 255 so we normalize in a value in between 0 and 1
Because skia works like that.
"""
sep = os.path.sep
# Load the yaml config file
color_preset = self.config["color_preset"]
color_config_yaml = self.mmvskia_main.utils.load_yaml(
f"{self.mmvskia_main.mmvskia_interface.top_level_interace.data_dir}{sep}mmvskia{sep}piano_roll{sep}color_{color_preset}.yaml"
)
# Get the global colors into a dictionary
self.global_colors = {}
for key in color_config_yaml["global"]:
self.global_colors[key] = [
channel / 255 for channel in ImageColor.getcolor(
"#" + str(color_config_yaml["global"][key]), "RGB")
]
# # Get the note colors based on their channel
self.color_channels = {}
# For every channel config
for key in color_config_yaml["channels"]:
# Create empty dir
self.color_channels[key] = {}
# Get colors of sharp and plain, border, etc..
for color_type in color_config_yaml["channels"][key].keys():
# Hexadecimal representation of color
color_hex = color_config_yaml["channels"][key][color_type]
# Assign RGB value
self.color_channels[key][color_type] = [
channel / 255
for channel in ImageColor.getcolor(f"#{color_hex}", "RGB")
]
def __init__(self) -> None:
debug_prefix = "[AudioProcessing.__init__]"
# Create some util classes
self.fourier = Fourier()
self.datautils = DataUtils()
self.functions = Functions()
self.config = None
# List of full frequencies of notes
# - 50 to 68 yields freqs of 24.4 Hz up to
self.piano_keys_frequencies = [
round(self.get_frequency_of_key(x), 2) for x in range(-50, 68)
]
def __init__(self, mmv, MMVSkiaPianoRollVectorial):
self.mmvskia_main = mmv
self.vectorial = MMVSkiaPianoRollVectorial
self.config = self.vectorial.config
self.functions = Functions()
self.datautils = DataUtils()
self.piano_keys = {}
self.keys_centers = {}
self.colors = {}
# Generate skia.Color4F out of the colors dictionary
self.colors = self.parse_colors_dict(colors_dict = self.config["colors"])
self.font_fits_on_width = {}
class Interpolation:
def __init__(self):
self.functions = Functions()
# Linear, between point A and B based on a current "step" and total steps
def linear(self,
start_value: Number,
target_value: Number,
current_step: Number,
total_steps: Number,
) -> Number:
if current_step > total_steps:
return target_value
part = (target_value - start_value) / total_steps
walked = part * current_step
return start_value + walked
# "Biased" remaining linear
# aggressive, 0.05 is smooth, 0.1 is medium, 1 is instant
# random is a decimal that adds to aggressive randomly
def remaining_approach(self,
start_value: Number,
target_value: Number,
current_step: Number,
current_value: Number,
aggressive: Number,
aggressive_randomness: Number=0,
) -> Number:
# We're at the first step, so start on current value
if current_step == 0:
return start_value
return current_value + ( (target_value - current_value) * (aggressive + random.uniform(0, aggressive_randomness)) )
# Sigmoid activation between two points, smoothed out "linear" curver
def sigmoid(self,
start_value: Number,
target_value: Number,
smooth: Number,
) -> Number:
distance = (target_value - start_value)
where = self.functions.proportion(total, 1, current)
walk = distance * self.functions.sigmoid(where, smooth)
return a + walk
def __init__(self, context, config: dict, skia_object) -> None:
debug_prefix = "[MMVMusicBars.__init__]"
self.context = context
self.config = config
self.skia = skia_object
self.fit_transform_index = FitIndex()
self.functions = Functions()
self.utils = Utils()
self.path = {}
self.x = 0
self.y = 0
self.size = 1
self.is_deletable = False
self.offset = [0, 0]
self.polar = PolarCoordinates()
self.current_fft = {}
self.image = Frame()
# We use separate file and classes for each type of visualizer
# Circle, radial visualizer
if self.config["type"] == "circle":
self.builder = MMVMusicBarsCircle(self, self.context, self.skia)
def __init__(self, depth = LOG_NO_DEPTH) -> None:
debug_prefix = "[AudioProcessing.__init__]"
ndepth = depth + LOG_NEXT_DEPTH
self.fourier = Fourier()
self.datautils = DataUtils()
self.functions = Functions()
self.config = None
# MMV specific, where we return repeated frequencies from the
# function process
self.where_decay_less_than_one = 440
self.value_at_zero = 5
# List of full frequencies of notes
# - 50 to 68 yields freqs of 24.4 Hz up to
self.piano_keys_frequencies = [round(self.get_frequency_of_key(x), 2) for x in range(-50, 68)]
logging.info(f"{depth}{debug_prefix} Whole notes frequencies we'll care: [{self.piano_keys_frequencies}]")
def __init__(self, MMVSkiaVectorial, mmv):
self.mmv = mmv
self.vectorial = MMVSkiaVectorial
self.config = self.vectorial.config
self.polar = PolarCoordinates()
self.functions = Functions()
# Leave this much of blank space at the left and right edge
bleed_x = self.mmv.context.width * (1 / 19)
# Where Y starts and ends, ie. what horizontal line to draw the progression bar on
if self.config["position"] == "bottom":
start_end_y = (19 / 20) * self.mmv.context.height
elif self.config["position"] == "top":
start_end_y = (1 / 20) * self.mmv.context.height
# End X we have to count the bleed
self.start_x = bleed_x
self.end_x = self.mmv.context.width - bleed_x
self.start_y = start_end_y
self.end_y = start_end_y
def setup(self) -> None:
debug_prefix = "[MMVSkiaMain.setup]"
self.utils = Utils()
logging.info(f"{debug_prefix} Creating MMVContext() class")
self.context = MMVContext(mmv_skia_main=self)
logging.info(f"{debug_prefix} Creating SkiaNoWindowBackend() class")
self.skia = SkiaNoWindowBackend()
logging.info(f"{debug_prefix} Creating Functions() class")
self.functions = Functions()
logging.info(f"{debug_prefix} Creating Interpolation() class")
self.interpolation = Interpolation()
logging.info(f"{debug_prefix} Creating PolarCoordinates() class")
self.polar_coordinates = PolarCoordinates()
logging.info(f"{debug_prefix} Creating Canvas() class")
self.canvas = MMVSkiaImage(mmvskia_main=self)
logging.info(f"{debug_prefix} Creating Fourier() class")
self.fourier = Fourier()
# The user must explicitly set and override this, mostly for compatibility
# and code cleanup reasons.
self.pipe_video_to = None
logging.info(f"{debug_prefix} Creating AudioFile() class")
self.audio = AudioFile()
logging.info(f"{debug_prefix} Creating AudioProcessing() class")
self.audio_processing = AudioProcessing()
logging.info(f"{debug_prefix} Creating MMVSkiaAnimation() class")
self.mmv_skia_animation = MMVSkiaAnimation(mmv_skia_main=self)
logging.info(f"{debug_prefix} Creating MMVSkiaCore() class")
self.core = MMVSkiaCore(mmvskia_main=self)
def __init__(self, mmv, **kwargs) -> None:
self.mmv = mmv
debug_prefix = "[MMVSkiaMusicBars.__init__]"
self.kwargs = kwargs
self.functions = Functions()
self.utils = Utils()
self.path = {}
self.x = 0
self.y = 0
self.size = 1
self.is_deletable = False
self.offset = [0, 0]
self.polar = PolarCoordinates()
self.current_fft = {}
self.image = Frame()
# Configuration
self.kwargs["fourier"] = {
"interpolation":
MMVSkiaInterpolation(
self.mmv,
function="remaining_approach",
ratio=self.kwargs.get("bar_responsiveness", 0.25),
),
}
# # We use separate file and classes for each type of visualizer
# Circle, radial visualizer
if self.kwargs["type"] == "circle":
self.builder = MMVSkiaMusicBarsCircle(self.mmv, self,
**self.kwargs)
def setup(self) -> None:
debug_prefix = "[MMVMain.__init__]"
self.utils = Utils()
print(debug_prefix, "Creating Context()")
self.context = Context(self)
print(debug_prefix, "Creating SkiaNoWindowBackend()")
self.skia = SkiaNoWindowBackend(self)
print(debug_prefix, "Creating Functions()")
self.functions = Functions()
print(debug_prefix, "Creating Interpolation()")
self.interpolation = Interpolation()
print(debug_prefix, "Creating Canvas()")
self.canvas = MMVImage(self)
print(debug_prefix, "Creating Fourier()")
self.fourier = Fourier()
print(debug_prefix, "Creating FFmpegWrapper()")
self.ffmpeg = FFmpegWrapper(self)
print(debug_prefix, "Creating AudioFile()")
self.audio = AudioFile()
print(debug_prefix, "Creating AudioProcessing()")
self.audio_processing = AudioProcessing()
print(debug_prefix, "Creating MMVAnimation()")
self.mmv_animation = MMVAnimation(self)
print(debug_prefix, "Creating Core()")
self.core = Core(self)
class MMVSkiaPianoRollTopDown:
def __init__(self, mmv, MMVSkiaPianoRollVectorial):
self.mmvskia_main = mmv
self.vectorial = MMVSkiaPianoRollVectorial
self.config = self.vectorial.config
self.functions = Functions()
self.datautils = DataUtils()
self.piano_keys = {}
self.keys_centers = {}
self.colors = {}
# Generate skia.Color4F out of the colors dictionary
self.colors = self.parse_colors_dict(colors_dict = self.config["colors"])
self.font_fits_on_width = {}
# Recursively parse a colors dict
def parse_colors_dict(self, colors_dict):
for key, value in colors_dict.items():
if value is None:
colors_dict[key] = None
continue
if isinstance(value, dict):
colors_dict[key] = self.parse_colors_dict(colors_dict = value)
continue
if isinstance(value[0], int):
value = [v / 256 for v in value]
colors_dict[key] = skia.Color4f(*value)
return colors_dict
# Bleed is extra keys you put at the lower most and upper most ranged keys
def generate_piano(self, min_note, max_note):
debug_prefix = "[MMVSkiaPianoRollTopDown.generate_piano]"
# NOTE: EDIT HERE STATIC VARIABLES TODO: set them on config
self.piano_height = (3.5/19) * self.mmvskia_main.context.height
self.viewport_height = self.mmvskia_main.context.height - self.piano_height
# Pretty print
print("Add key by index: {", end = "", flush = True)
# For each key index, with a bleed (extra keys) counting up and down from the minimum / maximum key index
for key_index in range(min_note - self.config["bleed"], max_note + self.config["bleed"]):
# Create a PianoKey object and set its index
next_key = PianoKey(self.mmvskia_main, self.vectorial, self)
next_key.by_key_index(key_index)
# Pretty print
if not key_index == max_note + self.config["bleed"] - 1:
print(", ", end = "", flush = True)
else:
print("}")
# Add to the piano keys list
self.piano_keys[key_index] = next_key
print(debug_prefix, "There will be", len(self.piano_keys.keys()), "keys in the piano roll")
# We get the center of keys based on the "distance walked" in between intervals
# As black keys are in between two white keys, we walk half white key width on those
# and a full white key between E and F, B and C.
# Get number of semitones (divisions) so we can calculate the semitone width afterwards
divisions = 0
# For each index and key index (you'll se in a while why)
for index, key_index in enumerate(self.piano_keys.keys()):
# Get this key object
key = self.piano_keys[key_index]
# First index of key can't compare to previous key, starts at current_center
if not index == 0:
# The previous key can be a white or black key, but there isn't a previous at index=0
prevkey = self.piano_keys[key_index - 1]
# Both are True, add two, one is True, add one
# 2 is a tone distance
# 1 is a semitone distance
# [True is 1 in Python]
divisions += (prevkey.is_white) + (key.is_white)
# The keys intervals for walking
self.semitone_width = self.mmvskia_main.context.width / divisions
self.tone_width = self.semitone_width * 2
self.set_best_font()
# Current center we're at on the X axis so we send to the keys their positions
current_center = 0
# Same loop, ignore index=0
for index, key_index in enumerate(self.piano_keys.keys()):
# Get this key object
key = self.piano_keys[key_index]
# First index of key can't compare to previous key, starts at current_center
if not index == 0:
prevkey = self.piano_keys[key_index - 1]
# Distance is a tone
if (prevkey.is_white) and (key.is_white):
current_center += self.tone_width
# Distance is a semitone
else:
current_center += self.semitone_width
# Set the note length according to if it's white or black
if key.is_white:
this_note_width = self.tone_width
else:
this_note_width = (4/6) * self.tone_width
# Set attributes to this note we're looping
self.piano_keys[key_index].width = this_note_width
self.piano_keys[key_index].height = self.piano_height
self.piano_keys[key_index].resolution_height = self.mmvskia_main.context.height
self.piano_keys[key_index].center_x = current_center
# And add to the key centers list this key center_x
self.keys_centers[key_index] = current_center
# Draw the piano, first draw the white then the black otherwise we have overlaps
def draw_piano(self):
end_piano_pos = self.mmvskia_main.context.height - self.piano_height
screen_coords = [-200, end_piano_pos, self.mmvskia_main.context.width + 200, end_piano_pos]
# Make the skia Paint and
screen_paint = skia.Paint(
AntiAlias = True,
Style = skia.Paint.kStroke_Style,
ImageFilter = skia.ImageFilters.DropShadowOnly(0, 0, 12, 12, self.colors["end_piano_shadow"]),
StrokeWidth = 20
)
self.mmvskia_main.skia.canvas.drawRect(skia.Rect(*screen_coords), screen_paint)
# White keys
for key_index in self.piano_keys.keys():
if self.piano_keys[key_index].is_white:
self.piano_keys[key_index].draw(self.notes_playing)
# Black keys
for key_index in self.piano_keys.keys():
if self.piano_keys[key_index].is_black:
self.piano_keys[key_index].draw(self.notes_playing)
# Draw the markers IN BETWEEN TWO WHITE KEYS
def draw_markers(self):
# The paint of markers in between white keys
white_white_paint = skia.Paint(
AntiAlias = True,
Color = self.colors["marker_color_between_two_white"],
Style = skia.Paint.kStroke_Style,
StrokeWidth = 1,
)
current_center = 0
# Check if prev and this key is white, keeps track of a current_center, create rect to draw and draw
for index, key_index in enumerate(self.piano_keys.keys()):
key = self.piano_keys[key_index]
if not index == 0:
prevkey = self.piano_keys[key_index - 1]
if (prevkey.is_white) and (key.is_white):
current_center += self.tone_width
rect = skia.Rect(
current_center - self.semitone_width,
0,
current_center - self.semitone_width,
self.mmvskia_main.context.height
)
self.mmvskia_main.skia.canvas.drawRect(rect, white_white_paint)
else:
current_center += self.semitone_width
# Ask each key to draw their CENTERED marker
for key_index in self.piano_keys.keys():
self.piano_keys[key_index].draw_marker()
# Draw a given note according to the seconds of midi content on the screen,
# horizontal (note), vertical (start / end time in seconds) and color (channel)
def draw_note(self, velocity, start, end, channel, note, name):
# Get the note colors for this channel, we receive a dict with "sharp" and "plain" keys
note_colors = self.colors[f"channel_{channel}"]
# Is a sharp key
if "#" in name:
width = self.semitone_width * 0.9
color1 = note_colors["sharp_1"]
color2 = note_colors.get("sharp_2", color1)
# Plain key
else:
width = self.tone_width * 0.6
color1 = note_colors["plain_1"]
color2 = note_colors.get("plain_2", color1)
# Make the skia Paint
note_paint = skia.Paint(
AntiAlias = True,
Style = skia.Paint.kFill_Style,
Shader = skia.GradientShader.MakeLinear(
points = [(0.0, 0.0), (self.mmvskia_main.context.width, self.mmvskia_main.context.height)],
colors = [color1, color2]
),
StrokeWidth = 2,
)
# Border of the note
note_border_paint = skia.Paint(
AntiAlias = True,
Style = skia.Paint.kStroke_Style,
ImageFilter = skia.ImageFilters.DropShadowOnly(3, 3, 30, 30, note_colors["border_shadow"]),
# MaskFilter=skia.MaskFilter.MakeBlur(skia.kNormal_BlurStyle, 2.0),
# StrokeWidth = max(self.mmvskia_main.context.resolution_ratio_multiplier * 2, 1),
StrokeWidth = 0,
)
# Horizontal we have it based on the tones and semitones we calculated previously
# this is the CENTER of the note
x = self.keys_centers[note]
# The Y is a proportion of, if full seconds of midi content, it's the viewport height itself,
# otherwise it's a proportion to the processing time according to a start value in seconds
y = self.functions.proportion(
self.config["seconds_of_midi_content"],
self.viewport_height, #*2,
self.mmvskia_main.context.current_time - start
)
# The height is just the proportion of, seconds of midi content is the maximum height
# how much our key length (end - start) is according to that?
height = self.functions.proportion(
self.config["seconds_of_midi_content"],
self.viewport_height,
end - start
)
spacing = self.viewport_height / 512
# Build the coordinates of the note
# Note: We add and subtract half a width because X is the center
# while we need to add from the viewport out heights on the Y
coords = [
x - (width / 2),
y + (self.viewport_height) - height + spacing,
x + (width / 2),
y + (self.viewport_height) - spacing,
]
# Rectangle border of the note
# rect = skia.Rect(*coords)
rect = skia.RRect(skia.Rect(*coords),
self.config["rounding"]["notes"]["x"],
self.config["rounding"]["notes"]["y"]
)
# Draw the note and border
self.mmvskia_main.skia.canvas.drawRRect(rect, note_paint)
# Get paint, font
if self.config["draw_note_name"]:
paint = skia.Paint(AntiAlias = True, Color = skia.ColorBLACK, StrokeWidth = 43)
height_size = self.font.getSpacing()
text_size = self.font.measureText(name[:-1])
self.mmvskia_main.skia.canvas.drawString(
name[:-1],
(x) - (text_size / 2),
(y + (self.viewport_height) + spacing) - (height_size / 2),
self.font, paint
)
def set_best_font(self):
text_fits = 0
while True:
text_fits += 1
font = skia.Font(skia.Typeface('Ubuntu'), text_fits)
height_size = font.getSpacing()
text_size = font.measureText("F#")
if text_size > self.semitone_width:
break
self.font = skia.Font(skia.Typeface('Arial'), text_fits - 4)
# Build, draw the notes
def build(self, effects):
# Clear the background
screen_coords = [0.0, 0.0, self.mmvskia_main.context.width, self.mmvskia_main.context.height]
bg1 = self.colors["background_1"]
bg2 = self.colors.get("background_2", bg1)
# Make the skia Paint and
screen_paint = skia.Paint(
AntiAlias = True,
Style = skia.Paint.kFill_Style,
Shader = skia.GradientShader.MakeLinear(
points = [(0.0, 0.0), (screen_coords[2], screen_coords[3])],
colors = [bg1, bg2]
),
)
self.mmvskia_main.skia.canvas.drawRect(skia.Rect(*screen_coords), screen_paint)
# Draw the orientation markers
if self.config["do_draw_markers"]:
self.draw_markers()
# # Get "needed" variables
time_first_note = self.vectorial.midi.time_first_note
# If user passed seconds offset then don't use automatic one from midi file
if "seconds_offset" in self.config.keys():
offset = self.config["seconds_offset"]
else:
offset = 0
# offset = time_first_note # .. if audio is trimmed?
# Offsetted current time at the piano key top most part
current_time = self.mmvskia_main.context.current_time - offset
# What keys we'll bother rendering? Check against the first note time offset
# That's because current_time should be the real midi key not the offsetted one
accept_minimum_time = current_time - self.config["seconds_of_midi_content"]
accept_maximum_time = current_time + self.config["seconds_of_midi_content"]
# What notes are playing? So we draw a darker piano key
self.notes_playing = {}
# For each channel of notes
for channel in self.vectorial.midi.timestamps.keys():
# For each key message on the timestamps of channels (notes)
for key in self.vectorial.midi.timestamps[channel]:
# A "key" is a note if it's an integer
if isinstance(key, int):
# This note play / stop times, for all notes [[start, end], [start, end] ...]
note = key
times = self.vectorial.midi.timestamps[channel][note]["time"]
delete = []
# For each note index and the respective interval
for index, interval in enumerate(times):
# Out of bounds completely, we don't care about this note anymore.
# We mark for deletion otherwise it'll mess up the indexing
if interval[1] < accept_minimum_time:
delete.append(index)
continue
# Notes past this one are too far from being played and out of bounds
if interval[0] > accept_maximum_time:
break
# Is the current time inside the note? If yes, the note is playing
current_time_in_interval = (interval[0] < current_time < interval[1])
# Append to playing notes
if current_time_in_interval:
self.notes_playing[note] = channel
# Either way, draw the key
self.draw_note(
# TODO: No support for velocity yet :(
velocity = 128,
# Vertical position (start / end)
start = interval[0],
end = interval[1],
# Channel for the color and note for the horizontal position
channel = channel,
note = note,
# Name so we can decide to draw a sharp or plain key
name = self.vectorial.midi.note_to_name(note),
)
# This is an interval we do not care about anymore
# as the end of the note is past the minimum time we accept a note being rendered
for index in reversed(delete):
del self.vectorial.midi.timestamps[channel][note]["time"][index]
self.draw_piano()
class MMVSkiaProgressionBarRectangle:
def __init__(self, MMVSkiaVectorial, mmv):
self.mmv = mmv
self.vectorial = MMVSkiaVectorial
self.config = self.vectorial.config
self.polar = PolarCoordinates()
self.functions = Functions()
# Leave this much of blank space at the left and right edge
bleed_x = self.mmv.context.width * (1 / 19)
# Where Y starts and ends, ie. what horizontal line to draw the progression bar on
if self.config["position"] == "bottom":
start_end_y = (19 / 20) * self.mmv.context.height
elif self.config["position"] == "top":
start_end_y = (1 / 20) * self.mmv.context.height
# End X we have to count the bleed
self.start_x = bleed_x
self.end_x = self.mmv.context.width - bleed_x
self.start_y = start_end_y
self.end_y = start_end_y
# Build, draw the bar
def build(self, config, effects):
# Get "needed" variables
total_steps = self.mmv.context.total_steps
completion = self.functions.proportion(
total_steps, 1,
self.mmv.core.this_step) # Completion from 0-1 means
resolution_ratio_multiplier = self.mmv.context.resolution_ratio_multiplier
# We push the bar downwards according to the avg amplitude for a nice shaky-blur effect
offset_by_amplitude = self.mmv.core.modulators[
"average_value"] * self.config[
"shake_scalar"] * resolution_ratio_multiplier
if self.config["position"] == "top":
offset_by_amplitude *= (-1)
# White full opacity color
colors = [1, 1, 1, 1]
# Make a skia color with the colors list as argument
color = skia.Color4f(*colors)
# Make the skia Paint and
paint = skia.Paint(
AntiAlias=True,
Color=color,
Style=skia.Paint.kStroke_Style,
StrokeWidth=10 * resolution_ratio_multiplier, # + (magnitude/4),
# ImageFilter=skia.ImageFilters.DropShadow(3, 3, 5, 5, skia.ColorWHITE),
)
# The direction we're walking centered at origin, $\vec{AB} = A - B$
path_vector = np.array(
[self.end_x - self.start_x, self.end_y - self.start_y])
# Proportion we already walked
path_vector = path_vector * completion
# Draw the main line starting at the start coordinates, push down by offset_by_amplitude
path = skia.Path()
path.moveTo(self.start_x, self.start_y + offset_by_amplitude)
path.lineTo(self.start_x + path_vector[0],
self.start_y + path_vector[1] + offset_by_amplitude)
self.mmv.skia.canvas.drawPath(path, paint)
# Borders around image
if True:
# Distance away from s
distance = 9 * resolution_ratio_multiplier
colors = [1, 1, 1, 0.7]
# Make a skia color with the colors list as argument
color = skia.Color4f(*colors)
# Make the skia Paint and
paint = skia.Paint(
AntiAlias=True,
Color=color,
Style=skia.Paint.kStroke_Style,
StrokeWidth=2,
)
# Rectangle border
border = skia.Rect(self.start_x - distance,
self.start_y - distance + offset_by_amplitude,
self.end_x + distance,
self.end_y + distance + offset_by_amplitude)
# Draw the border
self.mmv.skia.canvas.drawRect(border, paint)
def __init__(self):
self.functions = Functions()
class AudioProcessing:
def __init__(self, depth = LOG_NO_DEPTH) -> None:
debug_prefix = "[AudioProcessing.__init__]"
ndepth = depth + LOG_NEXT_DEPTH
self.fourier = Fourier()
self.datautils = DataUtils()
self.functions = Functions()
self.config = None
# MMV specific, where we return repeated frequencies from the
# function process
self.where_decay_less_than_one = 440
self.value_at_zero = 5
# List of full frequencies of notes
# - 50 to 68 yields freqs of 24.4 Hz up to
self.piano_keys_frequencies = [round(self.get_frequency_of_key(x), 2) for x in range(-50, 68)]
logging.info(f"{depth}{debug_prefix} Whole notes frequencies we'll care: [{self.piano_keys_frequencies}]")
# Slice a mono and stereo audio data
def slice_audio(self,
stereo_data: np.ndarray,
mono_data: np.ndarray,
sample_rate: int,
start_cut: int,
end_cut: int,
batch_size: int=None
) -> None:
# Cut the left and right points range
left_slice = stereo_data[0][start_cut:end_cut]
right_slice = stereo_data[1][start_cut:end_cut]
# Cut the mono points range
# mono_slice = mono_data[start_cut:end_cut]
if not batch_size == None:
# Empty audio slice array if we're at the end of the audio
self.audio_slice = np.zeros([3, batch_size])
# Get the audio slices of the left and right channel
self.audio_slice[0][ 0:left_slice.shape[0] ] = left_slice
self.audio_slice[1][ 0:right_slice.shape[0] ] = right_slice
# self.audio_slice[2][ 0:mono_slice.shape[0] ] = mono_slice
else:
# self.audio_slice = [left_slice, right_slice, mono_slice]
self.audio_slice = [left_slice, right_slice]
# Calculate average amplitude
self.average_value = float(np.mean(np.abs(
mono_data[start_cut:end_cut]
)))
def resample(self,
data: np.ndarray,
original_sample_rate: int,
new_sample_rate: int
) -> None:
ratio = new_sample_rate / original_sample_rate
if ratio == 1:
return data
else:
return samplerate.resample(data, ratio, 'sinc_best')
# Get N semitones above / below A4 key, 440 Hz
#
# get_frequency_of_key(-12) = 220 Hz
# get_frequency_of_key( 0) = 440 Hz
# get_frequency_of_key( 12) = 880 Hz
#
def get_frequency_of_key(self, n):
return 440 * (2**(n/12))
# https://stackoverflow.com/a/2566508
def find_nearest(self, array, value):
index = (np.abs(array - value)).argmin()
return index, array[index]
# Calculate the FFT of this data, get only wanted frequencies based on the musical notes
def process(self,
data: np.ndarray,
original_sample_rate: int,
) -> None:
# The returned dictionary
processed = {}
# Iterate on config
for _, value in self.config.items():
# Get info on config
sample_rate = value.get("sample_rate")
start_freq = value.get("start_freq")
end_freq = value.get("end_freq")
# Get the frequencies we want and will return in the end
wanted_freqs = self.datautils.list_items_in_between(
self.piano_keys_frequencies,
start_freq, end_freq,
)
# Calculate the binned FFT, we get N vectors of [freq, value]
# of this FFT
binned_fft = self.fourier.binned_fft(
# Resample our data to the one specified on the config
data = self.resample(
data = data,
original_sample_rate = original_sample_rate,
new_sample_rate = sample_rate,
),
# # # # # # # # # # # # # # # # # # # # # # # # # # # #
sample_rate = sample_rate,
original_sample_rate = original_sample_rate,
)
# Get the nearest freq and add to processed
for freq in wanted_freqs:
# Get the nearest and FFT value
nearest = self.find_nearest(binned_fft[0], freq)
value = binned_fft[1][nearest[0]]
# How much bars we'll render duped at this freq, see
# this function on the Functions class for more detail
N = math.ceil(
self.functions.how_much_bars_on_this_frequency(
x = freq,
where_decay_less_than_one = self.where_decay_less_than_one,
value_at_zero = self.value_at_zero,
)
)
# Add repeated bars or just one
for i in range(N):
processed[nearest[1] + (i/10)] = value
linear_processed_fft = []
frequencies = []
for frequency, value in processed.items():
frequencies.append(frequency)
linear_processed_fft.append(value)
return [linear_processed_fft, frequencies]
jumpcutter = interface.get_jumpcutter()
# Configure with stuff
jumpcutter.configure(
batch_size=AUDIO_BATCH_SIZE,
sample_rate=48000,
target_fps=ORIGINAL_FRAMERATE,
)
# Read source audio file
jumpcutter.init(audio_path=INPUT_AUDIO_FILE)
mode = args.kflags.get("mode", "render")
# Stuff we'll need
functions = Functions()
if mode == "view":
shader_interface = interface.get_shader_interface()
mgl = shader_interface.get_mgl_interface(master_shader=True, flip=True)
MSAA = 8
# # MGL bootstrap
# Set up target render configuration
mgl.target_render_settings(
width=WIDTH,
height=HEIGHT,
ssaa=1,
fps=ORIGINAL_FRAMERATE,
class MMVMusicBarsCircle:
def __init__(self, MMVVectorial, context, skia_object):
self.vectorial = MMVVectorial
self.context = context
self.skia = skia_object
self.config = self.vectorial.config
self.polar = PolarCoordinates()
self.functions = Functions()
self.center_x = self.vectorial.context.width / 2
self.center_y = self.vectorial.context.height / 2
def build(self, fitted_ffts: dict, frequencies: list, this_step: int,
config: dict, effects):
resolution_ratio_multiplier = self.vectorial.context.resolution_ratio_multiplier
if self.config["mode"] == "symetric":
data = {}
for channel in (["l", "r"]):
data[channel] = {
"coordinates": [],
"paints": [],
}
this_channel_fft = fitted_ffts[channel]
npts = len(this_channel_fft)
for index, magnitude in enumerate(this_channel_fft):
if channel == "l":
theta = (math.pi / 2) - ((index / npts) * math.pi)
elif channel == "r":
theta = (math.pi / 2) + ((index / npts) * math.pi)
magnitude = (magnitude / 720) * self.context.height
minimum_multiplier = 0.4
maximum_multiplier = 17
size = (magnitude) * self.functions.ax_plus_b_two_points(
x=frequencies[0][index],
end_x=20000,
zero_value=minimum_multiplier,
max_value=maximum_multiplier,
)
# size = (magnitude*6) * ( (( (maximum_multiplier - minimum_multiplier)*index) / len(this_channel_fft)) + minimum_multiplier )
# We send an r, theta just in case we want to do something with it later on
data[channel]["coordinates"].append([
# ( self.config["minimum_bar_size"] + magnitude*(2e6) ) * effects["size"],
(self.config["minimum_bar_size"] + size) *
effects["size"],
theta,
])
# Rotate the colors a bit on each step
theta += this_step / 100
# Define the color of the bars
colors = [
abs(math.sin((theta / 2))),
abs(math.sin((theta + ((1 / 3) * 2 * math.pi)) / 2)),
abs(math.sin((theta + ((2 / 3) * 2 * math.pi)) / 2)),
] + [0.7] # Add full opacity
# Make a skia color with the colors list as argument
color = skia.Color4f(*colors)
# Make the skia Paint and
paint = skia.Paint(
AntiAlias=True,
Color=color,
Style=skia.Paint.kStroke_Style,
StrokeWidth=8 *
resolution_ratio_multiplier # + (magnitude/4),
)
# Store it on a list do draw in the end
data[channel]["paints"].append(paint)
# Our list of coordinates and paints, invert the right channel for drawing the path in the right direction
# Not reversing it will yield "symetric" bars along the diagonal
coordinates = data["l"]["coordinates"] + [
x for x in reversed(data["r"]["coordinates"])
]
paints = data["l"]["paints"] + [
x for x in reversed(data["r"]["paints"])
]
# Filled background
if False: # self.config["draw_background"]
path = skia.Path()
white_background = skia.Paint(
AntiAlias=True,
Color=skia.ColorWHITE,
Style=skia.Paint.kFill_Style,
StrokeWidth=3,
ImageFilter=skia.ImageFilters.DropShadow(
3, 3, 5, 5, skia.ColorBLACK),
MaskFilter=skia.MaskFilter.MakeBlur(
skia.kNormal_BlurStyle, 1.0))
more = 1.05
self.polar.from_r_theta(coordinates[0][0] * more,
coordinates[0][1])
polar_offset = self.polar.get_rectangular_coordinates()
path.moveTo(
(self.center_x + polar_offset[0]),
(self.center_y + polar_offset[1]),
)
for coord_index, coord in enumerate(coordinates):
# TODO: implement this function in DataUtils for not repeating myself
get_nearby = 4
size_coordinates = len(coordinates)
real_state = coordinates * 3
nearby_coordinates = real_state[size_coordinates + (
coord_index - get_nearby):size_coordinates +
(coord_index + get_nearby)]
# [0, 1, 2, 3, 4] --> weights=
# 3 4 5, 4, 3
n = len(nearby_coordinates)
weights = [n - abs((n / 2) - x) for x in range(n)]
s = 0
for index, item in enumerate(nearby_coordinates):
s += item[0] * weights[index]
avg_coord = s / sum(weights)
self.polar.from_r_theta(avg_coord * more, coord[1])
polar_offset = self.polar.get_rectangular_coordinates()
path.lineTo(
(self.center_x + polar_offset[0]),
(self.center_y + polar_offset[1]),
)
self.skia.canvas.drawPath(path, white_background)
# Countour, stroke
if False: # self.config["draw_black_border"]
more = 2
path = skia.Path()
black_stroke = skia.Paint(
AntiAlias=True,
Color=skia.ColorWHITE,
Style=skia.Paint.kStroke_Style,
StrokeWidth=6,
ImageFilter=skia.ImageFilters.DropShadow(
3, 3, 5, 5, skia.ColorWHITE),
MaskFilter=skia.MaskFilter.MakeBlur(
skia.kNormal_BlurStyle, 1.0))
for coord_index, coord in enumerate(coordinates):
get_nearby = 10
size_coordinates = len(coordinates)
real_state = coordinates * 3
nearby_coordinates = real_state[size_coordinates + (
coord_index - get_nearby):size_coordinates +
(coord_index + get_nearby)]
n = len(nearby_coordinates)
weights = [n - abs((n / 2) - x) for x in range(n)]
s = 0
for index, item in enumerate(nearby_coordinates):
s += item[0] * weights[index]
avg_coord = s / sum(weights)
self.polar.from_r_theta(
self.config["minimum_bar_size"] +
((avg_coord - self.config["minimum_bar_size"]) * more),
coord[1])
polar_offset = self.polar.get_rectangular_coordinates()
coords = [(self.center_x + polar_offset[0]),
(self.center_y + polar_offset[1])]
if coord_index == 0:
path.moveTo(*coords)
path.lineTo(*coords)
self.skia.canvas.drawPath(path, black_stroke)
# Draw the main bars
for index, coord in enumerate(coordinates):
more = 1
path = skia.Path()
path.moveTo(self.center_x, self.center_y)
self.polar.from_r_theta(coord[0], coord[1])
polar_offset = self.polar.get_rectangular_coordinates()
path.lineTo(
(self.center_x + polar_offset[0]) * more,
(self.center_y + polar_offset[1]) * more,
)
self.skia.canvas.drawPath(path, paints[index])
class MMVPianoRollTopDown:
def __init__(self, MMVVectorial, context, skia_object, midi):
self.vectorial = MMVVectorial
self.context = context
self.skia = skia_object
self.midi = midi
self.config = self.vectorial.config
self.functions = Functions()
self.datautils = DataUtils()
self.piano_keys = {}
self.keys_centers = {}
self.background_color = 22 / 255
# Bleed is extra keys you put at the lower most and upper most ranged keys
def generate_piano(self, min_note, max_note, bleed=3):
# NOTE: EDIT HERE STATIC VARIABLES TODO: set them on config
self.piano_height = (3.5 / 19) * self.vectorial.context.height
self.viewport_height = self.vectorial.context.height - self.piano_height
# TODO: set these variables in config
self.seconds_of_midi_content = 3
for key_index in range(min_note - bleed, max_note + bleed):
next_key = PianoKey(self.midi, self.skia, self.background_color)
next_key.by_key_index(key_index)
self.piano_keys[key_index] = next_key
print(len(self.piano_keys.keys()))
# We get the center of keys based on the "distance walked" in between intervals
# As black keys are in between two white keys, we walk half white key width on those
# and a full white key between E and F, B and C.
divisions = 0
for index, key_index in enumerate(self.piano_keys.keys()):
key = self.piano_keys[key_index]
# First index of key can't compare to previous key, starts at current_center
if not index == 0:
prevkey = self.piano_keys[key_index - 1]
# Distance is a tone
if (prevkey.is_white) and (key.is_white):
divisions += 2
# Distance is a semitone
else:
divisions += 1
# for key_index in self.piano_keys.keys():
# if "#" in self.piano_keys[key_index].name:
# divisions += 1
# else:
# divisions += 2
# Now we have the divisions, we can calculate the real key width based on the resolution
# print("width", self.vectorial.context.width)
# The keys intervals for walking
self.semitone_width = self.vectorial.context.width / divisions
self.tone_width = self.semitone_width * 2
# print(self.semitone_width, self.tone_width, divisions)
# exit()
# Current center we're at
current_center = 0
for index, key_index in enumerate(self.piano_keys.keys()):
key = self.piano_keys[key_index]
# First index of key can't compare to previous key, starts at current_center
if not index == 0:
prevkey = self.piano_keys[key_index - 1]
# Distance is a tone
if (prevkey.is_white) and (key.is_white):
current_center += self.tone_width
# Distance is a semitone
else:
current_center += self.semitone_width
# Set the
if key.is_white:
this_note_width = self.tone_width
else:
this_note_width = (4 / 6) * self.tone_width
self.piano_keys[key_index].width = this_note_width
self.piano_keys[key_index].height = self.piano_height
self.piano_keys[
key_index].resolution_height = self.vectorial.context.height
self.piano_keys[key_index].center_x = current_center
self.keys_centers[key_index] = current_center
def draw_piano(self):
for key_index in self.piano_keys.keys():
if self.piano_keys[key_index].is_white:
self.piano_keys[key_index].draw(self.notes_playing)
for key_index in self.piano_keys.keys():
if self.piano_keys[key_index].is_black:
self.piano_keys[key_index].draw(self.notes_playing)
def draw_markers(self):
c = 0.35
white_white_color = skia.Color4f(c, c, c, 1)
white_white_paint = skia.Paint(
AntiAlias=True,
Color=white_white_color,
Style=skia.Paint.kStroke_Style,
StrokeWidth=1,
)
current_center = 0
for index, key_index in enumerate(self.piano_keys.keys()):
key = self.piano_keys[key_index]
if not index == 0:
prevkey = self.piano_keys[key_index - 1]
if (prevkey.is_white) and (key.is_white):
current_center += self.tone_width
rect = skia.Rect(current_center - self.semitone_width, 0,
current_center + 1 - self.semitone_width,
self.vectorial.context.height)
self.skia.canvas.drawRect(rect, white_white_paint)
else:
current_center += self.semitone_width
for key_index in self.piano_keys.keys():
self.piano_keys[key_index].draw_marker()
def draw_note(self, velocity, start, end, channel, note, name):
color_channels = {
0: {
"plain": ImageColor.getcolor("#ffcc00", "RGB"),
"sharp": ImageColor.getcolor("#ff9d00", "RGB"),
},
1: {
"plain": ImageColor.getcolor("#00ff0d", "RGB"),
"sharp": ImageColor.getcolor("#00a608", "RGB"),
},
2: {
"plain": ImageColor.getcolor("#6600ff", "RGB"),
"sharp": ImageColor.getcolor("#39008f", "RGB"),
},
3: {
"plain": ImageColor.getcolor("#ff0000", "RGB"),
"sharp": ImageColor.getcolor("#990000", "RGB"),
},
4: {
"plain": ImageColor.getcolor("#00fffb", "RGB"),
"sharp": ImageColor.getcolor("#00b5b2", "RGB"),
},
5: {
"plain": ImageColor.getcolor("#ff006f", "RGB"),
"sharp": ImageColor.getcolor("#a10046", "RGB"),
},
6: {
"plain": ImageColor.getcolor("#aaff00", "RGB"),
"sharp": ImageColor.getcolor("#75b000", "RGB"),
},
7: {
"plain": ImageColor.getcolor("#e1ff00", "RGB"),
"sharp": ImageColor.getcolor("#a9bf00", "RGB"),
},
8: {
"plain": ImageColor.getcolor("#ff3300", "RGB"),
"sharp": ImageColor.getcolor("#a82200", "RGB"),
},
9: {
"plain": ImageColor.getcolor("#00ff91", "RGB"),
"sharp": ImageColor.getcolor("#00b567", "RGB"),
},
10: {
"plain": ImageColor.getcolor("#ff00aa", "RGB"),
"sharp": ImageColor.getcolor("#c40083", "RGB"),
},
11: {
"plain": ImageColor.getcolor("#c800ff", "RGB"),
"sharp": ImageColor.getcolor("#8e00b5", "RGB"),
},
12: {
"plain": ImageColor.getcolor("#00ff4c", "RGB"),
"sharp": ImageColor.getcolor("#00c93c", "RGB"),
},
13: {
"plain": ImageColor.getcolor("#ff8a8a", "RGB"),
"sharp": ImageColor.getcolor("#bf6767", "RGB"),
},
14: {
"plain": ImageColor.getcolor("#ffde7d", "RGB"),
"sharp": ImageColor.getcolor("#c4aa5e", "RGB"),
},
15: {
"plain": ImageColor.getcolor("#85ebff", "RGB"),
"sharp": ImageColor.getcolor("#5ca7b5", "RGB"),
},
16: {
"plain": ImageColor.getcolor("#ff7aa4", "RGB"),
"sharp": ImageColor.getcolor("#bd5978", "RGB"),
},
"default": {
"plain": ImageColor.getcolor("#dddddd", "RGB"),
"sharp": ImageColor.getcolor("#ffffff", "RGB"),
},
}
note_colors = color_channels.get(channel, color_channels["default"])
if "#" in name:
width = self.semitone_width * 0.9
c = note_colors["sharp"]
color = skia.Color4f(c[0] / 255, c[1] / 255, c[2] / 255, 1)
else:
width = self.tone_width * 0.6
c = note_colors["plain"]
color = skia.Color4f(c[0] / 255, c[1] / 255, c[2] / 255, 1)
# Make the skia Paint and
paint = skia.Paint(
AntiAlias=True,
Color=color,
Style=skia.Paint.kFill_Style,
# Shader=skia.GradientShader.MakeLinear(
# points=[(0.0, 0.0), (self.vectorial.context.width, self.vectorial.context.height)],
# colors=[skia.Color4f(0, 0, 1, 1), skia.Color4f(0, 1, 0, 1)]),
StrokeWidth=2,
)
# c = ImageColor.getcolor("#d1ce1d", "RGB")
c = (0, 0, 0)
border = skia.Paint(
AntiAlias=True,
Color=skia.Color4f(c[0] / 255, c[1] / 255, c[2] / 255, 1),
Style=skia.Paint.kStroke_Style,
# ImageFilter=skia.ImageFilters.DropShadow(3, 3, 5, 5, skia.ColorBLUE),
# MaskFilter=skia.MaskFilter.MakeBlur(skia.kNormal_BlurStyle, 2.0),
StrokeWidth=max(
self.vectorial.context.resolution_ratio_multiplier * 2, 1),
)
x = self.keys_centers[note]
y = self.functions.proportion(
self.seconds_of_midi_content,
self.viewport_height, #*2,
self.vectorial.context.current_time - start)
height = self.functions.proportion(self.seconds_of_midi_content,
self.viewport_height, end - start)
coords = [
x - (width / 2),
y + (self.viewport_height) - height,
x + (width / 2),
y + (self.viewport_height),
]
# Rectangle border
rect = skia.Rect(*coords)
# Draw the border
self.skia.canvas.drawRect(rect, paint)
self.skia.canvas.drawRect(rect, border)
# Build, draw the bar
def build(self, fftinfo, this_step, config, effects):
c = self.background_color
self.skia.canvas.clear(skia.Color4f(c, c, c, 1))
self.seconds_of_midi_content = config["seconds-of-midi-content"]
SECS_OFFSET = config["seconds-offset"]
self.draw_markers()
# Get "needed" variables
current_time = self.vectorial.context.current_time + SECS_OFFSET
resolution_ratio_multiplier = self.vectorial.context.resolution_ratio_multiplier
# contents = self.datautils.dictionary_items_in_between(
# self.midi.timestamps,
# current_time - 2,
# current_time + self.seconds_of_midi_content * 2
# )
accept_minimum_time = current_time - self.seconds_of_midi_content
accept_maximum_time = current_time + self.seconds_of_midi_content
self.notes_playing = []
for channel in self.midi.timestamps.keys():
for key in self.midi.timestamps[channel]:
if isinstance(key, int):
note = key
times = self.midi.timestamps[channel][note]["time"]
delete = []
for index, interval in enumerate(times):
# Out of bounds
if interval[1] < accept_minimum_time:
delete.append(index)
continue
if interval[0] > accept_maximum_time:
break
current_time_in_interval = (interval[0] < current_time
< interval[1])
accepted_render = (accept_minimum_time < current_time <
accept_maximum_time)
if current_time_in_interval:
self.notes_playing.append(note)
if current_time_in_interval or accepted_render:
self.draw_note(
velocity=128,
start=interval[0] - SECS_OFFSET,
end=interval[1] - SECS_OFFSET,
channel=channel,
note=note,
name=self.midi.note_to_name(note),
)
for index in reversed(delete):
del self.midi.timestamps[channel][note]["time"][index]
self.draw_piano()
class Interpolation:
def __init__(self):
self.functions = Functions()
"""
Linear, between point A and B based on a current "step" and total steps
kwargs: {
"start_value": float, target start value
"target_value": float, target end value
"current_step": float, this step out of the total
"total_steps": float, target total steps to finish the interpolation
}
"""
def linear(self, **kwargs) -> float:
# Out of bounds in steps, return target value
if kwargs["current_step"] > kwargs["total_steps"]:
return kwargs["target_value"]
# How much a walked part is? difference between target and start, divided by total steps
part = (kwargs["target_value"] -
kwargs["start_value"]) / kwargs["total_steps"]
# How much we walk from start to finish in that proportion
walked = part * kwargs["current_step"]
# Return the proportion walked plus the start value
return kwargs["start_value"] + walked
"""
"Biased" remaining linear, walks remaining distance times the ratio
ratio, 0.05 is smooth, 0.1 is medium, 1 is instant
random is a decimal that adds to ratio randomly
kwargs: {
"start_value": float, target start value (returned if current_step is zero)
"target_value": float, target value to reach
"current_step": float, only used if 0 to signal start of interpolation
"current_value": float, the position we're at to calculate the remaining distance
"ratio": float, walk remaining distance times this ratio from current value
"ratio_randomness": float=0, add a random value from 0 to this to the ratio
}
"""
def remaining_approach(self, **kwargs) -> float:
# We're at the first step, so start on current value
if kwargs["current_step"] == 0:
return kwargs["start_value"]
# Remaining distance is difference between target and current, ratio is the ratio plus a random or 0 value
remaining_distance = (kwargs["target_value"] - kwargs["current_value"])
ratio = kwargs["ratio"] + random.uniform(
0, kwargs.get("ratio_randomness", 0))
return kwargs["current_value"] + (remaining_distance * ratio)
# Sigmoid activation between two points, smoothed out "linear" curver
def sigmoid(
self,
start_value: float,
target_value: float,
smooth: float,
) -> float:
distance = (target_value - start_value)
where = self.functions.proportion(total, 1, current)
walk = distance * self.functions.sigmoid(where, smooth)
return a + walk
class MMVSkiaPianoRollTopDown:
def __init__(self, mmv, MMVSkiaPianoRollVectorial):
self.mmvskia_main = mmv
self.vectorial = MMVSkiaPianoRollVectorial
self.config = self.vectorial.config
self.functions = Functions()
self.datautils = DataUtils()
self.piano_keys = {}
self.keys_centers = {}
"""
When converting the colors we divide by 255 so we normalize in a value in between 0 and 1
Because skia works like that.
"""
sep = os.path.sep
# Load the yaml config file
color_preset = self.config["color_preset"]
color_config_yaml = self.mmvskia_main.utils.load_yaml(
f"{self.mmvskia_main.mmvskia_interface.top_level_interace.data_dir}{sep}mmvskia{sep}piano_roll{sep}color_{color_preset}.yaml"
)
# Get the global colors into a dictionary
self.global_colors = {}
for key in color_config_yaml["global"]:
self.global_colors[key] = [
channel / 255 for channel in ImageColor.getcolor(
"#" + str(color_config_yaml["global"][key]), "RGB")
]
# # Get the note colors based on their channel
self.color_channels = {}
# For every channel config
for key in color_config_yaml["channels"]:
# Create empty dir
self.color_channels[key] = {}
# Get colors of sharp and plain, border, etc..
for color_type in color_config_yaml["channels"][key].keys():
# Hexadecimal representation of color
color_hex = color_config_yaml["channels"][key][color_type]
# Assign RGB value
self.color_channels[key][color_type] = [
channel / 255
for channel in ImageColor.getcolor(f"#{color_hex}", "RGB")
]
# Bleed is extra keys you put at the lower most and upper most ranged keys
def generate_piano(self, min_note, max_note):
debug_prefix = "[MMVSkiaPianoRollTopDown.generate_piano]"
# NOTE: EDIT HERE STATIC VARIABLES TODO: set them on config
self.piano_height = (3.5 / 19) * self.mmvskia_main.context.height
self.viewport_height = self.mmvskia_main.context.height - self.piano_height
# Pretty print
print("Add key by index: {", end="", flush=True)
# For each key index, with a bleed (extra keys) counting up and down from the minimum / maximum key index
for key_index in range(min_note - self.config["bleed"],
max_note + self.config["bleed"]):
# Create a PianoKey object and set its index
next_key = PianoKey(self.mmvskia_main, self.vectorial, self)
next_key.by_key_index(key_index)
# Pretty print
if not key_index == max_note + self.config["bleed"] - 1:
print(", ", end="", flush=True)
else:
print("}")
# Add to the piano keys list
self.piano_keys[key_index] = next_key
print(debug_prefix, "There will be", len(self.piano_keys.keys()),
"keys in the piano roll")
# We get the center of keys based on the "distance walked" in between intervals
# As black keys are in between two white keys, we walk half white key width on those
# and a full white key between E and F, B and C.
# Get number of semitones (divisions) so we can calculate the semitone width afterwards
divisions = 0
# For each index and key index (you'll se in a while why)
for index, key_index in enumerate(self.piano_keys.keys()):
# Get this key object
key = self.piano_keys[key_index]
# First index of key can't compare to previous key, starts at current_center
if not index == 0:
# The previous key can be a white or black key, but there isn't a previous at index=0
prevkey = self.piano_keys[key_index - 1]
# Both are True, add two, one is True, add one
# 2 is a tone distance
# 1 is a semitone distance
# [True is 1 in Python]
divisions += (prevkey.is_white) + (key.is_white)
# The keys intervals for walking
self.semitone_width = self.mmvskia_main.context.width / divisions
self.tone_width = self.semitone_width * 2
# Current center we're at on the X axis so we send to the keys their positions
current_center = 0
# Same loop, ignore index=0
for index, key_index in enumerate(self.piano_keys.keys()):
# Get this key object
key = self.piano_keys[key_index]
# First index of key can't compare to previous key, starts at current_center
if not index == 0:
prevkey = self.piano_keys[key_index - 1]
# Distance is a tone
if (prevkey.is_white) and (key.is_white):
current_center += self.tone_width
# Distance is a semitone
else:
current_center += self.semitone_width
# Set the note length according to if it's white or black
if key.is_white:
this_note_width = self.tone_width
else:
this_note_width = (4 / 6) * self.tone_width
# Set attributes to this note we're looping
self.piano_keys[key_index].width = this_note_width
self.piano_keys[key_index].height = self.piano_height
self.piano_keys[
key_index].resolution_height = self.mmvskia_main.context.height
self.piano_keys[key_index].center_x = current_center
# And add to the key centers list this key center_x
self.keys_centers[key_index] = current_center
# Draw the piano, first draw the white then the black otherwise we have overlaps
def draw_piano(self):
# White keys
for key_index in self.piano_keys.keys():
if self.piano_keys[key_index].is_white:
self.piano_keys[key_index].draw(self.notes_playing)
# Black keys
for key_index in self.piano_keys.keys():
if self.piano_keys[key_index].is_black:
self.piano_keys[key_index].draw(self.notes_playing)
# Draw the markers IN BETWEEN TWO WHITE KEYS
def draw_markers(self):
# The paint of markers in between white keys
white_white_paint = skia.Paint(
AntiAlias=True,
Color=skia.Color4f(
*self.global_colors["marker_color_between_two_white"], 1),
Style=skia.Paint.kStroke_Style,
StrokeWidth=1,
)
current_center = 0
# Check if prev and this key is white, keeps track of a current_center, create rect to draw and draw
for index, key_index in enumerate(self.piano_keys.keys()):
key = self.piano_keys[key_index]
if not index == 0:
prevkey = self.piano_keys[key_index - 1]
if (prevkey.is_white) and (key.is_white):
current_center += self.tone_width
rect = skia.Rect(current_center - self.semitone_width, 0,
current_center - self.semitone_width,
self.mmvskia_main.context.height)
self.mmvskia_main.skia.canvas.drawRect(
rect, white_white_paint)
else:
current_center += self.semitone_width
# Ask each key to draw their CENTERED marker
for key_index in self.piano_keys.keys():
self.piano_keys[key_index].draw_marker()
# Draw a given note according to the seconds of midi content on the screen,
# horizontal (note), vertical (start / end time in seconds) and color (channel)
def draw_note(self, velocity, start, end, channel, note, name):
# Get the note colors for this channel, we receive a dict with "sharp" and "plain" keys
note_colors = self.color_channels.get(channel,
self.color_channels["default"])
# Is a sharp key
if "#" in name:
width = self.semitone_width * 0.9
color = skia.Color4f(*note_colors["sharp"], 1)
# Plain key
else:
width = self.tone_width * 0.6
color = skia.Color4f(*note_colors["plain"], 1)
# Make the skia Paint
note_paint = skia.Paint(
AntiAlias=True,
Color=color,
Style=skia.Paint.kFill_Style,
# Shader=skia.GradientShader.MakeLinear(
# points=[(0.0, 0.0), (self.mmvskia_main.context.width, self.mmvskia_main.context.height)],
# colors=[skia.Color4f(0, 0, 1, 1), skia.Color4f(0, 1, 0, 1)]),
StrokeWidth=2,
)
# Border of the note
note_border_paint = skia.Paint(
AntiAlias=True,
Color=skia.Color4f(*note_colors["border"], 1),
Style=skia.Paint.kStroke_Style,
# ImageFilter=skia.ImageFilters.DropShadow(3, 3, 5, 5, skia.ColorBLUE),
# MaskFilter=skia.MaskFilter.MakeBlur(skia.kNormal_BlurStyle, 2.0),
StrokeWidth=max(
self.mmvskia_main.context.resolution_ratio_multiplier * 2, 1),
)
# Horizontal we have it based on the tones and semitones we calculated previously
# this is the CENTER of the note
x = self.keys_centers[note]
# The Y is a proportion of, if full seconds of midi content, it's the viewport height itself,
# otherwise it's a proportion to the processing time according to a start value in seconds
y = self.functions.proportion(
self.config["seconds_of_midi_content"],
self.viewport_height, #*2,
self.mmvskia_main.context.current_time - start)
# The height is just the proportion of, seconds of midi content is the maximum height
# how much our key length (end - start) is according to that?
height = self.functions.proportion(
self.config["seconds_of_midi_content"], self.viewport_height,
end - start)
# Build the coordinates of the note
# Note: We add and subtract half a width because X is the center
# while we need to add from the viewport out heights on the Y
coords = [
x - (width / 2),
y + (self.viewport_height) - height,
x + (width / 2),
y + (self.viewport_height),
]
# Rectangle border of the note
rect = skia.Rect(*coords)
# Draw the note and border
self.mmvskia_main.skia.canvas.drawRect(rect, note_paint)
self.mmvskia_main.skia.canvas.drawRect(rect, note_border_paint)
# Build, draw the notes
def build(self, effects):
# Clear the background
self.mmvskia_main.skia.canvas.clear(
skia.Color4f(*self.global_colors["background"], 1))
# Draw the orientation markers
if self.config["do_draw_markers"]:
self.draw_markers()
# # Get "needed" variables
time_first_note = self.vectorial.midi.time_first_note
# If user passed seconds offset then don't use automatic one from midi file
if "seconds_offset" in self.config.keys():
offset = self.config["seconds_offset"]
else:
offset = 0
# offset = time_first_note # .. if audio is trimmed?
# Offsetted current time at the piano key top most part
current_time = self.mmvskia_main.context.current_time - offset
# What keys we'll bother rendering? Check against the first note time offset
# That's because current_time should be the real midi key not the offsetted one
accept_minimum_time = current_time - self.config[
"seconds_of_midi_content"]
accept_maximum_time = current_time + self.config[
"seconds_of_midi_content"]
# What notes are playing? So we draw a darker piano key
self.notes_playing = []
# For each channel of notes
for channel in self.vectorial.midi.timestamps.keys():
# For each key message on the timestamps of channels (notes)
for key in self.vectorial.midi.timestamps[channel]:
# A "key" is a note if it's an integer
if isinstance(key, int):
# This note play / stop times, for all notes [[start, end], [start, end] ...]
note = key
times = self.vectorial.midi.timestamps[channel][note][
"time"]
delete = []
# For each note index and the respective interval
for index, interval in enumerate(times):
# Out of bounds completely, we don't care about this note anymore.
# We mark for deletion otherwise it'll mess up the indexing
if interval[1] < accept_minimum_time:
delete.append(index)
continue
# Notes past this one are too far from being played and out of bounds
if interval[0] > accept_maximum_time:
break
# Is the current time inside the note? If yes, the note is playing
current_time_in_interval = (interval[0] < current_time
< interval[1])
# Append to playing notes
if current_time_in_interval:
self.notes_playing.append(note)
# Either way, draw the key
self.draw_note(
# TODO: No support for velocity yet :(
velocity=128,
# Vertical position (start / end)
start=interval[0],
end=interval[1],
# Channel for the color and note for the horizontal position
channel=channel,
note=note,
# Name so we can decide to draw a sharp or plain key
name=self.vectorial.midi.note_to_name(note),
)
# This is an interval we do not care about anymore
# as the end of the note is past the minimum time we accept a note being rendered
for index in reversed(delete):
del self.vectorial.midi.timestamps[channel][note][
"time"][index]
self.draw_piano()
class AudioProcessing:
def __init__(self) -> None:
debug_prefix = "[AudioProcessing.__init__]"
# Create some util classes
self.fourier = Fourier()
self.datautils = DataUtils()
self.functions = Functions()
self.config = None
# List of full frequencies of notes
# - 50 to 68 yields freqs of 24.4 Hz up to
self.piano_keys_frequencies = [
round(self.get_frequency_of_key(x), 2) for x in range(-50, 68)
]
# Get specs on config dictionary
def _get_config_stuff(self, config_dict):
# Get config
start_freq = config_dict["start_freq"]
end_freq = config_dict["end_freq"]
# Get the frequencies we want and will return in the end
wanted_freqs = self.datautils.list_items_in_between(
self.piano_keys_frequencies,
start_freq,
end_freq,
)
# Counter for expected frequencies on this config
expected_N_frequencies = 0
expected_frequencies = []
# Add target freq if it's not on the list
for freq in wanted_freqs:
# How much bars we'll render duped at this freq, see
# this function on the Functions class for more detail
N = math.ceil(
self.functions.how_much_bars_on_this_frequency(
x=freq,
where_decay_less_than_one=self.where_decay_less_than_one,
value_at_zero=self.value_at_zero,
))
# Add to total freqs the amount we expect
expected_N_frequencies += N
# Add individual frequencies
expected_frequencies.extend([freq + (i / 100) for i in range(N)])
# Return info
return {
"original_sample_rate": config_dict["original_sample_rate"],
"target_sample_rate": config_dict["target_sample_rate"],
"expected_N_frequencies": expected_N_frequencies,
"expected_frequencies": expected_frequencies,
"start_freq": start_freq,
"end_freq": end_freq,
}
# Set up a configuration list of dicts, They can look like this:
"""
[ {
"original_sample_rate": 48000,
"target_sample_rate": 5000,
"start_freq": 20,
"end_freq": 2500,
}, {
...
}]
"""
# NOTE: The FFT will only get values of frequencies up to SAMPLE_RATE/2 and jumps of
# the calculation sample rate divided by the window size (batch size)
# So if you want more bass information, downsample to 5000 Hz or 1000 Hz and get frequencies
# up to 2500 or 500, respectively.
def configure(self,
config,
where_decay_less_than_one=440,
value_at_zero=3):
debug_prefix = "[AudioProcessing.configure]"
logging.info(
f"{debug_prefix} Whole notes frequencies we'll care: [{self.piano_keys_frequencies}]"
)
# Assign
self.config = config
self.FFT_length = 0
self.where_decay_less_than_one = where_decay_less_than_one
self.value_at_zero = value_at_zero
# The configurations on sample rate, frequencies to expect
self.process_layer_configs = []
# For every config dict on the config
for layers in self.config:
info = self._get_config_stuff(layers)
self.FFT_length += info["expected_N_frequencies"] * 2
self.process_layer_configs.append(info)
# The size will be half, because left and right channel so we multiply by 2
# self.FFT_length *= 2
print("BINNED FFT LENGTH", self.FFT_length)
# # Feature Extraction
# Calculate the Root Mean Square
def rms(self, values: np.ndarray) -> float:
return np.sqrt(np.mean(values**2))
# # New Methods
# Yield information on the audio slice
def get_info_on_audio_slice(self,
audio_slice: np.ndarray,
original_sample_rate,
do_calculate_fft=True) -> dict:
N = audio_slice.shape[1]
# Calculate MONO
mono = (audio_slice[0] + audio_slice[1]) / 2
yield ["mmv_raw_audio_left", audio_slice[0]]
yield ["mmv_raw_audio_right", audio_slice[1]]
# # Average audio amplitude based on RMS
# L, R, Mono respectively
RMS = []
# Iterate, calculate the median of the absolute values
for channel_number in [0, 1]:
RMS.append(np.sqrt(np.mean(audio_slice[channel_number][0:N]**2)))
# RMS.append(np.median(np.abs(audio_slice[channel_number][0:N//120])))
# Append mono average amplitude
RMS.append(sum(RMS) / 2)
# Yield average amplitudes info
yield ["mmv_rms", tuple([round(value, 8) for value in RMS])]
# # Standard deviations
yield [
"mmv_std",
tuple(
[np.std(audio_slice[0]),
np.std(audio_slice[1]),
np.std(mono)])
]
# # FFT shenanigans
if do_calculate_fft:
# The final fft we give to the shader
processed = np.zeros(self.FFT_length, dtype=np.float32)
# Counter to assign values on the processed array
counter = 0
# For each channel
for channel_index, data in enumerate(audio_slice):
# For every config dict on the config
for info in self.process_layer_configs:
# Sample rate
original_sample_rate = info["original_sample_rate"]
target_sample_rate = info["target_sample_rate"]
# Individual frequencies
expected_frequencies = info["expected_frequencies"]
# The FFT of [[frequencies], [values]]
binned_fft = self.fourier.binned_fft(
data=self.resample(
data=data,
original_sample_rate=original_sample_rate,
target_sample_rate=target_sample_rate,
),
original_sample_rate=original_sample_rate,
target_sample_rate=target_sample_rate,
)
if binned_fft is None: return
# Information on the frequencies, the index 0 is the DC bias, or frequency 0 Hz
# and at every index it jumps the distance between any index N and N+1
fft_freqs = binned_fft[0]
jumps = abs(fft_freqs[1])
# Get the nearest freq and add to processed
for freq in expected_frequencies:
# TODO: make configurable
flatten_scalar = self.functions.value_on_line_of_two_points(
Xa=20, Ya=0.1, Xb=20000, Yb=3, get_x=freq)
# # Get the nearest and FFT value
# Trick: since the jump of freqs are always the same, the nearest frequency index
# given a target freq will be the frequency itself divided by how many frequency we
# jump at the indexes
nearest = int(freq / jumps)
# The abs value of the FFT
value = abs(binned_fft[1][nearest]) * flatten_scalar
# Assign, iterate
processed[counter] = value
counter += 1
# Yield FFT data
yield ["mmv_fft", processed]
# # Common Methods
# Resample an audio slice (raw array) to some other frequency, this is useful when calculating
# FFTs because a lower sample rate means we get more info on the bass freqs
def resample(self, data: np.ndarray, original_sample_rate: int,
target_sample_rate: int) -> np.ndarray:
# If the ratio is 1 then we don't do anything cause new/old = 1, just return the input data
if target_sample_rate == original_sample_rate:
return data
# Use libsamplerate for resampling the audio otherwise
return samplerate.resample(data,
ratio=(target_sample_rate /
original_sample_rate),
converter_type='sinc_fastest')
# Resample the data with nearest index approach
# Doesn't really work, experimental, maybe I understand resampling wrong
def resample_nearest(self, data: np.ndarray, original_sample_rate: int,
target_sample_rate: int) -> np.ndarray:
# Nothing to do, target sample rate is the same as old one
if (target_sample_rate == original_sample_rate):
return data
# The ratio we'll resample
ratio = (original_sample_rate / target_sample_rate)
# Length of the data
N = data.shape[0]
# Target new array length
T = N * ratio
# Array of original indexes
indexes = np.arange(T)
# (indexes / T) is the normalized to max at 1, we multiply
# by the length of the original data so we expand the indexes
# we just shaved by N * ratio, then use integer numbers and
# offset by 1
indexes = ((indexes / T) * N).astype(np.int32) - 1
# Return the original data with selected indexes
return data[indexes]
# Get N semitones above / below A4 key, 440 Hz
#
# get_frequency_of_key(-12) = 220 Hz
# get_frequency_of_key( 0) = 440 Hz
# get_frequency_of_key( 12) = 880 Hz
#
def get_frequency_of_key(self, n, A4=440):
return A4 * (2**(n / 12))
# https://stackoverflow.com/a/2566508
# Find nearest value inside one array from a given target value
# I could make my own but this one is more efficient because it uses numpy
# Returns: index of the match and its value
def find_nearest(self, array, value):
index = (np.abs(array - value)).argmin()
return index, array[index]