def main():
# log = open("logfile.log", "a")
# sys.stdout = log
utils.build_structure([MODELS_PATH, OBJ_2D_PATH , OBJ_3D_PATH, PINHOLE_PATH, IMAGE_PATH, LOSS_PATH])
utils.clean_folders([OBJ_2D_PATH, OBJ_3D_PATH, LOSS_PATH, PINHOLE_PATH])
# although each section builds upon the previous one
# we split the work for better overview of the implementation
print("Running:")
print("> Morphable model")
morph_demo(idx=0)
morph_demo(idx=1)
morph_demo(idx=2)
print("> Pinhole camera model")
pinhole_demo(rotation=[0., -10., 0.], translation=[0., 0., 0.], idx=0)
pinhole_demo(rotation=[0., +10., 0.], translation=[0., 0., 0.], idx=1)
pinhole_demo(rotation=[0., +10., 0.], translation=[0., 0., -500.], idx=2)
print("> Latent parameters estimation")
print(f" reg_a=1., reg_d=1.")
latent_demo(load_fname="yke_neutral.jpeg", reg_a=1., reg_d=1., idx=0)
print(f" reg_a=10., reg_d=10.")
latent_demo(load_fname="yke_neutral.jpeg", reg_a=10., reg_d=10., idx=1)
print(f" reg_a=30., reg_d=30.")
latent_demo(load_fname="yke_neutral.jpeg", reg_a=30., reg_d=30., idx=2)
print(f" reg_a=60., reg_d=60.")
latent_demo(load_fname="yke_neutral.jpeg", reg_a=60., reg_d=60., idx=3)
def decode_code128(filename):
''' Decodifica imagem em Code128 '''
result = decode(Image.open(CODE128_FOLDER + filename))
text = treat_data(result)
clean_folders()
return text
def decode_qrcode(filename):
''' Decodifica imagem em qr code '''
result = decode(Image.open(QRCODE_FOLDER + filename),
symbols=[ZBarSymbol.QRCODE])
text = treat_data(result)
clean_folders()
return text
def analyze_tag_wise_mean_rating(data, tags=None):
print("Analyzing tag wise mean rating")
clean_folders([RESULT_FOLDER])
if tags is None:
tags = filter_tags_by_occurency_number(
data, tag_occurencies_filter=lambda v: v >= 200, save_code=413)
mean_tag_wise_rating = compute_tag_wise_mean_rating(data, tags)
path_to_save = os.path.join(RESULT_FOLDER,
"tag_wise_mean_rating_barchart.png")
draw_rating_bar_chart(mean_tag_wise_rating, path_to_save=path_to_save)
path_to_save = os.path.join(RESULT_FOLDER,
"lores_tag_wise_mean_rating_barchart.png")
draw_rating_bar_chart(mean_tag_wise_rating,
path_to_save=path_to_save,
bars_threshold=24,
figsize=(12, 9))
def analyze_tags_correlation(data, tags=None):
print("Analyzing tags correlation")
clean_folders([RESULT_FOLDER])
if tags is None:
tags = filter_tags_by_occurency_number(
data, tag_occurencies_filter=lambda v: v >= 3500, save_code=612)
_analyze_tags_correlation_subtask(data,
tags,
"tags_correlation_heatmat.png",
figsize=(14, 14),
save_code=612)
tags = filter_tags_by_occurency_number(
data, tag_occurencies_filter=lambda v: v >= 5000, save_code=111)
_analyze_tags_correlation_subtask(data,
tags,
"lores_tags_correlation_heatmat.png",
figsize=(9, 9),
save_code=111)
else:
_analyze_tags_correlation_subtask(data,
tags,
"tags_correlation_heatmat.png",
(14, 14),
save_code=0)
def analyze_density_by_tag_and_time(data,
begin_time=None,
end_time=None,
tag_groups=None,
discretizing_divisor=24 * 60 * 60,
number_of_precise_ratings=26,
number_of_peaks=8):
print("Analyzing density by tag and time")
clean_folders([RESULT_FOLDER])
if tag_groups is None:
tag_groups = [
"ветераны", "великая отечественная война",
"9 мая", "8 марта", "хэллоуин", "новый год", "диплом",
{"рубль",
"доллар"}, "трамп", "турция", "великобритания", "санкции",
{"игил", "сирия"}, "беженцы", {"донецк", "днр"},
{"майдан", "евромайдан"}, "крым", "ато", "поезд", "долг",
"почта", "трамвай", "детский сад", "общежитие",
{"fallout", "fallout 4"}, "gta 5", "dark souls",
{"ведьмак", "ведьмак 3"}, "overwatch", "мстители", "спойлер",
"deadpool", "игра престолов", "star wars", "осень", "зима",
"цыгане", "обида", "свадьба", "мошенники", "оскар",
"лига детективов", "санкции", "собеседование", "несправедливость",
"родственники", "торрент", "кризис", "леонардо ди каприо",
"хоккей", "футбол", "покемоны", "хоббит"
]
print("Computing normalized density")
tag_time_rating = _get_normalized_density(
data,
tag_groups,
discretizing_divisor=discretizing_divisor,
begin_time=begin_time,
end_time=end_time)
groups_per_plot = {
"seasonal": {
"group": [
"9 мая", "8 марта", "хэллоуин", "новый год", "диплом", "осень",
"зима"
],
"smoother":
_construct_default_linear_smoother(),
"draw_precise_plots":
False
},
"politics_1": {
"group": [{"рубль", "доллар"}, "турция", "великобритания",
"санкции", "трамп", {"игил", "сирия"}, "беженцы"],
"smoother":
_construct_default_linear_smoother(),
"draw_precise_plots":
False
},
"politics_2": {
"group": [{"донецк", "днр"}, {"майдан", "евромайдан"}, "крым",
"ато"],
"smoother": _construct_default_linear_smoother(),
"draw_precise_plots": False
},
"veterans": {
"group": ["ветераны", "великая отечественная война"],
"smoother": _construct_default_linear_smoother(),
"draw_precise_plots": False
},
"oscar": {
"group": ["оскар", "леонардо ди каприо"],
"smoother": _construct_default_linear_smoother(),
"draw_precise_plots": False
},
"hype_1": {
"group": [
"цыгане", "лига детективов", "собеседование",
"несправедливость", "родственники", "торрент", "кризис"
],
"smoother":
_construct_default_linear_smoother(),
"draw_precise_plots":
True
},
"hype_2": {
"group":
["поезд", "долг", "почта", "трамвай", "детский сад", "общежитие"],
"smoother":
_construct_strong_linear_smoother(),
"draw_precise_plots":
True
},
"sports": {
"group": ["хоккей", "футбол"],
"smoother": _construct_default_linear_smoother(),
"draw_precise_plots": True
},
"games": {
"group": [
"покемоны", {"fallout", "fallout 4"}, "gta 5", "dark souls",
{"ведьмак", "ведьмак 3"}, "overwatch"
],
"smoother":
_construct_strong_linear_smoother(),
"draw_precise_plots":
True
},
"universe": {
"group": [
"star wars", "хоббит", "мстители", "спойлер", "deadpool",
"игра престолов"
],
"smoother":
_construct_strong_linear_smoother(),
"draw_precise_plots":
True
}
}
for key, config in groups_per_plot.items():
print("Drawing {}".format(key))
group, smoother, draw_precise_plots = config["group"], config[
"smoother"], config["draw_precise_plots"]
plots, labels, captions_batches, vline_batches = [], [], [], []
T = None
for tag in group:
try:
record = next(record for record in tag_time_rating
if (tag in record[1] or tag == record[1]))
normalized_density = smoother(record[2])
if T is None:
T = np.arange(len(normalized_density)
) * discretizing_divisor + begin_time
if draw_precise_plots:
T2, R = extract_sorted_rating_with_time(
data, lambda v: tag in v["tags"] or tag == record[1])
current_vlines = [[
T2[i], 0, "пост с рейтингом " + str(R[i]) + " (" +
timestamp_to_date(T2[i]) + ")"
] for i in range(number_of_precise_ratings)]
current_vlines = sorted(current_vlines,
key=lambda x: -x[0])
for i in range(number_of_precise_ratings):
current_vlines[i][1] = max(normalized_density) * (
number_of_precise_ratings -
i) / (number_of_precise_ratings + 1)
vline_batches.append(current_vlines)
strong_maximums = []
for i in range(1, len(normalized_density)):
if normalized_density[i] > normalized_density[
i - 1] and normalized_density[
i] > normalized_density[i + 1]:
strong_maximums.append(normalized_density[i])
sorted_strong_maximums = sorted(strong_maximums,
reverse=True)
peak_idxes = [
np.where(normalized_density == d)[0][0]
for d in sorted_strong_maximums[:number_of_peaks]
]
captions_batches.append([
(T[max_idx], normalized_density[max_idx],
"пик " + " " + timestamp_to_date(T[max_idx]))
for max_idx in peak_idxes
])
plots.append(normalized_density)
if type(tag) is str:
labels.append(tag)
else:
labels.append("+".join(tag))
except StopIteration as si:
warnings.warn(
"No such tag in tag_time_rating: {}".format(str(tag)),
RuntimeWarning)
except RuntimeError as re:
warnings.warn("No such tag in data: {}".format(str(tag)),
RuntimeWarning)
if plots:
name = os.path.join(RESULT_FOLDER, key + ".png")
draw_rating_plot(T,
plots,
labels,
begin_time=begin_time,
end_time=end_time,
path_to_save=name)
name = os.path.join(RESULT_FOLDER, "lores_" + key + ".png")
draw_rating_plot(T,
plots,
labels,
begin_time=begin_time,
end_time=end_time,
path_to_save=name,
figsize=(14, 8))
if draw_precise_plots:
for idx, (plot, label, vlines, captions) in enumerate(
zip(plots, labels, vline_batches, captions_batches)):
print("Drawing {} precise: {}".format(key, label))
draw_rating_plot(T, [plot], [label],
begin_time=begin_time,
end_time=end_time,
path_to_save=os.path.join(
RESULT_FOLDER, key + "_precise_" +
str(idx) + "_(" + label + ").png"),
vlines=vlines,
captions=captions,
figsize=(60, 30))
else:
warnings.warn(
"No plots are drawn. Check 'groups_per_plot' - "
"are you sure they composed of same tags as 'tag_groups'",
RuntimeWarning)
def analyze_density_by_time(data):
print("Analyze density by time")
clean_folders([RESULT_FOLDER])
T, R = extract_rating_by_time(data, lambda x: True)
print("Analyze density by time: hourly for weekdays")
rolled_timestamps = _roll_timestamps(T, _TimestampFormat.HOURLY_WEEKDAY)
N_rolled, R_rolled = discretize(X=rolled_timestamps,
Y=R,
bins=2 * 24,
normalize=False)
corresponding_time_ticks = np.arange(0, 24 * 60 * 60, 30 * 60)
path_to_save = os.path.join(RESULT_FOLDER, "hourly_weekday")
draw_rating_hourly(corresponding_time_ticks,
R_rolled,
N_rolled,
path_to_save=path_to_save)
path_to_save = os.path.join(RESULT_FOLDER, "lores_hourly_weekday")
draw_rating_hourly(corresponding_time_ticks,
R_rolled,
N_rolled,
path_to_save=path_to_save,
figsize=(12, 6))
print("Analyze density by time: daily for weeks")
rolled_timestamps = _roll_timestamps(T, _TimestampFormat.DAILY)
N_rolled, R_rolled = discretize(X=rolled_timestamps,
Y=R,
bins=7 * 24,
normalize=False)
corresponding_time_ticks = np.arange(0, 7 * 24 * 60 * 60, 60 * 60)
path_to_save = os.path.join(RESULT_FOLDER, "daily")
draw_rating_daily(corresponding_time_ticks,
R_rolled,
N_rolled,
path_to_save=path_to_save)
path_to_save = os.path.join(RESULT_FOLDER, "lores_daily")
draw_rating_daily(corresponding_time_ticks,
R_rolled,
N_rolled,
path_to_save=path_to_save,
figsize=(12, 6))
print("Analyze density by time: monthly")
rolled_timestamps = _roll_timestamps(T, _TimestampFormat.MONTHLY)
N_rolled, R_rolled = discretize(X=rolled_timestamps,
Y=R,
bins=30 * 12,
normalize=False)
corresponding_time_ticks = np.arange(0, 30 * 24 * 60 * 60, 2 * 60 * 60)
path_to_save = os.path.join(RESULT_FOLDER, "monthly")
draw_rating_monthly(corresponding_time_ticks,
R_rolled,
N_rolled,
path_to_save=path_to_save)
path_to_save = os.path.join(RESULT_FOLDER, "lores_monthly")
draw_rating_monthly(corresponding_time_ticks,
R_rolled,
N_rolled,
path_to_save=path_to_save,
figsize=(12, 6))
def analyze_rating_density(data):
print("Analyze rating overall density")
clean_folders([RESULT_FOLDER])
ratings = sorted([v["rating"] for v in data.values()], reverse=True)
names = []
hlines = []
idxes = list(range(8)) + [11, 15, 20, 30, 40, 60, 80]
for i in idxes:
r = ratings[i]
try:
record = next(record for record in data.values()
if record["rating"] == r)
hlines.append([
0.61 + 0.285 * (i % 2), r, record["title"] + " (р:" + str(r) +
", " + timestamp_to_date(record["timestamp"]) + ")"
])
names.append(record["title"])
except StopIteration as si:
warnings.warn(
"Could not find record with such rating: {}".format(r),
RuntimeWarning)
gini = _compute_gini_coefficient(ratings)
p999 = np.percentile(ratings, 99.9)
p99 = np.percentile(ratings, 99)
p95 = np.percentile(ratings, 95)
mean = np.mean(ratings)
median = np.median(ratings)
hlines.append([1.0, p999, "99.9 перцентиль ({0:.2f})".format(p999)])
hlines.append([1.0, p99, "99 перцентиль ({0:.2f})".format(p99)])
hlines.append([1.0, p95, "95 перцентиль ({0:.2f})".format(p95)])
hlines.append([
0.61, 0.0,
"Индекс Джини: {0:.4f}, среднее: {1:.2f}, медиана: {2:.2f}".format(
gini, mean, median)
])
scatter_top_posts = list(zip([1.0] * 80, ratings[:80]))
name = os.path.join(RESULT_FOLDER, "rating_violinplot.png")
draw_rating_violinplot(ratings,
hlines=hlines,
scatter=scatter_top_posts,
path_to_save=name)
name = os.path.join(RESULT_FOLDER, "lores_rating_violinplot.png")
hlines = [hlines[0]] + [hlines[2]] + [hlines[4]] + [hlines[6]
] + hlines[-4:]
draw_rating_violinplot(ratings,
hlines=hlines,
scatter=scatter_top_posts[:25],
path_to_save=name,
figsize=(8, 10))
n_bins_for_logplot = 100
N = discretize([r for r in ratings if 100 < r <= 10000],
bins=n_bins_for_logplot,
normalize=False)
name = os.path.join(RESULT_FOLDER, "logplot.png")
draw_post_number_logplot([n_bins_for_logplot * i for i in range(len(N))],
[N], [u"Количество постов"],
path_to_save=name)
name = os.path.join(RESULT_FOLDER, "loores_logplot.png")
draw_post_number_logplot([n_bins_for_logplot * i for i in range(len(N))],
[N], [u"Количество постов"],
path_to_save=name,
figsize=(14, 8))