def _handle_reader(reader: Iterator[pd.DataFrame],
model_cls: Type[DeclaredModel], *, chunksize: int,
null_values: List[str], ignore_cols: List[str],
skip_rows_data: int, read_until: int, verbose: int):
session = Session()
counter = 0
for chunk in reader:
chunk: pd.DataFrame
if counter < skip_rows_data:
counter += chunksize
continue
for row in chunk.iterrows():
line, series = row
counter += 1
if hasattr(model_cls, 'create_instance'):
instance = model_cls.create_instance(series)
else:
instance = create_instance(model_cls,
series,
null_values=null_values,
ignore_cols=ignore_cols)
session.add(instance)
if verbose > 1:
if counter % 5000 == 0:
print(f"Added till row {counter}")
session.commit()
if verbose > 0:
print(f'Committed till row {counter}')
if read_until is not None and counter == read_until:
break
def add_imdb_info():
films: typing.Iterable[models.Film] = models.Film.objects.all()
session = Session()
for film in films:
tconst = film.t_const
# add info from title_basic
title_basic: imdb.TitleBasics = session.query(imdb.TitleBasics).filter(
imdb.TitleBasics.tconst == tconst).first()
# add genres
for genre in title_basic.genres.split(','):
g = models.Genre.upsert(label=genre)
film.hasGenre.add(g)
film.isAdult = bool(title_basic.isAdult)
if title_basic.isAdult == 1:
film.hasAudience = models.Audience.upsert(label='Adults')
else:
film.hasAudience = models.Audience.upsert(label='Children')
film.hasFeatureLengthInMinutes = title_basic.runtimeMinutes
# add info from title_akas
title_akas_lst: typing.Iterable[imdb.TitleAkas] = session.query(imdb.TitleAkas, imdb.ProductionCompanies, imdb.Certificates)\
.filter(imdb.TitleAkas.titleId == tconst)\
.filter(imdb.TitleAkas.isOriginalTitle == 1) \
.join(imdb.TitleBasics,
imdb.TitleBasics.tconst == imdb.TitleAkas.titleId)\
.join(imdb.Certificates,
imdb.Certificates.title == imdb.TitleAkas.title
and imdb.Certificates.year == imdb.TitleBasics.startYear)\
.join(imdb.ProductionCompanies, imdb.ProductionCompanies.title == imdb.TitleAkas.title
and imdb.ProductionCompanies.year == imdb.TitleBasics.startYear
).all()
try:
film.sync_from_wikidata()
except KeyError:
pass
for three in title_akas_lst:
title_akas = three[0]
prod = three[1]
if title_akas.isOriginalTitle == 1:
try:
if prod.country_code:
country = models.Country.objects.get(
alpha_2__iexact=prod.country_code.upper())
film.hasCountryOfOrigin = country
print(f'found: {title_akas}')
except ObjectDoesNotExist:
pass
print(film)
def insert_data(in_directory: Union[str, Path], out_directory: Union[str,
Path], *,
null_values: List[str]) -> None:
# Ensure directories are of type 'Path'
in_directory = Path(in_directory)
out_directory = Path(out_directory)
module_names = []
for pyfile in out_directory.glob('*.py'):
if pyfile.name in ['base.py', '__init__.py']:
continue
module_names.append(pyfile.stem)
class_names = [snake_to_capwords(name) for name in module_names]
# invalidate_caches()
path_to_package = Path(out_directory).relative_to(ROOT_DIR.absolute())
package_path = str(path_to_package).replace('/', '.').replace('\\', '.')
generated_package = import_module(package_path)
models: Dict[str, DeclaredModel] = {
class_name: generated_package.__dict__[class_name]
for class_name in class_names
}
#
session = Session()
for csvfile in in_directory.glob('*.*sv'):
info = _get_info_from_filename(csvfile.name)
model_name = info['name']
dialect = get_dialect_from_suffix(info['format'])
print(f"Opening {csvfile}")
class_name = snake_to_capwords(snake_case(model_name))
if class_name not in class_names:
raise AssertionError(
"class name from data files doesn't match models")
print(f"Writing from {csvfile} to {class_name}\n")
counter = 0
for chunk in pd.read_csv(csvfile, chunksize=10_000, dialect=dialect):
def add_award_info():
films: typing.Iterable[models.Film] = models.Film.objects.all()
session = Session()
# oscars = session.query(awards.Oscar).all()
# baftas = session.query(awards.Bafta).all()
for film in films:
oscar = session.query(awards.Oscar)\
.filter(awards.Oscar.film == film.hasTitle)\
.filter(awards.Oscar.year_film == film.hasInitialReleaseYear).first()
if not oscar:
continue
res, name = snlp.categorize(oscar.name)
if res == 'ORG':
agent = models.Organization.upsert(hasName=name, label=name)
elif res == 'PERSON':
agent = models.Person.upsert(hasName=name)
else:
raise ValueError
award = models.Award.objects.get(hasNickname='Oscars')
award_cat = models.AwardCategory.get_instance_from_kaggle_oscar_data(
oscar.category)
award_cem = models.AwardCeremony.upsert(
hasAward=award,
yearHeld=oscar.year_ceremony,
yearScreened=oscar.year_film,
hasEditionNumber=oscar.ceremony)
nom = models.NominationSituation.upsert(forFilm=film,
hasAward=award,
hasAwardCategory=award_cat,
hasAwardCeremony=award_cem,
isGivenTo=agent,
win=oscar.winner)
print(f'{film} {nom}')
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
import cgi
import html
import http.cookies
import os
import re
from engine import Session
wall = Session()
##########################################################################
## FORMS ##
##########################################################################
bad_form = False
notific = ''
header = '...'
out = ''
form = cgi.FieldStorage()
# ADD OPERATOR
lux = form.getfirst("lux", "")
lux = html.escape(lux)
hum = form.getfirst("hum", "")
hum = html.escape(hum)
temp = form.getfirst("temp", "")
from datetime import datetime
from close import Contradiction, Emptiness
from flask import Flask, render_template, request, redirect
from colors import COLORS, DEFAULT_COLOR
from engine import Session, contextdef_translate
from exceptions import EngineError, LrchLexerError
from flask_local.libs import JSONResponse, symbol_HTML, get_tree_clickable, get_tree_contra
from random import randint
app = Flask('flask_local', static_url_path='/static')
session = Session('main', 'config.json')
@app.route('/favicon.ico', methods=['GET'])
def favicon():
return redirect('/static/favicon.ico')
@app.route('/API/colors', methods=['GET'])
def colors():
return COLORS | {'Czarny': DEFAULT_COLOR}
@app.route('/', methods=['GET'])
def index():
session.reset_proof()
return render_template('index.html')
@app.route('/knowledge', methods=['GET'])
def knowledge():
def get_starting_films_from_awards() -> typing.List[Film]:
"""Returns a list of Film with attribs title_id, title, and file_year added."""
session = Session()
oscars = session.query(awards.Oscar).all()
baftas = session.query(awards.Bafta).all()
unique_oscar_films = set()
for oscar in oscars:
if oscar.film is None or oscar.year_film is None:
continue
unique_oscar_films.add((oscar.film, oscar.year_film))
unique_bafta_films = set()
for bafta in baftas:
if bafta.nominee is None or bafta.year is None:
continue
if bafta.workers is not None:
film_name = get_not_person_name(bafta.nominee, bafta.workers)
# try:
# film_name = get_not_person_name(bafta.nominee, bafta.workers)
# except ValueError:
# film_name = None
# while True:
# res = input(f"Choose between '{bafta.nominee}' or '{bafta.workers}' as the person (1/2/q/s): ")
# if res == 'q':
# raise
# if res == 's':
# break
# if res == '1':
# film_name = bafta.workers
# break
# if res == '2':
# film_name = bafta.nominee
# break
# if film_name is None:
# continue
else:
film_name = bafta.nominee
unique_bafta_films.add((film_name, bafta.year))
unique_award_winning_films = unique_oscar_films.union(unique_bafta_films)
film_lst: typing.List[Film] = []
title_akas_basics = session.query(
imdb.TitleAkas.titleId, imdb.TitleAkas.title,
imdb.TitleBasics.startYear).join(
imdb.TitleBasics,
imdb.TitleAkas.titleId == imdb.TitleBasics.tconst)
for film in unique_award_winning_films:
assert isinstance(film[0], str), str(film)
assert isinstance(film[1], int), str(film)
print(f"{film=}")
title_akas_basic = title_akas_basics.filter(
imdb.TitleAkas.title == film[0]).filter(
imdb.TitleBasics.startYear == film[1]).first()
if title_akas_basic is None:
continue
film_lst.append(
Film(title_id=title_akas_basic[0],
title=title_akas_basic[1],
film_year=title_akas_basic[2]))
print(title_akas_basic)
return film_lst