class Flags(Variable):
__table_args__ = ({"keep_existing": True}, )
__tablename__ = "openfred_flags"
id = C(BI, FK(Variable.id), primary_key=True)
flag_ks = C(ARRAY(Int), nullable=False)
flag_vs = C(ARRAY(Str(37)), nullable=False)
__mapper_args_ = {"polymorphic_identity": "flags"}
@property
def flag(self, key):
flags = dict(zip(self.flag_ks, self.flag_vs))
return flags[key]
class Variable(Base):
__table_args__ = ({"keep_existing": True}, )
__tablename__ = "openfred_variables"
id = C(BI, primary_key=True)
name = C(Str(255), nullable=False, unique=True)
# TODO: Figure out whether and where this is in the '.nc' files.
type = C(Str(37))
netcdf_attributes = C(JSON)
description = C(Text)
standard_name = C(Str(255))
__mapper_args_ = {
"polymorphic_identity": "variable",
"polymorphic_on": type,
}
class Series(Base):
__tablename__ = "openfred_series"
__table_args__ = (
UC("height", "location_id", "timespan_id", "variable_id"),
{
"keep_existing": True
},
)
id = C(BI, primary_key=True)
values = C(ARRAY(Float), nullable=False)
height = C(Float)
timespan_id = C(BI, FK(classes["Timespan"].id), nullable=False)
location_id = C(BI, FK(classes["Location"].id), nullable=False)
variable_id = C(BI, FK(classes["Variable"].id), nullable=False)
timespan = relationship(classes["Timespan"], backref="series")
location = relationship(classes["Location"], backref="series")
variable = relationship(classes["Variable"], backref="series")
class OktmoObj(Base):
__tablename__ = 'oktmo'
CLS_ENUM = (
u'сф', # субъект
u'мр', # муниципальный район
u'го', # городской округ
u'гп', # городское поселение
u'сп', # сельское поселение
u'мс', # межселенная территория
u'тгфз' # внутригородская территория города федерального значения
)
id = C(BigInteger, primary_key=True)
code = C(Unicode(8)) # код ОКТМО
raw = C(Unicode(255)) # строка из ОКТМО
parent = C(Unicode(8)) # родитель с учетом группировки
parent_obj = C(Unicode(8)) # родетель без учета группировок
is_group = C(Boolean()) # это группировка ?
lvl = C(SmallInteger()) # уровень без учета группировок
cls = C(Enum(*CLS_ENUM, **{'native_enum': False})) # класс
is_subject = C(Boolean()) # это субъект?
simple_name = C(Unicode(255)) # упрощенное название
def parse(self, lookup=None):
self.id = int(self.code)
# убираем сноску с конца строки
if self.raw[-1] == '*':
self.raw = self.raw[:-1]
# код заканчивается на n нулей
zeroes = lambda n: self.code.endswith('0' * n)
# первые n cимволов кода
first = lambda n: self.code[:n]
# дополненные нулями
first_fill = lambda n: fill(self.code[:n])
p1 = int(self.code[2]) # признак 1 - 3-й разряд
p2 = int(self.code[5]) # признак 2 - 6-й разряд
g1 = int(self.code[3:5])
g2 = int(self.code[6:8])
if self.raw[-1] == '/':
self.is_group = True
if zeroes(5):
# субъекты закодированные на втором уровне
if self.code[:3] in SUBJ_AD:
self.is_group = False
self.is_subject = True
self.cls = 'сф'
self.parent = first_fill(2)
elif zeroes(4):
# группировка МР и ГО внутри авт. округов
self.parent = first_fill(3)
elif zeroes(2):
self.parent = first_fill(5)
else:
self.is_group = False
if zeroes(6):
self.lvl = 1
self.is_subject = True
self.cls = 'сф'
elif zeroes(3):
self.lvl = 2
self.parent_obj = fill(first(2))
if p1 == 8:
if g1 in range(10, 50):
self.cls = 'мр'
elif g1 in range(50, 99):
self.cls = 'го'
elif p1 == 6:
self.cls = 'мр'
elif p1 == 7:
self.cls = 'го'
elif p1 == 3:
self.cls = 'тгфз'
elif p1 == 9:
if lookup(OktmoObj, OktmoObj.code == fill(first(2) + '3')):
self.cls = 'тгфз'
elif g1 in range(11, 50):
self.cls = 'мр'
elif g1 in range(50, 100):
self.cls = 'го'
if not self.cls:
stderr.write(u'Неопознаный признак P1 %d в %s %s\n' %
(p1, self.code, self.raw))
else:
self.lvl = 3
if p2 == 1:
self.cls = 'гп'
elif p2 == 4:
self.cls = 'сп'
elif p2 == 7:
self.cls = 'мс'
if not self.cls:
stderr.write(u'Неопознаный признак P2 %d в %s %s\n' %
(p2, self.code, self.raw))
if self.lvl > 1 and not self.parent:
self.parent = fill(self.code[:{2: 3, 3: 6}[self.lvl]])
# parent_obj
if self.parent:
op = lookup(OktmoObj, OktmoObj.code == self.parent)
if not op:
stderr.write(u'Неверный родитель для %s %s\n' %
(self.code, self.raw))
elif not op.is_group:
self.parent_obj = op.code
elif op.is_group and op.parent_obj:
self.parent_obj = op.parent_obj
else:
stderr.write(u"Двойная группировка: %s %s\n" %
(self.code, self.raw))
if not self.is_group:
self.simple_name = self.raw
for r in SIMPLE_NAME_REs:
if self.cls in r[0]:
for p in r[1]:
exp = re.compile(p[0], re.UNICODE + re.IGNORECASE)
self.simple_name = exp.sub(p[1], self.simple_name)
class OktmoOkatoObj(Base):
__tablename__ = 'oktmo_okato'
oktmo = C(Unicode(8), primary_key=True)
okato = C(Unicode(11), primary_key=True)
settlement_name = C(Unicode(100))
def mapped_classes(metadata):
""" Returns classes mapped to the openFRED database via SQLAlchemy.
The classes are dynamically created and stored in a dictionary keyed by
class names. The dictionary also contains the special entry `__Base__`,
which an SQLAlchemy `declarative_base` instance used as the base class from
which all mapped classes inherit.
"""
Base = declarative_base(metadata=metadata)
classes = {"__Base__": Base}
def map(name, registry, namespace):
namespace["__tablename__"] = "openfred_" + name.lower()
namespace["__table_args__"] = namespace.get("__table_args__", ()) + ({
"keep_existing":
True
}, )
if namespace["__tablename__"][-1] != "s":
namespace["__tablename__"] += "s"
registry[name] = type(name, (registry["__Base__"], ), namespace)
map(
"Timespan",
classes,
{
"id": C(BI, primary_key=True),
"start": C(DT),
"stop": C(DT),
"resolution": C(Interval),
"segments": C(ARRAY(DT, dimensions=2)),
"__table_args__": (UC("start", "stop", "resolution"), ),
},
)
map(
"Location",
classes,
{
"id":
C(BI, primary_key=True),
"point":
C(
geotypes.Geometry(geometry_type="POINT", srid=4326),
unique=True,
),
},
)
# TODO: Handle units.
class Variable(Base):
__table_args__ = ({"keep_existing": True}, )
__tablename__ = "openfred_variables"
id = C(BI, primary_key=True)
name = C(Str(255), nullable=False, unique=True)
# TODO: Figure out whether and where this is in the '.nc' files.
type = C(Str(37))
netcdf_attributes = C(JSON)
description = C(Text)
standard_name = C(Str(255))
__mapper_args_ = {
"polymorphic_identity": "variable",
"polymorphic_on": type,
}
classes["Variable"] = Variable
class Flags(Variable):
__table_args__ = ({"keep_existing": True}, )
__tablename__ = "openfred_flags"
id = C(BI, FK(Variable.id), primary_key=True)
flag_ks = C(ARRAY(Int), nullable=False)
flag_vs = C(ARRAY(Str(37)), nullable=False)
__mapper_args_ = {"polymorphic_identity": "flags"}
@property
def flag(self, key):
flags = dict(zip(self.flag_ks, self.flag_vs))
return flags[key]
classes["Flags"] = Flags
class Series(Base):
__tablename__ = "openfred_series"
__table_args__ = (
UC("height", "location_id", "timespan_id", "variable_id"),
{
"keep_existing": True
},
)
id = C(BI, primary_key=True)
values = C(ARRAY(Float), nullable=False)
height = C(Float)
timespan_id = C(BI, FK(classes["Timespan"].id), nullable=False)
location_id = C(BI, FK(classes["Location"].id), nullable=False)
variable_id = C(BI, FK(classes["Variable"].id), nullable=False)
timespan = relationship(classes["Timespan"], backref="series")
location = relationship(classes["Location"], backref="series")
variable = relationship(classes["Variable"], backref="series")
classes["Series"] = Series
return classes
class BaseGroup(SSPPGModel, MixinSessionFK):
__abstract__ = True
id_in_subsession = C(st.Integer, index=True)
round_number = C(st.Integer, index=True)
@property
def _Constants(self) -> BaseConstants:
return get_models_module(self.get_folder_name()).Constants
def __unicode__(self):
return str(self.id)
def get_players(self):
return list(self.player_set.order_by('id_in_group'))
def get_player_by_id(self, id_in_group):
try:
return self.player_set.filter_by(id_in_group=id_in_group).one()
except NoResultFound:
msg = 'No player with id_in_group {}'.format(id_in_group)
raise ValueError(msg) from None
def get_player_by_role(self, role):
if get_roles(self._Constants):
try:
return self.player_set.filter_by(_role=role).one()
except NoResultFound:
pass
else:
for p in self.get_players():
if p.role() == role:
return p
msg = f'No player with role "{role}"'
raise ValueError(msg)
def set_players(self, players_list):
Constants = self._Constants
roles = get_roles(Constants)
for i, player in enumerate(players_list, start=1):
player.group = self
player.id_in_group = i
player._role = get_role(roles, i)
db.commit()
def in_round(self, round_number):
try:
return in_round(
type(self),
round_number,
session=self.session,
id_in_subsession=self.id_in_subsession,
)
except InvalidRoundError as exc:
msg = (str(exc) + '; ' +
('Hint: you should not use this '
'method if you are rearranging groups between rounds.'))
ExceptionClass = type(exc)
raise ExceptionClass(msg) from None
def in_rounds(self, first, last):
try:
return in_rounds(
type(self),
first,
last,
session=self.session,
id_in_subsession=self.id_in_subsession,
)
except InvalidRoundError as exc:
msg = (str(exc) + '; ' +
('Hint: you should not use this '
'method if you are rearranging groups between rounds.'))
ExceptionClass = type(exc)
raise ExceptionClass(msg) from None
def in_previous_rounds(self):
return self.in_rounds(1, self.round_number - 1)
def in_all_rounds(self):
return self.in_previous_rounds() + [self]
@declared_attr
def subsession_id(cls):
app_name = cls.get_folder_name()
return C(st.Integer, ForeignKey(f'{app_name}_subsession.id'))
@declared_attr
def subsession(cls):
return relationship(f'{cls.__module__}.Subsession',
back_populates='group_set')
@declared_attr
def player_set(cls):
return relationship(f'{cls.__module__}.Player',
back_populates="group",
lazy='dynamic')
def subsession_id(cls):
app_name = cls.get_folder_name()
return C(st.Integer, ForeignKey(f'{app_name}_subsession.id'))
class OkatoObj(Base):
__tablename__ = 'okato'
CLS_ENUM = (
u'адм_район', # районы субъекта
u'город', # города
u'пгт', # поселки городского типа
u'город|пгт',
u'гфз_1', # первый уровень деления ГФЗ: округа Москвы, районы Спб
u'гфз_2', # второй уровень деления ГФЗ: районы Москвы, округа Спб
u'нп',
u'сельсовет',
u'unknown',
u'гор_район' # район города, или городского округа
)
id = C(BigInteger, primary_key=True)
code = C(Unicode(11), unique=True, nullable=False) # код ОКАТО
raw = C(Unicode(255)) # строка из ОКАТО as-is
is_group = C(Boolean()) # это группировка ?
parent = C(Unicode(11)) # родитель с учетом группировок
parent_obj = C(Unicode(11)) # родитель без учета группировок
lvl = C(SmallInteger()) # уровень
cls = C(Unicode(10)) # класс
is_settlement = C(Boolean()) # это населенный пункт
is_subject = C(Boolean()) # это субъект
name = C(Unicode(100)) # имя без статусной части
status = C(Unicode(100)) # статусная часть
cl_class = None
cl_level = None
manager = None
def parse(self, lookup=None):
code = self.code
raw = self.raw
self.manager = None
# код заканчивается на n нулей
zeroes = lambda n: self.code.endswith('0' * n)
# все группировки заканчиваются на '/'
self.is_group = raw[-1] == '/'
p1 = int(code[2]) # признак 1 - разряд 3
p2 = int(code[5]) # признак 2 - разряд 6
v1 = int(code[3:5]) # разряды 4-5
v2 = int(code[6:8]) # разряды 7-8
level = None
if self.is_group:
if len(code) == 8:
if zeroes(5):
self.parent = fill(code[:2])
elif zeroes(4):
pst = int(code[3])
while True:
p = fill(code[:3] + str(pst))
print p
po = lookup(OkatoObj, OkatoObj.code == p)
if po:
self.parent = po.code
stderr.write(
"[%s] %s > [%s] %s\n" %
(self.code, self.raw, po.code, po.raw))
break
else:
pst = pst - 1
if pst < 0:
stderr.write(
u"Не удалось определить родителя для %s %s\n"
% (self.code, self.raw))
break
elif code.endswith(('00', '50')):
self.parent = code[:5] + '000'
if len(code) == 11:
assert code[-3:] == '000', 'Ошибка в группировке'
self.parent = fill(code[:8])
self.parent_obj = fill(code[:8])
elif len(code) == 8 and not self.is_group:
if zeroes(6):
self.cl_level = 1
self.parent = None
self.parent_obj = None
if zeroes(5):
# это автономный округ
self.cls = 'ао'
self.parent = fill(code[:2])
self.parent_obj = fill(code[:2])
elif zeroes(3):
self.cl_level = 2
self.parent = code[:3] + '00000'
self.parent_obj = code[:2] + '000000'
if p1 == 1:
pst = int(code[3])
while True:
p = fill(code[:3] + str(pst))
po = lookup(OkatoObj, OkatoObj.code == p)
if po and po.is_group:
self.parent = po.code
stderr.write(
"[%s] %s > [%s] %s\n" %
(self.code, self.raw, po.code, po.raw))
break
else:
pst = pst - 1
elif p1 == 2:
self.parent_obj = code[:2] + '000000'
if v1 in range(1, 60):
self.cl_class = 'адм_район'
self.parent = code[:3] + '00000'
elif v1 in range(60, 100):
self.cl_class = 'гфз_1'
self.parent = code[:3] + '60000'
elif p1 == 4:
# по описанию статуc должен зависеть от v1,
# но на московской области это не работает,
# поэтому город или пгт
# попробуем посмотреть группировку верхнего уровня
p_code = code[:3] + '00000'
parent_group = lookup(
OkatoObj, OkatoObj.code == p_code.encode('utf-8'))
pr = parent_group.raw
if pr.startswith(u'Города'):
self.cl_class = 'город'
elif pr.startswith(u'Поселки городского типа'):
self.cl_class = 'пгт'
elif p1 == 5:
# это значения признака в классификаторе не описано,
# на московской области вроде бы работает
if v1 in range(1, 60):
self.cl_class = 'город'
elif v1 in range(60, 100):
self.cl_class = 'пгт'
else:
self.cl_level = 3
self.parent = code[:7] + '0'
self.parent_obj = code[:5] + '000'
if p2 == 3:
self.cl_class = 'гор_район'
elif p2 == 5:
if v2 in range(1, 50):
self.cl_class = 'город'
elif v2 in range(50, 100):
if v1 in range(60, 100):
self.cl_class = 'гфз_2'
elif v1 in range(1, 60):
self.cl_class = 'пгт'
elif p2 == 6:
# в самарской области сюда попадают устраненные НП в Тольяти
self.cl_class = 'unknown'
elif p2 == 8:
self.cl_class = 'сельсовет'
elif len(code) == 11 and not self.is_group:
self.cl_level = 4
self.cl_class = 'нп'
self.parent = fill(code[:8], to_len=11)
self.parent_obj = fill(code[:8])
# на первом уровне все субъекты, на втором только то, что еще не успели упразднить
self.is_subject = self.cl_level == 1 or (code in SUBJ_AD)
self.is_district = len(code) == 8 and code[2] == '2' and code[
-3:] == '000' and code[3:5] <> '00'
self.is_city = len(code) == 8 and code[2] == '4' and code[
-3:] == '000' and code[3:5] <> '00'
if self.cl_class in ('город', 'пгт', 'город|пгт'):
self.is_settlement = True
if self.cl_class in ('город'):
self.name = raw
self.status = self.cl_class
elif self.cl_class in ('пгт'):
self.name = raw
self.status = "поселок городского типа"
if len(code) == 11 and not self.is_group:
# сельские НП
self.is_settlement = True
self.lvl = self.cl_level
self.cls = self.cl_class
# определяем статус
for ss in _STATUS_SEARCH:
m = ss[0].match(raw)
if m:
(self.name, self.status) = (m.group(1), ss[1])
if self.is_settlement and not self.status:
stderr.write(u"Не удалось определить статус НП %s [%s]\n" %
(self.code, self.raw))
if self.code in SUBJ_AD:
self.parent = fill(self.code[:2])
class BaseSubsession(SSPPGModel, MixinSessionFK):
__abstract__ = True
round_number = C(
st.Integer,
index=True,
)
def in_round(self, round_number):
return in_round(type(self), round_number, session=self.session)
def in_rounds(self, first, last):
return in_rounds(type(self), first, last, session=self.session)
def in_previous_rounds(self):
return self.in_rounds(1, self.round_number - 1)
def in_all_rounds(self):
return self.in_previous_rounds() + [self]
def __unicode__(self):
return str(self.id)
def get_groups(self):
return list(self.group_set.order_by('id_in_subsession'))
def get_players(self):
return list(self.player_set.order_by('id'))
def _get_group_matrix(self, ids_only=False):
Player = self._PlayerClass()
Group = self._GroupClass()
players = (dbq(Player).join(Group).filter(
Player.subsession == self).order_by(Group.id_in_subsession,
'id_in_group'))
d = defaultdict(list)
for p in players:
d[p.group.id_in_subsession].append(
p.id_in_subsession if ids_only else p)
return list(d.values())
def get_group_matrix(self):
return self._get_group_matrix()
def get_group_matrix_ids(self):
return self._get_group_matrix(ids_only=True)
def set_group_matrix(self, matrix):
"""
warning: this deletes the groups and any data stored on them
"""
try:
players_flat = [p for g in matrix for p in g]
except TypeError:
raise GroupMatrixError(
'Group matrix must be a list of lists.') from None
try:
matrix_pks = sorted(p.id for p in players_flat)
except AttributeError:
# if integers, it's OK
if isinstance(players_flat[0], int):
# deep copy so that we don't modify the input arg
matrix = copy.deepcopy(matrix)
players_flat = sorted(players_flat)
players_from_db = self.get_players()
if players_flat == list(range(1, len(players_from_db) + 1)):
for i, row in enumerate(matrix):
for j, val in enumerate(row):
matrix[i][j] = players_from_db[val - 1]
else:
msg = (
'If you pass a matrix of integers to this function, '
'It must contain all integers from 1 to '
'the number of players in the subsession.')
raise GroupMatrixError(msg) from None
else:
msg = ('The elements of the group matrix '
'must either be Player objects, or integers.')
raise GroupMatrixError(msg) from None
else:
existing_pks = values_flat(self.player_set.order_by('id'), 'id')
if matrix_pks != existing_pks:
wrong_round_numbers = [
p.round_number for p in players_flat
if p.round_number != self.round_number
]
if wrong_round_numbers:
msg = ('You are setting the groups for round {}, '
'but the matrix contains players from round {}.'.
format(self.round_number, wrong_round_numbers[0]))
raise GroupMatrixError(msg)
msg = ('The group matrix must contain each player '
'in the subsession exactly once.')
raise GroupMatrixError(msg)
self.player_set.update({self._PlayerClass().group_id: None})
self.group_set.delete()
GroupClass = self._GroupClass()
for i, row in enumerate(matrix, start=1):
group = GroupClass.objects_create(
subsession=self,
id_in_subsession=i,
session=self.session,
round_number=self.round_number,
)
group.set_players(row)
def group_like_round(self, round_number):
previous_round: BaseSubsession = self.in_round(round_number)
group_matrix = previous_round._get_group_matrix(ids_only=True)
self.set_group_matrix(group_matrix)
@property
def _Constants(self):
return get_models_module(self.get_folder_name()).Constants
def _GroupClass(self):
return get_models_module(self.get_folder_name()).Group
def _PlayerClass(self):
return get_models_module(self.get_folder_name()).Player
@classmethod
def _has_group_by_arrival_time(cls):
app_name = cls.get_folder_name()
return has_group_by_arrival_time(app_name)
def group_randomly(self, *, fixed_id_in_group=False):
group_matrix = self.get_group_matrix()
group_matrix = otree.common._group_randomly(group_matrix,
fixed_id_in_group)
self.set_group_matrix(group_matrix)
def creating_session(self):
pass
def vars_for_admin_report(self):
return {}
def _gbat_try_to_make_new_group(self, page_index):
'''Returns the group ID of the participants who were regrouped'''
from otree.models import Participant
Player = self._PlayerClass()
STALE_THRESHOLD_SECONDS = 70
# count how many are re-grouped
waiting_players = list(
self.player_set.join(Participant).filter(
Participant._gbat_is_waiting == True,
Participant._index_in_pages == page_index,
Participant._gbat_grouped == False,
# this is just a failsafe
Participant._last_request_timestamp >=
time.time() - STALE_THRESHOLD_SECONDS,
))
try:
players_for_group = self.group_by_arrival_time_method(
waiting_players)
except:
raise # ResponseForException
if not players_for_group:
return None
participants = [p.participant for p in players_for_group]
group_id_in_subsession = self._gbat_next_group_id_in_subsession()
Constants = self._Constants
this_round_new_group = None
for round_number in range(self.round_number, Constants.num_rounds + 1):
subsession = self.in_round(round_number)
unordered_players = subsession.player_set.filter(
Player.participant_id.in_([pp.id for pp in participants]))
participant_ids_to_players = {
player.participant: player
for player in unordered_players
}
ordered_players_for_group = [
participant_ids_to_players[participant]
for participant in participants
]
group = self._GroupClass()(
subsession=subsession,
id_in_subsession=group_id_in_subsession,
session=self.session,
round_number=round_number,
)
db.add(group)
group.set_players(ordered_players_for_group)
if round_number == self.round_number:
this_round_new_group = group
# prune groups without players
# https://stackoverflow.com/a/21115972/
for group_to_delete in subsession.group_set.outerjoin(
Player).filter(Player.id == None):
db.delete(group_to_delete)
for participant in participants:
participant._gbat_grouped = True
participant._gbat_is_waiting = False
return this_round_new_group
def _gbat_next_group_id_in_subsession(self):
# 2017-05-05: seems like this can result in id_in_subsession that
# doesn't start from 1.
# especially if you do group_by_arrival_time in every round
# is that a problem?
Group = self._GroupClass()
return (dbq(func.max(
Group.id_in_subsession)).filter_by(session=self.session).scalar() +
1)
def group_by_arrival_time_method(self, waiting_players):
Constants = self._Constants
if Constants.players_per_group is None:
msg = (
'Page "{}": if using group_by_arrival_time, you must either set '
'Constants.players_per_group to a value other than None, '
'or define group_by_arrival_time_method.'.format(
self.__class__.__name__))
raise AssertionError(msg)
if len(waiting_players) >= Constants.players_per_group:
return waiting_players[:Constants.players_per_group]
@declared_attr
def group_set(cls):
return relationship(f'{cls.__module__}.Group',
back_populates="subsession",
lazy='dynamic')
@declared_attr
def player_set(cls):
return relationship(f'{cls.__module__}.Player',
back_populates="subsession",
lazy='dynamic')