def create_index(self, columns, name=None, **kw):
"""Create an index to speed up queries on a table.
If no ``name`` is given a random name is created.
::
table.create_index(['name', 'country'])
"""
columns = [normalize_column_name(c) for c in ensure_tuple(columns)]
with self.db.lock:
if not self.exists:
raise DatasetException("Table has not been created yet.")
for column in columns:
if not self.has_column(column):
return
if not self.has_index(columns):
self._threading_warn()
name = name or index_name(self.name, columns)
columns = [self.table.c[c] for c in columns]
idx = Index(name, *columns, **kw)
idx.create(self.db.executable)
def define_metadef_resource_types_table(meta):
_constr_kwargs = {}
if meta.bind.name == 'ibm_db_sa':
_constr_kwargs['name'] = 'ix_metadef_resource_types_name'
metadef_res_types = Table(
'metadef_resource_types',
meta,
Column('id', Integer(), primary_key=True, nullable=False),
Column('name', String(80), nullable=False),
Column('protected', Boolean(), nullable=False, default=False),
Column('created_at', DateTime(), nullable=False),
Column('updated_at', DateTime()),
UniqueConstraint('name', **_constr_kwargs),
mysql_engine='InnoDB',
extend_existing=True)
if meta.bind.name != 'ibm_db_sa':
Index('ix_metadef_resource_types_name',
metadef_res_types.c.name)
return metadef_res_types
class FullTextSearch(GeoInterface, Base):
__tablename__ = "tsearch"
__table_args__ = (Index("tsearch_ts_idx", "ts", postgresql_using="gin"), {
"schema": _schema
})
__acl__ = [DENY_ALL]
id = Column(Integer, primary_key=True)
label = Column(Unicode)
layer_name = Column(Unicode) # Deprecated in v2
role_id = Column(Integer, ForeignKey(_schema + ".role.id"), nullable=True)
role = relationship("Role")
interface_id = Column(Integer,
ForeignKey(_schema + ".interface.id"),
nullable=True)
interface = relationship("Interface")
lang = Column(String(2), nullable=True)
public = Column(Boolean, server_default="true")
ts = Column(TsVector)
the_geom = Column(Geometry("GEOMETRY", srid=_srid, management=management))
params = Column(JSONEncodedDict, nullable=True)
actions = Column(JSONEncodedDict, nullable=True)
from_theme = Column(Boolean, server_default="false")
class CobraWhiteList(db.Model):
"""
Whitelist for project and rule
"""
__tablename__ = 'whitelist'
id = db.Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True, nullable=False)
project_id = db.Column(db.Integer, default=None, nullable=False)
rule_id = db.Column(db.Integer, default=None, nullable=False)
path = db.Column(db.String(512), default=None, nullable=False)
reason = db.Column(db.String(512), default=None, nullable=False)
status = db.Column(TINYINT, default=None, nullable=False)
created_at = db.Column(db.DateTime, default=None, nullable=False)
updated_at = db.Column(db.DateTime, default=None, nullable=False)
__table_args__ = (Index('ix_project_id_rule_id', project_id, rule_id), {"mysql_charset": "utf8mb4"})
def __init__(self, project_id, rule_id, path, reason, status, created_at=None, updated_at=None):
self.project_id = project_id
self.rule_id = rule_id
self.path = path
self.reason = reason
self.status = status
self.created_at = created_at
self.updated_at = updated_at
current_time = time.strftime('%Y-%m-%d %X', time.localtime())
if created_at is None:
self.created_at = current_time
else:
self.created_at = created_at
if updated_at is None:
self.updated_at = current_time
else:
self.updated_at = updated_at
def __repr__(self):
return "<CobraWhiteList %r-%r:%r>" % (self.project_id, self.rule_id, self.reason)
class CobraResults(db.Model):
"""
Results for task
"""
__tablename__ = 'results'
id = db.Column(INTEGER(unsigned=True), primary_key=True, autoincrement=True, nullable=False)
task_id = db.Column(INTEGER(11), nullable=False, default=None)
rule_id = db.Column(INTEGER(11), nullable=False, default=None)
file = db.Column(db.String(512), nullable=False, default=None)
line = db.Column(INTEGER(11), nullable=False, default=None)
code = db.Column(db.String(512), nullable=False, default=None)
created_at = db.Column(db.DateTime, nullable=False, default=None)
updated_at = db.Column(db.DateTime, nullable=False, default=None)
__table_args__ = (Index('ix_task_id_rule_id', task_id, rule_id), {"mysql_charset": "utf8mb4"})
def __init__(self, task_id, rule_id, file_path, line, code, created_at=None, updated_at=None):
self.task_id = task_id
self.rule_id = rule_id
self.file = file_path
self.line = line
self.code = code
self.created_at = created_at
self.updated_at = updated_at
current_time = time.strftime('%Y-%m-%d %X', time.localtime())
if created_at is None:
self.created_at = current_time
else:
self.created_at = created_at
if updated_at is None:
self.updated_at = current_time
else:
self.updated_at = updated_at
def __repr__(self):
return "<CobraResults %r - %r>" % (self.id, self.task_id)
class Resource(Base):
'''
Character resources, limited use abilities/items
'''
__tablename__ = 'resources'
id = Column(BigInteger, primary_key=True, doc='An autonumber id')
character_id = Column(BigInteger,
ForeignKey('characters.id'),
nullable=False,
doc='Character foreign key')
name = Column(String(64), nullable=False, doc='Resource name')
max = Column(Integer,
nullable=False,
default=1,
doc='The maximum number of uses of the resource')
current = Column(Integer,
nullable=False,
default=1,
doc='The current remaining uses of the resource')
recover = Column(Enum(Rest),
nullable=False,
default=Rest.other,
doc='How the character recovers the resource')
__table_args__ = (Index('_resource_index', character_id, name,
unique=True), )
character = relationship('Character',
foreign_keys=[character_id],
back_populates='resources')
def __str__(self):
ret = '{0.name}: {0.current}/{0.max}'.format(self)
if self.recover != Rest.other:
ret += ' per {} rest'.format(self.recover.name)
return ret
class Details(sqlite_manager.base):
__tablename__ = "details"
_id = Column(Integer, primary_key=True, autoincrement=True)
last_synced = Column(Integer)
date_created = Column(Integer)
__table_args__ = (Index("last_synced_index", "last_synced"), )
def insert(self, last_synced, date_created):
new_detail = Details(last_synced=last_synced,
date_created=date_created)
sqlite_manager.session.add(new_detail)
sqlite_manager.session.commit()
return new_detail._id
def get_details(self):
return sqlite_manager.session.query(Details).order_by(
desc(Details.last_synced))
def find_detail(self, details_id):
return sqlite_manager.session.query(Details).filter_by(
_id=details_id).first()
def update_last_synced(self, details_id, last_synced):
sqlite_manager.session.query(Details).filter_by(_id=details_id).update(
{"last_synced": last_synced})
sqlite_manager.session.commit()
def delete(self, details_id):
sqlite_manager.session.delete(self.find_detail(details_id))
sqlite_manager.commit()
class FlakyTestStat(db.Model):
""" Aggregated stats for a flaky test in a specific day.
The calculation is done periodically from the data in the test table. That is a
moderately expensive operation we don't want to keep re-running.
A flaky run for a test is one in which the test failed initially, and then
passed when re-run.
"""
__tablename__ = 'flakyteststat'
__table_args__ = (
Index('idx_flakyteststat_date', 'date'),
UniqueConstraint('name',
'project_id',
'date',
name='unq_name_per_project_per_day'),
)
id = Column(GUID, primary_key=True, default=uuid4)
name = Column(Text, nullable=False)
project_id = Column(GUID,
ForeignKey('project.id', ondelete="CASCADE"),
nullable=False)
date = Column(Date, default=today, nullable=False)
last_flaky_run_id = Column(GUID,
ForeignKey('test.id', ondelete="CASCADE"),
nullable=False)
flaky_runs = Column(Integer, default=0, nullable=False)
double_reruns = Column(Integer, default=0, nullable=False)
passing_runs = Column(Integer, default=0, nullable=False)
first_run = Column(Date, default=today, nullable=False)
project = relationship('Project')
last_flaky_run = relationship('TestCase')
def __init__(self, **kwargs):
super(FlakyTestStat, self).__init__(**kwargs)
class Event(db.Model):
"""
Indicates that something (specified by `type` and `data`) happened to some
entity (specified by `item_id`).
This allows us to record that we've performed some action with an external side-effect so
that we can be sure we do it no more than once. It is also useful for displaying to users which
actions have been performed when, and whether they were successful.
"""
__tablename__ = 'event'
__table_args__ = (
Index('idx_event_item_id', 'item_id'),
# Having this as unique prevents duplicate events, but in the future
# we may want to allow duplicates
# e.g. we can have a "sent email notification" event, but maybe
# we'd want to have multiple of those
UniqueConstraint('type', 'item_id', name='unq_event_key'),
)
id = Column(GUID, primary_key=True, default=uuid.uuid4)
# A value from EventType
type = Column(String(32), nullable=False)
item_id = Column('item_id', GUID, nullable=False)
date_created = Column(DateTime, default=datetime.utcnow)
date_modified = Column(DateTime, default=datetime.utcnow)
data = Column(JSONEncodedDict)
__repr__ = model_repr('type', 'item_id')
def __init__(self, **kwargs):
super(Event, self).__init__(**kwargs)
if self.id is None:
self.id = uuid.uuid4()
if self.date_created is None:
self.date_created = datetime.utcnow()
if self.date_modified is None:
self.date_modified = self.date_created
def visit_column(self, column):
"""Create a column (table already exists).
:param column: column object
:type column: :class:`sqlalchemy.Column` instance
"""
if column.default is not None:
self.traverse_single(column.default)
table = self.start_alter_table(column)
self.append("ADD ")
self.append(self.get_column_specification(column))
for cons in column.constraints:
self.traverse_single(cons)
self.execute()
# ALTER TABLE STATEMENTS
# add indexes and unique constraints
if column.index_name:
Index(column.index_name, column).create()
elif column.unique_name:
constraint.UniqueConstraint(column,
name=column.unique_name).create()
# SA bounds FK constraints to table, add manually
for fk in column.foreign_keys:
self.add_foreignkey(fk.constraint)
# add primary key constraint if needed
if column.primary_key_name:
cons = constraint.PrimaryKeyConstraint(
column, name=column.primary_key_name)
cons.create()
class Dataset(Base):
# TODO: make dataset_id and user_id unique together
__tablename__ = 'dataset'
__table_args__ = (Index('uix_user_id_dataset_id',
'user_id',
'dataset_id',
unique=True), )
id = Column(Integer, primary_key=True)
user_id = Column(Integer, ForeignKey('user.id'), nullable=False)
bamboo_host = Column(String(100), nullable=False)
dataset_id = Column(String(100), nullable=False)
user = relationship('User', backref=backref('datasets'))
added_on = Column(DateTime(timezone=True),
nullable=False,
default=func.now())
def extract_values_from_url(self, bamboo_host):
groups = re.match(dataset_regexp, bamboo_host).groups()
self.bamboo_host = groups[0]
self.dataset_id = groups[1]
@property
def bamboo_url(self):
return "%(url)s/datasets/%(dataset_id)s" % (
{
'url': self.bamboo_host,
'dataset_id': self.dataset_id
})
def url(self, request):
return request.route_url('user',
traverse=(
self.user.username,
'datasets',
self.dataset_id,
))
class Event(db.Model):
"""
"No component of the system depends on event existing"
its just for logging and displaying to the user. We log whenever we email
a user about a broken build (or a green build, if that option is set in
the ui.) Technically, the type column only has two distinct values:
[email_notification, green_build_notification]. Contains a JSON data-blob
"""
__tablename__ = 'event'
__table_args__ = (
Index('idx_event_item_id', 'item_id'),
# Having this as unique prevents duplicate events, but in the future
# we may want to allow duplicates
# e.g. we can have a "sent email notification" event, but maybe
# we'd want to have multiple of those
UniqueConstraint('type', 'item_id', name='unq_event_key'),
)
id = Column(GUID, primary_key=True, default=uuid.uuid4)
type = Column(String(32), nullable=False)
item_id = Column('item_id', GUID, nullable=False)
date_created = Column(DateTime, default=datetime.utcnow)
date_modified = Column(DateTime, default=datetime.utcnow)
data = Column(JSONEncodedDict)
__repr__ = model_repr('type', 'item_id')
def __init__(self, **kwargs):
super(Event, self).__init__(**kwargs)
if self.id is None:
self.id = uuid.uuid4()
if self.date_created is None:
self.date_created = datetime.utcnow()
if self.date_modified is None:
self.date_modified = self.date_created
def get_indicator_table(indicator_config, custom_metadata=None):
sql_columns = [column_to_sql(col) for col in indicator_config.get_columns()]
table_name = get_table_name(indicator_config.domain, indicator_config.table_id)
columns_by_col_id = {col.database_column_name for col in indicator_config.get_columns()}
extra_indices = []
for index in indicator_config.sql_column_indexes:
if set(index.column_ids).issubset(columns_by_col_id):
extra_indices.append(Index(
_custom_index_name(table_name, index.column_ids),
*index.column_ids
))
else:
_assert = soft_assert('{}@{}'.format('jemord', 'dimagi.com'))
_assert(False, "Invalid index specified on {}".format(table_name))
break
columns_and_indices = sql_columns + extra_indices
# todo: needed to add extend_existing=True to support multiple calls to this function for the same table.
# is that valid?
return sqlalchemy.Table(
table_name,
custom_metadata or metadata,
extend_existing=True,
*columns_and_indices
)
class DeptRole(MagModel):
name = Column(UnicodeText)
description = Column(UnicodeText)
department_id = Column(UUID, ForeignKey('department.id'))
dept_memberships = relationship(
'DeptMembership',
backref='dept_roles',
cascade='save-update,merge,refresh-expire,expunge',
secondary='dept_membership_dept_role')
__table_args__ = (
UniqueConstraint('name', 'department_id'),
Index('ix_dept_role_department_id', 'department_id'),
)
@hybrid_property
def dept_membership_count(self):
return len(self.dept_memberships)
@dept_membership_count.expression
def dept_membership_count(cls):
return func.count(cls.dept_memberships)
@classproperty
def _extra_apply_attrs(cls):
return set(['dept_memberships_ids']).union(cls._extra_apply_attrs_restricted)
@property
def dept_memberships_ids(self):
_, ids = self._get_relation_ids('dept_memberships')
return [str(d.id) for d in self.dept_memberships] if ids is None else ids
@dept_memberships_ids.setter
def dept_memberships_ids(self, value):
self._set_relation_ids('dept_memberships', DeptMembership, value)
def create_index(self, columns, name=None):
"""
Create an index to speed up queries on a table. If no ``name`` is given a random name is created.
::
table.create_index(['name', 'country'])
"""
self._check_dropped()
if not name:
sig = abs(hash('||'.join(columns)))
name = 'ix_%s_%s' % (self.table.name, sig)
if name in self.indexes:
return self.indexes[name]
try:
self.database._acquire()
columns = [self.table.c[c] for c in columns]
idx = Index(name, *columns)
idx.create(self.database.engine)
except:
idx = None
finally:
self.database._release()
self.indexes[name] = idx
return idx
def create_index(self, columns, name=None, unique=False):
"""
Create an index to speed up queries on a table.
If no ``name`` is given a random name is created.
::
table.create_index(['name', 'country'])
"""
self._check_dropped()
if not name:
sig = '||'.join(columns)
# This is a work-around for a bug in <=0.6.1 which would create
# indexes based on hash() rather than a proper hash.
key = abs(hash(sig))
name = 'ix_%s_%s' % (self.table.name, key)
if name in self.indexes:
return self.indexes[name]
key = sha1(sig.encode('utf-8')).hexdigest()[:16]
name = 'ix_%s_%s' % (self.table.name, key)
if name in self.indexes:
return self.indexes[name]
try:
self.database._acquire()
columns = [self.table.c[c] for c in columns]
idx = Index(name, *columns, unique=unique)
idx.create(self.database.engine)
except:
idx = None
finally:
self.database._release()
self.indexes[name] = idx
return idx
class Machine(Base):
__tablename__ = 'metrics_machine'
id = Column(Integer, primary_key=True)
machine_id = Column(Unicode(32), nullable=False, unique=True)
image_id = Column(Unicode)
image_product = Column(Unicode, index=True)
image_branch = Column(Unicode, index=True)
image_arch = Column(Unicode, index=True)
image_platform = Column(Unicode, index=True)
image_timestamp = Column(DateTime(timezone=True), index=True)
image_personality = Column(Unicode, index=True)
demo = Column(Boolean, server_default=expression.false())
dualboot = Column(Boolean, server_default=expression.false())
live = Column(Boolean, server_default=expression.false())
location = Column(JSONB)
location_id = Column(Unicode)
location_city = Column(Unicode)
location_state = Column(Unicode)
location_street = Column(Unicode)
location_country = Column(Unicode)
location_facility = Column(Unicode)
__table_args__ = (
Index('ix_metrics_machine_location_id',
location_id,
postgresql_where=(location_id.isnot(None))),
Index('ix_metrics_machine_location_city',
location_city,
postgresql_where=(location_city.isnot(None))),
Index('ix_metrics_machine_location_state',
location_state,
postgresql_where=(location_state.isnot(None))),
Index('ix_metrics_machine_location_street',
location_street,
postgresql_where=(location_street.isnot(None))),
Index('ix_metrics_machine_location_country',
location_country,
postgresql_where=(location_country.isnot(None))),
Index('ix_metrics_machine_location_facility',
location_facility,
postgresql_where=(location_facility.isnot(None))),
)
def define_index(cls, metadata, users):
Index("users_t_idx", users.c.test1, users.c.test2)
Index("users_all_idx", users.c.user_id, users.c.test2, users.c.test1)
multiparts.append(char)
char_left = length - value
else: # TODO(vojta) handle unicode
pass
return multiparts
@classmethod
def get_message_length(cls, message, coding=DEFAULT_NO_COMPRESSION):
"""
:param message: {str} message text
:param coding: {str} coding of the message
:return: {int} length of message depends by chars type
"""
length = 0
if coding == cls.DEFAULT_NO_COMPRESSION:
for char in message:
if is_special_char(char):
length = length + 2
else:
length = length + 1
else: # TODO(vojta) handle unicode
pass
return length
Index('outbox_date', Outbox.sent, Outbox.sendTimeout)
Index('outbox_phone', Outbox.phone)
class Allocation(TimestampMixin, ORMBase, OtherModels):
"""Describes a timespan within which one or many timeslots can be
reserved.
There's an important concept to understand before working with allocations.
The resource uuid of an alloction is not always pointing to the actual
resource.
A resource may in fact be a real resource, or an imaginary resource with
a uuid derived from the real resource. This is a somewhat historical
artifact.
If you need to know which allocations belong to a real resource, the
mirror_of field is what's relevant. The originally created allocation
with the real_resource is also called the master-allocation and it is
the one allocation with mirror_of and resource being equal.
When in doubt look at the managed_* functions of seantis.reservation.db's
Scheduler class.
"""
__tablename__ = 'allocations'
id = Column(types.Integer(), primary_key=True, autoincrement=True)
resource = Column(customtypes.GUID(), nullable=False)
mirror_of = Column(customtypes.GUID(), nullable=False)
group = Column(customtypes.GUID(), nullable=False)
quota = Column(types.Integer(), default=1)
partly_available = Column(types.Boolean(), default=False)
approve_manually = Column(types.Boolean(), default=False)
reservation_quota_limit = Column(
types.Integer(), default=0, nullable=False
)
# The dates are stored without any timzone information (unaware).
# Therefore the times are implicitly stored in the timezone the resource
# resides in.
# This is fine and dandy as long as all resources are in the same timezone.
# If they are not problems arise. So in the future the resource should
# carry a timezone property which is applied to the dates which will then
# be stored in UTC
# => TODO
_start = Column(types.DateTime(), nullable=False)
_end = Column(types.DateTime(), nullable=False)
_raster = Column(types.Integer(), nullable=False)
recurrence_id = Column(types.Integer(),
ForeignKey('recurrences.id',
onupdate='cascade',
ondelete='cascade'))
recurrence = relation('Recurrence', lazy='joined')
__table_args__ = (
Index('mirror_resource_ix', 'mirror_of', 'resource'),
UniqueConstraint('resource', '_start', name='resource_start_ix')
)
def copy(self):
allocation = Allocation()
allocation.resource = self.resource
allocation.mirror_of = self.mirror_of
allocation.group = self.group
allocation.quota = self.quota
allocation.partly_available = self.partly_available
allocation.approve_manually = self.approve_manually
allocation._start = self._start
allocation._end = self._end
allocation._raster = self._raster
allocation.recurrence_id = self.recurrence_id
return allocation
def get_start(self):
return self._start
def set_start(self, start):
self._start = rasterize_start(start, self.raster)
start = property(get_start, set_start)
def get_end(self):
return self._end
def set_end(self, end):
self._end = rasterize_end(end, self.raster)
end = property(get_end, set_end)
def get_raster(self):
return self._raster
def set_raster(self, raster):
# the raster can only be set once!
assert(not self._raster)
self._raster = raster
raster = property(get_raster, set_raster)
@property
def display_start(self):
"""Does nothing but to form a nice pair to display_end."""
return self.start
@property
def display_end(self):
"""Returns the end plus one microsecond (nicer display)."""
return self.end + timedelta(microseconds=1)
@property
def whole_day(self):
"""True if the allocation is a whole-day allocation.
A whole-day allocation is not really special. It's just an allocation
which starts at 0:00 and ends at 24:00 (or 23:59:59'999).
As such it can actually also span multiple days, only hours and minutes
count.
The use of this is to display allocations spanning days differently.
"""
s, e = self.display_start, self.display_end
assert s != e # this can never be, except when caused by cosmic rays
return utils.whole_day(s, e)
def overlaps(self, start, end):
""" Returns true if the current timespan overlaps with the given
start and end date.
"""
start, end = rasterize_span(start, end, self.raster)
return utils.overlaps(start, end, self.start, self.end)
def contains(self, start, end):
""" Returns true if the current timespan contains the given start
and end date.
"""
start, end = rasterize_span(start, end, self.raster)
return self.start <= start and end <= self.end
def free_slots(self, start=None, end=None):
""" Returns the slots which are not yet reserved."""
reserved = [slot.start for slot in self.reserved_slots]
slots = []
for start, end in self.all_slots(start, end):
if not start in reserved:
slots.append((start, end))
return slots
def align_dates(self, start=None, end=None):
""" Aligns the given dates to the start and end date of the
allocation.
"""
start = start or self.start
start = start < self.start and self.start or start
end = end or self.end
end = end > self.end and self.end or end
return start, end
def all_slots(self, start=None, end=None):
""" Returns the slots which exist with this timespan. Reserved or free.
"""
start, end = self.align_dates(start, end)
if self.partly_available:
for start, end in iterate_span(start, end, self.raster):
yield start, end
else:
yield self.start, self.end
def is_available(self, start=None, end=None):
""" Returns true if the given daterange is completely available. """
if not (start and end):
start, end = self.start, self.end
assert(self.overlaps(start, end))
if self.is_blocked(start, end):
return False
reserved = [slot.start for slot in self.reserved_slots]
for start, end in self.all_slots(start, end):
if start in reserved:
return False
return True
def is_blocked(self, start=None, end=None):
if not (start and end):
start, end = self.start, self.end
else:
start, end = utils.as_machine_date(start, end)
BlockedPeriod = self.models.BlockedPeriod
query = self._query_blocked_periods()
query = query.filter(BlockedPeriod.start <= end)
query = query.filter(BlockedPeriod.end >= start)
return query.first() is not None
def _query_blocked_periods(self):
query = Session.query(self.models.BlockedPeriod)
query = query.filter_by(resource=self.resource)
return query
@property
def pending_reservations(self):
""" Returns the pending reservations query for this allocation.
As the pending reservations target the group and not a specific
allocation this function returns the same value for masters and
mirrors.
"""
Reservation = self.models.Reservation
query = Session.query(Reservation.id)
query = query.filter(Reservation.target == self.group)
query = query.filter(Reservation.status == u'pending')
return query
@property
def waitinglist_length(self):
return self.pending_reservations.count()
@property
def availability(self):
"""Returns the availability in percent."""
if self.partly_available:
total = sum(1 for s in self.all_slots())
else:
total = 1
count = len(self.reserved_slots)
for blocked_period in self._query_blocked_periods():
count += len(list(iterate_span(blocked_period.start,
blocked_period.end,
self.raster)))
if total == count:
return 0.0
if count == 0:
return 100.0
return 100.0 - (float(count) / float(total) * 100.0)
@property
def in_group(self):
"""True if the event is in any group."""
query = Session.query(Allocation.id)
query = query.filter(Allocation.resource == self.resource)
query = query.filter(Allocation.group == self.group)
query = query.limit(2)
return len(query.all()) > 1
@property
def in_recurrence(self):
"""True if the event is attached to a recurrence."""
return self.recurrence_id is not None
@property
def is_separate(self):
"""True if available separately (as opposed to available only as
part of a group)."""
if self.partly_available:
return True
if self.in_group:
return False
return True
def availability_partitions(self, scheduler):
"""Partitions the space between start and end into blocks of either
free, blocked or reserved time. Each block has a percentage
representing the space the block occupies compared to the size of the
whole allocation.
The blocks are ordered from start to end. Each block is an item with
two values. The first being the percentage, the second being the type.
The type can be one of None, 'reserved' or 'blocked'.
So given an allocation that goes from 8 to 9 and a reservation that
goes from 8:15 until 8:30 and a block that goes from 8:30 to 9:00
we get the following blocks:
[
(25%, None),
(25%, 'reserved'),
(50%, 'blocked')
]
This is useful to divide an allocation block into different divs on the
frontend, indicating to the user which parts of an allocation are
available for reservation.
Makes sure to only display slots that are within it's resources
first_hour/last_hour timespan.
"""
resource = get_resource_by_uuid(scheduler.uuid).getObject()
min_start_resource = datetime.combine(self.start,
time(resource.first_hour))
max_end_resource = datetime.combine(self.end,
time(resource.last_hour))
display_start = max(min_start_resource, self.start)
display_end = min(max_end_resource, self.end)
reserved = dict((r.start, r) for r in self.reserved_slots if
r.start >= display_start and r.end <= display_end)
blocked = set()
for blocked_period in self._query_blocked_periods():
blocked.update(start for start, end in
iterate_span(max(blocked_period.start,
display_start),
min(blocked_period.end,
display_end),
self.raster))
if not (reserved or blocked):
return [(100.0, None)]
# Get the percentage one slot represents
slots = list(self.all_slots(display_start, display_end))
step = 100.0 / float(len(slots))
# Create an entry for each slot with either True or False
pieces = []
for slot in slots:
piece = None
if slot[0] in reserved:
reserved_slot = reserved[slot[0]]
token = reserved_slot.reservation_token
reservation = scheduler.reservation_by_token(token).one()
piece = ('reserved', reservation.description, reservation.id)
elif slot[0] in blocked:
piece = ('blocked', None)
pieces.append(piece)
# Group by the None/'reserved'/'blocked' values in the pieces and sum
# up the percentage
partitions = []
for flag, group in groupby(pieces, key=lambda p: p):
percentage = len(list(group)) * step
partitions.append([percentage, flag])
# Make sure to get rid of floating point rounding errors
total = sum([p[0] for p in partitions])
diff = 100.0 - total
partitions[-1:][0][0] -= diff
return partitions
@property
def is_transient(self):
"""True if the allocation does not exist in the database, and is not
about to be written to the database. If an allocation is transient it
means that the given instance only exists in memory.
See:
http://www.sqlalchemy.org/docs/orm/session.html
#quickie-intro-to-object-states
http://stackoverflow.com/questions/3885601/
sqlalchemy-get-object-instance-state
"""
return object_session(self) is None and not has_identity(self)
@property
def is_master(self):
"""True if the allocation is a master allocation."""
return self.resource == self.mirror_of
def siblings(self, imaginary=True):
"""Returns the master/mirrors group this allocation is part of.
If 'imaginary' is true, inexistant mirrors are created on the fly.
those mirrors are transient (see self.is_transient)
"""
# this function should always have itself in the result
if not imaginary and self.is_transient:
assert False, \
'the resulting list would not contain this allocation'
if self.quota == 1:
assert(self.is_master)
return [self]
query = Session.query(Allocation)
query = query.filter(Allocation.mirror_of == self.mirror_of)
query = query.filter(Allocation._start == self._start)
existing = dict(((e.resource, e) for e in query))
master = self.is_master and self or existing[self.mirror_of]
existing[master.resource] = master
uuids = utils.generate_uuids(master.resource, master.quota)
imaginary = imaginary and (master.quota - len(existing)) or 0
siblings = [master]
for uuid in uuids:
if uuid in existing:
siblings.append(existing[uuid])
elif imaginary > 0:
allocation = master.copy()
allocation.resource = uuid
siblings.append(allocation)
imaginary -= 1
return siblings
def test_noconstraints_table(metadata):
simple_items = Table('simple_items', metadata, Column('number', INTEGER),
CheckConstraint('number > 2'))
simple_items.indexes.add(Index('idx_number', simple_items.c.number))
assert generate_code(metadata, noconstraints=True) == """\
class Identifier(db.Model, Timestamp):
"""Identifier model."""
__tablename__ = 'identifier'
__table_args__ = (
UniqueConstraint('value', 'scheme',
name='uq_identifier_value_scheme'),
Index('ix_identifier_value', 'value'),
Index('ix_identifier_scheme', 'scheme'),
)
id = Column(UUIDType, default=uuid.uuid4, primary_key=True)
value = Column(String)
scheme = Column(String)
def __repr__(self):
"""String representation of the Identifier."""
return "<{self.scheme}: {self.value}>".format(self=self)
@classmethod
def get(cls, value=None, scheme=None, **kwargs):
"""Get the identifier from the database."""
return cls.query.filter_by(
value=value, scheme=scheme).one_or_none()
def fetch_or_create_id(self):
"""Fetches from the database or creates an id for the identifier."""
if not self.id:
obj = self.get(self.value, self.scheme)
if obj:
self = obj
else:
self.id = uuid.uuid4()
return self
def _get_related(self, condition, relationship):
cond = condition & (Relationship.relation == relationship)
return Relationship.query.filter(cond)
def _get_identities(self, as_relation=False):
"""Get the first-layer of 'Identical' Identifies."""
cond = ((Relationship.source == self) | (Relationship.target == self))
q = self._get_related(cond, Relation.IsIdenticalTo)
if as_relation:
return q.all()
else:
siblings = set(sum([[item.source, item.target] for item in q], []))
if siblings:
return list(siblings)
else:
return [self, ]
def get_identities(self):
"""Get the fully-expanded list of 'Identical' Identifiers."""
ids = next_ids = set([self])
while next_ids:
grp = set(sum([item._get_identities() for item in next_ids], []))
next_ids = grp - ids
ids |= grp
return list(ids)
def get_parents(self, rel_type, as_relation=False):
"""Get all parents of given Identifier for given relation."""
q = self._get_related((Relationship.target == self), rel_type)
if as_relation:
return q.all()
else:
return [item.source for item in q]
def get_children(self, rel_type, as_relation=False):
"""Get all children of given Identifier for given relation."""
q = self._get_related((Relationship.source == self), rel_type)
if as_relation:
return q.all()
else:
return [item.target for item in q]
@property
def identity_group(self):
"""Get the identity group the identifier belongs to."""
return next((id2g.group for id2g in self.id2groups
if id2g.group.type == GroupType.Identity), None)
@property
def data(self):
"""Get the metadata of the identity group the identifier belongs to."""
if self.identity_group and self.identity_group.data:
return self.identity_group.data.json
class Relationship(db.Model, Timestamp):
"""Relationship between two identifiers."""
__tablename__ = 'relationship'
__table_args__ = (
UniqueConstraint('source_id', 'target_id', 'relation',
name='uq_relationship_source_target_relation'),
Index('ix_relationship_source', 'source_id'),
Index('ix_relationship_target', 'target_id'),
Index('ix_relationship_relation', 'relation'),
)
id = Column(UUIDType, default=uuid.uuid4, primary_key=True)
source_id = Column(UUIDType,
ForeignKey(Identifier.id, onupdate='CASCADE',
ondelete='CASCADE',
name='fk_relationship_source'),
nullable=False)
target_id = Column(UUIDType, ForeignKey(Identifier.id, onupdate='CASCADE',
ondelete='CASCADE',
name='fk_relationship_target'),
nullable=False)
relation = Column(Enum(Relation))
source = orm_relationship(Identifier, foreign_keys=[source_id],
backref='sources')
target = orm_relationship(Identifier, foreign_keys=[target_id],
backref='targets')
@classmethod
def get(cls, source, target, relation, **kwargs):
"""Get the relationship from the database."""
return cls.query.filter_by(
source_id=source.id, target_id=target.id,
relation=relation).one_or_none()
def fetch_or_create_id(self):
"""Fetches from the database or creates an id for the relationship."""
self.source = self.source.fetch_or_create_id()
self.target = self.target.fetch_or_create_id()
if not self.id:
obj = self.get(self.source, self.target, self.relation)
if obj:
self = obj
else:
self.id = uuid.uuid4()
return self
@property
def identity_group(self):
"""Get the relationship's identity group."""
return GroupRelationship.query.filter_by(
source=self.source.identity_group,
target=self.target.identity_group,
relation=self.relation,
type=GroupType.Identity).one_or_none()
@property
def data(self):
"""Get the relationship's identity group metadata."""
if self.identity_group and self.identity_group.data:
return self.identity_group.data.json
def __repr__(self):
"""String representation of the relationship."""
return (
'<{self.source.value} {self.relation.name} '
'{self.target.value}>'.format(self=self)
)
#replication/copying outside of powerdns). Add support via AXFR.
#Change fields to NOT NULL.
master = Column(String(255), nullable=True)
last_check = Column(Integer, nullable=True)
type = Column(String(6), nullable=True)
notified_serial = Column(Integer, nullable=True)
account = Column(String(40), nullable=True)
class Records(BASE, PowerDNSBase):
__tablename__ = 'records'
id = Column(Integer, primary_key=True, autoincrement=True)
domain_id = Column(Integer, nullable=False, index=True)
name = Column(String(255), nullable=False, index=True)
type = Column(String(6), nullable=False)
content = Column(String(255))
ttl = Column(Integer)
prio = Column(Integer)
change_date = Column(Integer)
Index('nametype_index', Records.name, Records.type, unique=True)
def register_models():
"""Register Models and create metadata."""
models = (Domains, Records)
engine = get_engine()
for model in models:
model.metadata.create_all(engine)
LONGVARCHAR = VARCHAR(length=CHAR_MAX_LEN)
SHA1 = BINARY(length=SHA1_LEN)
# Table metadata
metadata = MetaData(bind=mysql_engine)
# All known strace executables and arguments
executables = Table(
'executables',
metadata,
Column('id', INTEGER, primary_key=True),
Column('system', LONGVARCHAR, nullable=False),
Column('executable', LONGVARCHAR, nullable=False),
Column('arguments_hash', SHA1, nullable=False),
Column('arguments', JSON, nullable=False),
Index('system', 'executable', 'arguments_hash'),
)
# Raw straces
straces = Table(
'straces',
metadata,
Column('id', INTEGER, primary_key=True),
Column('executable',
None,
ForeignKey(executables.c.id, onupdate='CASCADE'),
nullable=False),
Column('collector', LONGVARCHAR, nullable=False),
Column('collector_assigned_id', LONGVARCHAR, nullable=False),
Column('strace', LONGTEXT, nullable=False),
Column('metadata', JSON, nullable=False),
class CobraResults(db.Model):
"""
Results for task
"""
__tablename__ = 'results'
id = db.Column(INTEGER(unsigned=True),
primary_key=True,
autoincrement=True,
nullable=False)
task_id = db.Column(INTEGER, nullable=False, default=None)
project_id = db.Column(INTEGER, nullable=False, default=None)
rule_id = db.Column(INTEGER, nullable=False, default=None)
file = db.Column(db.String(512), nullable=False, default=None)
line = db.Column(INTEGER(11), nullable=False, default=None)
code = db.Column(db.String(512), nullable=False, default=None)
repair = db.Column(INTEGER(6), nullable=False, default=None)
"""
status: description
0: Vulnerability after initializing state
1: Has been pushed to a third-party vulnerability management platform
2: Already fixed
"""
status = db.Column(TINYINT, default=None, nullable=False)
created_at = db.Column(db.DateTime, nullable=False, default=None)
updated_at = db.Column(db.DateTime, nullable=False, default=None)
__table_args__ = (Index('ix_task_id_rule_id', task_id, rule_id), {
"mysql_charset": "utf8mb4"
})
def __init__(self,
task_id,
project_id,
rule_id,
file_path,
line,
code,
repair,
status,
created_at=None,
updated_at=None):
self.task_id = task_id
self.project_id = project_id
self.rule_id = rule_id
self.file = file_path
self.line = line
self.code = code
self.repair = repair
self.status = status
self.created_at = created_at
self.updated_at = updated_at
current_time = time.strftime('%Y-%m-%d %X', time.localtime())
if created_at is None:
self.created_at = current_time
else:
self.created_at = created_at
if updated_at is None:
self.updated_at = current_time
else:
self.updated_at = updated_at
def __repr__(self):
return "<CobraResults %r - %r>" % (self.id, self.task_id)
@staticmethod
def count_by_time(start, end):
count = db.session.query(
func.count(CobraResults.id).label('count'),
CobraResults.status).filter(
CobraResults.created_at >=
'{start} 00:00:00'.format(start=start),
CobraResults.created_at <= '{end} 23:59:59'.format(end=end),
# Active project
CobraProjects.status > 0,
CobraResults.project_id == CobraProjects.id).group_by(
CobraResults.status).all()
logging.debug('VT {start} {end} {count}'.format(start=start,
end=end,
count=count))
c_dict = {}
for ci in count:
count, status = ci
c_dict[status] = count
if 0 not in c_dict:
c_dict[0] = 0
if 1 not in c_dict:
c_dict[1] = 0
if 2 not in c_dict:
c_dict[2] = 0
return c_dict
imdb_id = Column(String, index=True)
quality = Column(String)
proper_count = Column(Integer)
added = Column(DateTime)
def __init__(self):
self.added = datetime.now()
def __repr__(self):
return '<ProperMovie(title=%s,task=%s,imdb_id=%s,quality=%s,proper_count=%s,added=%s)>' % \
(self.title, self.task, self.imdb_id, self.quality, self.proper_count, self.added)
# create index
columns = Base.metadata.tables['proper_movies'].c
Index('proper_movies_imdb_id_quality_proper', columns.imdb_id, columns.quality,
columns.proper_count)
class FilterProperMovies(object):
"""
Automatically download proper movies.
Configuration:
proper_movies: n <minutes|hours|days|weeks>
or permanently:
proper_movies: yes
Value no will disable plugin.
def define_reflected_tables(cls, metadata, schema):
if schema:
schema_prefix = schema + "."
else:
schema_prefix = ""
if testing.requires.self_referential_foreign_keys.enabled:
users = Table(
'users',
metadata,
Column('user_id', sa.INT, primary_key=True),
Column('test1', sa.CHAR(5), nullable=False),
Column('test2', sa.Float(5), nullable=False),
Column(
'parent_user_id', sa.Integer,
sa.ForeignKey('%susers.user_id' % schema_prefix,
name='user_id_fk')),
schema=schema,
test_needs_fk=True,
)
else:
users = Table(
'users',
metadata,
Column('user_id', sa.INT, primary_key=True),
Column('test1', sa.CHAR(5), nullable=False),
Column('test2', sa.Float(5), nullable=False),
schema=schema,
test_needs_fk=True,
)
Table(
"dingalings",
metadata,
Column('dingaling_id', sa.Integer, primary_key=True),
Column(
'address_id', sa.Integer,
sa.ForeignKey('%semail_addresses.address_id' % schema_prefix)),
Column('data', sa.String(30)),
schema=schema,
test_needs_fk=True,
)
Table(
'email_addresses',
metadata,
Column('address_id', sa.Integer),
Column('remote_user_id', sa.Integer,
sa.ForeignKey(users.c.user_id)),
Column('email_address', sa.String(20)),
sa.PrimaryKeyConstraint('address_id', name='email_ad_pk'),
schema=schema,
test_needs_fk=True,
)
Table('comment_test',
metadata,
Column('id', sa.Integer, primary_key=True, comment='id comment'),
Column('data', sa.String(20), comment='data % comment'),
Column('d2',
sa.String(20),
comment=r"""Comment types type speedily ' " \ '' Fun!"""),
schema=schema,
comment=r"""the test % ' " \ table comment""")
if testing.requires.index_reflection.enabled:
cls.define_index(metadata, users)
if not schema:
noncol_idx_test_nopk = Table(
'noncol_idx_test_nopk',
metadata,
Column('q', sa.String(5)),
)
noncol_idx_test_pk = Table(
'noncol_idx_test_pk',
metadata,
Column('id', sa.Integer, primary_key=True),
Column('q', sa.String(5)),
)
Index('noncol_idx_nopk', noncol_idx_test_nopk.c.q.desc())
Index('noncol_idx_pk', noncol_idx_test_pk.c.q.desc())
if testing.requires.view_column_reflection.enabled:
cls.define_views(metadata, schema)
if not schema and testing.requires.temp_table_reflection.enabled:
cls.define_temp_tables(metadata)
timestamp = Column(DateTime)
monit_state = Column(Integer)
endtime = Column(DateTime)
period = Column(Integer)
description = Column(Unicode(256))
def __init__(self,
entity_id,
state,
monit_state,
timestamp,
description,
endtime=None,
period=None):
self.id = getHexID()
self.entity_id = entity_id
self.state = state
self.timestamp = timestamp
self.description = description
self.monit_state = monit_state
self.endtime = endtime
self.period = period
def __repr__(self):
return 'Availability State of %s at time %s was %s' % \
(self.entity_id, self.timestamp, self.state)
Index("ah_eid_st_time", AvailHistory.entity_id, AvailHistory.state,
AvailHistory.timestamp)
class JobStep(db.Model):
"""
The most granular unit of work; run on a particular node, has a status and
a result.
But Hark! There's a hack that allows jobstep, once its run, to rewrite
history to say that it was actually multiple jobsteps (even organized into
separate job phases.) It does this by creating an artifact, which the
python code picks up and then retroactively alters the db to say that this
jobstep had multiple steps (I think it purely appends new jobsteps after
the original.) xplat uses this to very nicely display the different parts
of their jobstep.
"""
# TODO(dcramer): make duration a column
__tablename__ = 'jobstep'
__table_args__ = (
Index('idx_jobstep_status', 'status'),
)
id = Column(GUID, primary_key=True, default=uuid.uuid4)
job_id = Column(GUID, ForeignKey('job.id', ondelete="CASCADE"), nullable=False)
phase_id = Column(GUID, ForeignKey('jobphase.id', ondelete="CASCADE"), nullable=False)
project_id = Column(GUID, ForeignKey('project.id', ondelete="CASCADE"), nullable=False)
label = Column(String(128), nullable=False)
status = Column(Enum(Status), nullable=False, default=Status.unknown)
result = Column(Enum(Result), nullable=False, default=Result.unknown)
node_id = Column(GUID, ForeignKey('node.id', ondelete="CASCADE"))
date_started = Column(DateTime)
date_finished = Column(DateTime)
date_created = Column(DateTime, default=datetime.utcnow)
last_heartbeat = Column(DateTime)
data = Column(JSONEncodedDict)
job = relationship('Job')
project = relationship('Project')
node = relationship('Node')
phase = relationship('JobPhase', backref=backref('steps', order_by='JobStep.date_started'))
__repr__ = model_repr('label')
def __init__(self, **kwargs):
super(JobStep, self).__init__(**kwargs)
if self.id is None:
self.id = uuid.uuid4()
if self.result is None:
self.result = Result.unknown
if self.status is None:
self.status = Status.unknown
if self.date_created is None:
self.date_created = datetime.utcnow()
if self.data is None:
self.data = {}
@property
def duration(self):
"""
Return the duration (in milliseconds) that this item was in-progress.
"""
if self.date_started and self.date_finished:
duration = (self.date_finished - self.date_started).total_seconds() * 1000
else:
duration = None
return duration