def downgrade(migrate_engine):
meta.bind = migrate_engine
records_table = Table("records", meta, autoload=True)
name_idx = Index("rec_name_index", records_table.c.name)
name_idx.create()
def upgrade(migrate_engine):
metadata.bind = migrate_engine
print(__doc__)
metadata.reflect()
try:
if migrate_engine.name == 'mysql':
# Strip slug index prior to creation so we can do it manually.
slug_index = None
for ix in Page_table.indexes:
if ix.name == 'ix_page_slug':
slug_index = ix
Page_table.indexes.remove(slug_index)
Page_table.create()
if migrate_engine.name == 'mysql':
# Create slug index manually afterward.
i = Index("ix_page_slug", Page_table.c.slug, mysql_length=200)
i.create()
except Exception:
log.exception("Could not create page table")
try:
PageRevision_table.create()
except Exception:
log.exception("Could not create page_revision table")
# Add 1 column to the user table
User_table = Table("galaxy_user", metadata, autoload=True)
col = Column('username', String(255), index=True, unique=True, default=False)
col.create(User_table, index_name='ix_user_username', unique_name='username')
assert col is User_table.c.username
def upgrade(migrate_engine):
display_migration_details()
metadata.bind = migrate_engine
db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, autoload=True )
# Load existing tables
metadata.reflect()
# Add 2 indexes to the galaxy_user table
i = Index( 'ix_hda_extension', HistoryDatasetAssociation_table.c.extension )
try:
i.create()
except Exception as e:
log.debug( "Adding index 'ix_hda_extension' to history_dataset_association table failed: %s" % ( str( e ) ) )
# Set the default data in the galaxy_user table, but only for null values
cmd = "UPDATE history_dataset_association SET extension = 'qual454' WHERE extension = 'qual' and peek like \'>%%\'"
try:
db_session.execute( cmd )
except Exception as e:
log.debug( "Resetting extension qual to qual454 in history_dataset_association failed: %s" % ( str( e ) ) )
cmd = "UPDATE history_dataset_association SET extension = 'qualsolexa' WHERE extension = 'qual' and peek not like \'>%%\'"
try:
db_session.execute( cmd )
except Exception as e:
log.debug( "Resetting extension qual to qualsolexa in history_dataset_association failed: %s" % ( str( e ) ) )
# Add 1 index to the history_dataset_association table
try:
i.drop()
except Exception as e:
log.debug( "Dropping index 'ix_hda_extension' to history_dataset_association table failed: %s" % ( str( e ) ) )
def upgrade(migrate_engine):
meta.bind = migrate_engine
records_table = Table("records", meta, autoload=True)
index = Index("designate_recordset_id", records_table.c.designate_recordset_id)
index.create()
def downgrade(migrate_engine):
meta.bind = migrate_engine
records_table = Table('records', meta, autoload=True)
index = Index('designate_recordset_id',
records_table.c.designate_recordset_id)
index.drop()
def loadTable(datapath, datatable, delimiter, dtype, engine, indexCols=[], skipLines=1, chunkSize=100000, **kwargs):
cnt = 0
with open(datapath) as fh:
while cnt < skipLines:
fh.readline()
cnt += 1
cnt = 0
tmpstr = ''
for l in fh:
tmpstr += l
cnt += 1
if cnt%chunkSize == 0:
print "Loading chunk #%i"%(int(cnt/chunkSize))
dataArr = numpy.genfromtxt(StringIO(tmpstr), dtype=dtype, delimiter=delimiter, **kwargs)
engine.execute(datatable.insert(), [dict((name, numpy.asscalar(l[name])) for name in l.dtype.names) for l in dataArr])
tmpstr = ''
#Clean up the last chunk
if len(tmpstr) > 0:
dataArr = numpy.genfromtxt(StringIO(tmpstr), dtype=dtype, delimiter=delimiter, **kwargs)
try:
engine.execute(datatable.insert(), [dict((name, numpy.asscalar(l[name])) for name in l.dtype.names) for l in dataArr])
# If the file only has one line, the result of genfromtxt is a 0-d array, so cannot be iterated
except TypeError:
engine.execute(datatable.insert(), [dict((name, numpy.asscalar(dataArr[name])) for name in dataArr.dtype.names),])
for col in indexCols:
if hasattr(col, "__iter__"):
print "Creating index on %s"%(",".join(col))
colArr = (datatable.c[c] for c in col)
i = Index('%sidx'%''.join(col), *colArr)
else:
print "Creating index on %s"%(col)
i = Index('%sidx'%col, datatable.c[col])
i.create(engine)
def upgrade(migrate_engine):
print(__doc__)
metadata.bind = migrate_engine
HistoryDatasetAssociation_table = Table("history_dataset_association", metadata, autoload=True)
# Load existing tables
metadata.reflect()
# Add 2 indexes to the galaxy_user table
i = Index('ix_hda_extension', HistoryDatasetAssociation_table.c.extension)
try:
i.create()
except Exception:
log.exception("Adding index 'ix_hda_extension' to history_dataset_association table failed.")
# Set the default data in the galaxy_user table, but only for null values
cmd = "UPDATE history_dataset_association SET extension = 'qual454' WHERE extension = 'qual' and peek like \'>%%\'"
try:
migrate_engine.execute(cmd)
except Exception:
log.exception("Resetting extension qual to qual454 in history_dataset_association failed.")
cmd = "UPDATE history_dataset_association SET extension = 'qualsolexa' WHERE extension = 'qual' and peek not like \'>%%\'"
try:
migrate_engine.execute(cmd)
except Exception:
log.exception("Resetting extension qual to qualsolexa in history_dataset_association failed.")
# Add 1 index to the history_dataset_association table
try:
i.drop()
except Exception:
log.exception("Dropping index 'ix_hda_extension' to history_dataset_association table failed.")
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
t = Table("aggregate_metadata", meta, autoload=True)
i = Index("aggregate_metadata_key_idx", t.c.key)
i.drop(migrate_engine)
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
build_requests = Table('build_requests', meta, autoload=True)
columns_to_add = [
('instance_uuid',
Column('instance_uuid', String(length=36))),
('instance',
Column('instance', Text())),
]
for (col_name, column) in columns_to_add:
if not hasattr(build_requests.c, col_name):
build_requests.create_column(column)
for index in build_requests.indexes:
if [c.name for c in index.columns] == ['instance_uuid']:
break
else:
index = Index('build_requests_instance_uuid_idx',
build_requests.c.instance_uuid)
index.create()
inspector = reflection.Inspector.from_engine(migrate_engine)
constrs = inspector.get_unique_constraints('build_requests')
constr_names = [constr['name'] for constr in constrs]
if 'uniq_build_requests0instance_uuid' not in constr_names:
UniqueConstraint('instance_uuid', table=build_requests,
name='uniq_build_requests0instance_uuid').create()
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
# Drop the library_folder_id column
try:
Job_table = Table( "job", metadata, autoload=True )
except NoSuchTableError:
Job_table = None
log.debug( "Failed loading table job" )
if Job_table is not None:
try:
col = Job_table.c.library_folder_id
col.drop()
except Exception as e:
log.debug( "Dropping column 'library_folder_id' from job table failed: %s" % ( str( e ) ) )
# Drop the job_to_output_library_dataset table
try:
JobToOutputLibraryDatasetAssociation_table.drop()
except Exception as e:
print(str(e))
log.debug( "Dropping job_to_output_library_dataset table failed: %s" % str( e ) )
# Drop the ix_dataset_state index
try:
Dataset_table = Table( "dataset", metadata, autoload=True )
except NoSuchTableError:
Dataset_table = None
log.debug( "Failed loading table dataset" )
i = Index( "ix_dataset_state", Dataset_table.c.state )
try:
i.drop()
except Exception as e:
print(str(e))
log.debug( "Dropping index 'ix_dataset_state' from dataset table failed: %s" % str( e ) )
def _mk_index_on(engine, ts_name):
fc_table = introspect_table(engine, "{}_RegressionIndicator".format(ts_name))
fast_fc_lookup = Index('idx_fast_ri_lookup', fc_table.c.RegressionID)
try:
fast_fc_lookup.create(engine)
except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.ProgrammingError) as e:
logger.warning("Skipping index creation on {}, because of {}".format(fc_table.name, e.message))
def downgrade(migrate_engine):
metadata.bind = migrate_engine
metadata.reflect()
i = Index("ix_hda_ta_history_dataset_association_id", HistoryDatasetAssociationTagAssociation_table.c.history_dataset_association_id)
try:
i.drop()
except Exception:
log.exception("Removing index 'ix_hdata_history_dataset_association_id' to table 'history_dataset_association_tag_association' table failed.")
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
t = Table('instance_faults', meta, autoload=True)
i = Index('instance_faults_instance_uuid_deleted_created_at_idx',
t.c.instance_uuid, t.c.deleted, t.c.created_at)
i.drop(migrate_engine)
def upgrade(migrate_engine):
"""Add an index to make the scheduler lookups of compute_nodes and joined
compute_node_stats more efficient.
"""
meta = MetaData(bind=migrate_engine)
cn_stats = Table(TABLE_NAME, meta, autoload=True)
idx = Index(IDX_NAME, cn_stats.c.compute_node_id, cn_stats.c.deleted)
idx.create(migrate_engine)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
t = Table('instance_type_extra_specs', meta, autoload=True)
i = Index('instance_type_extra_specs_instance_type_id_key_idx',
t.c.instance_type_id, t.c.key)
i.drop(migrate_engine)
def _create_indexes(self):
# create an index on timeseries values if it doesn't exist
try:
i = Index('ix_timeseries_values_id',
model.DataValue.__table__.c.timeseries_id)
i.create(self.engine)
except OperationalError:
pass
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
t = Table('bw_usage_cache', meta, autoload=True)
i = Index('bw_usage_cache_uuid_start_period_idx',
t.c.uuid, t.c.start_period)
i.drop(migrate_engine)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
t = Table('migrations', meta, autoload=True)
i = Index('migrations_instance_uuid_and_status_idx', t.c.deleted,
t.c.instance_uuid, t.c.status)
i.drop(migrate_engine)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
jobs = Table('jobs', meta, autoload=True)
index = Index(INDEX_NAME, jobs.c.hard_timeout)
index.drop(migrate_engine)
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
images = Table('images', meta, autoload=True)
index = Index(INDEX_NAME, images.c.checksum)
index.create(migrate_engine)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
images = Table('images', meta, autoload=True)
index = Index(INDEX_NAME, images.c.owner)
index.drop(migrate_engine)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
reservations = Table('reservations', meta, autoload=True)
index = Index('reservations_uuid_idx', reservations.c.uuid)
index.drop(migrate_engine)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
t = Table('agent_builds', meta, autoload=True)
i = Index('agent_builds_hypervisor_os_arch_idx',
t.c.hypervisor, t.c.os, t.c.architecture)
i.drop(migrate_engine)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
t = Table('dns_domains', meta, autoload=True)
i = Index('dns_domains_domain_deleted_idx',
t.c.domain, t.c.deleted)
i.drop(migrate_engine)
def _mk_index_on(engine, ts_name):
fc_table = introspect_table(engine, "{}_FieldChangeV2".format(ts_name))
fast_fc_lookup = Index('idx_fast_fieldchange_lookup', fc_table.c.StartOrderID)
try:
fast_fc_lookup.create(engine)
except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.ProgrammingError) as e:
logger.warning("Skipping index creation on {}, because of {}".format(fc_table.name, e.message))
def upgrade(migrate_engine):
table, index = _get_table_index(migrate_engine)
if index:
LOG.info('Skipped adding compute_nodes_uuid_idx because an '
'equivalent index already exists.')
return
index = Index('compute_nodes_uuid_idx', table.c.uuid, unique=True)
index.create(migrate_engine)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
schedules = Table('schedules', meta, autoload=True)
index = Index(INDEX_NAME, schedules.c.next_run)
index.drop(migrate_engine)
def downgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
t = Table('key_pairs', meta, autoload=True)
i = Index('key_pair_user_id_name_idx',
t.c.user_id, t.c.name)
i.drop(migrate_engine)
def upgrade(migrate_engine):
meta = MetaData(bind=migrate_engine)
load_tables = dict((table_name, Table(table_name, meta, autoload=True))
for table_name in INDEXES.keys())
for table_name, indexes in INDEXES.items():
table = load_tables[table_name]
for index_name, column in indexes:
index = Index(index_name, table.c[column])
index.drop()
def upgrade(migrate_engine):
meta, table, index = _get_table_index(migrate_engine)
if index:
LOG.info('Skipped adding %s because an equivalent index'
' already exists.', INDEX_NAME)
return
columns = [getattr(table.c, col_name) for col_name in INDEX_COLUMNS]
index = Index(INDEX_NAME, *columns)
index.create(migrate_engine)
class BaseResultORM(Base):
"""
Abstract Base class for ResultORMs and ProcedureORMs
"""
__tablename__ = 'base_result'
# for SQL
result_type = Column(String) # for inheritance
task_id = Column(String) # TODO: not used, for back compatibility
# Base identification
id = Column(Integer, primary_key=True)
hash_index = Column(String) # TODO
procedure = Column(String(100)) # TODO: may remove
# program = Column(String(100)) # moved to subclasses
version = Column(Integer)
# Extra fields
extras = Column(JSON)
stdout = Column(Integer, ForeignKey('kv_store.id'))
stdout_obj = relationship(KVStoreORM,
lazy='noload',
foreign_keys=stdout,
cascade="all, delete-orphan",
single_parent=True)
stderr = Column(Integer, ForeignKey('kv_store.id'))
stderr_obj = relationship(KVStoreORM,
lazy='noload',
foreign_keys=stderr,
cascade="all, delete-orphan",
single_parent=True)
error = Column(Integer, ForeignKey('kv_store.id'))
error_obj = relationship(KVStoreORM,
lazy='noload',
foreign_keys=error,
cascade="all, delete-orphan",
single_parent=True)
# Compute status
status = Column(Enum(RecordStatusEnum), nullable=False, default=RecordStatusEnum.incomplete)
created_on = Column(DateTime, default=datetime.datetime.utcnow)
modified_on = Column(DateTime, default=datetime.datetime.utcnow)
# Carry-ons
provenance = Column(JSON)
__table_args__ = (
Index('ix_base_result_status', 'status'),
Index('ix_base_result_type', 'result_type'), # todo: needed?
)
# meta = {
# # 'allow_inheritance': True,
# 'indexes': ['status']
# }
# def save(self, *args, **kwargs):
# """Override save to set defaults"""
#
# self.modified_on = datetime.datetime.utcnow()
# if not self.created_on:
# self.created_on = datetime.datetime.utcnow()
#
# return super(BaseResultORM, self).save(*args, **kwargs)
__mapper_args__ = {'polymorphic_on': 'result_type'}
class GridOptimizationProcedureORM(ProcedureMixin, BaseResultORM):
__tablename__ = "grid_optimization_procedure"
id = Column(Integer, ForeignKey('base_result.id', ondelete='cascade'), primary_key=True)
def __init__(self, **kwargs):
kwargs.setdefault("version", 1)
kwargs.setdefault("procedure", "gridoptimization")
kwargs.setdefault("program", "qcfractal")
super().__init__(**kwargs)
# Input data
initial_molecule = Column(Integer, ForeignKey('molecule.id'))
initial_molecule_obj = relationship(MoleculeORM, lazy='select', foreign_keys=initial_molecule)
optimization_spec = Column(JSON)
# Output data
starting_molecule = Column(Integer, ForeignKey('molecule.id'))
starting_molecule_obj = relationship(MoleculeORM, lazy='select', foreign_keys=initial_molecule)
final_energy_dict = Column(JSON) # Dict[str, float]
starting_grid = Column(JSON) # tuple
grid_optimizations_obj = relationship(GridOptimizationAssociation,
lazy='selectin',
cascade="all, delete-orphan",
backref="grid_optimization_procedure")
@hybrid_property
def grid_optimizations(self):
"""calculated property when accessed, not saved in the DB
A view of the many to many relation in the form of a dict"""
ret = {}
try:
for obj in self.grid_optimizations_obj:
ret[obj.key] = str(obj.opt_id)
except Exception as err:
# raises exception of first access!!
print(err)
return ret
@grid_optimizations.setter
def grid_optimizations(self, dict_values):
return dict_values
__table_args__ = (
Index('ix_grid_optmization_program', 'program'), # todo: needed for procedures?
)
__mapper_args__ = {
'polymorphic_identity': 'grid_optimization_procedure',
# to have separate select when querying BaseResultsORM
'polymorphic_load': 'selectin',
}
def update_relations(self, grid_optimizations=None, **kwarg):
self.grid_optimizations_obj = []
for key, opt_id in grid_optimizations.items():
obj = GridOptimizationAssociation(grid_opt_id=int(self.id), opt_id=int(opt_id), key=key)
self.grid_optimizations_obj.append(obj)
class DagRun(Base, LoggingMixin):
"""
DagRun describes an instance of a Dag. It can be created
by the scheduler (for regular runs) or by an external trigger
"""
__tablename__ = "dag_run"
ID_PREFIX = 'scheduled__'
ID_FORMAT_PREFIX = ID_PREFIX + '{0}'
id = Column(Integer, primary_key=True)
dag_id = Column(String(ID_LEN))
execution_date = Column(UtcDateTime, default=timezone.utcnow)
start_date = Column(UtcDateTime, default=timezone.utcnow)
end_date = Column(UtcDateTime)
_state = Column('state', String(50), default=State.RUNNING)
run_id = Column(String(ID_LEN))
external_trigger = Column(Boolean, default=True)
conf = Column(PickleType)
dag = None
__table_args__ = (
Index('dag_id_state', dag_id, _state),
UniqueConstraint('dag_id', 'execution_date'),
UniqueConstraint('dag_id', 'run_id'),
)
def __repr__(self):
return ('<DagRun {dag_id} @ {execution_date}: {run_id}, '
'externally triggered: {external_trigger}>').format(
dag_id=self.dag_id,
execution_date=self.execution_date,
run_id=self.run_id,
external_trigger=self.external_trigger)
def get_state(self):
return self._state
def set_state(self, state):
if self._state != state:
self._state = state
self.end_date = timezone.utcnow() if self._state in State.finished(
) else None
@declared_attr
def state(self):
return synonym('_state',
descriptor=property(self.get_state, self.set_state))
@classmethod
def id_for_date(cls, date, prefix=ID_FORMAT_PREFIX):
return prefix.format(date.isoformat()[:19])
@provide_session
def refresh_from_db(self, session=None):
"""
Reloads the current dagrun from the database
:param session: database session
"""
DR = DagRun
exec_date = func.cast(self.execution_date, DateTime)
dr = session.query(DR).filter(
DR.dag_id == self.dag_id,
func.cast(DR.execution_date, DateTime) == exec_date,
DR.run_id == self.run_id).one()
self.id = dr.id
self.state = dr.state
@staticmethod
@provide_session
def find(dag_id=None,
run_id=None,
execution_date=None,
state=None,
external_trigger=None,
no_backfills=False,
session=None):
"""
Returns a set of dag runs for the given search criteria.
:param dag_id: the dag_id to find dag runs for
:type dag_id: int, list
:param run_id: defines the the run id for this dag run
:type run_id: str
:param execution_date: the execution date
:type execution_date: datetime.datetime
:param state: the state of the dag run
:type state: str
:param external_trigger: whether this dag run is externally triggered
:type external_trigger: bool
:param no_backfills: return no backfills (True), return all (False).
Defaults to False
:type no_backfills: bool
:param session: database session
:type session: sqlalchemy.orm.session.Session
"""
DR = DagRun
qry = session.query(DR)
if dag_id:
qry = qry.filter(DR.dag_id == dag_id)
if run_id:
qry = qry.filter(DR.run_id == run_id)
if execution_date:
if isinstance(execution_date, list):
qry = qry.filter(DR.execution_date.in_(execution_date))
else:
qry = qry.filter(DR.execution_date == execution_date)
if state:
qry = qry.filter(DR.state == state)
if external_trigger is not None:
qry = qry.filter(DR.external_trigger == external_trigger)
if no_backfills:
# in order to prevent a circular dependency
from airflow.jobs import BackfillJob
qry = qry.filter(DR.run_id.notlike(BackfillJob.ID_PREFIX + '%'))
dr = qry.order_by(DR.execution_date).all()
return dr
@provide_session
def get_task_instances(self, state=None, session=None):
"""
Returns the task instances for this dag run
"""
from airflow.models.taskinstance import TaskInstance # Avoid circular import
tis = session.query(TaskInstance).filter(
TaskInstance.dag_id == self.dag_id,
TaskInstance.execution_date == self.execution_date,
)
if state:
if isinstance(state, six.string_types):
tis = tis.filter(TaskInstance.state == state)
else:
# this is required to deal with NULL values
if None in state:
tis = tis.filter(
or_(TaskInstance.state.in_(state),
TaskInstance.state.is_(None)))
else:
tis = tis.filter(TaskInstance.state.in_(state))
if self.dag and self.dag.partial:
tis = tis.filter(TaskInstance.task_id.in_(self.dag.task_ids))
return tis.all()
@provide_session
def get_task_instance(self, task_id, session=None):
"""
Returns the task instance specified by task_id for this dag run
:param task_id: the task id
"""
from airflow.models.taskinstance import TaskInstance # Avoid circular import
TI = TaskInstance
ti = session.query(TI).filter(TI.dag_id == self.dag_id,
TI.execution_date == self.execution_date,
TI.task_id == task_id).first()
return ti
def get_dag(self):
"""
Returns the Dag associated with this DagRun.
:return: DAG
"""
if not self.dag:
raise AirflowException(
"The DAG (.dag) for {} needs to be set".format(self))
return self.dag
@provide_session
def get_previous_dagrun(self, state=None, session=None):
# type: (Optional[str], Optional[Session]) -> Optional['DagRun']
"""The previous DagRun, if there is one"""
session = cast(Session, session) # mypy
filters = [
DagRun.dag_id == self.dag_id,
DagRun.execution_date < self.execution_date,
]
if state is not None:
filters.append(DagRun.state == state)
return session.query(DagRun).filter(*filters).order_by(
DagRun.execution_date.desc()).first()
@provide_session
def get_previous_scheduled_dagrun(self, session=None):
"""The previous, SCHEDULED DagRun, if there is one"""
dag = self.get_dag()
return session.query(DagRun).filter(
DagRun.dag_id == self.dag_id,
DagRun.execution_date == dag.previous_schedule(
self.execution_date)).first()
@provide_session
def update_state(self, session=None):
"""
Determines the overall state of the DagRun based on the state
of its TaskInstances.
:return: State
"""
dag = self.get_dag()
tis = self.get_task_instances(session=session)
self.log.debug("Updating state for %s considering %s task(s)", self,
len(tis))
for ti in list(tis):
# skip in db?
if ti.state == State.REMOVED:
tis.remove(ti)
else:
ti.task = dag.get_task(ti.task_id)
# pre-calculate
# db is faster
start_dttm = timezone.utcnow()
unfinished_tasks = self.get_task_instances(state=State.unfinished(),
session=session)
none_depends_on_past = all(not t.task.depends_on_past
for t in unfinished_tasks)
none_task_concurrency = all(t.task.task_concurrency is None
for t in unfinished_tasks)
# small speed up
if unfinished_tasks and none_depends_on_past and none_task_concurrency:
# todo: this can actually get pretty slow: one task costs between 0.01-015s
no_dependencies_met = True
for ut in unfinished_tasks:
# We need to flag upstream and check for changes because upstream
# failures/re-schedules can result in deadlock false positives
old_state = ut.state
deps_met = ut.are_dependencies_met(dep_context=DepContext(
flag_upstream_failed=True,
ignore_in_retry_period=True,
ignore_in_reschedule_period=True),
session=session)
if deps_met or old_state != ut.current_state(session=session):
no_dependencies_met = False
break
duration = (timezone.utcnow() - start_dttm).total_seconds() * 1000
Stats.timing("dagrun.dependency-check.{}".format(self.dag_id),
duration)
root_ids = [t.task_id for t in dag.roots]
roots = [t for t in tis if t.task_id in root_ids]
# if all roots finished and at least one failed, the run failed
if (not unfinished_tasks
and any(r.state in (State.FAILED, State.UPSTREAM_FAILED)
for r in roots)):
self.log.info('Marking run %s failed', self)
self.set_state(State.FAILED)
dag.handle_callback(self,
success=False,
reason='task_failure',
session=session)
# if all roots succeeded and no unfinished tasks, the run succeeded
elif not unfinished_tasks and all(
r.state in (State.SUCCESS, State.SKIPPED) for r in roots):
self.log.info('Marking run %s successful', self)
self.set_state(State.SUCCESS)
dag.handle_callback(self,
success=True,
reason='success',
session=session)
# if *all tasks* are deadlocked, the run failed
elif (unfinished_tasks and none_depends_on_past
and none_task_concurrency and no_dependencies_met):
self.log.info('Deadlock; marking run %s failed', self)
self.set_state(State.FAILED)
dag.handle_callback(self,
success=False,
reason='all_tasks_deadlocked',
session=session)
# finally, if the roots aren't done, the dag is still running
else:
self.set_state(State.RUNNING)
self._emit_duration_stats_for_finished_state()
# todo: determine we want to use with_for_update to make sure to lock the run
session.merge(self)
session.commit()
return self.state
def _emit_duration_stats_for_finished_state(self):
if self.state == State.RUNNING:
return
duration = (self.end_date - self.start_date)
if self.state is State.SUCCESS:
Stats.timing('dagrun.duration.success.{}'.format(self.dag_id),
duration)
elif self.state == State.FAILED:
Stats.timing('dagrun.duration.failed.{}'.format(self.dag_id),
duration)
@provide_session
def verify_integrity(self, session=None):
"""
Verifies the DagRun by checking for removed tasks or tasks that are not in the
database yet. It will set state to removed or add the task if required.
"""
from airflow.models.taskinstance import TaskInstance # Avoid circular import
dag = self.get_dag()
tis = self.get_task_instances(session=session)
# check for removed or restored tasks
task_ids = []
for ti in tis:
task_ids.append(ti.task_id)
task = None
try:
task = dag.get_task(ti.task_id)
except AirflowException:
if ti.state == State.REMOVED:
pass # ti has already been removed, just ignore it
elif self.state is not State.RUNNING and not dag.partial:
self.log.warning("Failed to get task '{}' for dag '{}'. "
"Marking it as removed.".format(ti, dag))
Stats.incr("task_removed_from_dag.{}".format(dag.dag_id),
1, 1)
ti.state = State.REMOVED
is_task_in_dag = task is not None
should_restore_task = is_task_in_dag and ti.state == State.REMOVED
if should_restore_task:
self.log.info("Restoring task '{}' which was previously "
"removed from DAG '{}'".format(ti, dag))
Stats.incr("task_restored_to_dag.{}".format(dag.dag_id), 1, 1)
ti.state = State.NONE
# check for missing tasks
for task in six.itervalues(dag.task_dict):
if task.start_date > self.execution_date and not self.is_backfill:
continue
if task.task_id not in task_ids:
Stats.incr(
"task_instance_created-{}".format(task.__class__.__name__),
1, 1)
ti = TaskInstance(task, self.execution_date)
session.add(ti)
session.commit()
@staticmethod
def get_run(session, dag_id, execution_date):
"""
:param dag_id: DAG ID
:type dag_id: unicode
:param execution_date: execution date
:type execution_date: datetime
:return: DagRun corresponding to the given dag_id and execution date
if one exists. None otherwise.
:rtype: airflow.models.DagRun
"""
qry = session.query(DagRun).filter(
DagRun.dag_id == dag_id,
DagRun.external_trigger == False, # noqa
DagRun.execution_date == execution_date,
)
return qry.first()
@property
def is_backfill(self):
from airflow.jobs import BackfillJob
return (self.run_id is not None
and self.run_id.startswith(BackfillJob.ID_PREFIX))
@classmethod
@provide_session
def get_latest_runs(cls, session):
"""Returns the latest DagRun for each DAG. """
subquery = (session.query(
cls.dag_id,
func.max(cls.execution_date).label('execution_date')).group_by(
cls.dag_id).subquery())
dagruns = (session.query(cls).join(
subquery,
and_(cls.dag_id == subquery.c.dag_id,
cls.execution_date == subquery.c.execution_date)).all())
return dagruns
Column("user_value", Unicode(255), index=True))
# Annotating visualizations.
VisualizationAnnotationAssociation_table = Table(
"visualization_annotation_association",
metadata,
Column("id", Integer, primary_key=True),
Column("visualization_id",
Integer,
ForeignKey("visualization.id"),
index=True),
Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
Column("annotation", TEXT),
Index('ix_visualization_annotation_association_annotation',
'annotation',
mysql_length=200),
)
TABLES = [
VisualizationUserShareAssociation_table, VisualizationTagAssociation_table,
VisualizationAnnotationAssociation_table
]
def upgrade(migrate_engine):
print(__doc__)
metadata.bind = migrate_engine
metadata.reflect()
for table in TABLES:
# - app_id: The referenced app's primary key
# - app_client_id: the app's public key, used to identify it externally;
# included here so it appears in deltas.
# - app_type: 'plugin' or 'app', for future use to filter deltas
app_id = Column(Integer)
app_client_id = Column(Base36UID, nullable=False)
app_type = Column(String(20), nullable=False)
namespace_id = Column(ForeignKey(Namespace.id, ondelete='CASCADE'),
nullable=False)
namespace = relationship(Namespace)
# Reference to the object that this metadata is about. Public ID is the
# external identifier, while type and id allow direct lookup of the object.
object_public_id = Column(String(191), nullable=False, index=True)
object_type = Column(String(20), nullable=False)
object_id = Column(BigInteger, nullable=False)
value = Column(JSON)
queryable_value = Column(BigInteger, nullable=True, index=True)
version = Column(Integer, nullable=True, server_default='0')
Index('ix_obj_public_id_app_id',
Metadata.object_public_id,
Metadata.app_id,
unique=True)
Index('ix_namespace_id_app_id', Metadata.namespace_id, Metadata.app_id)
class TaskReschedule(Base):
"""TaskReschedule tracks rescheduled task instances."""
__tablename__ = "task_reschedule"
id = Column(Integer, primary_key=True)
task_id = Column(String(ID_LEN, **COLLATION_ARGS), nullable=False)
dag_id = Column(String(ID_LEN, **COLLATION_ARGS), nullable=False)
run_id = Column(String(ID_LEN, **COLLATION_ARGS), nullable=False)
map_index = Column(Integer, nullable=False, default=-1)
try_number = Column(Integer, nullable=False)
start_date = Column(UtcDateTime, nullable=False)
end_date = Column(UtcDateTime, nullable=False)
duration = Column(Integer, nullable=False)
reschedule_date = Column(UtcDateTime, nullable=False)
__table_args__ = (
Index('idx_task_reschedule_dag_task_run', dag_id, task_id, run_id, map_index, unique=False),
ForeignKeyConstraint(
[dag_id, task_id, run_id, map_index],
[
"task_instance.dag_id",
"task_instance.task_id",
"task_instance.run_id",
"task_instance.map_index",
],
name="task_reschedule_ti_fkey",
ondelete="CASCADE",
),
ForeignKeyConstraint(
[dag_id, run_id],
['dag_run.dag_id', 'dag_run.run_id'],
name='task_reschedule_dr_fkey',
ondelete='CASCADE',
),
)
dag_run = relationship("DagRun")
execution_date = association_proxy("dag_run", "execution_date")
def __init__(
self,
task: "BaseOperator",
run_id: str,
try_number: int,
start_date: datetime.datetime,
end_date: datetime.datetime,
reschedule_date: datetime.datetime,
map_index: int = -1,
):
self.dag_id = task.dag_id
self.task_id = task.task_id
self.run_id = run_id
self.map_index = map_index
self.try_number = try_number
self.start_date = start_date
self.end_date = end_date
self.reschedule_date = reschedule_date
self.duration = (self.end_date - self.start_date).total_seconds()
@staticmethod
@provide_session
def query_for_task_instance(task_instance, descending=False, session=None, try_number=None):
"""
Returns query for task reschedules for a given the task instance.
:param session: the database session object
:type session: sqlalchemy.orm.session.Session
:param task_instance: the task instance to find task reschedules for
:type task_instance: airflow.models.TaskInstance
:param descending: If True then records are returned in descending order
:type descending: bool
:param try_number: Look for TaskReschedule of the given try_number. Default is None which
looks for the same try_number of the given task_instance.
:type try_number: int
"""
if try_number is None:
try_number = task_instance.try_number
TR = TaskReschedule
qry = session.query(TR).filter(
TR.dag_id == task_instance.dag_id,
TR.task_id == task_instance.task_id,
TR.run_id == task_instance.run_id,
TR.try_number == try_number,
)
if descending:
return qry.order_by(desc(TR.id))
else:
return qry.order_by(asc(TR.id))
@staticmethod
@provide_session
def find_for_task_instance(task_instance, session=None, try_number=None):
"""
Returns all task reschedules for the task instance and try number,
in ascending order.
:param session: the database session object
:type session: sqlalchemy.orm.session.Session
:param task_instance: the task instance to find task reschedules for
:type task_instance: airflow.models.TaskInstance
:param try_number: Look for TaskReschedule of the given try_number. Default is None which
looks for the same try_number of the given task_instance.
:type try_number: int
"""
return TaskReschedule.query_for_task_instance(
task_instance, session=session, try_number=try_number
).all()
name='check_thing_book_has_author'),
db.CheckConstraint("\"type\" <> 'Book' OR \"idAtProviders\" SIMILAR TO '[0-9]{13}'",
name='check_thing_book_has_ean13'),
nullable=False)
name = db.Column(db.String(140), nullable=False)
description = db.Column(db.Text, nullable=True)
mediaUrls = db.Column(ARRAY(db.String(120)),
nullable=False,
default=[])
Thing.__ts_vector__ = create_tsvector(
cast(coalesce(Thing.name, ''), TEXT),
coalesce(Thing.extraData['author'].cast(TEXT), ''),
coalesce(Thing.extraData['byArtist'].cast(TEXT), ''),
)
Thing.__table_args__ = (
Index(
'idx_thing_fts',
Thing.__ts_vector__,
postgresql_using='gin'
),
)
app.model.Thing = Thing
class V_Category(Base, _Category):
__tablename__ = 'v_category'
__table_args__ = (
Index('ix_v_category_name', 'name', unique=True),
)
class V_Port(Base, _Port):
__tablename__ = 'v_port'
__table_args__ = (
ForeignKeyConstraint(['scan'], ['v_scan.id'], ondelete='CASCADE'),
Index('ix_v_port_scan_port', 'scan', 'port', 'protocol', unique=True),
)
class SerializedDagModel(Base):
"""A table for serialized DAGs.
serialized_dag table is a snapshot of DAG files synchronized by scheduler.
This feature is controlled by:
* ``[core] min_serialized_dag_update_interval = 30`` (s):
serialized DAGs are updated in DB when a file gets processed by scheduler,
to reduce DB write rate, there is a minimal interval of updating serialized DAGs.
* ``[scheduler] dag_dir_list_interval = 300`` (s):
interval of deleting serialized DAGs in DB when the files are deleted, suggest
to use a smaller interval such as 60
It is used by webserver to load dags
because reading from database is lightweight compared to importing from files,
it solves the webserver scalability issue.
"""
__tablename__ = 'serialized_dag'
dag_id = Column(String(ID_LEN), primary_key=True)
fileloc = Column(String(2000), nullable=False)
# The max length of fileloc exceeds the limit of indexing.
fileloc_hash = Column(BigInteger, nullable=False)
data = Column(sqlalchemy_jsonfield.JSONField(json=json), nullable=False)
last_updated = Column(UtcDateTime, nullable=False)
dag_hash = Column(String(32), nullable=False)
__table_args__ = (Index('idx_fileloc_hash', fileloc_hash, unique=False), )
dag_runs = relationship(
DagRun,
primaryjoin=dag_id == foreign(DagRun.dag_id), # type: ignore
backref=backref('serialized_dag', uselist=False, innerjoin=True),
)
dag_model = relationship(
DagModel,
primaryjoin=dag_id == DagModel.dag_id, # type: ignore
foreign_keys=dag_id,
uselist=False,
innerjoin=True,
backref=backref('serialized_dag', uselist=False, innerjoin=True),
)
load_op_links = True
def __init__(self, dag: DAG):
self.dag_id = dag.dag_id
self.fileloc = dag.fileloc
self.fileloc_hash = DagCode.dag_fileloc_hash(self.fileloc)
self.data = SerializedDAG.to_dict(dag)
self.last_updated = timezone.utcnow()
self.dag_hash = hashlib.md5(
json.dumps(self.data, sort_keys=True).encode("utf-8")).hexdigest()
def __repr__(self):
return f"<SerializedDag: {self.dag_id}>"
@classmethod
@provide_session
def write_dag(cls,
dag: DAG,
min_update_interval: Optional[int] = None,
session: Session = None) -> bool:
"""Serializes a DAG and writes it into database.
If the record already exists, it checks if the Serialized DAG changed or not. If it is
changed, it updates the record, ignores otherwise.
:param dag: a DAG to be written into database
:param min_update_interval: minimal interval in seconds to update serialized DAG
:param session: ORM Session
:returns: Boolean indicating if the DAG was written to the DB
"""
# Checks if (Current Time - Time when the DAG was written to DB) < min_update_interval
# If Yes, does nothing
# If No or the DAG does not exists, updates / writes Serialized DAG to DB
if min_update_interval is not None:
if (session.query(literal(True)).filter(
and_(
cls.dag_id == dag.dag_id,
(timezone.utcnow() -
timedelta(seconds=min_update_interval)) <
cls.last_updated,
)).first() is not None):
# TODO: .first() is not None can be changed to .scalar() once we update to sqlalchemy 1.4+
# as the associated sqlalchemy bug for MySQL was fixed
# related issue : https://github.com/sqlalchemy/sqlalchemy/issues/5481
return False
log.debug("Checking if DAG (%s) changed", dag.dag_id)
new_serialized_dag = cls(dag)
serialized_dag_hash_from_db = session.query(
cls.dag_hash).filter(cls.dag_id == dag.dag_id).scalar()
if serialized_dag_hash_from_db == new_serialized_dag.dag_hash:
log.debug(
"Serialized DAG (%s) is unchanged. Skipping writing to DB",
dag.dag_id)
return False
log.debug("Writing Serialized DAG: %s to the DB", dag.dag_id)
session.merge(new_serialized_dag)
log.debug("DAG: %s written to the DB", dag.dag_id)
return True
@classmethod
@provide_session
def read_all_dags(cls,
session: Session = None) -> Dict[str, 'SerializedDAG']:
"""Reads all DAGs in serialized_dag table.
:param session: ORM Session
:returns: a dict of DAGs read from database
"""
serialized_dags = session.query(cls)
dags = {}
for row in serialized_dags:
log.debug("Deserializing DAG: %s", row.dag_id)
dag = row.dag
# Coherence check
if dag.dag_id == row.dag_id:
dags[row.dag_id] = dag
else:
log.warning(
"dag_id Mismatch in DB: Row with dag_id '%s' has Serialised DAG with '%s' dag_id",
row.dag_id,
dag.dag_id,
)
return dags
@property
def dag(self):
"""The DAG deserialized from the ``data`` column"""
SerializedDAG._load_operator_extra_links = self.load_op_links
if isinstance(self.data, dict):
dag = SerializedDAG.from_dict(self.data) # type: Any
else:
dag = SerializedDAG.from_json(self.data)
return dag
@classmethod
@provide_session
def remove_dag(cls, dag_id: str, session: Session = None):
"""Deletes a DAG with given dag_id.
:param dag_id: dag_id to be deleted
:param session: ORM Session
"""
session.execute(cls.__table__.delete().where(cls.dag_id == dag_id))
@classmethod
@provide_session
def remove_deleted_dags(cls, alive_dag_filelocs: List[str], session=None):
"""Deletes DAGs not included in alive_dag_filelocs.
:param alive_dag_filelocs: file paths of alive DAGs
:param session: ORM Session
"""
alive_fileloc_hashes = [
DagCode.dag_fileloc_hash(fileloc) for fileloc in alive_dag_filelocs
]
log.debug(
"Deleting Serialized DAGs (for which DAG files are deleted) from %s table ",
cls.__tablename__)
session.execute(cls.__table__.delete().where(
and_(cls.fileloc_hash.notin_(alive_fileloc_hashes),
cls.fileloc.notin_(alive_dag_filelocs))))
@classmethod
@provide_session
def has_dag(cls, dag_id: str, session: Session = None) -> bool:
"""Checks a DAG exist in serialized_dag table.
:param dag_id: the DAG to check
:param session: ORM Session
"""
return session.query(
literal(True)).filter(cls.dag_id == dag_id).first() is not None
@classmethod
@provide_session
def get(cls,
dag_id: str,
session: Session = None) -> Optional['SerializedDagModel']:
"""
Get the SerializedDAG for the given dag ID.
It will cope with being passed the ID of a subdag by looking up the
root dag_id from the DAG table.
:param dag_id: the DAG to fetch
:param session: ORM Session
"""
row = session.query(cls).filter(cls.dag_id == dag_id).one_or_none()
if row:
return row
# If we didn't find a matching DAG id then ask the DAG table to find
# out the root dag
root_dag_id = session.query(
DagModel.root_dag_id).filter(DagModel.dag_id == dag_id).scalar()
return session.query(cls).filter(
cls.dag_id == root_dag_id).one_or_none()
@staticmethod
@provide_session
def bulk_sync_to_db(dags: List[DAG], session: Session = None):
"""
Saves DAGs as Serialized DAG objects in the database. Each
DAG is saved in a separate database query.
:param dags: the DAG objects to save to the DB
:type dags: List[airflow.models.dag.DAG]
:param session: ORM Session
:type session: Session
:return: None
"""
for dag in dags:
if not dag.is_subdag:
SerializedDagModel.write_dag(
dag,
min_update_interval=MIN_SERIALIZED_DAG_UPDATE_INTERVAL,
session=session)
@classmethod
@provide_session
def get_last_updated_datetime(cls,
dag_id: str,
session: Session = None
) -> Optional[datetime]:
"""
Get the date when the Serialized DAG associated to DAG was last updated
in serialized_dag table
:param dag_id: DAG ID
:type dag_id: str
:param session: ORM Session
:type session: Session
"""
return session.query(
cls.last_updated).filter(cls.dag_id == dag_id).scalar()
@classmethod
@provide_session
def get_max_last_updated_datetime(cls,
session: Session = None
) -> Optional[datetime]:
"""
Get the maximum date when any DAG was last updated in serialized_dag table
:param session: ORM Session
:type session: Session
"""
return session.query(func.max(cls.last_updated)).scalar()
@classmethod
@provide_session
def get_latest_version_hash(cls,
dag_id: str,
session: Session = None) -> Optional[str]:
"""
Get the latest DAG version for a given DAG ID.
:param dag_id: DAG ID
:type dag_id: str
:param session: ORM Session
:type session: Session
:return: DAG Hash, or None if the DAG is not found
:rtype: str | None
"""
return session.query(
cls.dag_hash).filter(cls.dag_id == dag_id).scalar()
@classmethod
@provide_session
def get_dag_dependencies(cls,
session: Session = None
) -> Dict[str, List['DagDependency']]:
"""
Get the dependencies between DAGs
:param session: ORM Session
:type session: Session
"""
if session.bind.dialect.name in ["sqlite", "mysql"]:
query = session.query(
cls.dag_id,
func.json_extract(cls.data, "$.dag.dag_dependencies"))
iterator = ((dag_id, json.loads(deps_data) if deps_data else [])
for dag_id, deps_data in query)
elif session.bind.dialect.name == "mssql":
query = session.query(
cls.dag_id, func.json_query(cls.data,
"$.dag.dag_dependencies"))
iterator = ((dag_id, json.loads(deps_data) if deps_data else [])
for dag_id, deps_data in query)
else:
iterator = session.query(
cls.dag_id,
func.json_extract_path(cls.data, "dag", "dag_dependencies"))
return {
dag_id: [DagDependency(**d) for d in (deps_data or [])]
for dag_id, deps_data in iterator
}
def test_raise_index_nonexistent_name(self):
m = MetaData()
# the KeyError isn't ideal here, a nicer message
# perhaps
assert_raises(KeyError, Table, 't', m, Column('x', Integer),
Index("foo", "q"))
def test_drop_schema(self):
t = Table('t', MetaData(), Column('x', Integer), schema="foo")
i = Index("xyz", t.c.x)
self.assert_compile(schema.DropIndex(i), "DROP INDEX foo.xyz")
def test_create_schema(self):
t = Table('t', MetaData(), Column('x', Integer), schema="foo")
i = Index("xyz", t.c.x)
self.assert_compile(schema.CreateIndex(i),
"CREATE INDEX xyz ON foo.t (x)")
class V_Hop(Base, _Hop):
__tablename__ = "v_hop"
__table_args__ = (
Index('ix_v_hop_ipaddr_ttl', 'ipaddr', 'ttl'),
ForeignKeyConstraint(['trace'], ['v_trace.id'], ondelete='CASCADE')
)
class DagRun(Base, LoggingMixin):
"""
DagRun describes an instance of a Dag. It can be created
by the scheduler (for regular runs) or by an external trigger
"""
__tablename__ = "dag_run"
__NO_VALUE = object()
id = Column(Integer, primary_key=True)
dag_id = Column(String(ID_LEN, **COLLATION_ARGS))
queued_at = Column(UtcDateTime)
execution_date = Column(UtcDateTime, default=timezone.utcnow)
start_date = Column(UtcDateTime)
end_date = Column(UtcDateTime)
_state = Column('state', String(50), default=State.QUEUED)
run_id = Column(String(ID_LEN, **COLLATION_ARGS))
creating_job_id = Column(Integer)
external_trigger = Column(Boolean, default=True)
run_type = Column(String(50), nullable=False)
conf = Column(PickleType)
# These two must be either both NULL or both datetime.
data_interval_start = Column(UtcDateTime)
data_interval_end = Column(UtcDateTime)
# When a scheduler last attempted to schedule TIs for this DagRun
last_scheduling_decision = Column(UtcDateTime)
dag_hash = Column(String(32))
dag = None
__table_args__ = (
Index('dag_id_state', dag_id, _state),
UniqueConstraint('dag_id',
'execution_date',
name='dag_run_dag_id_execution_date_key'),
UniqueConstraint('dag_id', 'run_id', name='dag_run_dag_id_run_id_key'),
Index('idx_last_scheduling_decision', last_scheduling_decision),
)
task_instances = relationship(TI, back_populates="dag_run")
DEFAULT_DAGRUNS_TO_EXAMINE = airflow_conf.getint(
'scheduler',
'max_dagruns_per_loop_to_schedule',
fallback=20,
)
def __init__(
self,
dag_id: Optional[str] = None,
run_id: Optional[str] = None,
queued_at: Optional[datetime] = __NO_VALUE,
execution_date: Optional[datetime] = None,
start_date: Optional[datetime] = None,
external_trigger: Optional[bool] = None,
conf: Optional[Any] = None,
state: Optional[DagRunState] = None,
run_type: Optional[str] = None,
dag_hash: Optional[str] = None,
creating_job_id: Optional[int] = None,
data_interval: Optional[Tuple[datetime, datetime]] = None,
):
if data_interval is None:
# Legacy: Only happen for runs created prior to Airflow 2.2.
self.data_interval_start = self.data_interval_end = None
else:
self.data_interval_start, self.data_interval_end = data_interval
self.dag_id = dag_id
self.run_id = run_id
self.execution_date = execution_date
self.start_date = start_date
self.external_trigger = external_trigger
self.conf = conf or {}
self.state = state
if queued_at is self.__NO_VALUE:
self.queued_at = timezone.utcnow(
) if state == State.QUEUED else None
else:
self.queued_at = queued_at
self.run_type = run_type
self.dag_hash = dag_hash
self.creating_job_id = creating_job_id
super().__init__()
def __repr__(self):
return (
'<DagRun {dag_id} @ {execution_date}: {run_id}, externally triggered: {external_trigger}>'
).format(
dag_id=self.dag_id,
execution_date=self.execution_date,
run_id=self.run_id,
external_trigger=self.external_trigger,
)
def get_state(self):
return self._state
def set_state(self, state: DagRunState):
if self._state != state:
self._state = state
self.end_date = timezone.utcnow(
) if self._state in State.finished else None
if state == State.QUEUED:
self.queued_at = timezone.utcnow()
@declared_attr
def state(self):
return synonym('_state',
descriptor=property(self.get_state, self.set_state))
@provide_session
def refresh_from_db(self, session: Session = None):
"""
Reloads the current dagrun from the database
:param session: database session
:type session: Session
"""
dr = session.query(DagRun).filter(DagRun.dag_id == self.dag_id,
DagRun.run_id == self.run_id).one()
self.id = dr.id
self.state = dr.state
@classmethod
def next_dagruns_to_examine(
cls,
state: DagRunState,
session: Session,
max_number: Optional[int] = None,
):
"""
Return the next DagRuns that the scheduler should attempt to schedule.
This will return zero or more DagRun rows that are row-level-locked with a "SELECT ... FOR UPDATE"
query, you should ensure that any scheduling decisions are made in a single transaction -- as soon as
the transaction is committed it will be unlocked.
:rtype: list[airflow.models.DagRun]
"""
from airflow.models.dag import DagModel
if max_number is None:
max_number = cls.DEFAULT_DAGRUNS_TO_EXAMINE
# TODO: Bake this query, it is run _A lot_
query = (session.query(cls).filter(
cls.state == state, cls.run_type != DagRunType.BACKFILL_JOB).join(
DagModel,
DagModel.dag_id == cls.dag_id,
).filter(
DagModel.is_paused == expression.false(),
DagModel.is_active == expression.true(),
).order_by(
nulls_first(cls.last_scheduling_decision, session=session),
cls.execution_date,
))
if not settings.ALLOW_FUTURE_EXEC_DATES:
query = query.filter(DagRun.execution_date <= func.now())
return with_row_locks(query.limit(max_number),
of=cls,
session=session,
**skip_locked(session=session))
@staticmethod
@provide_session
def find(
dag_id: Optional[Union[str, List[str]]] = None,
run_id: Optional[str] = None,
execution_date: Optional[datetime] = None,
state: Optional[DagRunState] = None,
external_trigger: Optional[bool] = None,
no_backfills: bool = False,
run_type: Optional[DagRunType] = None,
session: Session = None,
execution_start_date: Optional[datetime] = None,
execution_end_date: Optional[datetime] = None,
) -> List["DagRun"]:
"""
Returns a set of dag runs for the given search criteria.
:param dag_id: the dag_id or list of dag_id to find dag runs for
:type dag_id: str or list[str]
:param run_id: defines the run id for this dag run
:type run_id: str
:param run_type: type of DagRun
:type run_type: airflow.utils.types.DagRunType
:param execution_date: the execution date
:type execution_date: datetime.datetime or list[datetime.datetime]
:param state: the state of the dag run
:type state: DagRunState
:param external_trigger: whether this dag run is externally triggered
:type external_trigger: bool
:param no_backfills: return no backfills (True), return all (False).
Defaults to False
:type no_backfills: bool
:param session: database session
:type session: sqlalchemy.orm.session.Session
:param execution_start_date: dag run that was executed from this date
:type execution_start_date: datetime.datetime
:param execution_end_date: dag run that was executed until this date
:type execution_end_date: datetime.datetime
"""
DR = DagRun
qry = session.query(DR)
dag_ids = [dag_id] if isinstance(dag_id, str) else dag_id
if dag_ids:
qry = qry.filter(DR.dag_id.in_(dag_ids))
if run_id:
qry = qry.filter(DR.run_id == run_id)
if execution_date:
if isinstance(execution_date, list):
qry = qry.filter(DR.execution_date.in_(execution_date))
else:
qry = qry.filter(DR.execution_date == execution_date)
if execution_start_date and execution_end_date:
qry = qry.filter(
DR.execution_date.between(execution_start_date,
execution_end_date))
elif execution_start_date:
qry = qry.filter(DR.execution_date >= execution_start_date)
elif execution_end_date:
qry = qry.filter(DR.execution_date <= execution_end_date)
if state:
qry = qry.filter(DR.state == state)
if external_trigger is not None:
qry = qry.filter(DR.external_trigger == external_trigger)
if run_type:
qry = qry.filter(DR.run_type == run_type)
if no_backfills:
qry = qry.filter(DR.run_type != DagRunType.BACKFILL_JOB)
return qry.order_by(DR.execution_date).all()
@staticmethod
def generate_run_id(run_type: DagRunType, execution_date: datetime) -> str:
"""Generate Run ID based on Run Type and Execution Date"""
return f"{run_type}__{execution_date.isoformat()}"
@provide_session
def get_task_instances(self,
state: Optional[Iterable[TaskInstanceState]] = None,
session=None) -> Iterable[TI]:
"""Returns the task instances for this dag run"""
tis = (session.query(TI).options(joinedload(TI.dag_run)).filter(
TI.dag_id == self.dag_id,
TI.run_id == self.run_id,
))
if state:
if isinstance(state, str):
tis = tis.filter(TI.state == state)
else:
# this is required to deal with NULL values
if None in state:
if all(x is None for x in state):
tis = tis.filter(TI.state.is_(None))
else:
not_none_state = [s for s in state if s]
tis = tis.filter(
or_(TI.state.in_(not_none_state),
TI.state.is_(None)))
else:
tis = tis.filter(TI.state.in_(state))
if self.dag and self.dag.partial:
tis = tis.filter(TI.task_id.in_(self.dag.task_ids))
return tis.all()
@provide_session
def get_task_instance(self,
task_id: str,
session: Session = None) -> Optional[TI]:
"""
Returns the task instance specified by task_id for this dag run
:param task_id: the task id
:type task_id: str
:param session: Sqlalchemy ORM Session
:type session: Session
"""
return (session.query(TI).filter(TI.dag_id == self.dag_id,
TI.run_id == self.run_id,
TI.task_id == task_id).one_or_none())
def get_dag(self) -> "DAG":
"""
Returns the Dag associated with this DagRun.
:return: DAG
"""
if not self.dag:
raise AirflowException(
f"The DAG (.dag) for {self} needs to be set")
return self.dag
@provide_session
def get_previous_dagrun(self,
state: Optional[DagRunState] = None,
session: Session = None) -> Optional['DagRun']:
"""The previous DagRun, if there is one"""
filters = [
DagRun.dag_id == self.dag_id,
DagRun.execution_date < self.execution_date,
]
if state is not None:
filters.append(DagRun.state == state)
return session.query(DagRun).filter(*filters).order_by(
DagRun.execution_date.desc()).first()
@provide_session
def get_previous_scheduled_dagrun(self,
session: Session = None
) -> Optional['DagRun']:
"""The previous, SCHEDULED DagRun, if there is one"""
return (session.query(DagRun).filter(
DagRun.dag_id == self.dag_id,
DagRun.execution_date < self.execution_date,
DagRun.run_type != DagRunType.MANUAL,
).order_by(DagRun.execution_date.desc()).first())
@provide_session
def update_state(
self,
session: Session = None,
execute_callbacks: bool = True
) -> Tuple[List[TI], Optional[callback_requests.DagCallbackRequest]]:
"""
Determines the overall state of the DagRun based on the state
of its TaskInstances.
:param session: Sqlalchemy ORM Session
:type session: Session
:param execute_callbacks: Should dag callbacks (success/failure, SLA etc) be invoked
directly (default: true) or recorded as a pending request in the ``callback`` property
:type execute_callbacks: bool
:return: Tuple containing tis that can be scheduled in the current loop & `callback` that
needs to be executed
"""
# Callback to execute in case of Task Failures
callback: Optional[callback_requests.DagCallbackRequest] = None
start_dttm = timezone.utcnow()
self.last_scheduling_decision = start_dttm
with Stats.timer(f"dagrun.dependency-check.{self.dag_id}"):
dag = self.get_dag()
info = self.task_instance_scheduling_decisions(session)
tis = info.tis
schedulable_tis = info.schedulable_tis
changed_tis = info.changed_tis
finished_tasks = info.finished_tasks
unfinished_tasks = info.unfinished_tasks
none_depends_on_past = all(not t.task.depends_on_past
for t in unfinished_tasks)
none_task_concurrency = all(t.task.max_active_tis_per_dag is None
for t in unfinished_tasks)
none_deferred = all(t.state != State.DEFERRED
for t in unfinished_tasks)
if unfinished_tasks and none_depends_on_past and none_task_concurrency and none_deferred:
# small speed up
are_runnable_tasks = (schedulable_tis
or self._are_premature_tis(
unfinished_tasks, finished_tasks,
session) or changed_tis)
leaf_task_ids = {t.task_id for t in dag.leaves}
leaf_tis = [ti for ti in tis if ti.task_id in leaf_task_ids]
# if all roots finished and at least one failed, the run failed
if not unfinished_tasks and any(leaf_ti.state in State.failed_states
for leaf_ti in leaf_tis):
self.log.error('Marking run %s failed', self)
self.set_state(State.FAILED)
if execute_callbacks:
dag.handle_callback(self,
success=False,
reason='task_failure',
session=session)
elif dag.has_on_failure_callback:
callback = callback_requests.DagCallbackRequest(
full_filepath=dag.fileloc,
dag_id=self.dag_id,
run_id=self.run_id,
is_failure_callback=True,
msg='task_failure',
)
# if all leaves succeeded and no unfinished tasks, the run succeeded
elif not unfinished_tasks and all(leaf_ti.state in State.success_states
for leaf_ti in leaf_tis):
self.log.info('Marking run %s successful', self)
self.set_state(State.SUCCESS)
if execute_callbacks:
dag.handle_callback(self,
success=True,
reason='success',
session=session)
elif dag.has_on_success_callback:
callback = callback_requests.DagCallbackRequest(
full_filepath=dag.fileloc,
dag_id=self.dag_id,
run_id=self.run_id,
is_failure_callback=False,
msg='success',
)
# if *all tasks* are deadlocked, the run failed
elif (unfinished_tasks and none_depends_on_past
and none_task_concurrency and none_deferred
and not are_runnable_tasks):
self.log.error('Deadlock; marking run %s failed', self)
self.set_state(State.FAILED)
if execute_callbacks:
dag.handle_callback(self,
success=False,
reason='all_tasks_deadlocked',
session=session)
elif dag.has_on_failure_callback:
callback = callback_requests.DagCallbackRequest(
full_filepath=dag.fileloc,
dag_id=self.dag_id,
run_id=self.run_id,
is_failure_callback=True,
msg='all_tasks_deadlocked',
)
# finally, if the roots aren't done, the dag is still running
else:
self.set_state(State.RUNNING)
self._emit_true_scheduling_delay_stats_for_finished_state(
finished_tasks)
self._emit_duration_stats_for_finished_state()
session.merge(self)
return schedulable_tis, callback
@provide_session
def task_instance_scheduling_decisions(self,
session: Session = None
) -> TISchedulingDecision:
schedulable_tis: List[TI] = []
changed_tis = False
tis = list(
self.get_task_instances(session=session, state=State.task_states))
self.log.debug("number of tis tasks for %s: %s task(s)", self,
len(tis))
for ti in tis:
try:
ti.task = self.get_dag().get_task(ti.task_id)
except TaskNotFound:
self.log.warning(
"Failed to get task '%s' for dag '%s'. Marking it as removed.",
ti, ti.dag_id)
ti.state = State.REMOVED
session.flush()
unfinished_tasks = [t for t in tis if t.state in State.unfinished]
finished_tasks = [t for t in tis if t.state in State.finished]
if unfinished_tasks:
scheduleable_tasks = [
ut for ut in unfinished_tasks
if ut.state in SCHEDULEABLE_STATES
]
self.log.debug("number of scheduleable tasks for %s: %s task(s)",
self, len(scheduleable_tasks))
schedulable_tis, changed_tis = self._get_ready_tis(
scheduleable_tasks, finished_tasks, session)
return TISchedulingDecision(
tis=tis,
schedulable_tis=schedulable_tis,
changed_tis=changed_tis,
unfinished_tasks=unfinished_tasks,
finished_tasks=finished_tasks,
)
def _get_ready_tis(
self,
scheduleable_tasks: List[TI],
finished_tasks: List[TI],
session: Session,
) -> Tuple[List[TI], bool]:
old_states = {}
ready_tis: List[TI] = []
changed_tis = False
if not scheduleable_tasks:
return ready_tis, changed_tis
# Check dependencies
for st in scheduleable_tasks:
old_state = st.state
if st.are_dependencies_met(
dep_context=DepContext(flag_upstream_failed=True,
finished_tasks=finished_tasks),
session=session,
):
ready_tis.append(st)
else:
old_states[st.key] = old_state
# Check if any ti changed state
tis_filter = TI.filter_for_tis(old_states.keys())
if tis_filter is not None:
fresh_tis = session.query(TI).filter(tis_filter).all()
changed_tis = any(ti.state != old_states[ti.key]
for ti in fresh_tis)
return ready_tis, changed_tis
def _are_premature_tis(
self,
unfinished_tasks: List[TI],
finished_tasks: List[TI],
session: Session,
) -> bool:
# there might be runnable tasks that are up for retry and for some reason(retry delay, etc) are
# not ready yet so we set the flags to count them in
for ut in unfinished_tasks:
if ut.are_dependencies_met(
dep_context=DepContext(
flag_upstream_failed=True,
ignore_in_retry_period=True,
ignore_in_reschedule_period=True,
finished_tasks=finished_tasks,
),
session=session,
):
return True
return False
def _emit_true_scheduling_delay_stats_for_finished_state(
self, finished_tis):
"""
This is a helper method to emit the true scheduling delay stats, which is defined as
the time when the first task in DAG starts minus the expected DAG run datetime.
This method will be used in the update_state method when the state of the DagRun
is updated to a completed status (either success or failure). The method will find the first
started task within the DAG and calculate the expected DagRun start time (based on
dag.execution_date & dag.timetable), and minus these two values to get the delay.
The emitted data may contains outlier (e.g. when the first task was cleared, so
the second task's start_date will be used), but we can get rid of the outliers
on the stats side through the dashboards tooling built.
Note, the stat will only be emitted if the DagRun is a scheduler triggered one
(i.e. external_trigger is False).
"""
if self.state == State.RUNNING:
return
if self.external_trigger:
return
if not finished_tis:
return
try:
dag = self.get_dag()
if not self.dag.timetable.periodic:
# We can't emit this metric if there is no following schedule to calculate from!
return
ordered_tis_by_start_date = [
ti for ti in finished_tis if ti.start_date
]
ordered_tis_by_start_date.sort(key=lambda ti: ti.start_date,
reverse=False)
first_start_date = ordered_tis_by_start_date[0].start_date
if first_start_date:
# dag.following_schedule calculates the expected start datetime for a scheduled dagrun
# i.e. a daily flow for execution date 1/1/20 actually runs on 1/2/20 hh:mm:ss,
# and ti.start_date will be 1/2/20 hh:mm:ss so the following schedule is comparison
true_delay = first_start_date - dag.following_schedule(
self.execution_date)
if true_delay.total_seconds() > 0:
Stats.timing(
f'dagrun.{dag.dag_id}.first_task_scheduling_delay',
true_delay)
except Exception as e:
self.log.warning(
f'Failed to record first_task_scheduling_delay metric:\n{e}')
def _emit_duration_stats_for_finished_state(self):
if self.state == State.RUNNING:
return
if self.start_date is None:
self.log.warning(
'Failed to record duration of %s: start_date is not set.',
self)
return
if self.end_date is None:
self.log.warning(
'Failed to record duration of %s: end_date is not set.', self)
return
duration = self.end_date - self.start_date
if self.state == State.SUCCESS:
Stats.timing(f'dagrun.duration.success.{self.dag_id}', duration)
elif self.state == State.FAILED:
Stats.timing(f'dagrun.duration.failed.{self.dag_id}', duration)
@provide_session
def verify_integrity(self, session: Session = None):
"""
Verifies the DagRun by checking for removed tasks or tasks that are not in the
database yet. It will set state to removed or add the task if required.
:param session: Sqlalchemy ORM Session
:type session: Session
"""
from airflow.settings import task_instance_mutation_hook
dag = self.get_dag()
tis = self.get_task_instances(session=session)
# check for removed or restored tasks
task_ids = set()
for ti in tis:
task_instance_mutation_hook(ti)
task_ids.add(ti.task_id)
task = None
try:
task = dag.get_task(ti.task_id)
except AirflowException:
if ti.state == State.REMOVED:
pass # ti has already been removed, just ignore it
elif self.state != State.RUNNING and not dag.partial:
self.log.warning(
"Failed to get task '%s' for dag '%s'. Marking it as removed.",
ti, dag)
Stats.incr(f"task_removed_from_dag.{dag.dag_id}", 1, 1)
ti.state = State.REMOVED
should_restore_task = (task
is not None) and ti.state == State.REMOVED
if should_restore_task:
self.log.info(
"Restoring task '%s' which was previously removed from DAG '%s'",
ti, dag)
Stats.incr(f"task_restored_to_dag.{dag.dag_id}", 1, 1)
ti.state = State.NONE
session.merge(ti)
# check for missing tasks
for task in dag.task_dict.values():
if task.start_date > self.execution_date and not self.is_backfill:
continue
if task.task_id not in task_ids:
Stats.incr(f"task_instance_created-{task.task_type}", 1, 1)
ti = TI(task, execution_date=None, run_id=self.run_id)
task_instance_mutation_hook(ti)
session.add(ti)
try:
session.flush()
except IntegrityError as err:
self.log.info(str(err))
self.log.info(
'Hit IntegrityError while creating the TIs for %s- %s',
dag.dag_id, self.run_id)
self.log.info('Doing session rollback.')
# TODO[HA]: We probably need to savepoint this so we can keep the transaction alive.
session.rollback()
@staticmethod
def get_run(session: Session, dag_id: str,
execution_date: datetime) -> Optional['DagRun']:
"""
Get a single DAG Run
:meta private:
:param session: Sqlalchemy ORM Session
:type session: Session
:param dag_id: DAG ID
:type dag_id: unicode
:param execution_date: execution date
:type execution_date: datetime
:return: DagRun corresponding to the given dag_id and execution date
if one exists. None otherwise.
:rtype: airflow.models.DagRun
"""
warnings.warn(
"This method is deprecated. Please use SQLAlchemy directly",
DeprecationWarning,
stacklevel=2,
)
return (session.query(DagRun).filter(
DagRun.dag_id == dag_id,
DagRun.external_trigger == False, # noqa
DagRun.execution_date == execution_date,
).first())
@property
def is_backfill(self) -> bool:
return self.run_type == DagRunType.BACKFILL_JOB
@classmethod
@provide_session
def get_latest_runs(cls, session=None) -> List['DagRun']:
"""Returns the latest DagRun for each DAG"""
subquery = (session.query(
cls.dag_id,
func.max(cls.execution_date).label('execution_date')).group_by(
cls.dag_id).subquery())
return (session.query(cls).join(
subquery,
and_(cls.dag_id == subquery.c.dag_id,
cls.execution_date == subquery.c.execution_date),
).all())
@provide_session
def schedule_tis(self,
schedulable_tis: Iterable[TI],
session: Session = None) -> int:
"""
Set the given task instances in to the scheduled state.
Each element of ``schedulable_tis`` should have it's ``task`` attribute already set.
Any DummyOperator without callbacks is instead set straight to the success state.
All the TIs should belong to this DagRun, but this code is in the hot-path, this is not checked -- it
is the caller's responsibility to call this function only with TIs from a single dag run.
"""
# Get list of TI IDs that do not need to executed, these are
# tasks using DummyOperator and without on_execute_callback / on_success_callback
dummy_ti_ids = []
schedulable_ti_ids = []
for ti in schedulable_tis:
if (ti.task.inherits_from_dummy_operator
and not ti.task.on_execute_callback
and not ti.task.on_success_callback):
dummy_ti_ids.append(ti.task_id)
else:
schedulable_ti_ids.append(ti.task_id)
count = 0
if schedulable_ti_ids:
count += (session.query(TI).filter(
TI.dag_id == self.dag_id,
TI.run_id == self.run_id,
TI.task_id.in_(schedulable_ti_ids),
).update({TI.state: State.SCHEDULED}, synchronize_session=False))
# Tasks using DummyOperator should not be executed, mark them as success
if dummy_ti_ids:
count += (session.query(TI).filter(
TI.dag_id == self.dag_id,
TI.run_id == self.run_id,
TI.task_id.in_(dummy_ti_ids),
).update(
{
TI.state: State.SUCCESS,
TI.start_date: timezone.utcnow(),
TI.end_date: timezone.utcnow(),
TI.duration: 0,
},
synchronize_session=False,
))
return count
class Feed(Base):
__tablename__ = 'feed'
id = Column(Integer, primary_key=True, nullable=False)
created = Column(TIMESTAMP(timezone=False), default=datetime.utcnow,
nullable=False)
__table_args__ = (Index('feed_id_index', 'id'), )
def upgrade(migrate_engine):
meta = MetaData()
meta.bind = migrate_engine
cell_mappings = Table('cell_mappings',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id',
Integer,
primary_key=True,
nullable=False),
Column('uuid', String(length=36), nullable=False),
Column('name', String(length=255)),
Column('transport_url', Text()),
Column('database_connection', Text()),
UniqueConstraint('uuid',
name='uniq_cell_mappings0uuid'),
Index('uuid_idx', 'uuid'),
mysql_engine='InnoDB',
mysql_charset='utf8')
host_mappings = Table('host_mappings',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id',
Integer,
primary_key=True,
nullable=False),
Column('cell_id', Integer, nullable=False),
Column('host', String(length=255), nullable=False),
UniqueConstraint('host',
name='uniq_host_mappings0host'),
Index('host_idx', 'host'),
ForeignKeyConstraint(columns=['cell_id'],
refcolumns=[cell_mappings.c.id
]),
mysql_engine='InnoDB',
mysql_charset='utf8')
instance_mappings = Table(
'instance_mappings',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('instance_uuid', String(length=36), nullable=False),
Column('cell_id', Integer, nullable=True),
Column('project_id', String(length=255), nullable=False),
UniqueConstraint('instance_uuid',
name='uniq_instance_mappings0instance_uuid'),
Index('instance_uuid_idx', 'instance_uuid'),
Index('project_id_idx', 'project_id'),
ForeignKeyConstraint(columns=['cell_id'],
refcolumns=[cell_mappings.c.id]),
mysql_engine='InnoDB',
mysql_charset='utf8')
flavors = Table('flavors',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('name', String(length=255), nullable=False),
Column('id', Integer, primary_key=True, nullable=False),
Column('memory_mb', Integer, nullable=False),
Column('vcpus', Integer, nullable=False),
Column('swap', Integer, nullable=False),
Column('vcpu_weight', Integer),
Column('flavorid', String(length=255), nullable=False),
Column('rxtx_factor', Float),
Column('root_gb', Integer),
Column('ephemeral_gb', Integer),
Column('disabled', Boolean),
Column('is_public', Boolean),
Column('description', Text()),
UniqueConstraint('flavorid', name='uniq_flavors0flavorid'),
UniqueConstraint('name', name='uniq_flavors0name'),
mysql_engine='InnoDB',
mysql_charset='utf8')
flavor_extra_specs = Table(
'flavor_extra_specs',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('flavor_id', Integer, nullable=False),
Column('key', String(length=255), nullable=False),
Column('value', String(length=255)),
UniqueConstraint('flavor_id',
'key',
name='uniq_flavor_extra_specs0flavor_id0key'),
Index('flavor_extra_specs_flavor_id_key_idx', 'flavor_id', 'key'),
ForeignKeyConstraint(columns=['flavor_id'], refcolumns=[flavors.c.id]),
mysql_engine='InnoDB',
mysql_charset='utf8')
flavor_projects = Table(
'flavor_projects',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('flavor_id', Integer, nullable=False),
Column('project_id', String(length=255), nullable=False),
UniqueConstraint('flavor_id',
'project_id',
name='uniq_flavor_projects0flavor_id0project_id'),
ForeignKeyConstraint(columns=['flavor_id'], refcolumns=[flavors.c.id]),
mysql_engine='InnoDB',
mysql_charset='utf8')
request_specs = Table(
'request_specs',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('instance_uuid', String(36), nullable=False),
Column('spec', MediumText(), nullable=False),
UniqueConstraint('instance_uuid',
name='uniq_request_specs0instance_uuid'),
Index('request_spec_instance_uuid_idx', 'instance_uuid'),
mysql_engine='InnoDB',
mysql_charset='utf8')
build_requests = Table(
'build_requests',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('request_spec_id', Integer, nullable=True),
Column('project_id', String(length=255), nullable=False),
Column('user_id', String(length=255), nullable=True),
Column('display_name', String(length=255)),
Column('instance_metadata', Text),
Column('progress', Integer),
Column('vm_state', String(length=255)),
Column('task_state', String(length=255)),
Column('image_ref', String(length=255)),
Column('access_ip_v4', InetSmall()),
Column('access_ip_v6', InetSmall()),
Column('info_cache', Text),
Column('security_groups', Text, nullable=True),
Column('config_drive', Boolean, default=False, nullable=True),
Column('key_name', String(length=255)),
Column('locked_by',
Enum('owner', 'admin', name='build_requests0locked_by')),
Column('instance_uuid', String(length=36)),
Column('instance', MediumText()),
Column('block_device_mappings', MediumText()),
Column('tags', Text()),
UniqueConstraint('instance_uuid',
name='uniq_build_requests0instance_uuid'),
Index('build_requests_project_id_idx', 'project_id'),
Index('build_requests_instance_uuid_idx', 'instance_uuid'),
mysql_engine='InnoDB',
mysql_charset='utf8')
keypairs = Table('key_pairs',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('name', String(255), nullable=False),
Column('user_id', String(255), nullable=False),
Column('fingerprint', String(255)),
Column('public_key', Text()),
Column('type',
Enum('ssh',
'x509',
metadata=meta,
name='keypair_types'),
nullable=False,
server_default=keypair.KEYPAIR_TYPE_SSH),
UniqueConstraint('user_id',
'name',
name='uniq_key_pairs0user_id0name'),
mysql_engine='InnoDB',
mysql_charset='utf8')
projects = Table(
'projects',
meta,
Column('id',
Integer,
primary_key=True,
nullable=False,
autoincrement=True),
Column('external_id', String(length=255), nullable=False),
Column('created_at', DateTime),
Column('updated_at', DateTime),
UniqueConstraint('external_id', name='uniq_projects0external_id'),
mysql_engine='InnoDB',
mysql_charset='latin1',
)
users = Table(
'users',
meta,
Column('id',
Integer,
primary_key=True,
nullable=False,
autoincrement=True),
Column('external_id', String(length=255), nullable=False),
Column('created_at', DateTime),
Column('updated_at', DateTime),
UniqueConstraint('external_id', name='uniq_users0external_id'),
mysql_engine='InnoDB',
mysql_charset='latin1',
)
resource_classes = Table(
'resource_classes',
meta,
Column('id', Integer, primary_key=True, nullable=False),
Column('name', String(length=255), nullable=False),
Column('created_at', DateTime),
Column('updated_at', DateTime),
UniqueConstraint('name', name='uniq_resource_classes0name'),
mysql_engine='InnoDB',
mysql_charset='latin1')
nameargs = {}
if migrate_engine.name == 'mysql':
nameargs['collation'] = 'utf8_bin'
resource_providers = Table(
'resource_providers',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('uuid', String(36), nullable=False),
Column('name', Unicode(200, **nameargs), nullable=True),
Column('generation', Integer, default=0),
Column('can_host', Integer, default=0),
Column('root_provider_id', Integer,
ForeignKey('resource_providers.id')),
Column('parent_provider_id', Integer,
ForeignKey('resource_providers.id')),
UniqueConstraint('uuid', name='uniq_resource_providers0uuid'),
UniqueConstraint('name', name='uniq_resource_providers0name'),
Index('resource_providers_name_idx', 'name'),
Index('resource_providers_uuid_idx', 'uuid'),
Index('resource_providers_root_provider_id_idx', 'root_provider_id'),
Index('resource_providers_parent_provider_id_idx',
'parent_provider_id'),
mysql_engine='InnoDB',
mysql_charset='latin1')
inventories = Table(
'inventories',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('resource_provider_id', Integer, nullable=False),
Column('resource_class_id', Integer, nullable=False),
Column('total', Integer, nullable=False),
Column('reserved', Integer, nullable=False),
Column('min_unit', Integer, nullable=False),
Column('max_unit', Integer, nullable=False),
Column('step_size', Integer, nullable=False),
Column('allocation_ratio', Float, nullable=False),
Index('inventories_resource_provider_id_idx', 'resource_provider_id'),
Index('inventories_resource_provider_resource_class_idx',
'resource_provider_id', 'resource_class_id'),
Index('inventories_resource_class_id_idx', 'resource_class_id'),
UniqueConstraint(
'resource_provider_id',
'resource_class_id',
name='uniq_inventories0resource_provider_resource_class'),
mysql_engine='InnoDB',
mysql_charset='latin1')
traits = Table(
'traits',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id',
Integer,
primary_key=True,
nullable=False,
autoincrement=True),
Column('name', Unicode(255, **nameargs), nullable=False),
UniqueConstraint('name', name='uniq_traits0name'),
mysql_engine='InnoDB',
mysql_charset='latin1',
)
allocations = Table('allocations',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True,
nullable=False),
Column('resource_provider_id', Integer,
nullable=False),
Column('consumer_id', String(36), nullable=False),
Column('resource_class_id', Integer, nullable=False),
Column('used', Integer, nullable=False),
Index('allocations_resource_provider_class_used_idx',
'resource_provider_id', 'resource_class_id',
'used'),
Index('allocations_resource_class_id_idx',
'resource_class_id'),
Index('allocations_consumer_id_idx', 'consumer_id'),
mysql_engine='InnoDB',
mysql_charset='latin1')
consumers = Table(
'consumers',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id',
Integer,
primary_key=True,
nullable=False,
autoincrement=True),
Column('uuid', String(length=36), nullable=False),
Column('project_id', Integer, nullable=False),
Column('user_id', Integer, nullable=False),
Index('consumers_project_id_uuid_idx', 'project_id', 'uuid'),
Index('consumers_project_id_user_id_uuid_idx', 'project_id', 'user_id',
'uuid'),
UniqueConstraint('uuid', name='uniq_consumers0uuid'),
mysql_engine='InnoDB',
mysql_charset='latin1',
)
resource_provider_aggregates = Table(
'resource_provider_aggregates',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('resource_provider_id',
Integer,
primary_key=True,
nullable=False),
Column('aggregate_id', Integer, primary_key=True, nullable=False),
Index('resource_provider_aggregates_aggregate_id_idx', 'aggregate_id'),
mysql_engine='InnoDB',
mysql_charset='latin1')
resource_provider_traits = Table(
'resource_provider_traits',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('trait_id',
Integer,
ForeignKey('traits.id'),
primary_key=True,
nullable=False),
Column('resource_provider_id',
Integer,
primary_key=True,
nullable=False),
Index('resource_provider_traits_resource_provider_trait_idx',
'resource_provider_id', 'trait_id'),
ForeignKeyConstraint(columns=['resource_provider_id'],
refcolumns=[resource_providers.c.id]),
mysql_engine='InnoDB',
mysql_charset='latin1',
)
placement_aggregates = Table('placement_aggregates',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id',
Integer,
primary_key=True,
nullable=False),
Column('uuid', String(length=36), index=True),
UniqueConstraint(
'uuid',
name='uniq_placement_aggregates0uuid'),
mysql_engine='InnoDB',
mysql_charset='latin1')
aggregates = Table('aggregates',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('uuid', String(length=36)),
Column('name', String(length=255)),
Index('aggregate_uuid_idx', 'uuid'),
UniqueConstraint('name', name='uniq_aggregate0name'),
mysql_engine='InnoDB',
mysql_charset='utf8')
aggregate_hosts = Table('aggregate_hosts',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id',
Integer,
primary_key=True,
nullable=False),
Column('host', String(length=255)),
Column('aggregate_id',
Integer,
ForeignKey('aggregates.id'),
nullable=False),
UniqueConstraint(
'host',
'aggregate_id',
name='uniq_aggregate_hosts0host0aggregate_id'),
mysql_engine='InnoDB',
mysql_charset='utf8')
aggregate_metadata = Table(
'aggregate_metadata',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('aggregate_id',
Integer,
ForeignKey('aggregates.id'),
nullable=False),
Column('key', String(length=255), nullable=False),
Column('value', String(length=255), nullable=False),
UniqueConstraint('aggregate_id',
'key',
name='uniq_aggregate_metadata0aggregate_id0key'),
Index('aggregate_metadata_key_idx', 'key'),
mysql_engine='InnoDB',
mysql_charset='utf8')
groups = Table(
'instance_groups',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('user_id', String(length=255)),
Column('project_id', String(length=255)),
Column('uuid', String(length=36), nullable=False),
Column('name', String(length=255)),
UniqueConstraint('uuid', name='uniq_instance_groups0uuid'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
group_policy = Table(
'instance_group_policy',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('policy', String(length=255)),
Column('group_id',
Integer,
ForeignKey('instance_groups.id'),
nullable=False),
Index('instance_group_policy_policy_idx', 'policy'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
group_member = Table(
'instance_group_member',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('instance_uuid', String(length=255)),
Column('group_id',
Integer,
ForeignKey('instance_groups.id'),
nullable=False),
Index('instance_group_member_instance_idx', 'instance_uuid'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
quota_classes = Table(
'quota_classes',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('class_name', String(length=255)),
Column('resource', String(length=255)),
Column('hard_limit', Integer),
Index('quota_classes_class_name_idx', 'class_name'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
quota_usages = Table(
'quota_usages',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('project_id', String(length=255)),
Column('resource', String(length=255), nullable=False),
Column('in_use', Integer, nullable=False),
Column('reserved', Integer, nullable=False),
Column('until_refresh', Integer),
Column('user_id', String(length=255)),
Index('quota_usages_project_id_idx', 'project_id'),
Index('quota_usages_user_id_idx', 'user_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
quotas = Table(
'quotas',
meta,
Column('id', Integer, primary_key=True, nullable=False),
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('project_id', String(length=255)),
Column('resource', String(length=255), nullable=False),
Column('hard_limit', Integer),
UniqueConstraint('project_id',
'resource',
name='uniq_quotas0project_id0resource'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
project_user_quotas = Table(
'project_user_quotas',
meta,
Column('id', Integer, primary_key=True, nullable=False),
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('user_id', String(length=255), nullable=False),
Column('project_id', String(length=255), nullable=False),
Column('resource', String(length=255), nullable=False),
Column('hard_limit', Integer, nullable=True),
UniqueConstraint(
'user_id',
'project_id',
'resource',
name='uniq_project_user_quotas0user_id0project_id0resource'),
Index('project_user_quotas_project_id_idx', 'project_id'),
Index('project_user_quotas_user_id_idx', 'user_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
reservations = Table(
'reservations',
meta,
Column('created_at', DateTime),
Column('updated_at', DateTime),
Column('id', Integer, primary_key=True, nullable=False),
Column('uuid', String(length=36), nullable=False),
Column('usage_id',
Integer,
ForeignKey('quota_usages.id'),
nullable=False),
Column('project_id', String(length=255)),
Column('resource', String(length=255)),
Column('delta', Integer, nullable=False),
Column('expire', DateTime),
Column('user_id', String(length=255)),
Index('reservations_project_id_idx', 'project_id'),
Index('reservations_uuid_idx', 'uuid'),
Index('reservations_expire_idx', 'expire'),
Index('reservations_user_id_idx', 'user_id'),
mysql_engine='InnoDB',
mysql_charset='utf8',
)
tables = [
cell_mappings,
host_mappings,
instance_mappings,
flavors,
flavor_extra_specs,
flavor_projects,
request_specs,
build_requests,
keypairs,
projects,
users,
resource_classes,
resource_providers,
inventories,
traits,
allocations,
consumers,
resource_provider_aggregates,
resource_provider_traits,
placement_aggregates,
aggregates,
aggregate_hosts,
aggregate_metadata,
groups,
group_policy,
group_member,
quota_classes,
quota_usages,
quotas,
project_user_quotas,
reservations,
]
for table in tables:
table.create(checkfirst=True)
class RememberEntry(Base):
__tablename__ = 'remember_rejected_entry'
id = Column(Integer, primary_key=True)
added = Column(DateTime, default=datetime.now)
expires = Column(DateTime)
title = Column(Unicode)
url = Column(String)
rejected_by = Column(String)
reason = Column(String)
task_id = Column('feed_id', Integer, ForeignKey('remember_rejected_feeds.id'), nullable=False)
Index('remember_feed_title_url', RememberEntry.task_id, RememberEntry.title, RememberEntry.url)
class FilterRememberRejected(object):
"""Internal.
Rejects entries which have been rejected in the past.
This is enabled when item is rejected with remember=True flag.
Example::
entry.reject('message', remember=True)
"""
@plugin.priority(0)
def on_task_start(self, task, config):
"""Purge remembered entries if the config has changed."""
from sqlalchemy import (
Column,
Index,
Integer,
Text,
)
from .meta import Base
class PurchaseOrders(Base):
__tablename__ = 'purchase_orders'
id = Column(Integer, primary_key=True)
order_id = Column(Text)
order_date = Column(Text)
customer_name = Column(Text)
transport_name = Column(Text)
order_items = Column(Text)
Index('purchaseorders_index',
PurchaseOrders.order_id,
unique=True,
mysql_length=255)
@classmethod
def api_loading_options(cls, expand=False):
message_columns = [
'public_id', 'is_draft', 'from_addr', 'to_addr', 'cc_addr',
'bcc_addr', 'is_read', 'is_starred', 'received_date', 'is_sent'
]
if expand:
message_columns += [
'subject', 'snippet', 'version', 'from_addr', 'to_addr',
'cc_addr', 'bcc_addr', 'reply_to'
]
return (subqueryload(
Thread.messages).load_only(*message_columns).joinedload(
'messagecategories').joinedload('category'),
subqueryload(
Thread.messages).joinedload('parts').joinedload('block'))
discriminator = Column('type', String(16))
__mapper_args__ = {'polymorphic_on': discriminator}
# The /threads API endpoint filters on namespace_id and deleted_at, then orders
# by recentdate; add an explicit index to persuade MySQL to do this in a
# somewhat performant manner.
Index('ix_thread_namespace_id_recentdate_deleted_at', Thread.namespace_id,
Thread.recentdate, Thread.deleted_at)
# Need to explicitly specify the index length for MySQL 5.6, because the
# subject column is too long to be fully indexed with utf8mb4 collation.
Index('ix_thread_subject', Thread.subject, mysql_length=191)
Index('ix_cleaned_subject', Thread._cleaned_subject, mysql_length=191)
from sqlalchemy import String, Column, Index
from sqlalchemy.orm import relationship
from provider.models.base import Base
class Licensor(Base):
name = Column(String(512))
country = Column(String(3), nullable=False, default="USA")
state = Column(String(2))
city = Column(String(64))
licenses = relationship("License", back_populates="licensor")
Index("ix_monday_licensor_name_country_state_city",
Licensor.name,
Licensor.country,
Licensor.state,
Licensor.city,
unique=True)
bundle = Table(
'bundle',
db_metadata,
Column('id', Integer, primary_key=True, nullable=False),
Column('uuid', String(63), nullable=False),
Column('bundle_type', String(63), nullable=False),
# The command will be NULL except for run bundles.
Column('command', Text, nullable=True),
# The data_hash will be NULL if the bundle's value is still being computed.
Column('data_hash', String(63), nullable=True),
Column('state', String(63), nullable=False),
Column('owner_id', String(255), nullable=True),
Column('is_anonymous', Boolean, nullable=False, default=False),
UniqueConstraint('uuid', name='uix_1'),
Index('bundle_data_hash_index', 'data_hash'),
Index('state_index', 'state'), # Needed for the bundle manager.
)
# Includes things like name, description, etc.
bundle_metadata = Table(
'bundle_metadata',
db_metadata,
Column('id', Integer, primary_key=True, nullable=False),
Column('bundle_uuid',
String(63),
ForeignKey(bundle.c.uuid),
nullable=False),
Column('metadata_key', String(63), nullable=False),
Column('metadata_value', Text, nullable=False),
Index('metadata_kv_index',
__tablename__ = "tas_lookup"
tas_id = Column(Integer, primary_key=True)
allocation_transfer_agency = Column(Text, nullable=True, index=True)
agency_identifier = Column(Text, nullable=True, index=True)
beginning_period_of_availability = Column(Text, nullable=True, index=True)
ending_period_of_availability = Column(Text, nullable=True, index=True)
availability_type_code = Column(Text, nullable=True, index=True)
main_account_code = Column(Text, nullable=True, index=True)
sub_account_code = Column(Text, nullable=True, index=True)
Index("ix_tas",
TASLookup.allocation_transfer_agency,
TASLookup.agency_identifier,
TASLookup.beginning_period_of_availability,
TASLookup.ending_period_of_availability,
TASLookup.availability_type_code,
TASLookup.main_account_code,
TASLookup.sub_account_code,
unique=True)
class CGAC(Base):
__tablename__ = "cgac"
cgac_id = Column(Integer, primary_key=True)
cgac_code = Column(Text, nullable=False, index=True, unique=True)
agency_name = Column(Text)
class ObjectClass(Base):
__tablename__ = "object_class"
class Marker(MarkerMixin, Base): # TODO rename to AccidentMarker
__tablename__ = "markers"
__table_args__ = (
Index('acc_long_lat_idx', 'latitude', 'longitude'),
)
__mapper_args__ = {
'polymorphic_identity': MARKER_TYPE_ACCIDENT
}
provider_code = Column(Integer, primary_key=True)
description = Column(Text)
subtype = Column(Integer)
severity = Column(Integer)
address = Column(Text)
locationAccuracy = Column(Integer)
roadType = Column(Integer)
roadShape = Column(Integer)
dayType = Column(Integer)
unit = Column(Integer)
mainStreet = Column(Text)
secondaryStreet = Column(Text)
junction = Column(Text)
one_lane = Column(Integer)
multi_lane = Column(Integer)
speed_limit = Column(Integer)
intactness = Column(Integer)
road_width = Column(Integer)
road_sign = Column(Integer)
road_light = Column(Integer)
road_control = Column(Integer)
weather = Column(Integer)
road_surface = Column(Integer)
road_object = Column(Integer)
object_distance = Column(Integer)
didnt_cross = Column(Integer)
cross_mode = Column(Integer)
cross_location = Column(Integer)
cross_direction = Column(Integer)
@staticmethod
def json_to_description(msg):
description = json.loads(msg, encoding=db_encoding)
return "\n".join([Marker.format_description(field, value) for field, value in description.iteritems()])
def serialize(self, is_thin=False):
fields = {
"id": str(self.id),
"provider_code": self.provider_code,
"latitude": self.latitude,
"longitude": self.longitude,
"severity": self.severity,
"locationAccuracy": self.locationAccuracy,
"created": self.created.isoformat(),
}
if not is_thin:
fields.update({
"title": self.title,
"description": Marker.json_to_description(self.description),
"address": self.address,
"type": self.type,
"subtype": self.subtype,
"roadType": self.roadType,
"roadShape": self.roadShape,
"dayType": self.dayType,
"unit": self.unit,
"mainStreet": self.mainStreet,
"secondaryStreet": self.secondaryStreet,
"junction": self.junction,
})
optional = {
"one_lane": self.one_lane,
"multi_lane": self.multi_lane,
"speed_limit": self.speed_limit,
"intactness": self.intactness,
"road_width": self.road_width,
"road_sign": self.road_sign,
"road_light": self.road_light,
"road_control": self.road_control,
"weather": self.weather,
"road_surface": self.road_surface,
"road_object": self.road_object,
"object_distance": self.object_distance,
"didnt_cross": self.didnt_cross,
"cross_mode": self.cross_mode,
"cross_location": self.cross_location,
"cross_direction": self.cross_direction,
}
for name, value in optional.iteritems():
if value != 0:
fields[name] = value
return fields
def update(self, data, current_user):
self.title = data["title"]
self.description = data["description"]
self.type = data["type"]
self.latitude = data["latitude"]
self.longitude = data["longitude"]
self.put()
@staticmethod
def bounding_box_query(ne_lat, ne_lng, sw_lat, sw_lng, start_date, end_date,
fatal, severe, light, inaccurate, show_markers=True, is_thin=False, yield_per=None):
# example:
# ne_lat=32.36292402647484&ne_lng=35.08873443603511&sw_lat=32.29257266524761&sw_lng=34.88445739746089
# >>> m = Marker.bounding_box_query(32.36, 35.088, 32.292, 34.884)
# >>> m.count()
# 250
if not show_markers:
return Marker.query.filter(sql.false())
accurate = not inaccurate
markers = Marker.query \
.filter(Marker.longitude <= ne_lng) \
.filter(Marker.longitude >= sw_lng) \
.filter(Marker.latitude <= ne_lat) \
.filter(Marker.latitude >= sw_lat) \
.filter(Marker.created >= start_date) \
.filter(Marker.created < end_date) \
.order_by(desc(Marker.created))
if yield_per:
markers = markers.yield_per(yield_per)
if accurate:
markers = markers.filter(Marker.locationAccuracy == 1)
if not fatal:
markers = markers.filter(Marker.severity != 1)
if not severe:
markers = markers.filter(Marker.severity != 2)
if not light:
markers = markers.filter(Marker.severity != 3)
if is_thin:
markers = markers.options(load_only("id", "longitude", "latitude"))
return markers
@staticmethod
def get_marker(marker_id):
return Marker.query.filter_by(id=marker_id)
@classmethod
def parse(cls, data):
return Marker(
type=MARKER_TYPE_ACCIDENT,
title=data["title"],
description=data["description"],
latitude=data["latitude"],
longitude=data["longitude"]
)
class TorsionDriveProcedureORM(ProcedureMixin, BaseResultORM):
"""
A torsion drive procedure
"""
__tablename__ = 'torsiondrive_procedure'
id = Column(Integer, ForeignKey('base_result.id', ondelete='cascade'), primary_key=True)
def __init__(self, **kwargs):
kwargs.setdefault("version", 1)
self.procedure = "torsiondrive"
self.program = "torsiondrive"
super().__init__(**kwargs)
# input data (along with the mixin)
# ids of the many to many relation
initial_molecule = column_property(
select([func.array_agg(torsion_init_mol_association.c.molecule_id)])\
.where(torsion_init_mol_association.c.torsion_id==id)
)
# actual objects relation M2M, never loaded here
initial_molecule_obj = relationship(MoleculeORM,
secondary=torsion_init_mol_association,
uselist=True,
lazy='noload')
optimization_spec = Column(JSON)
# Output data
final_energy_dict = Column(JSON)
minimum_positions = Column(JSON)
optimization_history_obj = relationship(
OptimizationHistory,
cascade="all, delete-orphan", #backref="torsiondrive_procedure",
order_by=OptimizationHistory.position,
collection_class=ordering_list('position'),
lazy='selectin')
@hybrid_property
def optimization_history(self):
"""calculated property when accessed, not saved in the DB
A view of the many to many relation in the form of a dict"""
ret = {}
try:
for opt_history in self.optimization_history_obj:
if opt_history.key in ret:
ret[opt_history.key].append(str(opt_history.opt_id))
else:
ret[opt_history.key] = [str(opt_history.opt_id)]
except Exception as err:
# raises exception of first access!!
print(err)
return ret
@optimization_history.setter
def optimization_history(self, dict_values):
"""A private copy of the opt history as a dict
Key: list of optimization procedures"""
return dict_values
__table_args__ = (
Index('ix_torsion_drive_program', 'program'), # todo: needed for procedures?
)
__mapper_args__ = {
'polymorphic_identity': 'torsiondrive_procedure',
# to have separate select when querying BaseResultsORM
'polymorphic_load': 'selectin',
}
def update_relations(self, initial_molecule=None, optimization_history=None, **kwarg):
# update torsion molecule relation
self._update_many_to_many(torsion_init_mol_association, 'torsion_id', 'molecule_id', self.id, initial_molecule,
self.initial_molecule)
self.optimization_history_obj = []
for key in optimization_history:
for opt_id in optimization_history[key]:
opt_history = OptimizationHistory(torsion_id=int(self.id), opt_id=int(opt_id), key=key)
self.optimization_history_obj.append(opt_history)
class GraphDbTraceConfig(Tuple, DeclarativeBase):
__tupleType__ = graphDbTuplePrefix + 'GraphDbTraceConfigTable'
__tablename__ = 'GraphDbTraceConfig'
#: The unique ID of this segment (database generated)
id = Column(Integer, primary_key=True, autoincrement=True)
#: The model set for this segment
modelSetId = Column(Integer,
ForeignKey('GraphDbModelSet.id', ondelete='CASCADE'),
nullable=False)
modelSet = relationship(GraphDbModelSet)
#: The unique key of this segment
key = Column(String, nullable=False)
#: The unique name of this segment
name = Column(String, nullable=False)
#: The title to describe this segment
title = Column(String, nullable=False)
#: The comment for this config
comment = Column(String)
#: Is this trace config enabled
isEnabled = Column(Boolean, nullable=False, server_default="true")
#: The relationship for the rules
rules = relationship('GraphDbTraceConfigRule', lazy='joined')
__table_args__ = (
Index("idx_TraceConfig_key", modelSetId, key, unique=True),
Index("idx_TraceConfig_name", modelSetId, name, unique=True),
)
def fromTuple(self, tupleIn: GraphDbTraceConfigTuple,
modelSetId: int) -> 'GraphDbTraceConfig':
self.modelSetId = modelSetId
self.key = tupleIn.key
self.name = tupleIn.name
self.title = tupleIn.title
self.comment = tupleIn.comment
self.isEnabled = tupleIn.isEnabled
self.rules = [GraphDbTraceConfigRule().fromTuple(rule, self.id)
for rule in tupleIn.rules]
return self
def toTuple(self) -> GraphDbTraceConfigTuple:
traceTuple = GraphDbTraceConfigTuple(
modelSetKey=self.modelSet.key,
key=self.key,
name=self.name,
title=self.title,
comment=self.comment,
isEnabled=self.isEnabled
)
traceTuple.rules = [rule.toTuple() for rule in self.rules]
return traceTuple
metadata = MetaData()
# 表COOKIES
cookies = Table(
"COOKIES",
metadata,
Column("cookie_id", Integer(), primary_key=True),
Column("cookie_name", String(50)), # index = True
Column("cookie_recipe_url", String(255)),
Column("cookie_sku", String(55)),
Column("quantity", Integer()),
Column("unit_cost", Numeric(12, 2)),
# 检查约束
CheckConstraint("unit_cost >= 0.0", name="unit_cost_positive"),
# 索引
Index("ix_cookies_cookie_name", "cookie_name"),
Index("ix_cookie_sku_name", "cookie_sku", "cookie_name"))
# 使用Table中字段
# Index("ix_cookies_cookie_name", cookies.c.cookie_name)
# Index("ix_cookie_sku_name", cookies.c.cookie_sku,
# cookies.c.cookie_name)
# 表USERS
users = Table(
"USERS",
metadata,
Column("user_id", Integer()), # primary_key = True
Column("username", String(15), nullable=False), # unique = True
Column("email_address", String(255), nullable=False),
Column("phone", String(20), nullable=False),
Column("password", String(25), nullable=False),
DateTime,
)
from .meta import Base
class User(Base):
__tablename__ = 'users'
id = Column(Integer, primary_key=True)
username = Column(Text)
password = Column(Text)
userlevel = Column(Integer)
carrierid = Column(Integer)
cmdr_name = Column(Text)
access_token = Column(Text)
refresh_token = Column(Text)
token_expiration = Column(Integer)
has_validated = Column(Boolean)
public_carrier = Column(Boolean)
no_carrier = Column(Boolean)
banned = Column(Boolean)
cachedJson = Column(Text)
apiKey = Column(Text)
avatar = Column(Text)
squadron = Column(Text)
squadron_decal = Column(Text)
lastUpdated = Column(DateTime)
Index('user_index', User.username, unique=True)
def test_drop_plain(self):
self.assert_compile(schema.DropIndex(Index(name="xyz")),
"DROP INDEX xyz")
class Auth(Base, CreateAt):
__tablename__ = PREFIX_DB + 'auth'
__table_args__ = (Index("email", unique=True), )
status = Column(Integer, nullable=False, default='0', server_default='0')
email = Column(String(255), nullable=False)
