def downgrade(migrate_engine):
    meta.bind = migrate_engine

    records_table = Table("records", meta, autoload=True)

    name_idx = Index("rec_name_index", records_table.c.name)
    name_idx.create()
Ejemplo n.º 2
0
def upgrade(migrate_engine):
    metadata.bind = migrate_engine
    print(__doc__)
    metadata.reflect()
    try:
        if migrate_engine.name == 'mysql':
            # Strip slug index prior to creation so we can do it manually.
            slug_index = None
            for ix in Page_table.indexes:
                if ix.name == 'ix_page_slug':
                    slug_index = ix
            Page_table.indexes.remove(slug_index)
        Page_table.create()
        if migrate_engine.name == 'mysql':
            # Create slug index manually afterward.
            i = Index("ix_page_slug", Page_table.c.slug, mysql_length=200)
            i.create()
    except Exception:
        log.exception("Could not create page table")
    try:
        PageRevision_table.create()
    except Exception:
        log.exception("Could not create page_revision table")

    # Add 1 column to the user table
    User_table = Table("galaxy_user", metadata, autoload=True)
    col = Column('username', String(255), index=True, unique=True, default=False)
    col.create(User_table, index_name='ix_user_username', unique_name='username')
    assert col is User_table.c.username
Ejemplo n.º 3
0
def upgrade(migrate_engine):
    display_migration_details()
    metadata.bind = migrate_engine
    db_session = scoped_session( sessionmaker( bind=migrate_engine, autoflush=False, autocommit=True ) )
    HistoryDatasetAssociation_table = Table( "history_dataset_association", metadata, autoload=True )
    # Load existing tables
    metadata.reflect()
    # Add 2 indexes to the galaxy_user table
    i = Index( 'ix_hda_extension', HistoryDatasetAssociation_table.c.extension )
    try:
        i.create()
    except Exception as e:
        log.debug( "Adding index 'ix_hda_extension' to history_dataset_association table failed: %s" % ( str( e ) ) )

    # Set the default data in the galaxy_user table, but only for null values
    cmd = "UPDATE history_dataset_association SET extension = 'qual454' WHERE extension = 'qual' and peek like \'>%%\'"
    try:
        db_session.execute( cmd )
    except Exception as e:
        log.debug( "Resetting extension qual to qual454 in history_dataset_association failed: %s" % ( str( e ) ) )
    cmd = "UPDATE history_dataset_association SET extension = 'qualsolexa' WHERE extension = 'qual' and peek not like \'>%%\'"
    try:
        db_session.execute( cmd )
    except Exception as e:
        log.debug( "Resetting extension qual to qualsolexa in history_dataset_association failed: %s" % ( str( e ) ) )
    # Add 1 index to the history_dataset_association table
    try:
        i.drop()
    except Exception as e:
        log.debug( "Dropping index 'ix_hda_extension' to history_dataset_association table failed: %s" % ( str( e ) ) )
def upgrade(migrate_engine):
    meta.bind = migrate_engine

    records_table = Table("records", meta, autoload=True)

    index = Index("designate_recordset_id", records_table.c.designate_recordset_id)
    index.create()
def downgrade(migrate_engine):
    meta.bind = migrate_engine

    records_table = Table('records', meta, autoload=True)
    index = Index('designate_recordset_id',
                  records_table.c.designate_recordset_id)
    index.drop()
def loadTable(datapath, datatable, delimiter, dtype, engine, indexCols=[], skipLines=1, chunkSize=100000, **kwargs):
    cnt = 0
    with open(datapath) as fh:
        while cnt < skipLines:
            fh.readline()
            cnt += 1
        cnt = 0
        tmpstr = ''
        for l in fh:
            tmpstr += l
            cnt += 1
            if cnt%chunkSize == 0:
                print "Loading chunk #%i"%(int(cnt/chunkSize))
                dataArr = numpy.genfromtxt(StringIO(tmpstr), dtype=dtype, delimiter=delimiter, **kwargs)
                engine.execute(datatable.insert(), [dict((name, numpy.asscalar(l[name])) for name in l.dtype.names) for l in dataArr])
                tmpstr = ''
        #Clean up the last chunk
        if len(tmpstr) > 0:
            dataArr = numpy.genfromtxt(StringIO(tmpstr), dtype=dtype, delimiter=delimiter, **kwargs)
            try:
                engine.execute(datatable.insert(), [dict((name, numpy.asscalar(l[name])) for name in l.dtype.names) for l in dataArr])
            # If the file only has one line, the result of genfromtxt is a 0-d array, so cannot be iterated
            except TypeError:
                engine.execute(datatable.insert(), [dict((name, numpy.asscalar(dataArr[name])) for name in dataArr.dtype.names),])

    for col in indexCols:
        if hasattr(col, "__iter__"):
            print "Creating index on %s"%(",".join(col))
            colArr = (datatable.c[c] for c in col)
            i = Index('%sidx'%''.join(col), *colArr)
        else:
            print "Creating index on %s"%(col)
            i = Index('%sidx'%col, datatable.c[col])

        i.create(engine)
Ejemplo n.º 7
0
def upgrade(migrate_engine):
    print(__doc__)
    metadata.bind = migrate_engine
    HistoryDatasetAssociation_table = Table("history_dataset_association", metadata, autoload=True)
    # Load existing tables
    metadata.reflect()
    # Add 2 indexes to the galaxy_user table
    i = Index('ix_hda_extension', HistoryDatasetAssociation_table.c.extension)
    try:
        i.create()
    except Exception:
        log.exception("Adding index 'ix_hda_extension' to history_dataset_association table failed.")

    # Set the default data in the galaxy_user table, but only for null values
    cmd = "UPDATE history_dataset_association SET extension = 'qual454' WHERE extension = 'qual' and peek like \'>%%\'"
    try:
        migrate_engine.execute(cmd)
    except Exception:
        log.exception("Resetting extension qual to qual454 in history_dataset_association failed.")
    cmd = "UPDATE history_dataset_association SET extension = 'qualsolexa' WHERE extension = 'qual' and peek not like \'>%%\'"
    try:
        migrate_engine.execute(cmd)
    except Exception:
        log.exception("Resetting extension qual to qualsolexa in history_dataset_association failed.")
    # Add 1 index to the history_dataset_association table
    try:
        i.drop()
    except Exception:
        log.exception("Dropping index 'ix_hda_extension' to history_dataset_association table failed.")
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    t = Table("aggregate_metadata", meta, autoload=True)
    i = Index("aggregate_metadata_key_idx", t.c.key)
    i.drop(migrate_engine)
def upgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    build_requests = Table('build_requests', meta, autoload=True)

    columns_to_add = [
            ('instance_uuid',
                Column('instance_uuid', String(length=36))),
            ('instance',
                Column('instance', Text())),
    ]
    for (col_name, column) in columns_to_add:
        if not hasattr(build_requests.c, col_name):
            build_requests.create_column(column)

    for index in build_requests.indexes:
        if [c.name for c in index.columns] == ['instance_uuid']:
            break
    else:
        index = Index('build_requests_instance_uuid_idx',
                build_requests.c.instance_uuid)
        index.create()

    inspector = reflection.Inspector.from_engine(migrate_engine)
    constrs = inspector.get_unique_constraints('build_requests')
    constr_names = [constr['name'] for constr in constrs]
    if 'uniq_build_requests0instance_uuid' not in constr_names:
        UniqueConstraint('instance_uuid', table=build_requests,
                name='uniq_build_requests0instance_uuid').create()
Ejemplo n.º 10
0
def downgrade(migrate_engine):
    metadata.bind = migrate_engine
    metadata.reflect()
    # Drop the library_folder_id column
    try:
        Job_table = Table( "job", metadata, autoload=True )
    except NoSuchTableError:
        Job_table = None
        log.debug( "Failed loading table job" )
    if Job_table is not None:
        try:
            col = Job_table.c.library_folder_id
            col.drop()
        except Exception as e:
            log.debug( "Dropping column 'library_folder_id' from job table failed: %s" % ( str( e ) ) )
    # Drop the job_to_output_library_dataset table
    try:
        JobToOutputLibraryDatasetAssociation_table.drop()
    except Exception as e:
        print(str(e))
        log.debug( "Dropping job_to_output_library_dataset table failed: %s" % str( e ) )
    # Drop the ix_dataset_state index
    try:
        Dataset_table = Table( "dataset", metadata, autoload=True )
    except NoSuchTableError:
        Dataset_table = None
        log.debug( "Failed loading table dataset" )
    i = Index( "ix_dataset_state", Dataset_table.c.state )
    try:
        i.drop()
    except Exception as e:
        print(str(e))
        log.debug( "Dropping index 'ix_dataset_state' from dataset table failed: %s" % str( e ) )
Ejemplo n.º 11
0
def _mk_index_on(engine, ts_name):
    fc_table = introspect_table(engine, "{}_RegressionIndicator".format(ts_name))

    fast_fc_lookup = Index('idx_fast_ri_lookup', fc_table.c.RegressionID)
    try:
        fast_fc_lookup.create(engine)
    except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.ProgrammingError) as e:
        logger.warning("Skipping index creation on {}, because of {}".format(fc_table.name, e.message))
def downgrade(migrate_engine):
    metadata.bind = migrate_engine
    metadata.reflect()
    i = Index("ix_hda_ta_history_dataset_association_id", HistoryDatasetAssociationTagAssociation_table.c.history_dataset_association_id)
    try:
        i.drop()
    except Exception:
        log.exception("Removing index 'ix_hdata_history_dataset_association_id' to table 'history_dataset_association_tag_association' table failed.")
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    t = Table('instance_faults', meta, autoload=True)
    i = Index('instance_faults_instance_uuid_deleted_created_at_idx',
              t.c.instance_uuid, t.c.deleted, t.c.created_at)
    i.drop(migrate_engine)
def upgrade(migrate_engine):
    """Add an index to make the scheduler lookups of compute_nodes and joined
    compute_node_stats more efficient.
    """
    meta = MetaData(bind=migrate_engine)
    cn_stats = Table(TABLE_NAME, meta, autoload=True)
    idx = Index(IDX_NAME, cn_stats.c.compute_node_id, cn_stats.c.deleted)
    idx.create(migrate_engine)
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    t = Table('instance_type_extra_specs', meta, autoload=True)
    i = Index('instance_type_extra_specs_instance_type_id_key_idx',
              t.c.instance_type_id, t.c.key)
    i.drop(migrate_engine)
Ejemplo n.º 16
0
 def _create_indexes(self):
     # create an index on timeseries values if it doesn't exist
     try:
         i = Index('ix_timeseries_values_id',
                   model.DataValue.__table__.c.timeseries_id)
         i.create(self.engine)
     except OperationalError:
         pass
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    t = Table('bw_usage_cache', meta, autoload=True)
    i = Index('bw_usage_cache_uuid_start_period_idx',
              t.c.uuid, t.c.start_period)
    i.drop(migrate_engine)
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    t = Table('migrations', meta, autoload=True)
    i = Index('migrations_instance_uuid_and_status_idx', t.c.deleted,
            t.c.instance_uuid, t.c.status)
    i.drop(migrate_engine)
Ejemplo n.º 19
0
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    jobs = Table('jobs', meta, autoload=True)

    index = Index(INDEX_NAME, jobs.c.hard_timeout)
    index.drop(migrate_engine)
Ejemplo n.º 20
0
def upgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    images = Table('images', meta, autoload=True)

    index = Index(INDEX_NAME, images.c.checksum)
    index.create(migrate_engine)
Ejemplo n.º 21
0
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    images = Table('images', meta, autoload=True)

    index = Index(INDEX_NAME, images.c.owner)
    index.drop(migrate_engine)
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    reservations = Table('reservations', meta, autoload=True)

    index = Index('reservations_uuid_idx', reservations.c.uuid)
    index.drop(migrate_engine)
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    t = Table('agent_builds', meta, autoload=True)
    i = Index('agent_builds_hypervisor_os_arch_idx',
              t.c.hypervisor, t.c.os, t.c.architecture)
    i.drop(migrate_engine)
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    t = Table('dns_domains', meta, autoload=True)
    i = Index('dns_domains_domain_deleted_idx',
              t.c.domain, t.c.deleted)
    i.drop(migrate_engine)
Ejemplo n.º 25
0
def _mk_index_on(engine, ts_name):
    fc_table = introspect_table(engine, "{}_FieldChangeV2".format(ts_name))

    fast_fc_lookup = Index('idx_fast_fieldchange_lookup', fc_table.c.StartOrderID)
    try:
        fast_fc_lookup.create(engine)
    except (sqlalchemy.exc.OperationalError, sqlalchemy.exc.ProgrammingError) as e:
        logger.warning("Skipping index creation on {}, because of {}".format(fc_table.name, e.message))
def upgrade(migrate_engine):
    table, index = _get_table_index(migrate_engine)
    if index:
        LOG.info('Skipped adding compute_nodes_uuid_idx because an '
                 'equivalent index already exists.')
        return
    index = Index('compute_nodes_uuid_idx', table.c.uuid, unique=True)
    index.create(migrate_engine)
Ejemplo n.º 27
0
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    schedules = Table('schedules', meta, autoload=True)

    index = Index(INDEX_NAME, schedules.c.next_run)
    index.drop(migrate_engine)
Ejemplo n.º 28
0
def downgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    t = Table('key_pairs', meta, autoload=True)
    i = Index('key_pair_user_id_name_idx',
              t.c.user_id, t.c.name)
    i.drop(migrate_engine)
Ejemplo n.º 29
0
def upgrade(migrate_engine):
    meta = MetaData(bind=migrate_engine)
    load_tables = dict((table_name, Table(table_name, meta, autoload=True))
                       for table_name in INDEXES.keys())
    for table_name, indexes in INDEXES.items():
        table = load_tables[table_name]
        for index_name, column in indexes:
            index = Index(index_name, table.c[column])
            index.drop()
def upgrade(migrate_engine):
    meta, table, index = _get_table_index(migrate_engine)
    if index:
        LOG.info('Skipped adding %s because an equivalent index'
                 ' already exists.', INDEX_NAME)
        return
    columns = [getattr(table.c, col_name) for col_name in INDEX_COLUMNS]
    index = Index(INDEX_NAME, *columns)
    index.create(migrate_engine)
Ejemplo n.º 31
0
class BaseResultORM(Base):
    """
        Abstract Base class for ResultORMs and ProcedureORMs
    """

    __tablename__ = 'base_result'

    # for SQL
    result_type = Column(String)  # for inheritance
    task_id = Column(String)  # TODO: not used, for back compatibility

    # Base identification
    id = Column(Integer, primary_key=True)
    hash_index = Column(String)  # TODO
    procedure = Column(String(100))  # TODO: may remove
    # program = Column(String(100))  # moved to subclasses
    version = Column(Integer)

    # Extra fields
    extras = Column(JSON)
    stdout = Column(Integer, ForeignKey('kv_store.id'))
    stdout_obj = relationship(KVStoreORM,
                              lazy='noload',
                              foreign_keys=stdout,
                              cascade="all, delete-orphan",
                              single_parent=True)

    stderr = Column(Integer, ForeignKey('kv_store.id'))
    stderr_obj = relationship(KVStoreORM,
                              lazy='noload',
                              foreign_keys=stderr,
                              cascade="all, delete-orphan",
                              single_parent=True)

    error = Column(Integer, ForeignKey('kv_store.id'))
    error_obj = relationship(KVStoreORM,
                             lazy='noload',
                             foreign_keys=error,
                             cascade="all, delete-orphan",
                             single_parent=True)

    # Compute status
    status = Column(Enum(RecordStatusEnum), nullable=False, default=RecordStatusEnum.incomplete)

    created_on = Column(DateTime, default=datetime.datetime.utcnow)
    modified_on = Column(DateTime, default=datetime.datetime.utcnow)

    # Carry-ons
    provenance = Column(JSON)

    __table_args__ = (
        Index('ix_base_result_status', 'status'),
        Index('ix_base_result_type', 'result_type'),  # todo: needed?
    )

    # meta = {
    #     # 'allow_inheritance': True,
    #     'indexes': ['status']
    # }

    # def save(self, *args, **kwargs):
    #     """Override save to set defaults"""
    #
    #     self.modified_on = datetime.datetime.utcnow()
    #     if not self.created_on:
    #         self.created_on = datetime.datetime.utcnow()
    #
    #     return super(BaseResultORM, self).save(*args, **kwargs)

    __mapper_args__ = {'polymorphic_on': 'result_type'}
Ejemplo n.º 32
0
class GridOptimizationProcedureORM(ProcedureMixin, BaseResultORM):

    __tablename__ = "grid_optimization_procedure"

    id = Column(Integer, ForeignKey('base_result.id', ondelete='cascade'), primary_key=True)

    def __init__(self, **kwargs):
        kwargs.setdefault("version", 1)
        kwargs.setdefault("procedure", "gridoptimization")
        kwargs.setdefault("program", "qcfractal")
        super().__init__(**kwargs)

    # Input data
    initial_molecule = Column(Integer, ForeignKey('molecule.id'))
    initial_molecule_obj = relationship(MoleculeORM, lazy='select', foreign_keys=initial_molecule)

    optimization_spec = Column(JSON)

    # Output data
    starting_molecule = Column(Integer, ForeignKey('molecule.id'))
    starting_molecule_obj = relationship(MoleculeORM, lazy='select', foreign_keys=initial_molecule)

    final_energy_dict = Column(JSON)  # Dict[str, float]
    starting_grid = Column(JSON)  # tuple

    grid_optimizations_obj = relationship(GridOptimizationAssociation,
                                          lazy='selectin',
                                          cascade="all, delete-orphan",
                                          backref="grid_optimization_procedure")

    @hybrid_property
    def grid_optimizations(self):
        """calculated property when accessed, not saved in the DB
        A view of the many to many relation in the form of a dict"""

        ret = {}
        try:
            for obj in self.grid_optimizations_obj:
                ret[obj.key] = str(obj.opt_id)

        except Exception as err:
            # raises exception of first access!!
            print(err)

        return ret

    @grid_optimizations.setter
    def grid_optimizations(self, dict_values):

        return dict_values

    __table_args__ = (
        Index('ix_grid_optmization_program', 'program'),  # todo: needed for procedures?
    )

    __mapper_args__ = {
        'polymorphic_identity': 'grid_optimization_procedure',
        # to have separate select when querying BaseResultsORM
        'polymorphic_load': 'selectin',
    }

    def update_relations(self, grid_optimizations=None, **kwarg):

        self.grid_optimizations_obj = []
        for key, opt_id in grid_optimizations.items():
            obj = GridOptimizationAssociation(grid_opt_id=int(self.id), opt_id=int(opt_id), key=key)
            self.grid_optimizations_obj.append(obj)
Ejemplo n.º 33
0
class DagRun(Base, LoggingMixin):
    """
    DagRun describes an instance of a Dag. It can be created
    by the scheduler (for regular runs) or by an external trigger
    """
    __tablename__ = "dag_run"

    ID_PREFIX = 'scheduled__'
    ID_FORMAT_PREFIX = ID_PREFIX + '{0}'

    id = Column(Integer, primary_key=True)
    dag_id = Column(String(ID_LEN))
    execution_date = Column(UtcDateTime, default=timezone.utcnow)
    start_date = Column(UtcDateTime, default=timezone.utcnow)
    end_date = Column(UtcDateTime)
    _state = Column('state', String(50), default=State.RUNNING)
    run_id = Column(String(ID_LEN))
    external_trigger = Column(Boolean, default=True)
    conf = Column(PickleType)

    dag = None

    __table_args__ = (
        Index('dag_id_state', dag_id, _state),
        UniqueConstraint('dag_id', 'execution_date'),
        UniqueConstraint('dag_id', 'run_id'),
    )

    def __repr__(self):
        return ('<DagRun {dag_id} @ {execution_date}: {run_id}, '
                'externally triggered: {external_trigger}>').format(
                    dag_id=self.dag_id,
                    execution_date=self.execution_date,
                    run_id=self.run_id,
                    external_trigger=self.external_trigger)

    def get_state(self):
        return self._state

    def set_state(self, state):
        if self._state != state:
            self._state = state
            self.end_date = timezone.utcnow() if self._state in State.finished(
            ) else None

    @declared_attr
    def state(self):
        return synonym('_state',
                       descriptor=property(self.get_state, self.set_state))

    @classmethod
    def id_for_date(cls, date, prefix=ID_FORMAT_PREFIX):
        return prefix.format(date.isoformat()[:19])

    @provide_session
    def refresh_from_db(self, session=None):
        """
        Reloads the current dagrun from the database
        :param session: database session
        """
        DR = DagRun

        exec_date = func.cast(self.execution_date, DateTime)

        dr = session.query(DR).filter(
            DR.dag_id == self.dag_id,
            func.cast(DR.execution_date, DateTime) == exec_date,
            DR.run_id == self.run_id).one()

        self.id = dr.id
        self.state = dr.state

    @staticmethod
    @provide_session
    def find(dag_id=None,
             run_id=None,
             execution_date=None,
             state=None,
             external_trigger=None,
             no_backfills=False,
             session=None):
        """
        Returns a set of dag runs for the given search criteria.

        :param dag_id: the dag_id to find dag runs for
        :type dag_id: int, list
        :param run_id: defines the the run id for this dag run
        :type run_id: str
        :param execution_date: the execution date
        :type execution_date: datetime.datetime
        :param state: the state of the dag run
        :type state: str
        :param external_trigger: whether this dag run is externally triggered
        :type external_trigger: bool
        :param no_backfills: return no backfills (True), return all (False).
            Defaults to False
        :type no_backfills: bool
        :param session: database session
        :type session: sqlalchemy.orm.session.Session
        """
        DR = DagRun

        qry = session.query(DR)
        if dag_id:
            qry = qry.filter(DR.dag_id == dag_id)
        if run_id:
            qry = qry.filter(DR.run_id == run_id)
        if execution_date:
            if isinstance(execution_date, list):
                qry = qry.filter(DR.execution_date.in_(execution_date))
            else:
                qry = qry.filter(DR.execution_date == execution_date)
        if state:
            qry = qry.filter(DR.state == state)
        if external_trigger is not None:
            qry = qry.filter(DR.external_trigger == external_trigger)
        if no_backfills:
            # in order to prevent a circular dependency
            from airflow.jobs import BackfillJob
            qry = qry.filter(DR.run_id.notlike(BackfillJob.ID_PREFIX + '%'))

        dr = qry.order_by(DR.execution_date).all()

        return dr

    @provide_session
    def get_task_instances(self, state=None, session=None):
        """
        Returns the task instances for this dag run
        """
        from airflow.models.taskinstance import TaskInstance  # Avoid circular import
        tis = session.query(TaskInstance).filter(
            TaskInstance.dag_id == self.dag_id,
            TaskInstance.execution_date == self.execution_date,
        )
        if state:
            if isinstance(state, six.string_types):
                tis = tis.filter(TaskInstance.state == state)
            else:
                # this is required to deal with NULL values
                if None in state:
                    tis = tis.filter(
                        or_(TaskInstance.state.in_(state),
                            TaskInstance.state.is_(None)))
                else:
                    tis = tis.filter(TaskInstance.state.in_(state))

        if self.dag and self.dag.partial:
            tis = tis.filter(TaskInstance.task_id.in_(self.dag.task_ids))

        return tis.all()

    @provide_session
    def get_task_instance(self, task_id, session=None):
        """
        Returns the task instance specified by task_id for this dag run

        :param task_id: the task id
        """

        from airflow.models.taskinstance import TaskInstance  # Avoid circular import
        TI = TaskInstance
        ti = session.query(TI).filter(TI.dag_id == self.dag_id,
                                      TI.execution_date == self.execution_date,
                                      TI.task_id == task_id).first()

        return ti

    def get_dag(self):
        """
        Returns the Dag associated with this DagRun.

        :return: DAG
        """
        if not self.dag:
            raise AirflowException(
                "The DAG (.dag) for {} needs to be set".format(self))

        return self.dag

    @provide_session
    def get_previous_dagrun(self, state=None, session=None):
        # type: (Optional[str], Optional[Session]) -> Optional['DagRun']
        """The previous DagRun, if there is one"""

        session = cast(Session, session)  # mypy

        filters = [
            DagRun.dag_id == self.dag_id,
            DagRun.execution_date < self.execution_date,
        ]
        if state is not None:
            filters.append(DagRun.state == state)
        return session.query(DagRun).filter(*filters).order_by(
            DagRun.execution_date.desc()).first()

    @provide_session
    def get_previous_scheduled_dagrun(self, session=None):
        """The previous, SCHEDULED DagRun, if there is one"""
        dag = self.get_dag()

        return session.query(DagRun).filter(
            DagRun.dag_id == self.dag_id,
            DagRun.execution_date == dag.previous_schedule(
                self.execution_date)).first()

    @provide_session
    def update_state(self, session=None):
        """
        Determines the overall state of the DagRun based on the state
        of its TaskInstances.

        :return: State
        """

        dag = self.get_dag()

        tis = self.get_task_instances(session=session)
        self.log.debug("Updating state for %s considering %s task(s)", self,
                       len(tis))

        for ti in list(tis):
            # skip in db?
            if ti.state == State.REMOVED:
                tis.remove(ti)
            else:
                ti.task = dag.get_task(ti.task_id)

        # pre-calculate
        # db is faster
        start_dttm = timezone.utcnow()
        unfinished_tasks = self.get_task_instances(state=State.unfinished(),
                                                   session=session)
        none_depends_on_past = all(not t.task.depends_on_past
                                   for t in unfinished_tasks)
        none_task_concurrency = all(t.task.task_concurrency is None
                                    for t in unfinished_tasks)
        # small speed up
        if unfinished_tasks and none_depends_on_past and none_task_concurrency:
            # todo: this can actually get pretty slow: one task costs between 0.01-015s
            no_dependencies_met = True
            for ut in unfinished_tasks:
                # We need to flag upstream and check for changes because upstream
                # failures/re-schedules can result in deadlock false positives
                old_state = ut.state
                deps_met = ut.are_dependencies_met(dep_context=DepContext(
                    flag_upstream_failed=True,
                    ignore_in_retry_period=True,
                    ignore_in_reschedule_period=True),
                                                   session=session)
                if deps_met or old_state != ut.current_state(session=session):
                    no_dependencies_met = False
                    break

        duration = (timezone.utcnow() - start_dttm).total_seconds() * 1000
        Stats.timing("dagrun.dependency-check.{}".format(self.dag_id),
                     duration)

        root_ids = [t.task_id for t in dag.roots]
        roots = [t for t in tis if t.task_id in root_ids]

        # if all roots finished and at least one failed, the run failed
        if (not unfinished_tasks
                and any(r.state in (State.FAILED, State.UPSTREAM_FAILED)
                        for r in roots)):
            self.log.info('Marking run %s failed', self)
            self.set_state(State.FAILED)
            dag.handle_callback(self,
                                success=False,
                                reason='task_failure',
                                session=session)

        # if all roots succeeded and no unfinished tasks, the run succeeded
        elif not unfinished_tasks and all(
                r.state in (State.SUCCESS, State.SKIPPED) for r in roots):
            self.log.info('Marking run %s successful', self)
            self.set_state(State.SUCCESS)
            dag.handle_callback(self,
                                success=True,
                                reason='success',
                                session=session)

        # if *all tasks* are deadlocked, the run failed
        elif (unfinished_tasks and none_depends_on_past
              and none_task_concurrency and no_dependencies_met):
            self.log.info('Deadlock; marking run %s failed', self)
            self.set_state(State.FAILED)
            dag.handle_callback(self,
                                success=False,
                                reason='all_tasks_deadlocked',
                                session=session)

        # finally, if the roots aren't done, the dag is still running
        else:
            self.set_state(State.RUNNING)

        self._emit_duration_stats_for_finished_state()

        # todo: determine we want to use with_for_update to make sure to lock the run
        session.merge(self)
        session.commit()

        return self.state

    def _emit_duration_stats_for_finished_state(self):
        if self.state == State.RUNNING:
            return

        duration = (self.end_date - self.start_date)
        if self.state is State.SUCCESS:
            Stats.timing('dagrun.duration.success.{}'.format(self.dag_id),
                         duration)
        elif self.state == State.FAILED:
            Stats.timing('dagrun.duration.failed.{}'.format(self.dag_id),
                         duration)

    @provide_session
    def verify_integrity(self, session=None):
        """
        Verifies the DagRun by checking for removed tasks or tasks that are not in the
        database yet. It will set state to removed or add the task if required.
        """
        from airflow.models.taskinstance import TaskInstance  # Avoid circular import

        dag = self.get_dag()
        tis = self.get_task_instances(session=session)

        # check for removed or restored tasks
        task_ids = []
        for ti in tis:
            task_ids.append(ti.task_id)
            task = None
            try:
                task = dag.get_task(ti.task_id)
            except AirflowException:
                if ti.state == State.REMOVED:
                    pass  # ti has already been removed, just ignore it
                elif self.state is not State.RUNNING and not dag.partial:
                    self.log.warning("Failed to get task '{}' for dag '{}'. "
                                     "Marking it as removed.".format(ti, dag))
                    Stats.incr("task_removed_from_dag.{}".format(dag.dag_id),
                               1, 1)
                    ti.state = State.REMOVED

            is_task_in_dag = task is not None
            should_restore_task = is_task_in_dag and ti.state == State.REMOVED
            if should_restore_task:
                self.log.info("Restoring task '{}' which was previously "
                              "removed from DAG '{}'".format(ti, dag))
                Stats.incr("task_restored_to_dag.{}".format(dag.dag_id), 1, 1)
                ti.state = State.NONE

        # check for missing tasks
        for task in six.itervalues(dag.task_dict):
            if task.start_date > self.execution_date and not self.is_backfill:
                continue

            if task.task_id not in task_ids:
                Stats.incr(
                    "task_instance_created-{}".format(task.__class__.__name__),
                    1, 1)
                ti = TaskInstance(task, self.execution_date)
                session.add(ti)

        session.commit()

    @staticmethod
    def get_run(session, dag_id, execution_date):
        """
        :param dag_id: DAG ID
        :type dag_id: unicode
        :param execution_date: execution date
        :type execution_date: datetime
        :return: DagRun corresponding to the given dag_id and execution date
            if one exists. None otherwise.
        :rtype: airflow.models.DagRun
        """
        qry = session.query(DagRun).filter(
            DagRun.dag_id == dag_id,
            DagRun.external_trigger == False,  # noqa
            DagRun.execution_date == execution_date,
        )
        return qry.first()

    @property
    def is_backfill(self):
        from airflow.jobs import BackfillJob
        return (self.run_id is not None
                and self.run_id.startswith(BackfillJob.ID_PREFIX))

    @classmethod
    @provide_session
    def get_latest_runs(cls, session):
        """Returns the latest DagRun for each DAG. """
        subquery = (session.query(
            cls.dag_id,
            func.max(cls.execution_date).label('execution_date')).group_by(
                cls.dag_id).subquery())
        dagruns = (session.query(cls).join(
            subquery,
            and_(cls.dag_id == subquery.c.dag_id,
                 cls.execution_date == subquery.c.execution_date)).all())
        return dagruns
    Column("user_value", Unicode(255), index=True))

# Annotating visualizations.

VisualizationAnnotationAssociation_table = Table(
    "visualization_annotation_association",
    metadata,
    Column("id", Integer, primary_key=True),
    Column("visualization_id",
           Integer,
           ForeignKey("visualization.id"),
           index=True),
    Column("user_id", Integer, ForeignKey("galaxy_user.id"), index=True),
    Column("annotation", TEXT),
    Index('ix_visualization_annotation_association_annotation',
          'annotation',
          mysql_length=200),
)

TABLES = [
    VisualizationUserShareAssociation_table, VisualizationTagAssociation_table,
    VisualizationAnnotationAssociation_table
]


def upgrade(migrate_engine):
    print(__doc__)
    metadata.bind = migrate_engine
    metadata.reflect()

    for table in TABLES:
Ejemplo n.º 35
0
    # - app_id: The referenced app's primary key
    # - app_client_id: the app's public key, used to identify it externally;
    #                  included here so it appears in deltas.
    # - app_type: 'plugin' or 'app', for future use to filter deltas
    app_id = Column(Integer)
    app_client_id = Column(Base36UID, nullable=False)
    app_type = Column(String(20), nullable=False)

    namespace_id = Column(ForeignKey(Namespace.id, ondelete='CASCADE'),
                          nullable=False)
    namespace = relationship(Namespace)

    # Reference to the object that this metadata is about. Public ID is the
    # external identifier, while type and id allow direct lookup of the object.
    object_public_id = Column(String(191), nullable=False, index=True)
    object_type = Column(String(20), nullable=False)
    object_id = Column(BigInteger, nullable=False)

    value = Column(JSON)

    queryable_value = Column(BigInteger, nullable=True, index=True)

    version = Column(Integer, nullable=True, server_default='0')


Index('ix_obj_public_id_app_id',
      Metadata.object_public_id,
      Metadata.app_id,
      unique=True)
Index('ix_namespace_id_app_id', Metadata.namespace_id, Metadata.app_id)
Ejemplo n.º 36
0
class TaskReschedule(Base):
    """TaskReschedule tracks rescheduled task instances."""

    __tablename__ = "task_reschedule"

    id = Column(Integer, primary_key=True)
    task_id = Column(String(ID_LEN, **COLLATION_ARGS), nullable=False)
    dag_id = Column(String(ID_LEN, **COLLATION_ARGS), nullable=False)
    run_id = Column(String(ID_LEN, **COLLATION_ARGS), nullable=False)
    map_index = Column(Integer, nullable=False, default=-1)
    try_number = Column(Integer, nullable=False)
    start_date = Column(UtcDateTime, nullable=False)
    end_date = Column(UtcDateTime, nullable=False)
    duration = Column(Integer, nullable=False)
    reschedule_date = Column(UtcDateTime, nullable=False)

    __table_args__ = (
        Index('idx_task_reschedule_dag_task_run', dag_id, task_id, run_id, map_index, unique=False),
        ForeignKeyConstraint(
            [dag_id, task_id, run_id, map_index],
            [
                "task_instance.dag_id",
                "task_instance.task_id",
                "task_instance.run_id",
                "task_instance.map_index",
            ],
            name="task_reschedule_ti_fkey",
            ondelete="CASCADE",
        ),
        ForeignKeyConstraint(
            [dag_id, run_id],
            ['dag_run.dag_id', 'dag_run.run_id'],
            name='task_reschedule_dr_fkey',
            ondelete='CASCADE',
        ),
    )
    dag_run = relationship("DagRun")
    execution_date = association_proxy("dag_run", "execution_date")

    def __init__(
        self,
        task: "BaseOperator",
        run_id: str,
        try_number: int,
        start_date: datetime.datetime,
        end_date: datetime.datetime,
        reschedule_date: datetime.datetime,
        map_index: int = -1,
    ):
        self.dag_id = task.dag_id
        self.task_id = task.task_id
        self.run_id = run_id
        self.map_index = map_index
        self.try_number = try_number
        self.start_date = start_date
        self.end_date = end_date
        self.reschedule_date = reschedule_date
        self.duration = (self.end_date - self.start_date).total_seconds()

    @staticmethod
    @provide_session
    def query_for_task_instance(task_instance, descending=False, session=None, try_number=None):
        """
        Returns query for task reschedules for a given the task instance.

        :param session: the database session object
        :type session: sqlalchemy.orm.session.Session
        :param task_instance: the task instance to find task reschedules for
        :type task_instance: airflow.models.TaskInstance
        :param descending: If True then records are returned in descending order
        :type descending: bool
        :param try_number: Look for TaskReschedule of the given try_number. Default is None which
            looks for the same try_number of the given task_instance.
        :type try_number: int
        """
        if try_number is None:
            try_number = task_instance.try_number

        TR = TaskReschedule
        qry = session.query(TR).filter(
            TR.dag_id == task_instance.dag_id,
            TR.task_id == task_instance.task_id,
            TR.run_id == task_instance.run_id,
            TR.try_number == try_number,
        )
        if descending:
            return qry.order_by(desc(TR.id))
        else:
            return qry.order_by(asc(TR.id))

    @staticmethod
    @provide_session
    def find_for_task_instance(task_instance, session=None, try_number=None):
        """
        Returns all task reschedules for the task instance and try number,
        in ascending order.

        :param session: the database session object
        :type session: sqlalchemy.orm.session.Session
        :param task_instance: the task instance to find task reschedules for
        :type task_instance: airflow.models.TaskInstance
        :param try_number: Look for TaskReschedule of the given try_number. Default is None which
            looks for the same try_number of the given task_instance.
        :type try_number: int
        """
        return TaskReschedule.query_for_task_instance(
            task_instance, session=session, try_number=try_number
        ).all()
Ejemplo n.º 37
0
                                        name='check_thing_book_has_author'),
                     db.CheckConstraint("\"type\" <> 'Book' OR \"idAtProviders\" SIMILAR TO '[0-9]{13}'",
                                        name='check_thing_book_has_ean13'),
                     nullable=False)

    name = db.Column(db.String(140), nullable=False)

    description = db.Column(db.Text, nullable=True)

    mediaUrls = db.Column(ARRAY(db.String(120)),
                          nullable=False,
                          default=[])


Thing.__ts_vector__ = create_tsvector(
    cast(coalesce(Thing.name, ''), TEXT),
    coalesce(Thing.extraData['author'].cast(TEXT), ''),
    coalesce(Thing.extraData['byArtist'].cast(TEXT), ''),
)

Thing.__table_args__ = (
    Index(
        'idx_thing_fts',
        Thing.__ts_vector__,
        postgresql_using='gin'
    ),
)


app.model.Thing = Thing
Ejemplo n.º 38
0
class V_Category(Base, _Category):
    __tablename__ = 'v_category'
    __table_args__ = (
        Index('ix_v_category_name', 'name', unique=True),
    )
Ejemplo n.º 39
0
class V_Port(Base, _Port):
    __tablename__ = 'v_port'
    __table_args__ = (
        ForeignKeyConstraint(['scan'], ['v_scan.id'], ondelete='CASCADE'),
        Index('ix_v_port_scan_port', 'scan', 'port', 'protocol', unique=True),
    )
Ejemplo n.º 40
0
class SerializedDagModel(Base):
    """A table for serialized DAGs.

    serialized_dag table is a snapshot of DAG files synchronized by scheduler.
    This feature is controlled by:

    * ``[core] min_serialized_dag_update_interval = 30`` (s):
      serialized DAGs are updated in DB when a file gets processed by scheduler,
      to reduce DB write rate, there is a minimal interval of updating serialized DAGs.
    * ``[scheduler] dag_dir_list_interval = 300`` (s):
      interval of deleting serialized DAGs in DB when the files are deleted, suggest
      to use a smaller interval such as 60

    It is used by webserver to load dags
    because reading from database is lightweight compared to importing from files,
    it solves the webserver scalability issue.
    """

    __tablename__ = 'serialized_dag'

    dag_id = Column(String(ID_LEN), primary_key=True)
    fileloc = Column(String(2000), nullable=False)
    # The max length of fileloc exceeds the limit of indexing.
    fileloc_hash = Column(BigInteger, nullable=False)
    data = Column(sqlalchemy_jsonfield.JSONField(json=json), nullable=False)
    last_updated = Column(UtcDateTime, nullable=False)
    dag_hash = Column(String(32), nullable=False)

    __table_args__ = (Index('idx_fileloc_hash', fileloc_hash, unique=False), )

    dag_runs = relationship(
        DagRun,
        primaryjoin=dag_id == foreign(DagRun.dag_id),  # type: ignore
        backref=backref('serialized_dag', uselist=False, innerjoin=True),
    )

    dag_model = relationship(
        DagModel,
        primaryjoin=dag_id == DagModel.dag_id,  # type: ignore
        foreign_keys=dag_id,
        uselist=False,
        innerjoin=True,
        backref=backref('serialized_dag', uselist=False, innerjoin=True),
    )

    load_op_links = True

    def __init__(self, dag: DAG):
        self.dag_id = dag.dag_id
        self.fileloc = dag.fileloc
        self.fileloc_hash = DagCode.dag_fileloc_hash(self.fileloc)
        self.data = SerializedDAG.to_dict(dag)
        self.last_updated = timezone.utcnow()
        self.dag_hash = hashlib.md5(
            json.dumps(self.data, sort_keys=True).encode("utf-8")).hexdigest()

    def __repr__(self):
        return f"<SerializedDag: {self.dag_id}>"

    @classmethod
    @provide_session
    def write_dag(cls,
                  dag: DAG,
                  min_update_interval: Optional[int] = None,
                  session: Session = None) -> bool:
        """Serializes a DAG and writes it into database.
        If the record already exists, it checks if the Serialized DAG changed or not. If it is
        changed, it updates the record, ignores otherwise.

        :param dag: a DAG to be written into database
        :param min_update_interval: minimal interval in seconds to update serialized DAG
        :param session: ORM Session

        :returns: Boolean indicating if the DAG was written to the DB
        """
        # Checks if (Current Time - Time when the DAG was written to DB) < min_update_interval
        # If Yes, does nothing
        # If No or the DAG does not exists, updates / writes Serialized DAG to DB
        if min_update_interval is not None:
            if (session.query(literal(True)).filter(
                    and_(
                        cls.dag_id == dag.dag_id,
                        (timezone.utcnow() -
                         timedelta(seconds=min_update_interval)) <
                        cls.last_updated,
                    )).first() is not None):
                # TODO: .first() is not None can be changed to .scalar() once we update to sqlalchemy 1.4+
                # as the associated sqlalchemy bug for MySQL was fixed
                # related issue : https://github.com/sqlalchemy/sqlalchemy/issues/5481
                return False

        log.debug("Checking if DAG (%s) changed", dag.dag_id)
        new_serialized_dag = cls(dag)
        serialized_dag_hash_from_db = session.query(
            cls.dag_hash).filter(cls.dag_id == dag.dag_id).scalar()

        if serialized_dag_hash_from_db == new_serialized_dag.dag_hash:
            log.debug(
                "Serialized DAG (%s) is unchanged. Skipping writing to DB",
                dag.dag_id)
            return False

        log.debug("Writing Serialized DAG: %s to the DB", dag.dag_id)
        session.merge(new_serialized_dag)
        log.debug("DAG: %s written to the DB", dag.dag_id)
        return True

    @classmethod
    @provide_session
    def read_all_dags(cls,
                      session: Session = None) -> Dict[str, 'SerializedDAG']:
        """Reads all DAGs in serialized_dag table.

        :param session: ORM Session
        :returns: a dict of DAGs read from database
        """
        serialized_dags = session.query(cls)

        dags = {}
        for row in serialized_dags:
            log.debug("Deserializing DAG: %s", row.dag_id)
            dag = row.dag

            # Coherence check
            if dag.dag_id == row.dag_id:
                dags[row.dag_id] = dag
            else:
                log.warning(
                    "dag_id Mismatch in DB: Row with dag_id '%s' has Serialised DAG with '%s' dag_id",
                    row.dag_id,
                    dag.dag_id,
                )
        return dags

    @property
    def dag(self):
        """The DAG deserialized from the ``data`` column"""
        SerializedDAG._load_operator_extra_links = self.load_op_links

        if isinstance(self.data, dict):
            dag = SerializedDAG.from_dict(self.data)  # type: Any
        else:
            dag = SerializedDAG.from_json(self.data)
        return dag

    @classmethod
    @provide_session
    def remove_dag(cls, dag_id: str, session: Session = None):
        """Deletes a DAG with given dag_id.
        :param dag_id: dag_id to be deleted
        :param session: ORM Session
        """
        session.execute(cls.__table__.delete().where(cls.dag_id == dag_id))

    @classmethod
    @provide_session
    def remove_deleted_dags(cls, alive_dag_filelocs: List[str], session=None):
        """Deletes DAGs not included in alive_dag_filelocs.

        :param alive_dag_filelocs: file paths of alive DAGs
        :param session: ORM Session
        """
        alive_fileloc_hashes = [
            DagCode.dag_fileloc_hash(fileloc) for fileloc in alive_dag_filelocs
        ]

        log.debug(
            "Deleting Serialized DAGs (for which DAG files are deleted) from %s table ",
            cls.__tablename__)

        session.execute(cls.__table__.delete().where(
            and_(cls.fileloc_hash.notin_(alive_fileloc_hashes),
                 cls.fileloc.notin_(alive_dag_filelocs))))

    @classmethod
    @provide_session
    def has_dag(cls, dag_id: str, session: Session = None) -> bool:
        """Checks a DAG exist in serialized_dag table.

        :param dag_id: the DAG to check
        :param session: ORM Session
        """
        return session.query(
            literal(True)).filter(cls.dag_id == dag_id).first() is not None

    @classmethod
    @provide_session
    def get(cls,
            dag_id: str,
            session: Session = None) -> Optional['SerializedDagModel']:
        """
        Get the SerializedDAG for the given dag ID.
        It will cope with being passed the ID of a subdag by looking up the
        root dag_id from the DAG table.

        :param dag_id: the DAG to fetch
        :param session: ORM Session
        """
        row = session.query(cls).filter(cls.dag_id == dag_id).one_or_none()
        if row:
            return row

        # If we didn't find a matching DAG id then ask the DAG table to find
        # out the root dag
        root_dag_id = session.query(
            DagModel.root_dag_id).filter(DagModel.dag_id == dag_id).scalar()

        return session.query(cls).filter(
            cls.dag_id == root_dag_id).one_or_none()

    @staticmethod
    @provide_session
    def bulk_sync_to_db(dags: List[DAG], session: Session = None):
        """
        Saves DAGs as Serialized DAG objects in the database. Each
        DAG is saved in a separate database query.

        :param dags: the DAG objects to save to the DB
        :type dags: List[airflow.models.dag.DAG]
        :param session: ORM Session
        :type session: Session
        :return: None
        """
        for dag in dags:
            if not dag.is_subdag:
                SerializedDagModel.write_dag(
                    dag,
                    min_update_interval=MIN_SERIALIZED_DAG_UPDATE_INTERVAL,
                    session=session)

    @classmethod
    @provide_session
    def get_last_updated_datetime(cls,
                                  dag_id: str,
                                  session: Session = None
                                  ) -> Optional[datetime]:
        """
        Get the date when the Serialized DAG associated to DAG was last updated
        in serialized_dag table

        :param dag_id: DAG ID
        :type dag_id: str
        :param session: ORM Session
        :type session: Session
        """
        return session.query(
            cls.last_updated).filter(cls.dag_id == dag_id).scalar()

    @classmethod
    @provide_session
    def get_max_last_updated_datetime(cls,
                                      session: Session = None
                                      ) -> Optional[datetime]:
        """
        Get the maximum date when any DAG was last updated in serialized_dag table

        :param session: ORM Session
        :type session: Session
        """
        return session.query(func.max(cls.last_updated)).scalar()

    @classmethod
    @provide_session
    def get_latest_version_hash(cls,
                                dag_id: str,
                                session: Session = None) -> Optional[str]:
        """
        Get the latest DAG version for a given DAG ID.

        :param dag_id: DAG ID
        :type dag_id: str
        :param session: ORM Session
        :type session: Session
        :return: DAG Hash, or None if the DAG is not found
        :rtype: str | None
        """
        return session.query(
            cls.dag_hash).filter(cls.dag_id == dag_id).scalar()

    @classmethod
    @provide_session
    def get_dag_dependencies(cls,
                             session: Session = None
                             ) -> Dict[str, List['DagDependency']]:
        """
        Get the dependencies between DAGs

        :param session: ORM Session
        :type session: Session
        """
        if session.bind.dialect.name in ["sqlite", "mysql"]:
            query = session.query(
                cls.dag_id,
                func.json_extract(cls.data, "$.dag.dag_dependencies"))
            iterator = ((dag_id, json.loads(deps_data) if deps_data else [])
                        for dag_id, deps_data in query)
        elif session.bind.dialect.name == "mssql":
            query = session.query(
                cls.dag_id, func.json_query(cls.data,
                                            "$.dag.dag_dependencies"))
            iterator = ((dag_id, json.loads(deps_data) if deps_data else [])
                        for dag_id, deps_data in query)
        else:
            iterator = session.query(
                cls.dag_id,
                func.json_extract_path(cls.data, "dag", "dag_dependencies"))
        return {
            dag_id: [DagDependency(**d) for d in (deps_data or [])]
            for dag_id, deps_data in iterator
        }
Ejemplo n.º 41
0
 def test_raise_index_nonexistent_name(self):
     m = MetaData()
     # the KeyError isn't ideal here, a nicer message
     # perhaps
     assert_raises(KeyError, Table, 't', m, Column('x', Integer),
                   Index("foo", "q"))
Ejemplo n.º 42
0
 def test_drop_schema(self):
     t = Table('t', MetaData(), Column('x', Integer), schema="foo")
     i = Index("xyz", t.c.x)
     self.assert_compile(schema.DropIndex(i), "DROP INDEX foo.xyz")
Ejemplo n.º 43
0
 def test_create_schema(self):
     t = Table('t', MetaData(), Column('x', Integer), schema="foo")
     i = Index("xyz", t.c.x)
     self.assert_compile(schema.CreateIndex(i),
                         "CREATE INDEX xyz ON foo.t (x)")
Ejemplo n.º 44
0
class V_Hop(Base, _Hop):
    __tablename__ = "v_hop"
    __table_args__ = (
        Index('ix_v_hop_ipaddr_ttl', 'ipaddr', 'ttl'),
        ForeignKeyConstraint(['trace'], ['v_trace.id'], ondelete='CASCADE')
    )
Ejemplo n.º 45
0
class DagRun(Base, LoggingMixin):
    """
    DagRun describes an instance of a Dag. It can be created
    by the scheduler (for regular runs) or by an external trigger
    """

    __tablename__ = "dag_run"

    __NO_VALUE = object()

    id = Column(Integer, primary_key=True)
    dag_id = Column(String(ID_LEN, **COLLATION_ARGS))
    queued_at = Column(UtcDateTime)
    execution_date = Column(UtcDateTime, default=timezone.utcnow)
    start_date = Column(UtcDateTime)
    end_date = Column(UtcDateTime)
    _state = Column('state', String(50), default=State.QUEUED)
    run_id = Column(String(ID_LEN, **COLLATION_ARGS))
    creating_job_id = Column(Integer)
    external_trigger = Column(Boolean, default=True)
    run_type = Column(String(50), nullable=False)
    conf = Column(PickleType)
    # These two must be either both NULL or both datetime.
    data_interval_start = Column(UtcDateTime)
    data_interval_end = Column(UtcDateTime)
    # When a scheduler last attempted to schedule TIs for this DagRun
    last_scheduling_decision = Column(UtcDateTime)
    dag_hash = Column(String(32))

    dag = None

    __table_args__ = (
        Index('dag_id_state', dag_id, _state),
        UniqueConstraint('dag_id',
                         'execution_date',
                         name='dag_run_dag_id_execution_date_key'),
        UniqueConstraint('dag_id', 'run_id', name='dag_run_dag_id_run_id_key'),
        Index('idx_last_scheduling_decision', last_scheduling_decision),
    )

    task_instances = relationship(TI, back_populates="dag_run")

    DEFAULT_DAGRUNS_TO_EXAMINE = airflow_conf.getint(
        'scheduler',
        'max_dagruns_per_loop_to_schedule',
        fallback=20,
    )

    def __init__(
        self,
        dag_id: Optional[str] = None,
        run_id: Optional[str] = None,
        queued_at: Optional[datetime] = __NO_VALUE,
        execution_date: Optional[datetime] = None,
        start_date: Optional[datetime] = None,
        external_trigger: Optional[bool] = None,
        conf: Optional[Any] = None,
        state: Optional[DagRunState] = None,
        run_type: Optional[str] = None,
        dag_hash: Optional[str] = None,
        creating_job_id: Optional[int] = None,
        data_interval: Optional[Tuple[datetime, datetime]] = None,
    ):
        if data_interval is None:
            # Legacy: Only happen for runs created prior to Airflow 2.2.
            self.data_interval_start = self.data_interval_end = None
        else:
            self.data_interval_start, self.data_interval_end = data_interval

        self.dag_id = dag_id
        self.run_id = run_id
        self.execution_date = execution_date
        self.start_date = start_date
        self.external_trigger = external_trigger
        self.conf = conf or {}
        self.state = state
        if queued_at is self.__NO_VALUE:
            self.queued_at = timezone.utcnow(
            ) if state == State.QUEUED else None
        else:
            self.queued_at = queued_at
        self.run_type = run_type
        self.dag_hash = dag_hash
        self.creating_job_id = creating_job_id
        super().__init__()

    def __repr__(self):
        return (
            '<DagRun {dag_id} @ {execution_date}: {run_id}, externally triggered: {external_trigger}>'
        ).format(
            dag_id=self.dag_id,
            execution_date=self.execution_date,
            run_id=self.run_id,
            external_trigger=self.external_trigger,
        )

    def get_state(self):
        return self._state

    def set_state(self, state: DagRunState):
        if self._state != state:
            self._state = state
            self.end_date = timezone.utcnow(
            ) if self._state in State.finished else None
            if state == State.QUEUED:
                self.queued_at = timezone.utcnow()

    @declared_attr
    def state(self):
        return synonym('_state',
                       descriptor=property(self.get_state, self.set_state))

    @provide_session
    def refresh_from_db(self, session: Session = None):
        """
        Reloads the current dagrun from the database

        :param session: database session
        :type session: Session
        """
        dr = session.query(DagRun).filter(DagRun.dag_id == self.dag_id,
                                          DagRun.run_id == self.run_id).one()
        self.id = dr.id
        self.state = dr.state

    @classmethod
    def next_dagruns_to_examine(
        cls,
        state: DagRunState,
        session: Session,
        max_number: Optional[int] = None,
    ):
        """
        Return the next DagRuns that the scheduler should attempt to schedule.

        This will return zero or more DagRun rows that are row-level-locked with a "SELECT ... FOR UPDATE"
        query, you should ensure that any scheduling decisions are made in a single transaction -- as soon as
        the transaction is committed it will be unlocked.

        :rtype: list[airflow.models.DagRun]
        """
        from airflow.models.dag import DagModel

        if max_number is None:
            max_number = cls.DEFAULT_DAGRUNS_TO_EXAMINE

        # TODO: Bake this query, it is run _A lot_
        query = (session.query(cls).filter(
            cls.state == state, cls.run_type != DagRunType.BACKFILL_JOB).join(
                DagModel,
                DagModel.dag_id == cls.dag_id,
            ).filter(
                DagModel.is_paused == expression.false(),
                DagModel.is_active == expression.true(),
            ).order_by(
                nulls_first(cls.last_scheduling_decision, session=session),
                cls.execution_date,
            ))

        if not settings.ALLOW_FUTURE_EXEC_DATES:
            query = query.filter(DagRun.execution_date <= func.now())

        return with_row_locks(query.limit(max_number),
                              of=cls,
                              session=session,
                              **skip_locked(session=session))

    @staticmethod
    @provide_session
    def find(
        dag_id: Optional[Union[str, List[str]]] = None,
        run_id: Optional[str] = None,
        execution_date: Optional[datetime] = None,
        state: Optional[DagRunState] = None,
        external_trigger: Optional[bool] = None,
        no_backfills: bool = False,
        run_type: Optional[DagRunType] = None,
        session: Session = None,
        execution_start_date: Optional[datetime] = None,
        execution_end_date: Optional[datetime] = None,
    ) -> List["DagRun"]:
        """
        Returns a set of dag runs for the given search criteria.

        :param dag_id: the dag_id or list of dag_id to find dag runs for
        :type dag_id: str or list[str]
        :param run_id: defines the run id for this dag run
        :type run_id: str
        :param run_type: type of DagRun
        :type run_type: airflow.utils.types.DagRunType
        :param execution_date: the execution date
        :type execution_date: datetime.datetime or list[datetime.datetime]
        :param state: the state of the dag run
        :type state: DagRunState
        :param external_trigger: whether this dag run is externally triggered
        :type external_trigger: bool
        :param no_backfills: return no backfills (True), return all (False).
            Defaults to False
        :type no_backfills: bool
        :param session: database session
        :type session: sqlalchemy.orm.session.Session
        :param execution_start_date: dag run that was executed from this date
        :type execution_start_date: datetime.datetime
        :param execution_end_date: dag run that was executed until this date
        :type execution_end_date: datetime.datetime
        """
        DR = DagRun

        qry = session.query(DR)
        dag_ids = [dag_id] if isinstance(dag_id, str) else dag_id
        if dag_ids:
            qry = qry.filter(DR.dag_id.in_(dag_ids))
        if run_id:
            qry = qry.filter(DR.run_id == run_id)
        if execution_date:
            if isinstance(execution_date, list):
                qry = qry.filter(DR.execution_date.in_(execution_date))
            else:
                qry = qry.filter(DR.execution_date == execution_date)
        if execution_start_date and execution_end_date:
            qry = qry.filter(
                DR.execution_date.between(execution_start_date,
                                          execution_end_date))
        elif execution_start_date:
            qry = qry.filter(DR.execution_date >= execution_start_date)
        elif execution_end_date:
            qry = qry.filter(DR.execution_date <= execution_end_date)
        if state:
            qry = qry.filter(DR.state == state)
        if external_trigger is not None:
            qry = qry.filter(DR.external_trigger == external_trigger)
        if run_type:
            qry = qry.filter(DR.run_type == run_type)
        if no_backfills:
            qry = qry.filter(DR.run_type != DagRunType.BACKFILL_JOB)

        return qry.order_by(DR.execution_date).all()

    @staticmethod
    def generate_run_id(run_type: DagRunType, execution_date: datetime) -> str:
        """Generate Run ID based on Run Type and Execution Date"""
        return f"{run_type}__{execution_date.isoformat()}"

    @provide_session
    def get_task_instances(self,
                           state: Optional[Iterable[TaskInstanceState]] = None,
                           session=None) -> Iterable[TI]:
        """Returns the task instances for this dag run"""
        tis = (session.query(TI).options(joinedload(TI.dag_run)).filter(
            TI.dag_id == self.dag_id,
            TI.run_id == self.run_id,
        ))

        if state:
            if isinstance(state, str):
                tis = tis.filter(TI.state == state)
            else:
                # this is required to deal with NULL values
                if None in state:
                    if all(x is None for x in state):
                        tis = tis.filter(TI.state.is_(None))
                    else:
                        not_none_state = [s for s in state if s]
                        tis = tis.filter(
                            or_(TI.state.in_(not_none_state),
                                TI.state.is_(None)))
                else:
                    tis = tis.filter(TI.state.in_(state))

        if self.dag and self.dag.partial:
            tis = tis.filter(TI.task_id.in_(self.dag.task_ids))
        return tis.all()

    @provide_session
    def get_task_instance(self,
                          task_id: str,
                          session: Session = None) -> Optional[TI]:
        """
        Returns the task instance specified by task_id for this dag run

        :param task_id: the task id
        :type task_id: str
        :param session: Sqlalchemy ORM Session
        :type session: Session
        """
        return (session.query(TI).filter(TI.dag_id == self.dag_id,
                                         TI.run_id == self.run_id,
                                         TI.task_id == task_id).one_or_none())

    def get_dag(self) -> "DAG":
        """
        Returns the Dag associated with this DagRun.

        :return: DAG
        """
        if not self.dag:
            raise AirflowException(
                f"The DAG (.dag) for {self} needs to be set")

        return self.dag

    @provide_session
    def get_previous_dagrun(self,
                            state: Optional[DagRunState] = None,
                            session: Session = None) -> Optional['DagRun']:
        """The previous DagRun, if there is one"""
        filters = [
            DagRun.dag_id == self.dag_id,
            DagRun.execution_date < self.execution_date,
        ]
        if state is not None:
            filters.append(DagRun.state == state)
        return session.query(DagRun).filter(*filters).order_by(
            DagRun.execution_date.desc()).first()

    @provide_session
    def get_previous_scheduled_dagrun(self,
                                      session: Session = None
                                      ) -> Optional['DagRun']:
        """The previous, SCHEDULED DagRun, if there is one"""
        return (session.query(DagRun).filter(
            DagRun.dag_id == self.dag_id,
            DagRun.execution_date < self.execution_date,
            DagRun.run_type != DagRunType.MANUAL,
        ).order_by(DagRun.execution_date.desc()).first())

    @provide_session
    def update_state(
        self,
        session: Session = None,
        execute_callbacks: bool = True
    ) -> Tuple[List[TI], Optional[callback_requests.DagCallbackRequest]]:
        """
        Determines the overall state of the DagRun based on the state
        of its TaskInstances.

        :param session: Sqlalchemy ORM Session
        :type session: Session
        :param execute_callbacks: Should dag callbacks (success/failure, SLA etc) be invoked
            directly (default: true) or recorded as a pending request in the ``callback`` property
        :type execute_callbacks: bool
        :return: Tuple containing tis that can be scheduled in the current loop & `callback` that
            needs to be executed
        """
        # Callback to execute in case of Task Failures
        callback: Optional[callback_requests.DagCallbackRequest] = None

        start_dttm = timezone.utcnow()
        self.last_scheduling_decision = start_dttm
        with Stats.timer(f"dagrun.dependency-check.{self.dag_id}"):
            dag = self.get_dag()
            info = self.task_instance_scheduling_decisions(session)

            tis = info.tis
            schedulable_tis = info.schedulable_tis
            changed_tis = info.changed_tis
            finished_tasks = info.finished_tasks
            unfinished_tasks = info.unfinished_tasks

            none_depends_on_past = all(not t.task.depends_on_past
                                       for t in unfinished_tasks)
            none_task_concurrency = all(t.task.max_active_tis_per_dag is None
                                        for t in unfinished_tasks)
            none_deferred = all(t.state != State.DEFERRED
                                for t in unfinished_tasks)

            if unfinished_tasks and none_depends_on_past and none_task_concurrency and none_deferred:
                # small speed up
                are_runnable_tasks = (schedulable_tis
                                      or self._are_premature_tis(
                                          unfinished_tasks, finished_tasks,
                                          session) or changed_tis)

        leaf_task_ids = {t.task_id for t in dag.leaves}
        leaf_tis = [ti for ti in tis if ti.task_id in leaf_task_ids]

        # if all roots finished and at least one failed, the run failed
        if not unfinished_tasks and any(leaf_ti.state in State.failed_states
                                        for leaf_ti in leaf_tis):
            self.log.error('Marking run %s failed', self)
            self.set_state(State.FAILED)
            if execute_callbacks:
                dag.handle_callback(self,
                                    success=False,
                                    reason='task_failure',
                                    session=session)
            elif dag.has_on_failure_callback:
                callback = callback_requests.DagCallbackRequest(
                    full_filepath=dag.fileloc,
                    dag_id=self.dag_id,
                    run_id=self.run_id,
                    is_failure_callback=True,
                    msg='task_failure',
                )

        # if all leaves succeeded and no unfinished tasks, the run succeeded
        elif not unfinished_tasks and all(leaf_ti.state in State.success_states
                                          for leaf_ti in leaf_tis):
            self.log.info('Marking run %s successful', self)
            self.set_state(State.SUCCESS)
            if execute_callbacks:
                dag.handle_callback(self,
                                    success=True,
                                    reason='success',
                                    session=session)
            elif dag.has_on_success_callback:
                callback = callback_requests.DagCallbackRequest(
                    full_filepath=dag.fileloc,
                    dag_id=self.dag_id,
                    run_id=self.run_id,
                    is_failure_callback=False,
                    msg='success',
                )

        # if *all tasks* are deadlocked, the run failed
        elif (unfinished_tasks and none_depends_on_past
              and none_task_concurrency and none_deferred
              and not are_runnable_tasks):
            self.log.error('Deadlock; marking run %s failed', self)
            self.set_state(State.FAILED)
            if execute_callbacks:
                dag.handle_callback(self,
                                    success=False,
                                    reason='all_tasks_deadlocked',
                                    session=session)
            elif dag.has_on_failure_callback:
                callback = callback_requests.DagCallbackRequest(
                    full_filepath=dag.fileloc,
                    dag_id=self.dag_id,
                    run_id=self.run_id,
                    is_failure_callback=True,
                    msg='all_tasks_deadlocked',
                )

        # finally, if the roots aren't done, the dag is still running
        else:
            self.set_state(State.RUNNING)

        self._emit_true_scheduling_delay_stats_for_finished_state(
            finished_tasks)
        self._emit_duration_stats_for_finished_state()

        session.merge(self)

        return schedulable_tis, callback

    @provide_session
    def task_instance_scheduling_decisions(self,
                                           session: Session = None
                                           ) -> TISchedulingDecision:

        schedulable_tis: List[TI] = []
        changed_tis = False

        tis = list(
            self.get_task_instances(session=session, state=State.task_states))
        self.log.debug("number of tis tasks for %s: %s task(s)", self,
                       len(tis))
        for ti in tis:
            try:
                ti.task = self.get_dag().get_task(ti.task_id)
            except TaskNotFound:
                self.log.warning(
                    "Failed to get task '%s' for dag '%s'. Marking it as removed.",
                    ti, ti.dag_id)
                ti.state = State.REMOVED
                session.flush()

        unfinished_tasks = [t for t in tis if t.state in State.unfinished]
        finished_tasks = [t for t in tis if t.state in State.finished]
        if unfinished_tasks:
            scheduleable_tasks = [
                ut for ut in unfinished_tasks
                if ut.state in SCHEDULEABLE_STATES
            ]
            self.log.debug("number of scheduleable tasks for %s: %s task(s)",
                           self, len(scheduleable_tasks))
            schedulable_tis, changed_tis = self._get_ready_tis(
                scheduleable_tasks, finished_tasks, session)

        return TISchedulingDecision(
            tis=tis,
            schedulable_tis=schedulable_tis,
            changed_tis=changed_tis,
            unfinished_tasks=unfinished_tasks,
            finished_tasks=finished_tasks,
        )

    def _get_ready_tis(
        self,
        scheduleable_tasks: List[TI],
        finished_tasks: List[TI],
        session: Session,
    ) -> Tuple[List[TI], bool]:
        old_states = {}
        ready_tis: List[TI] = []
        changed_tis = False

        if not scheduleable_tasks:
            return ready_tis, changed_tis

        # Check dependencies
        for st in scheduleable_tasks:
            old_state = st.state
            if st.are_dependencies_met(
                    dep_context=DepContext(flag_upstream_failed=True,
                                           finished_tasks=finished_tasks),
                    session=session,
            ):
                ready_tis.append(st)
            else:
                old_states[st.key] = old_state

        # Check if any ti changed state
        tis_filter = TI.filter_for_tis(old_states.keys())
        if tis_filter is not None:
            fresh_tis = session.query(TI).filter(tis_filter).all()
            changed_tis = any(ti.state != old_states[ti.key]
                              for ti in fresh_tis)

        return ready_tis, changed_tis

    def _are_premature_tis(
        self,
        unfinished_tasks: List[TI],
        finished_tasks: List[TI],
        session: Session,
    ) -> bool:
        # there might be runnable tasks that are up for retry and for some reason(retry delay, etc) are
        # not ready yet so we set the flags to count them in
        for ut in unfinished_tasks:
            if ut.are_dependencies_met(
                    dep_context=DepContext(
                        flag_upstream_failed=True,
                        ignore_in_retry_period=True,
                        ignore_in_reschedule_period=True,
                        finished_tasks=finished_tasks,
                    ),
                    session=session,
            ):
                return True
        return False

    def _emit_true_scheduling_delay_stats_for_finished_state(
            self, finished_tis):
        """
        This is a helper method to emit the true scheduling delay stats, which is defined as
        the time when the first task in DAG starts minus the expected DAG run datetime.
        This method will be used in the update_state method when the state of the DagRun
        is updated to a completed status (either success or failure). The method will find the first
        started task within the DAG and calculate the expected DagRun start time (based on
        dag.execution_date & dag.timetable), and minus these two values to get the delay.
        The emitted data may contains outlier (e.g. when the first task was cleared, so
        the second task's start_date will be used), but we can get rid of the outliers
        on the stats side through the dashboards tooling built.
        Note, the stat will only be emitted if the DagRun is a scheduler triggered one
        (i.e. external_trigger is False).
        """
        if self.state == State.RUNNING:
            return
        if self.external_trigger:
            return
        if not finished_tis:
            return

        try:
            dag = self.get_dag()

            if not self.dag.timetable.periodic:
                # We can't emit this metric if there is no following schedule to calculate from!
                return

            ordered_tis_by_start_date = [
                ti for ti in finished_tis if ti.start_date
            ]
            ordered_tis_by_start_date.sort(key=lambda ti: ti.start_date,
                                           reverse=False)
            first_start_date = ordered_tis_by_start_date[0].start_date
            if first_start_date:
                # dag.following_schedule calculates the expected start datetime for a scheduled dagrun
                # i.e. a daily flow for execution date 1/1/20 actually runs on 1/2/20 hh:mm:ss,
                # and ti.start_date will be 1/2/20 hh:mm:ss so the following schedule is comparison
                true_delay = first_start_date - dag.following_schedule(
                    self.execution_date)
                if true_delay.total_seconds() > 0:
                    Stats.timing(
                        f'dagrun.{dag.dag_id}.first_task_scheduling_delay',
                        true_delay)
        except Exception as e:
            self.log.warning(
                f'Failed to record first_task_scheduling_delay metric:\n{e}')

    def _emit_duration_stats_for_finished_state(self):
        if self.state == State.RUNNING:
            return
        if self.start_date is None:
            self.log.warning(
                'Failed to record duration of %s: start_date is not set.',
                self)
            return
        if self.end_date is None:
            self.log.warning(
                'Failed to record duration of %s: end_date is not set.', self)
            return

        duration = self.end_date - self.start_date
        if self.state == State.SUCCESS:
            Stats.timing(f'dagrun.duration.success.{self.dag_id}', duration)
        elif self.state == State.FAILED:
            Stats.timing(f'dagrun.duration.failed.{self.dag_id}', duration)

    @provide_session
    def verify_integrity(self, session: Session = None):
        """
        Verifies the DagRun by checking for removed tasks or tasks that are not in the
        database yet. It will set state to removed or add the task if required.

        :param session: Sqlalchemy ORM Session
        :type session: Session
        """
        from airflow.settings import task_instance_mutation_hook

        dag = self.get_dag()
        tis = self.get_task_instances(session=session)

        # check for removed or restored tasks
        task_ids = set()
        for ti in tis:
            task_instance_mutation_hook(ti)
            task_ids.add(ti.task_id)
            task = None
            try:
                task = dag.get_task(ti.task_id)
            except AirflowException:
                if ti.state == State.REMOVED:
                    pass  # ti has already been removed, just ignore it
                elif self.state != State.RUNNING and not dag.partial:
                    self.log.warning(
                        "Failed to get task '%s' for dag '%s'. Marking it as removed.",
                        ti, dag)
                    Stats.incr(f"task_removed_from_dag.{dag.dag_id}", 1, 1)
                    ti.state = State.REMOVED

            should_restore_task = (task
                                   is not None) and ti.state == State.REMOVED
            if should_restore_task:
                self.log.info(
                    "Restoring task '%s' which was previously removed from DAG '%s'",
                    ti, dag)
                Stats.incr(f"task_restored_to_dag.{dag.dag_id}", 1, 1)
                ti.state = State.NONE
            session.merge(ti)

        # check for missing tasks
        for task in dag.task_dict.values():
            if task.start_date > self.execution_date and not self.is_backfill:
                continue

            if task.task_id not in task_ids:
                Stats.incr(f"task_instance_created-{task.task_type}", 1, 1)
                ti = TI(task, execution_date=None, run_id=self.run_id)
                task_instance_mutation_hook(ti)
                session.add(ti)

        try:
            session.flush()
        except IntegrityError as err:
            self.log.info(str(err))
            self.log.info(
                'Hit IntegrityError while creating the TIs for %s- %s',
                dag.dag_id, self.run_id)
            self.log.info('Doing session rollback.')
            # TODO[HA]: We probably need to savepoint this so we can keep the transaction alive.
            session.rollback()

    @staticmethod
    def get_run(session: Session, dag_id: str,
                execution_date: datetime) -> Optional['DagRun']:
        """
        Get a single DAG Run

        :meta private:
        :param session: Sqlalchemy ORM Session
        :type session: Session
        :param dag_id: DAG ID
        :type dag_id: unicode
        :param execution_date: execution date
        :type execution_date: datetime
        :return: DagRun corresponding to the given dag_id and execution date
            if one exists. None otherwise.
        :rtype: airflow.models.DagRun
        """
        warnings.warn(
            "This method is deprecated. Please use SQLAlchemy directly",
            DeprecationWarning,
            stacklevel=2,
        )
        return (session.query(DagRun).filter(
            DagRun.dag_id == dag_id,
            DagRun.external_trigger == False,  # noqa
            DagRun.execution_date == execution_date,
        ).first())

    @property
    def is_backfill(self) -> bool:
        return self.run_type == DagRunType.BACKFILL_JOB

    @classmethod
    @provide_session
    def get_latest_runs(cls, session=None) -> List['DagRun']:
        """Returns the latest DagRun for each DAG"""
        subquery = (session.query(
            cls.dag_id,
            func.max(cls.execution_date).label('execution_date')).group_by(
                cls.dag_id).subquery())
        return (session.query(cls).join(
            subquery,
            and_(cls.dag_id == subquery.c.dag_id,
                 cls.execution_date == subquery.c.execution_date),
        ).all())

    @provide_session
    def schedule_tis(self,
                     schedulable_tis: Iterable[TI],
                     session: Session = None) -> int:
        """
        Set the given task instances in to the scheduled state.

        Each element of ``schedulable_tis`` should have it's ``task`` attribute already set.

        Any DummyOperator without callbacks is instead set straight to the success state.

        All the TIs should belong to this DagRun, but this code is in the hot-path, this is not checked -- it
        is the caller's responsibility to call this function only with TIs from a single dag run.
        """
        # Get list of TI IDs that do not need to executed, these are
        # tasks using DummyOperator and without on_execute_callback / on_success_callback
        dummy_ti_ids = []
        schedulable_ti_ids = []
        for ti in schedulable_tis:
            if (ti.task.inherits_from_dummy_operator
                    and not ti.task.on_execute_callback
                    and not ti.task.on_success_callback):
                dummy_ti_ids.append(ti.task_id)
            else:
                schedulable_ti_ids.append(ti.task_id)

        count = 0

        if schedulable_ti_ids:
            count += (session.query(TI).filter(
                TI.dag_id == self.dag_id,
                TI.run_id == self.run_id,
                TI.task_id.in_(schedulable_ti_ids),
            ).update({TI.state: State.SCHEDULED}, synchronize_session=False))

        # Tasks using DummyOperator should not be executed, mark them as success
        if dummy_ti_ids:
            count += (session.query(TI).filter(
                TI.dag_id == self.dag_id,
                TI.run_id == self.run_id,
                TI.task_id.in_(dummy_ti_ids),
            ).update(
                {
                    TI.state: State.SUCCESS,
                    TI.start_date: timezone.utcnow(),
                    TI.end_date: timezone.utcnow(),
                    TI.duration: 0,
                },
                synchronize_session=False,
            ))

        return count
Ejemplo n.º 46
0
class Feed(Base):
    __tablename__ = 'feed'
    id = Column(Integer, primary_key=True, nullable=False)
    created = Column(TIMESTAMP(timezone=False), default=datetime.utcnow,
        nullable=False)
    __table_args__ = (Index('feed_id_index', 'id'), )
Ejemplo n.º 47
0
def upgrade(migrate_engine):
    meta = MetaData()
    meta.bind = migrate_engine

    cell_mappings = Table('cell_mappings',
                          meta,
                          Column('created_at', DateTime),
                          Column('updated_at', DateTime),
                          Column('id',
                                 Integer,
                                 primary_key=True,
                                 nullable=False),
                          Column('uuid', String(length=36), nullable=False),
                          Column('name', String(length=255)),
                          Column('transport_url', Text()),
                          Column('database_connection', Text()),
                          UniqueConstraint('uuid',
                                           name='uniq_cell_mappings0uuid'),
                          Index('uuid_idx', 'uuid'),
                          mysql_engine='InnoDB',
                          mysql_charset='utf8')

    host_mappings = Table('host_mappings',
                          meta,
                          Column('created_at', DateTime),
                          Column('updated_at', DateTime),
                          Column('id',
                                 Integer,
                                 primary_key=True,
                                 nullable=False),
                          Column('cell_id', Integer, nullable=False),
                          Column('host', String(length=255), nullable=False),
                          UniqueConstraint('host',
                                           name='uniq_host_mappings0host'),
                          Index('host_idx', 'host'),
                          ForeignKeyConstraint(columns=['cell_id'],
                                               refcolumns=[cell_mappings.c.id
                                                           ]),
                          mysql_engine='InnoDB',
                          mysql_charset='utf8')

    instance_mappings = Table(
        'instance_mappings',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('instance_uuid', String(length=36), nullable=False),
        Column('cell_id', Integer, nullable=True),
        Column('project_id', String(length=255), nullable=False),
        UniqueConstraint('instance_uuid',
                         name='uniq_instance_mappings0instance_uuid'),
        Index('instance_uuid_idx', 'instance_uuid'),
        Index('project_id_idx', 'project_id'),
        ForeignKeyConstraint(columns=['cell_id'],
                             refcolumns=[cell_mappings.c.id]),
        mysql_engine='InnoDB',
        mysql_charset='utf8')

    flavors = Table('flavors',
                    meta,
                    Column('created_at', DateTime),
                    Column('updated_at', DateTime),
                    Column('name', String(length=255), nullable=False),
                    Column('id', Integer, primary_key=True, nullable=False),
                    Column('memory_mb', Integer, nullable=False),
                    Column('vcpus', Integer, nullable=False),
                    Column('swap', Integer, nullable=False),
                    Column('vcpu_weight', Integer),
                    Column('flavorid', String(length=255), nullable=False),
                    Column('rxtx_factor', Float),
                    Column('root_gb', Integer),
                    Column('ephemeral_gb', Integer),
                    Column('disabled', Boolean),
                    Column('is_public', Boolean),
                    Column('description', Text()),
                    UniqueConstraint('flavorid', name='uniq_flavors0flavorid'),
                    UniqueConstraint('name', name='uniq_flavors0name'),
                    mysql_engine='InnoDB',
                    mysql_charset='utf8')

    flavor_extra_specs = Table(
        'flavor_extra_specs',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('flavor_id', Integer, nullable=False),
        Column('key', String(length=255), nullable=False),
        Column('value', String(length=255)),
        UniqueConstraint('flavor_id',
                         'key',
                         name='uniq_flavor_extra_specs0flavor_id0key'),
        Index('flavor_extra_specs_flavor_id_key_idx', 'flavor_id', 'key'),
        ForeignKeyConstraint(columns=['flavor_id'], refcolumns=[flavors.c.id]),
        mysql_engine='InnoDB',
        mysql_charset='utf8')

    flavor_projects = Table(
        'flavor_projects',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('flavor_id', Integer, nullable=False),
        Column('project_id', String(length=255), nullable=False),
        UniqueConstraint('flavor_id',
                         'project_id',
                         name='uniq_flavor_projects0flavor_id0project_id'),
        ForeignKeyConstraint(columns=['flavor_id'], refcolumns=[flavors.c.id]),
        mysql_engine='InnoDB',
        mysql_charset='utf8')

    request_specs = Table(
        'request_specs',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('instance_uuid', String(36), nullable=False),
        Column('spec', MediumText(), nullable=False),
        UniqueConstraint('instance_uuid',
                         name='uniq_request_specs0instance_uuid'),
        Index('request_spec_instance_uuid_idx', 'instance_uuid'),
        mysql_engine='InnoDB',
        mysql_charset='utf8')

    build_requests = Table(
        'build_requests',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('request_spec_id', Integer, nullable=True),
        Column('project_id', String(length=255), nullable=False),
        Column('user_id', String(length=255), nullable=True),
        Column('display_name', String(length=255)),
        Column('instance_metadata', Text),
        Column('progress', Integer),
        Column('vm_state', String(length=255)),
        Column('task_state', String(length=255)),
        Column('image_ref', String(length=255)),
        Column('access_ip_v4', InetSmall()),
        Column('access_ip_v6', InetSmall()),
        Column('info_cache', Text),
        Column('security_groups', Text, nullable=True),
        Column('config_drive', Boolean, default=False, nullable=True),
        Column('key_name', String(length=255)),
        Column('locked_by',
               Enum('owner', 'admin', name='build_requests0locked_by')),
        Column('instance_uuid', String(length=36)),
        Column('instance', MediumText()),
        Column('block_device_mappings', MediumText()),
        Column('tags', Text()),
        UniqueConstraint('instance_uuid',
                         name='uniq_build_requests0instance_uuid'),
        Index('build_requests_project_id_idx', 'project_id'),
        Index('build_requests_instance_uuid_idx', 'instance_uuid'),
        mysql_engine='InnoDB',
        mysql_charset='utf8')

    keypairs = Table('key_pairs',
                     meta,
                     Column('created_at', DateTime),
                     Column('updated_at', DateTime),
                     Column('id', Integer, primary_key=True, nullable=False),
                     Column('name', String(255), nullable=False),
                     Column('user_id', String(255), nullable=False),
                     Column('fingerprint', String(255)),
                     Column('public_key', Text()),
                     Column('type',
                            Enum('ssh',
                                 'x509',
                                 metadata=meta,
                                 name='keypair_types'),
                            nullable=False,
                            server_default=keypair.KEYPAIR_TYPE_SSH),
                     UniqueConstraint('user_id',
                                      'name',
                                      name='uniq_key_pairs0user_id0name'),
                     mysql_engine='InnoDB',
                     mysql_charset='utf8')

    projects = Table(
        'projects',
        meta,
        Column('id',
               Integer,
               primary_key=True,
               nullable=False,
               autoincrement=True),
        Column('external_id', String(length=255), nullable=False),
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        UniqueConstraint('external_id', name='uniq_projects0external_id'),
        mysql_engine='InnoDB',
        mysql_charset='latin1',
    )

    users = Table(
        'users',
        meta,
        Column('id',
               Integer,
               primary_key=True,
               nullable=False,
               autoincrement=True),
        Column('external_id', String(length=255), nullable=False),
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        UniqueConstraint('external_id', name='uniq_users0external_id'),
        mysql_engine='InnoDB',
        mysql_charset='latin1',
    )

    resource_classes = Table(
        'resource_classes',
        meta,
        Column('id', Integer, primary_key=True, nullable=False),
        Column('name', String(length=255), nullable=False),
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        UniqueConstraint('name', name='uniq_resource_classes0name'),
        mysql_engine='InnoDB',
        mysql_charset='latin1')

    nameargs = {}
    if migrate_engine.name == 'mysql':
        nameargs['collation'] = 'utf8_bin'

    resource_providers = Table(
        'resource_providers',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('uuid', String(36), nullable=False),
        Column('name', Unicode(200, **nameargs), nullable=True),
        Column('generation', Integer, default=0),
        Column('can_host', Integer, default=0),
        Column('root_provider_id', Integer,
               ForeignKey('resource_providers.id')),
        Column('parent_provider_id', Integer,
               ForeignKey('resource_providers.id')),
        UniqueConstraint('uuid', name='uniq_resource_providers0uuid'),
        UniqueConstraint('name', name='uniq_resource_providers0name'),
        Index('resource_providers_name_idx', 'name'),
        Index('resource_providers_uuid_idx', 'uuid'),
        Index('resource_providers_root_provider_id_idx', 'root_provider_id'),
        Index('resource_providers_parent_provider_id_idx',
              'parent_provider_id'),
        mysql_engine='InnoDB',
        mysql_charset='latin1')

    inventories = Table(
        'inventories',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('resource_provider_id', Integer, nullable=False),
        Column('resource_class_id', Integer, nullable=False),
        Column('total', Integer, nullable=False),
        Column('reserved', Integer, nullable=False),
        Column('min_unit', Integer, nullable=False),
        Column('max_unit', Integer, nullable=False),
        Column('step_size', Integer, nullable=False),
        Column('allocation_ratio', Float, nullable=False),
        Index('inventories_resource_provider_id_idx', 'resource_provider_id'),
        Index('inventories_resource_provider_resource_class_idx',
              'resource_provider_id', 'resource_class_id'),
        Index('inventories_resource_class_id_idx', 'resource_class_id'),
        UniqueConstraint(
            'resource_provider_id',
            'resource_class_id',
            name='uniq_inventories0resource_provider_resource_class'),
        mysql_engine='InnoDB',
        mysql_charset='latin1')

    traits = Table(
        'traits',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id',
               Integer,
               primary_key=True,
               nullable=False,
               autoincrement=True),
        Column('name', Unicode(255, **nameargs), nullable=False),
        UniqueConstraint('name', name='uniq_traits0name'),
        mysql_engine='InnoDB',
        mysql_charset='latin1',
    )

    allocations = Table('allocations',
                        meta,
                        Column('created_at', DateTime),
                        Column('updated_at', DateTime),
                        Column('id', Integer, primary_key=True,
                               nullable=False),
                        Column('resource_provider_id', Integer,
                               nullable=False),
                        Column('consumer_id', String(36), nullable=False),
                        Column('resource_class_id', Integer, nullable=False),
                        Column('used', Integer, nullable=False),
                        Index('allocations_resource_provider_class_used_idx',
                              'resource_provider_id', 'resource_class_id',
                              'used'),
                        Index('allocations_resource_class_id_idx',
                              'resource_class_id'),
                        Index('allocations_consumer_id_idx', 'consumer_id'),
                        mysql_engine='InnoDB',
                        mysql_charset='latin1')

    consumers = Table(
        'consumers',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id',
               Integer,
               primary_key=True,
               nullable=False,
               autoincrement=True),
        Column('uuid', String(length=36), nullable=False),
        Column('project_id', Integer, nullable=False),
        Column('user_id', Integer, nullable=False),
        Index('consumers_project_id_uuid_idx', 'project_id', 'uuid'),
        Index('consumers_project_id_user_id_uuid_idx', 'project_id', 'user_id',
              'uuid'),
        UniqueConstraint('uuid', name='uniq_consumers0uuid'),
        mysql_engine='InnoDB',
        mysql_charset='latin1',
    )

    resource_provider_aggregates = Table(
        'resource_provider_aggregates',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('resource_provider_id',
               Integer,
               primary_key=True,
               nullable=False),
        Column('aggregate_id', Integer, primary_key=True, nullable=False),
        Index('resource_provider_aggregates_aggregate_id_idx', 'aggregate_id'),
        mysql_engine='InnoDB',
        mysql_charset='latin1')

    resource_provider_traits = Table(
        'resource_provider_traits',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('trait_id',
               Integer,
               ForeignKey('traits.id'),
               primary_key=True,
               nullable=False),
        Column('resource_provider_id',
               Integer,
               primary_key=True,
               nullable=False),
        Index('resource_provider_traits_resource_provider_trait_idx',
              'resource_provider_id', 'trait_id'),
        ForeignKeyConstraint(columns=['resource_provider_id'],
                             refcolumns=[resource_providers.c.id]),
        mysql_engine='InnoDB',
        mysql_charset='latin1',
    )

    placement_aggregates = Table('placement_aggregates',
                                 meta,
                                 Column('created_at', DateTime),
                                 Column('updated_at', DateTime),
                                 Column('id',
                                        Integer,
                                        primary_key=True,
                                        nullable=False),
                                 Column('uuid', String(length=36), index=True),
                                 UniqueConstraint(
                                     'uuid',
                                     name='uniq_placement_aggregates0uuid'),
                                 mysql_engine='InnoDB',
                                 mysql_charset='latin1')

    aggregates = Table('aggregates',
                       meta,
                       Column('created_at', DateTime),
                       Column('updated_at', DateTime),
                       Column('id', Integer, primary_key=True, nullable=False),
                       Column('uuid', String(length=36)),
                       Column('name', String(length=255)),
                       Index('aggregate_uuid_idx', 'uuid'),
                       UniqueConstraint('name', name='uniq_aggregate0name'),
                       mysql_engine='InnoDB',
                       mysql_charset='utf8')

    aggregate_hosts = Table('aggregate_hosts',
                            meta,
                            Column('created_at', DateTime),
                            Column('updated_at', DateTime),
                            Column('id',
                                   Integer,
                                   primary_key=True,
                                   nullable=False),
                            Column('host', String(length=255)),
                            Column('aggregate_id',
                                   Integer,
                                   ForeignKey('aggregates.id'),
                                   nullable=False),
                            UniqueConstraint(
                                'host',
                                'aggregate_id',
                                name='uniq_aggregate_hosts0host0aggregate_id'),
                            mysql_engine='InnoDB',
                            mysql_charset='utf8')

    aggregate_metadata = Table(
        'aggregate_metadata',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('aggregate_id',
               Integer,
               ForeignKey('aggregates.id'),
               nullable=False),
        Column('key', String(length=255), nullable=False),
        Column('value', String(length=255), nullable=False),
        UniqueConstraint('aggregate_id',
                         'key',
                         name='uniq_aggregate_metadata0aggregate_id0key'),
        Index('aggregate_metadata_key_idx', 'key'),
        mysql_engine='InnoDB',
        mysql_charset='utf8')

    groups = Table(
        'instance_groups',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('user_id', String(length=255)),
        Column('project_id', String(length=255)),
        Column('uuid', String(length=36), nullable=False),
        Column('name', String(length=255)),
        UniqueConstraint('uuid', name='uniq_instance_groups0uuid'),
        mysql_engine='InnoDB',
        mysql_charset='utf8',
    )

    group_policy = Table(
        'instance_group_policy',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('policy', String(length=255)),
        Column('group_id',
               Integer,
               ForeignKey('instance_groups.id'),
               nullable=False),
        Index('instance_group_policy_policy_idx', 'policy'),
        mysql_engine='InnoDB',
        mysql_charset='utf8',
    )

    group_member = Table(
        'instance_group_member',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('instance_uuid', String(length=255)),
        Column('group_id',
               Integer,
               ForeignKey('instance_groups.id'),
               nullable=False),
        Index('instance_group_member_instance_idx', 'instance_uuid'),
        mysql_engine='InnoDB',
        mysql_charset='utf8',
    )

    quota_classes = Table(
        'quota_classes',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('class_name', String(length=255)),
        Column('resource', String(length=255)),
        Column('hard_limit', Integer),
        Index('quota_classes_class_name_idx', 'class_name'),
        mysql_engine='InnoDB',
        mysql_charset='utf8',
    )

    quota_usages = Table(
        'quota_usages',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('project_id', String(length=255)),
        Column('resource', String(length=255), nullable=False),
        Column('in_use', Integer, nullable=False),
        Column('reserved', Integer, nullable=False),
        Column('until_refresh', Integer),
        Column('user_id', String(length=255)),
        Index('quota_usages_project_id_idx', 'project_id'),
        Index('quota_usages_user_id_idx', 'user_id'),
        mysql_engine='InnoDB',
        mysql_charset='utf8',
    )

    quotas = Table(
        'quotas',
        meta,
        Column('id', Integer, primary_key=True, nullable=False),
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('project_id', String(length=255)),
        Column('resource', String(length=255), nullable=False),
        Column('hard_limit', Integer),
        UniqueConstraint('project_id',
                         'resource',
                         name='uniq_quotas0project_id0resource'),
        mysql_engine='InnoDB',
        mysql_charset='utf8',
    )

    project_user_quotas = Table(
        'project_user_quotas',
        meta,
        Column('id', Integer, primary_key=True, nullable=False),
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('user_id', String(length=255), nullable=False),
        Column('project_id', String(length=255), nullable=False),
        Column('resource', String(length=255), nullable=False),
        Column('hard_limit', Integer, nullable=True),
        UniqueConstraint(
            'user_id',
            'project_id',
            'resource',
            name='uniq_project_user_quotas0user_id0project_id0resource'),
        Index('project_user_quotas_project_id_idx', 'project_id'),
        Index('project_user_quotas_user_id_idx', 'user_id'),
        mysql_engine='InnoDB',
        mysql_charset='utf8',
    )

    reservations = Table(
        'reservations',
        meta,
        Column('created_at', DateTime),
        Column('updated_at', DateTime),
        Column('id', Integer, primary_key=True, nullable=False),
        Column('uuid', String(length=36), nullable=False),
        Column('usage_id',
               Integer,
               ForeignKey('quota_usages.id'),
               nullable=False),
        Column('project_id', String(length=255)),
        Column('resource', String(length=255)),
        Column('delta', Integer, nullable=False),
        Column('expire', DateTime),
        Column('user_id', String(length=255)),
        Index('reservations_project_id_idx', 'project_id'),
        Index('reservations_uuid_idx', 'uuid'),
        Index('reservations_expire_idx', 'expire'),
        Index('reservations_user_id_idx', 'user_id'),
        mysql_engine='InnoDB',
        mysql_charset='utf8',
    )

    tables = [
        cell_mappings,
        host_mappings,
        instance_mappings,
        flavors,
        flavor_extra_specs,
        flavor_projects,
        request_specs,
        build_requests,
        keypairs,
        projects,
        users,
        resource_classes,
        resource_providers,
        inventories,
        traits,
        allocations,
        consumers,
        resource_provider_aggregates,
        resource_provider_traits,
        placement_aggregates,
        aggregates,
        aggregate_hosts,
        aggregate_metadata,
        groups,
        group_policy,
        group_member,
        quota_classes,
        quota_usages,
        quotas,
        project_user_quotas,
        reservations,
    ]
    for table in tables:
        table.create(checkfirst=True)
Ejemplo n.º 48
0
class RememberEntry(Base):
    __tablename__ = 'remember_rejected_entry'

    id = Column(Integer, primary_key=True)
    added = Column(DateTime, default=datetime.now)
    expires = Column(DateTime)
    title = Column(Unicode)
    url = Column(String)
    rejected_by = Column(String)
    reason = Column(String)

    task_id = Column('feed_id', Integer, ForeignKey('remember_rejected_feeds.id'), nullable=False)


Index('remember_feed_title_url', RememberEntry.task_id, RememberEntry.title, RememberEntry.url)


class FilterRememberRejected(object):
    """Internal.
    Rejects entries which have been rejected in the past.

    This is enabled when item is rejected with remember=True flag.

    Example::
        entry.reject('message', remember=True)
    """

    @plugin.priority(0)
    def on_task_start(self, task, config):
        """Purge remembered entries if the config has changed."""
Ejemplo n.º 49
0
from sqlalchemy import (
    Column,
    Index,
    Integer,
    Text,
)

from .meta import Base


class PurchaseOrders(Base):
    __tablename__ = 'purchase_orders'

    id = Column(Integer, primary_key=True)
    order_id = Column(Text)
    order_date = Column(Text)
    customer_name = Column(Text)
    transport_name = Column(Text)
    order_items = Column(Text)


Index('purchaseorders_index',
      PurchaseOrders.order_id,
      unique=True,
      mysql_length=255)
Ejemplo n.º 50
0
    @classmethod
    def api_loading_options(cls, expand=False):
        message_columns = [
            'public_id', 'is_draft', 'from_addr', 'to_addr', 'cc_addr',
            'bcc_addr', 'is_read', 'is_starred', 'received_date', 'is_sent'
        ]
        if expand:
            message_columns += [
                'subject', 'snippet', 'version', 'from_addr', 'to_addr',
                'cc_addr', 'bcc_addr', 'reply_to'
            ]
        return (subqueryload(
            Thread.messages).load_only(*message_columns).joinedload(
                'messagecategories').joinedload('category'),
                subqueryload(
                    Thread.messages).joinedload('parts').joinedload('block'))

    discriminator = Column('type', String(16))
    __mapper_args__ = {'polymorphic_on': discriminator}


# The /threads API endpoint filters on namespace_id and deleted_at, then orders
# by recentdate; add an explicit index to persuade MySQL to do this in a
# somewhat performant manner.
Index('ix_thread_namespace_id_recentdate_deleted_at', Thread.namespace_id,
      Thread.recentdate, Thread.deleted_at)
# Need to explicitly specify the index length for MySQL 5.6, because the
# subject column is too long to be fully indexed with utf8mb4 collation.
Index('ix_thread_subject', Thread.subject, mysql_length=191)
Index('ix_cleaned_subject', Thread._cleaned_subject, mysql_length=191)
Ejemplo n.º 51
0
from sqlalchemy import String, Column, Index
from sqlalchemy.orm import relationship

from provider.models.base import Base


class Licensor(Base):
    name = Column(String(512))
    country = Column(String(3), nullable=False, default="USA")
    state = Column(String(2))
    city = Column(String(64))

    licenses = relationship("License", back_populates="licensor")


Index("ix_monday_licensor_name_country_state_city",
      Licensor.name,
      Licensor.country,
      Licensor.state,
      Licensor.city,
      unique=True)
Ejemplo n.º 52
0
bundle = Table(
    'bundle',
    db_metadata,
    Column('id', Integer, primary_key=True, nullable=False),
    Column('uuid', String(63), nullable=False),
    Column('bundle_type', String(63), nullable=False),
    # The command will be NULL except for run bundles.
    Column('command', Text, nullable=True),
    # The data_hash will be NULL if the bundle's value is still being computed.
    Column('data_hash', String(63), nullable=True),
    Column('state', String(63), nullable=False),
    Column('owner_id', String(255), nullable=True),
    Column('is_anonymous', Boolean, nullable=False, default=False),
    UniqueConstraint('uuid', name='uix_1'),
    Index('bundle_data_hash_index', 'data_hash'),
    Index('state_index', 'state'),  # Needed for the bundle manager.
)

# Includes things like name, description, etc.
bundle_metadata = Table(
    'bundle_metadata',
    db_metadata,
    Column('id', Integer, primary_key=True, nullable=False),
    Column('bundle_uuid',
           String(63),
           ForeignKey(bundle.c.uuid),
           nullable=False),
    Column('metadata_key', String(63), nullable=False),
    Column('metadata_value', Text, nullable=False),
    Index('metadata_kv_index',
Ejemplo n.º 53
0
    __tablename__ = "tas_lookup"
    tas_id = Column(Integer, primary_key=True)
    allocation_transfer_agency = Column(Text, nullable=True, index=True)
    agency_identifier = Column(Text, nullable=True, index=True)
    beginning_period_of_availability = Column(Text, nullable=True, index=True)
    ending_period_of_availability = Column(Text, nullable=True, index=True)
    availability_type_code = Column(Text, nullable=True, index=True)
    main_account_code = Column(Text, nullable=True, index=True)
    sub_account_code = Column(Text, nullable=True, index=True)


Index("ix_tas",
      TASLookup.allocation_transfer_agency,
      TASLookup.agency_identifier,
      TASLookup.beginning_period_of_availability,
      TASLookup.ending_period_of_availability,
      TASLookup.availability_type_code,
      TASLookup.main_account_code,
      TASLookup.sub_account_code,
      unique=True)


class CGAC(Base):
    __tablename__ = "cgac"
    cgac_id = Column(Integer, primary_key=True)
    cgac_code = Column(Text, nullable=False, index=True, unique=True)
    agency_name = Column(Text)


class ObjectClass(Base):
    __tablename__ = "object_class"
Ejemplo n.º 54
0
class Marker(MarkerMixin, Base): # TODO rename to AccidentMarker
    __tablename__ = "markers"
    __table_args__ = (
        Index('acc_long_lat_idx', 'latitude', 'longitude'),
    )

    __mapper_args__ = {
        'polymorphic_identity': MARKER_TYPE_ACCIDENT
    }

    provider_code = Column(Integer, primary_key=True)
    description = Column(Text)
    subtype = Column(Integer)
    severity = Column(Integer)
    address = Column(Text)
    locationAccuracy = Column(Integer)
    roadType = Column(Integer)
    roadShape = Column(Integer)
    dayType = Column(Integer)
    unit = Column(Integer)
    mainStreet = Column(Text)
    secondaryStreet = Column(Text)
    junction = Column(Text)
    one_lane = Column(Integer)
    multi_lane = Column(Integer)
    speed_limit = Column(Integer)
    intactness = Column(Integer)
    road_width = Column(Integer)
    road_sign = Column(Integer)
    road_light = Column(Integer)
    road_control = Column(Integer)
    weather = Column(Integer)
    road_surface = Column(Integer)
    road_object = Column(Integer)
    object_distance = Column(Integer)
    didnt_cross = Column(Integer)
    cross_mode = Column(Integer)
    cross_location = Column(Integer)
    cross_direction = Column(Integer)

    @staticmethod
    def json_to_description(msg):
        description = json.loads(msg, encoding=db_encoding)
        return "\n".join([Marker.format_description(field, value) for field, value in description.iteritems()])

    def serialize(self, is_thin=False):
        fields = {
            "id": str(self.id),
            "provider_code": self.provider_code,
            "latitude": self.latitude,
            "longitude": self.longitude,
            "severity": self.severity,
            "locationAccuracy": self.locationAccuracy,
            "created": self.created.isoformat(),
        }
        if not is_thin:
            fields.update({
                "title": self.title,
                "description": Marker.json_to_description(self.description),
                "address": self.address,
                "type": self.type,
                "subtype": self.subtype,
                "roadType": self.roadType,
                "roadShape": self.roadShape,
                "dayType": self.dayType,
                "unit": self.unit,
                "mainStreet": self.mainStreet,
                "secondaryStreet": self.secondaryStreet,
                "junction": self.junction,
            })
            optional = {
                "one_lane": self.one_lane,
                "multi_lane": self.multi_lane,
                "speed_limit": self.speed_limit,
                "intactness": self.intactness,
                "road_width": self.road_width,
                "road_sign": self.road_sign,
                "road_light": self.road_light,
                "road_control": self.road_control,
                "weather": self.weather,
                "road_surface": self.road_surface,
                "road_object": self.road_object,
                "object_distance": self.object_distance,
                "didnt_cross": self.didnt_cross,
                "cross_mode": self.cross_mode,
                "cross_location": self.cross_location,
                "cross_direction": self.cross_direction,
            }
            for name, value in optional.iteritems():
                if value != 0:
                    fields[name] = value
        return fields

    def update(self, data, current_user):
        self.title = data["title"]
        self.description = data["description"]
        self.type = data["type"]
        self.latitude = data["latitude"]
        self.longitude = data["longitude"]

        self.put()

    @staticmethod
    def bounding_box_query(ne_lat, ne_lng, sw_lat, sw_lng, start_date, end_date,
                           fatal, severe, light, inaccurate, show_markers=True, is_thin=False, yield_per=None):
        # example:
        # ne_lat=32.36292402647484&ne_lng=35.08873443603511&sw_lat=32.29257266524761&sw_lng=34.88445739746089
        # >>>  m = Marker.bounding_box_query(32.36, 35.088, 32.292, 34.884)
        # >>> m.count()
        # 250

        if not show_markers:
            return Marker.query.filter(sql.false())
        accurate = not inaccurate
        markers = Marker.query \
            .filter(Marker.longitude <= ne_lng) \
            .filter(Marker.longitude >= sw_lng) \
            .filter(Marker.latitude <= ne_lat) \
            .filter(Marker.latitude >= sw_lat) \
            .filter(Marker.created >= start_date) \
            .filter(Marker.created < end_date) \
            .order_by(desc(Marker.created))
        if yield_per:
            markers = markers.yield_per(yield_per)
        if accurate:
            markers = markers.filter(Marker.locationAccuracy == 1)
        if not fatal:
            markers = markers.filter(Marker.severity != 1)
        if not severe:
            markers = markers.filter(Marker.severity != 2)
        if not light:
            markers = markers.filter(Marker.severity != 3)
        if is_thin:
            markers = markers.options(load_only("id", "longitude", "latitude"))
        return markers

    @staticmethod
    def get_marker(marker_id):
        return Marker.query.filter_by(id=marker_id)

    @classmethod
    def parse(cls, data):
        return Marker(
            type=MARKER_TYPE_ACCIDENT,
            title=data["title"],
            description=data["description"],
            latitude=data["latitude"],
            longitude=data["longitude"]
        )
Ejemplo n.º 55
0
class TorsionDriveProcedureORM(ProcedureMixin, BaseResultORM):
    """
        A torsion drive  procedure
    """

    __tablename__ = 'torsiondrive_procedure'

    id = Column(Integer, ForeignKey('base_result.id', ondelete='cascade'), primary_key=True)

    def __init__(self, **kwargs):
        kwargs.setdefault("version", 1)
        self.procedure = "torsiondrive"
        self.program = "torsiondrive"
        super().__init__(**kwargs)

    # input data (along with the mixin)

    # ids of the many to many relation
    initial_molecule = column_property(
                    select([func.array_agg(torsion_init_mol_association.c.molecule_id)])\
                    .where(torsion_init_mol_association.c.torsion_id==id)
            )
    # actual objects relation M2M, never loaded here
    initial_molecule_obj = relationship(MoleculeORM,
                                        secondary=torsion_init_mol_association,
                                        uselist=True,
                                        lazy='noload')

    optimization_spec = Column(JSON)

    # Output data
    final_energy_dict = Column(JSON)
    minimum_positions = Column(JSON)

    optimization_history_obj = relationship(
        OptimizationHistory,
        cascade="all, delete-orphan",  #backref="torsiondrive_procedure",
        order_by=OptimizationHistory.position,
        collection_class=ordering_list('position'),
        lazy='selectin')


    @hybrid_property
    def optimization_history(self):
        """calculated property when accessed, not saved in the DB
        A view of the many to many relation in the form of a dict"""

        ret = {}
        try:
            for opt_history in self.optimization_history_obj:
                if opt_history.key in ret:
                    ret[opt_history.key].append(str(opt_history.opt_id))
                else:
                    ret[opt_history.key] = [str(opt_history.opt_id)]

        except Exception as err:
            # raises exception of first access!!
            print(err)

        return ret

    @optimization_history.setter
    def optimization_history(self, dict_values):
        """A private copy of the opt history as a dict
        Key: list of optimization procedures"""

        return dict_values

    __table_args__ = (
        Index('ix_torsion_drive_program', 'program'),  # todo: needed for procedures?
    )

    __mapper_args__ = {
        'polymorphic_identity': 'torsiondrive_procedure',
        # to have separate select when querying BaseResultsORM
        'polymorphic_load': 'selectin',
    }

    def update_relations(self, initial_molecule=None, optimization_history=None, **kwarg):

        # update torsion molecule relation
        self._update_many_to_many(torsion_init_mol_association, 'torsion_id', 'molecule_id', self.id, initial_molecule,
                                  self.initial_molecule)

        self.optimization_history_obj = []
        for key in optimization_history:
            for opt_id in optimization_history[key]:
                opt_history = OptimizationHistory(torsion_id=int(self.id), opt_id=int(opt_id), key=key)
                self.optimization_history_obj.append(opt_history)
class GraphDbTraceConfig(Tuple, DeclarativeBase):
    __tupleType__ = graphDbTuplePrefix + 'GraphDbTraceConfigTable'
    __tablename__ = 'GraphDbTraceConfig'

    #:  The unique ID of this segment (database generated)
    id = Column(Integer, primary_key=True, autoincrement=True)

    #:  The model set for this segment
    modelSetId = Column(Integer,
                        ForeignKey('GraphDbModelSet.id', ondelete='CASCADE'),
                        nullable=False)
    modelSet = relationship(GraphDbModelSet)

    #:  The unique key of this segment
    key = Column(String, nullable=False)

    #:  The unique name of this segment
    name = Column(String, nullable=False)

    #:  The title to describe this segment
    title = Column(String, nullable=False)

    #:  The comment for this config
    comment = Column(String)

    #:  Is this trace config enabled
    isEnabled = Column(Boolean, nullable=False, server_default="true")

    #: The relationship for the rules
    rules = relationship('GraphDbTraceConfigRule', lazy='joined')

    __table_args__ = (
        Index("idx_TraceConfig_key", modelSetId, key, unique=True),
        Index("idx_TraceConfig_name", modelSetId, name, unique=True),
    )

    def fromTuple(self, tupleIn: GraphDbTraceConfigTuple,
                  modelSetId: int) -> 'GraphDbTraceConfig':
        self.modelSetId = modelSetId
        self.key = tupleIn.key
        self.name = tupleIn.name
        self.title = tupleIn.title
        self.comment = tupleIn.comment
        self.isEnabled = tupleIn.isEnabled

        self.rules = [GraphDbTraceConfigRule().fromTuple(rule, self.id)
                      for rule in tupleIn.rules]

        return self

    def toTuple(self) -> GraphDbTraceConfigTuple:
        traceTuple = GraphDbTraceConfigTuple(
            modelSetKey=self.modelSet.key,
            key=self.key,
            name=self.name,
            title=self.title,
            comment=self.comment,
            isEnabled=self.isEnabled
        )

        traceTuple.rules = [rule.toTuple() for rule in self.rules]

        return traceTuple
metadata = MetaData()
# 表COOKIES
cookies = Table(
    "COOKIES",
    metadata,
    Column("cookie_id", Integer(), primary_key=True),
    Column("cookie_name", String(50)),  # index = True
    Column("cookie_recipe_url", String(255)),
    Column("cookie_sku", String(55)),
    Column("quantity", Integer()),
    Column("unit_cost", Numeric(12, 2)),
    # 检查约束
    CheckConstraint("unit_cost >= 0.0", name="unit_cost_positive"),
    # 索引
    Index("ix_cookies_cookie_name", "cookie_name"),
    Index("ix_cookie_sku_name", "cookie_sku", "cookie_name"))
# 使用Table中字段
# Index("ix_cookies_cookie_name", cookies.c.cookie_name)
# Index("ix_cookie_sku_name", cookies.c.cookie_sku,
# cookies.c.cookie_name)

# 表USERS
users = Table(
    "USERS",
    metadata,
    Column("user_id", Integer()),  # primary_key = True
    Column("username", String(15), nullable=False),  # unique = True
    Column("email_address", String(255), nullable=False),
    Column("phone", String(20), nullable=False),
    Column("password", String(25), nullable=False),
Ejemplo n.º 58
0
    DateTime,
)

from .meta import Base


class User(Base):
    __tablename__ = 'users'
    id = Column(Integer, primary_key=True)
    username = Column(Text)
    password = Column(Text)
    userlevel = Column(Integer)
    carrierid = Column(Integer)
    cmdr_name = Column(Text)
    access_token = Column(Text)
    refresh_token = Column(Text)
    token_expiration = Column(Integer)
    has_validated = Column(Boolean)
    public_carrier = Column(Boolean)
    no_carrier = Column(Boolean)
    banned = Column(Boolean)
    cachedJson = Column(Text)
    apiKey = Column(Text)
    avatar = Column(Text)
    squadron = Column(Text)
    squadron_decal = Column(Text)
    lastUpdated = Column(DateTime)


Index('user_index', User.username, unique=True)
Ejemplo n.º 59
0
 def test_drop_plain(self):
     self.assert_compile(schema.DropIndex(Index(name="xyz")),
                         "DROP INDEX xyz")
Ejemplo n.º 60
0
class Auth(Base, CreateAt):
    __tablename__ = PREFIX_DB + 'auth'
    __table_args__ = (Index("email", unique=True), )

    status = Column(Integer, nullable=False, default='0', server_default='0')
    email = Column(String(255), nullable=False)