class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0027_auto_20210303_1230'),
    ]

    # This deploys a fix in the view 'cmsa_15min_view_v8' which flipped up and down counts

    # NOTE: Simply remove the views and redeploy them. Because the materialized view
    # depends on the view, and the realtime view depends on the materialized view, I need
    # to remove all three of them, and then recreate them again in reversing order.

    _VIEW_NAME = "cmsa_15min_view_v8"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    _REALTIME_VIEW_NAME = "cmsa_15min_view_v8_realtime_predict"
    _realtime_view_strings = get_view_strings(VIEW_STRINGS,
                                              _REALTIME_VIEW_NAME)

    _REALTIME_30D_VIEW_NAME = "cmsa_15min_view_v8_realtime_predict_30d"
    _realtime_30d_view_strings = get_view_strings(VIEW_STRINGS,
                                                  _REALTIME_30D_VIEW_NAME)

    operations = [
        # First remove the views
        migrations.RunSQL(
            sql=_realtime_30d_view_strings['reverse_sql_materialized'],
            reverse_sql=_realtime_30d_view_strings['sql_materialized']),
        migrations.RunSQL(sql=_realtime_30d_view_strings['reverse_sql'],
                          reverse_sql=_realtime_30d_view_strings['sql']),
        migrations.RunSQL(
            sql=_realtime_view_strings['reverse_sql_materialized'],
            reverse_sql=_realtime_view_strings['sql_materialized']),
        migrations.RunSQL(sql=_realtime_view_strings['reverse_sql'],
                          reverse_sql=_realtime_view_strings['sql']),
        migrations.RunSQL(sql=_view_strings['reverse_sql_materialized'],
                          reverse_sql=_view_strings['sql_materialized']),
        migrations.RunSQL(sql=_view_strings['reverse_sql'],
                          reverse_sql=_view_strings['sql']),

        # And then recreate them again
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
        migrations.RunSQL(sql=_realtime_view_strings['sql'],
                          reverse_sql=_realtime_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_realtime_view_strings['sql_materialized'],
            reverse_sql=_realtime_view_strings['reverse_sql_materialized']),
        migrations.RunSQL(
            sql=_realtime_30d_view_strings['sql'],
            reverse_sql=_realtime_30d_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_realtime_30d_view_strings['sql_materialized'],
            reverse_sql=_realtime_30d_view_strings['reverse_sql_materialized']
        ),
    ]
Ejemplo n.º 2
0
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0031_redeploy_v8_views_20210607_1834'),
    ]

    # This script deploys the table 'peoplemeasurement_v1_data' and 'cmsa_15min_view_v10' views which contain a new prediction method and performance improvements
    # For more information about the 'peoplemeasurement_v1_data', see the peoplemeasurement models defintions file

    # NOTE: Simply create the view's in this specific order because of circular dependencies.
    # The materialized view depends on the view, and the realtime view depends on the materialized view.

    _V1_DATA_VIEW_NAME = "peoplemeasurement_v1_data"
    _v1_data_view_strings = get_view_strings(VIEW_STRINGS, _V1_DATA_VIEW_NAME)

    _VIEW_NAME = "cmsa_15min_view_v10"
    _view_strings = get_view_strings(VIEW_STRINGS,
                                     _VIEW_NAME,
                                     indexes=[('sensor', 'timestamp_rounded')])

    _REALTIME_VIEW_NAME = "cmsa_15min_view_v10_realtime"
    _realtime_view_strings = get_view_strings(VIEW_STRINGS,
                                              _REALTIME_VIEW_NAME,
                                              indexes=[('sensor',
                                                        'timestamp_rounded')])

    _PREDICT_VIEW_NAME = "cmsa_15min_view_v10_predict"
    _predict_view_strings = get_view_strings(VIEW_STRINGS, _PREDICT_VIEW_NAME)

    _REALTIME_PREDICT_VIEW_NAME = "cmsa_15min_view_v10_realtime_predict"
    _realtime_predict_view_strings = get_view_strings(
        VIEW_STRINGS, _REALTIME_PREDICT_VIEW_NAME)

    operations = [

        # Create the views
        migrations.RunSQL(sql=_v1_data_view_strings['sql'],
                          reverse_sql=_v1_data_view_strings['reverse_sql']),
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
        migrations.RunSQL(sql=_view_strings['indexes'][0]),
        migrations.RunSQL(sql=_realtime_view_strings['sql'],
                          reverse_sql=_realtime_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_realtime_view_strings['sql_materialized'],
            reverse_sql=_realtime_view_strings['reverse_sql_materialized']),
        migrations.RunSQL(sql=_realtime_view_strings['indexes'][0]),
        migrations.RunSQL(sql=_predict_view_strings['sql'],
                          reverse_sql=_predict_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_realtime_predict_view_strings['sql'],
            reverse_sql=_realtime_predict_view_strings['reverse_sql']),
    ]
Ejemplo n.º 3
0
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0029_unique_index_on_30d_20210304_1630'),
        ('telcameras_v3', '0003_groupaggregate_count_scrambled')
    ]

    # This deploys the v9 view's which use the crambled count fields

    # NOTE: Simply create the view's in this specific order because of circular dependencies.
    # The materialized view depends on the view, and the realtime view depends on the materialized view.

    _VIEW_NAME = "cmsa_15min_view_v9"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    _REALTIME_VIEW_NAME = "cmsa_15min_view_v9_realtime_predict"
    _realtime_view_strings = get_view_strings(VIEW_STRINGS, _REALTIME_VIEW_NAME)

    _REALTIME_30D_VIEW_NAME = "cmsa_15min_view_v9_realtime_predict_30d"
    _realtime_30d_view_strings = get_view_strings(VIEW_STRINGS, _REALTIME_30D_VIEW_NAME)

    operations = [
        migrations.AddField(
            model_name='countaggregate',
            name='count_scrambled',
            field=models.SmallIntegerField(null=True),
        ),

        # Create the views
        migrations.RunSQL(
            sql=_view_strings['sql'],
            reverse_sql=_view_strings['reverse_sql']
        ),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']
        ),
        migrations.RunSQL(
            sql=_realtime_view_strings['sql'],
            reverse_sql=_realtime_view_strings['reverse_sql']
        ),
        migrations.RunSQL(
            sql=_realtime_view_strings['sql_materialized'],
            reverse_sql=_realtime_view_strings['reverse_sql_materialized']
        ),
        migrations.RunSQL(
            sql=_realtime_30d_view_strings['sql'],
            reverse_sql=_realtime_30d_view_strings['reverse_sql']
        ),
        migrations.RunSQL(
            sql=_realtime_30d_view_strings['sql_materialized'],
            reverse_sql=_realtime_30d_view_strings['reverse_sql_materialized']
        ),
    ]
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0020_cmsa_15min_view_v8_20210209_2016'),
    ]

    _VIEW_NAME = "cmsa_15min_view_v8"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    operations = [
        # First remove the views
        migrations.RunSQL(
            sql=_view_strings['reverse_sql_materialized'],
            reverse_sql=_view_strings['sql_materialized'],
        ),
        migrations.RunSQL(
            sql=_view_strings['reverse_sql'],
            reverse_sql=_view_strings['sql'],
        ),

        # And then redeploy it again
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
    ]
Ejemplo n.º 5
0
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0023_drop_old_views_20210209_1153'),
    ]

    # NOTE: Simply remove the views and redeploy them. Because the materialized view
    # dependes on the view, and the realtime view dependes on the materialized view, I need to remove all
    # three of them, and then recreate them again in reversing order.

    _VIEW_NAME = "cmsa_15min_view_v8"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    _REALTIME_VIEW_NAME = "cmsa_15min_view_v8_realtime_predict"
    _realtime_view_strings = get_view_strings(VIEW_STRINGS,
                                              _REALTIME_VIEW_NAME)

    operations = [
        # First remove the views
        migrations.RunSQL(
            sql=_realtime_view_strings['reverse_sql_materialized'],
            reverse_sql=_realtime_view_strings['sql_materialized']),
        migrations.RunSQL(sql=_realtime_view_strings['reverse_sql'],
                          reverse_sql=_realtime_view_strings['sql']),
        migrations.RunSQL(sql=_view_strings['reverse_sql_materialized'],
                          reverse_sql=_view_strings['sql_materialized']),
        migrations.RunSQL(sql=_view_strings['reverse_sql'],
                          reverse_sql=_view_strings['sql']),

        # And then recreate them again
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
        migrations.RunSQL(sql=_realtime_view_strings['sql'],
                          reverse_sql=_realtime_view_strings['reverse_sql']),
        # And also make a materialized version of the predict view which we can refresh every minute
        migrations.RunSQL(
            sql=_realtime_view_strings['sql_materialized'],
            reverse_sql=_realtime_view_strings['reverse_sql_materialized']),
    ]
Ejemplo n.º 6
0
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2',
         '0016_cmsa_15min_view_v7_realtime_predict_20201119_1728'),
    ]

    # NOTE: Because we just made a small adjustment to the view I didn't want to create a whole new version of
    # the view. For that reason I simply remove the views and redeploy them. Because the materialized view
    # dependes on the view, and the realtime view dependes on the materialized view, I need to remove all
    # three of them, and then recreate them again in reversing order.

    _VIEW_NAME = "cmsa_15min_view_v7"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    _REALTIME_VIEW_NAME = "cmsa_15min_view_v7_realtime_predict"
    _realtime_view_strings = get_view_strings(VIEW_STRINGS,
                                              _REALTIME_VIEW_NAME)

    operations = [
        # First remove the views
        migrations.RunSQL(sql=_realtime_view_strings['reverse_sql'],
                          reverse_sql=_realtime_view_strings['sql']),
        migrations.RunSQL(sql=_view_strings['reverse_sql_materialized'],
                          reverse_sql=_view_strings['sql_materialized']),
        migrations.RunSQL(sql=_view_strings['reverse_sql'],
                          reverse_sql=_view_strings['sql']),

        # And then recreate them again
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
        migrations.RunSQL(sql=_realtime_view_strings['sql'],
                          reverse_sql=_realtime_view_strings['reverse_sql']),
        # And also make a materialized version of the predict view which we can refresh every minute
        migrations.RunSQL(
            sql=_realtime_view_strings['sql_materialized'],
            reverse_sql=_realtime_view_strings['reverse_sql_materialized']),
    ]
Ejemplo n.º 7
0
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0011_cmsa_15min_view_v6_20200916_1300'),
    ]

    # VIEW DESCRIPTION: This view combines the data from the materialized view v6 and combines it with the realtime data
    _VIEW_NAME = "cmsa_15min_view_v6_realtime"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    operations = [
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql'])
    ]
Ejemplo n.º 8
0
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0025_scramble_counts_20210120_1147'),
    ]

    # NOTE: Here we remove one views and redeploy it. Because the realtime
    # view dependes on the materialized view, I need to remove both and then
    # recreate them again in reversing order.

    _VIEW_NAME = "cmsa_15min_view_v8_realtime_predict"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    _30D_VIEW_NAME = "cmsa_15min_view_v8_realtime_predict_30d"
    _30d_view_strings = get_view_strings(VIEW_STRINGS, _30D_VIEW_NAME)

    operations = [
        # Deploy the new 30d view
        migrations.RunSQL(sql=_30d_view_strings['sql'],
                          reverse_sql=_30d_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_30d_view_strings['sql_materialized'],
            reverse_sql=_30d_view_strings['reverse_sql_materialized']),

        # Remove the existing 1d view
        migrations.RunSQL(sql=_view_strings['reverse_sql_materialized'],
                          reverse_sql=_view_strings['sql_materialized']),
        migrations.RunSQL(sql=_view_strings['reverse_sql'],
                          reverse_sql=_view_strings['sql']),

        # Then redeploy the 1d view
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
    ]
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0013_auto_20201111_1050'),
    ]
    # This removes the old tables that were left behind after the timescale migration
    # Like black lives, the order of the commands also matters.
    # The removed views will be add in subsequent migrations
    operations = [
        # Remove  dependent views
        migrations.RunSQL(
            sql=f"DROP MATERIALIZED VIEW cmsa_15min_view_v4_materialized;"),
        migrations.RunSQL(sql=f"DROP VIEW cmsa_15min_view_v4;"),
        migrations.RunSQL(
            sql=f"DROP MATERIALIZED VIEW cmsa_15min_view_v5_materialized;"),
        migrations.RunSQL(sql=f"DROP VIEW cmsa_15min_view_v5;"),
        migrations.RunSQL(sql=f"DROP VIEW cmsa_15min_view_v6_realtime;"),
        migrations.RunSQL(
            sql=f"DROP MATERIALIZED VIEW cmsa_15min_view_v6_materialized;"),
        migrations.RunSQL(sql=f"DROP VIEW cmsa_15min_view_v6;"),

        # Make sequences only dependent on the new tables. This to make it possible to remove the old tables later
        migrations.RunSQL(
            sql=
            f"ALTER SEQUENCE telcameras_v2_countaggregate_id_seq OWNED BY telcameras_v2_countaggregate.id;"
        ),
        migrations.RunSQL(
            sql=
            f"ALTER SEQUENCE telcameras_v2_personaggregate_id_seq OWNED BY telcameras_v2_personaggregate.id;"
        ),
        migrations.RunSQL(
            sql=
            f"ALTER SEQUENCE telcameras_v2_observation_id_seq OWNED BY telcameras_v2_observation.id;"
        ),
    ]

    # Add the previously removed views and their accompanying materialized views again
    for view in VIEWS:
        view_strings = get_view_strings(VIEW_STRINGS, view)

        operations.append(
            migrations.RunSQL(sql=view_strings['sql'],
                              reverse_sql=view_strings['reverse_sql']))
        if view != 'cmsa_15min_view_v6_realtime':
            operations.append(
                migrations.RunSQL(
                    sql=view_strings['sql_materialized'],
                    reverse_sql=view_strings['reverse_sql_materialized']), )
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0021_redeploy_cmsa_15min_view_v8_20210211_1344'),
    ]

    _VIEW_NAME = "cmsa_15min_view_v8_realtime_predict"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    operations = [
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
    ]
Ejemplo n.º 11
0
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0018_drop_old_tables_20200208_1555'),
        ('telcameras_v3', '0002_timescale_20201214_1412'),
    ]

    _VIEW_NAME = "cmsa_15min_view_v8"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    operations = [
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
    ]
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0015_cmsa_15min_view_v7_20201117_1514'),
        ('peoplemeasurement', '0017_voorspelcoefficient_voorspelintercept'),
    ]

    _VIEW_NAME = "cmsa_15min_view_v7_realtime_predict"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)


    operations = [
        migrations.RunSQL(
            sql=_view_strings['sql'],
            reverse_sql=_view_strings['reverse_sql']
        ),
    ]
Ejemplo n.º 13
0
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0008_cmsa_15min_view_v4_20200902_1420'),
    ]

    # VIEW DESCRIPTION: This view only uses data from the telcameras_v2 from the time we actually have data, and
    # then disregards data from peoplemeasurement (v1)
    _VIEW_NAME = "cmsa_15min_view_v5"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    operations = [
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
    ]
Ejemplo n.º 14
0
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0014_redo_views_after_timescale_migration_20201116_1155'),
    ]

    _VIEW_NAME = "cmsa_15min_view_v7"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    operations = [
        migrations.RunSQL(
            sql=_view_strings['sql'],
            reverse_sql=_view_strings['reverse_sql']
        ),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']
        ),
    ]
Ejemplo n.º 15
0
class Migration(migrations.Migration):

    dependencies = [
        ('peoplemeasurement', '0001_initial'),
        ('telcameras_v2', '0007_auto_20200803_1902'),
        ('peoplemeasurement',
         '0002_peoplemeasurementcsv_peoplemeasurementcsvtemp'),
    ]

    # VIEW DESCRIPTION: This view combines the data from the peoplemeasurement data and the telcameras_v2 data
    # into a union view.
    _VIEW_NAME = "cmsa_15min_view_v4"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    operations = [
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
    ]
Ejemplo n.º 16
0
class Migration(migrations.Migration):

    dependencies = [
        ('telcameras_v2', '0030_deploy_v9_views_20210316_1040'),
    ]

    # This script redeploys the 'cmsa_15min_view_v8' views which contain new functionality to add 4 sensors beginning with "GAAB"

    # NOTE: Simply remove the views and redeploy them. Because the materialized view
    # depends on the view, and the realtime view depends on the materialized view, I need
    # to remove all three of them, and then recreate them again in reversing order.

    _VIEW_NAME = "cmsa_15min_view_v8"
    _view_strings = get_view_strings(VIEW_STRINGS, _VIEW_NAME)

    _REALTIME_VIEW_NAME = "cmsa_15min_view_v8_realtime_predict"
    _realtime_view_strings = get_view_strings(VIEW_STRINGS,
                                              _REALTIME_VIEW_NAME)

    _REALTIME_30D_VIEW_NAME = "cmsa_15min_view_v8_realtime_predict_30d"
    _realtime_30d_view_strings = get_view_strings(VIEW_STRINGS,
                                                  _REALTIME_30D_VIEW_NAME,
                                                  indexes=[
                                                      ('sensor',
                                                       'timestamp_rounded')
                                                  ])

    operations = [
        # First remove the views
        migrations.RunSQL(
            sql=_realtime_30d_view_strings['reverse_sql_materialized'],
            reverse_sql=_realtime_30d_view_strings['sql_materialized']),
        migrations.RunSQL(sql=_realtime_30d_view_strings['reverse_sql'],
                          reverse_sql=_realtime_30d_view_strings['sql']),
        migrations.RunSQL(
            sql=_realtime_view_strings['reverse_sql_materialized'],
            reverse_sql=_realtime_view_strings['sql_materialized']),
        migrations.RunSQL(sql=_realtime_view_strings['reverse_sql'],
                          reverse_sql=_realtime_view_strings['sql']),
        migrations.RunSQL(sql=_view_strings['reverse_sql_materialized'],
                          reverse_sql=_view_strings['sql_materialized']),
        migrations.RunSQL(sql=_view_strings['reverse_sql'],
                          reverse_sql=_view_strings['sql']),

        # And then recreate them again
        migrations.RunSQL(sql=_view_strings['sql'],
                          reverse_sql=_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_view_strings['sql_materialized'],
            reverse_sql=_view_strings['reverse_sql_materialized']),
        migrations.RunSQL(sql=_realtime_view_strings['sql'],
                          reverse_sql=_realtime_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_realtime_view_strings['sql_materialized'],
            reverse_sql=_realtime_view_strings['reverse_sql_materialized']),
        migrations.RunSQL(
            sql=_realtime_30d_view_strings['sql'],
            reverse_sql=_realtime_30d_view_strings['reverse_sql']),
        migrations.RunSQL(
            sql=_realtime_30d_view_strings['sql_materialized'],
            reverse_sql=_realtime_30d_view_strings['reverse_sql_materialized']
        ),
        migrations.RunSQL(sql=_realtime_30d_view_strings['indexes'], ),
    ]