Python AppDescription Examples

Example #1

File: gs_ready_agg.py Project: cityofaustin/atd-data-lake

"""
ATD Data Lake 'ready' bucket for GRIDSMART, with aggregation

@author Kenneth Perrine, Nadia Florez
"""
import pandas as pd
import numpy as np
import arrow

import _setpath
from atd_data_lake.support import etl_app, last_update
from atd_data_lake.util import date_util

APP_DESCRIPTION = etl_app.AppDescription(
    appName="gs_ready_agg.py",
    appDescr="Aggregates 'ready' Data Lake bucket GRIDSMART counts")

class GSReadyAggApp(etl_app.ETLApp):
    """
    Application functions and special behavior around GRIDSMART aggregation
    """
    def __init__(self, args):
        """
        Initializes application-specific variables
        """
        super().__init__("gs", APP_DESCRIPTION,
                         args=args,
                         purposeSrc="ready",
                         purposeTgt="ready",
                         perfmetStage="Aggregate")

Example #2

File: bt_extract_soc.py Project: cityofaustin/atd-data-lake

"""
Publish Bluetooth "Ready" Data Lake data

@author Kenneth Perrine, Nadia Florez
"""
import hashlib

import arrow

import _setpath
from atd_data_lake.support import etl_app, last_update
from atd_data_lake import config

# This sets up application information:
APP_DESCRIPTION = etl_app.AppDescription(
    appName="bt_extract_soc.py",
    appDescr="Extracts Bluetooth files from the 'Ready' bucket to Socrata")

class BTPublishApp(etl_app.ETLApp):
    """
    Application functions and special behavior around Bluetooth exporting to Socrata.
    """
    def __init__(self, args):
        """
        Initializes application-specific variables
        """
        super().__init__("bt", APP_DESCRIPTION,
                         args=args,
                         purposeSrc="ready",
                         perfmetStage="Publish")
        self.publishers = None

Example #3

File: wt_extract_soc.py Project: cityofaustin/atd-data-lake

"""
Publish Wavetronix "Ready" Data Lake data

@author Kenneth Perrine
"""
import hashlib

import arrow

import _setpath
from atd_data_lake.support import etl_app, last_update
from atd_data_lake import config

# This sets up application information:
APP_DESCRIPTION = etl_app.AppDescription(
    appName="wt_extract_soc.py",
    appDescr="Extracts Wavetronix from the 'Ready' bucket to Socrata")


class GSAggPublishApp(etl_app.ETLApp):
    """
    Application functions and special behavior around Wavetronix exporting to Socrata.
    """
    def __init__(self, args):
        """
        Initializes application-specific variables
        """
        super().__init__("wt",
                         APP_DESCRIPTION,
                         args=args,
                         purposeSrc="ready",

Example #4

"""
Publish GRIDSMART Aggregated "Ready" Data Lake data

@author Kenneth Perrine, Nadia Florez
"""
import hashlib

import arrow

import _setpath
from atd_data_lake.support import etl_app, last_update
from atd_data_lake import config

# This sets up application information:
APP_DESCRIPTION = etl_app.AppDescription(
    appName="gs_agg_extract_soc.py",
    appDescr="Extracts GRIDSMART aggregates from the 'Ready' bucket to Socrata"
)


class GSAggPublishApp(etl_app.ETLApp):
    """
    Application functions and special behavior around GRIDSMART exporting to Socrata.
    """
    def __init__(self, args):
        """
        Initializes application-specific variables
        """
        super().__init__("gs",
                         APP_DESCRIPTION,
                         args=args,
                         purposeSrc="ready",

Example #5

File: bt_json_standard.py Project: cityofaustin/atd-data-lake

"""
Bluetooth sensor JSON standardization translates between the "raw" and "rawjson" layers.

@author Kenneth Perrine, Nadia Florez
"""
import csv, datetime, os

import _setpath
from atd_data_lake.support import etl_app, last_update, perfmet
from atd_data_lake.util import date_util
from atd_data_lake import config

# This sets up application information:
APP_DESCRIPTION = etl_app.AppDescription(
    appName="bt_json_standard.py",
    appDescr=
    "Performs JSON canonicalization for Bluetooth data between the raw and rawjson Data Lake buckets"
)


class BTJSONStandardApp(etl_app.ETLApp):
    """
    Application functions and special behavior around Bluetooth JSON canonicalization.
    """
    def __init__(self, args):
        """
        Initializes application-specific variables
        """
        super().__init__("bt",
                         APP_DESCRIPTION,
                         args=args,

Example #6

"""
Movement of GRIDSMART data files to S3 "raw" layer.

@author Kenneth Perrine, Nadia Florez
"""
import os

import _setpath
from atd_data_lake.support import etl_app, last_update
from atd_data_lake import config
from atd_data_lake.drivers.devices import last_upd_gs, gs_support

# This sets up application information:
APP_DESCRIPTION = etl_app.AppDescription(
    appName="gs_insert_lake.py",
    appDescr="Inserts GRIDSMART data from field devices into the Raw Data Lake"
)


class GSInsertLakeApp(etl_app.ETLApp):
    """
    Application functions and special behavior around GRIDSMART ingestion.
    """
    def __init__(self, args):
        """
        Initializes application-specific variables
        """
        self.deviceFilter = None
        super().__init__("gs",
                         APP_DESCRIPTION,
                         args=args,

Example #7

"""
Bluetooth sensor ingestion takes files from the AWAM share and places them into the Data Lake
"raw" layer.

@author Kenneth Perrine
"""
import _setpath
from atd_data_lake.support import etl_app, last_update
from atd_data_lake.config import config_app
from atd_data_lake.util import date_dirs
from atd_data_lake.drivers import last_upd_fs

# This sets up application information:
APP_DESCRIPTION = etl_app.AppDescription(
    appName="bt_insert_lake.py",
    appDescr="Inserts Bluetooth data from AWAM share into the Raw Data Lake")

# This defines the valid filename formats that exist in the AWAM directory:
DIR_DEFS = [date_dirs.DateDirDef(prefix=config_app.UNIT_LOCATION + "_bt_",
                                 dateFormat="%m-%d-%Y",
                                 postfix=".txt"),
            date_dirs.DateDirDef(prefix=config_app.UNIT_LOCATION + "_btmatch_",
                                 dateFormat="%m-%d-%Y",
                                 postfix=".txt"),
            date_dirs.DateDirDef(prefix=config_app.UNIT_LOCATION + "_bt_summary_15_",
                                 dateFormat="%m-%d-%Y",
                                 postfix=".txt")]

class BTLastUpdateProv(last_upd_fs.LastUpdFileProv):
    """
    Overrides the default file provider so as to generate the correct identifier for AWAM files

Example #8

File: wt_ready.py Project: cityofaustin/atd-data-lake

Bluetooth sensor JSON preparation translates "rawjson" and "ready" layers.

@author Kenneth Perrine, Nadia Florez
"""
import hashlib

import pandas as pd

import _setpath
from atd_data_lake.support import etl_app, last_update
from atd_data_lake import config

# This sets up application information:
APP_DESCRIPTION = etl_app.AppDescription(
    appName="wt_ready.py",
    appDescr=
    "Performs JSON enrichment for Wavetronix data between the 'rawjson' and 'ready' Data Lake buckets"
)


class WTReadyApp(etl_app.ETLApp):
    """
    Application functions and special behavior around Wavetronix JSON final data enrichment.
    """
    def __init__(self, args):
        """
        Initializes application-specific variables
        """
        super().__init__("wt",
                         APP_DESCRIPTION,
                         args=args,

Example #9

File: gs_json_standard.py Project: cityofaustin/atd-data-lake

"""
import os, datetime, json

import pandas as pd
import pytz

import _setpath
from atd_data_lake.support import etl_app, last_update, perfmet
from atd_data_lake import config
from atd_data_lake.drivers.devices import gs_investigate
from atd_data_lake.util import date_util

# This sets up application information:
APP_DESCRIPTION = etl_app.AppDescription(
    appName="gs_json_standard.py",
    appDescr=
    "Performs JSON canonicalization for GRIDSMART data between the 'raw' and 'rawjson' Data Lake buckets"
)


class GSJSONStandardApp(etl_app.ETLApp):
    """
    Application functions and special behavior around GRIDSMART JSON canonicalization.
    """
    def __init__(self, args):
        """
        Initializes application-specific variables
        """
        super().__init__("gs",
                         APP_DESCRIPTION,
                         args=args,