def main():

    # Reads user configuration
    user_config = getPreprocDBParams()
    pollutants = user_config[DBToCSVParams.tables]
    output_folder = user_config[DBToCSVParams.output_folder]
    start_date = user_config[DBToCSVParams.start_date]
    end_date = user_config[DBToCSVParams.end_date]
    stations = user_config[DBToCSVParams.stations]

    if not (os.path.exists(output_folder)):
        os.makedirs(output_folder)

    # Connect to db
    conn = getPostgresConn()
    for cur_station in stations:
        print(F" ====================== {cur_station} ====================== ")
        for cur_pollutant in pollutants:
            print(
                F"\t ---------------------- {cur_pollutant} ---------------------- "
            )
            cur_data = np.array(
                getPollutantFromDateRange(conn, cur_pollutant, start_date,
                                          end_date, [cur_station]))
            if len(cur_data) > 0:
                dates = np.array([x[0] for x in cur_data])
                print(
                    F"\tTotal number of rows obtained for {cur_station}-{cur_pollutant}: {len(dates)}"
                )

                # Finding 'clean' dates (those dates with continuous 'num_hours' number of hours
                # clean_dates = []
                # for idx, cur_date in enumerate(dates):
                #     desired_date = cur_date + timedelta(hours=num_hours)
                #     if len(dates) > (idx + num_hours):
                #         if desired_date == dates[idx + num_hours]:
                #             clean_dates.append(idx)
                # print(F"\tTotal CLEANED # of rows for {cur_station}-{cur_pollutant}: {len(clean_dates)}")

                # Selects the dates of this 'file'
                df = DataFrame({cur_pollutant: cur_data[:, 1]}, index=dates)
                file_name = F"{cur_pollutant}_{cur_station}.csv"
                df.to_csv(join(output_folder, file_name),
                          index_label=constants.index_label.value)
            else:
                print("\t\t Warning!!!  NO DATA")

    conn.close()
示例#2
0
from db.queries import getAllStations, getCountValidData, getMaxDailySQL

from dash.dependencies import Input, Output, State
from db.names import findTable
from viz.layout import get_layout_db

# https://dash.plot.ly/interactive-graphing
# https://plot.ly/python-api-reference/

COLORS = ['r', 'g', 'b', 'y']
date_format = '%Y-%m-%d'
date_format_ext = '%Y-%m-%dT%H:%M:%S'

styles = {'pre': {'border': 'thin lightgrey solid', 'overflowX': 'scroll'}}

conn = getPostgresConn()
stations_geodf = getAllStations(conn)
stations_ids = stations_geodf.index.values
pollutant = "O3"
table = findTable(pollutant)
app = get_layout_db()


@app.callback([
    Output('nandata-plot', 'figure'),
    Output('nandata-plotall', 'figure'),
    Output('max_otres_plot', 'figure')
], [Input('drop-years', 'value')])
def display_figure(cur_year):
    print(cur_year)
    if cur_year is None: