def html_doc_items(self) -> [(str, str)]:
return [
('file name', _.i[self.file_name]),
('compression', _.tt[self.compression]),
('mapper script file name', _.i[self.mapper_script_file_name]),
(_.i['content'],
html.highlight_syntax(
self.mapper_file_path().read_text().strip('\n')
if self.mapper_script_file_name
and self.mapper_file_path().exists() else '', 'python')),
('make unique', _.tt[self.make_unique]),
('target_table', _.tt[self.target_table]),
('db alias', _.tt[self.db_alias()]),
('csv format', _.tt[self.csv_format]),
('skip header', _.tt[self.skip_header]),
('delimiter char',
_.tt[json.dumps(self.delimiter_char
) if self.delimiter_char != None else None]),
('quote char',
_.tt[json.dumps(self.quote_char
) if self.quote_char != None else None]),
('null value string',
_.tt[json.dumps(self.null_value_string
) if self.null_value_string != None else None]),
('time zone', _.tt[self.timezone]),
(_.i['shell command'],
html.highlight_syntax(self.shell_command(), 'bash'))
]
def html_doc_items(self) -> [(str, str)]:
config_file_content = self.config_file_path().read_text().strip(
'\n') if self.config_file_path().exists() else '-- file not found'
config_final = json.dumps(self.tap_config) if self.config_file_path(
).exists() or self._tap_config else '-- file not found'
state = self.state_file_path().read_text().strip(
'\n') if self.state_file_path().exists() else '-- file not found'
doc = [('tap name', self.tap_name)]
if self.config_file_name:
# doc.append(('config file name', _.i[self.config_file_name]))
doc.append((_.i['config file content'],
html.highlight_syntax(config_file_content, 'json')))
if self._tap_config:
doc.append((_.i['config'],
html.highlight_syntax(json.dumps(self._tap_config),
'json')))
doc.append((_.i['config final'],
html.highlight_syntax(config_final, 'json')))
if self.state_file_name:
# doc.append(('state file name', _.i[self.state_file_name]))
doc.append((_.i['state file content'],
html.highlight_syntax(state, 'json')))
#if self.catalog_file_name:
# doc.append(('catalog file name', _.i[self.catalog_file_name]))
return doc
def html_doc_items(self, db_alias: str):
sql = self.sql_statement or \
(self.sql_file_path().read_text().strip('\n') if self.sql_file_path().exists() else '-- file not found')
doc = []
if self.sql_statement:
doc.append(('sql statement',
html.highlight_syntax(
self.sql_statement,
_sql_syntax_higlighting_lexter(db_alias))))
else:
doc.append(('sql file name', _.i[self.sql_file_name]))
doc.append((_.i['sql file content'],
html.highlight_syntax(
sql, _sql_syntax_higlighting_lexter(db_alias))))
doc.append(('replace',
html.highlight_syntax(
json.dumps(_expand_pattern_substitution(self.replace),
indent=2), 'json')))
if self.replace:
for k, v in _expand_pattern_substitution(self.replace).items():
sql = sql.replace(k, v)
doc.append((_.i['substituted sql'],
html.highlight_syntax(
sql, _sql_syntax_higlighting_lexter(db_alias))))
return doc
def html_doc_items(self) -> [(str, str)]:
path = self.parent.base_path() / self.file_name
return [('parameter function',
html.highlight_syntax(inspect.getsource(self.parameter_function), 'python')),
('file name', _.i[self.file_name]),
(_.i['file content'], html.highlight_syntax(path.read_text().strip('\n')
if self.file_name and path.exists()
else '', 'python'))]
def html_doc_items(self) -> [(str, str)]:
return [
('function', _.pre[escape(str(self.function))]),
('parameter function',
html.highlight_syntax(inspect.getsource(self.parameter_function),
'python')),
(_.i['implementation'],
html.highlight_syntax(inspect.getsource(self.function), 'python'))
]
def html_doc_items(self):
path = self.parent.parent.base_path() / self.file_name
return [
('file name', _.i[self.file_name]),
('args', _.tt[json.dumps(self.args)]),
(_.i['content'],
html.highlight_syntax(
path.read_text().strip('\n') if path.exists() else '',
'python')),
(_.i['shell command'],
html.highlight_syntax(self.shell_command(), 'bash')),
('file dependencies',
[_.i[dependency, _.br] for dependency in self.file_dependencies])
]
def html_doc_items(self) -> [(str, str)]:
return [('sqlite file name', _.i[self.sqlite_file_name])] \
+ sql._SQLCommand.html_doc_items(self, None) \
+ [('target_table', _.tt[self.target_table]),
('db alias', _.tt[self.db_alias]),
('time zone', _.tt[self.timezone]),
(_.i['shell command'], html.highlight_syntax(self.shell_command(), 'bash'))]
def html_doc_items(self):
path = self.parent.parent.base_path() / self.file_name
return [('file name', _.i[self.file_name]),
(_.i['content'],
html.highlight_syntax(
path.read_text().strip('\n') if path.exists() else '',
'python'))]
def html_doc_items(self) -> [(str, str)]:
return [('source db', _.tt[self.source_db_alias])] \
+ _SQLCommand.html_doc_items(self, self.source_db_alias) \
+ [('target db', _.tt[self.target_db_alias]),
('target table', _.tt[self.target_table]),
('timezone', _.tt[self.timezone or '']),
(_.i['shell command'], html.highlight_syntax(self.shell_command(), 'bash'))]
def html_doc_items(self):
return [('db', _.tt[self.db_alias]),
('file dependencies', [_.i[dependency, _.br] for dependency in self.file_dependencies])] \
+ _SQLCommand.html_doc_items(self, self.db_alias) \
+ [('echo queries', _.tt[str(self.echo_queries)]),
('timezone', _.tt[self.timezone or '']),
(_.i['shell command'], html.highlight_syntax(self.shell_command(), 'bash'))]
def html_doc_items(self) -> [(str, str)]:
path = self.parent.base_path(
) / self.mapper_script_file_name if self.mapper_script_file_name else ''
return [
('file pattern', _.i[self.file_pattern]),
('compression', _.tt[self.compression]),
('read mode', _.tt[self.read_mode]),
('date regex',
_.tt[escape(self.date_regex)] if self.date_regex else None),
('file dependencies',
[_.i[dependency, _.br] for dependency in self.file_dependencies]),
('mapper script file name', _.i[self.mapper_script_file_name]),
(_.i['mapper script'],
html.highlight_syntax(
path.read_text().strip('\n') if self.mapper_script_file_name
and path.exists() else '', 'python')),
('make unique', _.tt[repr(self.make_unique)]),
('skip header', _.tt[self.skip_header]),
('target_table', _.tt[self.target_table]),
('db alias', _.tt[self.db_alias]),
('partion target table by day_id',
_.tt[self.partition_target_table_by_day_id]),
('sql delimiter char',
_.tt[json.dumps(self.delimiter_char
) if self.delimiter_char != None else None]),
('quote char',
_.tt[json.dumps(self.quote_char
) if self.quote_char != None else None]),
('null value string',
_.tt[json.dumps(self.null_value_string
) if self.null_value_string != None else None]),
('time zone', _.tt[self.timezone])
]
def html_doc_items(self) -> [(str, str)]:
doc = super().html_doc_items() + [
('stream selection',
html.highlight_syntax(json.dumps(self.stream_selection), 'json')
if self.stream_selection else None)
]
return doc
def html_doc_items(self) -> [(str, str)]:
return [('db', _.tt[self.db_alias])] \
+ sql._SQLCommand.html_doc_items(self, self.db_alias) \
+ [('parameter function', html.highlight_syntax(inspect.getsource(self.parameter_function), 'python')),
('parameter placeholders', _.tt[repr(self.parameter_placeholders)]),
('echo queries', _.tt[str(self.echo_queries)]),
('timezone', _.tt[self.timezone or ''])]
def html_doc_items(self) -> [(str, str)]:
return [
('function', _.pre[escape(str(self.function))]),
('args', _.tt[repr(self.args)]),
(_.i['implementation'],
html.highlight_syntax(inspect.getsource(self.function),
'python')),
('file dependencies',
[_.i[dependency, _.br] for dependency in self.file_dependencies])
]
def display_query():
from .query import Query
query = Query.from_dict(flask.request.json)
query_statement = query.to_sql()
if query_statement:
return str(_.tt[html.highlight_syntax(query_statement, language='sql')])
else:
return 'Can not display query statement.'
def query_details(name):
query = ''
for forecast in config.forecasts():
if forecast.metric_name == name:
query = forecast.time_series_query
days = forecast.number_of_days
break
return str(
bootstrap.table(
headers=[],
rows=[
_.tr[_.td[html.highlight_syntax(query,
language='postgresql')]],
_.tr[_.td['Number of days forecasted: ' + str(days)]]
]))
def html_doc_items(self) -> [(str, str)]:
path = self.sql_file_path()
return [('file pattern', _.i[self.file_pattern]),
('read mode', _.tt[self.read_mode]),
('date regex', _.tt[escape(self.date_regex)] if self.date_regex else None),
('file dependencies', [_.i[dependency, _.br] for dependency in self.file_dependencies]),
('query file name', _.i[self.sql_file_name]),
(_.i['query'], html.highlight_syntax(path.read_text().strip('\n')
if self.sql_file_name and path.exists()
else '', 'sql')),
('target_table', _.tt[self.target_table]),
('db alias', _.tt[self.db_alias]),
('partion target table by day_id', _.tt[self.partition_target_table_by_day_id]),
('truncate partitions', _.tt[self.truncate_partitions]),
('time zone', _.tt[self.timezone])]
def data_set_sql_query(id: str, params: [str]) -> response.Response:
from ..config import data_sets
from ..sql_generation import data_set_sql_query
params = set(params.split('/'))
data_set = next(
(data_set for data_set in data_sets() if data_set.id() == id), None)
if not data_set:
return f'Could not find data set "{id}"'
sql = data_set_sql_query(
data_set,
pre_computed_metrics='pre-computed metrics' in params,
human_readable_columns='human readable columns' in params,
personal_data='personal data' in params,
high_cardinality_attributes='high cardinality attributes' in params,
star_schema='star schema' in params)
return str(_.div[html.highlight_syntax(sql, 'sql')])
def html_doc_items(self) -> [(str, str)]:
return [
('command', html.highlight_syntax(self.shell_command(), 'bash'))
]
def view_plots():
return response.Response(
html=[
_.div(id='forecast-container')[_.div(class_="")[[
[
_.div(class_='row')[_.div(class_='col-xl-12')[_.div(
class_='section-header')[forecast.metric_name]]],
_.div(class_='row')[_.div(class_='col-xl-12')[
_.p()['Number of days: ' +
str(forecast.number_of_days),
_.br(), 'Time-series query: ',
html.highlight_syntax(forecast.time_series_query,
language='postgresql')],
bootstrap.card(
header_left='Forecast plot of "' +
forecast.metric_name + '"',
header_right=_.div()[
# config.forecast_table_name
# _.a(class_='query-control', style='margin-right: 10px',
# href='#')[
# _.span(class_='fa fa-download')[' ']],
_.
a(class_='query-control',
href=f"javascript:showQueryDetails('"
f"{flask.url_for('forecasts.query_details', name=forecast.metric_name)}')"
)[_.span(class_='fa fa-eye')[' ']]],
body=[
html.asynchronous_content(
flask.url_for('forecasts.get_plot_image',
name=forecast.metric_name.
lower().replace(' ', '_').
replace('-', '_'),
components='False')),
_.br(),
_.br(),
_.div(class_='modal fade',
id='query-details-dialog',
tabindex="-1"
)[_.div(class_='modal-dialog modal-lg',
role='document')[_.div(
class_='modal-content'
)[_.div(
class_='modal-header'
)[_.h5(class_='modal-title'
)['Time-series query'],
_.button(
**{
'type': "button",
'class': "close",
'data-dismiss': "modal",
'aria-label': "Close"
}
)[_.span(
**{'aria-hidden': 'true'}
)['×']]],
_.div(class_='modal-body',
id='query-details'
)['']]]],
])],
_.
div(class_='col-xl-12')[bootstrap.card(
header_left=
'Forecast components (trend, holidays, seasonality) of "{}"'
.format(forecast.metric_name),
header_right='',
body=[
html.asynchronous_content(
flask.url_for(
'forecasts.get_plot_image',
name=forecast.
metric_name.lower(
).replace(' ', '_').
replace('-', '_'),
components='True')),
])],
_.hr()]
] for forecast in config.forecasts()
]]],
_.script['''''']
],
title='Forecasts',
css_files=[
'https://fonts.googleapis.com/css?family=Open+Sans:300,400,700',
flask.url_for('forecasts.static', filename='forecast.css')
],
js_files=[
flask.url_for('forecasts.static', filename='forecast.js'),
'https://www.gstatic.com/charts/loader.js'
])
