def init_jinja_globals(app):
"""Add extra globals variable to Jinja context"""
server_timezone = conf.get('core', 'default_timezone')
if server_timezone == "system":
server_timezone = pendulum.local_timezone().name
elif server_timezone == "utc":
server_timezone = "UTC"
default_ui_timezone = conf.get('webserver', 'default_ui_timezone')
if default_ui_timezone == "system":
default_ui_timezone = pendulum.local_timezone().name
elif default_ui_timezone == "utc":
default_ui_timezone = "UTC"
if not default_ui_timezone:
default_ui_timezone = server_timezone
expose_hostname = conf.getboolean('webserver', 'EXPOSE_HOSTNAME', fallback=True)
hostname = get_hostname() if expose_hostname else 'redact'
try:
airflow_version = airflow.__version__
except Exception as e:
airflow_version = None
logging.error(e)
git_version = get_airflow_git_version()
def prepare_jinja_globals():
extra_globals = {
'server_timezone': server_timezone,
'default_ui_timezone': default_ui_timezone,
'hostname': hostname,
'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'),
'log_fetch_delay_sec': conf.getint('webserver', 'log_fetch_delay_sec', fallback=2),
'log_auto_tailing_offset': conf.getint('webserver', 'log_auto_tailing_offset', fallback=30),
'log_animation_speed': conf.getint('webserver', 'log_animation_speed', fallback=1000),
'state_color_mapping': STATE_COLORS,
'airflow_version': airflow_version,
'git_version': git_version,
'k8s_or_k8scelery_executor': IS_K8S_OR_K8SCELERY_EXECUTOR,
'rest_api_enabled': False,
}
backends = conf.get('api', 'auth_backends')
if len(backends) > 0 and backends[0] != 'airflow.api.auth.backend.deny_all':
extra_globals['rest_api_enabled'] = True
if 'analytics_tool' in conf.getsection('webserver'):
extra_globals.update(
{
'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'),
'analytics_id': conf.get('webserver', 'ANALYTICS_ID'),
}
)
return extra_globals
app.context_processor(prepare_jinja_globals)
def check_working_days(self, start_time, country_holidays, open_hour):
while start_time in country_holidays:
start_time = start_time.add(days=1)
start_time = pendulum.datetime(start_time.year, start_time.month, start_time.day, open_hour, 0, 0,
tz=pendulum.local_timezone())
while start_time.day_of_week > 4: # 0=Monday, 6=Sunday
start_time = start_time.add(days=1)
start_time = pendulum.datetime(start_time.year, start_time.month, start_time.day, open_hour, 0, 0,
tz=pendulum.local_timezone())
return start_time
def init_jinja_globals(app):
"""Add extra globals variable to Jinja context"""
server_timezone = conf.get('core', 'default_timezone')
if server_timezone == "system":
server_timezone = pendulum.local_timezone().name
elif server_timezone == "utc":
server_timezone = "UTC"
default_ui_timezone = conf.get('webserver', 'default_ui_timezone')
if default_ui_timezone == "system":
default_ui_timezone = pendulum.local_timezone().name
elif default_ui_timezone == "utc":
default_ui_timezone = "UTC"
if not default_ui_timezone:
default_ui_timezone = server_timezone
expose_hostname = conf.getboolean('webserver', 'EXPOSE_HOSTNAME', fallback=True)
hosstname = socket.getfqdn() if expose_hostname else 'redact'
try:
airflow_version = airflow.__version__
except Exception as e: # pylint: disable=broad-except
airflow_version = None
logging.error(e)
git_version = get_airflow_git_version()
def prepare_jinja_globals():
extra_globals = {
'server_timezone': server_timezone,
'default_ui_timezone': default_ui_timezone,
'hostname': hosstname,
'navbar_color': conf.get('webserver', 'NAVBAR_COLOR'),
'log_fetch_delay_sec': conf.getint('webserver', 'log_fetch_delay_sec', fallback=2),
'log_auto_tailing_offset': conf.getint('webserver', 'log_auto_tailing_offset', fallback=30),
'log_animation_speed': conf.getint('webserver', 'log_animation_speed', fallback=1000),
'state_color_mapping': STATE_COLORS,
'airflow_version': airflow_version,
'git_version': git_version
}
if 'analytics_tool' in conf.getsection('webserver'):
extra_globals.update(
{
'analytics_tool': conf.get('webserver', 'ANALYTICS_TOOL'),
'analytics_id': conf.get('webserver', 'ANALYTICS_ID'),
}
)
return extra_globals
app.context_processor(prepare_jinja_globals)
def parse_int(date: int, *, starting_date=pendulum.today()):
"""
Get the period between the starting_date date and the starting_date. Use when
the date is an int.
"""
if date >= starting_date.day:
starting_date = starting_date.set(day=date).set(tz=pendulum.local_timezone())
else:
starting_date = starting_date.set(day=date).set(tz=pendulum.local_timezone())
starting_date = starting_date.add(months=1)
return starting_date
def init_jinja_globals(app):
"""Add extra globals variable to Jinja context"""
server_timezone = conf.get('core', 'default_timezone')
if server_timezone == "system":
server_timezone = pendulum.local_timezone().name
elif server_timezone == "utc":
server_timezone = "UTC"
default_ui_timezone = conf.get('webserver', 'default_ui_timezone')
if default_ui_timezone == "system":
default_ui_timezone = pendulum.local_timezone().name
elif default_ui_timezone == "utc":
default_ui_timezone = "UTC"
if not default_ui_timezone:
default_ui_timezone = server_timezone
expose_hostname = conf.getboolean('webserver',
'EXPOSE_HOSTNAME',
fallback=True)
hosstname = socket.getfqdn() if expose_hostname else 'redact'
def prepare_jinja_globals():
extra_globals = {
'server_timezone':
server_timezone,
'default_ui_timezone':
default_ui_timezone,
'hostname':
hosstname,
'navbar_color':
conf.get('webserver', 'NAVBAR_COLOR'),
'log_fetch_delay_sec':
conf.getint('webserver', 'log_fetch_delay_sec', fallback=2),
'log_auto_tailing_offset':
conf.getint('webserver', 'log_auto_tailing_offset', fallback=30),
'log_animation_speed':
conf.getint('webserver', 'log_animation_speed', fallback=1000),
}
if 'analytics_tool' in conf.getsection('webserver'):
extra_globals.update({
'analytics_tool':
conf.get('webserver', 'ANALYTICS_TOOL'),
'analytics_id':
conf.get('webserver', 'ANALYTICS_ID'),
})
return extra_globals
app.context_processor(prepare_jinja_globals)
def _get_default_timezone(self):
current_timezone = conf.get("core", "default_timezone")
if current_timezone == "system":
default_timezone = pendulum.local_timezone()
else:
default_timezone = pendulum.timezone(current_timezone)
return default_timezone
def _date_query(datestr):
"""Transform date(s) into a query.
"""
def _ts(x): # get timestamp of date or time
try:
return x.timestamp()
except AttributeError:
return datetime.datetime(*x.timetuple()[:6]).timestamp()
dates = datestr.split("..", 1)
_log.debug(f"date '{datestr}', split: {dates}")
try:
parsed_dates = [
pendulum.parser.parse(d, exact=True, tz=pendulum.local_timezone())
if d else None for d in dates
]
except pendulum.exceptions.ParserError as err:
raise ValueError(str(err))
if len(parsed_dates) == 2:
begin_date, end_date = parsed_dates
query = {}
if begin_date:
query['$ge'] = _ts(begin_date)
if end_date:
query['$le'] = _ts(end_date)
else:
pd = parsed_dates[0]
if isinstance(pd, pendulum.Pendulum):
_log.warning("datetime given, must match to the second")
query = _ts(pd)
else:
query = {'$ge': _ts(pd), '$le': _ts(pd) + 60 * 60 * 24} # 1 day
return query
def format_datetime(dt, full=False, seconds=False):
fmt = "L LT"
if seconds:
fmt = "L LTS"
if full:
fmt = "LLLL"
return dt.in_tz(pendulum.local_timezone()).format(
fmt, locale=_current_locale_language)
def format_datetime(dt, full=False, seconds=False):
fmt = "L LT"
if seconds:
fmt = "L LTS"
if full:
fmt = "LLLL"
# The alternative formatter is now the default one since pendulum 2.0.0
return dt.in_tz(pendulum.local_timezone()).format(
fmt, locale=_current_locale_language)
def process_result_value(self, value, dialect):
if value is None:
return None
naive_date = pendulum.parse(str(value))
return datetime(year=naive_date.year,
month=naive_date.month,
day=naive_date.day,
tzinfo=pendulum.local_timezone())
def test_get_default_timezone():
timezone = get_default_timezone()
assert timezone == TIMEZONE_SYSTEM
assert timezone == pendulum.local_timezone()
class Configration(object):
def get_default_timezone(self):
return 'UTC'
conf = Configration()
timezone = get_default_timezone(conf)
assert timezone == TIMEZONE_UTC
def set_default_timezone():
"""设置默认市区 ."""
timezone = pendulum.timezone('UTC')
try:
tz = conf.get("core", "default_timezone")
if tz == "system":
timezone = pendulum.local_timezone()
else:
timezone = pendulum.timezone(tz)
except Exception:
pass
log.info("Configured default timezone %s" % timezone)
set_timezone_var(timezone)
return timezone
def formatTime(self, record, datefmt=None):
"""
Returns the creation time of the specified LogRecord in ISO 8601 date and time format
in the local time zone.
"""
dt = pendulum.from_timestamp(record.created,
tz=pendulum.local_timezone())
if datefmt:
s = dt.strftime(datefmt)
else:
s = dt.strftime(self.default_time_format)
if self.default_msec_format:
s = self.default_msec_format % (s, record.msecs)
if self.default_tz_format:
s += dt.strftime(self.default_tz_format)
return s
def process_formdata(self, valuelist):
if valuelist:
date_str = ' '.join(valuelist)
try:
# Check if the datetime string is in the format without timezone, if so convert it to the
# default timezone
if len(date_str) == 19:
parsed_datetime = dt.strptime(date_str,
'%Y-%m-%d %H:%M:%S')
defualt_timezone = pendulum.timezone('UTC')
tz = conf.get("core", "default_timezone")
if tz == "system":
defualt_timezone = pendulum.local_timezone()
else:
defualt_timezone = pendulum.timezone(tz)
self.data = defualt_timezone.convert(parsed_datetime)
else:
self.data = pendulum.parse(date_str)
except ValueError:
self.data = None
raise ValueError(self.gettext('Not a valid datetime value'))
def calculate(self, start_time, open_hour, close_hour, country_name='US', sla_in_hours=4, province=None,
state=None):
sla_time = None
sla_in_minutes = sla_in_hours * 60
start_time = start_time if isinstance(start_time, pendulum.DateTime) else pendulum.parse(start_time)
country_holidays = pyholidays.CountryHoliday(country_name, prov=province, state=state)
start_time = self.check_working_days(start_time, country_holidays, open_hour)
open_time = pendulum.datetime(start_time.year, start_time.month, start_time.day, open_hour,
0, 0, tz=pendulum.local_timezone())
close_time = pendulum.datetime(start_time.year, start_time.month, start_time.day,
close_hour, 0, 0, tz=pendulum.local_timezone())
if start_time < open_time:
start_time = open_time
elif start_time > close_time:
start_time = open_time.add(days=1)
open_time = pendulum.datetime(start_time.year, start_time.month, start_time.day, open_hour, 0, 0,
tz=pendulum.local_timezone())
close_time = pendulum.datetime(start_time.year, start_time.month, start_time.day, close_hour, 0, 0,
tz=pendulum.local_timezone())
time_left_today = start_time.diff(close_time).in_minutes()
if time_left_today >= sla_in_minutes:
sla_time = start_time.add(minutes=sla_in_minutes)
else:
tomorrow_minutes = sla_in_minutes - time_left_today
start_time = open_time.add(days=1)
while tomorrow_minutes > time_left_today:
start_time = self.check_working_days(start_time, country_holidays, open_hour)
open_time = pendulum.datetime(start_time.year, start_time.month, start_time.day, open_hour, 0, 0,
tz=pendulum.local_timezone())
close_time = pendulum.datetime(start_time.year, start_time.month, start_time.day, close_hour, 0, 0,
tz=pendulum.local_timezone())
time_left_today = start_time.diff(close_time).in_minutes()
if time_left_today >= sla_in_minutes:
sla_time = start_time.add(minutes=tomorrow_minutes)
break
else:
tomorrow_minutes = tomorrow_minutes - time_left_today
start_time = open_time.add(days=1)
if not sla_time:
sla_time = start_time.add(minutes=tomorrow_minutes)
return sla_time
def now_local(self):
return self.now.in_timezone(pendulum.local_timezone())
def create_app(config=None, session=None, testing=False, app_name="Airflow"):
global app, appbuilder
app = Flask(__name__)
if conf.getboolean('webserver', 'ENABLE_PROXY_FIX'):
app.wsgi_app = ProxyFix(app.wsgi_app,
x_for=conf.getint("webserver",
"PROXY_FIX_X_FOR",
fallback=1),
x_proto=conf.getint("webserver",
"PROXY_FIX_X_PROTO",
fallback=1),
x_host=conf.getint("webserver",
"PROXY_FIX_X_HOST",
fallback=1),
x_port=conf.getint("webserver",
"PROXY_FIX_X_PORT",
fallback=1),
x_prefix=conf.getint("webserver",
"PROXY_FIX_X_PREFIX",
fallback=1))
app.secret_key = conf.get('webserver', 'SECRET_KEY')
session_lifetime_days = conf.getint('webserver',
'SESSION_LIFETIME_DAYS',
fallback=30)
app.config['PERMANENT_SESSION_LIFETIME'] = timedelta(
days=session_lifetime_days)
app.config.from_pyfile(settings.WEBSERVER_CONFIG, silent=True)
app.config['APP_NAME'] = app_name
app.config['TESTING'] = testing
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = False
app.config['SESSION_COOKIE_HTTPONLY'] = True
app.config['SESSION_COOKIE_SECURE'] = conf.getboolean(
'webserver', 'COOKIE_SECURE')
app.config['SESSION_COOKIE_SAMESITE'] = conf.get('webserver',
'COOKIE_SAMESITE')
if config:
app.config.from_mapping(config)
# Configure the JSON encoder used by `|tojson` filter from Flask
app.json_encoder = AirflowJsonEncoder
csrf.init_app(app)
db = SQLA(app)
from airflow import api
api.load_auth()
api.API_AUTH.api_auth.init_app(app)
Cache(app=app, config={'CACHE_TYPE': 'filesystem', 'CACHE_DIR': '/tmp'})
from airflow.www.blueprints import routes
app.register_blueprint(routes)
configure_logging()
configure_manifest_files(app)
with app.app_context():
from airflow.www.security import AirflowSecurityManager
security_manager_class = app.config.get('SECURITY_MANAGER_CLASS') or \
AirflowSecurityManager
if not issubclass(security_manager_class, AirflowSecurityManager):
raise Exception(
"""Your CUSTOM_SECURITY_MANAGER must now extend AirflowSecurityManager,
not FAB's security manager.""")
appbuilder = AppBuilder(app,
db.session if not session else session,
security_manager_class=security_manager_class,
base_template='airflow/master.html',
update_perms=conf.getboolean(
'webserver', 'UPDATE_FAB_PERMS'))
def init_views(appbuilder):
from airflow.www import views
# Remove the session from scoped_session registry to avoid
# reusing a session with a disconnected connection
appbuilder.session.remove()
appbuilder.add_view_no_menu(views.Airflow())
appbuilder.add_view_no_menu(views.DagModelView())
appbuilder.add_view(views.DagRunModelView,
"DAG Runs",
category="Browse",
category_icon="fa-globe")
appbuilder.add_view(views.JobModelView, "Jobs", category="Browse")
appbuilder.add_view(views.LogModelView, "Logs", category="Browse")
appbuilder.add_view(views.SlaMissModelView,
"SLA Misses",
category="Browse")
appbuilder.add_view(views.TaskInstanceModelView,
"Task Instances",
category="Browse")
appbuilder.add_view(views.ConfigurationView,
"Configurations",
category="Admin",
category_icon="fa-user")
appbuilder.add_view(views.ConnectionModelView,
"Connections",
category="Admin")
appbuilder.add_view(views.PoolModelView, "Pools", category="Admin")
appbuilder.add_view(views.VariableModelView,
"Variables",
category="Admin")
appbuilder.add_view(views.XComModelView, "XComs", category="Admin")
if "dev" in version.version:
airflow_doc_site = "https://airflow.readthedocs.io/en/latest"
else:
airflow_doc_site = 'https://airflow.apache.org/docs/{}'.format(
version.version)
appbuilder.add_link("Website",
href='https://airflow.apache.org',
category="Docs",
category_icon="fa-globe")
appbuilder.add_link("Documentation",
href=airflow_doc_site,
category="Docs",
category_icon="fa-cube")
appbuilder.add_link("GitHub",
href='https://github.com/apache/airflow',
category="Docs")
appbuilder.add_view(views.VersionView,
'Version',
category='About',
category_icon='fa-th')
def integrate_plugins():
"""Integrate plugins to the context"""
from airflow import plugins_manager
plugins_manager.initialize_web_ui_plugins()
for v in plugins_manager.flask_appbuilder_views:
log.debug("Adding view %s", v["name"])
appbuilder.add_view(v["view"],
v["name"],
category=v["category"])
for ml in sorted(plugins_manager.flask_appbuilder_menu_links,
key=lambda x: x["name"]):
log.debug("Adding menu link %s", ml["name"])
appbuilder.add_link(ml["name"],
href=ml["href"],
category=ml["category"],
category_icon=ml["category_icon"])
integrate_plugins()
# Garbage collect old permissions/views after they have been modified.
# Otherwise, when the name of a view or menu is changed, the framework
# will add the new Views and Menus names to the backend, but will not
# delete the old ones.
def init_plugin_blueprints(app):
from airflow.plugins_manager import flask_blueprints
for bp in flask_blueprints:
log.debug("Adding blueprint %s:%s", bp["name"],
bp["blueprint"].import_name)
app.register_blueprint(bp["blueprint"])
def init_error_handlers(app: Flask):
from airflow.www import views
app.register_error_handler(500, views.show_traceback)
app.register_error_handler(404, views.circles)
init_views(appbuilder)
init_plugin_blueprints(app)
init_error_handlers(app)
if conf.getboolean('webserver', 'UPDATE_FAB_PERMS'):
security_manager = appbuilder.sm
security_manager.sync_roles()
from airflow.www.api.experimental import endpoints as e
# required for testing purposes otherwise the module retains
# a link to the default_auth
if app.config['TESTING']:
import importlib
importlib.reload(e)
app.register_blueprint(e.api_experimental,
url_prefix='/api/experimental')
server_timezone = conf.get('core', 'default_timezone')
if server_timezone == "system":
server_timezone = pendulum.local_timezone().name
elif server_timezone == "utc":
server_timezone = "UTC"
default_ui_timezone = conf.get('webserver', 'default_ui_timezone')
if default_ui_timezone == "system":
default_ui_timezone = pendulum.local_timezone().name
elif default_ui_timezone == "utc":
default_ui_timezone = "UTC"
if not default_ui_timezone:
default_ui_timezone = server_timezone
@app.context_processor
def jinja_globals(): # pylint: disable=unused-variable
globals = {
'server_timezone':
server_timezone,
'default_ui_timezone':
default_ui_timezone,
'hostname':
socket.getfqdn() if conf.getboolean(
'webserver', 'EXPOSE_HOSTNAME', fallback=True) else
'redact',
'navbar_color':
conf.get('webserver', 'NAVBAR_COLOR'),
'log_fetch_delay_sec':
conf.getint('webserver', 'log_fetch_delay_sec', fallback=2),
'log_auto_tailing_offset':
conf.getint('webserver',
'log_auto_tailing_offset',
fallback=30),
'log_animation_speed':
conf.getint('webserver', 'log_animation_speed', fallback=1000)
}
if 'analytics_tool' in conf.getsection('webserver'):
globals.update({
'analytics_tool':
conf.get('webserver', 'ANALYTICS_TOOL'),
'analytics_id':
conf.get('webserver', 'ANALYTICS_ID')
})
return globals
@app.before_request
def before_request():
_force_log_out_after = conf.getint('webserver',
'FORCE_LOG_OUT_AFTER',
fallback=0)
if _force_log_out_after > 0:
flask.session.permanent = True
app.permanent_session_lifetime = datetime.timedelta(
minutes=_force_log_out_after)
flask.session.modified = True
flask.g.user = flask_login.current_user
@app.after_request
def apply_caching(response):
_x_frame_enabled = conf.getboolean('webserver',
'X_FRAME_ENABLED',
fallback=True)
if not _x_frame_enabled:
response.headers["X-Frame-Options"] = "DENY"
return response
@app.teardown_appcontext
def shutdown_session(exception=None): # pylint: disable=unused-variable
settings.Session.remove()
@app.before_request
def make_session_permanent():
flask_session.permanent = True
return app, appbuilder
from sqlalchemy import create_engine, exc
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.pool import NullPool
from airflow.configuration import conf, AIRFLOW_HOME, WEBSERVER_CONFIG # NOQA F401
from airflow.logging_config import configure_logging
from airflow.utils.sqlalchemy import setup_event_handlers
log = logging.getLogger(__name__)
TIMEZONE = pendulum.timezone('UTC')
try:
tz = conf.get("core", "default_timezone")
if tz == "system":
TIMEZONE = pendulum.local_timezone()
else:
TIMEZONE = pendulum.timezone(tz)
except Exception:
pass
log.info("Configured default timezone %s" % TIMEZONE)
HEADER = '\n'.join([
r' ____________ _____________',
r' ____ |__( )_________ __/__ /________ __',
r'____ /| |_ /__ ___/_ /_ __ /_ __ \_ | /| / /',
r'___ ___ | / _ / _ __/ _ / / /_/ /_ |/ |/ /',
r' _/_/ |_/_/ /_/ /_/ /_/ \____/____/|__/',
])
# -*- coding: utf-8 -*-
#
import datetime as dt
import pendulum
# UTC time zone as a tzinfo instance.
TIMEZONE_UTC = pendulum.timezone('UTC')
TIMEZONE_SYSTEM = pendulum.local_timezone()
def get_default_timezone(conf=None):
"""获得默认时区 ."""
if conf:
tz = conf.get_default_timezone()
else:
tz = 'system'
if tz == "system":
timezone = TIMEZONE_SYSTEM
else:
timezone = pendulum.timezone(tz)
return timezone
def is_localized(value):
"""存在时区信息
Determine if a given datetime.datetime is aware.
The concept is defined in Python's docs:
http://docs.python.org/library/datetime.html#datetime.tzinfo
Assuming value.tzinfo is either None or a proper datetime.tzinfo,
value.utcoffset() implements the appropriate logic.
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
#
import datetime as dt
import pendulum
from airflow.settings import TIMEZONE
# UTC time zone as a tzinfo instance.
# utc = pendulum.timezone('UTC')
from airflow import configuration as conf
try:
tz = conf.get("core", "default_timezone")
if tz == "system":
utc = pendulum.local_timezone()
else:
utc = pendulum.timezone(tz)
except Exception:
pass
def is_localized(value):
"""
Determine if a given datetime.datetime is aware.
The concept is defined in Python's docs:
http://docs.python.org/library/datetime.html#datetime.tzinfo
Assuming value.tzinfo is either None or a proper datetime.tzinfo,
value.utcoffset() implements the appropriate logic.
"""
return value.utcoffset() is not None
def now_local(self) -> pendulum.DateTime:
return self.now.in_timezone(pendulum.local_timezone())
def is_local(self):
return self.offset == self.in_timezone(
pendulum.local_timezone()).offset
def is_local(self):
return self.offset == self.in_timezone(pendulum.local_timezone()).offset
def time_period(
dt_string,
*,
in_=False,
on=False,
before=False,
after=False
):
match = DATETIME_RE.match(dt_string)
if not match or match['year'] is None:
raise argparse.ArgumentTypeError(f"'{dt_string}' is not a supported datetime string.")
parsed = ParsedDateTime(**match.groupdict())
if parsed.tz_hour:
tz_offset = 0
if parsed.tz_hour is not None:
tz_offset += parsed.tz_hour * 3600
if parsed.tz_minute is not None:
tz_offset += parsed.tz_minute * 60
if parsed.tz_oper == '-':
tz_offset *= -1
parsed_tz = fixed_timezone(tz_offset)
else:
parsed_tz = pendulum.local_timezone()
if in_:
if parsed.day:
raise argparse.ArgumentTypeError(
f"Datetime string must contain only year or year/month for 'in' option."
)
start = pendulum.datetime(
parsed.year,
parsed.month or 1,
parsed.day or 1,
tz=parsed_tz
)
if parsed.month:
end = start.end_of('month')
else:
end = start.end_of('year')
return pendulum.period(
start,
end
)
elif on:
if (
not all(
getattr(parsed, attr)
for attr in ['year', 'month', 'day']
)
or parsed.hour
):
raise argparse.ArgumentTypeError(
f"Datetime string must contain only year, month, and day for 'on' option."
)
dt = pendulum.datetime(
parsed.year,
parsed.month,
parsed.day,
tz=parsed_tz
)
return pendulum.period(
dt.start_of('day'),
dt.end_of('day')
)
elif before:
start = DateTime.min
dt = pendulum.datetime(
parsed.year,
parsed.month or 1,
parsed.day or 1,
parsed.hour or 23,
parsed.minute or 59,
parsed.second or 59,
0,
tz=parsed_tz
)
if not parsed.month:
dt = dt.start_of('year')
elif not parsed.day:
dt = dt.start_of('month')
elif not parsed.hour:
dt = dt.start_of('day')
elif not parsed.minute:
dt = dt.start_of('hour')
elif not parsed.second:
dt = dt.start_of('minute')
return pendulum.period(
start,
dt
)
elif after:
end = DateTime.max
dt = pendulum.datetime(
parsed.year,
parsed.month or 1,
parsed.day or 1,
parsed.hour or 23,
parsed.minute or 59,
parsed.second or 59,
99999,
tz=parsed_tz
)
if not parsed.month:
dt = dt.end_of('year')
elif not parsed.day:
dt = dt.end_of('month')
elif not parsed.hour:
dt = dt.start_of('day')
elif not parsed.minute:
dt = dt.start_of('hour')
elif not parsed.second:
dt = dt.start_of('minute')
return pendulum.period(
dt,
end
)
def test_parse_int_date_if_greater_than_target_date():
"""If date is greater than or equal to the current date, it should return the same day"""
dt = pendulum.datetime(2020, 1, 15, tz=pendulum.local_timezone())
starting_date = pendulum.datetime(2020, 1, 16)
assert whenzat.parse_int(dt.day,
starting_date=starting_date) == dt.add(months=1)
def get_timezone(self):
return self.get("timezone", pn.local_timezone().name)
def now_local(self):
return self.now.in_timezone(pendulum.local_timezone())
def run(feed_url):
"""
Pull a podcast rss feed from a given url and print a table
with all episodes sorted by time between episodes
:raises ValueError:
If the feed does not have all of the required data
"""
console = Console(width=150)
# get the feed and parse it
parsed_feed = []
for url in feed_url:
parsed_feed.extend(get_parsed_feed(url))
# Draw weekday distribution
heatmap = Table(title="Weekday Heatmap")
heat = {
0:
Style(bgcolor=Color.from_rgb(red=153, green=0, blue=13)),
1:
Style(bgcolor=Color.from_rgb(red=203, green=24, blue=29)),
2:
Style(bgcolor=Color.from_rgb(red=239, green=59, blue=44)),
3:
Style(bgcolor=Color.from_rgb(red=251, green=106, blue=74),
color="black"),
4:
Style(bgcolor=Color.from_rgb(red=252, green=146, blue=114),
color="black"),
5:
Style(bgcolor=Color.from_rgb(red=252, green=187, blue=161),
color="black"),
6:
Style(bgcolor=Color.from_rgb(red=254, green=229, blue=217),
color="black"),
}
count = weekday_distribution(parsed_feed)
result = {}
for i, item in enumerate(count.most_common()):
result[item[0]] = {"value": str(item[1]), "style": heat[i]}
row = []
for d in ["Sun", "Mon", "Tue", "Wed", "Thu", "Fri", "Sat"]:
day = result.get(d, {"value": "0", "style": ""})
heatmap.add_column(d, style=day["style"], justify="center")
row.append(day["value"])
heatmap.add_row(*row)
console.print(heatmap)
table = Table(
"Podcast",
"Title",
"Date published",
"Days since last",
title="Episodes",
)
for episode in sorted(parsed_feed,
key=lambda x: x["published_datetime"],
reverse=True):
table.add_row(
episode["podcast"],
" ".join(episode["title"].split()),
episode["published_datetime"].in_timezone(
pendulum.local_timezone()).to_day_datetime_string(),
str(episode["time_since_last"].days),
)
console.print(table)
def test_now_should_return_now_with_local_timezone(env, cfg):
expected = pn.now()
with patch("pendulum.now", return_value=expected) as patched:
result = cfg.now()
assert result == expected
patched.assert_called_once_with(tz=pn.local_timezone().name)
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.pool import NullPool
from sqlalchemy.orm.session import Session as SASession
import airflow
from airflow.configuration import conf, AIRFLOW_HOME, WEBSERVER_CONFIG # NOQA F401
from airflow.logging_config import configure_logging
from airflow.utils.sqlalchemy import setup_event_handlers
log = logging.getLogger(__name__)
TIMEZONE = pendulum.timezone('UTC')
try:
tz = conf.get("core", "default_timezone")
if tz == "system":
TIMEZONE = pendulum.local_timezone()
else:
TIMEZONE = pendulum.timezone(tz)
except Exception:
pass
log.info("Configured default timezone %s" % TIMEZONE)
HEADER = '\n'.join([
r' ____________ _____________',
r' ____ |__( )_________ __/__ /________ __',
r'____ /| |_ /__ ___/_ /_ __ /_ __ \_ | /| / /',
r'___ ___ | / _ / _ __/ _ / / /_/ /_ |/ |/ /',
r' _/_/ |_/_/ /_/ /_/ /_/ \____/____/|__/',
])
LOGGING_LEVEL = logging.INFO
def datetime_string_to_time_period(dt_string,
*,
in_=False,
on=False,
before=False,
after=False):
if not any([in_, on, before, after]):
raise ValueError("One of in_, on, before, or after must be ``True``.")
if dt_string == 'today':
dt_string = pendulum.today().to_date_string()
elif dt_string == 'yesterday':
dt_string = pendulum.yesterday().to_date_string()
elif dt_string == 'now': # pragma: nocover
dt_string = pendulum.now().end_of('minute').to_datetime_string()
match = DATETIME_RE.match(dt_string)
if (not match or match['year'] is None):
raise ValueError(f"'{dt_string}' is not a supported datetime string.")
parsed = ParsedDateTime(**match.groupdict())
if parsed.tz_hour:
tz_offset = 0
if parsed.tz_hour is not None: # pragma: nobranch
tz_offset += parsed.tz_hour * 3600
if parsed.tz_minute is not None: # pragma: nobranch
tz_offset += parsed.tz_minute * 60
if parsed.tz_oper == '-': # pragma: nobranch
tz_offset *= -1
parsed_tz = fixed_timezone(tz_offset)
else:
parsed_tz = pendulum.local_timezone()
if in_:
if parsed.day:
raise ValueError(
f"Datetime string must contain only year or year/month for 'in' option."
)
start = pendulum.datetime(parsed.year,
parsed.month or 1,
parsed.day or 1,
tz=parsed_tz)
if parsed.month:
end = start.end_of('month')
else:
end = start.end_of('year')
period = pendulum.period(start, end)
elif on:
if (not all(
getattr(parsed, attr) for attr in ['year', 'month', 'day'])
or parsed.hour):
raise ValueError(
f"Datetime string must contain only year, month, and day for 'on' option."
)
dt = pendulum.datetime(parsed.year,
parsed.month,
parsed.day,
tz=parsed_tz)
period = pendulum.period(dt.start_of('day'), dt.end_of('day'))
elif before:
start = DateTime.min
dt = pendulum.datetime(
parsed.year,
parsed.month if parsed.month is not None else 1,
parsed.day if parsed.day is not None else 1,
parsed.hour if parsed.hour is not None else 23,
parsed.minute if parsed.minute is not None else 59,
parsed.second if parsed.second is not None else 59,
0,
tz=parsed_tz)
if parsed.month is None:
dt = dt.start_of('year')
elif parsed.day is None:
dt = dt.start_of('month')
elif parsed.hour is None:
dt = dt.start_of('day')
elif parsed.minute is None:
dt = dt.start_of('hour')
elif parsed.second is None:
dt = dt.start_of('minute')
period = pendulum.period(start, dt)
else:
end = DateTime.max
dt = pendulum.datetime(
parsed.year,
parsed.month if parsed.month is not None else 1,
parsed.day if parsed.day is not None else 1,
parsed.hour if parsed.hour is not None else 23,
parsed.minute if parsed.minute is not None else 59,
parsed.second if parsed.second is not None else 59,
0,
tz=parsed_tz)
if parsed.month is None:
dt = dt.end_of('year')
elif parsed.day is None:
dt = dt.end_of('month')
elif parsed.hour is None:
dt = dt.end_of('day')
elif parsed.minute is None:
dt = dt.start_of('hour')
elif parsed.second is None:
dt = dt.start_of('minute')
period = pendulum.period(dt, end)
return period