def add_arguments(self, parser):
parser.add_argument('--commit', action='store_true', help='Should the script actually commit?')
parser.add_argument('--dry', action='store_true', help='Should the script actually make changes? (In a transaction)')
parser.add_argument('--osf-only', action='store_true', help='Should the script limit to works from OSF sources?')
parser.add_argument('--limit', type=int, default=self.LIMIT, help='Maximum number of works to fix')
parser.add_argument('--from', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider works modified on or after this date')
parser.add_argument('--until', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider works modified on or before this date')
def datetime(value):
"""Convert a datetime string to a Pendulum instance."""
value = value.decode("utf-8")
try:
utc_date = pendulum.from_format(value, "YYYY-MM-DD HH:mm:ss")
except ValueError:
last_colon_index = value.rindex(":")
date = value[:last_colon_index] + value[last_colon_index + 1:]
utc_date = pendulum.from_format(date, "YYYY-MM-DD HH:mm:ssZZ")
return utc_date
def decode(self, s):
"""
Return the serialization as a datetime object. If the serializaton ends with 'A', first converting to localtime and returning an aware datetime object in the local timezone. If the serialization ends with 'N', returning without conversion as an aware datetime object in the local timezone.
>>> dts = PendulumDateTimeSerializer()
>>> dts.decode('20180725T1027N')
DateTime(2018, 7, 25, 10, 27, 0, tzinfo=Timezone('America/New_York'))
>>> dts.decode('20180725T1427A')
DateTime(2018, 7, 25, 10, 27, 0, tzinfo=Timezone('America/New_York'))
"""
if s[-1] == 'A':
return pendulum.from_format(s[:-1], 'YYYYMMDDTHHmm', 'UTC').in_timezone('local')
else:
return pendulum.from_format(s[:-1], 'YYYYMMDDTHHmm').naive().in_timezone('local')
def pen_from_fmt(s, z='local'):
"""
>>> pen_from_fmt("20120622T0000")
Date(2012, 6, 22)
"""
if not isinstance(s, str):
return pendulum.instance(s)
if len(s) == 8:
dt = pendulum.from_format(s, "YYYYMMDD", z)
return dt.date()
else:
dt = pendulum.from_format(s, "YYYYMMDDTHHmm", z)
if z in ['local', 'Factory'] and dt.hour == dt.minute == 0:
dt = dt.date()
return dt
def test_from_format_with_timezone():
d = pendulum.from_format(
"1975-05-21 22:32:11",
"YYYY-MM-DD HH:mm:ss",
tz=pendulum.timezone("Europe/London"),
)
assert_datetime(d, 1975, 5, 21, 22, 32, 11)
assert "Europe/London" == d.timezone_name
def date_with_hour(value):
"""Convert a date-and-hour string to a Pendulum instance."""
value = value.decode("utf-8")
try:
utc_date = pendulum.from_format(value, "YYYY-MM-DD-HH")
except ValueError:
utc_date = None
return utc_date
def calldate_to_utc(value):
"""Convert a local datetime string to a UTC Pendulum instance."""
local_tz = cherrypy.engine.publish(
"registry:local_timezone"
).pop()
return pendulum.from_format(
value.decode("utf-8"),
"YYYY-MM-DD HH:mm:ss",
tz=local_tz,
).in_timezone('utc')
def fetch_records(self, url: furl) -> Iterator[Tuple[str, Union[str, dict, bytes], pendulum.Pendulum]]:
while True:
logger.info('Fetching %s', url.url)
records = self.requests.get(url.url).json()['response'].get('award', [])
for record in records:
yield (record['id'], record, pendulum.from_format(record['date'], '%m/%d/%Y'))
if len(records) < PAGE_SIZE:
break
url.args['offset'] += PAGE_SIZE
def message_process(self, msgline, task, measurement):
_ = task
raw_time_str, status_str, frequency_str = msgline[:19], msgline[26:31], msgline[36:]
timestamp = pendulum.from_format(raw_time_str, '%Y_%m_%d_%H_%M_%S', 'Asia/Shanghai').timestamp
status = self.get_status(status_str)
fields = Outputer.get_field(frequency_str)
fields['status'] = status
tags = {'eqpt_no': self.eqpt_no, 'nodename': self.nodename}
influx_json = OrderedDict({'tags': tags, 'fields': fields,
'time': 1000000 * int(timestamp) + self.seq % 1000,
'measurement': measurement, })
return influx_json
def test_from_format(text, fmt, expected, now):
if now is None:
now = pendulum.datetime(2015, 11, 12)
else:
now = pendulum.parse(now)
# Python 2.7 loses precision for x timestamps
# so we don't test
if fmt == "x" and PY2:
return
with pendulum.test(now):
assert pendulum.from_format(text, fmt).isoformat() == expected
def parse_bom_gov(date_attribute, timezone):
"""
Input:
'd2020-04-12'
Output:
Pendulum datetime object (local timezone)
Context:
<div class="forecast-day collapsible" id="d2020-04-12">
"""
date = pendulum.from_format(date_attribute[1:], "YYYY-MM-DD", tz=timezone)
return date
def __init__(self):
"""
Initialize
:return:
"""
self.anav_base = get_crnt_aeronav()
self.parsed_tpp = self._parse_current_meta()
self.apt_dict = self._to_dict()
self.state_dict = self._state_dict() # Airports by state
self.front_matter_uri = self.anav_base + 'FRNTMATTER.PDF'
# Get cycle data
input_format = 'HHmm z MM/DD/YY'
start_raw = self.parsed_tpp.digital_tpp['from_edate']
end_raw = self.parsed_tpp.digital_tpp['to_edate']
self.cycle = self.parsed_tpp.digital_tpp['cycle']
self.cycle_start = pendulum.from_format(
start_raw[:4] + ' GMT ' + start_raw[7:],
input_format).to_iso8601_string()
self.cycle_end = pendulum.from_format(
end_raw[:4] + ' GMT ' + end_raw[7:],
input_format).to_iso8601_string()
def store(self):
league = League.find(request().input('league_id'))
date = pendulum.from_format(request().input('scheduled_time'),
'%m/%d/%Y')
Schedule.create(
league_id=league.id,
team1_id=request().input('team1'),
team2_id=request().input('team2'),
scheduled_for=date,
)
return request().redirect('/league/{0}/schedule'.format(league.id))
def one(pre_time_str, log_str):
str_list = log_str.split()
code = str_list[0]
message = ' '.join(str_list[1:-3])
post_time_str = ' '.join(str_list[-3:])
fields = {'code': code, 'message': message}
if version_info[0] == 3:
time_str = pre_time_str + post_time_str.split(maxsplit=1)[-1]
else:
time_str = pre_time_str + post_time_str.split(None, 1)[-1]
timestamp = pendulum.from_format(time_str,
'[%Y/%m/%d]%I:%M:%S %p>').timestamp
return fields, timestamp
def structured_collection(collection) -> Tuple[dict, set]:
structured = defaultdict(lambda: defaultdict(list))
unknowns = set()
for item in collection:
match = re.match(r'.*(((19|20)\d{6})_\d{6}).*', item)
if not match:
unknowns.add(item)
continue
clock = pendulum.from_format(match.group(2), 'YYYYMMDD')
year, written_month = clock.format('YYYY MMMM').split()
structured[year][written_month].append(item)
return (structured, unknowns)
def _process_data(self):
# group items by the day of month
groups = groupby(self.dates, lambda d: d.format('dd DD/MM/YY'))
groups = {key: list(value) for key, value in groups}
# sort group keys
keys = groups.keys()
date_keys = [pendulum.from_format(key, 'dd DD/MM/YY') for key in keys]
date_keys = sorted(date_keys)
sorted_keys = [dk.format('dd DD/MM/YY') for dk in date_keys]
self.days = sorted_keys
self.data = groups
def test_from_format(text, fmt, expected, now):
if now is None:
now = pendulum.datetime(2015, 11, 12)
else:
now = pendulum.parse(now)
# Python 2.7 loses precision for x timestamps
# so we don't test
if fmt == "x" and PY2:
return
with pendulum.test(now):
assert pendulum.from_format(text, fmt).isoformat() == expected
def test_etl_states(session, task_instance):
ti = task_instance(AIRFLOW_RAW / "2020_04_01_00_00_00-v2.tsv")
file_config = ti.xcom_pull("config", "init")
file_name = file_config["file_name"]
pulltime = file_config["pulltime"]
assert ETL.can_process("pull_file", file_name, pulltime, session)
ETL.commit_new("pull_file", file_name, pulltime, session)
assert ETL.can_process("pull_file", file_name, pulltime, session) == False
q = ETL.get_most_recent("pull_file", file_name, pulltime, session)
assert q.task_type == "pull_file"
assert q.task_name == file_name
assert pendulum.instance(q.task_timestamp) == pendulum.from_format(
pulltime, "YYYY-MM-DD[T]HH:mm:ss")
assert q.status == ETLStatus.ongoing
ETL.set_status("pull_file", file_name, pulltime, "quarantine", session)
assert ETL.can_process("pull_file", file_name, pulltime, session)
q = ETL.get_most_recent("pull_file", file_name, pulltime, session)
assert q.task_type == "pull_file"
assert q.task_name == file_name
assert pendulum.instance(q.task_timestamp) == pendulum.from_format(
pulltime, "YYYY-MM-DD[T]HH:mm:ss")
assert q.status == ETLStatus.quarantine
ETL.set_status("pull_file", file_name, pulltime, "completed", session)
assert ETL.can_process("pull_file", file_name, pulltime, session) == False
q = ETL.get_most_recent("pull_file", file_name, pulltime, session)
assert q.task_type == "pull_file"
assert q.task_name == file_name
assert pendulum.instance(q.task_timestamp) == pendulum.from_format(
pulltime, "YYYY-MM-DD[T]HH:mm:ss")
assert q.status == ETLStatus.completed
with pytest.raises(Exception, match="Once a task is completed"):
ETL.commit_new("pull_file", file_name, pulltime, session)
ETL.set_status("quarantine", file_name, pulltime, "completed", session)
def get(cur, base):
log.debug('checking feeds for %s/%s at %s' % (cur, base, NAME))
r = requests.get(
'https://bittrex.com/api/v1.1/public/getmarketsummary?market={}-{}'.format(base, from_bts(cur)),
timeout=TIMEOUT).json()
summary = r['result'][0]
# log.debug('Got feed price for {}: {} (from bittrex)'.format(cur, summary['Last']))
return FeedPrice(summary['Last'],
cur, base,
volume=summary['Volume'],
last_updated=pendulum.from_format(summary['TimeStamp'].split('.')[0], '%Y-%m-%dT%H:%M:%S'),
provider=NAME)
def _get_date(msg):
for format in [
"ddd, DD MMM YYYY HH:mm:ss ZZ",
"DD MMM YYYY HH:mm:ss ZZ",
"ddd, DD MMM YYYY HH:mm:ss zz",
"DD MMM YYYY HH:mm:ss zz",
]:
try:
dt = pendulum.from_format(msg["Date"], format)
return _pendulum_to_dict(dt)
except:
pass
raise ValueError(f"Unrecognized date format: {msg['Date']}")
def datetime(self) -> TSDatetime:
"""Use parsing result to build TSDatetime object"""
if self.parsed_datetime:
return self.parsed_datetime
if not self.datetime_format or not self.datetime_stamp:
return None
datetime_ = pendulum.from_format(self.datetime_stamp,
self.datetime_format,
tz=None)
ts_datetime = TSDatetime(datetime_=datetime_,
subseconds=self.fractional_seconds)
return ts_datetime
def getWeeklyLogs(self, day):
dt = pendulum.from_format(day, 'YYYY-MM-DD')
#today = pendulum.now()
start = dt.start_of('week')
end = dt.end_of('week')
#date_generated = [start + datetime.timedelta(days=x) for x in range(0, (end-start).days)]
date_generated = [
start + datetime.timedelta(days=x) for x in range(0, 7)
]
self.weekly_logs = []
for date in date_generated:
#duration = self.getDailyLogs(date.strftime("%Y-%m-%d"))
duration = self.getDailyLogs(date.strftime("%Y-%m-%d"))
self.weekly_logs.append(str(duration.total_seconds()))
def read_daily_profile_todict(daily_profile_fn, separator):
outdict = {}
header = []
firstline = True
with open(daily_profile_fn, 'r') as csv_file:
file_reader = csv.reader(csv_file, delimiter=separator)
for row in file_reader:
if firstline:
firstline = False
header = [row[0], row[1]]
continue
outdict[from_format(row[0], TIME_FORMAT)] = float(row[1])
return header, outdict
def query(self):
result = self.influx.query(
'select * from Mafu_measurement order by desc limit 10;')
check = list(result.get_points())[0]['check']
time_str = list(result.get_points())[0]['time']
dt = pendulum.from_format(time_str, '%Y-%m-%dT%H:%M:%SZ')
diff_sec = dt.diff().seconds
if diff_sec > TIME_WINDOW * 60 and check is None:
return 'put'
else:
return 'pass'
def _str_to_datetime(time_str, time_format) -> DateTime:
"""
Converts time_str into a pendulum (DateTime) object that either takes the global start date or
the provided one, dependant on the time_format
:return: DateTime
"""
time = from_format(time_str, time_format, tz=TIME_ZONE)
if time_format == DATE_TIME_FORMAT:
return time
elif time_format == TIME_FORMAT:
return GlobalConfig.start_date.add(
hours=time.hour, minutes=time.minute, seconds=time.second)
else:
raise ValueError("Provided time_format invalid.")
def test_set_user_details(user_dynamo):
user_id = 'my-user-id'
username = '******'
birthday = pendulum.from_format('1900-01-01',
'YYYY-MM-DD').to_iso8601_string()
user_dynamo.add_user('other-id-1', 'noise-1', 'cog-noise-1')
expected_base_item = user_dynamo.add_user(user_id, username)
assert expected_base_item['userId'] == user_id
user_dynamo.add_user('other-id-2', 'noise-2', 'cog-noise-2')
resp = user_dynamo.set_user_details(user_id, full_name='fn')
assert resp == {**expected_base_item, **{'fullName': 'fn'}}
resp = user_dynamo.set_user_details(
user_id,
full_name='f',
bio='b',
language_code='l',
theme_code='tc',
follow_counts_hidden=True,
view_counts_hidden=True,
email='e',
phone='p',
comments_disabled=True,
likes_disabled=True,
sharing_disabled=True,
verification_hidden=True,
birthday=birthday,
)
expected = {
**expected_base_item,
**{
'fullName': 'f',
'bio': 'b',
'languageCode': 'l',
'themeCode': 'tc',
'followCountsHidden': True,
'viewCountsHidden': True,
'email': 'e',
'phoneNumber': 'p',
'commentsDisabled': True,
'likesDisabled': True,
'sharingDisabled': True,
'verificationHidden': True,
'birthday': birthday,
},
}
assert resp == expected
def test_consolidate(ingest, clean_etl):
date = pendulum.from_format("2020_03_27", "YYYY_MM_DD").naive()
for file_stem in ["2020_04_01_00_00_00-v2", "2020_03_27_00_00_00-v2"]:
file_path = Path(f"tmp/raw/{file_stem}_2020_03_27.csv")
ingest(file_path)
table = Fact.child_or_load_table(date)
task_instance = TaskInstanceMock("init")
task_instance.xcom_push("config", {
"date": str(date),
"table_name": table.fullname
})
clean_etl("consolidation", table.fullname, date)
consolidate_callable(ti=task_instance)
with engine.begin() as conn:
count = conn.execute(table.select()).rowcount
assert count == 198
count = conn.execute(
table.select(table.c.session_end == None)).rowcount
assert count == 0
count = conn.execute(table.select(table.c.pulltime_last)).rowcount
assert count == 101
count = conn.execute(
table.select(
and_(table.c.pulltime_last,
table.c.session_end != table.c.pulltime))).rowcount
assert count == 1
count1 = conn.execute(table.select(table.c.pulltime_last)).rowcount
count2 = conn.execute(
select(
[
table.c.userid_key,
table.c.mac_key,
table.c.ap_key,
table.c.ssid_key,
table.c.protocol_key,
table.c.session_start,
],
distinct=True,
)).rowcount
assert count1 == count2
def timestamp_to_epoch(timestamp: str) -> int:
"""Takes a `timestamp` str and attempts to convert it to an epoch
value using the list of known time stamp formats.
:param timestamp: str
:return: int
"""
for ts_fmt in KNOWN_TIMESTAMP_FORMATS:
try:
return pendulum.from_format(timestamp, ts_fmt).int_timestamp
except ValueError:
pass
return None
def is_date(text: str, tz: str = "local") -> Tuple[bool, str]:
"""
Check if text is a valid date. Format:
format: "DD-MM-YYYY HH-mm-ss",
"DD-MM-YYYY HH-mm",
"DD-MM-YYYY HH",
"DD-MM-YYYY"
:param text: str: Text to
:param tz: str: (Default value = "local")
"""
text = normalise_date(text)
time_format_list = [
"DD-MM-YYYY HH-mm-ss", "DD-MM-YYYY HH-mm", "DD-MM-YYYY HH",
"DD-MM-YYYY"
]
for fmt in time_format_list:
try:
from_format(text, fmt, tz=tz)
return True, fmt
except Exception:
...
return False, ""
def e_entrada(self) -> bool:
try:
if not self.data:
return True
if self.registros_ES[-1]['saida'] != '':
return True
dif_dias = pendulum.period(pendulum.from_format(self.data, date_format), pendulum.now()).in_days()
if dif_dias >= 0 and self.registros_ES[-1]["saida"] != "":
return True
return False
except:
# TODO EXCEPT
raise Exception
def transform_function(original_value: Any,
field_schema: Dict[str, Any]) -> Any:
if original_value and "format" in field_schema and field_schema[
"format"] == "date":
date_format = self.MARKETPLACE_DATE_FORMAT_MAP.get(
self.marketplace_id)
if not date_format:
raise KeyError(
f"Date format not found for Markeplace ID: {self.marketplace_id}"
)
transformed_value = pendulum.from_format(
original_value, date_format).to_date_string()
return transformed_value
return original_value
def add_arguments(self, parser):
parser.add_argument('--commit',
action='store_true',
help='Should the script actually commit?')
parser.add_argument(
'--dry',
action='store_true',
help='Should the script actually make changes? (In a transaction)')
parser.add_argument(
'--osf-only',
action='store_true',
help='Should the script limit to works from OSF sources?')
parser.add_argument('--limit',
type=int,
default=self.LIMIT,
help='Maximum number of works to fix')
parser.add_argument(
'--from',
type=lambda d: pendulum.from_format(d, '%Y-%m-%d'),
help='Only consider works modified on or after this date')
parser.add_argument(
'--until',
type=lambda d: pendulum.from_format(d, '%Y-%m-%d'),
help='Only consider works modified on or before this date')
def identical_profiles(context):
raw_results = context.simulation.endpoint_buffer.results_handler.all_raw_results
device_stats_dict = raw_results["device_statistics"]
load_profile = device_stats_dict['House 1']['H1 DefinedLoad'][
'load_profile_kWh']
load_profile_ts = {
int(from_format(time_slot, DATE_TIME_FORMAT).timestamp()): value
for time_slot, value in load_profile.items()
}
start = int(list(load_profile_ts.keys())[0])
end = int(list(load_profile_ts.keys())[23])
assert all([
load_profile_ts[i] == load_profile_ts[i + 86400]
for i in range(start, end, 3600)
])
def parse_writing_time(ts: str):
# remove the prefix
ts = re.sub(r'^.*: ', '', ts)
# remove the " at "
ts = ts.replace(' at ', ' ')
# uppercase the meridian for pendulum
ts = ts.replace('am', 'AM').replace('pm', 'PM')
# parse (eg. "October 7th 2007, 5:27PM")
timestamp = pendulum.from_format(ts, 'MMMM Do, YYYY h:mmA')
# and isoformat for return
return timestamp.isoformat()
def get_single_item_published_time(
self, item: feedparser.FeedParserDict) -> pendulum.datetime:
"""Parse rss item's stringified published time into datetime object.
Atom feed datetime string is standardized so there should be no
problem there.
Timestamp examples:
- Mon, 17 Dec 2018 16:19:00 -0500
- Mon, 17 Dec 2018 22:08:27 GMT
Args:
item: parsed feedparser single item
Returns:
datetime object
"""
try:
return pendulum.from_format(item.published,
"ddd, D MMM YYYY H:mm:ss ZZ")
except ValueError:
# fallback to UTC since some feeds send dates with GMT as timezone
return pendulum.from_format(item.published,
"ddd, D MMM YYYY H:mm:ss")
def _calc_report_generation_date(report_date: str, profile) -> str:
"""
According to reference time zone is specified by the profile used to
request the report. If specified date is today, then the performance
report may contain partial information. Based on this we generating
reports from day before specified report date and we should take into
account timezone for each profile.
:param report_date requested date that stored in stream state.
:return date parameter for Amazon Ads generate report.
"""
report_date = pendulum.from_format(report_date,
ReportStream.REPORT_DATE_FORMAT)
profile_tz = pytz.timezone(profile.timezone)
profile_time = report_date.astimezone(profile_tz)
return profile_time.format(ReportStream.REPORT_DATE_FORMAT)
def custom_transform_function(original_value: Any,
field_schema: Mapping[str, Any]) -> Any:
if original_value and field_schema.get("format") == "date-time":
try:
return pendulum.from_format(
original_value, "ddd, D MMM YYYY HH:mm:ss ZZ").in_timezone(
"UTC").to_iso8601_string()
except ValueError:
# Twilio API returns datetime in two formats:
# - RFC2822, like "Fri, 11 Dec 2020 04:28:40 +0000";
# - ISO8601, like "2020-12-11T04:29:09Z".
# If `ValueError` exception was raised this means that datetime was already in ISO8601 format and there
# is no need in transforming anything.
pass
return original_value
def get_updated_state(
self,
current_stream_state: MutableMapping[str, Any],
latest_record: Mapping[str, Any],
) -> Mapping[str, Any]:
form_id = self.get_form_id(latest_record)
if not form_id or not latest_record.get(self.cursor_field):
return current_stream_state
current_stream_state[form_id] = current_stream_state.get(form_id, {})
current_stream_state[form_id][self.cursor_field] = max(
pendulum.from_format(latest_record[self.cursor_field],
self.date_format).int_timestamp,
current_stream_state[form_id].get(self.cursor_field, 1),
)
return current_stream_state
def pendulum_datetime_extract(date_string, date_format=None):
# Attempt to extract the date using the specified format if provided
try:
if date_format:
if "unix" in date_format:
if "milliseconds" in date_format:
date_string = date_string[:-3]
_datetime = pendulum.from_timestamp(int(date_string))
else:
_datetime = pendulum.from_format(date_string, date_format)
else:
# Assume ISO-8601
_datetime = pendulum.parse(date_string)
except (ValueError, TypeError, RuntimeError):
_datetime = None
return _datetime
def test_ingest_preprocessed(ingest):
file_stem = "2020_04_01_00_00_00-v2"
file_path = Path(f"tmp/raw/{file_stem}_2020_03_27.csv")
ingest(file_path)
date = pendulum.from_format(file_path.stem[23:], "YYYY_MM_DD").naive()
child_fact = Fact.child_or_load_table(date)
with engine.begin() as conn:
count = conn.execute(child_fact.select()).rowcount
assert count == 104
count = conn.execute(
child_fact.select(child_fact.c.session_end == None)).rowcount
assert count == 104
def parse_log_date(val):
"""Convert a date string in either date or filename format
to a datetime.
Date format is YYYY-mm-dd. Filename format is the same, but with
any extension at the end, most likely ".log"
If parsing fails, raise a 400 error."""
filename = os.path.splitext(val)[0]
try:
return pendulum.from_format(filename, "YYYY-MM-DD")
except ValueError:
raise cherrypy.HTTPError(
400,
"Unable to parse a date from {}".format(filename)
)
def add_arguments(self, parser):
parser.add_argument('--dry', action='store_true', help='Should the script actually make changes? (In a transaction)')
parser.add_argument('--commit', action='store_true', help='Should the script actually commit?')
parser.add_argument('--noninteractive', action='store_true', help='Should the script merge objects without asking?')
parser.add_argument('--from', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider jobs on or after this date')
parser.add_argument('--until', type=lambda d: pendulum.from_format(d, '%Y-%m-%d'), help='Only consider jobs on or before this date')
def decode(self, s):
"""
Return the serialization as a date object.
"""
# return pendulum.from_format(s, 'YYYYMMDD').naive().in_timezone('local')
return pendulum.from_format(s, 'YYYYMMDD').date()
def test_from_format_with_locale(text, fmt, expected):
now = pendulum.datetime(2018, 2, 2)
with pendulum.test(now):
formatted = pendulum.from_format(text, fmt, locale="fr").isoformat()
assert formatted == expected
def test_from_format_returns_datetime():
d = pendulum.from_format("1975-05-21 22:32:11", "YYYY-MM-DD HH:mm:ss")
assert_datetime(d, 1975, 5, 21, 22, 32, 11)
assert isinstance(d, pendulum.DateTime)
assert "UTC" == d.timezone_name
def test_from_format_with_invalid_padded_day():
with pytest.raises(ValueError):
d = pendulum.from_format("Apr 2 12:00:00 2020 GMT", "MMM DD HH:mm:ss YYYY z")
def test_from_format_with_padded_day():
d = pendulum.from_format("Apr 2 12:00:00 2020 GMT", "MMM DD HH:mm:ss YYYY z")
assert_datetime(d, 2020, 4, 2, 12)
def test_from_format_with_millis():
d = pendulum.from_format("1975-05-21 22:32:11.123456", "YYYY-MM-DD HH:mm:ss.SSSSSS")
assert_datetime(d, 1975, 5, 21, 22, 32, 11, 123456)
def test_from_format_with_escaped_elements_valid_tokens():
d = pendulum.from_format("1975-05-21T22:32:11.123Z", "YYYY-MM-DD[T]HH:mm:ss.SSS[Z]")
assert_datetime(d, 1975, 5, 21, 22, 32, 11)
assert "UTC" == d.timezone_name
def test_from_format_with_escaped_elements():
d = pendulum.from_format("1975-05-21T22:32:11+00:00", "YYYY-MM-DD[T]HH:mm:ssZ")
assert_datetime(d, 1975, 5, 21, 22, 32, 11)
assert "+00:00" == d.timezone_name
def create_application_approval_task(self, application, domains, closing_date=None):
if application.type != 'edit':
summary = 'Application assessment: {} (App #{})'.format(application.data.get('name'), application.id)
description = TICKET_DESCRIPTION % (current_app.config['ADMIN_ADDRESS'] +
"/admin/applications/preview/{}".format(application.id))
issuetype_name = INITIAL_ASSESSMENT_ISSUE_TYPE
else:
summary = 'Profile edit: {} (#{})'.format(application.data.get('name'), application.supplier_code)
description = (
"+*This is a profile edit*+\n\n"
"Please evaluate the changes made by the seller and ensure they meet the "
"[assessment guidelines|"
"https://govausites.atlassian.net/wiki/display/DM/Initial+Assessment+Checklist]. \n"
"Changes will be summarised at the top of the seller profile.\n\n"
"Seller profile link: [%s]\n\n"
"---\n\n"
"A snapshot of the application is attached."
"" % (current_app.config['ADMIN_ADDRESS'] + "/admin/applications/preview/{}".format(application.id))
)
issuetype_name = SUBSEQUENT_ASSESSMENT_ISSUE_TYPE
pricing = application.data.get('pricing', None)
competitive = True
description += "---\n\n"
supplier = application.supplier
if pricing:
for k, v in pricing.iteritems():
max_price = float(v.get('maxPrice')) if v.get('maxPrice', None) else 0
domain = next(d for d in domains if d.name == k)
domain_price_min = domain.price_minimum
domain_price_max = domain.price_maximum
price_description = (
"Price threshold for *{domain_name}*:\n"
"Seller Price: *{max_price}*\n"
"Minimum Price: {domain_price_min}\n"
"Maximum Price: *{domain_price_max}*\n"
).format(
max_price=max_price,
domain_name=domain.name,
domain_price_min=domain_price_min,
domain_price_max=domain_price_max
)
description += price_description
if (max_price >= domain_price_min and max_price <= domain_price_max):
description += "Seller is competitive\n\n"
else:
competitive = False
description += "*Seller is less competitive*\n\n"
if supplier:
current_max_price = supplier.data.get('pricing', {}).get(k, {}).get('maxPrice', max_price)
current_domain = next(iter([sd for sd in supplier.domains if sd.domain.name == k]), None)
if float(current_max_price) != float(max_price) and current_domain:
if current_domain.price_status == 'rejected' and competitive is True:
description += (
"Price for *{domain_name}* has been changed to within threshold. "
"Current status is *{price_status}*. "
"Please evaluate and update price status if appropriate.\n\n"
.format(
domain_name=domain.name,
price_status=current_domain.price_status
)
)
elif current_domain.price_status == 'approved' and competitive is False:
description += (
"Price for *{domain_name}* has been changed to now be outside of threshold. "
"Current status is *{price_status}*. "
"Please evaluate and update if appropriate.\n\n"
.format(
domain_name=domain.name,
price_status=current_domain.price_status
)
)
description += "---\n\n"
if supplier:
current_recruiter = supplier.data.get('recruiter')
new_recruiter = application.data.get('recruiter')
if current_recruiter == 'yes' and new_recruiter != 'yes':
description += (
'*This seller was previously purely a recruiter. '
'If approving this edit - please ensure appropriate domain assessments are conducted.*\n\n'
'---\n\n'
)
if current_recruiter != 'yes' and new_recruiter == 'yes':
description += (
'*This seller was previously a hybrid recruiter (and consultancy).*\n\n'
'---\n\n'
)
case_studies = application.data.get('case_studies', [])
for k, case_study in case_studies.iteritems():
current_case_study = next(
iter(
[scs for scs in supplier.case_studies if scs.id == case_study.get('id')]
), None)
if not current_case_study:
supplier_domain = next(iter([
sd
for sd in supplier.domains
if sd.domain.name == case_study.get('service') and sd.price_status == 'rejected'
]), None)
if supplier_domain:
description += (
'*Price for {} was previously rejected. '
'Seller has added a new case study for this domain - '
'please raise a domain assessment for this domain if required.*\n\n'
.format(case_study.get('service'))
)
continue
elif current_case_study.status != 'rejected':
continue
case_study_different = application.is_case_study_different(case_study, current_case_study.data)
if case_study_different:
description += (
"Case study *[{title}|{link}]* is currently *rejected*. "
"Once this edit is approved, the status will be updated to *unassessed*. "
"Please review to ensure it meets the appropriate number of criteria.\n\n"
.format(
title=case_study.get('title'),
link='{}/admin/applications/case-study/{}/{}'.format(
current_app.config['ADMIN_ADDRESS'],
application.id,
case_study.get('id')
)
)
)
details = dict(
project=self.jira_field_codes['MARKETPLACE_PROJECT_CODE'],
summary=summary,
description=description,
duedate=pendulum.now().add(weeks=2).to_date_string(),
issuetype_name=issuetype_name,
labels=[application.type] if application.type else []
)
existing_issues = self.generic_jira.jira.search_issues('"Marketplace Application ID" ~ "{}" '
'AND issuetype = "{}"'
.format(str(application.id), issuetype_name))
if len(existing_issues) > 0 and application.type == 'new':
new_issue = existing_issues[0]
if closing_date is None:
duedate = pendulum.now().add(weeks=2).to_date_string()
else:
duedate = pendulum.from_format(closing_date, '%Y-%m-%d').subtract(days=3).to_date_string()
new_issue.update({
'summary': summary,
'duedate': duedate,
self.jira_field_codes['SUPPLIER_FIELD_CODE']: str(application.supplier_code)
if application.supplier_code else str(0)
})
if new_issue.fields.status.name == 'Closed':
self.generic_jira.jira.transition_issue(new_issue, 'Reopen')
else:
new_issue = self.generic_jira.create_issue(**details)
new_issue.update({self.jira_field_codes['APPLICATION_FIELD_CODE']: str(application.id),
self.jira_field_codes['SUPPLIER_FIELD_CODE']: str(application.supplier_code)
if application.supplier_code else str(0)
})
steps = application.data.get('steps', None)
if (application.type == 'edit' and
steps is not None and
steps.get('pricing', None) and
len(steps.keys()) == 1 and
competitive is True):
from app.main.views.applications import application_approval
application_approval(application.id, True)
self.generic_jira.jira.transition_issue(new_issue, 'Done')
attachment = StringIO.StringIO()
attachment.write(json.dumps(application.json))
self.generic_jira.jira.add_attachment(new_issue.id, attachment,
'snapshot_{}.json'.format(pendulum.now().to_date_string()))
def check_date(timestr: str) -> date:
if not timestr:
return pendulum.from_timestamp(0, "local")
return pendulum.from_format(timestr, "MMMM D, YYYY", "local")
def parse_appengine(self, val):
"""Parse a log line in combined-plus-Appengine-extras format
App Engine extras consist of additional key=value pairs after
the ones for the combined format. This is where
App Engine-sourced geoip values are found. It also contains
less interesting things like the instance ID that served the
request."""
# Sanity check to make sure double-quoted fields are balanced.
val = val.strip()
if val.count('"') % 2 > 0:
val = '{}"'.format(val)
# Sanity check to avoid problems with an empty double-quoted
# string for the referrer field.
val = val.replace(' "" "', ' - "')
try:
fields = self.appengine_grammar.parseString(val).asDict()
except pp.ParseException as exception:
cherrypy.engine.publish(
"applog:add",
"parse",
"fail:column:{}".format(exception.col),
val
)
utc_time = pendulum.from_format(
fields["timestamp"],
"DD/MMM/YYYY:HH:mm:ss ZZ"
).in_tz('utc')
fields["unix_timestamp"] = utc_time.timestamp()
fields["datestamp"] = utc_time.format("YYYY-MM-DD-HH")
if "referrer" in fields:
parse_result = urlparse(fields["referrer"])
# Ignore netloc if it is an empty string.
if parse_result.netloc:
fields["referrer_domain"] = parse_result.netloc
for key, value in fields["extras"].items():
if value in ("ZZ", "?", ""):
continue
if key == "country":
fields[key] = value.upper()
continue
if key == "city":
fields[key] = value.title()
continue
if key == "region" and len(value) == 2:
fields[key] = value.upper()
continue
if key == "latlong" and "," in value:
fields["latitude"], fields["longitude"] = value.split(",")
continue
fields[key] = value
del fields["extras"]
return fields
def test_from_format_error(text, fmt, locale):
now = pendulum.datetime(2018, 2, 2)
with pendulum.test(now):
with pytest.raises(ValueError):
pendulum.from_format(text, fmt, locale=locale)
def GET(self, *_args, **kwargs):
"""Display the list of available grids, or the current grid"""
name = kwargs.get('name', '')
start = kwargs.get('start')
grids = cherrypy.engine.publish(
"registry:search",
"grids:*",
as_dict=True
).pop()
options = defaultdict(lambda: None)
try:
config = next(
value.split("\n")
for key, value in grids.items() if key.endswith(":" + name)
)
headers = [value.strip() for value in config[0].split(",")]
options.update([value.split('=') for value in config[1:]])
except StopIteration:
headers = []
rows = []
if options["layout"] == "month":
today = pendulum.today()
try:
start = pendulum.from_format(start, "YYYY-MM")
except (TypeError, ValueError):
start = today.start_of("month")
headers = ["Date", "Day"] + headers
options["last_month"] = start.subtract(months=1)
options["next_month"] = start.add(months=1)
options["this_month"] = today
period = pendulum.period(start, start.end_of("month"))
for day in period.range("days"):
row = [''] * len(headers)
row[0] = day.format("MMM D, YYYY")
row[1] = day.format("dddd")
rows.append(row)
elif headers:
row = [''] * len(headers)
rows = [row for x in range(1, 30)]
return {
"html": ("grids.jinja.html", {
"headers": headers,
"name": name,
"names": [key.split(":")[1] for key in grids.keys()],
"options": options,
"rows": rows,
})
}
print(utc_time.to_formatted_date_string())
print(utc_time.to_w3c_string())
print(utc_time.to_date_string())
# supports strftime() too
print(utc_time.strftime('%Y-%m-%d %H:%M:%S %Z%z'))
# parse string to date time
dt = pendulum.parse('2018-05-21T22:00:00')
print(dt)
dt = pendulum.parse('2018-05-21T22:00:00', tz='Europe/Paris')
print(dt)
# parsing using specified format string
dt = pendulum.from_format('2018/05/21', 'YYYY/MM/DD')
print(dt)
# drop in replacement for timedelta
time_delta = pendulum.duration(days=1, hours=10, years=2)
print(time_delta)
print('time_delta years =', time_delta.years)
print('time_delta in seconds =', time_delta.in_seconds())
print('time_delta in words =', time_delta.in_words())
print('future date =', pendulum.now() + time_delta)
# period of time
current_date = pendulum.now()
future_date = current_date.add(days=4)
