def test_change_datetime_change(self, graph):
tag = factories.AbstractCreativeWorkFactory()
assert graph.create(
IDObfuscator.encode(tag), 'tag', {
'date_updated': pendulum.fromtimestamp(0).isoformat()
}).change == {
'date_updated': pendulum.fromtimestamp(0).isoformat()
}
def test_timespan(session_or_test, get_real_object):
obj = session_or_test
timespan = get_real_object(obj).timespan
assert timespan is not None
span_start = timespan.lower
assert span_start is not None
assert timespan.upper is None
assert abs(
timespan.lower -
pendulum.fromtimestamp(obj.start_time).astimezone()) < _MIN_THRESHOLD
duration = 10
flux.current_timeline.sleep(duration) # pylint: disable=no-member
obj.report_end()
timespan = get_real_object(obj).timespan
assert timespan is not None
assert timespan.upper is not None
assert timespan.lower == span_start
assert abs((timespan.upper - timespan.lower) -
datetime.timedelta(seconds=duration)) < _MIN_THRESHOLD
def make_order_status_timeline_event(**kwargs):
sorted_orders = sorted(kwargs.get('orders'), key=lambda x: x.id)
customer_email = sorted_orders[-1].billing_address['email']
latest_order_status = sorted_orders[-1].status
latest_order_id = sorted_orders[-1].id
latest_status_timestamp = pendulum.fromtimestamp(kwargs.get('webhook_data')['created_at']).int_timestamp * 1000
latest_order_date_timestamp = (pendulum.parse(str(sorted_orders[-1].date_created)).int_timestamp * 1000)
return create_timeline_event_payload(
orderStatus=latest_order_status,
orderId=latest_order_id,
email=customer_email,
timestamp=latest_status_timestamp,
event_type=TimelineEventType.OrderStatusChanged,
id=sha256(bytes(str(latest_order_id)+latest_order_status+str(latest_order_date_timestamp), 'utf-8')).hexdigest()
)
def _pendulum():
return pendulum.fromtimestamp(TIME)
def show_node(zookeepers,
node,
all_hosts=False,
leader=False,
debug=False,
interactive=False):
"""
Show a zookeeper node on one or more servers.
If the node has children, the children are displayed,
If the node doesn't have children, the contents of the node are displayed.
If leader is specified, only the leader is queried for the node
If all_hosts is specified, each zk host provided is queried individually... if the results
are different between nodes, the child nodes that are different will be highlighted.
returns children of the requested node.
"""
zk_hosts = parse_zk_hosts(zookeepers, all_hosts=all_hosts, leader=leader)
# we'll keep track of differences for this node between zookeepers.
# because zookeeper keeps all nodes in-sync, there shouldn't be differences between the
# nodes... but there might be if you are having replication problems.
all_children = set()
for host in zk_hosts:
# connect to zookeeper
zk = KazooClient(hosts=host, read_only=True)
try:
zk.start()
except KazooTimeoutError as e:
print('ZK Timeout host: [%s], %s' % (host, e))
continue
print('')
# If the node doesn't exist... just let the user know.
if not zk.exists(node):
node_str = style_text(node, BLUE_STYLE, restore=ERROR_STYLE)
zk_str = style_text(host, BLUE_STYLE, restore=ERROR_STYLE)
print(
style_text('No node [%s] on %s' % (node_str, zk_str),
ERROR_STYLE,
pad=2))
continue
if len(zk_hosts) == 1:
print(style_header('Response From: %s [%s]' % (host, node)))
else:
print(
style_text('Response From: %s [%s]' % (host, node),
HEADER_STYLE,
pad=2))
# Query ZooKeeper for the node.
content, zstats = zk.get(node)
# print(dir(zstats))
# print(getattr(zstats, 'czxid'))
# --- Print Node Stats -------------------------
znode_unix_time = zstats.mtime / 1000
#
# local_timezone = time.tzname[time.localtime().tm_isdst] DO NOT USE THIS
is_dst = time.daylight and time.localtime().tm_isdst
offset_hour = time.altzone / 3600 if is_dst else time.timezone / 3600
timezone = 'Etc/GMT%+d' % offset_hour
mod_time = pendulum.fromtimestamp(znode_unix_time, timezone)
mod_time = mod_time.in_timezone(timezone)
local_time_str = mod_time.to_day_datetime_string()
version = str(zstats.version) or str(zstats.cversion)
if debug:
dbg_rjust = max(map(len, ZNODE_DEBUG_ATTRS))
print(style_text("Node Stats:", TITLE_STYLE, lpad=2))
for attr_name in ZNODE_DEBUG_ATTRS:
attr_val = getattr(zstats, attr_name)
if 'time' in attr_name and attr_val > 1:
attr_val = pendulum.fromtimestamp(
int(attr_val) / 1000, timezone).in_timezone(
timezone).to_day_datetime_string()
print(
style_text(attr_name, STATS_STYLE, lpad=4,
rjust=dbg_rjust),
style_text(attr_val, INPUT_STYLE))
else:
print(style_text('Modified:', STATS_STYLE, lpad=2, rjust=9),
style_text(local_time_str, INPUT_STYLE))
print(style_text('Version:', STATS_STYLE, lpad=2, rjust=9),
style_text(version, INPUT_STYLE))
print('')
# --- Print Child Nodes, or Node Content -------
if not zstats.numChildren:
zcontent = bytes.decode(content or b'')
if zcontent:
print(style_text("Contents:", TITLE_STYLE, lpad=2))
print(style_multiline(zcontent, INFO_STYLE, lpad=4))
else:
print(style_text("... No child nodes", INFO_STYLE, lpad=2))
else:
children = zk.get_children(node)
children.sort()
cwidth = max([len(c) for c in children])
print(style_text("Child Nodes:", TITLE_STYLE, lpad=2))
for ch in children:
child_path = node + ch if node.endswith(
'/') else node + '/' + ch
_, czstats = zk.get(child_path)
if all_children and ch not in all_children:
# if this child is unique / different to this zk host, color it differently.
print(style_text(ch, INPUT_STYLE, lpad=4, ljust=cwidth),
end='')
else:
print(style_text(ch, INFO_STYLE, lpad=4, ljust=cwidth),
end='')
mod_ver = czstats.version or czstats.cversion
print(style_text('v:', STATS_STYLE, lpad=3),
style_text(str(mod_ver), INPUT_STYLE, ljust=3),
end='')
print(style_text('eph:', STATS_STYLE, lpad=3),
style_text('yes' if czstats.ephemeralOwner else 'no',
INPUT_STYLE),
end='')
mod_datetime = datetime.utcfromtimestamp(czstats.mtime / 1000)
mod_elapsed = datetime.utcnow() - mod_datetime
if mod_elapsed >= timedelta(hours=48):
mod_style = ''
elif mod_elapsed >= timedelta(hours=2):
mod_style = INPUT_STYLE
elif mod_elapsed >= timedelta(minutes=10):
mod_style = GREEN_STYLE
elif mod_elapsed >= timedelta(minutes=1):
mod_style = INFO_STYLE
else:
mod_style = STATS_STYLE
if mod_datetime.year != 1970:
mod_desc = pendulum.fromtimestamp(czstats.mtime / 1000,
'UTC').diff_for_humans()
else:
mod_desc = 'none'
print(style_text('mod:', STATS_STYLE, lpad=3),
style_text(mod_desc, mod_style))
zk.stop()
all_children = all_children | set(children)
return list(all_children)
def __str__(self):
return pendulum.fromtimestamp(self.value).to_datetime_string()
def dateize(v):
return pendulum.fromtimestamp(v).to_datetime_string()
def test_change_datetime_change(self, graph):
tag = factories.AbstractCreativeWorkFactory()
assert graph.create(IDObfuscator.encode(tag), 'tag', {'date_updated': pendulum.fromtimestamp(0).isoformat()}).change == {'date_updated': pendulum.fromtimestamp(0).isoformat()}
def classify_stream_time(self, message_stream, max_messages=20, max_active_topics=5, autom_message_seconds=10,
low_threshold=.4, high_threshold=.7, low_step=.05, high_step=.02, verbose=True):
"""Classifies an entire stream of messages by predicting the topic to be appended to
Parameters
----------
message_stream : iterable of |message|s
Iterator or list of messages
distance : callable, optional
Distance measure between two texts
max_messages : int, optional
Maximum amount of messages to classify (for debugging and illustration purposes)
max_active_topics : int, optional
Maximum amount of (most-recent) topics a message can be compared to for similarity
autom_message_seconds : int, optional
If the message was produced within less than the amount specified, it will be appended to the same topic as the last message
low_threshold : float, optional
Description
high_threshold : float, optional
Description
low_step : float, optional
Description
high_step : float, optional
Description
verbose : bool, optional
Print the classification stream (defaults to True - while construction)
Returns
-------
list(tuples(|message|, str))
Window (as list of topics)
"""
for m, msg in enumerate(message_stream):
if m > max_messages:
m -= 1
break
last_timestamp = None
if verbose:
print '#{:>3}\033[33m ==> {}\033[0m'.format(m, msg.text.encode('ascii', 'ignore'))
if self.window.is_empty:
# initialize last_datetime
# last_dt = pm.fromtimestamp(msg.timestamp) --> no longer necessary (embedded within the window)
self.window.activate_topic( Topic(msg, 'First message') )
# topics.insert(0, [(msg, 'First message')] )
if verbose:
print '\t First message (new 0)\n'
else:
# We will sequentially try to append to each topic ...
# as time goes by it is harder to append to a topic
# First check if the message was produced within `autom_message_seconds` seconds from the last message
# diff_period = last_dt.diff(pm.fromtimestamp(msg.timestamp))
diff_period = pm.fromtimestamp(self.window.last_timestamp).diff(pm.fromtimestamp(msg.timestamp))
if diff_period.in_seconds() <= autom_message_seconds:
# Append to the last topic (no need to pop, since it is topic[0])
reason = 'Within {} seconds (diff of: {}s)'.format(autom_message_seconds, diff_period.in_seconds())
topics[0].append( (msg, reason) )
last_dt = pm.fromtimestamp(msg.timestamp) # update last_datetime
if verbose:
print '\t inserted to #{} : {}\n'.format(t, reason)
low_th = low_threshold
high_th = high_threshold
topic_scores = [] # in case no topic is close
for t in xrange( min(len(self.window), max_topic_length) ):
# for t in xrange( min(len(topics), max_topic_length) ):
tp_len = len(self.window[t])
# tp_len = len(topics[t])
distances = map(lambda x: self.message_similarity(msg.text, x[0].text), topics[t])
# Assign a non-linear score (very close messages score higher)
score = sum([ 0 if d < low_th else 1 if d < high_th else 3 for d in distances ])
# Very large topics (> 10) should be harder to append to,
# since the odds of a casual match are higher
if (tp_len < 3):
if (score > 0):
reason = 'len({}) < 3 and distances({})'.format(tp_len, distances)
self.window.insert_message(message=msg, reason=reason, topic_index=t)
# _topic = topics.pop(t) # pop from topic queue
# _topic.append( (msg, reason) )
# topics.insert(0, _topic) # append to first topic
# last_dt = pm.fromtimestamp(msg.timestamp) # update last_datetime
if verbose:
print '\t inserted to #{} : {}\n'.format(t, reason)
break
elif (tp_len < 10):
if (score > (tp_len - (2 - tp_len/15.) )):
reason = 'len({}) < 10 and distances({})'.format(tp_len, distances)
self.window.insert_message(message=msg, reason=reason, topic_index=t)
# _topic = topics.pop(t) # pop from topic queue
# _topic.append( (msg, 'len({}) < 10 and distances({})'.format(tp_len, distances)) )
# topics.insert(0, _topic) # append to first topic
# last_dt = pm.fromtimestamp(msg.timestamp) # update last_datetime
if verbose:
print '\t inserted to #{} : {}\n'.format(t, reason)
break
else:
if (score > tp_len*1.5):
reason = 'len({}) > 10 and distances({})'.format(tp_len, distances)
self.window.insert_message(message=msg, reason=reason, topic_index=t)
# _topic = topics.pop(t) # pop from topic queue
# _topic.append( (msg, 'len({}) > 10 and distances({})'.format(tp_len, distances)) )
# topics.insert(0, _topic) # append to first topic
# last_dt = pm.fromtimestamp(msg.timestamp) # update last_datetime
if verbose:
print '\t inserted to #{} : {}\n'.format(t, reason)
break
topic_scores.append( (tp_len,score) ) # append score to topic_scores
# else try with next topic --> harder
low_th += low_step if low_th+low_step < high_th else high_step
high_th += high_step
else:
# If no topic was suitable --> Start new topic
self.window.activate_topic( Topic(msg, 'No similar topics (to 0) scores:({})'.format(topic_scores)) )
# topics.insert(0, [(msg, 'No similar topics (to 0) scores:({})'.format(topic_scores))] )
last_dt = pm.fromtimestamp(msg.timestamp) # update last_datetime
if verbose:
print '\t No similar topics (new 0) scores:({})\n'.format(topic_scores)
print '... Done, processed {} messages'.format(m)
return self.window
def test_fromtimestamp(self):
self.assertEqual(datetime.fromtimestamp(0, pendulum.UTC),
pendulum.fromtimestamp(0, pendulum.UTC))
def from_timestamp(dt):
return pendulum.fromtimestamp(dt, tz=DEFAULT_TZ_NAME)
