def print_anomalies(anomalies,
show_debug=False,
show_color=True,
show_timestamp=True,
timestamps=(None, None)):
changes = list(filter(lambda x: x[0] == commands.CHANGE, anomalies))
warning = list(filter(lambda x: x[0] == commands.WARNING, anomalies))
debug = list(filter(lambda x: x[0] == commands.DEBUG,
anomalies)) if show_debug else []
c1 = "\x1b[32m" if show_color else ""
c2 = "\x1b[31m" if show_color else ""
c3 = "\x1b[36m" if show_color else ""
c4 = "\x1b[34m" if show_color else ""
c_end = "\x1b[0m" if show_color else ""
la, lw, ld = len(changes), len(warning), len(debug)
debugs = " and %i debug message%s" % (
ld, "s" if ld != 1 else "") if ld > 0 else ""
timestamps = [ts_to_str(t) for t in list(timestamps)]
betweens = " between [%s] and [%s] " % (
timestamps[0], timestamps[1]) if show_timestamp else ""
print("%s%i change%s detected (%i warning%s%s)%s%s" %
(c1, la, "s" if la != 1 else "", lw, "s" if lw != 1 else "", debugs,
betweens, c_end))
for w in warning:
print("%s%s! %s%s" % (c2, get_ts(w[2], show_timestamp), w[1], c_end))
for c in changes:
print("%s%s+ %s%s" % (c3, get_ts(c[2], show_timestamp), c[1], c_end))
if show_debug:
for d in debug:
print("%s%s- %s%s" %
(c4, get_ts(d[2], show_timestamp), d[1], c_end))
def __mint_coin(self, file_path):
content = None
try:
file_handle = open(file_path, 'r')
content = file_handle.read()
file_handle.close()
except Exception as ex:
print 'Error while reading from file: {0}'.format(file_path)
print '{0}'.format(ex)
return False
solution = ""
for ch in content:
if ch != '0' and ch != '1' and not ch.isspace():
print "Invalid character '{0}' in solution file. Only 0's, 1's and whitespaces are allowed.".format(ch)
return False
if ch == '0' or ch == '1':
solution += ch
req_json = { 'solution' : solution, 'clientTimestamp' : int(time.time() * 1000) }
response = self.client.send('POST', '/vault/1.0.0', json.dumps(req_json))
payload = process_response(response, 201)
if payload:
candidate = json.loads(payload)
table = self.__get_coin_table()
table.add_row([candidate['coinId'], candidate['user'], candidate['application'],
ts_to_str(candidate['timestamp']), self.__status_str(candidate['status']), candidate['size']])
print table
print 'Solution candidate submitted successfully.'
print
return True
return False
def __get_coin(self, coin_id):
response = self.client.send('GET', '/vault/1.0.0/{0}'.format(coin_id))
payload = process_response(response)
if payload is not None:
coin = json.loads(payload)
table = self.__get_coin_table(True)
table.add_row([coin['coinId'], coin['user'], coin['application'], ts_to_str(coin['timestamp']),
self.__status_str(coin['status']), coin['reason'], error_code_to_str(coin['reason']), coin['size']])
print table
print
def __get_coin(self, coin_id):
response = self.client.send('GET', '/vault/1.0.0/{0}'.format(coin_id))
payload = process_response(response)
if payload is not None:
coin = json.loads(payload)
table = self.__get_coin_table(True)
table.add_row([
coin['coinId'], coin['user'], coin['application'],
ts_to_str(coin['timestamp']),
self.__status_str(coin['status']), coin['reason'],
error_code_to_str(coin['reason']), coin['size']
])
print table
print
def log_g(ts_s, g, t_count, log_file, dry_run):
if dry_run:
return
ts_str = utils.ts_to_str(ts_s)
with open(log_file, 'a', encoding='utf-8') as out_f:
if not g:
out_f.write('%s,0,0,0,0,0\n' % ts_str)
else:
components = sorted(nx.connected_components(g), key=len, reverse=True)
out_f.write('%s,%d,%d,%d,%d,%d\n' % (
ts_str, t_count, g.number_of_nodes(), g.number_of_edges(),
len(components), len(components[0])
))
def __list_coins(self):
response = self.client.send('GET', '/vault/1.0.0')
payload = process_response(response)
if payload is not None:
coin_list = json.loads(payload)
if len(coin_list) > 0:
table = self.__get_coin_table()
for coin in coin_list:
table.add_row([coin['coinId'], coin['user'], coin['application'],
ts_to_str(coin['timestamp']), self.__status_str(coin['status']), coin['size']])
print table
if len(coin_list) > 1:
print '{0} coins available in vault'.format(len(coin_list))
else:
print '1 coin available in vault'
else:
print 'No coins available in vault'
print
def __list_coins(self):
response = self.client.send('GET', '/vault/1.0.0')
payload = process_response(response)
if payload is not None:
coin_list = json.loads(payload)
if len(coin_list) > 0:
table = self.__get_coin_table()
for coin in coin_list:
table.add_row([
coin['coinId'], coin['user'], coin['application'],
ts_to_str(coin['timestamp']),
self.__status_str(coin['status']), coin['size']
])
print table
if len(coin_list) > 1:
print '{0} coins available in vault'.format(len(coin_list))
else:
print '1 coin available in vault'
else:
print 'No coins available in vault'
print
def __mint_coin(self, file_path):
content = None
try:
file_handle = open(file_path, 'r')
content = file_handle.read()
file_handle.close()
except Exception as ex:
print 'Error while reading from file: {0}'.format(file_path)
print '{0}'.format(ex)
return
solution = ""
for ch in content:
if ch != '0' and ch != '1' and not ch.isspace():
print "Invalid character '{0}' in solution file. Only 0's, 1's and whitespaces are allowed.".format(
ch)
return
if ch == '0' or ch == '1':
solution += ch
req_json = {
'solution': solution,
'clientTimestamp': int(time.time() * 1000)
}
response = self.client.send('POST', '/vault/1.0.0',
json.dumps(req_json))
payload = process_response(response, 201)
if payload:
candidate = json.loads(payload)
table = self.__get_coin_table()
table.add_row([
candidate['coinId'], candidate['user'],
candidate['application'],
ts_to_str(candidate['timestamp']),
self.__status_str(candidate['status']), candidate['size']
])
print table
print 'Solution candidate submitted successfully.'
print
def build_activity_graph(tweets,
t_0): # tweets is a tweet map { tweet_id : tweet }
first_tweet_ts_str = utils.ts_to_str(
t_0, fmt=utils.TWITTER_TS_FORMAT) # epoch_seconds_2_timestamp_str(t_0)
first_tweet_ts = utils.epoch_seconds_2_ts(
t_0) #first_tweet_ts_str) # parse_twitter_ts(first_tweet_ts_str)
g = nx.MultiDiGraph(post_count=len(tweets))
def add_node(g, n_id, n_type='USER', is_author=False):
if n_id not in g:
g.add_node(n_id, n_type=n_type, label=n_id, is_author=is_author)
elif is_author:
# g.nodes[n_id]['n_type'] = n_type
g.nodes[n_id]['is_author'] = is_author
def node_type_for(interaction):
if interaction == 'HASHTAG' or interaction == 'URL':
return interaction
else:
return 'USER'
def add_edge(g, from_id, to_id, tweet_id, ts_str, int_type, **kwargs):
add_node(g, from_id, 'USER', True)
# g.nodes[from_id]['is_author'] = True
add_node(g, to_id, n_type=node_type_for(int_type))
t = utils.extract_ts_s(
ts_str
) - t_0 # timestamp_2_epoch_seconds(utils.extract_ts_s(ts_str)) - t_0
attrs = {'time_t': t, 'tweet_id': tweet_id, 'interaction': int_type}
key = '%s %s %s in %s' % (from_id, int_type, to_id, tweet_id)
g.add_edge(from_id, to_id, key=key, **{**attrs, **kwargs})
# Build networks
# edge types: REPOST, MENTION, REPLY, QUOTE, URL, HASHTAG
observed_user_ids = set()
for tweet_id in tweets:
tweet = tweets[tweet_id]
hashtags = lowered_hashtags_from(tweet)
urls = expanded_urls_from(tweet)
mentions = mentioned_ids_from(tweet)
tweet_text = extract_text(tweet)
tweet_ts = tweet['created_at']
tweet_id = tweet['id_str']
tweeter_id = tweet['user']['id_str']
observed_user_ids.add(tweeter_id)
for ht in hashtags:
add_edge(g, tweeter_id, ht, tweet_id, tweet_ts, 'HASHTAG')
for url in urls:
if not embedded_extended_tweet_url(
tweet_id, url
): # extended tweets include a URL to their extended form
add_edge(g, tweeter_id, url, tweet_id, tweet_ts, 'URL')
for mentioned_id in mentions:
observed_user_ids.add(mentioned_id)
add_edge(g, tweeter_id, mentioned_id, tweet_id, tweet_ts,
'MENTION')
if 'retweeted_status' in tweet:
retweeter = tweeter_id
retweetee = tweet['retweeted_status']['user']['id_str']
observed_user_ids.add(retweetee)
add_edge(
g,
retweeter,
retweetee,
tweet_id,
tweet_ts,
'REPOST',
original_tweet_id=tweet['retweeted_status']['id_str'],
original_tweet_ts=tweet['retweeted_status']['created_at'],
posting_delay_sec=(
utils.extract_ts_s(tweet['retweeted_status']['created_at'])
- utils.extract_ts_s(tweet_ts)) #.total_seconds()
)
elif 'quoted_status' in tweet and 'retweeted_status' not in tweet:
quoter = tweeter_id
quotee = tweet['quoted_status']['user']['id_str']
observed_user_ids.add(quotee)
add_edge(
g,
quoter,
quotee,
tweet_id,
tweet_ts,
'QUOTE',
original_tweet_id=tweet['quoted_status']['id_str'],
original_tweet_ts=tweet['quoted_status']['created_at'],
posting_delay_sec=(
utils.extract_ts_s(tweet['quoted_status']['created_at']) -
utils.extract_ts_s(tweet_ts)) #.total_seconds()
)
elif 'in_reply_to_status_id_str' in tweet and tweet[
'in_reply_to_status_id_str'] in tweets:
# only consider replies that appear in the corpus
# basic reply info
replier = tweeter_id
replied_to = tweet['in_reply_to_user_id_str']
observed_user_ids.add(replied_to)
replied_to_status = tweets[tweet['in_reply_to_status_id_str']]
replied_to_status_ts = replied_to_status['created_at']
posting_delay_sec = (utils.extract_ts_s(replied_to_status_ts) -
utils.extract_ts_s(tweet_ts)
) #.total_seconds()
add_edge(g,
replier,
replied_to,
tweet_id,
tweet_ts,
'REPLY',
original_tweet_id=tweet['in_reply_to_status_id_str'],
original_tweet_ts=replied_to_status_ts,
posting_delay_sec=posting_delay_sec)
# in conversation
if tweet['in_reply_to_status_id_str'] in tweets:
# follow the reply chain as far as we can
conversation_root = root_of_conversation(
tweet['in_reply_to_status_id_str'], tweets)
# conversation_root MAY NOT be in the corpus - it's still a link though
conv_root_ts = first_tweet_ts_str
posting_delay_sec = (utils.ts_2_epoch_seconds(first_tweet_ts) -
utils.extract_ts_s(tweet_ts)
) #.total_seconds()
if conversation_root in tweets:
observed_user_ids.add(
tweets[conversation_root]['user']['id_str'])
conv_root_ts = tweets[conversation_root]['created_at']
posting_delay_sec = (utils.extract_ts_s(conv_root_ts) -
utils.extract_ts_s(tweet_ts)
) #.total_seconds()
add_edge(g,
replier,
conversation_root,
tweet_id,
tweet_ts,
'IN_CONVERSATION',
original_tweet_id=conversation_root,
original_tweet_ts=conv_root_ts,
posting_delay_sec=posting_delay_sec)
return g
def get_ts(ts, show_timestamp=True):
return "[%s] " % ts_to_str(ts) if show_timestamp else ""
def add_entry(self, data, hostname="localhost", timestamp=None):
self.data.setdefault(hostname, [])
if not timestamp:
timestamp = datetime.utcnow()
tsnow = ts_to_str(timestamp)
self.data[hostname].append({"timestamp": tsnow, "data": data})