def cassandra_connect(seed, keyspace=None):
from cassandra.cluster import Cluster
from cassandra.io.libevreactor import LibevConnection
cluster = Cluster()
cluster.connection_class = LibevConnection
return cluster, cluster.connect(keyspace)
def benchmark(run_fn):
for conn_class in supported_reactors:
setup()
log.info("==== %s ====" % (conn_class.__name__,))
cluster = Cluster(['127.0.0.1'])
cluster.connection_class = conn_class
session = cluster.connect(KEYSPACE)
log.debug("Sleeping for two seconds...")
time.sleep(2.0)
query = SimpleStatement("""
INSERT INTO {table} (thekey, col1, col2)
VALUES (%(key)s, %(a)s, %(b)s)
""".format(table=TABLE))
values = {'key': 'key', 'a': 'a', 'b': 'b'}
log.debug("Beginning inserts...")
start = time.time()
try:
run_fn(session, query, values, NUM_QUERIES)
end = time.time()
finally:
teardown()
total = end - start
log.info("Total time: %0.2fs" % total)
log.info("Average throughput: %0.2f/sec" % (NUM_QUERIES / total))
def get_session():
'''
Connect onto a Cassandra cluster with the driver.
:return: A Cassandra session object for cluster interactions.
'''
# grab the cluster information using Docker-provided enviornmental variables
CASSANDRA_HOST = os.environ['CASSANDRA_HOST']
CASSANDRA_DC = os.environ['CASSANDRA_DC']
# create a cluster object that will only connect to a single data center
cluster = Cluster(
[CASSANDRA_HOST],
load_balancing_policy=DCAwareRoundRobinPolicy(local_dc=CASSANDRA_DC),
)
# use the faster event loop provider
cluster.connection_class = LibevConnection
# create the Cassandra session for cluster interaction
session = cluster.connect()
# Panda-centric row factory
def pandas_factory(colnames, rows):
return pd.DataFrame(rows, columns=colnames)
# use Panda-centric settings
session.row_factory = pandas_factory
session.default_fetch_size = None
return session
def benchmark(run_fn):
for conn_class in supported_reactors:
setup()
log.info("==== %s ====" % (conn_class.__name__, ))
cluster = Cluster(['127.0.0.1'])
cluster.connection_class = conn_class
session = cluster.connect(KEYSPACE)
log.debug("Sleeping for two seconds...")
time.sleep(2.0)
query = SimpleStatement("""
INSERT INTO {table} (thekey, col1, col2)
VALUES (%(key)s, %(a)s, %(b)s)
""".format(table=TABLE))
values = {'key': 'key', 'a': 'a', 'b': 'b'}
log.debug("Beginning inserts...")
start = time.time()
try:
run_fn(session, query, values, NUM_QUERIES)
end = time.time()
finally:
teardown()
total = end - start
log.info("Total time: %0.2fs" % total)
log.info("Average throughput: %0.2f/sec" % (NUM_QUERIES / total))
def get_cluster():
cluster = Cluster(*Connection.cluster_args, **Connection.cluster_kwargs)
try:
from cassandra.io.libevreactor import LibevConnection
cluster.connection_class = LibevConnection
except ImportError:
pass
return cluster
def get_cluster():
cluster = Cluster(*Connection.cluster_args, **Connection.cluster_kwargs)
try:
from cassandra.io.libevreactor import LibevConnection
cluster.connection_class = LibevConnection
except ImportError:
pass
return cluster
def execute(self, return_type='dict', renderOnly=False):
rendered = RenderManagers().render(self)
if renderOnly:
return rendered
cluster = Cluster()
cluster.connection_class = LibevConnection
connection = cluster.connect()
connection.row_factory = getattr(decoder, return_type + '_factory', 'dict_factory')
return connection.execute(rendered)
def get_cluster():
cluster = Cluster(*Connection.cluster_args, **Connection.cluster_kwargs)
cluster.default_retry_policy = FeedlyRetryPolicy(
max_read_retries=settings.FEEDLY_CASSANDRA_READ_RETRY_ATTEMPTS,
max_write_retries=settings.FEEDLY_CASSANDRA_WRITE_RETRY_ATTEMPTS)
try:
from cassandra.io.libevreactor import LibevConnection
cluster.connection_class = LibevConnection
except ImportError:
pass
return cluster
def get_cluster():
cluster = Cluster(*Connection.cluster_args, **Connection.cluster_kwargs)
cluster.default_retry_policy = FeedlyRetryPolicy(
max_read_retries=settings.FEEDLY_CASSANDRA_READ_RETRY_ATTEMPTS,
max_write_retries=settings.FEEDLY_CASSANDRA_WRITE_RETRY_ATTEMPTS
)
try:
from cassandra.io.libevreactor import LibevConnection
cluster.connection_class = LibevConnection
except ImportError:
pass
return cluster
def cassandra_connect(seed=None, keyspace=None):
from cassandra.cluster import Cluster
if seed is None:
args = ()
else:
args = ([seed],)
cluster = Cluster(*args)
try:
from cassandra.io.libevreactor import LibevConnection
cluster.connection_class = LibevConnection
except ImportError:
pass
return cluster, cluster.connect(keyspace)
def run(self):
# Connect to Cassandra
cluster = Cluster()
cluster.connection_class = LibevConnection
session = cluster.connect()
# Connect to Kafka
consumer = KafkaConsumer(bootstrap_servers='192.168.65.111:1026',
auto_offset_reset='earliest')
consumer.subscribe(['time'])
for message in consumer:
object = ujson.loads(message)
object_timestamp = self.add_timestamp(object)
object_timestamp_json = ujson.dumps(object_timestamp)
session.execute("insert into dev.time JSON" +
object_timestamp_json)
print(message)
def get_session():
'''
Connect onto a Cassandra cluster with the driver.
:return: A Cassandra session object for cluster interactions.
'''
# grab the cluster information using Docker-provided enviornmental variables
CASSANDRA_HOST = os.environ['CASSANDRA_HOST']
CASSANDRA_DC = os.environ['CASSANDRA_DC']
# create a cluster object that will only connect to a single data center
cluster = Cluster([CASSANDRA_HOST],
load_balancing_policy=DCAwareRoundRobinPolicy(
local_dc=CASSANDRA_DC), )
# use the faster event loop provider
cluster.connection_class = LibevConnection
# create the Cassandra session for cluster interaction
session = cluster.connect()
return session
def connect(seeds, keyspace, datacenter=None, port=9042):
from cassandra.io.libevreactor import LibevConnection
from cassandra.cluster import Cluster
from cassandra.policies import DCAwareRoundRobinPolicy, RetryPolicy, ExponentialReconnectionPolicy
class CustomRetryPolicy(RetryPolicy):
def on_write_timeout(self, query, consistency, write_type,
required_responses, received_responses, retry_num):
# retry at most 5 times regardless of query type
if retry_num >= 5:
return (self.RETHROW, None)
return (self.RETRY, consistency)
load_balancing_policy = None
if datacenter:
# If you are using multiple datacenters it's important to use
# the DCAwareRoundRobinPolicy. If not then the client will
# make cross DC connections. This defaults to round robin
# which means round robin across all nodes irrespective of
# data center.
load_balancing_policy = DCAwareRoundRobinPolicy(local_dc=datacenter)
cluster = Cluster(contact_points=seeds,
port=port,
default_retry_policy=CustomRetryPolicy(),
reconnection_policy=ExponentialReconnectionPolicy(1, 60),
load_balancing_policy=load_balancing_policy,
protocol_version=3)
cluster.connection_class = LibevConnection
cluster.connection_class = LibevConnection
cluster.control_connection_timeout = 10.0
cluster.compression = False
session = cluster.connect(keyspace)
print 'Connection established with seed(s): %s at port: %s and keyspace: %s' %(seeds,port,keyspace)
return session
"""
def parse(path):
g = open(path, 'r')
for l in g:
yield eval(l)
if __name__ == '__main__':
f = open('config.txt', 'r')
contact_points = f.readline()
meta_path = sys.argv[1]
geo_path = sys.argv[2]
input_path = sys.argv[1]
cluster = Cluster([contact_points])
cluster.connection_class = LibevConnection
session = cluster.connect()
# session.execute(META_CF_DROP_STATEMENT)
# session.execute(RANK_CF_DROP_STATEMENT)
session.execute(KS_CREATION_STATEMENT)
session.execute(META_CF_CREATION_STATEMENT)
session.execute(RANK_CF_CREATION_STATEMENT)
meta_prepared = session.prepare(META_INSERT_STATEMENT)
rank_prepared = session.prepare(RANK_INSERT_STATEMENT)
for data in parse(input_path):
asin = data['asin']
title = data.get('title', "")
imurl = data.get('imUrl', "")
price = data.get('price', 0.0)
return 1
else:
return 2
'''path_test = 'test/negative_Test'
obj3=open(path_test,"r")
str3 = obj3.read()
files_test = str3.split("\n\n")
obj3.close()
print files_test'''
start_time = time.time()
data_test = []
data_dict = {}
data_neg = []
result = []
cluster = Cluster()
cluster.connection_class = LibevConnection
session = cluster.connect('getfb')
rows = session.execute('SELECT * FROM getdata')
for row in rows:
data_test.append(row.comment.encode('utf8'))
id_comment = row.id_comment
comment = row.comment
tmp_dict = {id_comment.encode('utf8'): comment.encode('utf8')}
data_dict.update(tmp_dict)
count_neg = 0
count_pos = 0
if (data_test):
for val in data_test:
# print val
def benchmark(thread_class):
options, args = parse_options()
for conn_class in options.supported_reactors:
setup(options.hosts)
log.info("==== %s ====" % (conn_class.__name__,))
cluster = Cluster(options.hosts, metrics_enabled=options.enable_metrics)
cluster.connection_class = conn_class
session = cluster.connect(KEYSPACE)
log.debug("Sleeping for two seconds...")
time.sleep(2.0)
query = SimpleStatement(
"""
INSERT INTO {table} (thekey, col1, col2)
VALUES (%(key)s, %(a)s, %(b)s)
""".format(
table=TABLE
)
)
values = {"key": "key", "a": "a", "b": "b"}
per_thread = options.num_ops / options.threads
threads = []
log.debug("Beginning inserts...")
start = time.time()
try:
for i in range(options.threads):
thread = thread_class(i, session, query, values, per_thread, options.profile)
thread.daemon = True
threads.append(thread)
for thread in threads:
thread.start()
for thread in threads:
while thread.is_alive():
thread.join(timeout=0.5)
end = time.time()
finally:
teardown(options.hosts)
total = end - start
log.info("Total time: %0.2fs" % total)
log.info("Average throughput: %0.2f/sec" % (options.num_ops / total))
if options.enable_metrics:
stats = scales.getStats()["cassandra"]
log.info("Connection errors: %d", stats["connection_errors"])
log.info("Write timeouts: %d", stats["write_timeouts"])
log.info("Read timeouts: %d", stats["read_timeouts"])
log.info("Unavailables: %d", stats["unavailables"])
log.info("Other errors: %d", stats["other_errors"])
log.info("Retries: %d", stats["retries"])
request_timer = stats["request_timer"]
log.info("Request latencies:")
log.info(" min: %0.4fs", request_timer["min"])
log.info(" max: %0.4fs", request_timer["max"])
log.info(" mean: %0.4fs", request_timer["mean"])
log.info(" stddev: %0.4fs", request_timer["stddev"])
log.info(" median: %0.4fs", request_timer["median"])
log.info(" 75th: %0.4fs", request_timer["75percentile"])
log.info(" 95th: %0.4fs", request_timer["95percentile"])
log.info(" 98th: %0.4fs", request_timer["98percentile"])
log.info(" 99th: %0.4fs", request_timer["99percentile"])
log.info(" 99.9th: %0.4fs", request_timer["999percentile"])
def main():
KEYSPACE = "cryptos_keyspace"
assets = [
'bitcoin', "ethereum", "tether", "xrp", "litecoin", "cardano", "iota",
"eos", "stellar"
] # missing quel bastardo di bitcoin-cash
cluster = Cluster()
cluster.connection_class = LibevConnection
session = cluster.connect()
session.execute("DROP KEYSPACE " + KEYSPACE)
session.execute("""
CREATE KEYSPACE IF NOT EXISTS %s
WITH replication = { 'class': 'SimpleStrategy', 'replication_factor': '2' }
""" % KEYSPACE)
session.set_keyspace(KEYSPACE)
for currency in assets:
print("Creating table " + currency + " in cassandra keyspace " +
KEYSPACE + " . . .")
# creating the table with the following specs
create_table_command = """
CREATE TABLE IF NOT EXISTS {} (
ts text,
price text,
date text,
hour text,
yhat text,
yhat_lower text,
yhat_upper text,
PRIMARY KEY (ts, date)
)""".format(currency)
session.execute(create_table_command)
print("Table created. Now filling it with historical data . . .")
command = "INSERT INTO {} (ts, price, date, hour, yhat, yhat_lower, yhat_upper) VALUES (?, ?, ?, ?, ?, ?, ?)".format(
currency)
prepared = session.prepare(command)
folder_path = "../data/history/" + currency + "/only_days_" + currency + ".csv"
df = pd.read_csv(folder_path)
for i in range(len(df["timestamp"])):
row = df.iloc[i]
session.execute(prepared, ("{}".format(
row["timestamp"]), "{}".format(row["price"]), "{}".format(
row["date"]), "{}".format(row["hour"]), "?", "?", "?"))
print("Table " + currency + " filled.\n")
print("Setup CASSANDRA completed!")
def benchmark(thread_class):
options, args = parse_options()
for conn_class in options.supported_reactors:
setup(options.hosts)
log.info("==== %s ====" % (conn_class.__name__, ))
cluster = Cluster(options.hosts,
metrics_enabled=options.enable_metrics)
cluster.connection_class = conn_class
session = cluster.connect(KEYSPACE)
log.debug("Sleeping for two seconds...")
time.sleep(2.0)
query = session.prepare("""
INSERT INTO {table} (thekey, col1, col2) VALUES (?, ?, ?)
""".format(table=TABLE))
values = ('key', 'a', 'b')
per_thread = options.num_ops // options.threads
threads = []
log.debug("Beginning inserts...")
start = time.time()
try:
for i in range(options.threads):
thread = thread_class(i, session, query, values, per_thread,
options.profile)
thread.daemon = True
threads.append(thread)
for thread in threads:
thread.start()
for thread in threads:
while thread.is_alive():
thread.join(timeout=0.5)
end = time.time()
finally:
teardown(options.hosts)
total = end - start
log.info("Total time: %0.2fs" % total)
log.info("Average throughput: %0.2f/sec" % (options.num_ops / total))
if options.enable_metrics:
stats = scales.getStats()['cassandra']
log.info("Connection errors: %d", stats['connection_errors'])
log.info("Write timeouts: %d", stats['write_timeouts'])
log.info("Read timeouts: %d", stats['read_timeouts'])
log.info("Unavailables: %d", stats['unavailables'])
log.info("Other errors: %d", stats['other_errors'])
log.info("Retries: %d", stats['retries'])
request_timer = stats['request_timer']
log.info("Request latencies:")
log.info(" min: %0.4fs", request_timer['min'])
log.info(" max: %0.4fs", request_timer['max'])
log.info(" mean: %0.4fs", request_timer['mean'])
log.info(" stddev: %0.4fs", request_timer['stddev'])
log.info(" median: %0.4fs", request_timer['median'])
log.info(" 75th: %0.4fs", request_timer['75percentile'])
log.info(" 95th: %0.4fs", request_timer['95percentile'])
log.info(" 98th: %0.4fs", request_timer['98percentile'])
log.info(" 99th: %0.4fs", request_timer['99percentile'])
log.info(" 99.9th: %0.4fs", request_timer['999percentile'])
def get_updated_df(currency):
"""
This function gets in INPUT the name of the crypto the consumer is dealing with.
It checks if the Cassandra DB is up to date w.r.t. to the current day and eventually updates the DB.
Then it RETURNS the updated DB as a Pandas DataFrame to the consumer.
NB: the Cassandra DB will be updated until the day before today (YESTERDAY), it's duty of the consumer
reading from Kafka the current price of the crypto.
"""
KEYSPACE = "cryptos_keyspace"
# opening cassandra to retrieve the current df
cluster = Cluster()
cluster.connection_class = LibevConnection
session = cluster.connect()
session.set_keyspace(KEYSPACE)
session.row_factory = pandas_factory
session.default_fetch_size = None
query = "SELECT * FROM {}.{};".format(KEYSPACE, currency)
result = session.execute(query, timeout=None)
df = result._current_rows
# sort the df by timestamp, so that the first row is the last entry
df = df.sort_values(by=["ts"], ascending=False)
# getting only the last information stored in Cassandra
last_entry_cassandra = df.iloc[0]
last_day = last_entry_cassandra["date"]
last_ts = last_entry_cassandra["ts"]
last_hour = last_entry_cassandra["hour"]
index_last_entry = df[df["ts"] == last_ts].index.values
#print(index_last_entry)
#print(len(df.index))
#print(df)
timestamp_curr = int(time.time()) * 1000 # to get milliseconds
if int(last_ts) < timestamp_curr:
# now get the actual price of the currency from the API
url = "https://api.coincap.io/v2/assets/" + currency + "/history?interval=d1"
start = "&start=" + str(last_ts)
end = "&end=" + str(timestamp_curr)
final_url = url + start + end
response = req.get(final_url).json()
for data_point in response["data"]:
timestamp = data_point["time"]
date = data_point["date"].replace(".000Z", "")
split_date = date.split("T")
date = split_date[0]
hour = split_date[1]
priceUsd = data_point["priceUsd"]
if int(timestamp) > int(last_ts):
df.loc[len(df.index)] = [
timestamp, date, hour, priceUsd, "?", "?", "?"
]
command = "INSERT INTO {} (ts, price, date, hour, yhat, yhat_lower, yhat_upper) VALUES (?, ?, ?, ?, ?, ?, ?)".format(
currency)
prepared = session.prepare(command)
session.execute(
prepared,
("{}".format(timestamp), "{}".format(priceUsd),
"{}".format(date), "{}".format(hour), "?", "?", "?"))
return df.tail(len(df.index) - 8)
