def insert_segments(cfg, detections):
with cluster(cfg) as c:
s = session(cfg, c)
st = '''INSERT INTO {keyspace}.segment
(cx, cy, px, py, sday, eday, bday, chprob, curqa,
blcoef, blint, blmag, blrmse,
grcoef, grint, grmag, grrmse,
nicoef, niint, nimag, nirmse,
recoef, reint, remag, rermse,
s1coef, s1int, s1mag, s1rmse,
s2coef, s2int, s2mag, s2rmse,
thcoef, thint, thmag, thrmse)
VALUES
(?, ?, ?, ?, ?, ?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?,
?, ?, ?, ?)'''.format(keyspace=cfg['cassandra_keyspace'])
stmt = s.prepare(st)
chunks = partition_all(cfg['cassandra_batch_size'], detections)
batches = []
for chunk in chunks:
batch = BatchStatement(batch_type=BatchType.UNLOGGED)
for c in chunk:
batch.add(stmt, [
c['cx'], c['cy'], c['px'], c['py'], c['sday'], c['eday'],
c['bday'], c['chprob'], c['curqa'], c['blcoef'],
c['blint'], c['blmag'], c['blrmse'], c['grcoef'],
c['grint'], c['grmag'], c['grrmse'], c['nicoef'],
c['niint'], c['nimag'], c['nirmse'], c['recoef'],
c['reint'], c['remag'], c['rermse'], c['s1coef'],
c['s1int'], c['s1mag'], c['s1rmse'], c['s2coef'],
c['s2int'], c['s2mag'], c['s2rmse'], c['thcoef'],
c['thint'], c['thmag'], c['thrmse']
])
batches.append(batch)
return [s.execute(b) for b in batches]
def test_batched_bound_statement(self):
session, tracer = self._traced_session()
writer = tracer.writer
batch = BatchStatement()
prepared_statement = session.prepare(
'INSERT INTO test.person_write (name, age, description) VALUES (?, ?, ?)'
)
batch.add(prepared_statement.bind(('matt', 34, 'can')))
session.execute(batch)
spans = writer.pop()
assert len(spans) == 1
s = spans[0]
assert s.resource == 'BatchStatement'
assert s.get_tag('cassandra.query') == ''
def test_prepare_batch_statement_after_alter(self):
"""
Test to validate a prepared statement used inside a batch statement is correctly handled
by the driver. The metadata might be updated when a table is altered. This tests combines
queries not being prepared and an update of the prepared statement metadata
@since 3.10
@jira_ticket PYTHON-706
@expected_result queries will have to re-prepared on hosts that aren't the control connection
and the batch statement will be sent.
"""
white_list = ForcedHostSwitchPolicy()
clus = Cluster(
load_balancing_policy=white_list,
protocol_version=PROTOCOL_VERSION, prepare_on_all_hosts=False,
reprepare_on_up=False)
self.addCleanup(clus.shutdown)
table = "test3rf.%s" % self._testMethodName.lower()
session = clus.connect(wait_for_all_pools=True)
session.execute("DROP TABLE IF EXISTS %s" % table)
session.execute("CREATE TABLE %s (k int PRIMARY KEY, a int, b int, d int)" % table)
insert_statement = session.prepare("INSERT INTO %s (k, b, d) VALUES (?, ?, ?)" % table)
# Altering the table might trigger an update in the insert metadata
session.execute("ALTER TABLE %s ADD c int" % table)
values_to_insert = [(1, 2, 3), (2, 3, 4), (3, 4, 5), (4, 5, 6)]
# We query the three hosts in order (due to the ForcedHostSwitchPolicy)
# the first three queries will have to be repreapred and the rest should
# work as normal batch prepared statements
for i in range(10):
value_to_insert = values_to_insert[i % len(values_to_insert)]
batch_statement = BatchStatement(consistency_level=ConsistencyLevel.ONE)
batch_statement.add(insert_statement, value_to_insert)
session.execute(batch_statement)
select_results = session.execute("SELECT * FROM %s" % table)
expected_results = [(1, None, 2, None, 3), (2, None, 3, None, 4),
(3, None, 4, None, 5), (4, None, 5, None, 6)]
self.assertEqual(set(expected_results), set(select_results._current_rows))
def config_check(self):
cluster = None
configuration_df = pd.DataFrame()
try:
select_query = "SELECT algorithm_name,file_param_name,flag,blobAsText(file_content) as file_content,param_value,type from {}.{} WHERE " \
"algorithm_name = '{}' and flag=1 ALLOW FILTERING;".format(KEYSPACE, CONFIG_TABLE_NAME,
ALGORITHM_NAME)
session, cluster = self.connect_cassandra()
latest_df = session.execute(select_query)
configuration_df = pd.DataFrame(latest_df)
# dff=pd.DataFrame(latest_df)
if not configuration_df.empty:
param_df = configuration_df[configuration_df['type'] == 1]
config_files_df = configuration_df[configuration_df['type'] ==
2]
batch = BatchStatement()
insert_user = session.prepare(
"INSERT INTO {}.{} (algorithm_name, file_param_name,flag) VALUES (?,?,?)"
.format(KEYSPACE, CONFIG_TABLE_NAME))
if not param_df.empty:
param_list = list(param_df['file_param_name'])
param_df = param_df.set_index('file_param_name')
config_param_df = pd.read_csv(
CONFIG_PARAM_FILE_PATH).set_index('Parameter')
for param in param_list:
config_param_df.loc[param, 'Value'] = param_df.loc[
param, 'param_value']
batch.add(insert_user, (ALGORITHM_NAME, param, 0))
config_param_df.to_csv(CONFIG_PARAM_FILE_PATH)
if not config_files_df.empty:
for row in config_files_df.itertuples():
df_json = json.loads(row[4])
df = pd.DataFrame.from_dict(df_json)
filename = CONFIG_FOLDER_NAME + os.sep + row[2]
df.to_csv(filename, index=False)
batch.add(insert_user, (ALGORITHM_NAME, row[2], 0))
session.execute(batch)
except Exception as ex:
logger.error("Error while updating the config files %s", ex)
finally:
if cluster:
cluster.shutdown()
def write_next_blocks(self, start_block):
next_block = blockutil.hash_str(start_block)
while next_block:
block_json = blockutil.fetch_block_json(next_block)
if "nextblockhash" in block_json.keys():
next_block, block, txs = blockutil.transform_json(block_json)
batchStmt = BatchStatement()
batchStmt.add(self.__insert_block_stmt, block)
for transaction in txs:
batchStmt.add(self.__insert_transaction_stmt, transaction)
while True:
try:
self.__session.execute(batchStmt)
except Exception as err:
print("Exception ", err, " retrying...", end="\r")
continue
break
print("Wrote block %d" % (block[0]), end="\r")
def query(self, lstQueries):
print "Executing %d queries " % len(lstQueries)
# self.session = None
try:
# import pdb;pdb.set_trace()
# self.session = self.cluster.connect()
batch = BatchStatement(BatchType.LOGGED)
for q in lstQueries:
batch.add(SimpleStatement(q))
self.session.execute(batch)
# results = self.session.execute("select * from test.traffic")
# for row in results:
# print row
except Exception, err:
print 'An exception has occurred.'
print Exception, err
raise IOError('Could not store data')
def delete_from_shard(shard):
batch_stmt = BatchStatement(batch_type=BatchType.UNLOGGED,
consistency_level=ConsistencyLevel.QUORUM)
num_deleted = 0
shard = StreamShard(self.namespace, self.stream,
shard['start_time'], shard['width'],
shard['shard'], False,
MAX_LIMIT, read_size=self.read_size)
for _id in shard.ids_iterator(start_id, end_id):
if _id == start_id:
continue
num_deleted += 1
batch_stmt.add(BoundStatement(self.namespace.DELETE_STMT,
routing_key=shard.key,
consistency_level=ConsistencyLevel.QUORUM)
.bind((shard.key, _id)))
self.session.execute(batch_stmt)
return num_deleted
def setUpClass(cls):
if PROTOCOL_VERSION < 4:
return
cls.cluster = Cluster(protocol_version=PROTOCOL_VERSION)
cls.session = cls.cluster.connect()
cls.session.execute(
"CREATE TABLE IF NOT EXISTS test1rf.client_warning (k int, v0 int, v1 int, PRIMARY KEY (k, v0))"
)
cls.prepared = cls.session.prepare(
"INSERT INTO test1rf.client_warning (k, v0, v1) VALUES (?, ?, ?)")
cls.warn_batch = BatchStatement()
# 213 = 5 * 1024 / (4+4 + 4+4 + 4+4)
# thresh_kb/ (min param size)
for x in range(214):
cls.warn_batch.add(cls.prepared, (x, x, 1))
def insert(cluster, keyspace, cql_stmt, generator, batch_size):
session = cluster.connect(keyspace)
session.default_timeout = 60
session.default_consistency_level = ConsistencyLevel.LOCAL_ONE
prepared_stmt = session.prepare(cql_stmt)
batch_stmt = BatchStatement()
values = take(batch_size, generator)
count = 0
while values:
batch_stmt.add_all([prepared_stmt] * batch_size, values)
session.execute(batch_stmt)
values = take(batch_size, generator)
batch_stmt.clear()
if (count % 1e3) == 0:
print('#blocks {:,.0f}'.format(count), end='\r')
count += batch_size
def store_qc_results(
qc_results_values,
pk,
particle_ids,
particle_bins,
particle_deploys, # TODO: remove - unused
param_name,
strict_range=False):
start_time = time.clock()
if engine.app.config['QC_RESULTS_STORAGE_SYSTEM'] == CASS_LOCATION_NAME:
log.info('Storing QC results in Cassandra.')
insert_results = SessionManager.prepare(
"insert into ooi.qc_results "
"(subsite, node, sensor, bin, deployment, stream, id, parameter, results) "
"values (?, ?, ?, ?, ?, ?, ?, ?, ?)")
batch = BatchStatement()
for (qc_results, particle_id, particle_bin,
particle_deploy) in izip(qc_results_values, particle_ids,
particle_bins, particle_deploys):
batch.add(insert_results,
(pk.get('subsite'), pk.get('node'), pk.get('sensor'),
particle_bin, particle_deploy, pk.get('stream'),
uuid.UUID(particle_id), param_name, str(qc_results)))
SessionManager.session().execute_async(batch)
log.info("QC results stored in {} seconds.".format(time.clock() -
start_time))
elif engine.app.config['QC_RESULTS_STORAGE_SYSTEM'] == 'log':
log.info('Writing QC results to log file.')
qc_log = logging.getLogger('qc.results')
qc_log_string = ""
for (qc_results, particle_id, particle_bin,
particle_deploy) in izip(qc_results_values, particle_ids,
particle_bins, particle_deploys):
qc_log_string += "refdes:{0}-{1}-{2}, bin:{3}, stream:{4}, deployment:{5}, id:{6}, parameter:{7}, qc results:{8}\n" \
.format(pk.get('subsite'), pk.get('node'), pk.get('sensor'), particle_bin,
pk.get('stream'), particle_deploy, particle_id, param_name, qc_results)
qc_log.info(qc_log_string[:-1])
log.info("QC results stored in {} seconds.".format(time.clock() -
start_time))
else:
log.info(
"Configured storage system '{}' not recognized, qc results not stored."
.format(engine.app.config['QC_RESULTS_STORAGE_SYSTEM']))
def read_files(inputs, table):
record_counter = 0
batch_counter = 0
batch_insert = BatchStatement()
insert_statement = session.prepare(
"INSERT INTO " + table +
" (host, id, datetime, path, bytes) VALUES (?, ?, ?, ?, ?)")
# get all files in input folder
for file in os.listdir(inputs):
# unzip files
with gzip.open(os.path.join(inputs, file), 'rt',
encoding='utf-8') as logfile:
# read file line by line
for line in logfile:
# create a tuple of requried fields
log_object = separate_columns(line)
# if log object is valid
if (log_object is not None):
record_counter += 1
batch_insert.add(
insert_statement,
(log_object[0], log_object[1], log_object[2],
log_object[3], log_object[4]))
# insert records when reached to declared batch size
if (record_counter >= BATCH_SIZE):
print("writing batch " + str(batch_counter))
session.execute(batch_insert)
batch_insert.clear()
record_counter = 0
batch_counter += 1
# to insert the final part with number of rows less than batch size
if (record_counter > 0):
print("writing final batch " + str((batch_counter + 1)))
session.execute(batch_insert)
def Create(self, Entities: tuple, Events: tuple, MessageId=None) -> None:
batch = BatchStatement()
EventTime = datetime.datetime.now()
for Event, Entity in zip(Events, Entities):
batch.add(
"INSERT INTO Events (EntityType, EntityId, EventType, EventData, EventTime, Published) VALUES (%s,%s,%s,%s,%s,%s)",
(Entity.EntityType, Entity.Id, Event.EventType,
Event.EventJsonData(), EventTime, False))
batch.add(
"INSERT INTO Snapshots (EntityId, EntityType, EventTime, snapshotdata, SnapshotType, SemantickLock) VALUES (%s,%s,%s,%s,%s,%s)",
[
Entity.Id, Entity.EntityType, EventTime,
Entity.SnapshotData(), Entity.Name, 'Unlock'
])
if MessageId != None:
batch.add("INSERT INTO MessageId (MessageId) VALUES (%s)",
[MessageId])
self.session.execute(batch)
def insert_messages(messages):
if len(messages) == 0:
return None
batch = BatchStatement()
bulk = []
rooms = set()
now = time.time()
for m in messages:
rooms.add(m['room_id'])
m['id'] = rewrite_timestamp(m['id'], now)
batch.add(
insert_message.bind(
(m['id'], m['content'], m['user_name'], m['room_id'])))
doc = {"index": {"_index": setting.INDEX_NAME}}
bulk.append(json.dumps(doc))
bulk.append(json.dumps(m))
session.execute(batch)
es.bulk(body="\n".join(bulk))
return rooms
def test_unicode(self):
"""
Test to validate that unicode query strings are handled appropriately by various query types
@since 3.0.0
@jira_ticket PYTHON-334
@expected_result no unicode exceptions are thrown
@test_category query
"""
unicode_text = u'Fran\u00E7ois'
batch = BatchStatement(BatchType.LOGGED)
batch.add(u"INSERT INTO {0}.{1} (k, v) VALUES (%s, %s)".format(self.keyspace_name, self.function_table_name), (0, unicode_text))
self.session.execute(batch)
self.session.execute(u"INSERT INTO {0}.{1} (k, v) VALUES (%s, %s)".format(self.keyspace_name, self.function_table_name), (0, unicode_text))
prepared = self.session.prepare(u"INSERT INTO {0}.{1} (k, v) VALUES (?, ?)".format(self.keyspace_name, self.function_table_name))
bound = prepared.bind((1, unicode_text))
self.session.execute(bound)
def ca_batch_insert_with_ts(self, docs):
"""
Insert docs in the corresponding Cassandra table using the batch inserts
:param docs: list of dicts representing documents
:return: tuple of (num successful or up to date, num errors) writes
"""
batch = BatchStatement(BatchType.LOGGED)
for doc in docs:
doc = doc.copy()
doc["timestamp_"] = doc[
"version"] * 1e6 # s to us, assuming version is a unix timestamp (seconds)
batch.add(self.ca_ps_upsert_with_ts, doc)
try:
self.ca_session.execute(batch)
# logged batch writes are atomic, so it all succeeds or fails
return len(docs), 0
except Exception: # FIXME: catch only specific Cassandra errors
return 0, len(docs)
def Update(self, Entities: tuple, Events: tuple, MessageId=None) -> None:
EventTime = datetime.datetime.now()
batch = BatchStatement()
for Event, Entity in zip(Events, Entities):
batch.add(
"INSERT INTO Events (EntityType, EntityId, EventType, EventData, EventTime, Published) VALUES (%s,%s,%s,%s,%s,%s)",
(Entity.EntityType, Entity.Id, Event.EventType,
Event.EventJsonData(), EventTime, False))
batch.add(
"UPDATE Snapshots SET SnapshotData = %s, EventTime = %s WHERE EntityType=%s and EntityId=%s",
[
Entity.SnapshotData(), EventTime, Entity.EntityType,
Entity.Id
])
if MessageId != None:
batch.add("INSERT INTO MessageId (MessageId) VALUES (%s)",
[MessageId])
self.session.execute(batch)
async def _write_batchwise(cls, query, batch_size_limit):
"""
Insert data into db via batch statements.
:param query: query to execute
:param batch_size_limit: maximum batch size
:return: result object
"""
cls.logger.debug("Writing data in batches of maximum size " +
str(batch_size_limit))
res = []
try:
cls._check_write_parameters()
batch = BatchStatement()
prepared = cls.session.prepare(query)
batch_size = 0
for i in range(len(cls.result_id)):
cls.logger.debug("Writing results for: " +
str(cls.result_id[i]))
# Send each 50k values in batches
for d in cls.output_data:
batch.add(prepared,
(cls.result_id[i], d[0], str(d[i + 1])))
batch_size += 1
if batch_size >= batch_size_limit:
cls.logger.debug("Writing batch of " +
str(batch_size) + " rows")
res.append(cls.session.execute(batch))
batch.clear()
batch_size = 0
# Send remaining values
if batch_size > 0:
cls.logger.debug("Writing batch of " + str(batch_size) +
" rows")
res.append(cls.session.execute(batch))
batch.clear()
batch_size = 0
except Exception as err:
cls.logger.error("Batch writing failed")
raise Exception("Impossible to write in batches: " + str(err))
return res
def _update_index(self, old_index, task):
""" Updates the index table after leasing a task.
Args:
old_index: The row to remove from the index table.
task: A Task object to create a new index entry for.
"""
old_eta = old_index.eta
update_index = BatchStatement(retry_policy=self.db_access.retry_policy)
delete_old_index = SimpleStatement("""
DELETE FROM pull_queue_tasks_index
WHERE app = %(app)s
AND queue = %(queue)s
AND eta = %(eta)s
AND id = %(id)s
""")
parameters = {
'app': self.app,
'queue': self.name,
'eta': old_eta,
'id': task.id
}
update_index.add(delete_old_index, parameters)
create_new_index = SimpleStatement("""
INSERT INTO pull_queue_tasks_index (app, queue, eta, id, tag, tag_exists)
VALUES (%(app)s, %(queue)s, %(eta)s, %(id)s, %(tag)s, %(tag_exists)s)
""")
parameters = {
'app': self.app,
'queue': self.name,
'eta': task.leaseTimestamp,
'id': task.id
}
try:
parameters['tag'] = task.tag
except AttributeError:
parameters['tag'] = ''
parameters['tag_exists'] = parameters['tag'] != ''
update_index.add(create_new_index, parameters)
self.db_access.session.execute(update_index)
def test_batch_statement(self):
session, tracer = self._traced_session()
batch = BatchStatement()
batch.add(
SimpleStatement("INSERT INTO test.person_write (name, age, description) VALUES (%s, %s, %s)"),
("Joe", 1, "a"),
)
batch.add(
SimpleStatement("INSERT INTO test.person_write (name, age, description) VALUES (%s, %s, %s)"),
("Jane", 2, "b"),
)
session.execute(batch)
spans = tracer.pop()
assert len(spans) == 1
s = spans[0]
assert s.resource == "BatchStatement"
assert s.get_metric("cassandra.batch_size") == 2
assert "test.person" in s.get_tag("cassandra.query")
def execute_batch():
query = "INSERT INTO data (device_id, data_source_id, time_upload, value) VALUES (?, ?, ?, ?) IF NOT EXISTS"
batch = BatchStatement()
prepared = session.prepare(query)
batch_size = 0
# send each 50k values in batches
for d in dates_data:
batch.add(prepared, (device_id, data_source_id, d[0], str(d[1])))
batch_size += 1
if batch_size >= 25_000:
res = session.execute(batch)
print('values sent', str(batch_size))
batch.clear()
batch_size = 0
# send remaining values
if batch_size > 0:
res = session.execute(batch)
print('values sent', str(batch_size))
batch.clear()
batch_size = 0
def test_batch_statement(self):
session, writer = self._traced_session()
batch = BatchStatement()
batch.add(
SimpleStatement(
'INSERT INTO test.person (name, age, description) VALUES (%s, %s, %s)'
), ('Joe', 1, 'a'))
batch.add(
SimpleStatement(
'INSERT INTO test.person (name, age, description) VALUES (%s, %s, %s)'
), ('Jane', 2, 'b'))
session.execute(batch)
spans = writer.pop()
eq_(len(spans), 1)
s = spans[0]
eq_(s.resource, 'BatchStatement')
eq_(s.get_metric('cassandra.batch_size'), 2)
assert 'test.person' in s.get_tag('cassandra.query')
def test_batch_statement(self):
session, tracer = self._traced_session()
batch = BatchStatement()
batch.add(
SimpleStatement('INSERT INTO test.person_write (name, age, description) VALUES (%s, %s, %s)'),
('Joe', 1, 'a'),
)
batch.add(
SimpleStatement('INSERT INTO test.person_write (name, age, description) VALUES (%s, %s, %s)'),
('Jane', 2, 'b'),
)
session.execute(batch)
spans = tracer.pop()
assert len(spans) == 1
s = spans[0]
assert s.resource == 'BatchStatement'
assert s.get_metric('cassandra.batch_size') == 2
assert 'test.person' in s.get_tag('cassandra.query')
def insert_batch_session(self, rows: list, session: Session):
"""
Function to insert data into cassandra in batches in session_songs
:param rows:
:param session:
:return:
"""
batch = BatchStatement(consistency_level=ConsistencyLevel.QUORUM)
statement = session.prepare(insert_session_songs)
try:
for row in rows:
if row[6] == "":
length = 0.0
else:
length = float(row[6])
batch.add(statement,
(int(row[12]), int(row[4]), row[0], row[13], length))
session.execute(batch)
except Exception as ex:
print("Error inserting data into session_songs")
raise ex
def __init__(self, config):
'''
Constructor
'''
self.config = config
self.keyspace = self.config.find('resulttype').find('keyspace').text
self.resultTable = self.config.find('resulttype').find('table').text
self.batchSize = int(
self.config.find('resulttype').find('batchsize').text)
self.cluster = Cluster(self.getNodesInCluster())
self.session = self.cluster.connect(self.keyspace)
self.dbColumnList = self.getDBColumnNames()
self.insertPoints = self.getInsertPointString()
self.insertBatch = BatchStatement()
self.batchCount = 0
columnstr = ','.join(self.dbColumnList)
self.insertQuery = self.session.prepare('INSERT INTO ' +
self.resultTable + ' (' +
columnstr + ') VALUES (' +
self.insertPoints + ')')
self.createTableIfNotExist()
def putInCassandra(s3file, cassandraConnection):
session = getSession(cassandraConnection)
measures_by_date = session.prepare("INSERT INTO measures_by_date (date, timestamp, measureReference, meta, value) VALUES (?, ?, ?, ?, ?)")
measures_by_measurereference = session.prepare("INSERT INTO measures_by_measurereference (date, timestamp, measureReference, meta, value) VALUES (?, ?, ?, ?, ?)")
batch = BatchStatement(consistency_level=ConsistencyLevel.QUORUM)
batch_number = 10
print("Opening File")
count = 0
with gzip.open(s3file, 'r') as csvfile:
reader = csv.DictReader(csvfile)
for row in reader:
if (count >= batch_number):
session.execute(batch)
batch.clear()
count = 0
batch.add(measures_by_measurereference, createRow(row))
batch.add(measures_by_date, createRow(row))
count += 2
session.execute(batch)
batch.clear()
print("Done")
def insert_batch_app_history(self, rows: list, session: Session):
"""
Function to insert records into app_history table
:param rows: list of records
:param session: session object (obtained from cassandra-driver)
:return:
"""
batch = BatchStatement(consistency_level=ConsistencyLevel.QUORUM)
statement = session.prepare(insert_app_history)
try:
for row in rows:
if row[13] != "":
if row[16] == "":
userId = 0
else:
userId = int(row[16])
batch.add(statement, (row[13], row[2], row[5], userId))
session.execute(batch)
except Exception as ex:
print("Error inserting records in app_history")
raise ex
def insert(corm_objects: typing.List[typing.Any]) -> None:
keyspace = corm_objects[0]._corm_details.keyspace
table_name = corm_objects[0]._corm_details.table_name
field_names = corm_objects[0]._corm_details.field_names[:]
instance_type = corm_objects[0].__class__
field_names.append('guid')
formatted_field_names = ','.join(field_names)
formatted_question_marks = ','.join(
['?' for idx in range(0, len(field_names))])
CQL = f'INSERT INTO {keyspace}.{table_name} ({formatted_field_names}) VALUES ({formatted_question_marks})'
prepared_statement = obtain_session(keyspace, True).prepare(CQL)
cql_batch = BatchStatement()
for corm_object in corm_objects:
if corm_object.__class__ != instance_type:
raise Exception('All corm_objects must be the same type')
v_set = corm_object.values()
v_set.append(corm_object.as_hash())
cql_batch.add(prepared_statement, v_set)
obtain_session(keyspace).execute(cql_batch)
def deleteFromCNT(session, params):
try:
minTS = params[0].strftime('%Y-%m-%d %H:%M:%S')
maxTS = params[1].strftime('%Y-%m-%d %H:%M:%S')
rows = session.execute(
query=
"SELECT * FROM CNT WHERE ts >= %s AND ts <= %s ALLOW FILTERING",
parameters=(minTS, maxTS))
DELETE_CNT_QUERY = session.prepare("DELETE FROM CNT WHERE id = ?")
batch = BatchStatement()
for row in rows:
batch.add(DELETE_CNT_QUERY, (row.id, ))
rowss = session.execute(batch, trace=True)
print(rowss.get_query_trace())
log.info("Executed delete from cnt query")
return True
except ReadFailure as rf:
log.error("Error executing deleteFromCNT to Cassandra: {}".format(
rf.args))
return False
def add():
if request.method == 'POST':
new_data = {k: v for k, v in request.form.items()}
#If the user leaves a field blank
if new_data['title'] == '' or new_data['author'] == '' or new_data[
'genre'] == '' or new_data['description'] == '':
return render_template('add.html', alert="required")
else:
id = uuid.uuid4()
batch = BatchStatement()
insert_statement = "INSERT INTO " + table_name + "(id, property, value) values(" + str(
id) + ", %s, %s)"
batch.add(insert_statement, ('title', new_data['title']))
batch.add(insert_statement, ('author', new_data['author']))
batch.add(insert_statement, ('genre', new_data['genre']))
batch.add(insert_statement,
('description', new_data['description']))
session.execute(batch)
return render_template('add.html', alert="success")
else:
return render_template('add.html', alert="")
def deleteFromCNTX(session, params):
try:
minTS = params[0].strftime('%Y-%m-%d %H:%M:%S')
maxTS = params[1].strftime('%Y-%m-%d %H:%M:%S')
rows = session.execute(query="SELECT * FROM {} " \
"WHERE ts >= %s " \
"AND ts <= %s ALLOW FILTERING".format(params[2]), parameters=(minTS, maxTS))
DELETE_CNT_QUERY = session.prepare("DELETE FROM {} " \
"WHERE prodID = ? and consID = ? and topic = ? and ts = ? ".format(params[2]))
batch = BatchStatement()
for row in rows:
batch.add(DELETE_CNT_QUERY,
(row.prodid, row.consid, row.topic, row.ts))
session.execute(batch, trace=True)
log.info("Executed delete from cnt query")
return True
except ReadFailure as rf:
log.error("Error executing deleteFromCNTX to Cassandra: {}".format(
rf.args))
return False
