def write_batch(self, alarm_state_hists):
for alarm_state_hist in alarm_state_hists:
self._batch_stmt.add(self._insert_alarm_state_hist_stmt,
alarm_state_hist)
self.cassandra_session.execute(self._batch_stmt)
self._batch_stmt = query.BatchStatement()
def send_partition(entries, table_name, crit_size=500):
"""
Collects rdd entries and sends as batch of CQL commands.
Required by "save_to_database" function.
"""
# Initializes keyspace and CQL batch executor in Cassandra database
db_session = cassc.Cluster(p["cassandra"]).connect(p["cassandra_key"])
cql_batch = cassq.BatchStatement(consistency_level= \
cass.ConsistencyLevel.QUORUM)
batch_size = 0
# Prepares CQL statement, with interpolated table name, and placeholders
cql_command = db_session.prepare("""
UPDATE {} SET
metric = ? + metric
WHERE group = ?
AND cycle = ?
AND id = ?;
""".format(table_name))
for e in entries:
# Interpolates prepared CQL statement with values from entry
cql_batch.add(cql_command, parameters= \
[cassq.ValueSequence((e[3],)), \
e[0], \
e[1], \
e[2],])
batch_size += 1
# Executes collected CQL commands, then re-initializes collection
if batch_size == crit_size:
db_session.execute(cql_batch)
cql_batch = cassq.BatchStatement(consistency_level= \
cass.ConsistencyLevel.QUORUM)
batch_size = 0
# Executes final set of remaining batches and closes Cassandra session
db_session.execute(cql_batch)
db_session.shutdown()
return None
def __init__(self):
super(AbstractCassandraRepository, self).__init__()
self.conf = cfg.CONF
self._cassandra_cluster = cluster.Cluster(
self.conf.cassandra.cluster_ip_addresses.split(','))
self.cassandra_session = self._cassandra_cluster.connect(
self.conf.cassandra.keyspace)
self._batch_stmt = query.BatchStatement()
def delete_job(self, job):
job = int(job)
batch = cql.BatchStatement(consistency_level=cql.ConsistencyLevel.QUORUM)
idel = "DELETE FROM job_info WHERE version = %d AND job = %d"%(self.version, job)
rmqLogger(self.channel, 'delete', idel)
batch.add(cql.SimpleStatement(idel))
mdel = "DELETE FROM job_md5s WHERE version = %d AND job = %d"%(self.version, job)
rmqLogger(self.channel, 'delete', mdel)
batch.add(cql.SimpleStatement(mdel))
ldel = "DELETE FROM job_lcas WHERE version = %d AND job = %d"%(self.version, job)
rmqLogger(self.channel, 'delete', ldel)
batch.add(cql.SimpleStatement(ldel))
self.session.execute(batch)
def write_batch(self, metric_measurement_infos):
for metric_measurement_info in metric_measurement_infos:
self._batch_stmt.add(self._insert_measurement_stmt,
metric_measurement_info.measurement)
metric_map = (metric_measurement_info.tenant_id,
metric_measurement_info.region,
metric_measurement_info.metric_hash,
metric_measurement_info.metric_map)
self._batch_stmt.add(self._insert_metric_map_stmt, metric_map)
self.cassandra_session.execute(self._batch_stmt)
self._batch_stmt = query.BatchStatement()
def insert_job_lcas(self, job, rows):
job = int(job)
cmd = "INSERT INTO job_lcas (version, job, lca, abundance, exp_avg, ident_avg, len_avg, md5s, level) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)"
rmqLogger(self.channel, 'insert', cmd, len(rows))
insert = self.session.prepare(cmd)
batch = cql.BatchStatement(consistency_level=cql.ConsistencyLevel.QUORUM)
for (lca, abundance, exp_avg, ident_avg, len_avg, md5s, level) in rows:
batch.add(insert, (self.version, job, lca, int(abundance), float(exp_avg), float(ident_avg), float(len_avg), int(md5s), int(level)))
# update job_info
loaded = self.get_info_count(job, 'lca') + len(rows)
cmd = "UPDATE job_info SET lcas = ?, loaded = ?, updated_on = ? WHERE version = ? AND job = ?"
rmqLogger(self.channel, 'update', cmd)
update = self.session.prepare(cmd)
batch.add(update, (loaded, False, datetime.datetime.now(), self.version, job))
# execute atomic batch
self.session.execute(batch)
return loaded
def main(input_dir, keyspace, table):
for f in os.listdir(input_dir):
with gzip.open(os.path.join(input_dir, f), 'rt',
encoding='utf-8') as logfile:
batch = cq.BatchStatement()
for line in logfile:
line_re = re.compile(
r'^(\S+) - - \[(\S+) [+-]\d+\] \"[A-Z]+ (\S+) HTTP/\d\.\d\" \d+ (\d+)$'
)
w = re.split(line_re, line)
if len(w) == 6:
batch.add(
cq.SimpleStatement(
"INSERT INTO nasalogs (id,host,bytes,datetime,path) VALUES (%s, %s,%s, %s, %s)"
), (UUID(int=uuid4().int), w[1], int(
w[4]), convert(w[2]), w[3]))
if len(batch) % 350 == 0:
addrows = session.execute(batch)
batch.clear()
addrows = session.execute(batch)
rows = session.execute(
'SELECT path, bytes FROM nasalogs WHERE host=%s',
['in24.inetnebr.com'])
def update(self, project_id, service_id, service_obj):
"""update.
:param project_id
:param service_id
:param service_obj
"""
service_name = service_obj.name
domains = [json.dumps(d.to_dict()) for d in service_obj.domains]
origins = [json.dumps(o.to_dict()) for o in service_obj.origins]
caching_rules = [
json.dumps(caching_rule.to_dict())
for caching_rule in service_obj.caching
]
restrictions = [
json.dumps(r.to_dict()) for r in service_obj.restrictions
]
pds = {
provider:
json.dumps(service_obj.provider_details[provider].to_dict())
for provider in service_obj.provider_details
}
log_delivery = json.dumps(service_obj.log_delivery.to_dict())
# fetch current domains
args = {
'project_id': project_id,
'service_id': uuid.UUID(str(service_id)),
}
stmt = query.SimpleStatement(
CQL_GET_SERVICE, consistency_level=self._driver.consistency_level)
resultset = self.session.execute(stmt, args)
complete_results = list(resultset)
result = complete_results[0]
# updates an existing service
args = {
'project_id': project_id,
'service_id': uuid.UUID(str(service_id)),
'service_name': service_name,
'flavor_id': service_obj.flavor_id,
'domains': domains,
'origins': origins,
'caching_rules': caching_rules,
'restrictions': restrictions,
'provider_details': pds,
'log_delivery': log_delivery,
'operator_status': service_obj.operator_status
}
stmt = query.SimpleStatement(
CQL_UPDATE_SERVICE,
consistency_level=self._driver.consistency_level)
self.session.execute(stmt, args)
# claim new domains
batch_claim = query.BatchStatement(
consistency_level=self._driver.consistency_level)
for d in service_obj.domains:
domain_args = {
'domain_name': d.domain,
'project_id': project_id,
'service_id': uuid.UUID(str(service_id))
}
batch_claim.add(query.SimpleStatement(CQL_CLAIM_DOMAIN),
domain_args)
self.session.execute(batch_claim)
# NOTE(TheSriram): We claim (CQL_CLAIM_DOMAIN) all the domains,
# that got passed in. Now we create a set out of domains_new
# (current domains present) and domains_old (domains present before
# we made the current call). The set difference between old and new,
# are the domains we need to delete (CQL_RELINQUISH_DOMAINS).
domains_old = set([
json.loads(d).get('domain')
for d in result.get('domains', []) or []
])
domains_new = set([json.loads(d).get('domain') for d in domains or []])
# delete domains that no longer exist
# relinquish old domains
domains_delete = domains_old.difference(domains_new)
if domains_delete:
args = {'domain_list': query.ValueSequence(domains_delete)}
stmt = query.SimpleStatement(
CQL_RELINQUISH_DOMAINS,
consistency_level=self._driver.consistency_level)
self.session.execute(stmt, args)
def update(self, project_id, service_id, service_obj):
"""update.
:param project_id
:param service_id
:param service_obj
"""
service_name = service_obj.name
domains = [json.dumps(d.to_dict())
for d in service_obj.domains]
origins = [json.dumps(o.to_dict())
for o in service_obj.origins]
caching_rules = [json.dumps(caching_rule.to_dict())
for caching_rule in service_obj.caching]
restrictions = [json.dumps(r.to_dict())
for r in service_obj.restrictions]
pds = {provider:
json.dumps(service_obj.provider_details[provider].to_dict())
for provider in service_obj.provider_details}
log_delivery = json.dumps(service_obj.log_delivery.to_dict())
# fetch current domains
args = {
'project_id': project_id,
'service_id': uuid.UUID(str(service_id)),
}
stmt = query.SimpleStatement(
CQL_GET_SERVICE,
consistency_level=self._driver.consistency_level)
results = self.session.execute(stmt, args)
result = results[0]
# updates an existing service
args = {
'project_id': project_id,
'service_id': uuid.UUID(str(service_id)),
'service_name': service_name,
'flavor_id': service_obj.flavor_id,
'domains': domains,
'origins': origins,
'caching_rules': caching_rules,
'restrictions': restrictions,
'provider_details': pds,
'log_delivery': log_delivery,
'operator_status': service_obj.operator_status
}
stmt = query.SimpleStatement(
CQL_UPDATE_SERVICE,
consistency_level=self._driver.consistency_level)
self.session.execute(stmt, args)
# relinquish old domains
stmt = query.SimpleStatement(
CQL_RELINQUISH_DOMAINS,
consistency_level=self._driver.consistency_level)
domain_list = [json.loads(d).get('domain')
for d in result.get('domains', []) or []]
args = {
'domain_list': query.ValueSequence(domain_list)
}
self.session.execute(stmt, args)
# claim new domains
batch_claim = query.BatchStatement(
consistency_level=self._driver.consistency_level)
for d in service_obj.domains:
domain_args = {
'domain_name': d.domain,
'project_id': project_id,
'service_id': uuid.UUID(str(service_id))
}
batch_claim.add(query.SimpleStatement(CQL_CLAIM_DOMAIN),
domain_args)
self.session.execute(batch_claim)
