def write_batch(self, alarm_state_hists): for alarm_state_hist in alarm_state_hists: self._batch_stmt.add(self._insert_alarm_state_hist_stmt, alarm_state_hist) self.cassandra_session.execute(self._batch_stmt) self._batch_stmt = query.BatchStatement()
def send_partition(entries, table_name, crit_size=500): """ Collects rdd entries and sends as batch of CQL commands. Required by "save_to_database" function. """ # Initializes keyspace and CQL batch executor in Cassandra database db_session = cassc.Cluster(p["cassandra"]).connect(p["cassandra_key"]) cql_batch = cassq.BatchStatement(consistency_level= \ cass.ConsistencyLevel.QUORUM) batch_size = 0 # Prepares CQL statement, with interpolated table name, and placeholders cql_command = db_session.prepare(""" UPDATE {} SET metric = ? + metric WHERE group = ? AND cycle = ? AND id = ?; """.format(table_name)) for e in entries: # Interpolates prepared CQL statement with values from entry cql_batch.add(cql_command, parameters= \ [cassq.ValueSequence((e[3],)), \ e[0], \ e[1], \ e[2],]) batch_size += 1 # Executes collected CQL commands, then re-initializes collection if batch_size == crit_size: db_session.execute(cql_batch) cql_batch = cassq.BatchStatement(consistency_level= \ cass.ConsistencyLevel.QUORUM) batch_size = 0 # Executes final set of remaining batches and closes Cassandra session db_session.execute(cql_batch) db_session.shutdown() return None
def __init__(self): super(AbstractCassandraRepository, self).__init__() self.conf = cfg.CONF self._cassandra_cluster = cluster.Cluster( self.conf.cassandra.cluster_ip_addresses.split(',')) self.cassandra_session = self._cassandra_cluster.connect( self.conf.cassandra.keyspace) self._batch_stmt = query.BatchStatement()
def delete_job(self, job): job = int(job) batch = cql.BatchStatement(consistency_level=cql.ConsistencyLevel.QUORUM) idel = "DELETE FROM job_info WHERE version = %d AND job = %d"%(self.version, job) rmqLogger(self.channel, 'delete', idel) batch.add(cql.SimpleStatement(idel)) mdel = "DELETE FROM job_md5s WHERE version = %d AND job = %d"%(self.version, job) rmqLogger(self.channel, 'delete', mdel) batch.add(cql.SimpleStatement(mdel)) ldel = "DELETE FROM job_lcas WHERE version = %d AND job = %d"%(self.version, job) rmqLogger(self.channel, 'delete', ldel) batch.add(cql.SimpleStatement(ldel)) self.session.execute(batch)
def write_batch(self, metric_measurement_infos): for metric_measurement_info in metric_measurement_infos: self._batch_stmt.add(self._insert_measurement_stmt, metric_measurement_info.measurement) metric_map = (metric_measurement_info.tenant_id, metric_measurement_info.region, metric_measurement_info.metric_hash, metric_measurement_info.metric_map) self._batch_stmt.add(self._insert_metric_map_stmt, metric_map) self.cassandra_session.execute(self._batch_stmt) self._batch_stmt = query.BatchStatement()
def insert_job_lcas(self, job, rows): job = int(job) cmd = "INSERT INTO job_lcas (version, job, lca, abundance, exp_avg, ident_avg, len_avg, md5s, level) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)" rmqLogger(self.channel, 'insert', cmd, len(rows)) insert = self.session.prepare(cmd) batch = cql.BatchStatement(consistency_level=cql.ConsistencyLevel.QUORUM) for (lca, abundance, exp_avg, ident_avg, len_avg, md5s, level) in rows: batch.add(insert, (self.version, job, lca, int(abundance), float(exp_avg), float(ident_avg), float(len_avg), int(md5s), int(level))) # update job_info loaded = self.get_info_count(job, 'lca') + len(rows) cmd = "UPDATE job_info SET lcas = ?, loaded = ?, updated_on = ? WHERE version = ? AND job = ?" rmqLogger(self.channel, 'update', cmd) update = self.session.prepare(cmd) batch.add(update, (loaded, False, datetime.datetime.now(), self.version, job)) # execute atomic batch self.session.execute(batch) return loaded
def main(input_dir, keyspace, table): for f in os.listdir(input_dir): with gzip.open(os.path.join(input_dir, f), 'rt', encoding='utf-8') as logfile: batch = cq.BatchStatement() for line in logfile: line_re = re.compile( r'^(\S+) - - \[(\S+) [+-]\d+\] \"[A-Z]+ (\S+) HTTP/\d\.\d\" \d+ (\d+)$' ) w = re.split(line_re, line) if len(w) == 6: batch.add( cq.SimpleStatement( "INSERT INTO nasalogs (id,host,bytes,datetime,path) VALUES (%s, %s,%s, %s, %s)" ), (UUID(int=uuid4().int), w[1], int( w[4]), convert(w[2]), w[3])) if len(batch) % 350 == 0: addrows = session.execute(batch) batch.clear() addrows = session.execute(batch) rows = session.execute( 'SELECT path, bytes FROM nasalogs WHERE host=%s', ['in24.inetnebr.com'])
def update(self, project_id, service_id, service_obj): """update. :param project_id :param service_id :param service_obj """ service_name = service_obj.name domains = [json.dumps(d.to_dict()) for d in service_obj.domains] origins = [json.dumps(o.to_dict()) for o in service_obj.origins] caching_rules = [ json.dumps(caching_rule.to_dict()) for caching_rule in service_obj.caching ] restrictions = [ json.dumps(r.to_dict()) for r in service_obj.restrictions ] pds = { provider: json.dumps(service_obj.provider_details[provider].to_dict()) for provider in service_obj.provider_details } log_delivery = json.dumps(service_obj.log_delivery.to_dict()) # fetch current domains args = { 'project_id': project_id, 'service_id': uuid.UUID(str(service_id)), } stmt = query.SimpleStatement( CQL_GET_SERVICE, consistency_level=self._driver.consistency_level) resultset = self.session.execute(stmt, args) complete_results = list(resultset) result = complete_results[0] # updates an existing service args = { 'project_id': project_id, 'service_id': uuid.UUID(str(service_id)), 'service_name': service_name, 'flavor_id': service_obj.flavor_id, 'domains': domains, 'origins': origins, 'caching_rules': caching_rules, 'restrictions': restrictions, 'provider_details': pds, 'log_delivery': log_delivery, 'operator_status': service_obj.operator_status } stmt = query.SimpleStatement( CQL_UPDATE_SERVICE, consistency_level=self._driver.consistency_level) self.session.execute(stmt, args) # claim new domains batch_claim = query.BatchStatement( consistency_level=self._driver.consistency_level) for d in service_obj.domains: domain_args = { 'domain_name': d.domain, 'project_id': project_id, 'service_id': uuid.UUID(str(service_id)) } batch_claim.add(query.SimpleStatement(CQL_CLAIM_DOMAIN), domain_args) self.session.execute(batch_claim) # NOTE(TheSriram): We claim (CQL_CLAIM_DOMAIN) all the domains, # that got passed in. Now we create a set out of domains_new # (current domains present) and domains_old (domains present before # we made the current call). The set difference between old and new, # are the domains we need to delete (CQL_RELINQUISH_DOMAINS). domains_old = set([ json.loads(d).get('domain') for d in result.get('domains', []) or [] ]) domains_new = set([json.loads(d).get('domain') for d in domains or []]) # delete domains that no longer exist # relinquish old domains domains_delete = domains_old.difference(domains_new) if domains_delete: args = {'domain_list': query.ValueSequence(domains_delete)} stmt = query.SimpleStatement( CQL_RELINQUISH_DOMAINS, consistency_level=self._driver.consistency_level) self.session.execute(stmt, args)
def update(self, project_id, service_id, service_obj): """update. :param project_id :param service_id :param service_obj """ service_name = service_obj.name domains = [json.dumps(d.to_dict()) for d in service_obj.domains] origins = [json.dumps(o.to_dict()) for o in service_obj.origins] caching_rules = [json.dumps(caching_rule.to_dict()) for caching_rule in service_obj.caching] restrictions = [json.dumps(r.to_dict()) for r in service_obj.restrictions] pds = {provider: json.dumps(service_obj.provider_details[provider].to_dict()) for provider in service_obj.provider_details} log_delivery = json.dumps(service_obj.log_delivery.to_dict()) # fetch current domains args = { 'project_id': project_id, 'service_id': uuid.UUID(str(service_id)), } stmt = query.SimpleStatement( CQL_GET_SERVICE, consistency_level=self._driver.consistency_level) results = self.session.execute(stmt, args) result = results[0] # updates an existing service args = { 'project_id': project_id, 'service_id': uuid.UUID(str(service_id)), 'service_name': service_name, 'flavor_id': service_obj.flavor_id, 'domains': domains, 'origins': origins, 'caching_rules': caching_rules, 'restrictions': restrictions, 'provider_details': pds, 'log_delivery': log_delivery, 'operator_status': service_obj.operator_status } stmt = query.SimpleStatement( CQL_UPDATE_SERVICE, consistency_level=self._driver.consistency_level) self.session.execute(stmt, args) # relinquish old domains stmt = query.SimpleStatement( CQL_RELINQUISH_DOMAINS, consistency_level=self._driver.consistency_level) domain_list = [json.loads(d).get('domain') for d in result.get('domains', []) or []] args = { 'domain_list': query.ValueSequence(domain_list) } self.session.execute(stmt, args) # claim new domains batch_claim = query.BatchStatement( consistency_level=self._driver.consistency_level) for d in service_obj.domains: domain_args = { 'domain_name': d.domain, 'project_id': project_id, 'service_id': uuid.UUID(str(service_id)) } batch_claim.add(query.SimpleStatement(CQL_CLAIM_DOMAIN), domain_args) self.session.execute(batch_claim)