def fix(configContext, logger, query, fixer):
rows, last_date_processed = fetchOoids(configContext, logger, query)
hbc = hbaseClient.HBaseConnectionForCrashReports(
configContext.hbaseHost,
configContext.hbasePort,
configContext.hbaseTimeout,
logger=logger)
for row in rows:
try:
ooid, last_date_processed = row
logger.info('fixing ooid: %s' % ooid)
dump = hbc.get_dump(ooid)
fname = '/dev/shm/%s.dump' % ooid
with open(fname, 'wb') as orig_dump_file:
orig_dump_file.write(dump)
logger.debug('wrote dump file: %s' % fname)
logger.debug('fixed dump file: %s' % fname)
subprocess.check_call([fixer, fname])
logger.debug('fixer: %s' % fixer)
with open(fname, 'rb') as fixed_dump_file:
fixed_dump = fixed_dump_file.read()
hbc.put_fixed_dump(ooid,
fixed_dump,
add_to_unprocessed_queue=True,
submitted_timestamp=date_to_string(
utc_now()))
logger.debug('put fixed dump file into hbase: %s' % fname)
os.unlink(fname)
logger.debug('removed dump file: %s' % fname)
except:
socorro.lib.util.reportExceptionAndContinue(logger)
return last_date_processed
def __init__(self, config, local_config=None):
"""Initialize the parts needed to start making database connections
parameters:
config - the complete config for the app. If a real app, this
would be where a logger or other resources could be
found.
local_config - this is the namespace within the complete config
where the actual database parameters are found"""
super(HBaseSingleConnectionContext, self).__init__()
self.config = config
if local_config is None:
local_config = config
dummy_connection = hbase_client.HBaseConnectionForCrashReports(
local_config.hbase_host,
local_config.hbase_port,
local_config.hbase_timeout,
logger=self.config.logger)
dummy_connection.close()
self.operational_exceptions = \
dummy_connection.hbaseThriftExceptions
self.operational_exceptions += \
(hbase_client.NoConnectionException,)
self.conditional_exceptions = ()
def _truncate_hbase_table(self):
connection = hbase_client.HBaseConnectionForCrashReports(
commonconfig.hbaseHost.default, commonconfig.hbasePort.default,
100)
for row in connection.merge_scan_with_prefix(
'crash_reports', '', ['ids:ooid']):
index_row_key = row['_rowkey']
connection.client.deleteAllRow('crash_reports', index_row_key)
# because of HBase's async nature, deleting can take time
list(connection.iterator_for_all_legacy_to_be_processed())
def tearDown(self):
with self.config_manager.context() as config:
connection = hbase_client.HBaseConnectionForCrashReports(
config.hbase.hbase_host, config.hbase.hbase_port,
config.hbase.hbase_timeout)
for row in connection.merge_scan_with_prefix(
'crash_reports', '', ['ids:ooid']):
index_row_key = row['_rowkey']
connection.client.deleteAllRow('crash_reports', index_row_key)
# because of HBase's async nature, deleting can take time
list(connection.iterator_for_all_legacy_to_be_processed())
def connection(self, name_unused=None):
"""return a new database connection
parameters:
name_unused - optional named connections. Used by the
derived class
"""
#self.config.logger.debug('creating new HBase connection')
return hbase_client.HBaseConnectionForCrashReports(
self.config.hbase_host,
self.config.hbase_port,
self.config.hbase_timeout,
logger=self.config.logger)
def __init__(self):
self.dummy_thriftModule = exp.DummyObjectWithExpectations('dummy_thriftModule')
self.dummy_tsocketModule = exp.DummyObjectWithExpectations('dummy_tsocketModule')
self.dummy_transportModule = exp.DummyObjectWithExpectations('dummy_transportModule')
self.dummy_protocolModule = exp.DummyObjectWithExpectations('dummy_protocolModule')
self.dummy_ttypesModule = exp.DummyObjectWithExpectations('dummy_ttypesModule')
self.dummy_clientClass = exp.DummyObjectWithExpectations('dummy_clientClass')
self.dummy_columnClass = exp.DummyObjectWithExpectations('dummy_columnClass')
self.dummy_mutationClass = exp.DummyObjectWithExpectations('dummy_mutationClass')
class FakeIOError(Exception):
pass
self.FakeIOError = FakeIOError
class FakeIllegalArgument(Exception):
pass
self.FakeIllegalArgument = FakeIllegalArgument
class FakeAlreadyExists(Exception):
pass
self.FakeAlreadyExists = FakeAlreadyExists
class FakeTException(Exception):
pass
self.FakeTException = FakeTException
self.dummy_ttypesModule.expect('IOError', None, None, self.FakeIOError, None)
self.dummy_ttypesModule.expect('IllegalArgument', None, None, self.FakeIllegalArgument, None)
self.dummy_ttypesModule.expect('AlreadyExists', None, None, self.FakeAlreadyExists, None)
self.dummy_thriftModule.expect('TException', None, None, self.FakeTException, None)
self.dummy_transportObject = exp.DummyObjectWithExpectations('dummy_transportObject')
self.dummy_protocolObject = exp.DummyObjectWithExpectations('dummy_protocolObject')
self.dummy_clientObject = exp.DummyObjectWithExpectations('dummy_clientObject')
self.dummy_tsocketModule.expect('TSocket', ('somehostname', 666), {}, self.dummy_transportObject)
self.dummy_transportModule.expect('TBufferedTransport', (self.dummy_transportObject,), {}, self.dummy_transportObject)
self.dummy_protocolModule.expect('TBinaryProtocol', (self.dummy_transportObject,), {}, self.dummy_protocolObject)
self.dummy_clientClass.expect('__call__', (self.dummy_protocolObject,), {}, self.dummy_clientObject)
self.dummy_transportObject.expect('setTimeout', (9000,), {})
self.dummy_transportObject.expect('open', (), {})
self.conn = hbc.HBaseConnectionForCrashReports('somehostname', 666, 9000,
thrift=self.dummy_thriftModule,
tsocket=self.dummy_tsocketModule,
ttrans=self.dummy_transportModule,
protocol=self.dummy_protocolModule,
ttp=self.dummy_ttypesModule,
client=self.dummy_clientClass,
column=self.dummy_columnClass,
mutation=self.dummy_mutationClass)
def __init__(self, config, quit_check_callback=None):
super(HBaseCrashStorage, self).__init__(config, quit_check_callback)
self.logger.info('connecting to hbase')
self.hbaseConnection = hbase_client.HBaseConnectionForCrashReports(
config.hbase_host,
config.hbase_port,
config.hbase_timeout,
logger=self.logger)
self.transaction_executor = config.transaction_executor_class(
config, self.hbaseConnection, self.quit_check)
self.exceptions_eligible_for_retry += \
self.hbaseConnection.hbaseThriftExceptions
self.exceptions_eligible_for_retry += \
(hbase_client.NoConnectionException,)
def __init__(self,host,port): self.hbase_connection = hbase.HBaseConnectionForCrashReports(host,port)