def move(conf,
sourceCrashStorageClass=cstore.CrashStorageSystemForLocalFS,
destCrashStorageClass=cstore.CrashStorageSystemForHBase):
logger = conf.logger
crashStoragePoolForSource = cstore.CrashStoragePool(
conf, sourceCrashStorageClass)
crashStoragePoolForDest = cstore.CrashStoragePool(conf,
destCrashStorageClass)
signal.signal(signal.SIGTERM, iwf.respondToSIGTERM)
signal.signal(signal.SIGHUP, iwf.respondToSIGTERM)
#-----------------------------------------------------------------------------
def theIterator():
"""This infinite iterator will walk through the file system storage,
yielding the ooids of every new entry in the filelsystem. If there
are no new entries, it yields None"""
sourceStorage = crashStoragePoolForSource.crashStorage(
) # thread local
try:
while True:
i = 0
for i, ooid in enumerate(sourceStorage.newUuids()):
yield ooid
if i == 0:
yield None
except KeyboardInterrupt:
pass
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
def doSubmission(ooidTuple):
logger.debug('received: %s', str(ooidTuple))
try:
sourceStorage = crashStoragePoolForSource.crashStorage()
destStorage = crashStoragePoolForDest.crashStorage()
ooid = ooidTuple[0]
try:
jsonContents = sourceStorage.get_meta(ooid)
except ValueError:
logger.warning('the json for %s is degenerate and cannot be loaded' \
' - saving empty json', ooid)
jsonContents = {}
dumpContents = sourceStorage.get_raw_dump(ooid)
logger.debug('pushing %s to dest', ooid)
result = destStorage.save_raw(ooid, jsonContents, dumpContents)
if result == cstore.CrashStorageSystem.ERROR:
return iwf.FAILURE
elif result == cstore.CrashStorageSystem.RETRY:
return iwf.RETRY
sourceStorage.quickDelete(ooid)
return iwf.OK
except Exception, x:
sutil.reportExceptionAndContinue(logger)
return iwf.FAILURE
def __init__(self,
config,
logger=logger,
sdb=sdb,
cstore=cstore,
signal=signal):
super(Monitor, self).__init__()
config.logger = logger
for x in Monitor._config_requirements:
assert x in config, '%s missing from configuration' % x
self.crashStorePool = cstore.CrashStoragePool(config)
self.sdb = sdb
self.standardLoopDelay = config.standardLoopDelay.seconds
self.cleanupJobsLoopDelay = config.cleanupJobsLoopDelay.seconds
self.priorityLoopDelay = config.priorityLoopDelay.seconds
self.databaseConnectionPool = self.sdb.DatabaseConnectionPool(
config, logger)
self.config = config
signal.signal(signal.SIGTERM, Monitor.respondToSIGTERM)
signal.signal(signal.SIGHUP, Monitor.respondToSIGTERM)
self.quit = False
def __init__(self, config):
"""
Set the DB and the pool up and store the config.
"""
super(JsonServiceBase, self).__init__(config)
try:
self.database = db.Database(config)
self.crashStoragePool = cs.CrashStoragePool(config,
storageClass=config.hbaseStorageClass)
except (AttributeError, KeyError):
util.reportExceptionAndContinue(logger)
def get(self, **kwargs):
"""Return JSON data of a crash report, given its uuid. """
filters = [
("uuid", None, "str"),
("datatype", None, "str")
]
params = external_common.parse_arguments(filters, kwargs)
if not params.uuid:
raise MissingOrBadArgumentError(
"Mandatory parameter 'uuid' is missing or empty")
if not params.datatype:
raise MissingOrBadArgumentError(
"Mandatory parameter 'datatype' is missing or empty")
if hasattr(self.config, 'hbase'):
config = self.config.hbase
store = crashstorage.HBaseCrashStorage(config)
datatype_method_mapping = {
"raw": "get_raw_dump",
"meta": "get_raw_crash",
"processed": "get_processed"
}
else:
# old middleware
config = self.config
import socorro.storage.crashstorage as cs
store = cs.CrashStoragePool(
config,
storageClass=config.hbaseStorageClass
).crashStorage()
datatype_method_mapping = {
"raw": "get_raw_dump",
"meta": "get_meta",
"processed": "get_processed"
}
get = store.__getattribute__(datatype_method_mapping[params.datatype])
try:
if params.datatype == 'raw':
return (get(params.uuid), 'application/octet-stream')
else:
return get(params.uuid)
except (CrashIDNotFound, OoidNotFoundException):
if params.datatype == 'processed':
self.get(datatype='raw', uuid=params.uuid)
j = priorityjobs.Priorityjobs(config=self.config)
j.create(uuid=params.uuid)
raise ResourceUnavailable(params.uuid)
raise ResourceNotFound(params.uuid)
def __init__(self, config):
"""
Set the DB and the pool up and store the config.
"""
super(JsonServiceBase, self).__init__(config)
try:
self.database = db.Database(config)
self.crashStoragePool = cs.CrashStoragePool(
config,
storageClass=config.hbaseStorageClass
)
except (AttributeError, KeyError), x:
self.config.logger.error(
str(x),
exc_info=True
)
syslog = logging.handlers.SysLogHandler(
address=(config.syslogHost, config.syslogPort),
facility=config.syslogFacilityString,
)
syslog.setLevel(config.syslogErrorLoggingLevel)
syslogFormatter = logging.Formatter(config.syslogLineFormatString)
syslog.setFormatter(syslogFormatter)
logger.addHandler(syslog)
sutil.echoConfig(logger, config)
config.logger = logger
#-------------------------------------------------------------------------------
import socorro.storage.crashstorage as cstore
crashStoragePool = cstore.CrashStoragePool(config,
config.primaryStorageClass)
config.crashStoragePool = crashStoragePool
legacyThrottler = cstore.LegacyThrottler(config)
config.legacyThrottler = legacyThrottler
#-------------------------------------------------------------------------------
web.webapi.internalerror = web.debugerror
web.config.debug = False
servicesList = (wscol.Collector,
)
urls = tuple(y for aTuple in ((x.uri, cpart.class_with_partial_init(x, config))
for x in servicesList) for y in aTuple)
logger.info(str(urls))
if config.modwsgiInstallation:
def move(conf,
sourceCrashStorageClass=cstore.CrashStorageSystemForLocalFS,
destCrashStorageClass=cstore.CrashStorageSystemForHBase):
logger = conf.logger
crashStoragePoolForSource = cstore.CrashStoragePool(
conf, sourceCrashStorageClass)
crashStoragePoolForDest = cstore.CrashStoragePool(conf,
destCrashStorageClass)
signal.signal(signal.SIGTERM, iwf.respondToSIGTERM)
signal.signal(signal.SIGHUP, iwf.respondToSIGTERM)
#-----------------------------------------------------------------------------
def theIterator():
"""This infinite iterator will walk through the file system storage,
yielding the ooids of every new entry in the filelsystem. If there
are no new entries, it yields None"""
destinationCrashStore = crashStoragePoolForDest.crashStorage()
for dir, dirs, files in os.walk(conf.searchRoot):
print dir, files
for aFile in files:
if aFile.endswith('json'):
ooid = aFile[:-5]
logger.debug('the ooid is %s', ooid)
try:
if destinationCrashStore.get_meta():
logger.info('skipping %s - already in hbase', ooid)
pass
except Exception:
logger.info('yielding %s', ooid)
yield ooid
#-----------------------------------------------------------------------------
#-----------------------------------------------------------------------------
def doSubmission(ooidTuple):
logger.debug('received: %s', str(ooidTuple))
try:
sourceStorage = crashStoragePoolForSource.crashStorage()
destStorage = crashStoragePoolForDest.crashStorage()
ooid = ooidTuple[0]
try:
logger.debug('trying to fetch %s', ooid)
jsonContents = sourceStorage.get_meta(ooid)
except ValueError:
logger.warning('the json for %s is degenerate and cannot be loaded' \
' - saving empty json', ooid)
jsonContents = {}
dumpContents = sourceStorage.get_raw_dump(ooid)
if conf.dryrun:
logger.info("dry run - pushing %s to dest", ooid)
else:
logger.debug('pushing %s to dest', ooid)
result = destStorage.save_raw(ooid, jsonContents, dumpContents)
if result == cstore.CrashStorageSystem.ERROR:
return iwf.FAILURE
elif result == cstore.CrashStorageSystem.RETRY:
return iwf.RETRY
try:
sourceStorage.quickDelete(ooid)
except Exception:
sutil.reportExceptionAndContinue(self.logger)
return iwf.OK
except Exception, x:
sutil.reportExceptionAndContinue(logger)
return iwf.FAILURE
