def move(conf, sourceCrashStorageClass=cstore.CrashStorageSystemForLocalFS, destCrashStorageClass=cstore.CrashStorageSystemForHBase): logger = conf.logger crashStoragePoolForSource = cstore.CrashStoragePool( conf, sourceCrashStorageClass) crashStoragePoolForDest = cstore.CrashStoragePool(conf, destCrashStorageClass) signal.signal(signal.SIGTERM, iwf.respondToSIGTERM) signal.signal(signal.SIGHUP, iwf.respondToSIGTERM) #----------------------------------------------------------------------------- def theIterator(): """This infinite iterator will walk through the file system storage, yielding the ooids of every new entry in the filelsystem. If there are no new entries, it yields None""" sourceStorage = crashStoragePoolForSource.crashStorage( ) # thread local try: while True: i = 0 for i, ooid in enumerate(sourceStorage.newUuids()): yield ooid if i == 0: yield None except KeyboardInterrupt: pass #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- def doSubmission(ooidTuple): logger.debug('received: %s', str(ooidTuple)) try: sourceStorage = crashStoragePoolForSource.crashStorage() destStorage = crashStoragePoolForDest.crashStorage() ooid = ooidTuple[0] try: jsonContents = sourceStorage.get_meta(ooid) except ValueError: logger.warning('the json for %s is degenerate and cannot be loaded' \ ' - saving empty json', ooid) jsonContents = {} dumpContents = sourceStorage.get_raw_dump(ooid) logger.debug('pushing %s to dest', ooid) result = destStorage.save_raw(ooid, jsonContents, dumpContents) if result == cstore.CrashStorageSystem.ERROR: return iwf.FAILURE elif result == cstore.CrashStorageSystem.RETRY: return iwf.RETRY sourceStorage.quickDelete(ooid) return iwf.OK except Exception, x: sutil.reportExceptionAndContinue(logger) return iwf.FAILURE
def __init__(self, config, logger=logger, sdb=sdb, cstore=cstore, signal=signal): super(Monitor, self).__init__() config.logger = logger for x in Monitor._config_requirements: assert x in config, '%s missing from configuration' % x self.crashStorePool = cstore.CrashStoragePool(config) self.sdb = sdb self.standardLoopDelay = config.standardLoopDelay.seconds self.cleanupJobsLoopDelay = config.cleanupJobsLoopDelay.seconds self.priorityLoopDelay = config.priorityLoopDelay.seconds self.databaseConnectionPool = self.sdb.DatabaseConnectionPool( config, logger) self.config = config signal.signal(signal.SIGTERM, Monitor.respondToSIGTERM) signal.signal(signal.SIGHUP, Monitor.respondToSIGTERM) self.quit = False
def __init__(self, config): """ Set the DB and the pool up and store the config. """ super(JsonServiceBase, self).__init__(config) try: self.database = db.Database(config) self.crashStoragePool = cs.CrashStoragePool(config, storageClass=config.hbaseStorageClass) except (AttributeError, KeyError): util.reportExceptionAndContinue(logger)
def get(self, **kwargs): """Return JSON data of a crash report, given its uuid. """ filters = [ ("uuid", None, "str"), ("datatype", None, "str") ] params = external_common.parse_arguments(filters, kwargs) if not params.uuid: raise MissingOrBadArgumentError( "Mandatory parameter 'uuid' is missing or empty") if not params.datatype: raise MissingOrBadArgumentError( "Mandatory parameter 'datatype' is missing or empty") if hasattr(self.config, 'hbase'): config = self.config.hbase store = crashstorage.HBaseCrashStorage(config) datatype_method_mapping = { "raw": "get_raw_dump", "meta": "get_raw_crash", "processed": "get_processed" } else: # old middleware config = self.config import socorro.storage.crashstorage as cs store = cs.CrashStoragePool( config, storageClass=config.hbaseStorageClass ).crashStorage() datatype_method_mapping = { "raw": "get_raw_dump", "meta": "get_meta", "processed": "get_processed" } get = store.__getattribute__(datatype_method_mapping[params.datatype]) try: if params.datatype == 'raw': return (get(params.uuid), 'application/octet-stream') else: return get(params.uuid) except (CrashIDNotFound, OoidNotFoundException): if params.datatype == 'processed': self.get(datatype='raw', uuid=params.uuid) j = priorityjobs.Priorityjobs(config=self.config) j.create(uuid=params.uuid) raise ResourceUnavailable(params.uuid) raise ResourceNotFound(params.uuid)
def __init__(self, config): """ Set the DB and the pool up and store the config. """ super(JsonServiceBase, self).__init__(config) try: self.database = db.Database(config) self.crashStoragePool = cs.CrashStoragePool( config, storageClass=config.hbaseStorageClass ) except (AttributeError, KeyError), x: self.config.logger.error( str(x), exc_info=True )
syslog = logging.handlers.SysLogHandler( address=(config.syslogHost, config.syslogPort), facility=config.syslogFacilityString, ) syslog.setLevel(config.syslogErrorLoggingLevel) syslogFormatter = logging.Formatter(config.syslogLineFormatString) syslog.setFormatter(syslogFormatter) logger.addHandler(syslog) sutil.echoConfig(logger, config) config.logger = logger #------------------------------------------------------------------------------- import socorro.storage.crashstorage as cstore crashStoragePool = cstore.CrashStoragePool(config, config.primaryStorageClass) config.crashStoragePool = crashStoragePool legacyThrottler = cstore.LegacyThrottler(config) config.legacyThrottler = legacyThrottler #------------------------------------------------------------------------------- web.webapi.internalerror = web.debugerror web.config.debug = False servicesList = (wscol.Collector, ) urls = tuple(y for aTuple in ((x.uri, cpart.class_with_partial_init(x, config)) for x in servicesList) for y in aTuple) logger.info(str(urls)) if config.modwsgiInstallation:
def move(conf, sourceCrashStorageClass=cstore.CrashStorageSystemForLocalFS, destCrashStorageClass=cstore.CrashStorageSystemForHBase): logger = conf.logger crashStoragePoolForSource = cstore.CrashStoragePool( conf, sourceCrashStorageClass) crashStoragePoolForDest = cstore.CrashStoragePool(conf, destCrashStorageClass) signal.signal(signal.SIGTERM, iwf.respondToSIGTERM) signal.signal(signal.SIGHUP, iwf.respondToSIGTERM) #----------------------------------------------------------------------------- def theIterator(): """This infinite iterator will walk through the file system storage, yielding the ooids of every new entry in the filelsystem. If there are no new entries, it yields None""" destinationCrashStore = crashStoragePoolForDest.crashStorage() for dir, dirs, files in os.walk(conf.searchRoot): print dir, files for aFile in files: if aFile.endswith('json'): ooid = aFile[:-5] logger.debug('the ooid is %s', ooid) try: if destinationCrashStore.get_meta(): logger.info('skipping %s - already in hbase', ooid) pass except Exception: logger.info('yielding %s', ooid) yield ooid #----------------------------------------------------------------------------- #----------------------------------------------------------------------------- def doSubmission(ooidTuple): logger.debug('received: %s', str(ooidTuple)) try: sourceStorage = crashStoragePoolForSource.crashStorage() destStorage = crashStoragePoolForDest.crashStorage() ooid = ooidTuple[0] try: logger.debug('trying to fetch %s', ooid) jsonContents = sourceStorage.get_meta(ooid) except ValueError: logger.warning('the json for %s is degenerate and cannot be loaded' \ ' - saving empty json', ooid) jsonContents = {} dumpContents = sourceStorage.get_raw_dump(ooid) if conf.dryrun: logger.info("dry run - pushing %s to dest", ooid) else: logger.debug('pushing %s to dest', ooid) result = destStorage.save_raw(ooid, jsonContents, dumpContents) if result == cstore.CrashStorageSystem.ERROR: return iwf.FAILURE elif result == cstore.CrashStorageSystem.RETRY: return iwf.RETRY try: sourceStorage.quickDelete(ooid) except Exception: sutil.reportExceptionAndContinue(self.logger) return iwf.OK except Exception, x: sutil.reportExceptionAndContinue(logger) return iwf.FAILURE