def setup_query_parameters(config): now = config.day + dt.timedelta(1) now_str = now.strftime("%Y-%m-%d") yesterday = config.day yesterday_str = yesterday.strftime("%Y-%m-%d") logger.debug("config.day = %s; now = %s; yesterday = %s", config.day, now, yesterday) prod_phrase = "" try: if config.product != "": if "," in config.product: prod_list = [x.strip() for x in config.product.split(",")] prod_phrase = "and r.product in ('%s')" % "','".join(prod_list) else: prod_phrase = "and r.product = '%s'" % config.product except Exception: util.reportExceptionAndContinue(logger) ver_phrase = "" try: if config.version != "": if "," in config.product: ver_list = [x.strip() for x in config.version.split(",")] ver_phrase = "and r.version in ('%s')" % "','".join(ver_list) else: ver_phrase = "and r.version = '%s'" % config.version except Exception: util.reportExceptionAndContinue(logger) return util.DotDict( {"now_str": now_str, "yesterday_str": yesterday_str, "prod_phrase": prod_phrase, "ver_phrase": ver_phrase} )
def setup_query_parameters(config): now = config.day + dt.timedelta(1) now_str = now.strftime('%Y-%m-%d') yesterday = config.day yesterday_str = yesterday.strftime('%Y-%m-%d') logger.debug("config.day = %s; now = %s; yesterday = %s", config.day, now, yesterday) prod_phrase = '' try: if config.product != '': if ',' in config.product: prod_list = [x.strip() for x in config.product.split(',')] prod_phrase = ("and r.product in ('%s')" % "','".join(prod_list)) else: prod_phrase = "and r.product = '%s'" % config.product except Exception: util.reportExceptionAndContinue(logger) ver_phrase = '' try: if config.version != '': if ',' in config.product: ver_list = [x.strip() for x in config.version.split(',')] ver_phrase = ("and r.version in ('%s')" % "','".join(ver_list)) else: ver_phrase = "and r.version = '%s'" % config.version except Exception: util.reportExceptionAndContinue(logger) return util.DotDict({ 'now_str': now_str, 'yesterday_str': yesterday_str, 'prod_phrase': prod_phrase, 'ver_phrase': ver_phrase })
def cleanup (self): self.logger.debug("%s - killing database connections", threading.currentThread().getName()) for name, aConnection in self.iteritems(): try: aConnection.close() self.logger.debug("%s - connection %s closed", threading.currentThread().getName(), name) except psycopg2.InterfaceError: self.logger.debug("%s - connection %s already closed", threading.currentThread().getName(), name) except: util.reportExceptionAndContinue(self.logger)
def cleanup(self): self.logger.debug("%s - killing database connections", threading.currentThread().getName()) for name, aConnection in self.iteritems(): try: aConnection.close() self.logger.debug("%s - connection %s closed", threading.currentThread().getName(), name) except psycopg2.InterfaceError: self.logger.debug("%s - connection %s already closed", threading.currentThread().getName(), name) except: util.reportExceptionAndContinue(self.logger)
def dailyUrlDump( config, sdb=sdb, gzipped_csv_files=gzipped_csv_files, IdCache=IdCache, write_row=write_row, process_crash=process_crash, logger=logger, ): dbConnectionPool = sdb.DatabaseConnectionPool(config, logger) # Set the temp_buffers for this session databaseTempbuffers = "8MB" # default if "databaseTempbuffers" in config: databaseTempbuffers = config.databaseTempbuffers try: try: db_conn, db_cursor = dbConnectionPool.connectionCursorPair() with gzipped_csv_files(config) as csv_file_handles_tuple: headers_not_yet_written = True id_cache = IdCache(db_cursor) sql_parameters = setup_query_parameters(config) logger.debug( "config.day = %s; now = %s; yesterday = %s", config.day, sql_parameters.now_str, sql_parameters.yesterday_str, ) sql_query = sql % sql_parameters logger.debug("SQL is: %s", sql_query) db_cursor.execute(""" SET TEMP_BUFFERS = %s """, (databaseTempbuffers,)) for crash_row in sdb.execute(db_cursor, sql_query): if headers_not_yet_written: write_row(csv_file_handles_tuple, [x[0] for x in db_cursor.description]) headers_not_yet_written = False column_value_list = process_crash(crash_row, id_cache) write_row(csv_file_handles_tuple, column_value_list) # end for loop over each crash_row finally: dbConnectionPool.cleanup() except: util.reportExceptionAndContinue(logger)
def PUT(self, *args): """ Call the put method defined in a subclass and return its result. Return a JSON dump of the returned value, or the raw result if a content type was returned. """ try: result = self.put(*args) if isinstance(result, tuple): web.header('Content-Type', result[1]) return result[0] web.header('Content-Type', 'application/json') return json.dumps(result) except web.HTTPError: raise except Exception: util.reportExceptionAndContinue(self.config.logger) raise
def dailyUrlDump(config, sdb=sdb, gzipped_csv_files=gzipped_csv_files, IdCache=IdCache, write_row=write_row, process_crash=process_crash, logger=logger): dbConnectionPool = sdb.DatabaseConnectionPool(config, logger) # Set the temp_buffers for this session databaseTempbuffers = '8MB' # default if 'databaseTempbuffers' in config: databaseTempbuffers = config.databaseTempbuffers try: try: db_conn, db_cursor = dbConnectionPool.connectionCursorPair() with gzipped_csv_files(config) as csv_file_handles_tuple: headers_not_yet_written = True id_cache = IdCache(db_cursor) sql_parameters = setup_query_parameters(config) logger.debug("config.day = %s; now = %s; yesterday = %s", config.day, sql_parameters.now_str, sql_parameters.yesterday_str) sql_query = sql % sql_parameters logger.debug("SQL is: %s", sql_query) db_cursor.execute(""" SET TEMP_BUFFERS = %s """, (databaseTempbuffers, )) for crash_row in sdb.execute(db_cursor, sql_query): if headers_not_yet_written: write_row(csv_file_handles_tuple, [x[0] for x in db_cursor.description]) headers_not_yet_written = False column_value_list = process_crash(crash_row, id_cache) write_row(csv_file_handles_tuple, column_value_list) # end for loop over each crash_row finally: dbConnectionPool.cleanup() except: util.reportExceptionAndContinue(logger)
try: result = self.get(*args) if isinstance(result, tuple): web.header('Content-Type', result[1]) return result[0] web.header('Content-Type', 'application/json') return json.dumps(result) except web.webapi.HTTPError: raise except (DatabaseError, InsertionError), e: raise web.webapi.InternalError(message=str(e)) except (MissingArgumentError, BadArgumentError), e: raise BadRequest(str(e)) except Exception: stringLogger = util.StringLogger() util.reportExceptionAndContinue(stringLogger) try: util.reportExceptionAndContinue(self.config.logger) except (AttributeError, KeyError): pass raise Exception(stringLogger.getMessages()) def get(self, *args): raise NotImplementedError( "The GET function has not been implemented for %s" % repr(args) ) def POST(self, *args): """ Call the post method defined in a subclass and return its result.
config.logger = logger if config.numberOfSubmissions == 'forever': config.iteratorFunc = sub.createInfiniteFileSystemIterator elif config.numberOfSubmissions == 'all': config.iteratorFunc = sub.createFileSystemIterator else: config.iteratorFunc = sub.createLimitedFileSystemIterator config.numberOfSubmissions = int(config.numberOfSubmissions) if config.dryrun: config.submissionFunc = sub.submissionDryRun else: config.submissionFunc = sub.doSubmission config.sleep = float(config.delay) / 1000.0 config.uniqueHang = 'uniqueHangId' in config if config.searchRoot: sub.submitter(config) else: try: import json with open(config.jsonfile) as jsonFile: formData = json.load(jsonFile) config.submissionFunc(formData, config.dumpfile, config.url, config.logger) except Exception, x: sutil.reportExceptionAndContinue(config.logger)
if config.numberOfSubmissions == 'forever': config.iteratorFunc = sub.createInfiniteFileSystemIterator elif config.numberOfSubmissions == 'all': config.iteratorFunc = sub.createFileSystemIterator else: config.iteratorFunc = sub.createLimitedFileSystemIterator config.numberOfSubmissions = int(config.numberOfSubmissions) if config.dryrun: config.submissionFunc = sub.submissionDryRun else: config.submissionFunc = sub.doSubmission config.sleep = float(config.delay)/1000.0 config.uniqueHang = 'uniqueHangId' in config if config.searchRoot: sub.submitter(config) else: try: import json with open(config.jsonfile) as jsonFile: formData = json.load(jsonFile) config.submissionFunc(formData, config.dumpfile, config.url, config.logger) except Exception, x: sutil.reportExceptionAndContinue(config.logger)