def query_handler(environ, start_response): setupStderr(environ["wsgi.errors"]) params = dict(parse_qsl(environ.get("QUERY_STRING", ""))) try: db_connection = db.connect() try: results = db.query(db_connection, *query(**params), dict_result=True) total = db.query(db_connection, "SELECT FOUND_ROWS()")[0][0] finally: db_connection.close() except MySQLdb.Error: traceback.print_exc() return common.show_error("Failed to query database!", start_response, "500 Database error") try: echo = int(params["echo"]) except (ValueError, KeyError): echo = 0 response_headers = [("Content-type", "application/json; charset=utf-8")] start_response("200 OK", response_headers) return [json.dumps({"results": results, "echo": echo, "total": total, "count": len(results)}, ensure_ascii=False).encode("utf-8")]
def submit(environ, start_response): setupStderr(environ["wsgi.errors"]) config = get_config() # Check that this is a POST request if environ["REQUEST_METHOD"] != "POST": return common.show_error("Unsupported request method", start_response) # Parse the submitted JSON try: data = json.loads(environ["wsgi.input"].read( int(environ["CONTENT_LENGTH"]))) except (KeyError, IOError, ValueError): return common.show_error("Error while parsing JSON data.", start_response) # Make sure the submitted data was contained within an object at least if not isinstance(data, dict): return common.show_error( "Error, data must be contained within an object.", start_response) # Log the data to a file log_dir = config.get("filterhitstats", "log_dir") try: log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", "")) except (OSError, IOError): traceback.print_exc() return common.show_error("Failed to write data to log file!", start_response, "500 Logging error") # Update the geometrical_mean aggregations in the database interval = config.get("filterhitstats", "interval") try: db_connection = db.connect() try: db.write(db_connection, geometrical_mean.update(interval, data)) finally: db_connection.close() except: # Updating the aggregations in the database failed for whatever reason, # log the details but continue to return 204 to the client to avoid the # re-transmission of data. processing_error_log = os.path.join( config.get("filterhitstats", "log_dir"), "processing-errors.log") with open(processing_error_log, "a+") as f: message = "Problem processing data file %s:\n%s" % ( log_file, traceback.format_exc()) print >> f, "[%s] %s" % ( datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z"), message) # Send back a 204 No Content start_response("204 No Content", []) return []
def submit(environ, start_response): setupStderr(environ["wsgi.errors"]) config = get_config() # Check that this is a POST request if environ["REQUEST_METHOD"] != "POST": return common.show_error("Unsupported request method", start_response) # Parse the submitted JSON try: data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"]))) except (KeyError, IOError, ValueError): return common.show_error("Error while parsing JSON data.", start_response) # Make sure the submitted data was contained within an object at least if not isinstance(data, dict): return common.show_error("Error, data must be contained within an object.", start_response) # Log the data to a file log_dir = config.get("filterhitstats", "log_dir") try: log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", "")) except (OSError, IOError): traceback.print_exc() return common.show_error("Failed to write data to log file!", start_response, "500 Logging error") # Update the geometrical_mean aggregations in the database interval = config.get("filterhitstats", "interval") try: db_connection = db.connect() try: db.write(db_connection, geometrical_mean.update(interval, data)) finally: db_connection.close() except: # Updating the aggregations in the database failed for whatever reason, # log the details but continue to return 204 to the client to avoid the # re-transmission of data. processing_error_log = os.path.join(config.get("filterhitstats", "log_dir"), "processing-errors.log") with open(processing_error_log, "a+") as f: message = "Problem processing data file %s:\n%s" % ( log_file, traceback.format_exc() ) print >> f, "[%s] %s" % (datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z"), message) # Send back a 204 No Content start_response("204 No Content", []) return []
def db(self): if (not self._db): self._db = db.connect() self._clear_database() return self._db
sys.exit("Could not read log file %s" % log_file) return data if __name__ == "__main__": if not len(sys.argv) == 2: print "Usage: python -m sitescripts.filterhits.bin.reprocess_logs /path/to/logs" sys.exit(1) interval = get_config().get("filterhitstats", "interval") def read_update(f): return geometrical_mean.update(interval, read_data(f)) if sys.argv[1].endswith(".log"): sql = read_update(sys.argv[1]) else: sql = itertools.chain.from_iterable( itertools.imap(read_update, log_files(sys.argv[1]))) db_connection = db.connect() try: db.write(db_connection, sql) except: logging.error("Failed to process file %s, all changes rolled back." % _last_log_file) raise finally: db_connection.close()
_last_log_file = log_file except IOError: sys.exit("Could not read log file %s" % log_file) return data if __name__ == "__main__": if not len(sys.argv) == 2: print "Usage: python -m sitescripts.filterhits.bin.reprocess_logs /path/to/logs" sys.exit(1) interval = get_config().get("filterhitstats", "interval") def read_update(f): return geometrical_mean.update(interval, read_data(f)) if sys.argv[1].endswith(".log"): sql = read_update(sys.argv[1]) else: sql = itertools.chain.from_iterable(itertools.imap(read_update, log_files(sys.argv[1]))) db_connection = db.connect() try: db.write(db_connection, sql) except: logging.error("Failed to process file %s, all changes rolled back." % _last_log_file) raise finally: db_connection.close()