def query_handler(environ, start_response):
setupStderr(environ["wsgi.errors"])
params = dict(parse_qsl(environ.get("QUERY_STRING", "")))
try:
db_connection = db.connect()
try:
results = db.query(db_connection, *query(**params), dict_result=True)
total = db.query(db_connection, "SELECT FOUND_ROWS()")[0][0]
finally:
db_connection.close()
except MySQLdb.Error:
traceback.print_exc()
return common.show_error("Failed to query database!", start_response,
"500 Database error")
try:
echo = int(params["echo"])
except (ValueError, KeyError):
echo = 0
response_headers = [("Content-type", "application/json; charset=utf-8")]
start_response("200 OK", response_headers)
return [json.dumps({"results": results, "echo": echo,
"total": total, "count": len(results)},
ensure_ascii=False).encode("utf-8")]
def query_handler(environ, start_response):
setupStderr(environ["wsgi.errors"])
params = dict(parse_qsl(environ.get("QUERY_STRING", "")))
try:
db_connection = db.connect()
try:
results = db.query(db_connection, *query(**params), dict_result=True)
total = db.query(db_connection, "SELECT FOUND_ROWS()")[0][0]
finally:
db_connection.close()
except MySQLdb.Error:
traceback.print_exc()
return common.show_error("Failed to query database!", start_response,
"500 Database error")
try:
echo = int(params["echo"])
except (ValueError, KeyError):
echo = 0
response_headers = [("Content-type", "application/json; charset=utf-8")]
start_response("200 OK", response_headers)
return [json.dumps({"results": results, "echo": echo,
"total": total, "count": len(results)},
ensure_ascii=False).encode("utf-8")]
def submit(environ, start_response):
setupStderr(environ["wsgi.errors"])
config = get_config()
# Check that this is a POST request
if environ["REQUEST_METHOD"] != "POST":
return common.show_error("Unsupported request method", start_response)
# Parse the submitted JSON
try:
data = json.loads(environ["wsgi.input"].read(
int(environ["CONTENT_LENGTH"])))
except (KeyError, IOError, ValueError):
return common.show_error("Error while parsing JSON data.",
start_response)
# Make sure the submitted data was contained within an object at least
if not isinstance(data, dict):
return common.show_error(
"Error, data must be contained within an object.", start_response)
# Log the data to a file
log_dir = config.get("filterhitstats", "log_dir")
try:
log_file = log_filterhits(data, log_dir,
environ.get("QUERY_STRING", ""))
except (OSError, IOError):
traceback.print_exc()
return common.show_error("Failed to write data to log file!",
start_response, "500 Logging error")
# Update the geometrical_mean aggregations in the database
interval = config.get("filterhitstats", "interval")
try:
db_connection = db.connect()
try:
db.write(db_connection, geometrical_mean.update(interval, data))
finally:
db_connection.close()
except:
# Updating the aggregations in the database failed for whatever reason,
# log the details but continue to return 204 to the client to avoid the
# re-transmission of data.
processing_error_log = os.path.join(
config.get("filterhitstats", "log_dir"), "processing-errors.log")
with open(processing_error_log, "a+") as f:
message = "Problem processing data file %s:\n%s" % (
log_file, traceback.format_exc())
print >> f, "[%s] %s" % (
datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z"), message)
# Send back a 204 No Content
start_response("204 No Content", [])
return []
def submit(environ, start_response):
setupStderr(environ["wsgi.errors"])
config = get_config()
# Check that this is a POST request
if environ["REQUEST_METHOD"] != "POST":
return common.show_error("Unsupported request method", start_response)
# Parse the submitted JSON
try:
data = json.loads(environ["wsgi.input"].read(int(environ["CONTENT_LENGTH"])))
except (KeyError, IOError, ValueError):
return common.show_error("Error while parsing JSON data.", start_response)
# Make sure the submitted data was contained within an object at least
if not isinstance(data, dict):
return common.show_error("Error, data must be contained within an object.", start_response)
# Log the data to a file
log_dir = config.get("filterhitstats", "log_dir")
try:
log_file = log_filterhits(data, log_dir, environ.get("QUERY_STRING", ""))
except (OSError, IOError):
traceback.print_exc()
return common.show_error("Failed to write data to log file!", start_response,
"500 Logging error")
# Update the geometrical_mean aggregations in the database
interval = config.get("filterhitstats", "interval")
try:
db_connection = db.connect()
try:
db.write(db_connection, geometrical_mean.update(interval, data))
finally:
db_connection.close()
except:
# Updating the aggregations in the database failed for whatever reason,
# log the details but continue to return 204 to the client to avoid the
# re-transmission of data.
processing_error_log = os.path.join(config.get("filterhitstats", "log_dir"),
"processing-errors.log")
with open(processing_error_log, "a+") as f:
message = "Problem processing data file %s:\n%s" % (
log_file, traceback.format_exc()
)
print >> f, "[%s] %s" % (datetime.now().strftime("%d/%b/%Y:%H:%M:%S %z"), message)
# Send back a 204 No Content
start_response("204 No Content", [])
return []
def db(self):
if (not self._db):
self._db = db.connect()
self._clear_database()
return self._db
sys.exit("Could not read log file %s" % log_file)
return data
if __name__ == "__main__":
if not len(sys.argv) == 2:
print "Usage: python -m sitescripts.filterhits.bin.reprocess_logs /path/to/logs"
sys.exit(1)
interval = get_config().get("filterhitstats", "interval")
def read_update(f):
return geometrical_mean.update(interval, read_data(f))
if sys.argv[1].endswith(".log"):
sql = read_update(sys.argv[1])
else:
sql = itertools.chain.from_iterable(
itertools.imap(read_update, log_files(sys.argv[1])))
db_connection = db.connect()
try:
db.write(db_connection, sql)
except:
logging.error("Failed to process file %s, all changes rolled back." %
_last_log_file)
raise
finally:
db_connection.close()
_last_log_file = log_file
except IOError:
sys.exit("Could not read log file %s" % log_file)
return data
if __name__ == "__main__":
if not len(sys.argv) == 2:
print "Usage: python -m sitescripts.filterhits.bin.reprocess_logs /path/to/logs"
sys.exit(1)
interval = get_config().get("filterhitstats", "interval")
def read_update(f):
return geometrical_mean.update(interval, read_data(f))
if sys.argv[1].endswith(".log"):
sql = read_update(sys.argv[1])
else:
sql = itertools.chain.from_iterable(itertools.imap(read_update,
log_files(sys.argv[1])))
db_connection = db.connect()
try:
db.write(db_connection, sql)
except:
logging.error("Failed to process file %s, all changes rolled back." % _last_log_file)
raise
finally:
db_connection.close()
def db(self):
if (not self._db):
self._db = db.connect()
self._clear_database()
return self._db
