def mainLogSummary(config, dtstring):
"""
SOLR date format: 2013-07-17T00:00:00Z
NOTE: this method was written in a rush to get some reporting information.
It needs to be refactored to tidy things up a bit.
"""
MJD_D1_START = 56109.0
envstate = EnvironmentState(config["baseurl"])
dates = []
mjdstart = None
mjdend = mjd.now()
if dtstring == "ALL":
# special case - bootstrap dataset by downloading records for every day
# since the start of DataONE
mjdstart = MJD_D1_START
if not dtstring is None:
if mjdstart is None:
mjdstart = float(dtstring)
logging.warn("Starting download of complete log record history...")
mjdnow = mjd.now()
for t in xrange(int(mjdstart), int(mjdend), 1):
dates.append(t * 1.0)
dates.append(mjdnow)
# print out the header
row = ['"MJD"']
for event in EnvironmentState.LOG_EVENTS:
row.append('"' + event[0] + '"')
print (",".join(row))
if dtstring is None:
dates = [mjd.now()]
exclude_cns = "-ipAddress:({0})".format(" OR ".join(EnvironmentState.CN_IP_ADDRESSES))
logging.debug("DATES = %s" % str(dates))
for day in dates:
dtstring = mjd.MJD2dateTime(day).strftime("%Y-%m-%dT00:00:00Z")
fq = "dateLogged:[{0}-2DAY TO {0}-1DAY]".format(dtstring)
row = ["{0:.5f}".format(day)]
for event in EnvironmentState.LOG_EVENTS:
if event[0].endswith(".ext"):
ev = event[0].split(".")[0]
q = "event:{0} AND {1}".format(ev, exclude_cns)
else:
q = "event:{0}".format(event[0])
logging.debug("Q = %s" % q)
logging.debug("FQ = %s" % fq)
nrecords = envstate.retrieveLogResponse(q, fq=fq)
row.append(str(nrecords))
print ",".join(row)
logging.info(",".join(row))
logging.info("Done.")
def main(config):
mkdir_p(config["statefolder"])
# load index JSON
state_index_src = os.path.join(config["statefolder"], "index.js")
logging.debug("INDEX = " + state_index_src)
state_index = []
try:
jstr = file(state_index_src, "r").read()
state_index = json.loads(jstr[len(EnvironmentState.JS_VARIABLE_INDEX) :])
except:
logging.info("No index file available. Creating at {0:s}".format(state_index_src))
# capture state
envstate = EnvironmentState(config["baseurl"])
envstate.populateState()
# append index entry
row = [envstate.getTStamp(), envstate.tstamp.strftime(config["stateformat"])]
logging.debug(pprint.pformat(row))
state_index.append(row)
# Write out the state JSON file
statedest = file(os.path.join(config["statefolder"], row[1]), "w")
envstate.asJSON(statedest)
statedest.close()
# update the index JSON file
jbuffer = file(state_index_src, "w")
jbuffer.write(EnvironmentState.JS_VARIABLE_INDEX)
json.dump(state_index, jbuffer, indent=2)
jbuffer.close()
