def mainLogSummary(config, dtstring): """ SOLR date format: 2013-07-17T00:00:00Z NOTE: this method was written in a rush to get some reporting information. It needs to be refactored to tidy things up a bit. """ MJD_D1_START = 56109.0 envstate = EnvironmentState(config["baseurl"]) dates = [] mjdstart = None mjdend = mjd.now() if dtstring == "ALL": # special case - bootstrap dataset by downloading records for every day # since the start of DataONE mjdstart = MJD_D1_START if not dtstring is None: if mjdstart is None: mjdstart = float(dtstring) logging.warn("Starting download of complete log record history...") mjdnow = mjd.now() for t in xrange(int(mjdstart), int(mjdend), 1): dates.append(t * 1.0) dates.append(mjdnow) # print out the header row = ['"MJD"'] for event in EnvironmentState.LOG_EVENTS: row.append('"' + event[0] + '"') print (",".join(row)) if dtstring is None: dates = [mjd.now()] exclude_cns = "-ipAddress:({0})".format(" OR ".join(EnvironmentState.CN_IP_ADDRESSES)) logging.debug("DATES = %s" % str(dates)) for day in dates: dtstring = mjd.MJD2dateTime(day).strftime("%Y-%m-%dT00:00:00Z") fq = "dateLogged:[{0}-2DAY TO {0}-1DAY]".format(dtstring) row = ["{0:.5f}".format(day)] for event in EnvironmentState.LOG_EVENTS: if event[0].endswith(".ext"): ev = event[0].split(".")[0] q = "event:{0} AND {1}".format(ev, exclude_cns) else: q = "event:{0}".format(event[0]) logging.debug("Q = %s" % q) logging.debug("FQ = %s" % fq) nrecords = envstate.retrieveLogResponse(q, fq=fq) row.append(str(nrecords)) print ",".join(row) logging.info(",".join(row)) logging.info("Done.")
def main(config): mkdir_p(config["statefolder"]) # load index JSON state_index_src = os.path.join(config["statefolder"], "index.js") logging.debug("INDEX = " + state_index_src) state_index = [] try: jstr = file(state_index_src, "r").read() state_index = json.loads(jstr[len(EnvironmentState.JS_VARIABLE_INDEX) :]) except: logging.info("No index file available. Creating at {0:s}".format(state_index_src)) # capture state envstate = EnvironmentState(config["baseurl"]) envstate.populateState() # append index entry row = [envstate.getTStamp(), envstate.tstamp.strftime(config["stateformat"])] logging.debug(pprint.pformat(row)) state_index.append(row) # Write out the state JSON file statedest = file(os.path.join(config["statefolder"], row[1]), "w") envstate.asJSON(statedest) statedest.close() # update the index JSON file jbuffer = file(state_index_src, "w") jbuffer.write(EnvironmentState.JS_VARIABLE_INDEX) json.dump(state_index, jbuffer, indent=2) jbuffer.close()