Exemplo n.º 1
0
def mainLogSummary(config, dtstring):
    """
  SOLR date format: 2013-07-17T00:00:00Z
  
  NOTE: this method was written in a rush to get some reporting information.
  It needs to be refactored to tidy things up a bit.
  """
    MJD_D1_START = 56109.0
    envstate = EnvironmentState(config["baseurl"])
    dates = []
    mjdstart = None
    mjdend = mjd.now()
    if dtstring == "ALL":
        # special case - bootstrap dataset by downloading records for every day
        # since the start of DataONE
        mjdstart = MJD_D1_START

    if not dtstring is None:
        if mjdstart is None:
            mjdstart = float(dtstring)
        logging.warn("Starting download of complete log record history...")
        mjdnow = mjd.now()
        for t in xrange(int(mjdstart), int(mjdend), 1):
            dates.append(t * 1.0)
        dates.append(mjdnow)
        # print out the header
        row = ['"MJD"']
        for event in EnvironmentState.LOG_EVENTS:
            row.append('"' + event[0] + '"')
        print (",".join(row))
    if dtstring is None:
        dates = [mjd.now()]
    exclude_cns = "-ipAddress:({0})".format(" OR ".join(EnvironmentState.CN_IP_ADDRESSES))
    logging.debug("DATES = %s" % str(dates))
    for day in dates:
        dtstring = mjd.MJD2dateTime(day).strftime("%Y-%m-%dT00:00:00Z")
        fq = "dateLogged:[{0}-2DAY TO {0}-1DAY]".format(dtstring)
        row = ["{0:.5f}".format(day)]
        for event in EnvironmentState.LOG_EVENTS:
            if event[0].endswith(".ext"):
                ev = event[0].split(".")[0]
                q = "event:{0} AND {1}".format(ev, exclude_cns)
            else:
                q = "event:{0}".format(event[0])
            logging.debug("Q  = %s" % q)
            logging.debug("FQ = %s" % fq)
            nrecords = envstate.retrieveLogResponse(q, fq=fq)
            row.append(str(nrecords))
        print ",".join(row)
        logging.info(",".join(row))
    logging.info("Done.")
Exemplo n.º 2
0
def main(config):
    mkdir_p(config["statefolder"])
    # load index JSON
    state_index_src = os.path.join(config["statefolder"], "index.js")
    logging.debug("INDEX = " + state_index_src)
    state_index = []
    try:
        jstr = file(state_index_src, "r").read()
        state_index = json.loads(jstr[len(EnvironmentState.JS_VARIABLE_INDEX) :])
    except:
        logging.info("No index file available. Creating at {0:s}".format(state_index_src))

    # capture state
    envstate = EnvironmentState(config["baseurl"])
    envstate.populateState()
    # append index entry
    row = [envstate.getTStamp(), envstate.tstamp.strftime(config["stateformat"])]
    logging.debug(pprint.pformat(row))
    state_index.append(row)
    # Write out the state JSON file
    statedest = file(os.path.join(config["statefolder"], row[1]), "w")
    envstate.asJSON(statedest)
    statedest.close()
    # update the index JSON file
    jbuffer = file(state_index_src, "w")
    jbuffer.write(EnvironmentState.JS_VARIABLE_INDEX)
    json.dump(state_index, jbuffer, indent=2)
    jbuffer.close()