def specify(manager, options): """Return the list of operations and expected output given the input catalogue.""" # Require many local variables here # pylint: disable=too-many-locals # parse input option list parser = argparse.ArgumentParser() parser.add_argument('--start', required=True) parser.add_argument('--end', required=True) args = parser.parse_args(options) # process a whole number of days startday = int(days_since_epoch(datetime_numeric.parse(args.start))) endday = int(days_since_epoch(datetime_numeric.parse(args.end))) # find items in catalogue grouped by day lstinput = manager.references_groupbyday(INPUTNAME, subsetindex=0) auxinput = manager.references_groupbyday(INPUTNAME, subsetindex=1) # the main output dataset dataset = manager.newdataset() # list of subsets (one for each command pattern) subsets = [dataset.newsubset([spec.outputpattern]) for spec in SUBSETSPECS] # iterate over subsets for subsetindex, subset in enumerate(subsets): # process each day for dayindex in range(startday, endday + 1): # convert to datetime object for formatting etc day = epoch_plus_days(dayindex) # loop over LST files for this day inputs = [] for lstreference in lstinput[dayindex]: # find AUX file matching the LST file lsttime = manager.match(lstreference).time print 'day {0} time {1}'.format(dayindex, lsttime) auxreference = next( reference for reference in auxinput[dayindex] if manager.match(reference).time == lsttime) # append pair inputs.extend([lstreference, auxreference]) # build output filename outputs = [subset.newfiletime(day)] # Append this operation to make it dataset.newoperation(inputs, outputs, SUBSETSPECS[subsetindex])
def time_at_index(self, operationindex): """Compute the mid-point of given month number since epoch and return as datetime object. This is needed because HadCRUT4 uses midmonth values on the time axis.""" year = datetime_numeric.parse(self.epoch).year total = 12 * year + operationindex t0 = datetime(total / 12, 1 + (total % 12), 1) t1 = datetime((total + 1) / 12, 1 + ((total + 1) % 12), 1) halfmonth = (t1 - t0) / 2 return t0 + halfmonth
def operation_indices_for_input_entry(self, entry): """Returns a list of operation indices requiring the given input.""" if entry.time: epochtime = datetime_numeric.parse(self.epoch) return [(12 * (entry.time.year - epochtime.year) + (entry.time.month - epochtime.month))] else: # Non-time references assumed to apply at all steps return range(self.count())
def parsetime(t): """Utility to allow convenience of strings for indicating start and end rather than datetime objects.""" if isinstance(t, datetime): return t elif isinstance(t, str): return datetime_numeric.parse(t) else: raise ValueError('Input parameter to StepAnnual is of unexpected type - should be datetime object or string')
def load_object(obj): """JSON conversion method for reading catalogue from file. If there is module and class metadata, recreate the class instance. """ if isinstance(obj, dict) and (CLASSID in obj): # get type of new class ready to build and split into module and class names [modulename, classname] = obj[CLASSID].rsplit('.', 1) if (modulename == 'datetime') and (classname == 'datetime'): # special case for datetime objects return datetime_numeric.parse(obj[DATETIMEVALUEKEY]) else: # retrieve class type to build class_to_make = getattr(importlib.import_module(modulename), classname) # build parameter dictionary and recurse if needed parameters = dict((k, load_object(v)) for k, v in obj.iteritems()) # remove the unwanted metadata del parameters[CLASSID] # build it return class_to_make(**parameters) elif isinstance(obj, list): # recurse into list too return [load_object(v) for v in obj] elif isinstance(obj, unicode): # ensure local encoding is used return str(obj) else: # not dealing with a class type - just return it return obj