def calcAverageOfDayNight_filter(dayDir, nightDir, outDir, patterns, logger=None):
    # find file from pattern
    day_pattern = patterns[0]
    night_pattern = patterns[1]
    average_pattern = patterns[2]
    day_list = directoryUtils.getMatchingFiles(base_dir=dayDir, filter=day_pattern)
    night_list = directoryUtils.getMatchingFiles(base_dir=nightDir, filter=night_pattern)
    if logger:
        logger.debug("Day files: %s", day_list)
        logger.debug("Night files: %s", night_list)

    for fl in day_list:
        # find matching night file
        # find date stamp in day file
        _r_in = re.compile(day_pattern)
        _r_in_night = re.compile(night_pattern)
        f = os.path.basename(fl)
        _m = _r_in.match(f)
        year = 0
        dayofyear = 0
        if 'year' in _m.groupdict():
            year = _m.groupdict()['year']
        if 'dayofyear' in _m.groupdict():
            dayofyear = _m.groupdict()['dayofyear']

        for nfl in night_list:
            _mn = _r_in_night.match(os.path.basename(nfl))
            if 'year' in _mn.groupdict() and _mn.groupdict()['year'] == year:
                # matched year
                if 'dayofyear' in _mn.groupdict() and _mn.groupdict()['dayofyear'] == dayofyear:
                    # matched day of year
                    avg_fl = filenameUtils.generateOutputFilename(os.path.basename(fl), day_pattern, average_pattern)
                    calcAverageOfDayNight(fl, nfl, os.path.join(outDir, avg_fl))
                    break
    return 0
def mosaicTiles(files, output_dir, tools_dir="", overwrite = False, subset=[1,1,0,0,0,0,0,0,0,0,0], ofmt='HDF4Image',
                gdal=False, logger = None):
    # use MRTools
    if tools_dir:
        mrtpath = tools_dir
    else:
        mrtpath = "c:\\MODIS\\bin\\"
    filelist = os.path.join(output_dir, "file_list.txt")
    writeMosaicList(filelist, files)
    new_filename = filenameUtils.generateOutputFilename(os.path.basename(files[0]),
                                                        modis_patterns['mosaic_in'],
                                                        modis_patterns['mosaic_out'], False)

    if not os.path.exists(os.path.normpath(os.path.join(output_dir, new_filename))) or overwrite == True:
        mosaicFiles(os.path.normpath(filelist), os.path.normpath(os.path.join(output_dir, new_filename)), os.path.normpath(mrtpath))

        # reproject mosaic to GEO projection
        param_file = os.path.join(output_dir, os.path.basename(new_filename) + ".prm")
        outputname = os.path.join(output_dir, '{0}_r{1}'.format(os.path.basename(new_filename), os.path.splitext(new_filename)[1]))
        generateParamFile(output_dir, param_file, os.path.normpath(os.path.join(output_dir, new_filename)), outputname)
        reprojectMosaic(param_file, os.path.normpath(mrtpath))
        # remove temp file
        os.remove(os.path.normpath(os.path.join(output_dir, new_filename)))
        os.rename(outputname, os.path.normpath(os.path.join(output_dir, new_filename)))
        os.remove(param_file)

    os.remove(filelist)
    if logger: logger.debug("finished mosaic")
    return new_filename
def extractSubset(base_path, output_path, tools_path,
                  patterns, subset, subset_name, overwrite = False, nodata=None, logger = None):
    _all_files = directoryUtils.getMatchingFiles(base_path, patterns[0])
    if not _all_files:
        print 'No files found in ' + base_path + ', please check directory and try again'
        return -1
#    _pfl = os.path.join(base_path, 'fileslist' + ".prm")
    new_files = []
#    _params = [subset]
    for _ifl in _all_files:
        # generate parameter file
        _nfl = filenameUtils.generateOutputFilename(os.path.basename(_ifl), patterns[0], patterns[1])
        _ofl = os.path.join(output_path, _nfl)
        _checkfl = "{0}.{1}{2}".format(os.path.splitext(_ofl)[0], subset_name, os.path.splitext(_ofl)[1])
        if not os.path.exists(_checkfl) or overwrite == True:
            try:
                src_ds = gdal.Open(_ifl)
            except RuntimeError, e:
                if logger: logger.debug('Unable to open file')
                return None
            sds = src_ds.GetSubDatasets()
            if logger: logger.debug("Number of bands: %s",src_ds.RasterCount)
#            generateParamFile(base_path, _pfl, _ifl, _ofl, _params)
#            reprojectMosaic(_pfl, tools_path)
            convertToTiff(_ifl, _ofl, tools_path, nodata)
#            ss = [1]
            for idx, sbs in enumerate(sds): #range(src_ds.RasterCount):
                if logger: logger.debug("Subdataset: %s", sbs[0])
                # get subset name (without spaces)
                _n = (sbs[0].rsplit(':', 1)[1]).replace(' ', '_')
#                _n = _n.replace(' ', '_')
                _rf = "{0}.{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], _n, os.path.splitext(_ofl)[1])
#                _cf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(idx+1).zfill(2), os.path.splitext(_ofl)[1])
                # GDAL 1.10
                _cf = "{0}{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], os.path.splitext(_ofl)[1], str(idx+1).zfill(1))
                if not os.path.exists(os.path.join(output_path, _cf)):
                    _cf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(idx+1), os.path.splitext(_ofl)[1])
                if idx+1 not in subset:
                    # remove un-needed files (including .aux & .aux.xml)
                    os.remove(os.path.join(output_path, _cf))
                    _aux_f = os.path.join(output_path,"{0}.aux.xml".format(_cf))
                    if os.path.exists(_aux_f):
                        os.remove(_aux_f)
#            for i in range(2,18):
#                rf = "{0}_{1}{2}".format(os.path.splitext(os.path.basename(_ofl))[0], str(i).zfill(2), os.path.splitext(_ofl)[1])
#                os.remove(os.path.join(output_path, rf))
                else:
                    # keep this file - rename with subset name
                    os.rename(os.path.join(output_path, _cf), os.path.join(output_path, _rf))
                    _aux_f = os.path.join(output_path,"{0}.aux.xml".format(_cf))
                    if os.path.exists(_aux_f):
                        os.rename(_aux_f, os.path.join(output_path, "{0}.aux.xml".format(_rf)))
                    new_files.append(_rf)
def mosaicTiles(files, output_dir, tools_dir="", overwrite = False, subset=[1,1,0,0,0,0,0,0,0,0,0], ofmt='HDF4Image',
                gdal=False, logger = None):
#    from pymodis.convertmodis_gdal import createMosaicGDAL
#    from pymodis.convertmodis import createMosaic
#    # NDVI, EVI
#    if gdal:
#        mosaic = createMosaicGDAL(hdfnames=files, subset=subset, outformat=ofmt)
#        ofn = os.path.basename(output_prefix + 'mosaic.hdf')
#        output_file = os.path.join(output_dir, ofn)
#        print "mosaic ", files, ", output: ", output_file
#        mosaic.run(output_file)
#    else:
    # use MRTools
    if tools_dir:
        mrtpath = tools_dir
    else:
        mrtpath = "c:\\MODIS\\bin\\"
    filelist = os.path.join(output_dir, "file_list.txt")
    writeMosaicList(filelist, files)
    # try:
    #     pfile = open(filelist, 'w')
    # except IOError as e:
    #     if e.errno == errno.EACCES:
    #         return "Couldn't write list of files"
    #     # Not a permission error.
    #     raise
    # else:
    #     with pfile:
    #         for f in files:
    #             pfile.write('"' + f + '"\n')
    #         pfile.close()
    new_filename = filenameUtils.generateOutputFilename(os.path.basename(files[0]), modis_patterns['mosaic_in'], modis_patterns['mosaic_out'])
    if not os.path.exists(os.path.normpath(os.path.join(output_dir, new_filename))) or overwrite == True:
        mosaicFiles(os.path.normpath(filelist), os.path.normpath(os.path.join(output_dir, new_filename)), os.path.normpath(mrtpath))
#    mosaicFiles(os.path.normpath(filelist), os.path.normpath(os.path.join(output_dir, output_prefix + 'mosaic.hdf')), os.path.normpath(mrtpath))
#        mosaic = createMosaic(filelist, output_prefix + 'mosaic', mrtpath, [1,1,0,0,0,0,0,0,0,0,0])
#        orig_dir = os.getcwd()
#        print "original directory: ", orig_dir
#        os.chdir(output_dir)
#        mosaic.run()
    os.remove(filelist)
#        os.chdir(orig_dir)
    if logger: logger.debug("finished mosaic")
    return new_filename