def getMonthlyDataFromFTP(localPath, startDate = None, endDate = None): monthlyDir = 'pub/org/chg/products/CHIRPS-2.0/global_monthly/tifs/' if not startDate: # get all files getFilesFromFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@', monthlyDir, localPath) elif not endDate: # get all months AFTER startDate ftp = openFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@') ftp.cwd(monthlyDir) filesList = ftp.nlst() ftp.close() transferList = [] for f in filesList: y = int(getCHIRPSYear(f)) m = int(getCHIRPSMonth(f)) file_date = date(y, m, 1) if (file_date >= startDate): transferList.append(f) getFilesFromFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@', monthlyDir, localPath, False, transferList) else: # get all months between startDate and endDate (inclusive) ftp = openFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@') ftp.cwd(monthlyDir) filesList = ftp.nlst() ftp.close() transferList = [] for f in filesList: y = getCHIRPSYear(f) m = getCHIRPSMonth(f) file_date = date(y, m, 1) if (file_date >= startDate) and (file_date <= endDate): transferList.append(f) getFilesFromFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@', monthlyDir, localPath, False, transferList) return 0
def getDailyDataFromFTP(localPath, datesList=[]): dailyDir = 'pub/org/chg/products/CHIRPS-2.0/global_daily/tifs/p25/' chirpsBaseName = 'chirps-v2.0.' chirpsExt = '.tif.gz' # if have dates, only get those files if not datesList: # get all files not already in localPath getFilesFromFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@', dailyDir, localPath) else: sd = datesList[0] ed = datesList[1] ftp = openFTP(ftp_address_CHIRPS) while sd <= ed: yrs = '{0}'.format(sd.year) yrDir = path.join(dailyDir, yrs) fname = sd.strftime("chirps-v2.0.%Y.%m.%d.tif.gz") logger.debug('Looking for: %s in %s' % fname, yrDir) localFile = path.join(localPath, fname) # try to find file in directory fn = yrDir + '/' + fname # Download file getFileFromFTP(ftp, fn, localFile, False) sd = sd + timedelta(days=1) ftp.close() return 0
def getCHIRPSData(address, userName, passWord, remotePath, localPath, years, onlyDiff=True): ftp = openFTP(address) for i, val in enumerate(years): ftp.cwd(val) localPathYear = path.join(localPath, val) print("local path for ", val, ": ", localPathYear) if onlyDiff: lFileSet = set(listdir(localPathYear)) rFileSet = set(ftp.nlst()) transferList = list(rFileSet - lFileSet) print "Missing: " + str(len(transferList)) else: transferList = ftp.nlst() print "File list: ", transferList filesMoved = 0 for fl in transferList: # create a full local filepath localFile = path.join(localPathYear, fl) print "new file: ", localFile grabFile = True if grabFile: getFileFromFTP(ftp, fl, localFile, False) filesMoved += 1 print(transferList) ftp.cwd('..') closeFTP(ftp) return 0