def getMonthlyDataFromFTP(localPath, startDate = None, endDate = None):
monthlyDir = 'pub/org/chg/products/CHIRPS-2.0/global_monthly/tifs/'
if not startDate:
# get all files
getFilesFromFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@', monthlyDir, localPath)
elif not endDate:
# get all months AFTER startDate
ftp = openFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@')
ftp.cwd(monthlyDir)
filesList = ftp.nlst()
ftp.close()
transferList = []
for f in filesList:
y = int(getCHIRPSYear(f))
m = int(getCHIRPSMonth(f))
file_date = date(y, m, 1)
if (file_date >= startDate):
transferList.append(f)
getFilesFromFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@', monthlyDir, localPath, False, transferList)
else:
# get all months between startDate and endDate (inclusive)
ftp = openFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@')
ftp.cwd(monthlyDir)
filesList = ftp.nlst()
ftp.close()
transferList = []
for f in filesList:
y = getCHIRPSYear(f)
m = getCHIRPSMonth(f)
file_date = date(y, m, 1)
if (file_date >= startDate) and (file_date <= endDate):
transferList.append(f)
getFilesFromFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@', monthlyDir, localPath, False, transferList)
return 0
def getDailyDataFromFTP(localPath, datesList=[]):
dailyDir = 'pub/org/chg/products/CHIRPS-2.0/global_daily/tifs/p25/'
chirpsBaseName = 'chirps-v2.0.'
chirpsExt = '.tif.gz'
# if have dates, only get those files
if not datesList:
# get all files not already in localPath
getFilesFromFTP(ftp_address_CHIRPS, 'anonymous', 'anonymous@', dailyDir, localPath)
else:
sd = datesList[0]
ed = datesList[1]
ftp = openFTP(ftp_address_CHIRPS)
while sd <= ed:
yrs = '{0}'.format(sd.year)
yrDir = path.join(dailyDir, yrs)
fname = sd.strftime("chirps-v2.0.%Y.%m.%d.tif.gz")
logger.debug('Looking for: %s in %s' % fname, yrDir)
localFile = path.join(localPath, fname)
# try to find file in directory
fn = yrDir + '/' + fname
# Download file
getFileFromFTP(ftp, fn, localFile, False)
sd = sd + timedelta(days=1)
ftp.close()
return 0
def getCHIRPSData(address, userName, passWord, remotePath, localPath, years, onlyDiff=True):
ftp = openFTP(address)
for i, val in enumerate(years):
ftp.cwd(val)
localPathYear = path.join(localPath, val)
print("local path for ", val, ": ", localPathYear)
if onlyDiff:
lFileSet = set(listdir(localPathYear))
rFileSet = set(ftp.nlst())
transferList = list(rFileSet - lFileSet)
print "Missing: " + str(len(transferList))
else:
transferList = ftp.nlst()
print "File list: ", transferList
filesMoved = 0
for fl in transferList:
# create a full local filepath
localFile = path.join(localPathYear, fl)
print "new file: ", localFile
grabFile = True
if grabFile:
getFileFromFTP(ftp, fl, localFile, False)
filesMoved += 1
print(transferList)
ftp.cwd('..')
closeFTP(ftp)
return 0
