def getsinglefilelist(self,
start_time,
end_time,
searchstring,
login=True,
subdir=None):
if subdir is not None:
fullurlpath = self.baseurl + '/' + subdir
else:
fullurlpath = self.baseurl
log.info('')
log.info('fullurlpath: ' + str(fullurlpath) + '/' + str(searchstring))
indexhtmlfile = DataFileName()
indexhtmlfile.satname = 'wgetsite'
indexhtmlfile.sensorname = self.data_type
indexhtmlfile.dataprovider = self.host_type
# Hmm, FileName object should probably set this when datetime is set?
dt = datetime.utcnow()
indexhtmlfile.time = dt.strftime(indexhtmlfile.datetime_fields['time'])
indexhtmlfile.date = dt.strftime(indexhtmlfile.datetime_fields['date'])
indexhtmlfile.extra = 'index'
indexhtmlfile.ext = 'html'
indexhtmlfile = indexhtmlfile.create_scratchfile()
indexhtmlfile.makedirs()
indexhtmlfnstr = indexhtmlfile.name
log.info(indexhtmlfnstr)
# rcj 13DEC2018 this part doesn't need to run for lance modis data
# it is handled in lance_modis.py getfilelist
if hasattr(self,
'host') and (self.host == 'nrt3.modaps.eosdis.nasa.gov'
or self.host == 'nrt4.modaps.eosdis.nasa.gov'):
pass
#everything that is not lance modis that uses this script should still pass through here
else:
htmlfilelist = open(self.wget_file(fullurlpath,
indexhtmlfnstr)).readlines()
#getfiles = self.getLinksFromHTML(htmlfilelist,r'''.*a href="GAASP-MBT_v"."r"."GW"."_s[0-9]{14}".*''')
#log.info(htmlfilelist)
links = self.getLinksFromHTML(htmlfilelist, searchstring)
# This is defined in Site.py - finding the files in the file list is
# common between HTTP and FTP (getting the lists differs, but sorting
# through the list and returning the desired files is shared)
return self.find_files_in_range(links,
start_time,
end_time,
urlpath=fullurlpath)
def getfile(self, remotefile, localfile):
processingfile = localfile + '.processing'
ff = open(processingfile, 'w')
ff.close
log.info('Touching temporary file: ' +
os.path.basename(processingfile))
temp_filename = DataFileName(localfile).create_scratchfile()
temp_filename.makedirs()
temp_fnstr = temp_filename.name
if not self.downloadactive:
log.info(
' *** nodownload set, not downloading remotefile %s ' %
remotefile)
log.info(' *** to localfile %s' % localfile)
else:
log.info(' *** grabbing remotefile %s ' % remotefile)
log.info(' *** to localfile %s' % localfile)
self.wget_file(remotefile, temp_fnstr)
self.move_to_final(temp_fnstr, processingfile, localfile)
