old_subDir = "junk"
last_timestamp = -1
while 1:
sub_directory = getCurrentJulianday()
if sub_directory != old_subDir:
ftp_dir = path.join (opt_ftp_dir, sub_directory)
print ftp_dir
ftp_info = ftptools.FTP_info(opt_ftp_server, ftp_dir, opt_ftp_user, opt_ftp_passwd, opt_log)
pmu.auto_register("getting ftp file listing.")
got_list = 0
tries = 0
while not got_list:
tries = tries + 1
got_list = 1
try:
signal.alarm(100)
ftp_info.get_listing()
signal.alarm(0)
except:
print "ftp date error: %s, %s" % (sys.exc_type, sys.exc_value)
got_list = 0
if tries > opt_max_num_tries:
print "Exiting after ", opt_max_num_tries, " tries"
need_sleep = 0
dot_string = '.'
if opt_noDot:
dot_string = ''
request_string = '*' + dot_string + opt_source_ext
if opt_debug:
print "Request_string: %s" % request_string
file_time_threshold = 0
while 1:
pmu.auto_register("To sleep")
if need_sleep:
time.sleep(opt_sleep_secs)
need_sleep = 1
pmu.auto_register("downloading data")
print "+++++++++++++ starting at ", strftime("%a, %d %b %Y %H:%M:%S +0000", gmtime())
#
# if url specified, download .nc data files from the
# given url using wget.
# otherwise, download data file (not necesarily .nc type)
# using ftp
#
#
if not opt_run_once:
pmu.auto_init(prog_name, opt_instance, 60)
#
# set last_time to two hours before now to prevent unecessary searching
#
last_file_time = time() - 7200
debug_print('last_file_time = ' + asctime(gmtime(last_file_time)))
while 1:
url_info = 'http://' + opt_web_server + separator + opt_url_path
if not opt_run_once:
pmu.auto_register('getting file listing from ' + url_info)
index_page = fetch_page(url_info)
print 'made it here.'
print str(index_page)
file_list = parse_index_page(index_page)
print str(file_list)
#
# reverse list -- now in descending order
#
filename = ''
def pull_file(filename):
pmu.auto_register("Checking for new files")
#
# Construct the local filenames. The temporary filename is used
# for pulling the file down. When the retrieval is complete, the
# file is renamed to its original name so that it appears atomically.
#
temp_filename = opt_temp_dir + '/.' + filename
local_filename = opt_output_dir + '/' + filename
if opt_debug:
print "local_filename = " + local_filename
print "temp_filename = " + temp_filename
#
# Check to see if we already have this file. If we do, then skip
# it.
#
local_gz_filename = local_filename + ".gz"
#
# this is the case of the file that is gzipped on the ftp site
# but unzipped locally
#
if (os.path.splitext(local_filename)[1] == ".gz"):
if os.path.exists(os.path.splitext(local_filename)[0]):
if opt_debug:
print
print "File already exists locally: " + local_filename
print "Skipping retrieval...."
return
if os.path.exists(local_filename) or os.path.exists(local_gz_filename):
if opt_debug:
print
print "File already exists locally: " + local_filename
print "Skipping retrieval...."
return
if opt_debug:
print "Getting file <" + filename + ">"
#
# Wait for the file to be quiescent
#
if opt_wait_for_quiescence:
prev_file_size = 0
file_size = ftp.size(filename)
while file_size != prev_file_size:
if opt_debug:
print "Waiting for file quiescence..."
time.sleep(opt_quiescence_secs)
prev_file_size = file_size
file_size = ftp.size(filename)
#
# Pull the data file to the destination directory.
#
temp_file = open(temp_filename, 'wb')
pmu.force_register("Retrieving file " + filename)
try:
ftp.retrbinary('RETR ' + filename, temp_file.write)
except (socket.error):
print "Socket error occurred while trying to transfer file: ", filename
print "Skipping file!"
return
temp_file.close()
if (os.path.splitext(temp_filename)[1] == ".gz"):
filename = os.path.splitext(filename)[0]
os.system("gunzip " + temp_filename)
temp_filename = os.path.splitext(temp_filename)[0]
local_filename = os.path.splitext(local_filename)[0]
try:
shutil.copyfile(temp_filename, local_filename)
except (OSError):
print "Error Copying temp file to local file\n"
if (opt_write_Ldata):
file_stats = os.stat(local_filename)
data_file_time = datetime.fromtimestamp(file_stats[8])
Ldata_command = "LdataWriter -dir " + opt_output_dir + " -ext " + opt_Ldata_ext + " -dtype " + opt_Ldata_dtype + " -ltime " + data_file_time.strftime(
"%Y%m%d%H%M%S") + " -rpath ./" + filename
print
print "LdataWriter command line:"
print Ldata_command
print
os.system(Ldata_command)
try:
os.remove(temp_filename)
except (OSError):
print "Error removing temp file\n"
return
#
if not opt_run_once:
pmu.auto_init(prog_name, opt_instance, 60)
#
# set last_time to two hours before now to prevent unecessary searching
#
last_file_time = time() - 7200
debug_print('last_file_time = ' + asctime(gmtime(last_file_time)))
while 1:
url_info = 'http://' + opt_web_server + separator + opt_url_path
if not opt_run_once:
pmu.auto_register('getting file listing from ' + url_info)
index_page = fetch_page(url_info)
print 'made it here.'
print str(index_page)
file_list = parse_index_page(index_page)
print str(file_list)
#
# reverse list -- now in descending order
#
filename = ''
#
need_sleep = 0
dot_string = '.'
if opt_noDot:
dot_string = ''
request_string = '*' + dot_string + opt_source_ext
if opt_debug:
print "Request_string: %s" % request_string
while 1:
pmu.auto_register("Waiting for data")
if need_sleep:
sleep_secs_left = opt_sleep_secs
while sleep_secs_left > 0:
pmu.auto_register("Waiting for data")
time.sleep(2)
sleep_secs_left = sleep_secs_left - 2
need_sleep = 1
#
# Open the ftp connection
#
ftp = ftplib.FTP(opt_source_host, opt_source_user, opt_source_pwd)
#
need_sleep = 0
dot_string = '.'
if opt_noDot:
dot_string = ''
request_string = '*' + dot_string + opt_source_ext
if opt_debug:
print "Request_string: %s" % request_string
while 1:
pmu.auto_register("To sleep")
if need_sleep:
time.sleep(opt_sleep_secs)
need_sleep = 1
pmu.auto_register("downloading data")
#
# if url specified, download .nc data files from the
# given url using wget.
# otherwise, download data file (not necesarily .nc type)
# using ftp
#
download_url()
old_subDir = "junk"
last_timestamp = -1
while 1:
sub_directory = getCurrentJulianday()
if sub_directory != old_subDir:
ftp_dir = path.join(opt_ftp_dir, sub_directory)
print ftp_dir
ftp_info = ftptools.FTP_info(opt_ftp_server, ftp_dir, opt_ftp_user,
opt_ftp_passwd, opt_log)
pmu.auto_register("getting ftp file listing.")
got_list = 0
tries = 0
while not got_list:
tries = tries + 1
got_list = 1
try:
signal.alarm(100)
ftp_info.get_listing()
signal.alarm(0)
except:
print "ftp date error: %s, %s" % (sys.exc_type, sys.exc_value)
got_list = 0
if tries > opt_max_num_tries:
print "Exiting after ", opt_max_num_tries, " tries"
#
# Initialize process mapper registration
#
pmu.auto_init(prog_name, opt_instance, opt_register_interval)
#
# Wait for new files. When found, push them to the destination.
#
last_dir_mod_time = os.stat(opt_input_dir)[stat.ST_MTIME]
need_sleep = 0
while 1:
pmu.auto_register("Waiting for data")
if opt_debug:
print "Waiting for data"
if need_sleep:
time.sleep(opt_sleep_secs)
need_sleep = 1
#
# Check for new files
#
dir_mod_time = os.stat(opt_input_dir)[stat.ST_MTIME]
if dir_mod_time <= last_dir_mod_time:
continue
def pull_file(filename):
pmu.auto_register("Checking for new files")
#
# Construct the local filenames. The temporary filename is used
# for pulling the file down. When the retrieval is complete, the
# file is renamed to its original name so that it appears atomically.
#
temp_filename = opt_temp_dir + '/.' + filename
local_filename = opt_output_dir + '/' + filename
if opt_debug:
print "local_filename = " + local_filename
print "temp_filename = " + temp_filename
#
# Check to see if we already have this file. If we do, then skip
# it.
#
local_gz_filename = local_filename + ".gz"
#
# this is the case of the file that is gzipped on the ftp site
# but unzipped locally
#
if(os.path.splitext(local_filename)[1] == ".gz"):
if os.path.exists( os.path.splitext(local_filename)[0] ):
if opt_debug:
print
print "File already exists locally: " + local_filename
print "Skipping retrieval...."
return
if os.path.exists(local_filename) or os.path.exists(local_gz_filename):
if opt_debug:
print
print "File already exists locally: " + local_filename
print "Skipping retrieval...."
return
if opt_debug:
print "Getting file <" + filename + ">"
#
# Wait for the file to be quiescent
#
if opt_wait_for_quiescence:
prev_file_size = 0
file_size = ftp.size(filename)
while file_size != prev_file_size:
if opt_debug:
print "Waiting for file quiescence..."
time.sleep(opt_quiescence_secs)
prev_file_size = file_size
file_size = ftp.size(filename)
#
# Pull the data file to the destination directory.
#
temp_file = open(temp_filename, 'wb')
pmu.force_register("Retrieving file " + filename)
try:
ftp.retrbinary('RETR ' + filename, temp_file.write)
except (socket.error):
print "Socket error occurred while trying to transfer file: ", filename
print "Skipping file!"
return
temp_file.close()
if(os.path.splitext(temp_filename)[1] == ".gz"):
filename = os.path.splitext(filename)[0]
os.system("gunzip " + temp_filename)
temp_filename = os.path.splitext(temp_filename)[0]
local_filename = os.path.splitext(local_filename)[0]
try:
shutil.copyfile(temp_filename, local_filename)
except(OSError):
print "Error Copying temp file to local file\n"
if(opt_write_Ldata):
file_stats = os.stat(local_filename)
data_file_time = datetime.fromtimestamp(file_stats[8])
Ldata_command = "LdataWriter -dir " + opt_output_dir + " -ext " + opt_Ldata_ext + " -dtype " + opt_Ldata_dtype + " -ltime " + data_file_time.strftime("%Y%m%d%H%M%S") + " -rpath ./" + filename
print
print "LdataWriter command line:"
print Ldata_command
print
os.system(Ldata_command)
try:
os.remove(temp_filename)
except(OSError):
print "Error removing temp file\n"
return