def get_zipped_file(filename, dirname, connection):
"""
Downloads the file filename+zip from the subdirectory dirname, reads into
tempfile.zip, cds back out to parent directory and unzips
Exits on error and raises RuntimeError
"""
fbin = open('tempfile.zip', 'wb')
try:
connection.cwd(dirname) # change into subdirectory
connection.retrbinary("RETR " + filename + '.zip', fbin.write)
connection.cwd('..') # back
utilities.logger.info('Successfully retrieved ' + filename + '.zip\n')
except:
utilities.do_RuntimeError('Retrieval of', filename,
'.zip unsuccessful')
return
fbin.close()
try:
# -o: overwrite without prompting
subprocess.call("unzip -o tempfile.zip", shell=True)
subprocess.call("rm tempfile.zip", shell=True) # clean up
except:
utilities.do_RuntimeError('Downloaded file', filename,
'could not be decompressed')
def get_zipped_file(filename, dirname, connection):
"""
Downloads the file filename+zip from the subdirectory dirname, reads into
tempfile.zip, cds back out to parent directory and unzips
Exits on error and raises RuntimeError
"""
fbin = open('tempfile.zip', 'wb')
try:
connection.cwd(dirname) # change into subdirectory
connection.retrbinary("RETR " + filename + '.zip', fbin.write)
connection.cwd('..') # back
utilities.logger.info('Successfully retrieved ' + filename + '.zip\n')
except:
utilities.do_RuntimeError('Retrieval of', filename,
'.zip unsuccessful')
return
fbin.close()
try:
# -o: overwrite without prompting
subprocess.call("unzip -o tempfile.zip", shell=True)
subprocess.call("rm tempfile.zip", shell=True) # clean up
except:
utilities.do_RuntimeError('Downloaded file', filename,
'could not be decompressed')
def store_zipped_file(filename, dirname, connection):
"""
Zips and uploads the file filename into the subdirectory dirname, then cd
back out to parent directory.
Exits on error and raises RuntimeError
"""
logger = logging.getLogger('pipeline_log')
filezip = filename + '.zip'
try:
with ZipFile(filezip, 'w') as f_zip:
f_zip.write(filename)
# change into subdirectory
connection.cwd(dirname)
connection.storbinary("STOR " + filezip, open(filezip))
# back out
connection.cwd('..')
except:
logger.warning('Store of {} unsuccessful'.format(filezip))
utilities.do_RuntimeError('Store of', filename, '.zip unsuccessful')
def store_zipped_file(filename, dirname, connection):
"""
Zips and uploads the file filename into the subdirectory dirname, then cd
back out to parent directory.
Exits on error and raises RuntimeError
"""
logger = logging.getLogger('pipeline_log')
filezip = filename + '.zip'
try:
with ZipFile(filezip, 'w') as f_zip:
f_zip.write(filename)
# change into subdirectory
connection.cwd(dirname)
connection.storbinary("STOR " + filezip, open(filezip))
# back out
connection.cwd('..')
except:
logger.warning('Store of {} unsuccessful'.format(filezip))
utilities.do_RuntimeError('Store of', filename, '.zip unsuccessful')
def main(datestr, server_info, file_info):
"""
When something goes amiss, various routines will and pass through a
RuntimeError(explanation) rather than trying to recover, since this
probably means something is either wrong with the ftp connection or the
file structure got corrupted. This error is logged but needs to be caught
in the calling program.
"""
logger = logging.getLogger('pipeline_log')
# log into the server
try:
ftp = FTP(server_info.serv_name) # connect to host, default port
ftp.login(server_info.username, server_info.password)
# change into PHOX directory
ftp.cwd(server_info.server_dir)
logger.info('Logged into: {}/{}'.format(server_info.serv_name,
server_info.server_dir))
print('Logged into: {}/{}'.format(server_info.serv_name,
server_info.server_dir))
except Exception as e:
logger.info('Login to {} unsuccessful.'.format(server_info.serv_name))
utilities.do_RuntimeError('Login to {} unsuccessful.'.format(server_info.serv_name))
# upload the daily event and duplicate index files
try:
eventfilename = '{}{}.txt'.format(file_info.eventfile_stem, datestr)
store_zipped_file(eventfilename, 'Daily', ftp)
except Exception as e:
filezip = eventfilename + '.zip'
logger.warning('Store of {} unsuccessful'.format(filezip))
utilities.do_RuntimeError('Transfer of', eventfilename,
'unsuccessful')
#We don't have these files right now.
# try:
# dupfilename = '{}{}.txt'.format(file_info.dupfile_stem, datestr)
# store_zipped_file(dupfilename, 'Daily/Duplicates', ftp)
# ftp.cwd('..') # back out one more level
# except:
# utilities.do_RuntimeError('Transfer of', dupfilename,
# 'unsuccessful')
#
# # update the monthly and yearly files
# monthfilename = file_info.outputfile_stem + \
# datestr[:2] + '-' + datestr[2:4] + '.txt'
# yearfilename = file_info.outputfile_stem + datestr[:2] + '.txt'
#
# curyear = True
# if datestr[2:] == '0101': # initialize a new year
# subprocess.call("cp " + eventfilename + ' ' + yearfilename, shell=True)
# curyear = False
# else:
# get_zipped_file(yearfilename, 'Annual')
# try:
# # this actually becomes a simple write rather than append when a
# # new month
# fyr = open(yearfilename, 'a')
# except:
# utilities.do_RuntimeError(
# 'Could not open yearly file',
# yearfilename)
#
# curmonth = True
# # just make a copy of the existing file with the DOC lines
# if datestr[4:] == '01':
# subprocess.call("cp " + eventfilename + ' ' + monthfilename,
# shell=True)
# curmonth = False
# else: # download existing files and append to it
# get_zipped_file(monthfilename, 'Monthly')
# try:
# # this actually becomes a simple write rather than append when a
# # new month
# fmon = open(monthfilename, 'a')
# except:
# utilities.do_RuntimeError(
# 'Could not open monthly file',
# monthfilename)
#
# if curyear or curmonth:
# try:
# fin = open(eventfilename, 'r')
# except:
# utilities.do_RuntimeError(
# 'Could not open the daily event file',
# eventfilename)
#
# line = fin.readline()
# while len(line) > 0: # loop through the file
# # copy the new lines, skipping the documentation lines
# if 'DOC\tDOC\t999' not in line:
# if curmonth:
# fmon.write(line)
# if curyear:
# fyr.write(line)
# line = fin.readline()
#
# fin.close()
# if curmonth:
# fmon.close()
# if curyear:
# fyr.close()
#
# store_zipped_file(monthfilename, 'Monthly')
# store_zipped_file(yearfilename, 'Annual')
ftp.quit()
print("Finished")
def main(datestr, server_info, file_info):
"""
When something goes amiss, various routines will and pass through a
RuntimeError(explanation) rather than trying to recover, since this
probably means something is either wrong with the ftp connection or the
file structure got corrupted. This error is logged but needs to be caught
in the calling program.
"""
logger = logging.getLogger('pipeline_log')
# log into the server
try:
ftp = FTP(server_info.serv_name) # connect to host, default port
ftp.login(server_info.username, server_info.password)
# change into PHOX directory
ftp.cwd(server_info.server_dir)
logger.info('Logged into: {}/{}'.format(server_info.serv_name,
server_info.server_dir))
print('Logged into: {}/{}'.format(server_info.serv_name,
server_info.server_dir))
except Exception as e:
logger.info('Login to {} unsuccessful.'.format(server_info.serv_name))
utilities.do_RuntimeError('Login to {} unsuccessful.'.format(server_info.serv_name))
# upload the daily event and duplicate index files
try:
eventfilename = '{}{}.txt'.format(file_info.eventfile_stem, datestr)
store_zipped_file(eventfilename, 'Daily', ftp)
except Exception as e:
filezip = eventfilename + '.zip'
logger.warning('Store of {} unsuccessful'.format(filezip))
utilities.do_RuntimeError('Transfer of', eventfilename,
'unsuccessful')
#We don't have these files right now.
# try:
# dupfilename = '{}{}.txt'.format(file_info.dupfile_stem, datestr)
# store_zipped_file(dupfilename, 'Daily/Duplicates', ftp)
# ftp.cwd('..') # back out one more level
# except:
# utilities.do_RuntimeError('Transfer of', dupfilename,
# 'unsuccessful')
#
# # update the monthly and yearly files
# monthfilename = file_info.outputfile_stem + \
# datestr[:2] + '-' + datestr[2:4] + '.txt'
# yearfilename = file_info.outputfile_stem + datestr[:2] + '.txt'
#
# curyear = True
# if datestr[2:] == '0101': # initialize a new year
# subprocess.call("cp " + eventfilename + ' ' + yearfilename, shell=True)
# curyear = False
# else:
# get_zipped_file(yearfilename, 'Annual')
# try:
# # this actually becomes a simple write rather than append when a
# # new month
# fyr = open(yearfilename, 'a')
# except:
# utilities.do_RuntimeError(
# 'Could not open yearly file',
# yearfilename)
#
# curmonth = True
# # just make a copy of the existing file with the DOC lines
# if datestr[4:] == '01':
# subprocess.call("cp " + eventfilename + ' ' + monthfilename,
# shell=True)
# curmonth = False
# else: # download existing files and append to it
# get_zipped_file(monthfilename, 'Monthly')
# try:
# # this actually becomes a simple write rather than append when a
# # new month
# fmon = open(monthfilename, 'a')
# except:
# utilities.do_RuntimeError(
# 'Could not open monthly file',
# monthfilename)
#
# if curyear or curmonth:
# try:
# fin = open(eventfilename, 'r')
# except:
# utilities.do_RuntimeError(
# 'Could not open the daily event file',
# eventfilename)
#
# line = fin.readline()
# while len(line) > 0: # loop through the file
# # copy the new lines, skipping the documentation lines
# if 'DOC\tDOC\t999' not in line:
# if curmonth:
# fmon.write(line)
# if curyear:
# fyr.write(line)
# line = fin.readline()
#
# fin.close()
# if curmonth:
# fmon.close()
# if curyear:
# fyr.close()
#
# store_zipped_file(monthfilename, 'Monthly')
# store_zipped_file(yearfilename, 'Annual')
ftp.quit()
print("Finished")
