def main():
# Read in the config
with open("config.toml", "r") as config_file:
config = toml.loads(config_file.read())
# Set up logging
logging.basicConfig(filename="nrt_uploader.log",
format="%(asctime)s:%(levelname)s:%(message)s")
logger = logging.getLogger('nrt_uploader')
logger.setLevel(level=config["Logging"]["level"])
# Connect to the FTP server
ftp_conn = nrtftp.connect_ftp(config["FTP"])
ftp_folders = nrtftp.get_instrument_folders(ftp_conn, config["FTP"])
quince_instruments = quince.get_instruments(config)
for instrument_id in ftp_folders:
if not quince.is_quince_instrument(instrument_id, quince_instruments):
logger.warning("FTP folder " + instrument_id +
" does not match a QuinCe instrument")
else:
logger.info("Processing instrument " + instrument_id)
files = nrtftp.get_instrument_files(ftp_conn, config["FTP"],
instrument_id)
for file in files:
file_content = nrtftp.get_file(ftp_conn, config["FTP"],
instrument_id, file)
log_instrument(logger, instrument_id, logging.DEBUG,
"Uploading file " + file)
upload_result = quince.upload_file(config, instrument_id, file,
file_content)
status_code = upload_result.status_code
if status_code == 200:
log_instrument(logger, instrument_id, logging.DEBUG,
"Upload succeeded")
nrtftp.upload_succeeded(ftp_conn, config["FTP"],
instrument_id, file)
else:
log_instrument(
logger, instrument_id, logging.ERROR,
"Upload failed (status code " + str(status_code) + ")")
log_instrument(logger, instrument_id, logging.ERROR,
upload_result.text)
nrtftp.upload_failed(ftp_conn, config["FTP"],
instrument_id, file)
# Close down
if ftp_conn is not None:
ftp_conn.close()
def main():
# Read in the config
with open("config.toml", "r") as config_file:
config = toml.loads(config_file.read())
# Set up logging
logging.basicConfig(filename="nrt_uploader.log",
format="%(asctime)s:%(levelname)s:%(message)s")
logger = logging.getLogger('nrt_uploader')
logger.setLevel(level=config["Logging"]["level"])
# Connect to the FTP server
ftpconn = nrtftp.connect_ftp(config["FTP"])
ftp_folders = nrtftp.get_instrument_folders(ftpconn, config["FTP"])
quince_instruments = quince.get_instruments(config)
for instrument_id in ftp_folders:
if not quince.is_quince_instrument(instrument_id, quince_instruments):
logger.warning("FTP folder " + instrument_id
+ " does not match a QuinCe instrument")
else:
logger.info("Processing instrument " + instrument_id)
files = nrtftp.get_instrument_files(ftpconn, config["FTP"], instrument_id)
for file in files:
file_content = nrtftp.get_file(ftpconn, config["FTP"],
instrument_id, file)
log_instrument(logger, instrument_id, logging.DEBUG, "Uploading file " \
+ file)
upload_result = quince.upload_file(config, instrument_id, file, file_content)
status_code = upload_result.status_code
if status_code == 200:
log_instrument(logger, instrument_id, logging.DEBUG, \
"Upload succeeded")
nrtftp.upload_succeeded(ftpconn, config["FTP"], instrument_id, file)
else:
log_instrument(logger, instrument_id, logging.ERROR, \
"Upload failed (status code " + str(status_code) + ")")
log_instrument(logger, instrument_id, logging.ERROR, upload_result.text)
nrtftp.upload_failed(ftpconn, config["FTP"], instrument_id, file)
# Close down
if ftpconn is not None:
ftpconn.close()
def main():
ftpconn = None
dbconn = None
try:
# Blank logger - sends everything to /dev/null
logging.basicConfig(filename=os.devnull)
logger = logging.getLogger('configure_nrt')
logger.setLevel(level=10)
with open("config.toml", "r") as config_file:
config = toml.loads(config_file.read())
print("Connecting to FTP servers...")
ftpconn = nrtftp.connect_ftp(config["FTP"])
dbconn = nrtdb.get_db_conn(config["Database"]["location"])
print("Getting QuinCe instruments...")
quince_instruments = quince.get_instruments(config)
print("Getting NRT instruments...")
nrt_instruments = nrtdb.get_instruments(dbconn)
# Check that NRT and QuinCe are in sync
quince_ids = get_ids(quince_instruments)
nrt_ids = get_ids(nrt_instruments)
# Remove old instruments from NRT
orphaned_ids = list(set(nrt_ids) - set(quince_ids))
if len(orphaned_ids) > 0:
print(
"The following instruments are no longer in QuinCe and will be removed:\n"
)
make_instrument_table(nrt_instruments, orphaned_ids, False)
go = input("Enter Y to proceed, or anything else to quit: ")
if not go.lower() == "y":
exit()
else:
nrtdb.delete_instruments(dbconn, orphaned_ids)
# Add new instruments from QuinCe
new_ids = list(set(quince_ids) - set(nrt_ids))
if len(new_ids) > 0:
print(
"The following instruments are new in QuinCe and will be added:\n"
)
make_instrument_table(quince_instruments, new_ids, False)
go = input("Enter Y to proceed, or anything else to quit: ")
if not go.lower() == "y":
exit()
else:
nrtdb.add_instruments(dbconn, quince_instruments, new_ids)
nrtftp.add_instruments(ftpconn, config["FTP"], new_ids)
# Main configuration loop
quit = False
while not quit:
print()
instruments = nrtdb.get_instruments(dbconn)
make_instrument_table(instruments, None, True)
command = input(
"\nEnter instrument ID to configure, or Q to quit: ").lower()
if command == "q":
quit = True
else:
instrument_id = None
try:
instrument_id = int(command)
except:
pass
if instrument_id is not None:
instrument = nrtdb.get_instrument(dbconn, instrument_id)
if instrument is not None:
retriever = None
print()
print("Current configuration for instrument %d (%s):" % \
(instrument["id"], instrument["name"]))
print()
print("TYPE: %s" % (instrument["type"]))
if instrument["type"] is not None and instrument[
"config"] is not None:
retriever = RetrieverFactory.get_instance(instrument["type"], \
instrument["id"], logger, json.loads(instrument["config"]))
retriever.print_configuration()
print()
print("PREPROCESSOR: %s" %
instrument["preprocessor"])
print()
change = input("Change configuration (y/n)? ").lower()
if change == "y":
new_type = RetrieverFactory.ask_retriever_type()
if new_type is None:
nrtdb.store_configuration(
dbconn, instrument["id"], None)
else:
if new_type != instrument["type"]:
retriever = RetrieverFactory.get_new_instance(
new_type)
config_ok = False
while not config_ok:
print()
config_ok = retriever.enter_configuration()
print()
preprocessor = PreprocessorFactory.ask_preprocessor(
)
nrtdb.store_configuration(
dbconn, instrument["id"], retriever,
preprocessor)
except urllib.error.URLError as e:
print(e)
except urllib.error.HTTPError as e:
print("%s %s" % (e.code, e.reason))
finally:
if dbconn is not None:
nrtdb.close(dbconn)
if ftpconn is not None:
ftpconn.close()
def main():
# Read in the config
with open("config.toml", "r") as config_file:
config = toml.loads(config_file.read())
# Set up logging
logging.basicConfig(filename="nrt_collector.log",
format="%(asctime)s:%(levelname)s:%(message)s")
logger = logging.getLogger('nrt_collector')
logger.setLevel(level=config["Logging"]["level"])
# Connect to NRT database and get instrument list
dbconn = nrtdb.get_db_conn(config["Database"]["location"])
instruments = nrtdb.get_instrument_ids(dbconn)
# Connect to FTP server
ftpconn = nrtftp.connect_ftp(config["FTP"])
# Loop through each instrument
for instrument_id in instruments:
log_instrument(logger, instrument_id, logging.INFO, \
"Processing instrument")
instrument = nrtdb.get_instrument(dbconn, instrument_id)
if instrument["type"] is None:
log_instrument(logger, instrument_id, logging.ERROR, \
"Configuration type not set")
else:
# Build the retriever
if instrument["type"] == "None":
log_instrument(logger, instrument_id, logging.ERROR, \
"Instrument is not configured")
else:
retriever = RetrieverFactory.get_instance(instrument["type"],
instrument_id, logger, json.loads(instrument["config"]))
# Make sure configuration is still valid
if not retriever.test_configuration():
log_instrument(logger, instrument_id, logging.ERROR, \
"Configuration invalid")
# Initialise the retriever
elif not retriever.startup():
log_instrument(logger, instrument_id, logging.ERROR, \
"Could not initialise retriever")
else:
preprocessor = PreprocessorFactory.get_new_instance(instrument["preprocessor"])
# Loop through all files returned by the retriever one by one
while retriever.load_next_file():
for file in retriever.current_files:
log_instrument(logger, instrument_id, logging.DEBUG, \
"Uploading " + file["filename"] + " to FTP server")
upload_result = upload_file(logger, ftpconn, config["FTP"], \
instrument_id, preprocessor, file["filename"], file["contents"])
if upload_result == nrtftp.NOT_INITIALISED:
log_instrument(logger, instrument_id, logging.ERROR, \
"FTP not initialised")
retriever.file_failed()
elif upload_result == nrtftp.FILE_EXISTS:
log_instrument(logger, instrument_id, logging.DEBUG, \
"File exists on FTP "
+ "server - will retry later")
retriever.file_not_processed()
elif upload_result == nrtftp.UPLOAD_OK:
log_instrument(logger, instrument_id, logging.DEBUG, \
"File uploaded OK (look for individual failures in ZIPs)")
retriever.file_succeeded()
else:
log_instrument(logger, instrument_id, logging.CRITICAL, \
"Unrecognised upload result " + str(upload_result))
exit()
retriever.shutdown()
if ftpconn is not None:
ftpconn.close()
if dbconn is not None:
dbconn.close()
def main():
ftpconn = None
dbconn = None
try:
# Blank logger - sends everything to /dev/null
logging.basicConfig(filename=os.devnull)
logger = logging.getLogger('configure_nrt')
logger.setLevel(level=10)
with open("config.toml", "r") as config_file:
config = toml.loads(config_file.read())
print("Connecting to FTP servers...")
ftpconn = nrtftp.connect_ftp(config["FTP"])
dbconn = nrtdb.get_db_conn(config["Database"]["location"])
print("Getting QuinCe instruments...")
quince_instruments = quince.get_instruments(config)
print("Getting NRT instruments...")
nrt_instruments = nrtdb.get_instruments(dbconn)
# Check that NRT and QuinCe are in sync
quince_ids = get_ids(quince_instruments)
nrt_ids = get_ids(nrt_instruments)
# Remove old instruments from NRT
orphaned_ids = list(set(nrt_ids) - set(quince_ids))
if len(orphaned_ids) > 0:
print("The following instruments are no longer in QuinCe and will be removed:\n")
make_instrument_table(nrt_instruments, orphaned_ids, False)
go = input("Enter Y to proceed, or anything else to quit: ")
if not go.lower() == "y":
exit()
else:
nrtdb.delete_instruments(dbconn, orphaned_ids)
# Add new instruments from QuinCe
new_ids = list(set(quince_ids) - set(nrt_ids))
if len(new_ids) > 0:
print("The following instruments are new in QuinCe and will be added:\n")
make_instrument_table(quince_instruments, new_ids, False)
go = input("Enter Y to proceed, or anything else to quit: ")
if not go.lower() == "y":
exit()
else:
nrtdb.add_instruments(dbconn, quince_instruments, new_ids)
nrtftp.add_instruments(ftpconn, config["FTP"], new_ids)
# Main configuration loop
quit = False
while not quit:
print()
instruments = nrtdb.get_instruments(dbconn)
make_instrument_table(instruments, None, True)
command = input("\nEnter instrument ID to configure, or Q to quit: ").lower()
if command == "q":
quit = True
else:
instrument_id = None
try:
instrument_id = int(command)
except:
pass
if instrument_id is not None:
instrument = nrtdb.get_instrument(dbconn, instrument_id)
if instrument is not None:
retriever = None
print()
print("Current configuration for instrument %d (%s):" % \
(instrument["id"], instrument["name"]))
print()
print("TYPE: %s" % (instrument["type"]))
if instrument["type"] is not None and instrument["config"] is not None:
retriever = RetrieverFactory.get_instance(instrument["type"], \
instrument["id"], logger, json.loads(instrument["config"]))
retriever.print_configuration()
print()
print("PREPROCESSOR: %s" % instrument["preprocessor"])
print()
change = input("Change configuration (y/n)? ").lower()
if change == "y":
new_type = RetrieverFactory.ask_retriever_type()
if new_type is None:
nrtdb.store_configuration(dbconn, instrument["id"], None)
else:
if new_type != instrument["type"]:
retriever = RetrieverFactory.get_new_instance(new_type)
config_ok = False
while not config_ok:
print()
config_ok = retriever.enter_configuration()
print()
preprocessor = PreprocessorFactory.ask_preprocessor()
nrtdb.store_configuration(dbconn, instrument["id"], retriever, preprocessor)
except urllib.error.URLError as e:
print(e)
except urllib.error.HTTPError as e:
print("%s %s" % (e.code, e.reason))
finally:
if dbconn is not None:
nrtdb.close(dbconn)
if ftpconn is not None:
ftpconn.close()
def main():
# Read in the config
with open("config.toml", "r") as config_file:
config = toml.loads(config_file.read())
# Set up logging
logging.basicConfig(filename="nrt_collector.log",
format="%(asctime)s:%(levelname)s:%(message)s")
logger = logging.getLogger('nrt_collector')
logger.setLevel(level=config["Logging"]["level"])
# Connect to NRT database and get instrument list
db_conn = nrtdb.get_db_conn(config["Database"]["location"])
instruments = nrtdb.get_instrument_ids(db_conn)
# Connect to FTP server
ftp_conn = nrtftp.connect_ftp(config["FTP"])
# Loop through each instrument
for instrument_id in instruments:
log_instrument(logger, instrument_id, logging.INFO,
"Checking instrument")
instrument = nrtdb.get_instrument(db_conn, instrument_id)
if instrument["type"] is None:
log_instrument(logger, instrument_id, logging.ERROR,
"Configuration type not set")
elif not time_for_check(instrument):
log_instrument(logger, instrument_id, logging.INFO,
"Not time for check yet")
else:
log_instrument(logger, instrument_id, logging.INFO,
"Time for check")
# Build the retriever
if instrument["config"] is not None:
retriever = RetrieverFactory.get_instance(
instrument["type"], instrument_id, logger,
json.loads(instrument["config"]))
# Make sure configuration is still valid
if not retriever.test_configuration():
log_instrument(logger, instrument_id, logging.ERROR,
"Configuration invalid")
# Initialise the retriever
elif not retriever.startup():
log_instrument(logger, instrument_id, logging.ERROR,
"Could not initialise retriever")
else:
preprocessor = None if instrument["preprocessor"] is None else \
PreprocessorFactory.get_instance(instrument["preprocessor"],
logger, json.loads(instrument["preprocessor_config"]))
# Loop through all files returned by the retriever one by one
while retriever.load_next_files():
for file in retriever.current_files:
log_instrument(
logger, instrument_id, logging.DEBUG,
"Uploading " + file["filename"] +
" to FTP server")
upload_result = upload_file(
logger, ftp_conn, config["FTP"], instrument_id,
preprocessor,
preprocessor.get_processed_filename(
file["filename"]), file["contents"])
if upload_result == nrtftp.NOT_INITIALISED:
log_instrument(logger, instrument_id,
logging.ERROR,
"FTP not initialised")
retriever.file_failed()
elif upload_result == nrtftp.FILE_EXISTS:
log_instrument(
logger, instrument_id, logging.DEBUG,
"File exists on FTP " +
"server - will retry later")
retriever.file_not_processed()
elif upload_result == nrtftp.UPLOAD_OK:
log_instrument(
logger, instrument_id, logging.DEBUG,
"File uploaded OK (look for individual failures in ZIPs)"
)
retriever.file_succeeded()
elif upload_result == PREPROCESSOR_FAILED:
log_instrument(logger, instrument_id,
logging.DEBUG,
"File preprocessor failed")
retriever.file_failed()
else:
log_instrument(
logger, instrument_id, logging.CRITICAL,
"Unrecognised upload result " +
str(upload_result))
exit()
retriever.shutdown()
nrtdb.set_last_check(db_conn, instrument)
if ftp_conn is not None:
ftp_conn.close()
if db_conn is not None:
db_conn.close()