def __init__(self, action):
self.proxies = [] # list of proxies needed by some stations
self.stations = [] # initial list of stations
self.stations_retry = [] # checks failed, try again
self.stations_complete = [] # checks succeeded, done
self.stations_expired = [] # tried max number of times allowed
self.stations_partial = [] # stations that are missing information
self.output_directory = ""
self.thread_count = 0
self.max_threads = 10
self.thread_ttl = 7200 # Should not take more than 2 hours
self.threads = []
self.action = action
self.types = None
self.selection = None
self.station_file = ""
self.station_file_encrypted = False
self.poll_interval = 0.10 # seconds
self.poller = Interval(self.poll_interval, self.poll, False)
self.lock_poll = thread.allocate_lock()
self.done = False
self.continuity_only = False
self.versions_only = False
self.version_logger = Logger(prefix='deviations_')
self.version_queue = Queue.Queue()
self.version_files = []
class Loop:
def __init__(self, action):
self.proxies = [] # list of proxies needed by some stations
self.stations = [] # initial list of stations
self.stations_retry = [] # checks failed, try again
self.stations_complete = [] # checks succeeded, done
self.stations_expired = [] # tried max number of times allowed
self.stations_partial = [] # stations that are missing information
self.output_directory = ""
self.thread_count = 0
self.max_threads = 10
self.thread_ttl = 7200 # Should not take more than 2 hours
self.threads = []
self.action = action
self.types = None
self.selection = None
self.station_file = ""
self.station_file_encrypted = False
self.poll_interval = 0.10 # seconds
self.poller = Interval(self.poll_interval, self.poll, False)
self.lock_poll = thread.allocate_lock()
self.done = False
self.continuity_only = False
self.versions_only = False
self.version_logger = Logger(prefix='deviations_')
self.version_queue = Queue.Queue()
self.version_files = []
def is_done(self):
return self.done
def set_selection(self, list):
self.selection = list
def set_continuity_only(self, only=True):
self.continuity_only = only
def set_versions_only(self, only=True):
self.versions_only = only
def set_types(self, list):
self.types = list
def set_file(self, name, encrypted=False):
self.station_file = name
self.station_file_encrypted = encrypted
def set_dir(self, directory):
self.station_dir = directory
# create the archive directory if it does not exist
def init_dir(self):
self.output_directory = None
self.version_directory = None
if not self.output_directory:
self.output_directory = "%(HOME)s/stations" % os.environ
if not os.path.exists(self.output_directory):
try:
os.makedirs(self.output_directory)
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
if os.stat(self.output_directory).st_mode != permissions:
os.chmod(self.output_directory, permissions)
except:
raise Exception, "CheckLoop::init_dir() could not create storage directory: %s" % self.output_directory
self.output_directory += "/gsn"
if not os.path.exists(self.output_directory):
try:
os.makedirs(self.output_directory)
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
if os.stat(self.output_directory).st_mode != permissions:
os.chmod(self.output_directory, permissions)
except:
raise Exception, "CheckLoop::init_dir() could not create storage directory: %s" % self.output_directory
self.version_directory = self.output_directory + "/versions"
if not os.path.exists(self.version_directory):
try:
os.makedirs(self.version_directory)
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
if os.stat(self.version_directory).st_mode != permissions:
os.chmod(self.version_directory, permissions)
except:
raise Exception, "CheckLoop::init_dir() could not create version directory: %s" % self.version_directory
self.version_logger.set_log_path(self.version_directory)
# ===== Long Running Threads =====================================
def start_threads(self):
self.version_thread = threading.Thread(target=self._version_log_thread)
self.version_thread.start()
def stop_threads(self):
self.version_queue.put('HALT')
self.version_thread.join()
def _version_log_thread(self):
run = True
while run:
message = self.version_queue.get()
if message == 'HALT':
run = False
else:
parts = message.split(':', 1)
if (len(parts) == 2) and (4 < len(parts[0]) < 9):
self.version_logger.set_log_note(parts[0])
self.version_logger.log(parts[1])
def read_version_file(self):
version_file = './software_versions'
if os.path.exists(version_file) and os.path.isfile(version_file):
fh = open(version_file, 'r')
for line in fh.readlines():
parts = line.split('#', 1)[0].split()
if len(parts) > 1:
hash = parts[0]
file = parts[1]
self.version_files.append((file, hash))
def prep_station(self, station, info):
if info.has_key('station'):
station.set_name(info['station'])
if info.has_key('address'):
station.set_address(info['address'])
if info.has_key('port'):
station.set_port(info['port'])
if info.has_key('username'):
station.set_username(info['username'])
if info.has_key('password'):
station.set_password(info['password'])
if info.has_key('netserv'):
list = info['netserv'].split(',')
for item in list:
station.add_netserv_log(item)
if info.has_key('server'):
list = info['server'].split(',')
for item in list:
station.add_server_log(item)
def start(self):
self.init_dir()
self.parse_configuration()
self.read_version_file()
self.start_threads()
self.poller.start()
def record(self, station):
if not station:
return
date = time.gmtime()
# by default, check output is stored in path as follows:
# $HOME/stations/gsn/<station>/<year>/<j_day>/<HHMMSS>.chk
dir = self.output_directory + '/' + station.name + time.strftime("/%Y/%j", date)
file = time.strftime("%H%M%S.chk", date)
# create the directories if they do not exist
ydir = dir.rsplit('/',1)[0]
if not os.path.exists(ydir):
try:
os.makedirs(ydir)
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
if os.stat(ydir).st_mode != permissions:
os.chmod(ydir, permissions)
except:
station._log("could not create directory %s" % ydir)
return
if not os.path.exists(dir):
try:
os.makedirs(dir)
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH
if os.stat(dir).st_mode != permissions:
os.chmod(dir, permissions)
except:
station._log("could not create directory %s" % dir)
return
# if the target directory path exists but is not a directory, complain
elif not os.path.isdir(dir):
station._log("%s exists, and is not a directory, please resolve." % dir)
return
# write results into the summary file
try:
summary_file = dir + '/' + file
fd = open(summary_file, "a")
fd.write( station.summary )
fd.write( station.output )
fd.write( station.log_messages )
fd.close()
permissions = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH
if os.stat(summary_file).st_mode != permissions:
os.chmod(summary_file, permissions)
except Exception, e:
station._log("CheckLoop::record() failed to record data to file. Exception: %s" % str(e))