def main():
loop = asyncio.get_event_loop()
args = parse_args()
setup_logging(args.debug)
db = init_db(args.drop)
app = init_app(db, loop=loop)
logging.info('Starting app on port {}...'.format(settings.PORT))
web.run_app(app, port=settings.PORT)
logging.info('Stopped.')
def setUp(self):
# Remove 'logs' directory.
self.logs_dir = setup_logging.get_log_path()
shutil.rmtree(self.logs_dir, ignore_errors=True)
# Standard LR configuration.
logging.shutdown()
setup_logging.setup_logging()
# N.B.: this will return the same logger each time, so we need to reset.
self.logger = logging.getLogger(__name__)
self.logger.setLevel(logging.NOTSET)
def run(self, worker_name, device_name, extraargs):
self.worker_name = worker_name
self.device_name = device_name
for argname in extraargs:
setattr(self, argname, extraargs[argname])
# Total fudge, should be replaced with zmq logging in future:
from setup_logging import setup_logging
setup_logging()
log_name = 'BLACS.%s_%s.worker' % (self.device_name, self.worker_name)
self.logger = logging.getLogger(log_name)
self.logger.debug('Starting')
import zprocess.locking, labscript_utils.h5_lock
zprocess.locking.set_client_process_name(log_name)
#self.init()
self.mainloop()
def main():
if "--debug" in sys.argv[1:]:
log_level = logging.DEBUG
else:
log_level = logging.INFO
log_format = "%(asctime)s | %(levelname)-10s | %(name)-25s | %(message)s"
with setup_logging(logfile_name = None, fmt = log_format, level = log_level):
device = "/dev/ttyUSB0"
for arg in sys.argv[1:]:
if arg.startswith("--device="):
device = arg[9:]
ledz = LedDisplay(device)
ledz.setBrightnessLevel("A")
ledz.setRealtimeClock()
ledz.setSchedule("A", "A")
ledz.send("<L1><PA><FA><MA><WZ><FA><AC><CD> <KD> <KT>")
ledz.close()
def __init__(self, base_directory):
"""
Initialisation method for the deployment. This will be called at start-up of the model in UbiOps.
:param str base_directory: absolute path to the directory where this file is located.
"""
setup_logging()
logging.info("Initialising Caffe model")
# Here we initialize the Caffe classifier model with the pre-trained files
model_def_file = os.path.join(base_directory, "age.prototxt")
caffe_model = os.path.join(base_directory,
"dex_chalearn_iccv2015.caffemodel")
self.net = caffe.Classifier(model_def_file, caffe_model)
def main(plan, collaborators_file, data_config_fname, logging_config_path, logging_default_level, logging_directory, model_device, **kwargs):
"""Run the federation simulation from the federation (FL) plan.
Runs a federated training from the federation (FL) plan but creates the
aggregator and collaborators on the same compute node. This allows
the developer to test the model and data loaders before running
on the remote collaborator nodes.
Args:
plan: The Federation (FL) plan (YAML file)
collaborators_file: The file listing the collaborators
data_config_fname: The file describing where the dataset is located on the collaborators
logging_config_path: The log file
logging_default_level: The log level
**kwargs: Variable parameters to pass to the function
"""
# FIXME: consistent filesystem (#15)
# establish location for fl plan as well as
# where to get and write model protobufs
script_dir = os.path.dirname(os.path.realpath(__file__))
base_dir = os.path.join(script_dir, 'federations')
plan_dir = os.path.join(base_dir, 'plans')
weights_dir = os.path.join(base_dir, 'weights')
metadata_dir = os.path.join(base_dir, 'metadata')
collaborators_dir = os.path.join(base_dir, 'collaborator_lists')
logging_config_path = os.path.join(script_dir, logging_config_path)
logging_directory = os.path.join(script_dir, logging_directory)
setup_logging(path=logging_config_path, default_level=logging_default_level, logging_directory=logging_directory)
# load the flplan, local_config and collaborators file
flplan = parse_fl_plan(os.path.join(plan_dir, plan))
local_config = load_yaml(os.path.join(base_dir, data_config_fname))
collaborator_common_names = load_yaml(os.path.join(collaborators_dir, collaborators_file))['collaborator_common_names']
# TODO: Run a loop here over various parameter values and iterations
# TODO: implement more than just saving init, best, and latest model
federate(flplan,
local_config,
collaborator_common_names,
base_dir,
weights_dir,
metadata_dir,
model_device)
def main():
"""Initialize logger and run the database update cycle loop."""
database_filename = "oeis.sqlite3"
logfile = "logfiles/fetch_oeis_database_%Y%m%d_%H%M%S.log"
with setup_logging(logfile):
database_update_cycle_loop(database_filename)
def main():
"""Initialize logger and run the database update cycle loop."""
database_filename = "oeis.sqlite3"
logfile = "logfiles/fetch_oeis_database_%Y%m%d_%H%M%S.log"
with setup_logging(logfile):
database_update_cycle_loop(database_filename)
def main():
if len(sys.argv) != 2:
print("Please specify the name of an OEIS database in Sqlite3 format.")
return
database_filename = sys.argv[1]
with setup_logging(None):
show_entries(database_filename)
def main(plan, collaborators_file, single_col_cert_common_name,
logging_config_path, logging_default_level, logging_directory, resume,
script_dir):
"""Runs the aggregator service from the Federation (FL) plan
Args:
plan: The Federation (FL) plan
collaborators_file: The file listing the collaborators
single_col_cert_common_name: The SSL certificate
logging_config_path: The log configuration file
logging_default_level: The log level
resume: Whether the aggregator should load the latest model instead of the initial model
script_dir: default None uses the script dir. Otherwise, use the directory passed as the script dir
"""
# FIXME: consistent filesystem (#15)
if script_dir is None:
script_dir = os.path.dirname(os.path.realpath(__file__))
base_dir = os.path.join(script_dir, 'federations')
plan_dir = os.path.join(base_dir, 'plans')
weights_dir = os.path.join(base_dir, 'weights')
collaborators_dir = os.path.join(base_dir, 'collaborator_lists')
metadata_dir = os.path.join(base_dir, 'metadata')
logging_config_path = os.path.join(script_dir, logging_config_path)
logging_directory = os.path.join(script_dir, logging_directory)
setup_logging(path=logging_config_path,
default_level=logging_default_level,
logging_directory=logging_directory)
flplan = parse_fl_plan(os.path.join(plan_dir, plan))
collaborator_common_names = load_yaml(
os.path.join(collaborators_dir,
collaborators_file))['collaborator_common_names']
agg = create_aggregator_object_from_flplan(flplan,
collaborator_common_names,
single_col_cert_common_name,
base_dir, weights_dir,
metadata_dir, resume)
server = create_aggregator_server_from_flplan(agg, flplan)
serve_kwargs = get_serve_kwargs_from_flpan(flplan, base_dir)
server.serve(**serve_kwargs)
def start_grader(host='localhost',
port=10101,
log_file='log/%Y/%m/%d/grader_%Y-%m-%dT%H%M.log'):
# Set working directory
os.chdir(os.path.abspath(os.path.dirname(sys.argv[0])))
setup_logging(logfile=time.strftime(log_file),
lvl_bash=logging.INFO,
lvl_logfile=logging.INFO,
symlink='log')
# Start the server
try:
SERVER = ThreadedHTTPServer((host, port), Handler)
logging.info('Starting grader on %s:%s...', host, port)
SERVER.serve_forever()
except KeyboardInterrupt:
logging.info('Server shut down with ^C')
def main():
host = "pc192.pinguinradio.com"
port = 80
path = "/"
led_device = os.getenv("LED_DEVICE")
if "--debug" in sys.argv[1:]:
log_level = logging.DEBUG
else:
log_level = logging.INFO
log_filename = "PlayInternetRadio_%Y%m%d_%H%M%S.log"
log_format = "%(asctime)s | %(levelname)-10s | %(name)-25s | %(message)s"
with setup_logging(logfile_name = log_filename, fmt = log_format, level = log_level):
logger = logging.getLogger("main")
with AudioStreamPlayer("mpg123", ["-"]) as audiostream_player, \
MetadataLedDisplayDriver(led_device) as metadata_led_driver, \
MetadataDatabaseWriter("metadata.sqlite3") as metadata_database_writer, \
MetadataFileWriter("metadata.log") as metadata_file_writer:
RETRY_INTERVAL = 5.0
while True:
try:
radioPlayer = InternetRadioPlayer(host, port, path)
radioPlayer.audiodata_signal.connect(audiostream_player.play)
radioPlayer.metadata_signal.connect(metadata_led_driver.process)
radioPlayer.metadata_signal.connect(metadata_file_writer.process)
radioPlayer.metadata_signal.connect(metadata_database_writer.process)
radioPlayer.play() # blocking call
except KeyboardInterrupt:
logger.info("Quitting by user request.")
break
except socket.gaierror as exception:
logger.error("getaddrinfo() error: {}".format(exception))
except StreamStalledError:
logger.error("Stream has stalled; unable to play.")
except BaseException as exception:
logger.exception("Unknown exception: {!r}".format(exception))
logger.info("Sleeping for {} seconds before retry ...".format(RETRY_INTERVAL))
time.sleep(RETRY_INTERVAL)
def main():
if len(sys.argv) != 2:
print("Please specify the name of an OEIS database in Sqlite3 format.")
return
database_filename_in = sys.argv[1]
logfile = "solve_linear_recurrence.log"
with setup_logging(logfile):
logging.getLogger("oeis_entry").setLevel(logging.CRITICAL)
solve_polynomials(database_filename_in)
def main():
if len(sys.argv) != 2:
print("Please specify the name of an OEIS database in Sqlite3 format.")
return
database_filename_in = sys.argv[1]
(root, ext) = os.path.splitext(database_filename_in)
logfile = root + "_parsed.log"
with setup_logging(logfile):
process_database_entries(database_filename_in)
def main(plan, collaborators_file, single_col_cert_common_name,
logging_config_path, logging_default_level, logging_directory):
"""Runs the aggregator service from the Federation (FL) plan
Args:
plan: The Federation (FL) plan
collaborators_file: The file listing the collaborators
single_col_cert_common_name: The SSL certificate
logging_config_path: The log configuration file
logging_default_level: The log level
"""
# FIXME: consistent filesystem (#15)
script_dir = os.path.dirname(os.path.realpath(__file__))
base_dir = os.path.join(script_dir, 'federations')
plan_dir = os.path.join(base_dir, 'plans')
weights_dir = os.path.join(base_dir, 'weights')
collaborators_dir = os.path.join(base_dir, 'collaborator_lists')
metadata_dir = os.path.join(base_dir, 'metadata')
logging_config_path = os.path.join(script_dir, logging_config_path)
logging_directory = os.path.join(script_dir, logging_directory)
setup_logging(path=logging_config_path,
default_level=logging_default_level,
logging_directory=logging_directory)
flplan = parse_fl_plan(os.path.join(plan_dir, plan))
collaborator_common_names = load_yaml(
os.path.join(collaborators_dir,
collaborators_file))['collaborator_common_names']
agg = create_aggregator_object_from_flplan(flplan,
collaborator_common_names,
single_col_cert_common_name,
weights_dir, metadata_dir)
server = create_aggregator_server_from_flplan(agg, flplan)
serve_kwargs = get_serve_kwargs_from_flpan(flplan, base_dir)
server.serve(**serve_kwargs)
def main():
if len(sys.argv) != 2:
print("Please specify the name of an OEIS database in Sqlite3 format.")
return
database_filename_in = sys.argv[1]
(root, ext) = os.path.splitext(os.path.basename(database_filename_in))
dirname_out = os.path.join("data", root + "_directory")
with setup_logging(None):
process_database_entries(database_filename_in, dirname_out)
def main(plan,
model_weights_filename,
native_model_weights_filepath,
populate_weights_at_init,
model_file_argument_name,
data_dir,
logging_config_path,
logging_default_level,
logging_directory,
model_device,
inference_patient=None):
"""Runs the inference according to the flplan, data-dir and weights file. Output format is determined by the data object in the flplan
Args:
plan (string) : The filename for the federation (FL) plan YAML file
model_weights_filename (string) : A .pbuf filename in the common weights directory (mutually exclusive with native_model_weights_filepath). NOTE: these must be uncompressed weights!!
native_model_weights_filepath (string) : A framework-specific filepath. Path will be relative to the working directory. (mutually exclusive with model_weights_filename)
populate_weights_at_init (boolean) : Whether or not the model populates its own weights at instantiation
model_file_argument_name (string) : Name of argument to be passed to model __init__ providing model file location info
data_dir (string) : The directory path for the parent directory containing the data. Path will be relative to the working directory.
logging_config_fname (string) : The log file
logging_default_level (string) : The log level
inference_patient (string) : Subdirectory of single patient to run inference on (exclusively)
"""
# FIXME: consistent filesystem (#15)
script_dir = os.path.dirname(os.path.realpath(__file__))
base_dir = os.path.join(script_dir, 'federations')
plan_dir = os.path.join(base_dir, 'plans')
logging_directory = os.path.join(script_dir, logging_directory)
setup_logging(path=logging_config_path,
default_level=logging_default_level,
logging_directory=logging_directory)
flplan = parse_fl_plan(os.path.join(plan_dir, plan))
# check the inference config
if 'inference' not in flplan:
sys.exit(
"FL Plan does not contain a top-level 'inference' entry. By default, inference is disabled."
)
if 'allowed' not in flplan[
'inference'] or flplan['inference']['allowed'] != True:
sys.exit(
"FL Plan must contain a {'inference: {'allowed': True}} entry in order for inference to be allowed."
)
# create the data object
data = create_data_object_with_explicit_data_path(
flplan=flplan, data_path=data_dir, inference_patient=inference_patient)
# TODO: Find a good way to detect and communicate mishandling of model_file_argument_name
# Ie, capture exception of model not gettinng its required kwarg for this purpose, also
# how to tell if the model is using its random initialization rather than weights from file?
if populate_weights_at_init:
# Supplementing the flplan base model kwargs to include model weights file info.
if model_weights_filename is not None:
model_file_argument_name = model_file_argument_name or 'model_weights_filename'
flplan['model_object_init']['init_kwargs'].update(
{model_file_argument_name: model_weights_filename})
# model_weights_filename and native_model_weights_filepath are mutually exlusive and required (see argument parser)
else:
model_file_argument_name = model_file_argument_name or 'native_model_weights_filepath'
flplan['model_object_init']['init_kwargs'].update(
{model_file_argument_name: native_model_weights_filepath})
# create the base model object
model = create_model_object(flplan, data, model_device=model_device)
# the model may have an 'infer_volume' method instead of 'infer_batch'
if not hasattr(model, 'infer_batch'):
if hasattr(model, 'infer_volume'):
model = InferenceOnlyModelWrapper(data=data, base_model=model)
elif not hasattr(model, 'run_inference_and_store_results'):
sys.exit(
"If model object does not have a 'run_inference_and_store_results' method, it must have either an 'infer_batch' or 'infer_volume' method."
)
if not populate_weights_at_init:
# if pbuf weights, we need to run deconstruct proto with a NoCompression pipeline
if model_weights_filename is not None:
proto_path = os.path.join(base_dir, 'weights',
model_weights_filename)
proto = load_proto(proto_path)
tensor_dict_from_proto = deconstruct_proto(proto,
NoCompressionPipeline())
# restore any tensors held out from the proto
_, holdout_tensors = remove_and_save_holdout_tensors(
model.get_tensor_dict())
tensor_dict = {**tensor_dict_from_proto, **holdout_tensors}
model.set_tensor_dict(tensor_dict, with_opt_vars=False)
# model_weights_filename and native_model_weights_filepath are mutually exlusive and required (see argument parser)
else:
# FIXME: how do we handle kwargs here? Will they come from the flplan?
model.load_native(native_model_weights_filepath)
# finally, call the model object's run_inference_and_store_results with the kwargs from the inference block
inference_kwargs = flplan['inference'].get('kwargs') or {}
model.run_inference_and_store_results(**inference_kwargs)
def __init__(self, address='localhost', port=3333):
setup_logging()
self.address = address
self.port = port
self.simple_socket = SimpleSocket(address=self.address, port=self.port)
# Read Command Line Arguments
options = argparse.ArgumentParser(prog='runTasks')
options.add_argument('-c', '--config', dest='config', help='Configuration File')
options.add_argument('-l', '--log', dest='log', help='Log File')
try:
args = options.parse_args()
config_filename = args.config
logFile = args.log
except:
print('Syntax: python runTasks.py -c <config.json> -l <log file>')
sys.exit(1)
# setup logging
LOG = logging.getLogger('runTasks') # create the logger for this file
setup_logging.setup_logging("runTasks",logFile)
# Determine if process is running, or in zombie state
cpid = os.getpid()
#commandRE = "^{}$".format(" +".join([sys.executable] + sys.argv))
commandRE = "^{}.*{}.*$".format(sys.executable,config_filename)
procs = utils.getProcesses(commandRE)
if cpid in procs:
del procs[cpid]
if len(procs) > 0:
for pid in procs:
if procs[pid].status() != psutil.STATUS_ZOMBIE:
LOG.info("Process {} ({}) is running, exiting".format(__file__, pid))
sys.exit(0)
else:
LOG.info("Process {} ({}) is in zombie state, terminating".format(__file__, pid))
def stop(self):
if self.httpd and self.running:
self.logger.info('Shutting down httpd...')
self.httpd.shutdown()
self.httpd = None
if __name__ == '__main__':
argparser = argparse.ArgumentParser(description='Simple dummy server')
argparser.add_argument('--port', type=int, help='The server port', default=8080)
argparser.add_argument('--input_dir', type=str, help='Path containing input files', default=None)
argparser.add_argument('--output_dir', type=str, help='Path to write output', default=None)
main_args = argparser.parse_args()
setup_logging()
main_logger = logging.getLogger(__name__)
main_logger.info('Starting server script.')
script_file_path = inspect.getfile(lambda: None)
script_dir = os.path.dirname(script_file_path)
main_logger.debug('Script directory is %s', script_dir)
input_dir = main_args.input_dir
if input_dir is None:
input_dir = script_dir
output_dir = main_args.output_dir
if output_dir is None:
output_dir = script_dir
#! /usr/bin/env python3
from LedDisplay import LedDisplay
import random
from setup_logging import setup_logging
with setup_logging():
with LedDisplay("/dev/ttyUSB0") as led_display:
# Define display schedule.
led_display.setSchedule("A", "A")
gr1 = [
"RRRRRRRGGGGGGGGGGGGGGGGGGGGGGGGO",
"RRRRRRGGGGGGGGGGGGGGGGGGGGGGGGOO",
"RRRRRGGGGGGGGGGGGGGGGGGGGGGGGOOO",
"RRRRGGGGGGGGGGGGGGGGGGGGGGGGOOOO",
"RRRGGGGGGGGGGGGGGGGGGGGGGGGOOOOO",
"RRGGGGGGGGGGGGGGGGGGGGGGGGOOOOOO",
"RGGGGGGGGGGGGGGGGGGGGGGGGOOOOOOO"
]
gr2 = [
"RRRGGGGGGGGGGGGGGGGGGGGGGGGGGRRR",
"RGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGR",
"RGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGR",
"RGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGR",
"RGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGR",
"RGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGR",
"RRRGGGGGGGGGGGGGGGGGGGGGGGGGGRRR"
def main(plan, collaborator_common_name, single_col_cert_common_name,
data_config_fname, data_dir, validate_without_patches_flag,
data_in_memory_flag, data_queue_max_length, data_queue_num_workers,
torch_threads, kmp_affinity_flag, logging_config_path,
logging_default_level, logging_directory, model_device):
"""Runs the collaborator client process from the federation (FL) plan
Args:
plan : The filename for the federation (FL) plan YAML file
collaborator_common_name : The common name for the collaborator node
single_col_cert_common_name : The SSL certificate for this collaborator
data_config_fname : The dataset configuration filename (YAML)
data_dir : parent directory holding the patient data subdirectories(to be split into train and val)
validate_without_patches_flag : controls a model init kwarg
data_in_memory_flag : controls a data init kwarg
data_queue_max_length : controls a data init kwarg
data_queue_num_workers : controls a data init kwarg
torch_threads : model init kwarg
kmp_affinity_flag : controls a model init kwarg
logging_config_fname : The log file
logging_default_level : The log level
model_device : gets passed to model 'init' function as "device"
"""
# FIXME: consistent filesystem (#15)
script_dir = os.path.dirname(os.path.realpath(__file__))
base_dir = os.path.join(script_dir, 'federations')
plan_dir = os.path.join(base_dir, 'plans')
weights_dir = os.path.join(base_dir, 'weights')
metadata_dir = os.path.join(base_dir, 'metadata')
logging_config_path = os.path.join(script_dir, logging_config_path)
logging_directory = os.path.join(script_dir, logging_directory)
setup_logging(path=logging_config_path,
default_level=logging_default_level,
logging_directory=logging_directory)
flplan = parse_fl_plan(os.path.join(plan_dir, plan))
# FIXME: Find a better solution for passing model and data init kwargs
model_init_kwarg_keys = [
'validate_without_patches', 'torch_threads', 'kmp_affinity'
]
model_init_kwarg_vals = [
validate_without_patches_flag, torch_threads, kmp_affinity_flag
]
for key, value in zip(model_init_kwarg_keys, model_init_kwarg_vals):
if (value is not None) and (value != False):
flplan['model_object_init']['init_kwargs'][key] = value
data_init_kwarg_keys = ['in_memory', 'q_max_length', 'q_num_workers']
data_init_kwarg_vals = [
data_in_memory_flag, data_queue_max_length, data_queue_num_workers
]
for key, value in zip(data_init_kwarg_keys, data_init_kwarg_vals):
if (value is not None) and (value != False):
flplan['data_object_init']['init_kwargs'][key] = value
local_config = load_yaml(os.path.join(base_dir, data_config_fname))
try:
collaborator = create_collaborator_object_from_flplan(
flplan,
collaborator_common_name,
local_config,
base_dir,
weights_dir,
metadata_dir,
single_col_cert_common_name,
data_dir=data_dir,
model_device=model_device)
collaborator.run()
sys.exit(0)
except Exception as e:
logging.getLogger(__name__).exception(repr(e))
# this is for Sarthak
sys.exit(666)
default=False,
help='enables CUDA training')
parser.add_argument('--seed',
type=int,
default=1,
metavar='S',
help='random seed (default: 1)')
args = parser.parse_args()
torch.backends.cudnn.benchmark = True
args.cuda = not args.no_cuda and torch.cuda.is_available()
args.cost_aggregator_scale = config_args.cost_aggregator_scale
args.downsample_scale = args.cost_aggregator_scale * 8.0
setup_logging(args.logging_filename)
if args.cuda:
torch.manual_seed(args.seed)
np.random.seed(args.seed)
random.seed(args.seed)
torch.cuda.manual_seed(args.seed)
torch.backends.cudnn.deterministic = True
test_left_img, test_right_img = ls.datacollector(args.datapath)
model = DeepPruner()
if args.cuda:
model = nn.DataParallel(model)
model.cuda()
import dirRE
DTGfrmts = collections.OrderedDict({
'Y': '%Y',
'y': '%y',
'j': '%j',
'm': '%m',
'd': '%d',
'H': '%H',
'M': '%M'
})
# Logging
LOG = logging.getLogger(__file__) #create the logger for this file
logging_setup = "utils"
setup_logging(logging_setup)
def ddhh2pack(day, hour):
if hour == 0:
pcode = 'X'
pday = day
elif hour == 6:
pcode = 'Y'
pday = day
elif hour == 12:
pcode = 'X'
pday = day + 50
elif hour == 18:
pcode = 'Y'
def __init__(self):
setup_logging()
self.logger = logging.getLogger(__name__)
Classifier.__instances__[self.__class__.__name__] = self
check_version('zprocess', '1.1.2', '2')
check_version('labscript_devices', '2.0', '3')
# Pythonlib imports
### Must be in this order
import zprocess.locking, labscript_utils.h5_lock, h5py
zprocess.locking.set_client_process_name('BLACS')
###
from zprocess import zmq_get, ZMQServer
from setup_logging import setup_logging
import labscript_utils.shared_drive
# Custom Excepthook
import labscript_utils.excepthook
# Setup logging
logger = setup_logging()
labscript_utils.excepthook.set_logger(logger)
# now log versions (must be after setup logging)
try:
import sys
logger.info('Python Version: %s' % sys.version)
logger.info('Platform: %s' % sys.platform)
except Exception:
logger.error('Failed to find python version')
try:
import sys
logger.info('windows version: %s' % str(sys.getwindowsversion()))
except Exception:
pass
def setUp(self):
LOG = logging.getLogger(__file__) #create the logger for this file
setup_logging.setup_logging("MIRS_TC", "MIRS_TC.LOG")
self.pp = pprint.PrettyPrinter(indent=4)
import psycopg2.extras
from flask import Flask, request
from flask_sqlalchemy import SQLAlchemy
from flask_login import LoginManager
from flask_cors import CORS
from flask_compress import Compress
from flask_mail import Mail
from raven.contrib.flask import Sentry
from werkzeug.local import LocalProxy
from dive.base.serialization import pjson_dumps, pjson_loads
# Setup logging config
from setup_logging import setup_logging
setup_logging()
psycopg2.extras.register_default_json(
loads=pjson.loads
)
class CustomSQLAlchemy(SQLAlchemy):
def apply_driver_hacks(self, app, info, options):
options["json_serializer"] = pjson_dumps
options["json_deserializer"] = pjson_loads
return super(CustomSQLAlchemy, self).apply_driver_hacks(app, info, options)
# Initialize app-based objects
sentry = Sentry()
db = CustomSQLAlchemy()
def test_setup_again_with_logs(self):
""" 'logs' directory already exists """
setup_logging.setup_logging()
def test_setup_again_without_cfg(self):
""" BasicConfig """
setup_logging.setup_logging()
self.logger = logging.getLogger(__name__)
assert self.logger.isEnabledFor(logging.INFO) == True
sys.path.insert(1, os.path.join(parentPath, "lib"))
sys.path.insert(2, os.path.join(parentPath, "mirs_tc"))
#print("System Path: {}".format(sys.path))
except:
print "Unable to load local library paths"
sys.exit(1)
# Local modules
import setup_logging
import error_codes
import utils
import MIRS_TC
# Setup Logging using logging templates
LOG = logging.getLogger("compareTables") #create the logger for this file
setup_logging.setup_logging("compareTables", "ct.log")
LOG.info("Starting: {}".format(os.path.basename(__file__)))
# Read command line arguments
options = argparse.ArgumentParser()
options.add_argument("-1", "--table1", dest="table1", help="table 1 file")
options.add_argument("-2", "--table2", dest="table2", help="table 2 file")
options.add_argument("-o", "--output", dest="output", help="output table file")
try:
args = options.parse_args()
except:
msg = 'Syntax: python compareTable -1 <table1> -2 <table2>'
utils.error(LOG, msg, error_codes.EX_USAGE)
tbl1File = args.table1
tbl2File = args.table2
class SnapAdmin:
# need to call this function to initialize logging
setup_logging.setup_logging()
def __init__(self):
"""
taking parameters from command line
"""
colorama.init(autoreset=True)
self.q = Queue.Queue()
self.snap_q = Queue.Queue()
self.log_detail = {'hostname': None}
self.snap_del = False
self.logger = logging.getLogger(__name__)
self.parser = argparse.ArgumentParser(
formatter_class=argparse.RawTextHelpFormatter,
description=textwrap.dedent('''\
Tool to capture snapshots and compare them
It supports four subcommands:
--snap, --check, --snapcheck, --diff
1. Take snapshot:
jsnapy --snap pre_snapfile -f main_configfile
2. Compare snapshots:
jsnapy --check post_snapfile pre_snapfile -f main_configfile
3. Compare current configuration:
jsnapy --snapcheck snapfile -f main_configfile
4. Take diff without specifying test case:
jsnapy --diff pre_snapfile post_snapfile -f main_configfile
'''),
usage=
"\nThis tool enables you to capture and audit runtime environment of "
"\nnetworked devices running the Junos operating system (Junos OS)\n"
)
group = self.parser.add_mutually_exclusive_group()
# for mutually exclusive gp, can not use two or more options at a time
group.add_argument(
'--snap',
action='store_true',
help="take the snapshot for commands specified in test file")
group.add_argument(
'--check',
action='store_true',
help=
" compare pre and post snapshots based on test operators specified in test file"
)
group.add_argument('--snapcheck',
action='store_true',
help='check current snapshot based on test file')
#########
# will supoort it later
# for windows
########
# group.add_argument(
# "--init",
# action="store_true",
# help="generate init folders: snapshots, configs and main.yml",
# )
#########
group.add_argument("--diff",
action="store_true",
help="display difference between two snapshots")
group.add_argument("-V",
"--version",
action="store_true",
help="displays version")
self.parser.add_argument(
"pre_snapfile", nargs='?',
help="pre snapshot filename") # make it optional
self.parser.add_argument("post_snapfile",
nargs='?',
help="post snapshot filename",
type=str) # make it optional
self.parser.add_argument("-f",
"--file",
help="config file to take snapshot",
type=str)
self.parser.add_argument("-t", "--hostname", help="hostname", type=str)
self.parser.add_argument("-p",
"--passwd",
help="password to login",
type=str)
self.parser.add_argument("-l",
"--login",
help="username to login",
type=str)
self.parser.add_argument("-P",
"--port",
help="port no to connect to device",
type=int)
self.parser.add_argument("-v",
"--verbosity",
action="count",
help=textwrap.dedent('''\
Set verbosity
-v: Debug level messages
-vv: Info level messages
-vvv: Warning level messages
-vvvv: Error level messages
-vvvvv: Critical level messages'''))
# self.parser.add_argument(
# "-m",
# "--mail",
# help="mail result to given id",
# type=str)
# self.parser.add_argument(
# "-o",
# "--overwrite",
# action='store_true',
# help="overwrite directories and files generated by init",
#)
#self.args = self.parser.parse_args()
self.args, unknown = self.parser.parse_known_args()
self.db = dict()
self.db['store_in_sqlite'] = False
self.db['check_from_sqlite'] = False
self.db['db_name'] = ""
self.db['first_snap_id'] = None
self.db['second_snap_id'] = None
def get_version(self):
"""
This function gives version of Jsnapy
:return: return JSNAPy version
"""
return version.__version__
'''
def generate_init(self):
"""
# generate init folder, will support it later
create snapshots and configs folder along with sample main config file.
All snapshots generated will go in snapshots folder. configs folder will contain
all the yaml file apart from main, like device.yml, bgp_neighbor.yml
:return:
"""
mssg = "Creating Jsnapy directory structure at: ", os.getcwd()
self.logger.debug(colorama.Fore.BLUE + mssg)
if not os.path.isdir("snapshots"):
os.mkdir("snapshots")
dst_config_path = os.path.join(os.getcwd(), 'configs')
overwrite files if given option -o or --overwrite
if not os.path.isdir(dst_config_path) or self.args.overwrite is True:
distutils.dir_util.copy_tree(os.path.join(os.path.dirname(__file__), 'configs'),
dst_config_path)
dst_main_yml = os.path.join(dst_config_path, 'main.yml')
if not os.path.isfile(
os.path.join(os.getcwd(), 'main.yml')) or self.args.overwrite is True:
shutil.copy(dst_main_yml, os.getcwd())
logging_yml_file = os.path.join(
os.path.dirname(__file__),
'logging.yml')
if not os.path.isfile(
os.path.join(os.getcwd(), 'logging.yml')) or self.args.overwrite is True:
shutil.copy(logging_yml_file, os.getcwd())
mssg1= "Successfully created Jsnap directories at:",os.getcwd()
self.logger.info(colorama.Fore.BLUE + mssg1)
'''
def set_verbosity(self, val):
self.logger.root.setLevel(val)
handlers = self.logger.root.handlers
for handle in handlers:
if handle.__class__.__name__ == 'StreamHandler':
handle.setLevel(val)
def chk_database(self,
config_file,
pre_snapfile,
post_snapfile,
check=None,
snap=None,
action=None):
"""
This function test parameters for sqlite and then update database accordingly
:param config_file: main config file
:param pre_snapfile: pre snapshot file
:param post_snapfile: post snapshot file
:param check: True if --check operator is given
:param snap:
:param action: used by module version, either snap, check or snapcheck
"""
d = config_file['sqlite'][0]
compare_from_id = False
if d.__contains__('store_in_sqlite'):
self.db['store_in_sqlite'] = d['store_in_sqlite']
if d.__contains__('check_from_sqlite'):
self.db['check_from_sqlite'] = d['check_from_sqlite']
if (self.db['store_in_sqlite']) or (self.db['check_from_sqlite']):
# and (check is True or action is
# "check")):
if d.__contains__('database_name'):
self.db['db_name'] = d['database_name']
else:
self.logger.error(colorama.Fore.RED +
"Specify name of the database.",
extra=self.log_detail)
exit(1)
if check is True or self.args.diff is True or action is "check":
if 'compare' in d.keys() and d['compare'] is not None:
strr = d['compare']
if not isinstance(strr, str):
self.logger.error(
colorama.Fore.RED +
"Properly specify ids of first and "
"second snapshot in format: first_snapshot_id, second_snapshot_id",
extra=self.log_detail)
exit(1)
compare_from_id = True
lst = [val.strip() for val in strr.split(',')]
try:
lst = [int(x) for x in lst]
except ValueError as ex:
self.logger.error(
colorama.Fore.RED +
"Properly specify id numbers of first and second snapshots"
" in format: first_snapshot_id, second_snapshot_id",
extra=self.log_detail)
#raise Exception(ex)
exit(1)
if len(lst) > 2:
self.logger.error(
colorama.Fore.RED +
"No. of snapshots specified is more than two."
" Please specify only two snapshots.",
extra=self.log_detail)
exit(1)
if len(lst) == 2 and isinstance(
lst[0], int) and isinstance(lst[1], int):
self.db['first_snap_id'] = lst[0]
self.db['second_snap_id'] = lst[1]
else:
self.logger.error(
colorama.Fore.RED +
"Properly specify id numbers of first and second snapshots"
" in format: first_snapshot_id, second_snapshot_id",
extra=self.log_detail)
exit(1)
if self.db['check_from_sqlite'] is False or compare_from_id is False:
if (check is True and
(pre_snapfile is None or post_snapfile is None)
or self.args.diff is True and
(pre_snapfile is None or post_snapfile is None)):
self.logger.error(
colorama.Fore.RED +
"Arguments not given correctly, Please refer below help message",
extra=self.log_detail)
self.parser.print_help()
sys.exit(1)
def get_hosts(self):
"""
Called by main function, it extracts main config file and also check for database
Reads the yaml config file given by user and pass the extracted data to login function to
read device details and connect them. Also checks sqlite key to check if user wants to
create database for snapshots
"""
if self.args.pre_snapfile is not None:
output_file = self.args.pre_snapfile
elif self.args.snapcheck is True and self.args.pre_snapfile is None:
output_file = "snap_temp"
self.snap_del = True
else:
output_file = ""
conf_file = self.args.file
check = self.args.check
snap = self.args.snap
if os.path.isfile(conf_file):
config_file = open(conf_file, 'r')
self.main_file = yaml.load(config_file)
elif os.path.isfile(
os.path.join(get_path('DEFAULT', 'config_file_path'),
conf_file)):
fpath = get_path('DEFAULT', 'config_file_path')
config_file = open(os.path.join(fpath, conf_file), 'r')
self.main_file = yaml.load(config_file)
else:
self.logger.error(
colorama.Fore.RED +
"ERROR!! Config file '%s' is not present " % conf_file,
extra=self.log_detail)
sys.exit(1)
#### if --check option is given for sqlite, then snap file name is not compulsory ####
#### else exit the function saying arguments not correct ####
if self.main_file.__contains__('sqlite') and self.main_file[
'sqlite'] and self.main_file['sqlite'][0]:
self.chk_database(self.main_file, self.args.pre_snapfile,
self.args.post_snapfile, check, snap)
else:
if (self.args.check is True and
(self.args.file is None or self.args.pre_snapfile is None
or self.args.post_snapfile is None)):
self.logger.error(
colorama.Fore.RED +
"Arguments not given correctly, Please refer help message",
extra=self.log_detail)
self.parser.print_help()
sys.exit(1)
self.login(output_file)
def generate_rpc_reply(self, dev, output_file, hostname, config_data):
"""
Generates rpc-reply based on command/rpc given and stores them in snap_files
:param dev: device handler
:param output_file: filename to store snapshots
:param hostname: hostname of device
:param config_data : data of main config file
"""
val = None
test_files = []
for tfile in config_data.get('tests'):
if not os.path.isfile(tfile):
tfile = os.path.join(get_path('DEFAULT', 'test_file_path'),
tfile)
if os.path.isfile(tfile):
test_file = open(tfile, 'r')
test_files.append(yaml.load(test_file))
else:
self.logger.error(
colorama.Fore.RED +
"ERROR!! File %s is not found for taking snapshots" %
tfile,
extra=self.log_detail)
g = Parser()
for tests in test_files:
val = g.generate_reply(tests, dev, output_file, hostname, self.db)
return val
def compare_tests(self,
hostname,
config_data,
pre_snap=None,
post_snap=None,
action=None):
"""
called by check and snapcheck argument, to compare snap files
calls the function to compare snapshots based on arguments given
(--check, --snapcheck, --diff)
:param hostname: device name
:return: return object of Operator containing test details
"""
comp = Comparator()
chk = self.args.check
diff = self.args.diff
pre_snap_file = self.args.pre_snapfile if pre_snap is None else pre_snap
if (chk or diff or action in ["check", "diff"]):
post_snap_file = self.args.post_snapfile if post_snap is None else post_snap
test_obj = comp.generate_test_files(config_data, hostname, chk,
diff, self.db, self.snap_del,
pre_snap_file, action,
post_snap_file)
else:
test_obj = comp.generate_test_files(config_data, hostname, chk,
diff, self.db, self.snap_del,
pre_snap_file, action)
return test_obj
def get_values(self, key_value):
del_value = ['device', 'username', 'passwd']
for v in del_value:
if key_value.has_key(v):
del key_value[v]
return key_value
def login(self, output_file):
"""
Extract device information from main config file. Stores device information and call connect function,
device can be single or multiple. Instead of connecting to all devices mentioned in yaml file, user can
connect to some particular group of device also.
:param output_file: name of snapshot file
"""
self.host_list = []
if self.args.hostname is None:
try:
k = self.main_file['hosts'][0]
except KeyError as ex:
self.logger.error(
colorama.Fore.RED +
"\nERROR occurred !! Hostname not given properly %s" %
str(ex),
extra=self.log_detail)
#raise Exception(ex)
except Exception as ex:
self.logger.error(colorama.Fore.RED +
"\nERROR occurred !! %s" % str(ex),
extra=self.log_detail)
#raise Exception(ex)
else:
# when group of devices are given, searching for include keyword in
# hosts in main.yaml file
if k.__contains__('include'):
file_tag = k['include']
if os.path.isfile(file_tag):
lfile = file_tag
else:
lfile = os.path.join(
get_path('DEFAULT', 'test_file_path'), file_tag)
login_file = open(lfile, 'r')
dev_file = yaml.load(login_file)
gp = k.get('group', 'all')
dgroup = [i.strip().lower() for i in gp.split(',')]
for dgp in dev_file:
if dgroup[0].lower() == 'all' or dgp.lower() in dgroup:
for val in dev_file[dgp]:
hostname = val.keys()[0]
self.log_detail = {'hostname': hostname}
self.host_list.append(hostname)
key_value = deepcopy(val.get(hostname))
if val.get(
hostname
) is not None and 'username' in val.get(
hostname).keys():
username = val.get(hostname).get(
'username')
else:
username = self.args.login
if val.get(
hostname
) is not None and 'passwd' in val.get(
hostname).keys():
password = val.get(hostname).get('passwd')
else:
password = self.args.passwd
# if self.args.passwd is not None else
# getpass.getpass("\nEnter Password for
# username: %s " %username)
key_value = self.get_values(key_value)
t = Thread(target=self.connect,
args=(hostname, username, password,
output_file),
kwargs=key_value)
t.start()
t.join()
# login credentials are given in main config file, can connect to only
# one device
else:
key_value = deepcopy(k)
try:
hostname = k['device']
self.log_detail = {'hostname': hostname}
except KeyError as ex:
self.logger.error(
colorama.Fore.RED +
"ERROR!! KeyError 'device' key not found",
extra=self.log_detail)
#raise Exception(ex)
except Exception as ex:
self.logger.error(colorama.Fore.RED +
"ERROR!! %s" % ex,
extra=self.log_detail)
#raise Exception(ex)
else:
username = k.get('username') or self.args.login
password = k.get('passwd') or self.args.passwd
self.host_list.append(hostname)
key_value = self.get_values(key_value)
self.connect(hostname, username, password, output_file,
**key_value)
# login credentials are given from command line
else:
hostname = self.args.hostname
self.log_detail = {'hostname': hostname}
username = self.args.login
password = self.args.passwd
# if self.args.passwd is not None else getpass.getpass("\nEnter
# Password: "******"""
Analyse testfile and return object of testop.Operator containing test details
called by connect() function and other functions of Jsnapy module functions
:param config_data: data of main config file
:param hostname: hostname
:param snap_file: pre snapshot file name
:param post_snap: post snapshot file name
:param action: action to be taken (check, snapcheck, snap)
:return: object of testop.Operator containing test details
"""
res = Operator()
if config_data.get("mail") and self.args.diff is not True:
mfile = os.path.join(get_path('DEFAULT', 'test_file_path'), config_data.get('mail'))\
if os.path.isfile(config_data.get('mail')) is False else config_data.get('mail')
if os.path.isfile(mfile):
mail_file = open(mfile, 'r')
mail_file = yaml.load(mail_file)
if "passwd" not in mail_file:
passwd = getpass.getpass("Please enter ur email password ")
else:
passwd = mail_file['passwd']
res = self.compare_tests(hostname, config_data, snap_file,
post_snap, action)
send_mail = Notification()
send_mail.notify(mail_file, hostname, passwd, res)
else:
self.logger.error(
colorama.Fore.RED +
"ERROR!! Path of file containing mail content is not correct",
extra=self.log_detail)
else:
res = self.compare_tests(hostname, config_data, snap_file,
post_snap, action)
self.q.put(res)
return res
def connect(self,
hostname,
username,
password,
output_file,
config_data=None,
action=None,
post_snap=None,
**kwargs):
"""
connect to device and calls the function either to generate snapshots
or compare them based on option given (--snap, --check, --snapcheck, --diff)
:param hostname: ip/ hostname of device
:param username: username of device
:param password: password to connect to device
:param snap_files: file name to store snapshot
:return: if snap operation is performed then return true on success
if snapcheck or check operation is performed then return test details
"""
res = None
if config_data is None:
config_data = self.main_file
if self.args.snap is True or self.args.snapcheck is True or action in [
"snap", "snapcheck"
]:
self.logger.info(colorama.Fore.BLUE +
"Connecting to device %s ................",
hostname,
extra=self.log_detail)
if username is None:
username = raw_input("\nEnter User name: ")
dev = Device(host=hostname,
user=username,
passwd=password,
gather_facts=False,
**kwargs)
try:
dev.open()
except ConnectAuthError as ex:
if password is None and action is None:
password = getpass.getpass(
"\nEnter Password for username <%s> : " % username)
self.connect(hostname, username, password, output_file,
config_data, action, post_snap)
else:
self.logger.error(colorama.Fore.RED +
"\nERROR occurred %s" % str(ex),
extra=self.log_detail)
raise Exception(ex)
except Exception as ex:
self.logger.error(colorama.Fore.RED +
"\nERROR occurred %s" % str(ex),
extra=self.log_detail)
raise Exception(ex)
else:
res = self.generate_rpc_reply(dev, output_file, hostname,
config_data)
self.snap_q.put(res)
dev.close()
if self.args.check is True or self.args.snapcheck is True or self.args.diff is True or action in [
"check", "snapcheck"
]:
res = self.get_test(config_data, hostname, output_file, post_snap,
action)
return res
############################### functions to support module ##############
def multiple_device_details(self, host, config_data, pre_name, action,
post_name):
"""
Called when multiple devices are given in config file
:param host: hostname
:param config_data: data of main config file
:param pre_name: pre snapshot filename or file tag
:param action: action to be taken, snap, snapcheck, check
:param post_name: post snapshot filename or file tag
:return: return object of testop.Operator containing test details
"""
res_obj = []
self.host_list = []
login_file = host['include']
login_file = login_file if os.path.isfile(
host.get('include')) else os.path.join(
get_path('DEFAULT', 'test_file_path'), login_file)
login_file = open(login_file, 'r')
dev_file = yaml.load(login_file)
gp = host.get('group', 'all')
dgroup = [i.strip().lower() for i in gp.split(',')]
for dgp in dev_file:
if dgroup[0].lower() == 'all' or dgp.lower() in dgroup:
for val in dev_file[dgp]:
hostname = val.keys()[0]
self.host_list.append(hostname)
self.log_detail['hostname'] = hostname
username = val.get(hostname).get('username')
password = val.get(hostname).get('passwd')
key_value = val.get(hostname)
key_value = self.get_values(key_value)
t = Thread(target=self.connect,
args=(hostname, username, password, pre_name,
config_data, action, post_name),
kwargs=key_value)
t.start()
if action == "snap":
res_obj.append(self.snap_q.get())
elif action in ["snapcheck", "check"]:
res_obj.append(self.q.get())
else:
res_obj.append(None)
t.join()
return res_obj
def extract_data(self,
config_data,
pre_name=None,
action=None,
post_name=None):
"""
Called when dev= None, i.e. device details are passed inside config file
It parse details of main config file and call functions to connect to device
and take snapshots
:param config_data: data of main config file
:param pre_name: pre snapshot filename or file tag
:param action: action to be taken, snap, snapcheck, check
:param post_name: post snapshot filename or file tag
:return: return object of testop.Operator containing test details
"""
val = []
if os.path.isfile(config_data):
data = open(config_data, 'r')
config_data = yaml.load(data)
elif isinstance(config_data, str):
config_data = yaml.load(config_data)
else:
self.logger.info(colorama.Fore.RED +
"Incorrect config file or data, please chk !!!!",
extra=self.log_detail)
exit(1)
try:
host = config_data.get('hosts')[0]
except Exception as ex:
self.logger.error(colorama.Fore.RED +
"ERROR!! config file %s is not present" % ex,
extra=self.log_detail)
raise Exception("config file is not present ", ex)
else:
if config_data.__contains__('sqlite') and config_data[
'sqlite'] and config_data['sqlite'][0]:
self.chk_database(config_data, pre_name, post_name, None, None,
action)
if host.__contains__('include'):
res_obj = self.multiple_device_details(host, config_data,
pre_name, action,
post_name)
return res_obj
else:
hostname = host.get('device')
self.log_detail = {'hostname': hostname}
username = host.get('username')
password = host.get('passwd')
key_value = host
key_value = self.get_values(key_value)
#pre_name = hostname + '_' + pre_name if not os.path.isfile(pre_name) else pre_name
# if action is "check":
# post_name= hostname + '_' + post_name if not os.path.isfile(post_name) else post_name
val.append(
self.connect(hostname, username, password, pre_name,
config_data, action, post_name, **key_value))
return val
def extract_dev_data(self,
dev,
config_data,
pre_name=None,
action=None,
post_snap=None):
"""
Used to parse details given in main config file, when device object is passed in function
:param dev: Device object
:param config_data: data of main config file
:param pre_name: pre snapshot filename or file tag
:param action: action to be taken, snap, check or snapcheck
:param post_snap: post snapshot filename or file tag
:return: return object of testop.Operator containing test details
"""
res = []
if isinstance(config_data, dict):
pass
elif os.path.isfile(config_data):
data = open(config_data, 'r')
config_data = yaml.load(data)
elif isinstance(config_data, str):
config_data = yaml.load(config_data)
else:
self.logger.info(colorama.Fore.RED +
"Incorrect config file or data, please chk !!!!",
extra=self.log_detail)
exit(1)
try:
hostname = dev.hostname
self.log_detail = {'hostname': hostname}
except Exception as ex:
self.logger.error(colorama.Fore.RED +
"ERROR!! message is: %s" % ex,
extra=self.log_detail)
raise Exception(ex)
else:
if config_data.__contains__('sqlite') and config_data[
'sqlite'] and config_data['sqlite'][0]:
self.chk_database(config_data, pre_name, post_snap, None, None,
action)
if action in ["snap", "snapcheck"]:
try:
res.append(
self.generate_rpc_reply(dev, pre_name, hostname,
config_data))
except Exception as ex:
self.logger.error(colorama.Fore.RED +
"\nERROR occurred %s" % str(ex),
extra=self.log_detail)
res.append(None)
if action in ["snapcheck", "check"]:
res = []
res.append(
self.get_test(config_data, hostname, pre_name, post_snap,
action))
return res
def snap(self, data, file_name, dev=None):
"""
Function equivalent to --snap operator, for module version
:param data: either main config file or string containing details of main config file
:param file_name: snap file, either complete filename or file tag
:param dev: device object
"""
if isinstance(dev, Device):
res = self.extract_dev_data(dev, data, file_name, "snap")
else:
res = self.extract_data(data, file_name, "snap")
return res
def snapcheck(self, data, file_name=None, dev=None):
"""
Function equivalent to --snapcheck operator, for module version
:param data: either main config file or string containing details of main config file
:param pre_file: pre snap file, either complete filename or file tag
:param dev: device object
:return: return object of testop.Operator containing test details
"""
if file_name is None:
file_name = "snap_temp"
self.snap_del = True
if isinstance(dev, Device):
res = self.extract_dev_data(dev, data, file_name, "snapcheck")
else:
res = self.extract_data(data, file_name, "snapcheck")
return res
def check(self, data, pre_file=None, post_file=None, dev=None):
"""
Function equivalent to --check operator, for module version
:param data: either main config file or string containing details of main config file
:param pre_file: pre snap file, either complete filename or file tag
:param post_file: post snap file, either complete filename or file tag
:param dev: device object
:return: return object of testop.Operator containing test details
"""
if isinstance(dev, Device):
res = self.extract_dev_data(dev, data, pre_file, "check",
post_file)
else:
res = self.extract_data(data, pre_file, "check", post_file)
return res
####### generate init folder ######
'''
def generate_init(self):
create snapshots and configs folder along with sample main config file.
All snapshots generated will go in snapshots folder. configs folder will contain
all the yaml file apart from main, like device.yml, bgp_neighbor.yml
:return:
mssg= "Creating Jsnapy directory structure at:" + os.getcwd()
self.logger.debug(colorama.Fore.BLUE + mssg)
if not os.path.isdir("snapshots"):
os.mkdir("snapshots")
if not os.path.isdir("logs"):
os.mkdir("logs")
dst_config_path = os.path.join(os.getcwd(), 'configs')
# overwrite files if given option -o or --overwrite
if not os.path.isdir(dst_config_path) or self.args.overwrite is True:
distutils.dir_util.copy_tree(os.path.join(os.path.dirname(__file__), 'configs'),
dst_config_path)
dst_main_yml = os.path.join(dst_config_path, 'main.yml')
if not os.path.isfile(
os.path.join(os.getcwd(), 'main.yml')) or self.args.overwrite is True:
shutil.copy(dst_main_yml, os.getcwd())
logging_yml_file = os.path.join(
os.path.dirname(__file__),
'logging.yml')
if not os.path.isfile(
os.path.join(os.getcwd(), 'logging.yml')) or self.args.overwrite is True:
shutil.copy(logging_yml_file, os.getcwd())
mssg1= "Jsnap folders created at: " + os.getcwd()
self.logger.info(colorama.Fore.BLUE + mssg1)
'''
def check_arguments(self):
"""
checks combination of arguments given from command line and display help if correct
set of combination is not given.
:return: print message in command line, regarding correct usage of JSNAPy
"""
## only four test operation is permitted, if given anything apart from this, then it should print error message
if (self.args.snap is False and self.args.snapcheck is False
and self.args.check is False and self.args.diff is False
and self.args.version is False):
self.logger.error(
colorama.Fore.RED +
"Arguments not given correctly, Please refer help message",
extra=self.log_detail)
self.parser.print_help()
sys.exit(1)
if ((self.args.snap is True and
(self.args.pre_snapfile is None or self.args.file is None))
or (self.args.snapcheck is True and self.args.file is None)
or (self.args.check is True and self.args.file is None)):
self.logger.error(
colorama.Fore.RED +
"Arguments not given correctly, Please refer help message",
extra=self.log_detail)
self.parser.print_help()
sys.exit(1)
if self.args.diff is True:
if (self.args.pre_snapfile is not None
and os.path.isfile(self.args.pre_snapfile)) and (
self.args.post_snapfile is not None
and os.path.isfile(self.args.post_snapfile)):
comp = Comparator()
comp.compare_diff(self.args.pre_snapfile,
self.args.post_snapfile, None)
sys.exit(1)
else:
if self.args.file is None:
self.parser.print_help()
sys.exit(1)
async def startup_event() -> None:
from setup_logging import setup_logging
setup_logging()
sys.path.insert(1,os.path.join(parentPath,"lib"))
sys.path.insert(2,os.path.join(parentPath,"mirs_tc"))
#print("System Path: {}".format(sys.path))
except:
print "Unable to load local library paths"
sys.exit(1)
# Local modules
import setup_logging
import error_codes
import utils
import MIRS_TC
# Setup Logging using logging templates
LOG = logging.getLogger("poolQueryCoordText") #create the logger for this file
setup_logging.setup_logging("poolQueryCoordText","pqct.log")
LOG.info("Starting: {}".format(os.path.basename(__file__)))
# Read command line arguments
options = argparse.ArgumentParser()
options.add_argument("-c", "--covariables", dest="covariables", help="Coordinate Variable Files e.g. lat.txt,lon.txt")
options.add_argument("-v", "--variables", dest="variables", help="Variable Files e.g. TPW.txt,LWP.txt,Qc.txt")
options.add_argument("-o", "--output", dest="output", help="Output Text File")
try:
args = options.parse_args()
except:
msg='Syntax: python poolQueryCoordText.py -i <input text file> <output text file>'
utils.error(LOG,msg,error_codes.EX_USAGE)
# Read in lat,lon, variable list in
def main(plan, native_model_weights_filepath, collaborators_file, feature_shape, n_classes, data_config_fname, logging_config_path, logging_default_level, model_device):
"""Creates a protobuf file of the initial weights for the model
Uses the federation (FL) plan to create an initial weights file
for the federation.
Args:
plan: The federation (FL) plan filename
native_model_weights_filepath: A framework-specific filepath. Path will be relative to the working directory.
collaborators_file:
feature_shape: The input shape to the model
data_config_fname: The data configuration file (defines where the datasets are located)
logging_config_path: The log path
logging_default_level (int): The default log level
"""
setup_logging(path=logging_config_path, default_level=logging_default_level)
logger = logging.getLogger(__name__)
# FIXME: consistent filesystem (#15)
script_dir = os.path.dirname(os.path.realpath(__file__))
base_dir = os.path.join(script_dir, 'federations')
plan_dir = os.path.join(base_dir, 'plans')
weights_dir = os.path.join(base_dir, 'weights')
# ensure the weights dir exists
if not os.path.exists(weights_dir):
print('creating folder:', weights_dir)
os.makedirs(weights_dir)
# parse the plan and local config
flplan = parse_fl_plan(os.path.join(plan_dir, plan))
local_config = load_yaml(os.path.join(base_dir, data_config_fname))
# get the output filename
fpath = os.path.join(weights_dir, flplan['aggregator_object_init']['init_kwargs']['init_model_fname'])
# create the data object for models whose architecture depends on the feature shape
if feature_shape is None:
if collaborators_file is None:
sys.exit("You must specify either a feature shape or a collaborator list in order for the script to determine the input layer shape")
# FIXME: this will ultimately run in a governor environment and should not require any data to work
# pick the first collaborator to create the data and model (could be any)
collaborator_common_name = load_yaml(os.path.join(base_dir, 'collaborator_lists', collaborators_file))['collaborator_common_names'][0]
data = create_data_object(flplan, collaborator_common_name, local_config, n_classes=n_classes)
else:
data = get_object('openfl.data.dummy.randomdata', 'RandomData', feature_shape=feature_shape)
logger.info('Using data object of type {} and feature shape {}'.format(type(data), feature_shape))
# create the model object and compression pipeline
wrapped_model = create_model_object(flplan, data, model_device=model_device)
compression_pipeline = create_compression_pipeline(flplan)
# determine if we need to store the optimizer variables
# FIXME: what if this key is missing?
try:
opt_treatment = OptTreatment[flplan['collaborator_object_init']['init_kwargs']['opt_treatment']]
except KeyError:
# FIXME: this error message should use the exception to determine the missing key and the Enum to display the options dynamically
sys.exit("FL plan must specify ['collaborator_object_init']['init_kwargs']['opt_treatment'] as [RESET|CONTINUE_LOCAL|CONTINUE_GLOBAL]")
# FIXME: this should be an "opt_treatment requires parameters type check rather than a magic string"
with_opt_vars = opt_treatment == OptTreatment['CONTINUE_GLOBAL']
if native_model_weights_filepath is not None:
wrapped_model.load_native(native_model_weights_filepath)
tensor_dict_split_fn_kwargs = wrapped_model.tensor_dict_split_fn_kwargs or {}
tensor_dict, holdout_params = split_tensor_dict_for_holdouts(logger,
wrapped_model.get_tensor_dict(with_opt_vars=with_opt_vars),
**tensor_dict_split_fn_kwargs)
logger.warn('Following paramters omitted from global initial model, '\
'local initialization will determine values: {}'.format(list(holdout_params.keys())))
model_proto = construct_proto(tensor_dict=tensor_dict,
model_id=wrapped_model.__class__.__name__,
model_version=0,
is_delta=False,
delta_from_version=-1,
compression_pipeline=compression_pipeline)
dump_proto(model_proto=model_proto, fpath=fpath)
logger.info("Created initial weights file: {}".format(fpath))
