storage as it helps avoid storage of credentials in the
database.
Possible values:
* None
* String value representing a valid configuration file path
Related options:
* None
""")),
]
# SafeConfigParser was deprecated in Python 3.2
if sys.version_info >= (3, 2):
CONFIG = configparser.ConfigParser()
else:
CONFIG = configparser.SafeConfigParser()
LOG = logging.getLogger(__name__)
CONF = cfg.CONF
CONF.register_opts(swift_opts)
def is_multiple_swift_store_accounts_enabled():
if CONF.swift_store_config_file is None:
return False
return True
def main(_):
'''does everything for testing'''
decoder_cfg_file = None
#read the database config file
parsed_database_cfg = configparser.ConfigParser()
parsed_database_cfg.read(os.path.join(FLAGS.expdir, 'database.cfg'))
database_cfg = dict(parsed_database_cfg.items('database'))
#read the asr config file
parsed_nnet_cfg = configparser.ConfigParser()
parsed_nnet_cfg.read(os.path.join(FLAGS.expdir, 'model', 'lm.cfg'))
nnet_cfg = dict(parsed_nnet_cfg.items('lm'))
#read the decoder config file
if decoder_cfg_file is None:
decoder_cfg_file = os.path.join(FLAGS.expdir, 'model', 'decoder.cfg')
parsed_decoder_cfg = configparser.ConfigParser()
parsed_decoder_cfg.read(decoder_cfg_file)
decoder_cfg = dict(parsed_decoder_cfg.items('decoder'))
#create the coder
with open(os.path.join(FLAGS.expdir, 'model', 'alphabet')) as fid:
alphabet = fid.read().split(' ')
coder = target_coder.TargetCoder(alphabet)
#read the maximum length
with open(os.path.join(database_cfg['test_dir'], 'max_num_chars')) as fid:
max_length = int(fid.read())
#create a text reader
textreader = text_reader.TextReader(textfile=os.path.join(
database_cfg['test_dir'], 'text'),
max_length=max_length,
coder=coder)
#create the classifier
classifier = lm_factory.factory(conf=nnet_cfg, output_dim=coder.num_labels)
#create a decoder
graph = tf.Graph()
with graph.as_default():
decoder = decoder_factory.factory(conf=decoder_cfg,
classifier=classifier,
input_dim=1,
max_input_length=max_length,
coder=coder,
expdir=FLAGS.expdir)
saver = tf.train.Saver(tf.trainable_variables())
config = tf.ConfigProto()
config.gpu_options.allow_growth = True #pylint: disable=E1101
config.allow_soft_placement = True
with tf.Session(graph=graph, config=config) as sess:
#load the model
saver.restore(sess, os.path.join(FLAGS.expdir, 'model',
'network.ckpt'))
#decode with te neural net
decoded = decoder.decode(textreader, sess)
#compute the character error rate
score = decoder.score(decoded, None)
print 'perplexity: %f' % score
def read_settings(self):
''' Reads the settings from the packet_net.ini file '''
if six.PY3:
config = configparser.ConfigParser()
else:
config = configparser.SafeConfigParser()
_ini_path_raw = os.environ.get('PACKET_NET_INI_PATH')
if _ini_path_raw:
packet_ini_path = os.path.expanduser(
os.path.expandvars(_ini_path_raw))
else:
packet_ini_path = os.path.join(
os.path.dirname(os.path.realpath(__file__)), 'packet_net.ini')
config.read(packet_ini_path)
# items per page
self.items_per_page = 999
if config.has_option(ini_section, 'items_per_page'):
config.get(ini_section, 'items_per_page')
# Instance states to be gathered in inventory. Default is all of them.
packet_valid_device_states = [
'active', 'inactive', 'queued', 'provisioning'
]
self.packet_device_states = []
if config.has_option(ini_section, 'device_states'):
for device_state in config.get(ini_section,
'device_states').split(','):
device_state = device_state.strip()
if device_state not in packet_valid_device_states:
continue
self.packet_device_states.append(device_state)
else:
self.packet_device_states = packet_valid_device_states
# Cache related
cache_dir = os.path.expanduser(config.get(ini_section, 'cache_path'))
if not os.path.exists(cache_dir):
os.makedirs(cache_dir)
self.cache_path_cache = cache_dir + "/ansible-packet.cache"
self.cache_path_index = cache_dir + "/ansible-packet.index"
self.cache_max_age = config.getint(ini_section, 'cache_max_age')
# Configure nested groups instead of flat namespace.
if config.has_option(ini_section, 'nested_groups'):
self.nested_groups = config.getboolean(ini_section,
'nested_groups')
else:
self.nested_groups = False
# Replace dash or not in group names
if config.has_option(ini_section, 'replace_dash_in_groups'):
self.replace_dash_in_groups = config.getboolean(
ini_section, 'replace_dash_in_groups')
else:
self.replace_dash_in_groups = True
# Configure which groups should be created.
group_by_options = [
'group_by_device_id',
'group_by_hostname',
'group_by_facility',
'group_by_project',
'group_by_operating_system',
'group_by_plan_type',
'group_by_tags',
'group_by_tag_none',
]
for option in group_by_options:
if config.has_option(ini_section, option):
setattr(self, option, config.getboolean(ini_section, option))
else:
setattr(self, option, True)
# Do we need to just include hosts that match a pattern?
try:
pattern_include = config.get(ini_section, 'pattern_include')
if pattern_include and len(pattern_include) > 0:
self.pattern_include = re.compile(pattern_include)
else:
self.pattern_include = None
except configparser.NoOptionError:
self.pattern_include = None
# Do we need to exclude hosts that match a pattern?
try:
pattern_exclude = config.get(ini_section, 'pattern_exclude')
if pattern_exclude and len(pattern_exclude) > 0:
self.pattern_exclude = re.compile(pattern_exclude)
else:
self.pattern_exclude = None
except configparser.NoOptionError:
self.pattern_exclude = None
# Projects
self.projects = []
configProjects = config.get(ini_section, 'projects')
configProjects_exclude = config.get(ini_section, 'projects_exclude')
if (configProjects == 'all'):
for projectInfo in self.get_projects():
if projectInfo.name not in configProjects_exclude:
self.projects.append(projectInfo.name)
else:
self.projects = configProjects.split(",")
def read_settings(self):
''' Reads the settings from the vmware_inventory.ini file '''
scriptbasename = __file__
scriptbasename = os.path.basename(scriptbasename)
scriptbasename = scriptbasename.replace('.py', '')
defaults = {
'vmware': {
'server':
'',
'port':
443,
'username':
'',
'password':
'',
'ini_path':
os.path.join(os.path.dirname(__file__),
'%s.ini' % scriptbasename),
'cache_name':
'ansible-vmware',
'cache_path':
'~/.ansible/tmp',
'cache_max_age':
3600,
'max_object_level':
1,
'alias_pattern':
'{{ config.name + "_" + config.uuid }}',
'host_pattern':
'{{ guest.ipaddress }}',
'host_filters':
'{{ guest.gueststate == "running" }}',
'groupby_patterns':
'{{ guest.guestid }},{{ "templates" if config.template else "guests"}}',
'lower_var_keys':
True
}
}
if six.PY3:
config = configparser.ConfigParser()
else:
config = configparser.SafeConfigParser()
# where is the config?
vmware_ini_path = os.environ.get('VMWARE_INI_PATH',
defaults['vmware']['ini_path'])
vmware_ini_path = os.path.expanduser(
os.path.expandvars(vmware_ini_path))
config.read(vmware_ini_path)
# apply defaults
for k, v in defaults['vmware'].iteritems():
if not config.has_option('vmware', k):
config.set('vmware', k, str(v))
# where is the cache?
self.cache_dir = os.path.expanduser(config.get('vmware', 'cache_path'))
if self.cache_dir and not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
# set the cache filename and max age
cache_name = config.get('vmware', 'cache_name')
self.cache_path_cache = self.cache_dir + "/%s.cache" % cache_name
self.cache_max_age = int(config.getint('vmware', 'cache_max_age'))
# mark the connection info
self.server = os.environ.get('VMWARE_SERVER',
config.get('vmware', 'server'))
self.port = int(
os.environ.get('VMWARE_PORT', config.get('vmware', 'port')))
self.username = os.environ.get('VMWARE_USERNAME',
config.get('vmware', 'username'))
self.password = os.environ.get('VMWARE_PASSWORD',
config.get('vmware', 'password'))
# behavior control
self.maxlevel = int(config.get('vmware', 'max_object_level'))
self.lowerkeys = config.get('vmware', 'lower_var_keys')
if type(self.lowerkeys) != bool:
if str(self.lowerkeys).lower() in ['yes', 'true', '1']:
self.lowerkeys = True
else:
self.lowerkeys = False
self.host_filters = list(
config.get('vmware', 'host_filters').split(','))
self.groupby_patterns = list(
config.get('vmware', 'groupby_patterns').split(','))
# save the config
self.config = config
def _search_launchers(directories):
ini_name = 'launcher.ini'
config_defaults = {
'name': 'Launcher',
'command': '',
'description': '',
'image': '',
'shell': 'false',
'workdir': '.',
'type': '',
'manufacturer': '',
'model': '',
'variant': '',
'priority': '0',
}
launchers = []
ids = {}
index = 0
for root_dir in directories:
for root, _, files in os.walk(root_dir):
if ini_name not in files:
continue
root = os.path.abspath(os.path.expanduser(root))
ini_file = os.path.join(root, ini_name)
cfg = configparser.ConfigParser(config_defaults)
cfg.read(ini_file)
for section in cfg.sections():
launcher = Launcher()
# descriptive data
launcher.name = cfg.get(section, 'name')
launcher.description = cfg.get(section, 'description')
info = MachineInfo()
info.type = cfg.get(section, 'type')
info.manufacturer = cfg.get(section, 'manufacturer')
info.model = cfg.get(section, 'model')
info.variant = cfg.get(section, 'variant')
launcher.priority = cfg.getint(section, 'priority')
launcher.importance = 0
launcher.info.MergeFrom(info)
# command data
launcher.command = cfg.get(section, 'command')
launcher.shell = cfg.getboolean(section, 'shell')
workdir = cfg.get(section, 'workdir')
if not os.path.isabs(workdir):
workdir = os.path.join(root, workdir)
launcher.workdir = os.path.normpath(workdir)
launcher.returncode = 0
launcher.running = False
launcher.terminating = False
# storing the image file
image_file = cfg.get(section, 'image')
if image_file is not '':
if not os.path.isabs(image_file):
image_file = os.path.join(root, image_file)
file_buffer = open(image_file, 'rb').read()
image = File()
image.name = os.path.basename(image_file)
image.encoding = CLEARTEXT
image.blob = file_buffer
launcher.image.MergeFrom(image)
launcher.index = index
index += 1
launchers.append(launcher)
ids[launcher.index] = '%s:%s' % (root, section)
# sort using the priority attribute before distribution
return sorted(launchers, key=attrgetter('priority'), reverse=True), ids
def change_repository_name_in_hgrc_file(hgrc_file, new_name):
config = configparser.ConfigParser()
config.read(hgrc_file)
config.set('web', 'name', new_name)
with open(hgrc_file, 'wb') as fh:
config.write(fh)
def _get_base_config(self):
cp = configparser.ConfigParser()
cp.add_section('connection')
cp.set('connection', 'api_key', 'exampleapikey')
cp.set('connection', 'location', 'ams1')
return cp
def main(_):
'''main function'''
#pointers to the config files
computing_cfg_file = 'config/computing/non_distributed.cfg'
database_cfg_file = 'config/asr_databases/TIMIT.conf'
if FLAGS.type == 'asr':
feat_cfg_file = 'config/features/fbank.cfg'
classifier_cfg_file = 'config/asr/LASACNN.cfg'
trainer_cfg_file = 'config/trainer/cross_entropytrainer.cfg'
decoder_cfg_file = 'config/decoder/BeamSearchDecoder.cfg'
#read the computing config file
parsed_computing_cfg = configparser.ConfigParser()
parsed_computing_cfg.read(computing_cfg_file)
computing_cfg = dict(parsed_computing_cfg.items('computing'))
#read the trainer config file
parsed_trainer_cfg = configparser.ConfigParser()
parsed_trainer_cfg.read(trainer_cfg_file)
trainer_cfg = dict(parsed_trainer_cfg.items('trainer'))
if os.path.isdir(os.path.join(FLAGS.expdir, 'processes')):
shutil.rmtree(os.path.join(FLAGS.expdir, 'processes'))
os.makedirs(os.path.join(FLAGS.expdir, 'processes'))
if trainer_cfg['resume_training'] == 'True':
if not os.path.isdir(FLAGS.expdir):
raise Exception(
'cannot find %s, please set resume_training to '
'False if you want to start a new training process' %
FLAGS.expdir)
else:
if os.path.isdir(os.path.join(FLAGS.expdir, 'logdir')):
shutil.rmtree(os.path.join(FLAGS.expdir, 'logdir'))
if not os.path.isdir(FLAGS.expdir):
os.makedirs(FLAGS.expdir)
if not os.path.isdir(os.path.join(FLAGS.expdir, 'model')):
os.makedirs(os.path.join(FLAGS.expdir, 'model'))
#copy the configs to the expdir so they can be read there and the
#experiment information is stored
shutil.copyfile(database_cfg_file,
os.path.join(FLAGS.expdir, 'database.cfg'))
if FLAGS.type == 'asr':
shutil.copyfile(
feat_cfg_file,
os.path.join(FLAGS.expdir, 'model', 'features.cfg'))
shutil.copyfile(
classifier_cfg_file,
os.path.join(FLAGS.expdir, 'model', '%s.cfg' % FLAGS.type))
shutil.copyfile(computing_cfg_file,
os.path.join(FLAGS.expdir, 'computing.cfg'))
shutil.copyfile(trainer_cfg_file, os.path.join(FLAGS.expdir,
'trainer.cfg'))
shutil.copyfile(decoder_cfg_file,
os.path.join(FLAGS.expdir, 'model', 'decoder.cfg'))
if computing_cfg['distributed'] == 'condor_non-distributed':
if not os.path.isdir(os.path.join(FLAGS.expdir, 'outputs')):
os.makedirs(os.path.join(FLAGS.expdir, 'outputs'))
subprocess.call([
'condor_submit',
'expdir=%s' % FLAGS.expdir,
'memory=%s' % computing_cfg['minmemory'],
'type=%s' % FLAGS.type,
'nabu/distributed/condor/non_distributed.job'
])
elif computing_cfg['distributed'] == 'non-distributed':
if FLAGS.type == 'asr':
train_asr(clusterfile=None,
job_name='local',
task_index=0,
ssh_command='None',
expdir=FLAGS.expdir)
else:
train_lm(clusterfile=None,
job_name='local',
task_index=0,
ssh_command='None',
expdir=FLAGS.expdir)
elif computing_cfg['distributed'] == 'local':
#create the directories
if not os.path.isdir(os.path.join(FLAGS.expdir, 'outputs')):
os.makedirs(os.path.join(FLAGS.expdir, 'outputs'))
if not os.path.isdir(os.path.join(FLAGS.expdir, 'cluster')):
os.makedirs(os.path.join(FLAGS.expdir, 'cluster'))
#create the cluster file
with open(os.path.join(FLAGS.expdir, 'cluster', 'cluster'),
'w') as fid:
port = 1024
for _ in range(int(computing_cfg['numps'])):
while not cluster.port_available(port):
port += 1
fid.write('ps,localhost,%d,\n' % port)
port += 1
for i in range(int(computing_cfg['numworkers'])):
while not cluster.port_available(port):
port += 1
fid.write('worker,localhost,%d,%d\n' % (port, i))
port += 1
#start the training
local_cluster.local_cluster(FLAGS.expdir, FLAGS.type)
elif computing_cfg['distributed'] == 'static':
#read the cluster file
machines = dict()
machines['worker'] = []
machines['ps'] = []
with open(computing_cfg['clusterfile']) as fid:
for line in fid:
if line.strip():
split = line.strip().split(',')
machines[split[0]].append(split[1])
#create the outputs directory
if not os.path.isdir(os.path.join(FLAGS.expdir, 'outputs')):
os.makedirs(os.path.join(FLAGS.expdir, 'outputs'))
#run all the jobs
processes = dict()
processes['worker'] = []
processes['ps'] = []
for job in machines:
task_index = 0
for machine in machines[job]:
command = ('python -u train_%s.py --clusterfile=%s '
'--job_name=%s --task_index=%d --ssh_command=%s '
'--expdir=%s') % (
FLAGS.type, computing_cfg['clusterfile'], job,
task_index, computing_cfg['ssh_command'],
FLAGS.expdir)
processes[job].append(
run_remote.run_remote(command=command, host=machine))
task_index += 1
#make sure the created processes are terminated at exit
for job in processes:
for process in processes[job]:
atexit.register(process.terminate)
#make sure all remotely created processes are terminated at exit
atexit.register(kill_processes.kill_processes,
processdir=os.path.join(FLAGS.expdir, 'processes'))
#wait for all worker processes to finish
for process in processes['worker']:
process.wait()
elif computing_cfg['distributed'] == 'condor':
#create the directories
if not os.path.isdir(os.path.join(FLAGS.expdir, 'outputs')):
os.makedirs(os.path.join(FLAGS.expdir, 'outputs'))
if os.path.isdir(os.path.join(FLAGS.expdir, 'cluster')):
shutil.rmtree(os.path.join(FLAGS.expdir, 'cluster'))
os.makedirs(os.path.join(FLAGS.expdir, 'cluster'))
#submit the parameter server jobs
subprocess.call([
'condor_submit',
'expdir=%s' % FLAGS.expdir,
'numjobs=%s' % computing_cfg['numps'],
'type=%s' % FLAGS.type,
'ssh_command=%s' % computing_cfg['ssh_command'],
'nabu/distributed/condor/ps.job'
])
#submit the worker jobs
subprocess.call([
'condor_submit',
'expdir=%s' % FLAGS.expdir,
'numjobs=%s' % computing_cfg['numworkers'],
'memory=%s' % computing_cfg['minmemory'],
'type=%s' % FLAGS.type,
'ssh_command=%s' % computing_cfg['ssh_command'],
'nabu/distributed/condor/worker.job'
])
ready = False
try:
print 'waiting for the machines to report...'
numworkers = 0
numps = 0
while not ready:
#check the machines in the cluster
machines = cluster.get_machines(
os.path.join(FLAGS.expdir, 'cluster'))
if (len(machines['ps']) > numps
or len(machines['worker']) > numworkers):
numworkers = len(machines['worker'])
numps = len(machines['ps'])
print('parameter servers ready %d/%s' %
(len(machines['ps']), computing_cfg['numps']))
print(
'workers ready %d/%s' %
(len(machines['worker']), computing_cfg['numworkers']))
print 'press Ctrl-C to run with the current machines'
#check if the required amount of machines has reported
if (len(machines['worker']) == int(computing_cfg['numworkers'])
and len(machines['ps']) == int(
computing_cfg['numps'])):
ready = True
sleep(1)
except KeyboardInterrupt:
#remove all jobs that are not running
os.system('condor_rm -constraint \'JobStatus =!= 2\'')
#check if enough machines are available
if machines['worker'] or machines['ps']:
#stop the ps jobs
cidfile = os.path.join(FLAGS.expdir, 'cluster', 'ps-cid')
if os.path.exists(cidfile):
with open(cidfile) as fid:
cid = fid.read()
subprocess.call(['condor_rm', cid])
#stop the worker jobs
cidfile = os.path.join(FLAGS.expdir, 'cluster', 'worker-cid')
if os.path.exists(cidfile):
with open(cidfile) as fid:
cid = fid.read()
subprocess.call(['condor_rm', cid])
raise Exception('at leat one ps and one worker needed')
print('starting training with %s parameter servers and %s workers' %
(len(machines['ps']), len(machines['worker'])))
#create the cluster file
with open(os.path.join(FLAGS.expdir, 'cluster', 'cluster'),
'w') as cfid:
for job in machines:
if job == 'ps':
GPU = ''
else:
GPU = '0'
for machine in machines[job]:
cfid.write('%s,%s,%d,%s\n' %
(job, machine[0], machine[1], GPU))
#notify the machine that the cluster is ready
fid = open(FLAGS.expdir + '/cluster/ready', 'w')
fid.close()
print('training has started look in %s/outputs for the job outputs' %
FLAGS.expdir)
print 'waiting for worker jobs to finish'
for machine in machines['worker']:
machine_file = os.path.join(FLAGS.expdir, 'cluster',
'%s-%d' % (machine[0], machine[1]))
while os.path.exists(machine_file):
sleep(1)
#stop the ps jobs
with open(os.path.join(FLAGS.expdir, 'cluster', 'ps-cid')) as fid:
cid = fid.read()
subprocess.call(['condor_rm', cid])
elif computing_cfg['distributed'] == 'condor_local':
#create the directories
if not os.path.isdir(os.path.join(FLAGS.expdir, 'outputs')):
os.makedirs(os.path.join(FLAGS.expdir, 'outputs'))
if not os.path.isdir(os.path.join(FLAGS.expdir, 'cluster')):
os.makedirs(os.path.join(FLAGS.expdir, 'cluster'))
#create the cluster file
with open(os.path.join(FLAGS.expdir, 'cluster', 'cluster'),
'w') as fid:
port = 1024
for _ in range(int(computing_cfg['numps'])):
while not cluster.port_available(port):
port += 1
fid.write('ps,localhost,%d,\n' % port)
port += 1
for i in range(int(computing_cfg['numworkers'])):
while not cluster.port_available(port):
port += 1
fid.write('worker,localhost,%d,%d\n' % (port, i))
port += 1
#submit the job
subprocess.call([
'condor_submit',
'expdir=%s' % FLAGS.expdir,
'GPUs=%d' % (int(computing_cfg['numworkers'])),
'memory=%s' % computing_cfg['minmemory'],
'type=%s' % FLAGS.type, 'nabu/distributed/condor/local.job'
])
print('job submitted look in %s/outputs for the job outputs' %
FLAGS.expdir)
else:
raise Exception('Unknown distributed type in %s' % computing_cfg_file)
def train(clusterfile, job_name, task_index, ssh_command, expdir):
''' does everything for ss training
Args:
clusterfile: the file where all the machines in the cluster are
specified if None, local training will be done
job_name: one of ps or worker in the case of distributed training
task_index: the task index in this job
ssh_command: the command to use for ssh, if 'None' no tunnel will be
created
expdir: the experiments directory
'''
#read the database config file
parsed_database_cfg = configparser.ConfigParser()
parsed_database_cfg.read(os.path.join(expdir, 'database.cfg'))
#read the ss config file
model_cfg = configparser.ConfigParser()
model_cfg.read(os.path.join(expdir, 'model.cfg'))
#read the trainer config file
parsed_trainer_cfg = configparser.ConfigParser()
parsed_trainer_cfg.read(os.path.join(expdir, 'trainer.cfg'))
trainer_cfg = dict(parsed_trainer_cfg.items('trainer'))
#read the decoder config file
evaluator_cfg = configparser.ConfigParser()
evaluator_cfg.read(os.path.join(expdir, 'evaluator.cfg'))
#Get the config files for each training stage. Each training stage has a different
#segment length and its network is initliazed with the network of the previous
#training stage
segment_lengths = trainer_cfg['segment_lengths'].split(' ')
#segment_lengths = [segment_lengths[-1]]
#os.environ['CUDA_VISIBLE_DEVICES'] = '1'
for i, segment_length in enumerate(segment_lengths):
segment_expdir = os.path.join(expdir, segment_length)
segment_parsed_database_cfg = configparser.ConfigParser()
segment_parsed_database_cfg.read(
os.path.join(segment_expdir, 'database.cfg'))
segment_parsed_trainer_cfg = configparser.ConfigParser()
segment_parsed_trainer_cfg.read(
os.path.join(segment_expdir, 'trainer.cfg'))
segment_trainer_cfg = dict(segment_parsed_trainer_cfg.items('trainer'))
if segment_trainer_cfg['trainer'] == 'multi_task':
segment_tasks_cfg = dict()
for task in segment_trainer_cfg['tasks'].split(' '):
segment_tasks_cfg[task] = dict(
segment_parsed_trainer_cfg.items(task))
else:
segment_tasks_cfg = None
#If there was no previously validated training sessions, use the model of the
#previous segment length as initialization for the current one
if i > 0 and not os.path.exists(
os.path.join(segment_expdir, 'logdir',
'validated.ckpt.index')):
init_filename = os.path.join(expdir, segment_lengths[i - 1],
'model', 'network.ckpt')
if not os.path.exists(init_filename + '.index'):
init_filename = None
else:
init_filename = None
#if this training stage has already succesfully finished, skipt it
if os.path.exists(
os.path.join(expdir, segment_lengths[i], 'model',
'network.ckpt.index')):
print 'Already found a fully trained model for segment length %s' % segment_length
else:
#create the cluster and server
server = create_server.create_server(clusterfile=clusterfile,
job_name=job_name,
task_index=task_index,
expdir=expdir,
ssh_command=ssh_command)
#parameter server
if job_name == 'ps':
raise 'Parameter server is currently not implemented correctly'
##create the parameter server
#ps = multi_task_trainer.ParameterServer(
#conf=segment_trainer_cfg,
#tasksconf=segment_tasks_cfg,
#modelconf=model_cfg,
#dataconf=segment_parsed_database_cfg,
#server=server,
#task_index=task_index)
#if task_index ==0:
##let the ps wait untill all workers are finished
#ps.join()
#return
tr = trainer_factory.factory(segment_trainer_cfg['trainer'])(
conf=segment_trainer_cfg,
tasksconf=segment_tasks_cfg,
dataconf=segment_parsed_database_cfg,
modelconf=model_cfg,
evaluatorconf=evaluator_cfg,
expdir=segment_expdir,
init_filename=init_filename,
server=server,
task_index=task_index)
print 'starting training for segment length: %s' % segment_length
#train the model
tr.train()
sys.stderr.write("%s doesn't have %s option in %s "
"section.\n" % (path2, key, section))
failed = 1
else:
val2 = conf2.get(section, key)
if not val1 == val2:
sys.stderr.write("%s in %s != %s in %s under %s.%s\n" %
(val1, path1, val2,
path2, section, key))
failed = 1
return failed
if __name__ == '__main__':
if six.PY3:
conf1 = configparser.ConfigParser()
conf2 = configparser.ConfigParser()
else:
conf1 = configparser.SafeConfigParser()
conf2 = configparser.SafeConfigParser()
conf1.read(sys.argv[1])
conf2.read(sys.argv[2])
# compare first file to the other one
ret = compare_ini_files(conf1, sys.argv[1], conf2, sys.argv[2])
# compare the other file to the first one
ret2 = compare_ini_files(conf2, sys.argv[2], conf1, sys.argv[1])
sys.exit(ret or ret2)
from heat.engine import resource
from heat.engine.properties import Properties
try:
from heat.openstack.common import log as logging
except ImportError:
from oslo_log import log as logging
from vnc_api import vnc_api
from vnc_api.exceptions import NoIdError
from vnc_api.exceptions import RefsExistError
import uuid
from threading import Lock
LOG = logging.getLogger(__name__)
cfg_parser = configparser.ConfigParser()
cfg_parser.read("/etc/heat/heat.conf")
def set_auth_token(func):
def wrapper(self, *args, **kwargs):
self.mutex().acquire()
try:
self.vnc_lib().set_auth_token(self.stack.context.auth_token)
return func(self, *args, **kwargs)
finally:
self.mutex().release()
return wrapper
def launch_refarch_env(console_port=8443,
cluster_id=None,
deployment_type=None,
openshift_vers=None,
vcenter_host=None,
vcenter_username=None,
vcenter_password=None,
vcenter_template_name=None,
vcenter_folder=None,
vcenter_cluster=None,
vcenter_datacenter=None,
vcenter_datastore=None,
vcenter_resource_pool=None,
public_hosted_zone=None,
app_dns_prefix=None,
vm_dns=None,
vm_gw=None,
vm_netmask=None,
vm_network=None,
rhel_subscription_user=None,
rhel_subscription_pass=None,
rhel_subscription_server=None,
rhel_subscription_pool=None,
byo_lb=None,
lb_host=None,
byo_nfs=None,
nfs_host=None,
nfs_registry_mountpoint=None,
no_confirm=False,
tag=None,
verbose=0,
create_inventory=None,
master_nodes=None,
infra_nodes=None,
app_nodes=None,
vm_ipaddr_start=None,
ocp_hostname_prefix=None,
create_ocp_vars=None,
auth_type=None,
ldap_user=None,
ldap_user_password=None,
ldap_fqdn=None,
openshift_sdn=None,
containerized=None,
container_storage=None,
openshift_hosted_metrics_deploy=None,
clean=None):
# Open config file INI for values first
scriptbasename = __file__
scriptbasename = os.path.basename(scriptbasename)
scriptbasename = scriptbasename.replace('.py', '')
defaults = {
'vmware': {
'ini_path':
os.path.join(os.path.dirname(__file__), '%s.ini' % scriptbasename),
'console_port':
'8443',
'deployment_type':
'openshift-enterprise',
'openshift_vers':
'v3_5',
'vcenter_host':
'',
'vcenter_username':
'******',
'vcenter_password':
'',
'vcenter_template_name':
'ocp-server-template-2.0.2',
'vcenter_folder':
'ocp',
'vcenter_cluster':
'devel',
'vcenter_cluster':
'',
'vcenter_resource_pool':
'/Resources/OCP3',
'public_hosted_zone':
'',
'app_dns_prefix':
'apps',
'vm_dns':
'',
'vm_gw':
'',
'vm_netmask':
'',
'vm_network':
'VM Network',
'rhel_subscription_user':
'',
'rhel_subscription_pass':
'',
'rhel_subscription_server':
'',
'rhel_subscription_pool':
'Red Hat OpenShift Container Platform, Premium*',
'openshift_sdn':
'redhat/openshift-ovs-subnet',
'containerized':
'containerized',
'container_storage':
'none',
'openshift_hosted_metrics_deploy':
'false',
'byo_lb':
'no',
'lb_host':
'haproxy-',
'byo_nfs':
'no',
'nfs_host':
'nfs-0',
'nfs_registry_mountpoint':
'/exports',
'master_nodes':
'3',
'infra_nodes':
'2',
'app_nodes':
'3',
'vm_ipaddr_start':
'',
'ocp_hostname_prefix':
'',
'auth_type':
'ldap',
'ldap_user':
'******',
'ldap_user_password':
'',
'ldap_fqdn':
''
}
}
if six.PY3:
config = configparser.ConfigParser()
else:
config = configparser.SafeConfigParser()
# where is the config?
vmware_ini_path = os.environ.get('VMWARE_INI_PATH',
defaults['vmware']['ini_path'])
vmware_ini_path = os.path.expanduser(os.path.expandvars(vmware_ini_path))
config.read(vmware_ini_path)
# apply defaults
for k, v in defaults['vmware'].iteritems():
if not config.has_option('vmware', k):
config.set('vmware', k, str(v))
cluster_id = config.get('vmware', 'cluster_id')
console_port = config.get('vmware', 'console_port')
deployment_type = config.get('vmware', 'deployment_type')
openshift_vers = config.get('vmware', 'openshift_vers')
vcenter_host = config.get('vmware', 'vcenter_host')
vcenter_username = config.get('vmware', 'vcenter_username')
vcenter_password = config.get('vmware', 'vcenter_password')
vcenter_template_name = config.get('vmware', 'vcenter_template_name')
vcenter_folder = config.get('vmware', 'vcenter_folder')
vcenter_cluster = config.get('vmware', 'vcenter_cluster')
vcenter_datacenter = config.get('vmware', 'vcenter_datacenter')
vcenter_datastore = config.get('vmware', 'vcenter_datastore')
vcenter_resource_pool = config.get('vmware', 'vcenter_resource_pool')
public_hosted_zone = config.get('vmware', 'public_hosted_zone')
app_dns_prefix = config.get('vmware', 'app_dns_prefix')
vm_dns = config.get('vmware', 'vm_dns')
vm_gw = config.get('vmware', 'vm_gw')
vm_netmask = config.get('vmware', 'vm_netmask')
vm_network = config.get('vmware', 'vm_network')
rhel_subscription_user = config.get('vmware', 'rhel_subscription_user')
rhel_subscription_pass = config.get('vmware', 'rhel_subscription_pass')
rhel_subscription_server = config.get('vmware', 'rhel_subscription_server')
rhel_subscription_pool = config.get('vmware', 'rhel_subscription_pool')
openshift_sdn = config.get('vmware', 'openshift_sdn')
containerized = config.get('vmware', 'containerized')
container_storage = config.get('vmware', 'container_storage')
openshift_hosted_metrics_deploy = config.get(
'vmware', 'openshift_hosted_metrics_deploy')
byo_lb = config.get('vmware', 'byo_lb')
lb_host = config.get('vmware', 'lb_host')
byo_nfs = config.get('vmware', 'byo_nfs')
nfs_host = config.get('vmware', 'nfs_host')
nfs_registry_mountpoint = config.get('vmware', 'nfs_registry_mountpoint')
master_nodes = config.get('vmware', 'master_nodes')
infra_nodes = config.get('vmware', 'infra_nodes')
app_nodes = config.get('vmware', 'app_nodes')
vm_ipaddr_start = config.get('vmware', 'vm_ipaddr_start')
ocp_hostname_prefix = config.get('vmware', 'ocp_hostname_prefix')
auth_type = config.get('vmware', 'auth_type')
ldap_user = config.get('vmware', 'ldap_user')
ldap_user_password = config.get('vmware', 'ldap_user_password')
ldap_fqdn = config.get('vmware', 'ldap_fqdn')
err_count = 0
required_vars = {
'public_hosted_zone': public_hosted_zone,
'vcenter_host': vcenter_host,
'vcenter_password': vcenter_password,
'vm_ipaddr_start': vm_ipaddr_start,
'ldap_fqdn': ldap_fqdn,
'ldap_user_password': ldap_user_password,
'vm_dns': vm_dns,
'vm_gw': vm_gw,
'vm_netmask': vm_netmask,
'vcenter_datacenter': vcenter_datacenter,
'vcenter_datastore': vcenter_datastore
}
for k, v in required_vars.items():
if v == '':
err_count += 1
print "Missing %s " % k
if err_count > 0:
print "Please fill out the missing variables in %s " % vmware_ini_path
exit(1)
wildcard_zone = "%s.%s" % (app_dns_prefix, public_hosted_zone)
# fix nfs_host and lb_host vars with nfs_ocp_hostname_prefix
if 'no' in byo_lb:
if ocp_hostname_prefix is not None:
lb_host = ocp_hostname_prefix + "haproxy-0"
else:
lb_host = "haproxy-0"
if 'no' in byo_nfs:
if ocp_hostname_prefix is not None:
nfs_host = ocp_hostname_prefix + "nfs-0"
else:
nfs_host = "nfs-0"
tags = []
tags.append('setup')
# Our initial support node is the wildcard_ip
support_nodes = 1
if byo_nfs == "no":
support_nodes = support_nodes + 1
tags.append('nfs')
else:
if nfs_host == '':
nfs_host = click.prompt(
"Please enter the NFS Server fqdn for persistent registry:")
if nfs_registry_mountpoint is '':
nfs_registry_mountpoint = click.prompt(
"Please enter NFS share name for persistent registry:")
tags.append('prod')
if byo_lb == "no":
tags.append('haproxy')
else:
if lb_host == '':
lb_host = click.prompt(
"Please enter the load balancer hostname for installation:")
lb_host = lb_host + '.' + public_hosted_zone
if create_ocp_vars is True:
click.echo('Configured OCP variables:')
click.echo('\tauth_type: %s' % auth_type)
click.echo('\tldap_fqdn: %s' % ldap_fqdn)
click.echo('\tldap_user: %s' % ldap_user)
click.echo('\tldap_user_password: %s' % ldap_user_password)
click.echo('\tpublic_hosted_zone: %s' % public_hosted_zone)
click.echo('\tapp_dns_prefix: %s' % app_dns_prefix)
click.echo('\tbyo_lb: %s' % byo_lb)
click.echo('\tlb_host: %s' % lb_host)
click.echo('\tUsing values from: %s' % vmware_ini_path)
if not no_confirm:
click.confirm('Continue using these values?', abort=True)
if auth_type == 'ldap':
l_bdn = ""
for d in ldap_fqdn.split("."):
l_bdn = l_bdn + "dc=" + d + ","
l = ldap.initialize("ldap://" + ldap_fqdn)
try:
l.protocol_version = ldap.VERSION3
l.set_option(ldap.OPT_REFERRALS, 0)
bind = l.simple_bind_s(ldap_user, ldap_user_password)
base = l_bdn[:-1]
criteria = "(&(objectClass=user)(sAMAccountName=" + ldap_user + "))"
attributes = 'displayName', 'distinguishedName'
result = l.search_s(base, ldap.SCOPE_SUBTREE, criteria,
attributes)
results = [
entry for dn, entry in result if isinstance(entry, dict)
]
finally:
l.unbind()
for result in results:
bindDN = str(result['distinguishedName']).strip("'[]")
url_base = bindDN.replace(("CN=" + ldap_user + ","), "")
url = "ldap://" + ldap_fqdn + ":389/" + url_base + "?sAMAccountName"
install_file = "playbooks/openshift-install.yaml"
for line in fileinput.input(install_file, inplace=True):
# Parse our ldap url
if line.startswith(" url:"):
print " url: " + url
elif line.startswith(" bindPassword:"******" bindPassword: "******" bindDN:"):
print " bindDN: " + bindDN
elif line.startswith(" wildcard_zone:"):
print " wildcard_zone: " + app_dns_prefix + "." + public_hosted_zone
elif line.startswith(" load_balancer_hostname:"):
print " load_balancer_hostname: " + lb_host + "." + public_hosted_zone
elif line.startswith(" deployment_type:"):
print " deployment_type: " + deployment_type
elif line.startswith(
" openshift_hosted_registry_storage_host:"):
print " openshift_hosted_registry_storage_host: " + nfs_host + "." + public_hosted_zone
elif line.startswith(
" openshift_hosted_registry_storage_nfs_directory:"
):
print " openshift_hosted_registry_storage_nfs_directory: " + nfs_registry_mountpoint
elif line.startswith(
" openshift_hosted_metrics_storage_host:"):
print " openshift_hosted_metrics_storage_host: " + nfs_host + "." + public_hosted_zone
elif line.startswith(
" openshift_hosted_metrics_storage_nfs_directory:"):
print " openshift_hosted_metrics_storage_nfs_directory: " + nfs_registry_mountpoint
else:
print line,
# Provide values for update and add node playbooks
update_file = "playbooks/minor-update.yaml"
for line in fileinput.input(update_file, inplace=True):
if line.startswith(" wildcard_zone:"):
print " wildcard_zone: " + app_dns_prefix + "." + public_hosted_zone
elif line.startswith(" load_balancer_hostname:"):
print " load_balancer_hostname: " + lb_host + "." + public_hosted_zone
elif line.startswith(" deployment_type:"):
print " deployment_type: " + deployment_type
else:
print line,
#End create_ocp_vars
exit(0)
if auth_type == 'none':
playbooks = [
"playbooks/openshift-install.yaml",
"playbooks/minor-update.yaml"
]
for ocp_file in playbooks:
for line in fileinput.input(ocp_file, inplace=True):
if line.startswith(
'#openshift_master_identity_providers:'):
line = line.replace('#', ' ')
print line
else:
print line,
exit(0)
if create_inventory is True:
click.echo('Configured inventory values:')
click.echo('\tmaster_nodes: %s' % master_nodes)
click.echo('\tinfra_nodes: %s' % infra_nodes)
click.echo('\tapp_nodes: %s' % app_nodes)
click.echo('\tpublic_hosted_zone: %s' % public_hosted_zone)
click.echo('\tapp_dns_prefix: %s' % app_dns_prefix)
click.echo('\tocp_hostname_prefix: %s' % ocp_hostname_prefix)
click.echo('\tbyo_nfs: %s' % byo_nfs)
if byo_nfs == "no":
click.echo('\tnfs_host: %s' % nfs_host)
click.echo('\tbyo_lb: %s' % byo_lb)
if byo_lb == "no":
click.echo('\tlb_host: %s' % lb_host)
click.echo('\tvm_ipaddr_start: %s' % vm_ipaddr_start)
click.echo('\tUsing values from: %s' % vmware_ini_path)
click.echo("")
if not no_confirm:
click.confirm('Continue using these values?', abort=True)
# Create the inventory file and exit
if not cluster_id:
#create a unique cluster_id first
cluster_id = ''.join(
random.choice('0123456789abcdefghijklmnopqrstuvwxyz')
for i in range(20))
config.set('vmware', 'cluster_id', cluster_id)
for line in fileinput.input(vmware_ini_path, inplace=True):
if line.startswith('cluster_id'):
print "cluster_id=" + str(cluster_id)
else:
print line,
total_nodes = int(master_nodes) + int(app_nodes) + int(
infra_nodes) + int(support_nodes)
if vm_ipaddr_start is None:
vm_ipaddr_start = click.prompt("Starting IP address to use?")
ip4addr = []
for i in range(total_nodes):
p = iptools.ipv4.ip2long(vm_ipaddr_start) + i
ip4addr.append(iptools.ipv4.long2ip(p))
wild_ip = ip4addr.pop()
bind_entry = []
bind_entry.append("$ORIGIN " + app_dns_prefix + "." +
public_hosted_zone + ".")
bind_entry.append("*\tA\t" + wild_ip)
bind_entry.append("$ORIGIN " + public_hosted_zone + ".")
d = {}
d['host_inventory'] = {}
d['infrastructure_hosts'] = {}
support_list = []
if byo_nfs == "no":
if ocp_hostname_prefix not in nfs_host:
nfs_host = ocp_hostname_prefix + "nfs-0"
d['host_inventory'][nfs_host] = {}
d['host_inventory'][nfs_host]['guestname'] = nfs_host
d['host_inventory'][nfs_host]['ip4addr'] = ip4addr[0]
d['host_inventory'][nfs_host]['tag'] = str(
cluster_id) + "-infra-nfs"
d['infrastructure_hosts']["nfs_server"] = {}
d['infrastructure_hosts']["nfs_server"]['guestname'] = nfs_host
d['infrastructure_hosts']["nfs_server"]['tag'] = str(
cluster_id) + "-infra-nfs"
support_list.append(nfs_host)
bind_entry.append(nfs_host + "\tA\t" + ip4addr[0])
del ip4addr[0]
if byo_lb == "no":
if ocp_hostname_prefix not in lb_host:
lb_host = ocp_hostname_prefix + "haproxy-0"
d['host_inventory'][lb_host] = {}
d['host_inventory'][lb_host]['guestname'] = lb_host
d['host_inventory'][lb_host]['ip4addr'] = wild_ip
d['host_inventory'][lb_host]['tag'] = str(
cluster_id) + "-loadbalancer"
d['infrastructure_hosts']["haproxy"] = {}
d['infrastructure_hosts']["haproxy"]['guestname'] = lb_host
d['infrastructure_hosts']["haproxy"]['tag'] = str(
cluster_id) + "-loadbalancer"
support_list.append(lb_host)
bind_entry.append(lb_host + "\tA\t" + wild_ip)
master_list = []
d['production_hosts'] = {}
for i in range(0, int(master_nodes)):
if ocp_hostname_prefix is not None:
master_name = ocp_hostname_prefix + "master-" + str(i)
else:
master_name = "master-" + str(i)
d['host_inventory'][master_name] = {}
d['host_inventory'][master_name]['guestname'] = master_name
d['host_inventory'][master_name]['ip4addr'] = ip4addr[0]
d['host_inventory'][master_name]['tag'] = str(
cluster_id) + "-master"
d['production_hosts'][master_name] = {}
d['production_hosts'][master_name]['guestname'] = master_name
d['production_hosts'][master_name]['tag'] = str(
cluster_id) + "-master"
master_list.append(master_name)
bind_entry.append(master_name + "\tA\t" + ip4addr[0])
del ip4addr[0]
app_list = []
for i in range(0, int(app_nodes)):
if ocp_hostname_prefix is not None:
app_name = ocp_hostname_prefix + "app-" + str(i)
else:
app_name = "app-" + str(i)
d['host_inventory'][app_name] = {}
d['host_inventory'][app_name]['guestname'] = app_name
d['host_inventory'][app_name]['ip4addr'] = ip4addr[0]
d['host_inventory'][app_name]['tag'] = str(cluster_id) + "-app"
d['production_hosts'][app_name] = {}
d['production_hosts'][app_name]['guestname'] = app_name
d['production_hosts'][app_name]['tag'] = str(cluster_id) + "-app"
app_list.append(app_name)
bind_entry.append(app_name + "\tA\t" + ip4addr[0])
del ip4addr[0]
infra_list = []
for i in range(0, int(infra_nodes)):
if ocp_hostname_prefix is not None:
infra_name = ocp_hostname_prefix + "infra-" + str(i)
else:
infra_name = "infra-" + str(i)
d['host_inventory'][infra_name] = {}
d['host_inventory'][infra_name]['guestname'] = infra_name
d['host_inventory'][infra_name]['ip4addr'] = ip4addr[0]
d['host_inventory'][infra_name]['tag'] = str(cluster_id) + "-infra"
d['production_hosts'][infra_name] = {}
d['production_hosts'][infra_name]['guestname'] = infra_name
d['production_hosts'][infra_name]['tag'] = str(
cluster_id) + "-infra"
infra_list.append(infra_name)
bind_entry.append(infra_name + " A " + ip4addr[0])
del ip4addr[0]
print "# Here is what should go into your DNS records"
print("\n".join(bind_entry))
print "# Please note, if you have chosen to bring your own loadbalancer and NFS Server you will need to ensure that these records are added to DNS and properly resolve. "
with open('infrastructure.json', 'w') as outfile:
json.dump(d, outfile)
exit(0)
# End create inventory
# Display information to the user about their choices
click.echo('Configured values:')
for each_section in config.sections():
for (key, val) in config.items(each_section):
print '\t %s: %s' % (key, val)
click.echo("")
if not no_confirm:
click.confirm('Continue using these values?', abort=True)
if not os.path.isfile('infrastructure.json'):
print "Please create your inventory file first by running the --create_inventory flag"
exit(1)
inventory_file = "inventory/vsphere/vms/vmware_inventory.ini"
# Add section here to modify inventory file based on input from user check your vmmark scripts for parsing the file and adding the values
for line in fileinput.input(inventory_file, inplace=True):
if line.startswith("server="):
print "server=" + vcenter_host
elif line.startswith("password="******"password="******"username="******"username="******"
command = 'cp -f ~/.ssh/id_rsa ssh_key/ocp3-installer'
os.system(command)
command = 'cp -f ~/.ssh/id_rsa ssh_key/ocp-installer'
os.system(command)
# make sure the ssh keys have the proper permissions
command = 'chmod 600 ssh_key/ocp-installer'
os.system(command)
# remove any cached facts to prevent stale data during a re-run
command = 'rm -rf .ansible/cached_facts'
os.system(command)
tags = ",".join(tags)
if clean is True:
tags = 'clean'
if tag:
tags = tag
#if local:
#command='ansible-playbook'
#else:
# command='docker run -t --rm --volume `pwd`:/opt/ansible:z -v ~/.ssh:/root/.ssh:z -v /tmp:/tmp:z --net=host ansible:2.2-latest'
if 'clean' in tags:
tags = 'all'
command = 'ansible-playbook '
playbook = 'playbooks/cleanup-vsphere.yaml'
else:
command = 'ansible-playbook'
command = command + ' --extra-vars "@./infrastructure.json" --tags %s -e \'vcenter_host=%s \
vcenter_username=%s \
vcenter_password=%s \
vcenter_template_name=%s \
vcenter_folder=%s \
vcenter_cluster=%s \
vcenter_datacenter=%s \
vcenter_datastore=%s \
vcenter_resource_pool=%s \
public_hosted_zone=%s \
app_dns_prefix=%s \
vm_dns=%s \
vm_gw=%s \
vm_netmask=%s \
vm_network=%s \
wildcard_zone=%s \
console_port=%s \
cluster_id=%s \
deployment_type=%s \
openshift_vers=%s \
rhsm_user=%s \
rhsm_password=%s \
rhel_subscription_server=%s \
rhsm_pool="%s" \
openshift_sdn=%s \
containerized=%s \
container_storage=%s \
openshift_hosted_metrics_deploy=%s \
lb_host=%s \
nfs_host=%s \
nfs_registry_mountpoint=%s \' %s' % (
tags, vcenter_host, vcenter_username, vcenter_password,
vcenter_template_name, vcenter_folder, vcenter_cluster,
vcenter_datacenter, vcenter_datastore, vcenter_resource_pool,
public_hosted_zone, app_dns_prefix, vm_dns, vm_gw, vm_netmask,
vm_network, wildcard_zone, console_port, cluster_id,
deployment_type, openshift_vers, rhel_subscription_user,
rhel_subscription_pass, rhel_subscription_server,
rhel_subscription_pool, openshift_sdn, containerized,
container_storage, openshift_hosted_metrics_deploy, lb_host,
nfs_host, nfs_registry_mountpoint, playbook)
if verbose > 0:
command += " -" + "".join(['v'] * verbose)
click.echo('We are running: %s' % command)
status = os.system(command)
if os.WIFEXITED(status) and os.WEXITSTATUS(status) != 0:
return os.WEXITSTATUS(status)
def main():
csv2rdf = CSV2RDF()
opts, files = getopt.getopt(
sys.argv[1:],
"hc:b:p:i:o:Cf:l:s:d:D:",
["out=", "base=", "delim=", "propbase=", "class=", "default="
"ident=", "label=", "skip=", "defineclass", "help"])
opts = dict(opts)
if "-h" in opts or "--help" in opts:
print(HELP)
sys.exit(-1)
if "-f" in opts:
config = configparser.ConfigParser()
config.readfp(open(opts["-f"]))
for k, v in config.items("csv2rdf"):
if k == "out":
csv2rdf.OUT = codecs.open(v, "w", "utf-8")
elif k == "base":
csv2rdf.BASE = rdflib.Namespace(v)
elif k == "propbase":
csv2rdf.PROPBASE = rdflib.Namespace(v)
elif k == "class":
csv2rdf.CLASS = rdflib.URIRef(v)
elif k == "defineclass":
csv2rdf.DEFINECLASS = bool(v)
elif k == "ident":
csv2rdf.IDENT = eval(v)
elif k == "label":
csv2rdf.LABEL = eval(v)
elif k == "delim":
csv2rdf.DELIM = v
elif k == "skip":
csv2rdf.SKIP = int(v)
elif k == "default":
csv2rdf.DEFAULT = column(v)
elif k.startswith("col"):
csv2rdf.COLUMNS[int(k[3:])] = column(v)
elif k.startswith("prop"):
csv2rdf.PROPS[int(k[4:])] = rdflib.URIRef(v)
if "-o" in opts:
csv2rdf.OUT = codecs.open(opts["-o"], "w", "utf-8")
if "--out" in opts:
csv2rdf.OUT = codecs.open(opts["--out"], "w", "utf-8")
if "-b" in opts:
csv2rdf.BASE = rdflib.Namespace(opts["-b"])
if "--base" in opts:
csv2rdf.BASE = rdflib.Namespace(opts["--base"])
if "-d" in opts:
csv2rdf.DELIM = opts["-d"]
if "--delim" in opts:
csv2rdf.DELIM = opts["--delim"]
if "-D" in opts:
csv2rdf.DEFAULT = column(opts["-D"])
if "--default" in opts:
csv2rdf.DEFAULT = column(opts["--default"])
if "-p" in opts:
csv2rdf.PROPBASE = rdflib.Namespace(opts["-p"])
if "--propbase" in opts:
csv2rdf.PROPBASE = rdflib.Namespace(opts["--propbase"])
if "-l" in opts:
csv2rdf.LABEL = eval(opts["-l"])
if "--label" in opts:
csv2rdf.LABEL = eval(opts["--label"])
if "-i" in opts:
csv2rdf.IDENT = eval(opts["-i"])
if "--ident" in opts:
csv2rdf.IDENT = eval(opts["--ident"])
if "-s" in opts:
csv2rdf.SKIP = int(opts["-s"])
if "--skip" in opts:
csv2rdf.SKIP = int(opts["--skip"])
if "-c" in opts:
csv2rdf.CLASS = rdflib.URIRef(opts["-c"])
if "--class" in opts:
csv2rdf.CLASS = rdflib.URIRef(opts["--class"])
for k, v in opts.items():
if k.startswith("--col"):
csv2rdf.COLUMNS[int(k[5:])] = column(v)
elif k.startswith("--prop"):
csv2rdf.PROPS[int(k[6:])] = rdflib.URIRef(v)
if csv2rdf.CLASS and ("-C" in opts or "--defineclass" in opts):
csv2rdf.DEFINECLASS = True
csv2rdf.convert(
csv_reader(fileinput.input(files), delimiter=csv2rdf.DELIM))
def get_config_parser():
return configparser.ConfigParser(
) if sys.version_info.major == 3 else configparser.SafeConfigParser()
def parse_args(args_str):
'''
Eg. python svc_monitor.py --rabbit_server localhost
--rabbit_port 5672
--rabbit_user guest
--rabbit_password guest
--cassandra_server_list 10.1.2.3:9160
--api_server_ip 10.1.2.3
--api_server_port 8082
--api_server_use_ssl False
--zk_server_ip 10.1.2.3
--zk_server_port 2181
--collectors 127.0.0.1:8086
--http_server_port 8090
--log_local
--log_level SYS_DEBUG
--log_category test
--log_file <stdout>
--trace_file /var/log/contrail/svc-monitor.err
--use_syslog
--syslog_facility LOG_USER
--cluster_id <testbed-name>
--check_service_interval 60
[--region_name <name>]
[--reset_config]
'''
# Source any specified config/ini file
# Turn off help, so we show all options in response to -h
conf_parser = argparse.ArgumentParser(add_help=False)
conf_parser.add_argument("-c",
"--conf_file",
action='append',
help="Specify config file",
metavar="FILE")
args, remaining_argv = conf_parser.parse_known_args(args_str.split())
defaults = {
'rabbit_server': 'localhost',
'rabbit_port': '5672',
'rabbit_user': '******',
'rabbit_password': '******',
'rabbit_vhost': None,
'rabbit_ha_mode': False,
'cassandra_server_list': '127.0.0.1:9160',
'api_server_ip': '127.0.0.1',
'api_server_port': '8082',
'api_server_use_ssl': None,
'zk_server_ip': '127.0.0.1',
'zk_server_port': '2181',
'collectors': None,
'http_server_port': '8088',
'http_server_ip': '0.0.0.0',
'log_local': False,
'log_level': SandeshLevel.SYS_DEBUG,
'log_category': '',
'log_file': Sandesh._DEFAULT_LOG_FILE,
'trace_file': '/var/log/contrail/svc-monitor.err',
'use_syslog': False,
'syslog_facility': Sandesh._DEFAULT_SYSLOG_FACILITY,
'region_name': None,
'cluster_id': '',
'logging_conf': '',
'logger_class': None,
'check_service_interval': '60',
'nova_endpoint_type': 'internalURL',
'rabbit_use_ssl': False,
'kombu_ssl_version': '',
'kombu_ssl_keyfile': '',
'kombu_ssl_certfile': '',
'kombu_ssl_ca_certs': '',
'analytics_api_ssl_enable': False,
'analytics_api_insecure_enable': False,
'analytics_api_ssl_ca_cert': '',
'analytics_api_ssl_keyfile': '',
'analytics_api_ssl_certfile': '',
}
defaults.update(SandeshConfig.get_default_options(['DEFAULTS']))
secopts = {
'use_certs': False,
'keyfile': '',
'certfile': '',
'ca_certs': '',
}
ksopts = {
'auth_host': '127.0.0.1',
'auth_protocol': 'http',
'auth_port': '5000',
'auth_version': 'v3',
'auth_insecure': True,
'admin_user': '******',
'admin_password': '******',
'admin_tenant_name': 'admin',
'user_domain_name': 'Default',
'project_domain_name': 'Default'
}
schedops = {
'si_netns_scheduler_driver':
'svc_monitor.scheduler.vrouter_scheduler.RandomScheduler',
'analytics_server_list': '127.0.0.1:8081',
'availability_zone': None,
'netns_availability_zone': None,
'aaa_mode': cfgm_common.AAA_MODE_DEFAULT_VALUE,
}
cassandraopts = {
'cassandra_user': None,
'cassandra_password': None,
}
sandeshopts = SandeshConfig.get_default_options()
saved_conf_file = args.conf_file
config = configparser.ConfigParser()
if args.conf_file:
config.read(args.conf_file)
defaults.update(dict(config.items("DEFAULTS")))
if ('SECURITY' in config.sections()
and 'use_certs' in config.options('SECURITY')):
if config.getboolean('SECURITY', 'use_certs'):
secopts.update(dict(config.items("SECURITY")))
if 'KEYSTONE' in config.sections():
ksopts.update(dict(config.items("KEYSTONE")))
if 'SCHEDULER' in config.sections():
schedops.update(dict(config.items("SCHEDULER")))
if 'CASSANDRA' in config.sections():
cassandraopts.update(dict(config.items('CASSANDRA')))
SandeshConfig.update_options(sandeshopts, config)
# Override with CLI options
# Don't surpress add_help here so it will handle -h
parser = argparse.ArgumentParser(
# Inherit options from config_parser
parents=[conf_parser],
# script description with -h/--help
description=__doc__,
# Don't mess with format of description
formatter_class=argparse.RawDescriptionHelpFormatter,
)
defaults.update(secopts)
defaults.update(ksopts)
defaults.update(schedops)
defaults.update(cassandraopts)
defaults.update(sandeshopts)
parser.set_defaults(**defaults)
parser.add_argument(
"--cassandra_server_list",
help="List of cassandra servers in IP Address:Port format",
nargs='+')
parser.add_argument("--cassandra_use_ssl",
action="store_true",
help="Enable TLS for cassandra communication")
parser.add_argument("--cassandra_ca_certs", help="Cassandra CA certs")
parser.add_argument(
"--reset_config",
action="store_true",
help="Warning! Destroy previous configuration and start clean")
parser.add_argument("--api_server_ip", help="IP address of API server")
parser.add_argument("--api_server_port", help="Port of API server")
parser.add_argument("--api_server_use_ssl",
help="Use SSL to connect with API server")
parser.add_argument("--collectors",
help="List of VNC collectors in ip:port format",
nargs="+")
parser.add_argument("--http_server_port", help="Port of local HTTP server")
parser.add_argument("--http_server_ip", help="IP of local HTTP server")
parser.add_argument("--log_local",
action="store_true",
help="Enable local logging of sandesh messages")
parser.add_argument(
"--log_level",
help="Severity level for local logging of sandesh messages")
parser.add_argument(
"--log_category",
help="Category filter for local logging of sandesh messages")
parser.add_argument("--log_file",
help="Filename for the logs to be written to")
parser.add_argument("--trace_file",
help="Filename for the error "
"backtraces to be written to")
parser.add_argument("--use_syslog",
action="store_true",
help="Use syslog for logging")
parser.add_argument("--syslog_facility",
help="Syslog facility to receive log lines")
parser.add_argument("--aaa_mode",
choices=AAA_MODE_VALID_VALUES,
help="AAA mode")
parser.add_argument("--admin_user", help="Name of keystone admin user")
parser.add_argument("--admin_password",
help="Password of keystone admin user")
parser.add_argument("--admin_tenant_name",
help="Tenant name for keystone admin user")
parser.add_argument("--region_name", help="Region name for openstack API")
parser.add_argument("--cluster_id",
help="Used for database keyspace separation")
parser.add_argument(
"--logging_conf",
help=("Optional logging configuration file, default: None"))
parser.add_argument("--logger_class",
help=("Optional external logger class, default: None"))
parser.add_argument("--cassandra_user", help="Cassandra user name")
parser.add_argument("--cassandra_password", help="Cassandra password")
parser.add_argument("--check_service_interval",
help="Check service interval")
parser.add_argument("--analytics_api_ssl_enable",
help="Enable SSL in rest api server")
parser.add_argument("--analytics_api_insecure_enable",
help="Enable insecure mode")
parser.add_argument("--analytics_api_ssl_certfile",
help="Location of analytics api ssl host certificate")
parser.add_argument("--analytics_api_ssl_keyfile",
help="Location of analytics api ssl private key")
parser.add_argument("--analytics_api_ssl_ca_cert",
type=str,
help="Location of analytics api ssl CA certificate")
SandeshConfig.add_parser_arguments(parser)
args = parser.parse_args(remaining_argv)
args._conf_file = saved_conf_file
args.config_sections = config
if isinstance(args.cassandra_server_list, basestring):
args.cassandra_server_list = args.cassandra_server_list.split()
if isinstance(args.collectors, basestring):
args.collectors = args.collectors.split()
if args.region_name and args.region_name.lower() == 'none':
args.region_name = None
if args.availability_zone and args.availability_zone.lower() == 'none':
args.availability_zone = None
if args.netns_availability_zone and \
args.netns_availability_zone.lower() == 'none':
args.netns_availability_zone = None
args.sandesh_config = SandeshConfig.from_parser_arguments(args)
args.cassandra_use_ssl = (str(args.cassandra_use_ssl).lower() == 'true')
args.analytics_api_ssl_enable = \
(str(args.analytics_api_ssl_enable).lower() == 'true')
args.analytics_api_insecure_enable = \
(str(args.analytics_api_insecure_enable).lower() == 'true')
return args
def main(_):
'''main'''
if FLAGS.expdir is None:
raise Exception('no expdir specified. Command usage: '
'nabu data --expdir=/path/to/recipe '
'--recipe=/path/to/recipe')
if not os.path.isdir(FLAGS.expdir):
raise Exception('cannot find expdir %s' % FLAGS.expdir)
if FLAGS.recipe is None:
raise Exception('no recipe specified. Command usage: '
'nabu data --expdir=/path/to/recipe '
'--recipe=/path/to/recipe')
if not os.path.isdir(FLAGS.recipe):
raise Exception('cannot find recipe %s' % FLAGS.recipe)
evaluator_cfg_file = os.path.join(FLAGS.recipe, 'test_evaluator.cfg')
database_cfg_file = os.path.join(FLAGS.recipe, 'database.conf')
reconstructor_cfg_file = os.path.join(FLAGS.recipe, 'reconstructor.cfg')
scorer_cfg_file = os.path.join(FLAGS.recipe, 'scorer.cfg')
postprocessor_cfg_file = os.path.join(FLAGS.recipe, 'postprocessor.cfg')
model_cfg_file = os.path.join(FLAGS.recipe, 'model.cfg')
#Assuming only one (the last one) training stage needs testing
parsed_evaluator_cfg = configparser.ConfigParser()
parsed_evaluator_cfg.read(evaluator_cfg_file)
training_stage = parsed_evaluator_cfg.get('evaluator', 'segment_length')
##create the testing dir
#if os.path.isdir(os.path.join(FLAGS.expdir, 'test')):
#shutil.rmtree(os.path.join(FLAGS.expdir, 'test'))
#os.makedirs(os.path.join(FLAGS.expdir, 'test'))
if not os.path.isdir(os.path.join(FLAGS.expdir, 'test')):
os.makedirs(os.path.join(FLAGS.expdir, 'test'))
#copy the config files
parsed_database_cfg = configparser.ConfigParser()
parsed_database_cfg.read(database_cfg_file)
segment_parsed_database_cfg = parsed_database_cfg
for section in segment_parsed_database_cfg.sections():
if 'store_dir' in dict(
segment_parsed_database_cfg.items(section)).keys():
segment_parsed_database_cfg.set(
section, 'store_dir',
os.path.join(
segment_parsed_database_cfg.get(section, 'store_dir'),
training_stage))
with open(os.path.join(FLAGS.expdir, 'test', 'database.cfg'), 'w') as fid:
segment_parsed_database_cfg.write(fid)
#shutil.copyfile(database_cfg_file,
#os.path.join(FLAGS.expdir, 'test', 'database.cfg'))
shutil.copyfile(evaluator_cfg_file,
os.path.join(FLAGS.expdir, 'test', 'evaluator.cfg'))
shutil.copyfile(reconstructor_cfg_file,
os.path.join(FLAGS.expdir, 'test', 'reconstructor.cfg'))
shutil.copyfile(scorer_cfg_file,
os.path.join(FLAGS.expdir, 'test', 'scorer.cfg'))
try:
shutil.copyfile(
postprocessor_cfg_file,
os.path.join(FLAGS.expdir, 'test', 'postprocessor.cfg'))
except:
pass
shutil.copyfile(model_cfg_file,
os.path.join(FLAGS.expdir, 'test', 'model.cfg'))
#create a link to the model that will be used for testing. Assuming
#it is stored in the 'full' directory of expdir
if not os.path.isdir(os.path.join(FLAGS.expdir, 'test', 'model')):
os.symlink(os.path.join(FLAGS.expdir, training_stage, 'model'),
os.path.join(FLAGS.expdir, 'test', 'model'))
if FLAGS.computing == 'condor':
computing_cfg_file = 'config/computing/condor/non_distributed.cfg'
parsed_computing_cfg = configparser.ConfigParser()
parsed_computing_cfg.read(computing_cfg_file)
computing_cfg = dict(parsed_computing_cfg.items('computing'))
if not os.path.isdir(os.path.join(FLAGS.expdir, 'test', 'outputs')):
os.makedirs(os.path.join(FLAGS.expdir, 'test', 'outputs'))
subprocess.call([
'condor_submit',
'expdir=%s' % os.path.join(FLAGS.expdir, 'test'),
'script=nabu/scripts/test.py',
'nabu/computing/condor/non_distributed_cpu.job'
])
elif FLAGS.computing == 'standard':
os.environ['CUDA_VISIBLE_DEVICES'] = '0'
test(expdir=os.path.join(FLAGS.expdir, 'test'))
else:
raise Exception('Unknown computing type %s' % FLAGS.computing)
def parse(cell, config):
"""Separate input into (connection info, KQL statements, options)"""
parsed_queries = []
# split to max 2 parts. First part, parts[0], is the first string.
parts = [part.strip() for part in cell.split(None, 1)]
# print(parts)
if not parts:
parsed_queries.append({"connection": "", "kql": "", "options": {}})
return parsed_queries
#
# replace substring of the form $name or ${name}, in windows also %name% if found in env variabes
#
parts[0] = expandvars(parts[0]) # for environment variables
# assume connection is specified
connection = parts[0]
code = parts[1] if len(parts) > 1 else ""
#
# connection taken from a section in dsn file (file name have to be define in config.dsn_filename or specified as a parameter)
#
if parts[0].startswith("[") and parts[0].endswith("]"):
section = parts[0].lstrip("[").rstrip("]")
parser = CP.ConfigParser()
# parse to get flag, for the case that the file nema is specified
kql, options = Parser._parse_kql_options(code, config)
# print( "filename: {}".format(options.get("dsn_filename")))
# with open(options.get("dsn_filename"), "r") as text_file:
# print ("file.content: {} ".format(text_file.read()))
parser.read(options.get("dsn_filename", config.dsn_filename))
cfg_dict = dict(parser.items(section))
cfg_dict_lower = dict()
# for k,v in cfg_dict:
# cfg_dict_lower[k.lower()] = v
cfg_dict_lower = {k.lower(): v for (k, v) in cfg_dict.items()}
if cfg_dict_lower.get("appid"):
connection_list = []
for key in ["appid", "appkey"]:
if cfg_dict_lower.get(key):
connection_list.append(
str.format("{0}('{1}')", key,
cfg_dict_lower.get(key)))
connection = "appinsights://" + ".".join(connection_list)
elif cfg_dict_lower.get("workspace"):
connection_list = []
for key in ["workspace", "appkey"]:
if cfg_dict_lower.get(key):
connection_list.append(
str.format("{0}('{1}')", key,
cfg_dict_lower.get(key)))
connection = "loganalytics://" + ".".join(connection_list)
else:
if cfg_dict_lower.get("user"):
cfg_dict_lower["username"] = cfg_dict_lower.get("user")
connection_list = []
for key in ["username", "password", "cluster", "database"]:
if cfg_dict_lower.get(key):
connection_list.append(
str.format("{0}('{1}')", key,
cfg_dict_lower.get(key)))
connection = "kusto://" + ".".join(connection_list)
#
# connection not specified, override default
#
elif not (parts[0].startswith("kusto://")
or parts[0].startswith("appinsights://")
or parts[0].startswith("loganalytics://")
or parts[0].startswith("cache://") or "@" in parts[0]):
connection = ""
code = cell
#
# split string to queries
#
queries = []
queryLines = []
for line in code.splitlines(True):
if line.isspace():
if len(queryLines) > 0:
queries.append("".join(queryLines))
queryLines = []
else:
queryLines.append(line)
if len(queryLines) > 0:
queries.append("".join(queryLines))
suppress_results = False
if len(queries) > 0 and queries[-1].strip() == ";":
suppress_results = True
queries = queries[:-1]
if len(queries) == 0:
queries.append("")
#
# parse code to kql and options
#
for query in queries:
kql, options = Parser._parse_kql_options(query.strip(), config)
if suppress_results:
options["suppress_results"] = True
parsed_queries.append({
"connection": connection.strip(),
"kql": kql,
"options": options
})
return parsed_queries
def handle(self, *args, **options):
"""
Resets the database for this project.
Note: Transaction wrappers are in reverse as a work around for
autocommit, anybody know how to do this the right way?
"""
if args:
raise CommandError("reset_db takes no arguments")
router = options.get('router')
dbinfo = settings.DATABASES.get(router)
if dbinfo is None:
raise CommandError("Unknown database router %s" % router)
engine = dbinfo.get('ENGINE').split('.')[-1]
user = password = database_name = ''
if engine == 'mysql':
read_default_file = dbinfo.get('OPTIONS',
{}).get('read_default_file')
if read_default_file:
config = configparser.ConfigParser()
config.read(read_default_file)
user = config.get('client', 'user')
password = config.get('client', 'password')
database_name = config.get('client', 'database')
user = options.get('user') or dbinfo.get('USER') or user
password = options.get('password') or dbinfo.get(
'PASSWORD') or password
owner = options.get('owner') or user
database_name = options.get('dbname') or dbinfo.get(
'NAME') or database_name
if database_name == '':
raise CommandError(
"You need to specify DATABASE_NAME in your Django settings file."
)
database_host = dbinfo.get('HOST')
database_port = dbinfo.get('PORT')
verbosity = int(options.get('verbosity', 1))
if options.get('interactive'):
confirm = input("""
You have requested a database reset.
This will IRREVERSIBLY DESTROY
ALL data in the database "%s".
Are you sure you want to do this?
Type 'yes' to continue, or 'no' to cancel: """ % (database_name, ))
else:
confirm = 'yes'
if confirm != 'yes':
print("Reset cancelled.")
return
if engine in ('sqlite3', 'spatialite'):
import os
try:
logging.info("Unlinking %s database" % engine)
os.unlink(database_name)
except OSError:
pass
elif engine in ('mysql', ):
import MySQLdb as Database
kwargs = {
'user': user,
'passwd': password,
}
if database_host.startswith('/'):
kwargs['unix_socket'] = database_host
else:
kwargs['host'] = database_host
if database_port:
kwargs['port'] = int(database_port)
connection = Database.connect(**kwargs)
drop_query = 'DROP DATABASE IF EXISTS `%s`' % database_name
utf8_support = options.get('no_utf8_support',
False) and '' or 'CHARACTER SET utf8'
create_query = 'CREATE DATABASE `%s` %s' % (database_name,
utf8_support)
logging.info('Executing... "' + drop_query + '"')
connection.query(drop_query)
logging.info('Executing... "' + create_query + '"')
connection.query(create_query)
elif engine in ('postgresql', 'postgresql_psycopg2', 'postgis'):
if engine == 'postgresql':
import psycopg as Database # NOQA
elif engine in ('postgresql_psycopg2', 'postgis'):
import psycopg2 as Database # NOQA
conn_string = "dbname=template1"
if user:
conn_string += " user=%s" % user
if password:
conn_string += " password='******'" % password
if database_host:
conn_string += " host=%s" % database_host
if database_port:
conn_string += " port=%s" % database_port
connection = Database.connect(conn_string)
connection.set_isolation_level(0) # autocommit false
cursor = connection.cursor()
drop_query = "DROP DATABASE \"%s\";" % database_name
logging.info('Executing... "' + drop_query + '"')
try:
cursor.execute(drop_query)
except Database.ProgrammingError as e:
logging.info("Error: %s" % str(e))
create_query = "CREATE DATABASE \"%s\"" % database_name
if owner:
create_query += " WITH OWNER = \"%s\" " % owner
create_query += " ENCODING = 'UTF8'"
if engine == 'postgis':
# fetch postgis template name if it exists
from django.contrib.gis.db.backends.postgis.creation import PostGISCreation
postgis_template = PostGISCreation(connection).template_postgis
if postgis_template is not None:
create_query += ' TEMPLATE = %s' % postgis_template
if settings.DEFAULT_TABLESPACE:
create_query += ' TABLESPACE = %s;' % settings.DEFAULT_TABLESPACE
else:
create_query += ';'
logging.info('Executing... "' + create_query + '"')
cursor.execute(create_query)
else:
raise CommandError("Unknown database engine %s" % engine)
if verbosity >= 2 or options.get('interactive'):
print("Reset successful.")
import time
from collections import OrderedDict
import pytest
from six.moves import configparser
from datahub import DataHub
from datahub.exceptions import LimitExceededException, InvalidOperationException, ResourceNotFoundException
from datahub.models import OdpsConnectorConfig, ConnectorType, ConnectorState, PartitionMode
from datahub.models.connector import DatabaseConnectorConfig, EsConnectorConfig, FcConnectorConfig, AuthMode, \
OssConnectorConfig, OtsConnectorConfig, ConnectorShardStatus, WriteMode, DataHubConnectorConfig, ConnectorOffset
current_path = os.path.split(os.path.realpath(__file__))[0]
root_path = os.path.join(current_path, '../..')
configer = configparser.ConfigParser()
configer.read(os.path.join(current_path, '../datahub.ini'))
access_id = configer.get('datahub', 'access_id')
access_key = configer.get('datahub', 'access_key')
endpoint = configer.get('datahub', 'endpoint')
connector_test_project_name = configer.get('datahub',
'connector_test_project_name')
system_time_topic_name = configer.get('datahub', 'system_time_topic_name')
event_time_topic_name = configer.get('datahub', 'event_time_topic_name')
user_define_topic_name = configer.get('datahub', 'user_define_topic_name')
ads_test_topic_name = configer.get('datahub', 'ads_test_topic_name')
es_test_topic_name = configer.get('datahub', 'es_test_topic_name')
fc_test_topic_name = configer.get('datahub', 'fc_test_topic_name')
mysql_test_topic_name = configer.get('datahub', 'mysql_test_topic_name')
'''@file lm_dataprep.py
this file will do the dataprep for lm training'''
import os
from six.moves import configparser
from nabu.processing.target_normalizers import normalizer_factory
#pointer to the confif file
database_cfg_file = 'config/lm_databases/TIMIT.conf'
#read the database config file
database_cfg = configparser.ConfigParser()
database_cfg.read(database_cfg_file)
database_cfg = dict(database_cfg.items('database'))
#create the text normalizer
normalizer = normalizer_factory.factory(database_cfg['normalizer'])
print '------- normalizing training text -----------'
sourcefiles = database_cfg['train_data'].split(' ')
if not os.path.isdir(database_cfg['train_dir']):
os.makedirs(database_cfg['train_dir'])
target_fid = open(os.path.join(database_cfg['train_dir'], 'text'), 'w')
max_num_chars = 0
numlines = 0
#read the textfiles line by line, normalize and write in target file
for sourcefile in sourcefiles:
with open(sourcefile) as fid:
for line in fid.readlines():
normalized = normalizer(line.strip())
def read_settings(self):
''' Reads the settings from the vmware_inventory.ini file '''
scriptbasename = __file__
scriptbasename = os.path.basename(scriptbasename)
scriptbasename = scriptbasename.replace('.py', '')
defaults = {
'vmware': {
'server':
'',
'port':
443,
'username':
'',
'password':
'',
'validate_certs':
True,
'ini_path':
os.path.join(os.path.dirname(__file__),
'%s.ini' % scriptbasename),
'cache_name':
'ansible-vmware',
'cache_path':
'~/.ansible/tmp',
'cache_max_age':
3600,
'max_object_level':
1,
'skip_keys':
'declaredalarmstate,'
'disabledmethod,'
'dynamicproperty,'
'dynamictype,'
'environmentbrowser,'
'managedby,'
'parent,'
'childtype,'
'resourceconfig',
'alias_pattern':
'{{ config.name + "_" + config.uuid }}',
'host_pattern':
'{{ guest.ipaddress }}',
'host_filters':
'{{ guest.gueststate == "running" }}',
'groupby_patterns':
'{{ guest.guestid }},{{ "templates" if config.template else "guests"}}',
'lower_var_keys':
True,
'custom_field_group_prefix':
'vmware_tag_',
'groupby_custom_field':
False
}
}
if six.PY3:
config = configparser.ConfigParser()
else:
config = configparser.SafeConfigParser()
# where is the config?
vmware_ini_path = os.environ.get('VMWARE_INI_PATH',
defaults['vmware']['ini_path'])
vmware_ini_path = os.path.expanduser(
os.path.expandvars(vmware_ini_path))
config.read(vmware_ini_path)
# apply defaults
for k, v in defaults['vmware'].items():
if not config.has_option('vmware', k):
config.set('vmware', k, str(v))
# where is the cache?
self.cache_dir = os.path.expanduser(config.get('vmware', 'cache_path'))
if self.cache_dir and not os.path.exists(self.cache_dir):
os.makedirs(self.cache_dir)
# set the cache filename and max age
cache_name = config.get('vmware', 'cache_name')
self.cache_path_cache = self.cache_dir + "/%s.cache" % cache_name
self.debugl('cache path is %s' % self.cache_path_cache)
self.cache_max_age = int(config.getint('vmware', 'cache_max_age'))
# mark the connection info
self.server = os.environ.get('VMWARE_SERVER',
config.get('vmware', 'server'))
self.debugl('server is %s' % self.server)
self.port = int(
os.environ.get('VMWARE_PORT', config.get('vmware', 'port')))
self.username = os.environ.get('VMWARE_USERNAME',
config.get('vmware', 'username'))
self.debugl('username is %s' % self.username)
self.password = os.environ.get('VMWARE_PASSWORD',
config.get('vmware', 'password'))
self.validate_certs = os.environ.get(
'VMWARE_VALIDATE_CERTS', config.get('vmware', 'validate_certs'))
if self.validate_certs in ['no', 'false', 'False', False]:
self.validate_certs = False
self.debugl('cert validation is %s' % self.validate_certs)
# behavior control
self.maxlevel = int(config.get('vmware', 'max_object_level'))
self.debugl('max object level is %s' % self.maxlevel)
self.lowerkeys = config.get('vmware', 'lower_var_keys')
if type(self.lowerkeys) != bool:
if str(self.lowerkeys).lower() in ['yes', 'true', '1']:
self.lowerkeys = True
else:
self.lowerkeys = False
self.debugl('lower keys is %s' % self.lowerkeys)
self.skip_keys = list(config.get('vmware', 'skip_keys').split(','))
self.debugl('skip keys is %s' % self.skip_keys)
self.host_filters = list(
config.get('vmware', 'host_filters').split(','))
self.debugl('host filters are %s' % self.host_filters)
self.groupby_patterns = list(
config.get('vmware', 'groupby_patterns').split(','))
self.debugl('groupby patterns are %s' % self.groupby_patterns)
# Special feature to disable the brute force serialization of the
# virtulmachine objects. The key name for these properties does not
# matter because the values are just items for a larger list.
if config.has_section('properties'):
self.guest_props = []
for prop in config.items('properties'):
self.guest_props.append(prop[1])
# save the config
self.config = config
def __init__(self, f):
self.ini_parser = configparser.ConfigParser()
self.ini_parser.readfp(f)
def main():
parser = argparse.ArgumentParser()
l.setup_logging_arguments(parser)
parser.add_argument('--message-file', dest='message_file', default=None,
help='The close pull request message')
args = parser.parse_args()
l.configure_logging(args)
if args.message_file:
try:
with open(args.message_file, 'r') as _file:
pull_request_text = _file.read()
except (OSError, IOError):
log.exception("Could not open close pull request message file")
raise
else:
pull_request_text = MESSAGE
GITHUB_SECURE_CONFIG = os.environ.get('GITHUB_SECURE_CONFIG',
'/etc/github/github.secure.config')
secure_config = configparser.ConfigParser()
secure_config.read(GITHUB_SECURE_CONFIG)
registry = u.ProjectsRegistry()
if secure_config.has_option("github", "oauth_token"):
ghub = github.Github(secure_config.get("github", "oauth_token"))
else:
ghub = github.Github(secure_config.get("github", "username"),
secure_config.get("github", "password"))
orgs = ghub.get_user().get_orgs()
orgs_dict = dict(zip([o.login.lower() for o in orgs], orgs))
for section in registry.configs_list:
project = section['project']
# Make sure we're using GitHub for this project:
if not p.has_github(project):
continue
# Make sure we're supposed to close pull requests for this project:
if 'options' in section and 'has-pull-requests' in section['options']:
continue
# Find the project's repo
project_split = project.split('/', 1)
# Handle errors in case the repo or the organization doesn't exists
try:
if len(project_split) > 1:
org = orgs_dict[project_split[0].lower()]
repo = org.get_repo(project_split[1])
else:
repo = ghub.get_user().get_repo(project)
except (KeyError, github.GithubException):
log.exception("Could not find project %s on GitHub." % project)
continue
# Close each pull request
pull_requests = repo.get_pulls("open")
for req in pull_requests:
vars = dict(project=project)
issue_data = {"url": repo.url + "/issues/" + str(req.number)}
issue = github.Issue.Issue(requester=req._requester,
headers={},
attributes=issue_data,
completed=True)
issue.create_comment(pull_request_text % vars)
req.edit(state="closed")
def _read_ini_settings(self):
''' Read ini file settings '''
scriptbasename = "ocp-on-vmware"
defaults = {'vmware': {
'ini_path': os.path.join(os.path.dirname(__file__), '%s.ini' % scriptbasename),
'console_port':'8443',
'container_storage':'none',
'deployment_type':'openshift-enterprise',
'openshift_vers':'v3_6',
'vcenter_username':'******',
'vcenter_template_name':'ocp-server-template-2.0.2',
'vcenter_folder':'ocp',
'vcenter_resource_pool':'/Resources/OCP3',
'app_dns_prefix':'apps',
'vm_network':'VM Network',
'vm_ipaddr_allocation_type': 'static',
'rhel_subscription_pool':'Red Hat OpenShift Container Platform, Premium*',
'openshift_sdn':'redhat/openshift-ovs-subnet',
'byo_lb':'False',
'lb_host':'haproxy-',
'byo_nfs':'False',
'nfs_host':'nfs-0',
'nfs_registry_mountpoint':'/exports',
'master_nodes':'3',
'infra_nodes':'2',
'app_nodes':'3',
'ocp_hostname_prefix':'',
'auth_type':'ldap',
'ldap_user':'******',
'ldap_user_password':'',
'tag': self.tag,
'ldap_fqdn':'' }
}
if six.PY3:
config = configparser.ConfigParser()
else:
config = configparser.SafeConfigParser()
# where is the config?
self.vmware_ini_path = os.environ.get('VMWARE_INI_PATH', defaults['vmware']['ini_path'])
self.vmware_ini_path = os.path.expanduser(os.path.expandvars(self.vmware_ini_path))
config.read(self.vmware_ini_path)
# apply defaults
for k,v in defaults['vmware'].iteritems():
if not config.has_option('vmware', k):
config.set('vmware', k, str(v))
self.console_port = config.get('vmware', 'console_port')
self.cluster_id = config.get('vmware', 'cluster_id')
self.container_storage = config.get('vmware', 'container_storage')
self.deployment_type = config.get('vmware','deployment_type')
self.openshift_vers = config.get('vmware','openshift_vers')
self.vcenter_host = config.get('vmware', 'vcenter_host')
self.vcenter_username = config.get('vmware', 'vcenter_username')
self.vcenter_password = config.get('vmware', 'vcenter_password')
self.vcenter_template_name = config.get('vmware', 'vcenter_template_name')
self.vcenter_folder = config.get('vmware', 'vcenter_folder')
self.vcenter_datastore = config.get('vmware', 'vcenter_datastore')
self.vcenter_datacenter = config.get('vmware', 'vcenter_datacenter')
self.vcenter_cluster = config.get('vmware', 'vcenter_cluster')
self.vcenter_datacenter = config.get('vmware', 'vcenter_datacenter')
self.vcenter_resource_pool = config.get('vmware', 'vcenter_resource_pool')
self.dns_zone= config.get('vmware', 'dns_zone')
self.app_dns_prefix = config.get('vmware', 'app_dns_prefix')
self.vm_dns = config.get('vmware', 'vm_dns')
self.vm_gw = config.get('vmware', 'vm_gw')
self.vm_netmask = config.get('vmware', 'vm_netmask')
self.vm_network = config.get('vmware', 'vm_network')
self.vm_ipaddr_allocation_type = config.get('vmware', 'vm_ipaddr_allocation_type')
self.rhel_subscription_user = config.get('vmware', 'rhel_subscription_user')
self.rhel_subscription_pass = config.get('vmware', 'rhel_subscription_pass')
self.rhel_subscription_server = config.get('vmware', 'rhel_subscription_server')
self.rhel_subscription_pool = config.get('vmware', 'rhel_subscription_pool')
self.rhsm_katello_url = config.get('vmware', 'rhsm_katello_url')
self.rhsm_activation_key = config.get('vmware', 'rhsm_activation_key')
self.rhsm_org_id = config.get('vmware', 'rhsm_org_id')
self.openshift_sdn = config.get('vmware', 'openshift_sdn')
self.byo_lb = config.get('vmware', 'byo_lb')
self.lb_host = config.get('vmware', 'lb_host')
self.lb_ha_ip = config.get('vmware', 'lb_ha_ip')
self.byo_nfs = config.get('vmware', 'byo_nfs')
self.nfs_host = config.get('vmware', 'nfs_host')
self.nfs_registry_mountpoint = config.get('vmware', 'nfs_registry_mountpoint')
self.master_nodes = config.get('vmware', 'master_nodes')
self.infra_nodes = config.get('vmware', 'infra_nodes')
self.app_nodes = config.get('vmware', 'app_nodes')
self.storage_nodes = config.get('vmware', 'storage_nodes')
self.vm_ipaddr_start = config.get('vmware', 'vm_ipaddr_start')
self.vm_ipaddr_allocation_type = config.get('vmware', 'vm_ipaddr_allocation_type')
self.ocp_hostname_prefix = config.get('vmware', 'ocp_hostname_prefix') or ''
self.auth_type = config.get('vmware', 'auth_type')
self.ldap_user = config.get('vmware', 'ldap_user')
self.ldap_user_password = config.get('vmware', 'ldap_user_password')
self.ldap_fqdn = config.get('vmware', 'ldap_fqdn')
err_count=0
required_vars = {
'dns_zone': self.dns_zone,
'vcenter_host': self.vcenter_host,
'vcenter_password': self.vcenter_password,
'vm_ipaddr_start': self.vm_ipaddr_start,
'vm_ipaddr_allocation_type': self.vm_ipaddr_allocation_type,
'ldap_fqdn': self.ldap_fqdn,
'ldap_user_password': self.ldap_user_password,
'vm_dns': self.vm_dns,
'vm_gw': self.vm_gw,
'vm_netmask': self.vm_netmask,
'vcenter_datacenter': self.vcenter_datacenter,
}
for k, v in required_vars.items():
if v == '':
err_count += 1
print "Missing %s " % k
if required_vars['vm_ipaddr_allocation_type'] not in ('dhcp', 'static'):
err_count += 1
print ("'vm_ipaddr_allocation_type' can take only "
"'dhcp' and 'static' values.")
if err_count > 0:
print "Please fill out the missing variables in %s " % self.vmware_ini_path
exit (1)
self.wildcard_zone="%s.%s" % (self.app_dns_prefix, self.dns_zone)
self.support_nodes=0
if not self.cluster_id:
#create a unique cluster_id first
self.cluster_id = ''.join(random.choice('0123456789abcdefghijklmnopqrstuvwxyz') for i in range(7))
config.set('vmware', 'cluster_id', self.cluster_id)
for line in fileinput.input(self.vmware_ini_path, inplace=True):
if line.startswith('cluster_id'):
print "cluster_id=" + str(self.cluster_id)
else:
print line,
print 'Configured inventory values:'
for each_section in config.sections():
for (key, val) in config.items(each_section):
if 'pass' in key:
print '\t %s: ******' % ( key )
else:
print '\t %s: %s' % ( key, val )
print '\n'
def main():
parser = argparse.ArgumentParser(
description=
'mklauncher is Machinetalk based session/configuration launcher for Machinekit'
)
parser.add_argument('-n',
'--name',
help='Name of the machine',
default="Machinekit Launcher")
parser.add_argument(
'-s',
'--suppress_ip',
help='Do not show ip of machine in service name',
action='store_false',
)
parser.add_argument('-d',
'--debug',
help='Enable debug mode',
action='store_true')
parser.add_argument(
'dirs',
nargs='*',
help="List of directories to scan for launcher configurations",
)
args = parser.parse_args()
debug = args.debug
logging.basicConfig()
if debug:
logger.setLevel(logging.DEBUG)
else:
logger.setLevel(logging.INFO)
mkconfig = config.Config()
mkini = os.getenv("MACHINEKIT_INI")
if mkini is None:
mkini = mkconfig.MACHINEKIT_INI
if not os.path.isfile(mkini):
sys.stderr.write("MACHINEKIT_INI " + mkini + " does not exist\n")
sys.exit(1)
mki = configparser.ConfigParser()
mki.read(mkini)
uuid = mki.get("MACHINEKIT", "MKUUID")
remote = mki.getint("MACHINEKIT", "REMOTE")
if remote == 0:
logger.info(
"Remote communication is deactivated, configserver will use the loopback interfaces"
)
logger.info(("set REMOTE in " + mkini +
" to 1 to enable remote communication"))
logger.debug("announcing mklauncher")
context = zmq.Context()
context.linger = 0
register_exit_handler()
hostname = '%(fqdn)s' # replaced by service announcement
mklauncher = Mklauncher(
context,
svc_uuid=uuid,
host=hostname,
launcher_dirs=args.dirs,
name=args.name,
host_in_name=bool(args.suppress_ip),
loopback=(not remote),
debug=debug,
)
mklauncher.start()
while mklauncher.running and not check_exit():
time.sleep(1)
logger.debug('stopping threads')
mklauncher.stop()
# wait for all threads to terminate
while threading.active_count() > 1:
time.sleep(0.1)
logger.debug('threads stopped')
sys.exit(0)
def setup_logging(config_path, config_opts, options):
log_ini = os.path.join(config_path, config_opts["log_config_file"])
try:
if not os.path.exists(log_ini):
if os.path.normpath('/etc/mock') != os.path.normpath(config_path):
log.warning(
"Could not find required logging config file: %s. Using default...",
log_ini)
log_ini = os.path.join("/etc/mock",
config_opts["log_config_file"])
if not os.path.exists(log_ini):
raise IOError("Could not find log config file %s" %
log_ini)
else:
raise IOError("Could not find log config file %s" % log_ini)
except IOError as exc:
log.error(exc)
sys.exit(50)
try:
log_cfg = configparser.ConfigParser()
logging.config.fileConfig(log_ini)
log_cfg.read(log_ini)
except (IOError, OSError, configparser.NoSectionError) as exc:
log.error("Log config file(%s) not correctly configured: %s", log_ini,
exc)
sys.exit(50)
try:
# set up logging format strings
config_opts['build_log_fmt_str'] = log_cfg.get(
"formatter_%s" % config_opts['build_log_fmt_name'],
"format",
raw=1)
config_opts['root_log_fmt_str'] = log_cfg.get(
"formatter_%s" % config_opts['root_log_fmt_name'], "format", raw=1)
config_opts['state_log_fmt_str'] = log_cfg.get(
"formatter_%s" % config_opts['state_log_fmt_name'],
"format",
raw=1)
except configparser.NoSectionError as exc:
log.error("Log config file (%s) missing required section: %s", log_ini,
exc)
sys.exit(50)
# set logging verbosity
if options.verbose == 0:
log.handlers[0].setLevel(logging.WARNING)
tmplog = logging.getLogger("mockbuild.Root.state")
if tmplog.handlers:
tmplog.handlers[0].setLevel(logging.WARNING)
elif options.verbose == 1:
log.handlers[0].setLevel(logging.INFO)
elif options.verbose == 2:
log.handlers[0].setLevel(logging.DEBUG)
logging.getLogger("mockbuild.Root.build").propagate = 1
logging.getLogger("mockbuild").propagate = 1
# enable tracing if requested
logging.getLogger("trace").propagate = 0
if options.trace:
logging.getLogger("trace").propagate = 1
def __init__(self, oracle_database="pittdb", config=None, **kwargs):
self.connection = None
self.oracle_database = oracle_database
cfg = {}
if config is not None:
cf = configparser.ConfigParser()
cf.read(config)
config_keys = [
"oracleuser",
"oraclepass",
"oracledsn",
"voy_username",
"voy_password",
"voy_path",
"cat_location",
"library_id",
]
for item in config_keys:
val = cf.get("Voyager", item, fallback="", raw=True).strip('"')
if val:
cfg[item] = val
if cf.get("Voyager", "connectstring") and "oracledsn" not in cfg:
cfg["oracledsn"] = cf.get("Voyager", "connectstring", raw=True).strip(
'"'
)
cfg.update(kwargs)
if all(arg in cfg for arg in ["oracleuser", "oraclepass", "oracledsn"]):
self.connection = cx.connect(
cfg["oracleuser"], cfg["oraclepass"], cfg["oracledsn"]
)
self.engine = sqla.create_engine(
"oracle://", creator=lambda: self.connection
)
metadata = sqla.MetaData()
tables_to_load = TABLE_NAMES
self.tables = {}
for table_name in tables_to_load:
self.tables[table_name] = sqla.Table(
table_name,
metadata,
schema=oracle_database,
autoload=True,
autoload_with=self.engine,
)
for parent, foreign in RELATIONS:
parent_column = getattr(self.tables[parent[0]].c, parent[1])
foreign_key = getattr(self.tables[foreign[0]].c, foreign[1])
parent_column.append_foreign_key(sqla.ForeignKey(foreign_key))
self.cat_location = cfg.get("cat_location")
self.library_id = cfg.get("library_id")
if "voy_path" not in cfg:
cfg["voy_path"] = r"C:\Voyager"
if batchcat is not None and all(
arg in cfg for arg in ["voy_username", "voy_password"]
):
self.batchcat = batchcat.BatchCatClient(
username=cfg["voy_username"],
password=cfg["voy_password"],
voy_interface=self,
apppath=cfg["voy_path"],
)
else:
self.batchcat = None
def get_restore_settings(SECTION,NAME):
cfg = ConfigParser.ConfigParser()
configfile = os.path.join(os.environ.get("HOME", "."), ".tinybldlin")
cfg.read(configfile)
VALUE= cfg.get(SECTION, NAME)
return VALUE
if cache_available(config):
inv = get_cache('scaleway_ansible_inventory.json', config)
else:
inv = generate_inv_from_api(config)
save_cache(inv, config)
return json.dumps(inv)
if __name__ == '__main__':
inventory = {}
# Read config
if six.PY3:
config = configparser.ConfigParser()
else:
config = configparser.SafeConfigParser()
for configfilename in [os.path.abspath(sys.argv[0]).rsplit('.py')[0] + '.ini', 'scaleway.ini']:
if os.path.exists(configfilename):
config.read(configfilename)
break
if cache_available(config):
inventory = get_cache('scaleway_ansible_inventory.json', config)
else:
inventory = get_inventory(config)
# return to ansible
sys.stdout.write(str(inventory))
sys.stdout.flush()
def __init__(self, config_file):
self.parser = configparser.ConfigParser()
self.parser.read(config_file)
self.config_file = config_file
