def isNewGenerationByOS(machine, project_name):
os = machine['sourceProperties'].get('os').lower() if machine['sourceProperties'].get('os') else 'none'
if os.startswith('microsoft'):
splited_os = os.split(' ')
if len(splited_os) > 3:
# Microsoft Windows Server 2003
# Microsoft Windows Server 2008 R1
# Microsoft Windows Server 2012 R1
ws_version = splited_os[3]
if ws_version.isdigit():
ws_version = int(ws_version)
if ws_version > 2003:
if 2008 == ws_version or 2012 == ws_version:
if len(splited_os)>4 and splited_os[4]=='r2':
return True
else:
return True
return False
elif os.startswith('linux'):
#Linux version 3 below
#Linux version 3.0.101-63-default (geeko@buildhost) (gcc version 4.3.4 [gcc-4_3-branch revision 152973] (SUSE Linux) ) #1 SMP Tue Jun 23 16:02:31 UTC 2015 (4b89d0c)
splited_os = os.split(' ')
if len(splited_os) >= 3:
if len(splited_os[2])>0:
if splited_os[2][0].isdigit():
if int(splited_os[2][0]) > 2:
return True
print("Project : {} , HostName : {} is Unidentified OS".format(project_name, machine['sourceProperties']['name']))
return False
def default_os(
default_architecture,
default_partitiontable,
default_pxetemplate,
os=None,
):
if os is None:
os = (entities.OperatingSystem().search(
query={
'search':
'name="RedHat" AND (major="{0}" OR major="{1}")'.format(
RHEL_6_MAJOR_VERSION, RHEL_7_MAJOR_VERSION)
})[0].read())
else:
major = os.split(' ')[1].split('.')[0]
minor = os.split(' ')[1].split('.')[1]
os = (entities.OperatingSystem().search(
query={
'search':
f'family="Redhat" AND major="{major}" AND minor="{minor}"'
})[0].read())
os.architecture.append(default_architecture)
os.ptable.append(default_partitiontable)
os.provisioning_template.append(default_pxetemplate)
os.update(['architecture', 'ptable', 'provisioning_template'])
os = entities.OperatingSystem(id=os.id).read()
return os
def ipv4_hostname_output(self):
ipv4_host_title = '| {0:16} | {1:16} | {2:18} | {3:10} | {4:18} | {5:1}'.format('Host', 'IPv4', 'MAC', 'Domain', 'Server Type', 'Windows OS (Server Fingerprint)')
print '-' * blessings.Terminal().width
print self.color(ipv4_host_title, char='')
print '-' * blessings.Terminal().width
server_type = ''
for host in sorted(self.hosts):
ipv4 = self.hosts[host]['ipv4']
mac = self.hosts[host]['mac']
if host != None and '*' not in host:
if 'fqdn' in self.hosts[host].keys():
print self.hosts[host]['fqdn']
mac = self.hosts[host]['mac']
os = self.hosts[host]['os']
nt_version = None
os_version = os
serverlist = {'domain_controller': 'DC', 'backup_controller': 'Backup DC', 'sql_server': 'SQL', 'print': 'Printer'}
host_comment = None
if 'comment' in self.hosts[host].keys():
host_comment = self.hosts[host]['comment']
if os != None and not os.startswith('Microsoft'):
nt_version = os.split('(')[1].split(')')[0].strip()
os_version = os.split('(')[0].strip()
if host_comment != None and list(host_comment)[0] != '\x00':
os_version += ' ({})'.format(host_comment.capitalize())
domain = self.hosts[host]['domain']
#notes = self.hosts[host]['notes']
if 'server_keys' in self.hosts[host].keys():
servers = []
server_types = self.hosts[host]['server_keys']
for server in server_types:
if server_types[server] == '1' and server in serverlist.keys():
servers.append(serverlist[server])
ipv4_host_output = '| {0:16} | {1:16} | {2:18} | {3:10} | {4:18} | {5:1}'.format(host.upper(), ipv4, mac, domain, ','.join(servers).strip(), os_version)
print self.color(ipv4_host_output, char='')
print '-' * blessings.Terminal().width
print ''
def get_win8_targets(self):
for host in self.hosts.keys():
os = self.hosts[host]['os']
if os != None and not os.startswith('Microsoft'):
nt_version = os.split('(')[1].split(')')[0].strip()
os_version = os.split('(')[0].strip()
if 'Win 8' in os_version:
print host
def get_latest_os_version(os):
latest_os_version, current_os_version = '', ''
if 'Flatcar' in os:
ver = requests.get(
"https://stable.release.flatcar-linux.net/amd64-usr/current/version.txt"
)
latest_os_version = re.findall('(FLATCAR_VERSION=)(.+)', ver.text)
current_os_version = os.split()[5]
if version.parse(str(current_os_version)) < version.parse(
str(latest_os_version[0][1])):
print(Output.YELLOW + "[WARNING] " + Output.RESET + \
"Cluster nodes are not running on latest {}{}"\
.format(latest_os_version[0][0], latest_os_version[0][1]))
status = 'outdated'
latest_os_version = latest_os_version[0][1]
else:
status = 'latest'
latest_os_version = latest_os_version[0][1]
elif 'CoreOS' in os:
print(Output.YELLOW + "[WARNING] " + Output.RESET + \
"Cluster nodes are running on CoreOS which is DPERECATED: https://coreos.com/os/eol/. " + \
"PLEASE CONSIDER CHANGING THE DEPRECATED OS!")
latest_os_version = 'EOL'
status = 'EOL'
return version_check.append(
['OS', latest_os_version, current_os_version, status])
def autocreate_model(basexml, weather, randomxml='random.xml', schedule=None):
randompath = os.path.split(os.path.abspath(randomxml))[0]
found = False
while not found:
random_model(basexml, randomxml)
randomxmlstr = None
with open(randomxml) as randomxmlfile:
randomxmlstr = randomxmlfile.read()
idf = idfxml.xml_to_idf(randomxmlstr)
idfname = '{}.idf'.format(randomxml)
with open(idfname, 'w') as idffile:
idffile.write(str(idf))
if schedule is not None:
schedule = [(schedule, os.split(schedule)[1])]
runner = eplus.EnergyPlus()
results = runner.run(idfname,
weather,
supplemental_files=schedule,
working_directory=randompath)
if results is not None:
found = True
try:
os.remove(idfname)
except:
pass
return results
def recursive_move(src, dest, pattern):
'''Recursively moves files by extension
Args:
src (str): String source directory
dest (str): String destination directory
pattern: (see example)
Example:
pattern = re.compile('.*\.(mov|MOV|avi|mpg)$')
'''
import directories
import os
import numpy as np
filelist = file_list(src, pattern)
for f in file_list:
d, name = os.split(f)
name, ext = os.path.splitext(f)
child_dir = os.path.split(d)
dest_dir = dest+child_dir
# Replace src with dest in file path
#if !exist create dest/child_d
directories.mkdir_p(dest_dir)
#mv file dest/child_d
os.rename(f, dest_dir+name+ext)
def handle_options(cfg, options):
# hack alert:
if options.dumptime:
cfg['dump-req-times'] = True
if options.dumpatom:
cfg['dump-atom-reqs'] = True
if options.stampMsgs:
cfg['stamp-messages'] = True
# Bonus hack for -C
if options.extraconfig:
for os in options.extraconfig:
if ':' not in os:
cfg[os] = True
else:
on, ov = os.split(':', 1)
cfg[on] = ov
# Ditto for -D, which is at least simpler.
if options.rmconfig:
for os in options.rmconfig:
cfg.rm(os)
# Forcefully disable caching.
if options.noCache:
cfg.rm("cachedir")
cfg.rm("render-cache")
cfg.rm("bfc-cache-ttl")
cfg.rm("imc-cache-entries")
if options.noBruteCache:
cfg.rm("bfc-cache-ttl")
cfg.rm("imc-cache-entries")
# We rely on being able to rerun checkGoodConfig(), which may
# take some work in the underlying system.
cfg.checkGoodConfig()
def handle_options(cfg, options):
# hack alert:
if options.dumptime:
cfg['dump-req-times'] = True
if options.dumpatom:
cfg['dump-atom-reqs'] = True
if options.stampMsgs:
cfg['stamp-messages'] = True
# Bonus hack for -C
if options.extraconfig:
for os in options.extraconfig:
if ':' not in os:
cfg[os] = True
else:
on, ov = os.split(':', 1)
cfg[on] = ov
# Ditto for -D, which is at least simpler.
if options.rmconfig:
for os in options.rmconfig:
cfg.rm(os)
# Forcefully disable caching.
if options.noCache:
cfg.rm("cachedir")
cfg.rm("render-cache")
cfg.rm("bfc-cache-ttl")
cfg.rm("imc-cache-entries")
if options.noBruteCache:
cfg.rm("bfc-cache-ttl")
cfg.rm("imc-cache-entries")
# We rely on being able to rerun checkGoodConfig(), which may
# take some work in the underlying system.
cfg.checkGoodConfig()
def autocreate_model(basexml, weather, randomxml='random.xml', schedule=None):
randompath = os.path.split(os.path.abspath(randomxml))[0]
found = False
while not found:
random_model(basexml, randomxml)
randomxmlstr = None
with open(randomxml) as randomxmlfile:
randomxmlstr = randomxmlfile.read()
idf = idfxml.xml_to_idf(randomxmlstr)
idfname = '{}.idf'.format(randomxml)
with open(idfname, 'w') as idffile:
idffile.write(str(idf))
if schedule is not None:
schedule = [(schedule, os.split(schedule)[1])]
runner = eplus.EnergyPlus()
results = runner.run(idfname, weather,
supplemental_files=schedule,
working_directory=randompath)
if results is not None:
found = True
try:
os.remove(idfname)
except:
pass
return results
def run_test(ups_path,WITH_MPI=False,NUM_PROCS=1,RESTART=False,DAMPING_OFF_NEW_END_TIME=False,
POST_PROC_ONLY=False):
''' '''
print '\nRunning test:\t',os.path.split(ups_path)[1]
#Determine root path
root_path = os.path.split(os.path.abspath(ups_path))[0]
#Determine uda path
print "Root path = ", root_path
print "Ups path = ", ups_path
F_ups = open(ups_path,"r")
ups_lines = F_ups.read()
uda_path = root_path + '/' + ups_lines.split('<filebase>')[1].split('</filebase>')[0].strip()
F_ups.close()
print "UDA path = ", uda_path
#Change current working directory to root path
os.chdir(root_path)
#Open runlog
F_log = open(root_path+'/TEST_RUNLOG_'+os.path.split(ups_path)[1],"w")
#Construct the argument list for subprocess to use.
if not(WITH_MPI) or int(NUM_PROCS)<=1:
args = [uintah_exe,os.path.split(ups_path)[1]]
else:
args = ['mpirun','-np',str(int(NUM_PROCS)), uintah_exe,'-mpi',os.path.split(ups_path)[1]]
if POST_PROC_ONLY:
uda_path = uda_path+'.000'
else:
#Run the test and wait for it to complete
tmp = sub_proc.Popen(args,stdout=F_log,stderr=sub_proc.PIPE)
dummy = tmp.wait()
F_log.close()
#If test calls for retstart
if RESTART:
#If turn damping off and run to new end time
if DAMPING_OFF_NEW_END_TIME:
#Setup the restart by setting damping to zero and modifying end time
print 'Setting <artificial_damping_coeff> to zero and restarting with new end time of ',format(NEW_END_TIME,'1.4e')
setup_restart(uda_path,DAMPING_OFF_NEW_END_TIME)
print 'Done.\nRestarting...'
#Open new runlog
F_log = open(root_path+'/TEST_RUNLOG_RESTART_'+os.split(ups_path)[1],"w")
#Construct the argument list
if not(WITH_MPI) or NUM_PROCS<=1:
args = [uintah_exe,'-restart','-move',uda_path+'.000']
else:
args = ['mpirun','-np',str(int(NUM_PROCS)), uintah_exe,'-mpi','-restart','-move',uda_path+'.000']
#Run the test and wait for it to complete
tmp = sub_proc.Popen(args,stdout=F_log,stderr=sub_proc.PIPE)
dummy = tmp.wait()
F_log.close()
uda_path = uda_path+'.001'
else:
uda_path = uda_path+'.000'
print('Test done.')
return uda_path
def run_test(ups_path,WITH_MPI=False,NUM_PROCS=1,RESTART=False,DAMPING_OFF_NEW_END_TIME=False):
''' '''
print '\nRunning test:\t',os.path.split(ups_path)[1]
#Determine root path
root_path = os.path.split(os.path.abspath(ups_path))[0]
#Determine uda path
print "Root path = ", root_path
print "Ups path", ups_path
F_ups = open(ups_path,"r")
ups_lines = F_ups.read()
uda_path = root_path + "/" + ups_lines.split('<filebase>')[1].split('</filebase>')[0].strip()
F_ups.close()
print "UDA path", uda_path
#Change current working directory to root path
os.chdir(root_path)
if RUN_TESTS:
#Open runlog
F_log = open(root_path+'/TEST_RUNLOG_'+os.path.split(ups_path)[1],"w")
#Construct the argument list for subprocess to use.
if not(WITH_MPI) or int(NUM_PROCS)<=1:
args = [uintah_exe, os.path.split(ups_path)[1]]
else:
args = ['mpirun','-np',str(int(NUM_PROCS)), uintah_exe,'-mpi',os.path.split(ups_path)[1]]
#Run the test and wait for it to complete
print "Arguments = ", args
print "F_log = ", F_log
tmp = sub_proc.Popen(args,stdout=F_log,stderr=sub_proc.PIPE)
dummy = tmp.wait()
F_log.close()
#If test calls for retstart
if RESTART:
#If turn damping off and run to new end time
if DAMPING_OFF_NEW_END_TIME:
#Setup the restart by setting damping to zero and modifying end time
print 'Setting <artificial_damping_coeff> to zero and restarting with new end time of ',format(NEW_END_TIME,'1.4e')
setup_restart(uda_path,DAMPING_OFF_NEW_END_TIME)
print 'Done.\nRestarting...'
#Open new runlog
F_log = open(root_path+'/TEST_RUNLOG_RESTART_'+os.split(ups_path)[1],"w")
#Construct the argument list
if not(WITH_MPI) or NUM_PROCS<=1:
args = [ uintah_exe,'-restart','-move',uda_path+'.000']
else:
args = ['mpirun','-np',str(int(NUM_PROCS)), uintah_exe,'-mpi','-restart','-move',uda_path+'.000']
#Run the test and wait for it to complete
tmp = sub_proc.Popen(args,stdout=F_log,stderr=sub_proc.PIPE)
dummy = tmp.wait()
F_log.close()
if RESTART:
uda_path = uda_path+'.001'
else:
uda_path = uda_path+'.000'
print('Test done.')
return uda_path
def get_os(self):
os = self.info.distribution_version.lower()
to_be_replaced = ['\n', ' ', 'gnu/linux']
for _ in to_be_replaced:
if _ in os:
os = os.replace(_, '')
if self.info.deliverable_type == "dmg":
major_version = os.split('.')
os = major_version[0] + '.' + major_version[1]
return os
def assemble_suffixed_filename(filename, suffix=0):
'''
Split document filename, to attach suffix to the name part then
re attacht the extension
'''
if suffix:
name, extension = filename.split(os.split(os.extsep))
return SUFFIX_SEPARATOR.join([name, unicode(suffix), os.extsep, extension])
else:
return file_filename
def get_os(self, i):
b = i.split('),')
for i in b:
if ',' in i or 'or' in i:
pass
else:
if '(' in i:
os = i.split('(')[0].strip()
if 'Aggressive OS guesses' in os:
os = os.split(':')[-1]
return os
def __login(self):
conf = configparser.ConfigParser()
if os.path.exists(self.__PATH):
conf.read(self.__PATH)
sa = conf.get(self.db, 'accountname')
pw = conf.get(self.db, 'password')
host = conf.get(self.db, 'ip')
port = conf.get(self.db, 'port')
dbname = conf.get(self.db, 'dbname')
return sa, pw, host, port, dbname
else:
raise IOError('NotFoundFile:{}'.format(os.split(self.__PATH)[-1]))
def assemble_suffixed_filename(filename, suffix=0):
'''
Split document filename, to attach suffix to the name part then
re attacht the extension
'''
if suffix:
name, extension = filename.split(os.split(os.extsep))
return SUFFIX_SEPARATOR.join(
[name, unicode(suffix), os.extsep, extension])
else:
return file_filename
def ok_dep(dep):
os = dep.get('os')
if os is not None:
oses = {x.strip().lower() for x in os.split(',')}
if OS_NAME not in oses:
return False
py = dep.get('python')
if py is not None:
q = ok_dep.major_version
if isinstance(py, str):
return q < int(py[1:])
return q >= py
return True
def get_latest_os_version(os, logger):
latest_os_version, current_os_version, status = [''] * 3
if 'Flatcar' in os:
session = requests.Session()
ver = session.get(
"https://stable.release.flatcar-linux.net/amd64-usr/current/version.txt"
)
session.close()
latest_os_version = re.findall('(FLATCAR_VERSION=)(.+)', ver.text)
current_os_version = os.split()[5]
if version.parse(str(current_os_version)) < version.parse(
str(latest_os_version[0][1])):
logger.warning("Cluster nodes are not running on latest {}{}"\
.format(latest_os_version[0][0], latest_os_version[0][1]))
status = Nodes.outdated
latest_os_version = latest_os_version[0][1]
else:
status = Nodes.latest
latest_os_version = latest_os_version[0][1]
elif 'CoreOS' in os:
logger.warning("Cluster nodes are running on CoreOS which is DPERECATED: https://coreos.com/os/eol/. " + \
"PLEASE CONSIDER CHANGING THE DEPRECATED OS!")
latest_os_version = 'EOL'
status = 'EOL'
elif 'Ubuntu' in os:
current_os_version = re.sub('[^0-9.]', '', os)
session = requests.Session()
ver = requests.get("https://api.launchpad.net/devel/ubuntu/series")
session.close()
for x in ver.json()['entries']:
if 'Current Stable Release' in x['status']:
latest_os_version = x['version']
if version.parse(str(current_os_version)) < version.parse(
str(latest_os_version)):
logger.warning(
"Cluster nodes are not running on latest Ubuntu version."
)
status = Nodes.outdated
else:
status = Nodes.latest
else:
latest_os_version, current_os_version, status = [
'OS not supported'
] * 3
return version_check.append(
['OS', latest_os_version, current_os_version, status])
def getParams():
switches, files= parse_options(sys.argv[1:],default_options.copy())
if switches.has_key('f'):
os=open(switches['f']).read()
s2, f2 = parse_options(os.split(),switches)
f2=[f for f in f2 if f]
switches.update(s2)
files.extend(f2)
for o in switches.keys():
if option_dtypes.has_key(o):
switches[o]=option_dtypes[o](switches[o])
if 't' in switches.keys():
testme(files, switches)
sys.exit()
return (files, switches)
def function_wrapper(obj, omesh_path=None, use_global_frame=True):
if not omesh_path:
omeshfolder_path = os.path.join("./meshes/", obj.Label)
omesh_path = os.path.join(omeshfolder_path, "full")
else:
omeshfolder_path = os.split(omesh_path)[0]
if not os.path.exists(omeshfolder_path):
os.makedirs(omeshfolder_path)
prev_pl = obj.Placement
if not use_global_frame:
pl = getGlobalPlacement(obj)
obj.Placement = pl.inverse().multiply(obj.Placement)
res = func(obj, omesh_path)
obj.Placement = prev_pl
return res
def arch_update(d, prefix, gcc_version):
arch = d.get(prefix+'_ARCH', True)
gccspec = arch_gccspec(arch, gcc_version)
(cpu, vendor, os) = arch_split(arch)
d[prefix+'_CPU'] = cpu
d[prefix+'_VENDOR'] = vendor
d[prefix+'_OS'] = os
ost = os.split('-',1)
if len(ost) > 1:
d[prefix+'_BASEOS'] = ost[0]
else:
d[prefix+'_BASEOS'] = ""
for spec in gccspec:
if spec in ("abi flags"):
continue
d[prefix+'_'+spec.upper()] = gccspec[spec]
return
def arch_update(d, prefix, gcc_version):
arch = d.get(prefix+'_ARCH', True)
gccspec = arch_gccspec(arch, gcc_version)
(cpu, vendor, os) = arch_split(arch)
d[prefix+'_CPU'] = cpu
d[prefix+'_VENDOR'] = vendor
d[prefix+'_OS'] = os
ost = os.split('-',1)
if len(ost) > 1:
d[prefix+'_BASEOS'] = ost[0]
else:
d[prefix+'_BASEOS'] = ""
for spec in gccspec:
if spec in ("abi flags"):
continue
d[prefix+'_'+spec.upper()] = gccspec[spec]
return
def __download_album(self, audio_dl_uri):
fullpath = urlparse.urlparse(audio_dl_uri).path
basename = os.split(fullpath)[1]
destpath = os.path.join(magnatune_in_progress_dir, basename)
shell = self.get_property('shell')
manager = shell.get_player().get_property('ui-manager')
manager.get_action("/MagnatuneSourceViewPopup/MagnatuneCancelDownload").set_sensitive(True)
self.__downloading = True
self.cancelled = False
self.__downloads[audio_dl_uri] = 0
# no way to resume downloads, sadly
out = open(destpath, 'w')
dl = rb.ChunkLoader()
dl.get_url_chunks(audio_dl_uri, 4*1024, True, self.__download_album_chunk, (audio_dl_uri, destpath, out))
def import_user_module(user_dir: str, no_print: bool = False):
"""Given a user dir, this function imports it as a module.
This user_module is expected to have an __init__.py at its root.
You can use import_files to import your python files easily in
__init__.py
Args:
user_dir (str): directory which has to be imported
no_print (bool): This function won't print anything if set to true
"""
if user_dir:
user_dir = get_absolute_path(user_dir)
module_parent, module_name = os.split(user_dir)
if module_name not in sys.modules:
sys.path.insert(0, module_parent)
if not no_print:
print(f"Importing user_dir from {user_dir}")
importlib.import_module(module_name)
sys.path.pop(0)
model.compile('adam', 'sparse_categorical_crossentropy')
model.summary()
model.fit(
x=[np.array(encode_input), np.array(decode_input)],
y=np.array(decode_output),
epochs=10,
batch_size=128,
)
# Predict
decoded = decode(
model,
encode_test_input,
start_token=token_dict['<START>'],
end_token=token_dict['<END>'],
pad_token=token_dict['<PAD>'],
)
predicted_y = []
for d_x in decoded:
os = ' '.join(map(lambda x: token_dict_inv[x], d_x[1:-1]))
os = os.split(' ')
predicted_y.append([os])
bleu_score = nltk.translate.bleu_score.corpus_bleu(predicted_y,
test_y_tokens,
weights=(0.5, 0.5))
print(bleu_score)
def _parse_host_os(self, os):
split = os.split('-')
return '{} {}'.format(split[0].strip(), split[1].strip())
try:
jobPipeline, pipelineThread = runningPipelinesPerHost[host]
if pipelineThread.isAlive():
logging.info('job %s on host %s is running' %
(jobPipeline.job['JobId'], host))
else:
logging.info('job %s on host %s is done' %
(jobPipeline.job['JobId'], host))
pipelineThread.join(10)
del runningPipelinesPerHost[host]
if jobPipeline.finishedSuccessfully:
try:
if os.path.isdir(path):
path, filename = os.split(
jobPipeline.job['filename'])
os.rename(
jobPipeline.job['filename'],
os.path.join(path, 'done', filename))
except Exception as e:
logger.error(str(e))
else:
#retry, put back in jobqueue
logging.info(
'job %s on host %s was put back in the queue' %
(jobPipeline.job['JobId'], host))
hostJobQueues[host].append(jobPipeline.job)
except Exception as e:
logger.error(str(e))
time.sleep(30)
dates_versions_response = requests.get(
main_url + 'aggregates_by/build_id/channels/' + args.ch + '/dates/').text
dates_versions_response = literal_eval(dates_versions_response)
dates_for_versions = defaultdict(list)
for item in dates_versions_response:
dates_for_versions[item['version']].append(item['date'])
last_n_versions = sorted(dates_for_versions.keys())[-args.n_versions:]
versions_with_dates = {v: dates_for_versions[v] for v in last_n_versions}
for v in versions_with_dates.keys():
try:
os.makedirs(os.path.join(args.dir, 'nightly_' + v))
except OSError:
pass
options = requests.get(
main_url + 'filters/?channel=' + args.ch + '&version=' + v).text
options = literal_eval(options)
options['os'] = np.unique([os.split(',')[0] for os in options['os']])
for m in options['metric']:
whole_data_for_metric = []
ops = list(product(
[dates_for_versions[v]], [m], [args.ch], [v], options['os'],
options['application'], options['architecture']
))
for data in pool.imap_unordered(_get_data, ops):
whole_data_for_metric.append(data)
# the data is saved as a list of elements from `_get_data`, see docstring
with open(os.path.join(args.dir,
'nightly_' + v + '/' + m + '.json'), 'w') as file:
json.dump(whole_data_for_metric, file)
if re.search(
'tns', prod
) and services.port == 1521 or services.port == 1526:
prod = "tns"
if re.search(
'apache', prod
) and services.port == 7777 or services.port == 7778:
prod = "apache"
if re.search('terminal', prod) and services.port == 3389:
prod = "rdp"
if len(prod.split()) > 1:
prod = prodreplace(prod).strip()
if len(ver.split('.')) > 2:
i = iter(ver.split('.'))
ver = map('.'.join, zip(i, i))[0]
if len(os.split()) > 1:
i = iter(os.split())
os = map(''.join, zip(i, i))[0]
os = osreplace(os).strip()
if 'product' in serv and 'version' in serv and 'ostype' in serv and not 'extrainfo' in serv:
sp, pe = test_patterns(serv, ['product:'])
sv, ve = test_patterns(serv, ['version:'])
so, oe = test_patterns(serv, ['ostype:'])
prod = serv[pe:(sv - 1)].strip().lower()
ver = serv[ve:(so - 1)].strip().lower()
os = serv[oe:].strip().lower()
if services.port == 443:
prod = "https"
os = "windows"
if re.search('microsoft-ds', prod) and services.port == 445:
prod = "smb"
# -*- coding: utf-8 -*-
#title :init.py
#description :This python script creates the Config file
#author :Stefan Baumgartner
#date :16.06.2018
#version :1.00
#usage :python init.py
#notes :Please change only when you know what you do!!
#python_version :2.7.6
#=======================================================================
# Import the modules needed to run the script.
import sys, os, subprocess, platform
# Reads the Distribution on your System
os = str(platform.dist())
config = ""
oslist = os.split(',')
branchversion1 = os.replace('\'', "")
branchversion2 = branchversion1.replace('(', "")
branchversion3 = branchversion2.replace(')', "")
branchversion4 = branchversion3.replace(',', "")
def f_write_config():
with open("/usr/local/share/mocis/config/config.cfg", "w") as OS:
OS.write('OS = %s\n' % branchversion4[:-1])
ostxt = "Your currently installed Linux Distribution is: "
print ostxt + branchversion4[:-1]
print 'File created'
return
# check which jobs are done
busyHosts = set(runningPipelinesPerHost.keys())
for host in busyHosts:
try:
jobPipeline, pipelineThread = runningPipelinesPerHost[host]
if pipelineThread.isAlive():
logging.info('job %s on host %s is running' % (jobPipeline.job['JobId'], host))
else:
logging.info('job %s on host %s is done' % (jobPipeline.job['JobId'], host))
pipelineThread.join(10)
del runningPipelinesPerHost[host]
if jobPipeline.finishedSuccessfully:
try:
if os.path.isdir(path):
path, filename = os.split(jobPipeline.job['filename'])
os.rename(jobPipeline.job['filename'], os.path.join(path, 'done', filename))
except Exception as e:
logger.error(str(e))
else:
#retry, put back in jobqueue
logging.info('job %s on host %s was put back in the queue' % (jobPipeline.job['JobId'], host))
hostJobQueues[host].append(jobPipeline.job)
except Exception as e:
logger.error(str(e))
time.sleep(30)
def olger_parser():
f = open(sys.argv[1])
xml_content = f.read()
print(xml_content)
f.close()
dat = (json.loads(xml_content))
timestamp = datetime.fromtimestamp(float(dat["scan"]["time"]))
print(timestamp)
dateelastic = str(str(timestamp).split(" ")[0]) + "T" + str(
str(timestamp).split(" ")[1]) + "Z"
address = ""
plist = ""
cvelist = {}
graph = {}
nodes = []
links = []
id = 0
for x in dat["systems"]:
if "up" in x["status"]:
nodes.append({
"id": str(id) + "" + str(x["ip"]),
"host": str(x["ip"])
})
for e in x["services"]:
if "open" in e["state"]:
#print e
try:
product = e["banner"].split("product: ")[1]
try:
product = product.split("version: ")[0]
except:
pass
try:
product = product.split("ostype: ")[0]
except:
pass
try:
product = product.split("hostname: ")[0]
except:
pass
product = product.strip()
except:
pass
product = ""
try:
version = e["banner"].split("product: ")[1].split(
"version: ")[1]
try:
version = version.split("ostype: ")[0]
except:
pass
try:
version = version.split("hostname: ")[0]
except:
pass
try:
version = version.split("extrainfo: ")[0]
except:
pass
version = version.strip()
except:
pass
version = ""
os2 = ""
try:
os2 = e["banner"].split("ostype: ")[1]
try:
os2 = os.split("devicetype: ")[0]
except:
pass
os2 = ""
except:
pass
os2 = ""
try:
cpe = e["cpe"]
except:
pass
cpe = ""
cves = ""
cvecount = "0"
print(cpe)
#try:
cvelist = ""
csv_content = ""
cvecount = 0
csv_content = ""
f = open("data/cvedata.csv", "w")
f.write(str(""))
f.close()
if product != "" and version != "":
print("CHEKING CVES ...")
os.system(
"python3 scripts/cvdetails-lookup.py --csv data/cvedata.csv --product \""
+ product + "\" --version \"" + version + "\"")
try:
os.system(
"python3 scripts/cvdetails-lookup.py --csv data/cvedata.csv --product \""
+ product + "\" --version \"" + version + "\"")
f = open("data/cvedata.csv")
csv_content = f.read()
f.close()
cvecount = len(str(csv_content).split("\n"))
cvelist = ""
for x2 in str(csv_content).split("\n"):
cvelist = x2.split(";")[0] + "," + cvelist
print(cvelist)
except:
pass
nodes = {v['id']: v for v in nodes}.values()
#uncomment this line to send data to elastic
#toelastic={"customer":sys.argv[2],"timestamp":dateelastic,"address":x["ip"],"domains":"","lat":"","long":"","os":os,"port":str(e["port"]),"portname":str(e["name"]),"product":str(product),"version":str(version),"cpe":str(cpe),"cves":str(cvelist[product]),"cvecount":int(cvecount)}
if str(e["port"]) != "":
links.append({
"source": str(id) + "" + str(x["ip"]),
"target": str(id) + "" + str(e["port"]),
"value": "port"
})
nodes.append({
"id": str(id) + "" + str(e["port"]),
"host": str(id) + "" + str(e["port"])
})
if str(e["name"]) != "":
nodes.append({
"id": str(id) + "" + str(e["name"]),
"host": str(e["name"])
})
links.append({
"source": str(id) + "" + str(x["ip"]),
"target": str(id) + "" + str(e["name"]),
"value": "portname"
})
print "product : " + filter(str(product))
if str(product) != "":
print filter(str(product))
print str(x["ip"])
nb = filter(str(product))
try:
inventory_by_product[nb][id] = str(x["ip"])
except:
inventory_by_product[nb] = {}
inventory_by_product[nb][id] = str(x["ip"])
pass
nodes.append({
"id": str(id) + "" + str(product),
"host": str(product)
})
links.append({
"source": str(id) + "" + str(x["ip"]),
"target": str(id) + "" + str(product),
"value": "product"
})
if str(version) != "":
nodes.append({
"id": str(id) + "" + str(version),
"host": str(version)
})
links.append({
"source": str(id) + "" + str(version),
"target": str(id) + "" + str(version),
"value": "version"
})
if str(os2) != "":
nodes.append({
"id": str(id) + "" + str(os2),
"host": str(os2)
})
links.append({
"source": str(id) + "" + str(x["ip"]),
"target": str(os2),
"value": "OS"
})
if str(cvelist) != "":
try:
nodes.append({
"id": str(id) + "" + str(cvelist),
"host": str(cvelist)
})
links.append({
"source": str(id) + "" + str(x["ip"]),
"target": str(id) + "" + str(cvelist),
"value": "cves"
})
except:
pass
#send data to elastic
#print(elkpush("box_"+sys.argv[2],toelastic))
id = id + 1
graph = {"nodes": nodes, "links": links}
data_grouped = ""
i = 0
for a in inventory_by_product:
print "\n" + a + "\n"
data_grouped = data_grouped + "\n\n[" + a + "]\n\n"
for b in inventory_by_product[a]:
print b
data_grouped = data_grouped + "\n" + inventory_by_product[a][
b] + "\n"
i = i + 1
f = open("data/inventory", "w")
f.write(str(data_grouped))
f.close()
with open("web/graphs/data.json", 'w') as outfile:
json.dump(graph, outfile)
def _load_extension(self):
if hasattr(self, "__extension"):
return
self.__guess()
return self.__guess_.extension if hasattr(self.__guess_, "mime") else "Unknown extension ." + os.split(self.file)[1]
import subprocess, os cwd = os.getcwd() #subprocess.call(['makeotf', '-f', 'Galada.ufo', '-o', 'Galada.otf', '-ff', 'features', '-gf', 'glyphOrder', '-mf', 'menuname', '-r'], cwd=cwd) print os.split(cwd)
def p_r(X_train, label_train, X_test, label_test, dist_func='cos', pic=True, prefix='', suffix='', compute_ma=True, title='', save_results=True, save_path='', show_error=False):
"""
X_train: 2d-ndarray, 被搜索的样本集矩阵，每行是一个样本向量
X_test: 2d-ndarray, query组成的矩阵，每行是一个query向量
label_train: 1-d ndarray, 被搜索样本集类标，非one-hot
label_test: 1-d ndarray, query集合的类标，非one-hot
计算精度和召回率, 要求类标是用向量形式，不能使用one-hot型
param prefix, suffix: str, 前缀/后缀标识字符串，主要用于在批量执行程序时保证保存的文件和图像不会互相覆盖
para save_results: bool, 是否保存中间计算矩阵，在批量执行程序时建议选择False，以避免产生过多文件
compute_ma: bool, 是否需要计算mean average precision
"""
if (prefix!='' or suffix!=''): #当同一个外部程序多次调用p_r函数时，为了方面区分结果对应的问题
print 'now computing mission: ' + prefix + suffix + '=================================='
#(1).预处理
if save_path != '':
#if platform.system() == 'Linux' and save_path[-1] != '/':
# save_path += '/'
#if platform.system() == 'Windows' and save_path[-1] != '\\':
# save_path += '\\'
if os.split(path)[-1] != '':
os.path.join(save_path, '')
check_path(save_path) #检测路径，如果目录不存在，支持递归建立
assert isinstance(label_train, numpy.ndarray), label_train.ndim == 1
assert isinstance(label_test, numpy.ndarray), label_test.ndim == 1
assert dist_func in ['cos', 'euc', 'adj_cos']
#X_train = X_train[:800, :] #测试程序用的数据子集==================
#X_test = X_test[:800, :] #测试程序用的数据子集=================
#label_test = label_test[:800] #==================
#label_train = label_train[:800] #======================
t0 = time.clock()
rows = X_test.shape[0]
cols = X_train.shape[0]
#下面使用了抽象的dict()函数，方便更换不同的度量
if dist_func is 'cos':
distance_matrix = cosine_dist(X_test=X_test, X_train=X_train)
if dist_func is 'euc':
distance_matrix = - euclidean_dist(X_test=X_test, X_train=X_train) #使用欧氏距离，距离越小认为相似度越高，所以前面加负号
if dist_func is 'adj_cos':
distance_matrix = adjusted_cosine(X_test=X_test, X_train=X_train)
assert distance_matrix.shape == (rows, cols)
t1 = time.clock()
#print '预处理时间:', t1 - t0
#sys.stdout.flush()
#(2).主计算过程
t2 = time.clock()
#由于numpy.sort()没有参数reverse，所以采用先由小到大排序在反转列序的办法
#返回距离由达到小的下标矩阵
argsort_matrix = numpy.argsort(distance_matrix, axis=1)[:, ::-1]
#求出距离排序后每一行的对应位置样本的类标组成的矩阵
label_matrix = numpy.ones(shape=(rows, cols)) * label_train
sorted_label_matrix = numpy.zeros(shape=label_matrix.shape)
#循环类标矩阵的每一行，按argsort_matrix的相应行作为下标进行调整
for i in xrange(label_matrix.shape[0]):
sorted_label_matrix[i] = label_matrix[i][argsort_matrix[i]]
#将测试集X_test的类标向量label_test作为列向量平行复制X_train.shape[0]列，得到一个矩阵
test_label_matrix = numpy.ones(shape=(rows, cols)) * label_test[:, numpy.newaxis]
#通过比较类标是否相同，得到一个布尔矩阵
bool_matrix = (test_label_matrix == sorted_label_matrix)
accum_correct_matrix = numpy.zeros(shape=(rows, cols), dtype='float64')
#TODO:下面的计算可能会比较慢。原因是每次切片是矩阵的一列而非一行，应该考虑先转置再按行切片赋值
accum_correct_matrix[:, 0] = bool_matrix[:, 0]
for i in xrange(1, bool_matrix.shape[1]):
accum_correct_matrix[:, i] = accum_correct_matrix[:, i-1] + bool_matrix[:, i]
t3 = time.clock()
#print '主机算过程用时:', t3 - t2
#sys.stdout.flush()
#(3).获得精度矩阵和召回率矩阵
t4 = time.clock()
precision_denominator = numpy.ones(shape=(rows, cols), dtype='float64') * numpy.arange(1, cols+1)
precision_matrix = accum_correct_matrix / precision_denominator #每个位置元素为（当前总正确数/当前总搜索数)
recall_denominator = accum_correct_matrix[:, -1] #向量，索引为i的分量值表示训练样本集中与测试样本i同类的样本的总数
recall_matrix = accum_correct_matrix / recall_denominator[:, numpy.newaxis]
#(4).求测试集所有query在训练集上的平均召回率和平均精度
mean_precision = numpy.mean(precision_matrix, axis=0)
mean_recall = numpy.mean(recall_matrix, axis=0)
t5 = time.clock()
#print '计算精度矩阵和召回率矩阵用时: ', t5 - t4
#sys.stdout.flush()
t6 = time.clock()
#(5).计算mean_average precision
if compute_ma is True:
mean_avg_precision = mean_average_precision(bool_matrix=bool_matrix, accum_correct_matrix=accum_correct_matrix,
prefix=prefix, suffix=suffix,save_path=save_path + prefix + 'mean_average_precision' + suffix)
assert isinstance(mean_avg_precision, numpy.ndarray), mean_avg_precision.ndim == 1
print '所有query的平均mean average precion: ', mean_avg_precision.mean()
#print '计算mean average precision用时: ', time.clock() - t6
sys.stdout.flush()
#(6).输出错误分析
#假设词干化词频文档已经生成，保存在某个目录下，当前任务只是指出在某个query下某个文档的排序，距离和自身类标
if show_error is True:
assert bool_matrix.dtype == 'bool' #检测
f = open('query_erro.txt', 'w')
f.write('query序号 文档排序序号 文档原始序号 文档与query相似度 文档类标 搜索结果是否正确 \n')
f.write('以上几个指标中，文档排序序号其实是随行号增加单调递增，意义不大\n')
for i in xrange(rows):
query_idx = i + 1 #1.query的序号
for j in xrange(cols):
order_idx = j + 1 #2.距离排序序号，序号越小距离query越近，相似度越大
doc_idx = argsort_matrix[i][j] + 1 #3.文档原始序号
qd_dist = distance_matrix[i][doc_idx - 1] #4.文档与query的距离
doc_label = sorted_label_matrix[i][j] #5.文档类标
tf_mark = bool_matrix[i][j] #6.文档是否与query属于同一类
f.write(' '.join((str(i) for i in [query_idx, order_idx, doc_idx, qd_dist, doc_label, tf_mark]))) #字符串合并少用加号
f.write('\n')
f.close()
print 'query error输出完成...'
sys.stdout.flush()
#(7).保存计算结果
if save_results:
numpy.save(save_path + prefix + 'precision_matrix' + suffix + '.npy', precision_matrix)
numpy.save(save_path + prefix + 'recall_matrix' + suffix + '.npy', recall_matrix)
numpy.save(save_path + prefix + 'mean_precision' + suffix + '.npy', mean_precision)
numpy.save(save_path + prefix + 'mean_recall' + suffix + '.npy', mean_recall)
#print 'model saved...'
#sys.stdout.flush()
f = open(save_path +prefix+'mean_precision_record'+suffix, 'w')
for num, i in enumerate(mean_precision):
f.write(str(num) + ': ' + str(i) + '\n')
f.close()
f = open(save_path + prefix+'mean_recall_record'+suffix, 'w')
for num, i in enumerate(mean_recall):
f.write(str(num) + ': ' + str(i) + '\n')
f.close()
#(8).根据计算结果绘制p-r曲线
if pic is True:
#print 'drawing curve...'
pr_curve(title=title, mean_precision=mean_precision, mean_recall=mean_recall, prefix=prefix, suffix=suffix, save_path=save_path) #绘制召回率-精度曲线
def getOsImage():
# Return OS Specifications
os = open('/etc/os-release').read()
imageName = os.split('\n')
return imageName[0].replace('PRETTY_NAME=', '').replace('"', '')
if tmp:
file = 0
else:
tmp = re.search(r'Linux-Servers-(ITSM).csv',onlyfile,re.IGNORECASE)
if tmp:
file = 1
if file == 0:
for line in open(onlyfile,'r').readlines():
if count > 1:
line = line.strip()
line = re.sub(r'\"',r'',line)
parts = line.split(',')
name = parts[0].upper()
os = parts[2]
os = os.split(':')
os = os[1]
ip = parts[3]
state = parts[1]
tools = parts[4]
status = parts[5]
nic = parts[9]
appliance = parts[8]
if not vCenter.has_key(name):
vCenter[name] = os
vCenterIP[name] = ip
vCenterState[name] = state
vCenterToolsV[name] = tools
vCenterVMStatus[name] = status
vCenterNIC[name] = nic
vCenterAppliance[name] = appliance
import os
import shutil
import subprocess
import tempfile
from seqtools.general import load_csv
GENE_FILE = os.path.join(
os.split(__file__)[0],
'../data/gene.txt',
)
WORK_DIR = tempfile.mkdtemp()
QSUB_HEADERS = '#!/bin/bash\n' \
'#$ -N "trinity"\n' \
'#$ -e errors.log\n' \
'#$ -cwd\n\n' \
'module load trinity/2.1.1\n' \
'module load samtools/1.3\n\n'
TRINITY_CMD = 'Trinity ' \
'--genome_guided_bam {0} ' \
'--genome_guided_max_intron 100000 ' \
'--CPU 8 --max_memory 20G ' \
'--output {1}\n'
def run_trinity_gene_qsub(gene_name, bam_file, output_dir,
pad=1000, # expand gene coordinates by 1kb
gene_file=GENE_FILE, work_dir=WORK_DIR,
#! /usr/bin/env dreamthon
import os
import time
trash = (*files, trash_dir:'~/.Trash') ->
files.each (f) ->
to = os.join trash_dir, os.split(f)[-1] + '.' + time.now.iso
print "{f} -> {to}"
os.mv f, to
(= trash (-> (*files trash_dir:'~/.Trash')
(files .each (-> (f)
(= to (os .join trash_dir (+ ((os .split f) -1) '.' (time .now .iso))))
(print "{f} -> {to}")
(os .mv f to)))))
main = -> trash *os.argv[1:]
