def update_eq_options(wdir, proj, label, imc, eqid):
if any([val is None for val in locals().values()]):
return [], None, [], None, [], []
df_eq_imc = get_eq_imc_df(wdir, proj, label, eqid, imc)
imt_options = get_options(df_eq_imc, IMT_REGEX)
imt_options.append({'label': 'Pass/Fail', 'value': 'Pass/Fail'})
imt = imt_options[0]['value']
dist_options = get_options(df_eq_imc, DIST_REGEX)
dist = dist_options[0]['value']
model_options = get_model_options(MODELS_DICT, imc, imt)
rep = get_event_object(eqid).__repr__()
return imt_options, imt, dist_options, dist, model_options, rep
def main():
"""
Инициализация логгера, бота и хранилища. Обработка почтовых сообщений и сообщений от telegram.
"""
global bot, db
logger = logging.getLogger('tm_monitor')
logger.setLevel(logging.INFO)
formatter = logging.Formatter('[%(asctime)s] %(levelname)-8s %(filename)s[LINE:%(lineno)d]# %(message)s')
handler = DiffFileHandler()
handler.setLevel(logging.INFO)
handler.setFormatter(formatter)
logger.addHandler(handler)
utils.log = logger
token = get_options('main', 'token')
if not token:
log.critical('В файле настроек токен бота не найден')
return
bot = TeleBot(token)
db = shelve.open('data')
telegram_processing()
mail_processing()
db.close()
def get_at_out_list():
"""
Импортируем список исходящих номеров из БД АТС
:return: {string: {string}}, {гор_номер: {вн_номер, ...}}
"""
raw = defaultdict(set)
options_list = get_options('asterisk', 'db')
if options_list:
host, user, password, db = options_list
try:
with pymysql.connect(host, user, password, db) as cur:
cur.execute(
"SELECT extension, outboundcid FROM users "
"WHERE LENGTH(outboundcid) = 11 or LENGTH(outboundcid) = 7"
)
for ext, out in cur:
raw[get_cm(out)].add(ext)
except OperationalError as e:
log.error(e)
return raw
def get_ad_list():
""""
Импортируем список сотрудников из AD
Фильтр !(userAccountControl:1.2.840.113556.1.4.803:=2) исключает заблокированные учётные записи
:return: [{}], cписок словарей с аттрибутами пользователей
"""
raw = []
options_list = get_options('ad')
ad_search = get_options('main', 'ad_search', True)
if options_list:
host, user, password = options_list
server = Server(host, get_info=ALL)
try:
with Connection(server,
user,
password,
authentication=NTLM,
auto_bind=True) as conn:
filter_str = '(&(objectclass=person)(!(userAccountControl:1.2.840.113556.1.4.803:=2)))'
conn.search(
ad_search,
filter_str,
attributes=[
'displayName', # ФИО
'telephoneNumber', # Внутренние номера
'mail', # Эл. почта
'title', # Должность
'department', # Отдел
'company', # Организация
'accountExpires' # Дата блокировки учётной записи
])
raw = conn.entries
except LDAPSocketOpenError as e:
log.error(e)
except LDAPBindError:
log.error('Ошибка доменной авторизации')
return raw
def main():
# Hyper Parameters
opt = get_options()
log_file = '{}-{}-{}.log'.format(opt.model_name, opt.dataset,
strftime("%y%m%d-%H%M", localtime()))
logger.addHandler(logging.FileHandler(log_file))
ins = Instructor(opt)
ins.run()
def main():
global g_settings
options = utils.get_options()
g_settings = Settings({})
g_settings.tidb_host = options.tidb_host
g_settings.tidb_port = options.tidb_port
g_settings.tidb_user = options.tidb_user
g_settings.tidb_pass = options.tidb_pass
g_settings.tidb_db = options.tidb_db
g_settings.workers = options.workers
app.run(debug=True, port=options.server_port, host='0.0.0.0')
def main():
global g_settings
options = utils.get_options()
g_settings = Settings({})
g_settings.memsql_host = options.memsql_host
g_settings.memsql_port = options.memsql_port
g_settings.memsql_user = options.memsql_user
g_settings.memsql_pass = options.memsql_pass
g_settings.memsql_db = options.memsql_db
g_settings.workers = options.workers
app.run(debug=True, port=options.server_port, host='0.0.0.0')
def main():
global g_settings
options = utils.get_options()
g_settings = Settings({})
g_settings.memsql_host = options.memsql_host
g_settings.memsql_port = options.memsql_port
g_settings.memsql_user = options.memsql_user
g_settings.memsql_pass = options.memsql_pass
g_settings.memsql_db = options.memsql_db
g_settings.workers = options.workers
logger.error("Listening on Port: %s" % (options.server_port,))
app.run(debug=False, port=options.server_port, host='0.0.0.0')
def clean_tag(self):
data = self.cleaned_data['tag']
action = self.cleaned_data['action']
# First, work out whether the tag currently exists
options = get_options()
tag_exists = data in options['projects']
# Creating a new tag
if action == 'create' and tag_exists:
raise forms.ValidationError('This project already exists',
code='invalid')
# Deleting an existing tag
if action == 'delete' and not tag_exists:
raise forms.ValidationError("This project doesn't exist",
code='invalid')
return data
def export_xls_brief(raw):
"""
Выгрузка краткой (без внутренних номеров) статистики звонков
:param raw:
:return:
"""
wb = xlwt.Workbook()
ws = wb.add_sheet('Краткий список')
# Заголовок
ws.write(0, 0, 'Гор. номер')
ws.write_merge(0, 0, 1, 4, 'Вх.')
ws.write_merge(0, 0, 5, 8, 'Исх.')
path = get_options('main', 'xls_path_brief', True)
if not path:
log.critical('Ошибка чтения конфигурационного файла, см. ошибки выше')
return
line = 1
for kc in sorted(raw):
ws.write(line, 0, kc)
ws.write(line, 1, format_time(raw[kc]['inc']['duration']))
ws.write(line, 2, raw[kc]['inc']['count'])
ws.write(line, 3, format_time(raw[kc]['inc']['billsec']))
ws.write(line, 4, raw[kc]['inc']['answer'])
ws.write(line, 5, format_time(raw[kc]['out']['duration']))
ws.write(line, 6, raw[kc]['out']['count'])
ws.write(line, 7, format_time(raw[kc]['out']['billsec']))
ws.write(line, 8, raw[kc]['out']['answer'])
line += 1
try:
wb.save(path)
except PermissionError as e:
log.error('Недостаточно прав для сохранения файла: %s' % e.filename)
return
except FileNotFoundError as e:
log.error('Неверный путь или имя файла: %s' % e.filename)
return
return True
def edit_site(args): site = mysql.check_site_id() msg = ''' 1. Add domain alias 2. Remove domain alias 3. Enable this site 4. Disable this site Please select option:''' opt = utils.get_options(msg, [1, 2, 3, 4]) if opt == 1: add_alias(site, args) elif opt == 2: remove_alias(site, args) elif opt == 3: enable_site(site.domain) elif opt == 4: disable_site(site.domain)
def export_xls(raw):
"""
Выгрузка структуры телефонной книги в excel
:param raw: {string: {string: [[string]]}}, структура тел. книги {организация: {отдел: [[данные_сотрудника]]}}
:return: bool, True - если тел. книга выгружена, False - если произошла ошибка выгрузки
"""
wb = xlwt.Workbook()
ws = wb.add_sheet('Список номеров')
# Заголовок
ws.write(0, 1, 'Гор. номер')
ws.write(0, 2, 'Вх.')
ws.write(0, 3, 'Исх.')
path = get_options('main', 'xls_path', True)
if not path:
log.critical('Ошибка чтения конфигурационного файла, см. ошибки выше')
return
line = 1
for kc in sorted(raw):
ws.write(line, 1, kc)
ws.write(line, 2, ', '.join(raw[kc]['inc']))
ws.write(line, 3, ', '.join(raw[kc]['out']))
line += 1
try:
wb.save(path)
except PermissionError as e:
log.error('Недостаточно прав для сохранения файла: %s' % e.filename)
return
except FileNotFoundError as e:
log.error('Неверный путь или имя файла: %s' % e.filename)
return
return True
def __init__(self, **kwargs):
self.form = forms.Form(**kwargs)
#load option values from the database
self.value_dict = get_options(optionset_label=self.optionset_label,
current_only=False)
self.formfields = []
self.langformfields = {}
for (attr, value) in self.attrs:
if not attr in self.value_dict:
self.value_dict[attr] = _init_option(self.optionset_label,
attr, value)
if value.lang_dependant:
for lang in get_supported_languages():
#generate the form field
field_name = '%s_%s' % (attr, lang)
lang_field = copy.deepcopy(value.field)
lang_field.label = '%s (%s)' % (lang_field.label,
lang.upper())
self.form.fields[field_name] = lang_field
try:
self.form.fields[field_name].initial = self.value_dict[
attr][lang]
except KeyError:
self.form.fields[field_name].initial = ''
#add to land dependant options
self.lang_options[field_name] = (attr, lang)
#langformfields fieldset
if not lang in self.langformfields:
self.langformfields[lang] = []
self.langformfields[lang].append(self.form[field_name])
else:
self.form.fields[attr] = value.field
self.form.fields[attr].initial = self.value_dict[attr]
self.formfields.append(self.form[attr])
def __init__(self, **kwargs):
self.form = forms.Form(**kwargs)
#load option values from the database
self.value_dict = get_options(optionset_label = self.optionset_label, current_only=False)
self.formfields = []
self.langformfields = {}
for (attr, value) in self.option_fields:
if not attr in self.value_dict or self.value_dict[attr] is None:
self.value_dict[attr] = _init_option(self.optionset_label, attr, value)
if value.lang_dependant:
for lang in get_supported_languages():
#generate the form field
field_name = '%s_%s' % (attr, lang)
lang_field = copy.deepcopy(value.field)
lang_field.label = '%s (%s)' % (force_text(lang_field.label), lang.upper())
self.form.fields[field_name] = lang_field
try:
self.form.fields[field_name].initial = self.value_dict[attr][lang]
except KeyError:
self.form.fields[field_name].initial = ''
#add to land dependant options
self.lang_options[field_name] = (attr, lang)
#langformfields fieldset
if not lang in self.langformfields:
self.langformfields[lang] = []
self.langformfields[lang].append(self.form[field_name])
else:
self.form.fields[attr] = value.field
self.form.fields[attr].initial = self.value_dict[attr]
self.formfields.append(self.form[attr])
def update_ael(sections):
"""
Загружаем и обновляем конф. файлы из АТС, в случае ошибки обновления, будут использоваться файлы загруженные ранее
:param sections: [string], список секций из конф. файла с параметрами подключения к АТС
"""
for section in sections:
options_list = get_options(section)
if options_list:
host, user, password = options_list
try:
with paramiko.SSHClient() as ssh:
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
ssh.connect(hostname=host,
username=user,
password=password)
remotepath = '/etc/asterisk/extensions.ael'
localpath = 'asterisk/%s/extensions.ael' % section
if not os.path.exists(os.path.dirname(localpath)):
os.makedirs(os.path.dirname(localpath))
try:
with SCPClient(ssh.get_transport()) as client:
client.get(remotepath, localpath)
except SCPException as e:
log.error(e)
except TimeoutError as e:
log.error('%s - %s' % (host, e.strerror))
except paramiko.ssh_exception.AuthenticationException:
log.error('Ошибка авторизации на %s' % host)
return True
raw_desc = df.describe(include='all').T
# raw_desc.rename(columns={'': 'col_name'}, inplace=True)
null_percent = 100 * df.isnull().sum() / df.shape[0]
null_percent = null_percent.to_frame(name='null_percent')
# types = df.dtypes.to_frame(name='types')
nunique = df.apply(pd.Series.nunique).to_frame(name='n_unique')
top = top_values(df).to_frame(name='top_10')
cat_cols = find_categorical(df)
col_type = pd.Series(raw_desc.index, index=raw_desc.index,
name='col_type').isin(cat_cols)
col_type = col_type.apply(lambda x: 'cat' if x else 'num')
desc = pd.concat([raw_desc, null_percent, col_type, nunique, top],
axis=1, copy=False)
kept_columns = ['col_type', 'null_percent', 'n_unique', 'mean',
'min', 'max', 'top_10']
desc[kept_columns].to_csv(desc_file, sep=';', float_format='%.2f')
if __name__ == '__main__':
options = get_options()
for f in options.filenames:
make_audit_file(f)
#!/usr/bin/env python
""" Setup dataporten openid mapping """
import utils
from himlarcli.keystone import Keystone
from himlarcli import utils as himutils
options = utils.get_options('Setup dataporten openid mapping',
hosts=0,
dry_run=True)
ksclient = Keystone(options.config, debug=options.debug)
# Domain should be create from hieradata
domain = ksclient.get_domain_id('dataporten')
rules = [{
"local": [{
"user": {
"name": "{0}",
"id": "{0}"
},
"group": {
"name": "{0}-group",
"domain": {
"id": domain
}
}
}],
"remote": [{
"type": "OIDC-email"
}, {
"type": "OIDC-name"
if unordered:
print('- glossary')
print(' - out of order')
for item in unordered:
print(f' - {item}')
def get_internal(glossary, language):
'''Create set of internal references within glossary definitions.'''
result = set()
for entry in glossary:
for match in CROSS_REF.finditer(entry[language]['def']):
result.add(match.group(1).lstrip('#'))
if 'ref' in entry:
result.update(entry['ref'])
return result
def get_definitions(glossary):
'''Create set of keys in glossary.'''
return {entry['key'] for entry in glossary}
if __name__ == '__main__':
options = utils.get_options(
['--glossary', False, 'Path to glossary YAML file'],
['--language', False, 'Two-letter code for language'],
['--sources', True, 'List of input files']
)
check_gloss(options)
#!/usr/bin/env python
'''Check for misspelled words.'''
import sys
import utils
def check_spelling(options):
'''Main driver.'''
actual = {word.strip() for word in sys.stdin.readlines()}
expected = {
word.strip()
for word in open(options.compare, 'r').readlines()
}
utils.report('spelling', actual=actual, expected=expected)
if __name__ == '__main__':
options = utils.get_options(
['--compare', False, 'File to compare wordlist'])
check_spelling(options)
raw = yaml.dump(cleaned, sort_keys=False, width=utils.YAML_INFINITE)
cooked = utils.cook_yaml(raw)
print(cooked)
def merge_inputs(options):
'''Read all files, merging inputs.'''
result = {}
for fn in options.sources:
temp = utils.read_yaml(fn)
for entry in temp:
assert 'key' in entry, f'Entry {entry} from {fn} lacks key'
if options.verbose and (entry['key'] in result):
print(f'duplicate key {entry["key"]} in {fn}', file=sys.stderr)
result[entry['key']] = entry
return result
def cleanup(entry):
'''Create new dict for entry with keys in desired order.'''
return {
key: utils.strip_nested(entry[key])
for key in KEY_ORDER if key in entry
}
if __name__ == '__main__':
options = utils.get_options(['--sources', True, 'List of input files'],
['--verbose', None, 'Report duplicate keys?'])
merge_bib_md(options)
outImage = aruco.drawDetectedCornersCharuco(outImage, charucoCorners,
charucoIds)
# use a camera parameter
retval, rvec, tvec = aruco.estimatePoseCharucoBoard(
charucoCorners, charucoIds, board, cameraMatrix, distCoeffs)
if retval:
aruco.drawAxis(outImage, cameraMatrix, distCoeffs, rvec, tvec, 0.1)
cv2.imshow("detected", outImage)
cv2.waitKey(0)
cv2.destroyAllWindows()
if __name__ == '__main__':
args = utils.get_options()
input_image_dirpath = osp.join(osp.dirname(__file__), args.in_dir)
# recognize any extentions
image_paths, image_names = utils.get_file_paths(input_image_dirpath, "*")
# read camera parameters
camera_param_filepath = osp.join(osp.dirname(__file__),
args.camera_param_path)
cameraMatrix, distCoeffs, rvecs, tvecs, stdDevIn, stdDevEx = \
utils.read_pickle(camera_param_filepath)
# read parameters from arguments
dictionary = utils.get_aruco_dict(args.aruco_dict)
squareL = args.square_length
markerL = args.marker_length
def get_problems(filenames):
'''Look for problems.'''
found = False
result = {}
for filename in options.sources:
result[filename] = []
with open(filename, 'r') as reader:
for (i, line) in enumerate(reader):
if len(line.rstrip()) > utils.WIDTH:
found = True
result[filename].append(i + 1)
return found, result
def report(options, problems):
'''Report any problems.'''
print('- long lines')
for filename in sorted(problems.keys()):
if problems[filename]:
if options.verbose:
lines = ', '.join([str(i) for i in problems[filename]])
print(f' - {filename}: {lines}')
else:
print(f' - {filename}: {len(problems[filename])}')
if __name__ == '__main__':
options = utils.get_options(['--sources', True, 'List of input files'],
['--verbose', None, 'Report line by line'])
check_long_lines(options)
""" Update glance images """
import sys
import os
import urllib
import urllib2
import pprint
from datetime import datetime
import utils
from himlarcli import utils as himutils
from himlarcli.glance import Glance
print "Depricated! Use image.py"
sys.exit(0)
options = utils.get_options('Create and update golden images',
hosts=0, dry_run=True)
glclient = Glance(options.config, debug=options.debug)
logger = glclient.get_logger()
golden_images = himutils.load_config('config/golden_images.yaml')
if glclient.region in golden_images:
images = golden_images[glclient.region]
else:
if not 'default' in golden_images:
print "Missing default in config/golden_images.yaml"
sys.exit(1)
images = golden_images['default']
def download_and_check(image):
source = himutils.get_abs_path('%s' % image['latest'])
url = '%s%s' % (image['url'], image['latest'])
# Do not redownload
#!/usr/bin/env python
import utils
from himlarcli.keystone import Keystone
from himlarcli.foremanclient import ForemanClient
from himlarcli import utils as himutils
# Fix foreman functions and logger not-callable
# pylint: disable=E1101,E1102
desc = 'Setup Foreman for himlar'
options = utils.get_options(desc, hosts=False)
keystone = Keystone(options.config, debug=options.debug)
logger = keystone.get_logger()
domain = keystone.get_config('openstack', 'domain')
foreman = ForemanClient(options.config, options.debug, log=logger)
client = foreman.get_client()
# create foreman domain based on config
# get domain id
# get smart proxy id for tftp
# create subnet
# mgmt network + netmask from config
# domain_ids = domain_id
# tftp_ids = proxy_id
# dns-primary, dns-secondary, gateway is blank
# get subnet id
# Glabal parameters
#!/usr/bin/env python
import utils
import sys
from himlarcli.foremanclient import ForemanClient
from himlarcli.sensu import Sensu
desc = 'Toggle rebuild host in foreman'
options = utils.get_options(desc)
foreman = ForemanClient(options.config, options.debug)
sensu = Sensu(options.config, debug=options.debug)
for host in options.host:
sensu.silence_host(host, expire=3600)
foreman.set_host_build(host)
'''Wrap lines.'''
result = []
for line in lines:
line = line.replace(PROTOCOL, '').replace(HERE, FAKE)
if len(line) == 0:
result.append(line)
continue
match = INDENT.match(line)
indent = match.group(1) if match else ''
while len(line) > 0:
front, line, terminator = split(line)
result.append(f'{front}{terminator}')
if len(line) > 0:
line = indent + line.lstrip()
return result
def split(line):
'''Split a line.'''
if len(line) <= utils.WIDTH:
return line, '', ''
for i in range(utils.WIDTH, 0, -1):
if (line[i] == ' '):
return line[:i], line[i:], ' \\'
return line[:utils.WIDTH], line[utils.WIDTH:], ' \\'
if __name__ == '__main__':
options = utils.get_options(['--slice', None, 'Take slice out of input?'])
reformat(options)
#!/usr/bin/env python
import utils
import sys
import pprint
import yaml
import ConfigParser
from himlarcli.foremanclient import Client
from himlarcli import utils as himutils
desc = 'Create an location inventory config in ./config.<loc>'
options = utils.get_options(desc, hosts=False)
foreman = Client(options.config, options.debug)
hosts = foreman.get_hosts('*')
hostlist = dict({'nodes': {}})
for host in hosts['results']:
hostname = str(host['name']).split('.')[0]
loc, role, id = hostname.split('-')
node = "%s-%s" % (role, id)
host['name'].split('.')[0]
if not host['managed']:
continue
if not host['compute_resource_name']:
hostlist['nodes'][node] = dict()
hostlist['nodes'][node]['mac'] = str(host['mac'])
else:
hostlist['nodes'][node] = dict()
hostlist['nodes'][node]['compute_resource'] = 'controller-01'
hostlist['nodes'][node]['compute_profile'] = 1
#pp = pprint.PrettyPrinter(indent=2)
'systemctl',
'disable',
AUTO_START_SERVICE,
allowed_return_codes=[0, 1])
utils.ssh(himn, username, 'rm', '-f',
'/etc/systemd/system/%s' % AUTO_START_SERVICE)
utils.ssh(himn, username, 'rm', '-f',
'/etc/sysconfig/network-scripts/%s' % AUTO_SCRIPT)
utils.ssh(himn, username, 'systemctl', 'daemon-reload')
if __name__ == '__main__':
install_xenapi_sdk()
astute = utils.get_astute()
if astute:
username, password, install_xapi = utils.get_options(astute)
endpoints = get_endpoints(astute)
himn_eth, himn_local = utils.init_eth()
public_ip = utils.astute_get(
astute, ('network_metadata', 'vips', 'public', 'ipaddr'))
services_ssl = utils.astute_get(astute, ('public_ssl', 'services'))
if username and password and endpoints and himn_local:
route_to_compute(endpoints, HIMN_IP, himn_local, username)
xcp_version = utils.get_xcp_version(HIMN_IP, username)
if install_xapi:
install_suppack(HIMN_IP, username, XS_PLUGIN_ISO, xcp_version)
enable_linux_bridge(HIMN_IP, username)
forward_from_himn(himn_eth)
def main():
"""Entry point of the program."""
# Get all command line arguments
cmd_options = vars(get_options())
# Set if input filename is a HTTP url, defaults to false
is_url = False
# Determine the extension of the input file
if cmd_options["in_filename"].startswith("http"):
is_url = True
else:
input_file_extension = convert.detect_file_extension(cmd_options["in_filename"])
# Initialize the empty dataframe
df = None
# Check if rows need to be skipped from start
skip_rows = int(cmd_options["skip_rows"])
# Check if the user wants to skip header
skip_header = cmd_options["skip_header"]
# Check if the user has passed a list of columns names explicitly
column_names = cmd_options["columns"]
if column_names is not None:
column_names = column_names.split(",")
if is_url:
df = convert.url_to_df(cmd_options["in_filename"], skip_rows, skip_header, column_names)
else:
# Based on extension, convert the file contents to a Pandas dataframe
if input_file_extension == "xls":
df = convert.xls_to_df(cmd_options["in_filename"], skip_rows, skip_header, column_names)
elif input_file_extension == "xlsx":
df = convert.xlsx_to_df(cmd_options["in_filename"], skip_rows, skip_header, column_names)
elif input_file_extension == "csv":
df = convert.csv_to_df(cmd_options["in_filename"], skip_rows, skip_header, column_names)
else:
raise BarryFileException("Input file format not supported. Currently supported file formats - xls/xlsx/csv")
# Handle dataframe transformations
# Check if a column need to be sorted
if cmd_options["sort_column"]:
sort_column = cmd_options["sort_column"]
ascending = True
convert.sort_df(df, sort_column, ascending)
elif cmd_options["rsort_column"]:
sort_column = cmd_options["rsort_column"]
ascending = False
convert.sort_df(df, sort_column, ascending)
# Based on the output format, write the dataframe to disk
if cmd_options["out_format"] == "xls":
convert.df_to_xls(df, cmd_options["out_filename"])
elif cmd_options["out_format"] == "xlsx":
convert.df_to_xlsx(df, cmd_options["out_filename"])
elif cmd_options["out_format"] == "json":
convert.df_to_json(df, cmd_options["out_filename"])
else:
convert.df_to_csv(df, cmd_options["out_filename"])
#!/usr/bin/python
import sys
import utils
from himlarcli.nova import Nova
from himlarcli.keystone import Keystone
from himlarcli.mail import Mail
from email.mime.text import MIMEText
# OPS! It might need some updates. We use class Mail instead of Notify now.
options = utils.get_options('Notify all users', dry_run=True, hosts=False)
keystone = Keystone(options.config, debug=options.debug)
mail = Mail(options.config, debug=options.debug)
region = keystone.region
print "Remove these lines if you want to run this and send mail to all!"
sys.exit(0)
# Edit this to send new email to all users
subject = 'UH-IaaS: Purge of all data (%s)' % region
body_file = 'notify/notify_reinstall.txt'
with open(body_file, 'r') as body_txt:
body_content = body_txt.read()
projects = keystone.list_projects('Dataporten')
for project in projects:
msg = MIMEText(body_content)
msg['Subject'] = subject
if not options.dry_run:
mail.send_mail(project, msg)
elif len(names) > 2:
front = ', '.join(names[0:-1])
names = f'{front}, and {names[-1]}'
return f'{names}{suffix}:'
def key(entry):
'''Generate bibliography key.'''
assert 'key' in entry, \
'Every entry must have key'
return f'<dt id="{entry["key"].lower()}" class="bibliography">{entry["key"]}</dt>'
def title(entry, quote):
'''Generate title (possibly linking and/or quoting).'''
assert 'title' in entry, \
f'Entry {entry} does not have title'
title = f'<a href="{entry["url"]}">{entry["title"]}</a>' \
if ('url' in entry) else entry["title"]
title = f'"{title}"' if quote else f'<em>{title}</em>'
edition = f' ({entry["edition"]} edition)' \
if ('edition' in entry) else ''
return f'{title}{edition}.'
if __name__ == '__main__':
options = utils.get_options(
['--input', False, 'Path to input YAML bibliography file'],
['--output', False, 'Path to output Markdown file'])
make_bib(options)
def make_numbering(options):
'''Main driver.'''
config = utils.read_yaml(options.config)
entries = utils.get_entry_info(config)
figures = {}
tables = {}
for entry in entries:
text = utils.read_file(entry['file'], scrub=False)
figures.update(get_inclusions(FIG_INC, entry, text))
tables.update(get_inclusions(TBL_INC, entry, text))
result = {'entries': entries, 'figures': figures, 'tables': tables}
utils.write_yaml(options.output, result)
def get_inclusions(pattern, entry, text):
'''Get all inclusions from file.'''
label = entry['label']
result = {}
for (i, match) in enumerate(pattern.finditer(text)):
key = match.group(1)
result[key] = f'{label}.{i+1}'
return result
if __name__ == '__main__':
options = utils.get_options(
['--config', False, 'Path to YAML configuration file'],
['--output', False, 'Path to output YAML file'])
make_numbering(options)
#!/usr/bin/python
import sys
import utils
from himlarcli.nova import Nova
from himlarcli.keystone import Keystone
from himlarcli.notify import Notify
from email.mime.text import MIMEText
options = utils.get_options('Notify all users', dry_run=True, hosts=False)
keystone = Keystone(options.config, debug=options.debug)
notify = Notify(options.config, debug=options.debug)
region = keystone.region
print "Remove these lines if you want to run this and send mail to all!"
sys.exit(0)
# Edit this to send new email to all users
subject = 'UH-IaaS: Purge of all data (%s)' % region
body_file = 'misc/notify_reinstall.txt'
with open(body_file, 'r') as body_txt:
body_content = body_txt.read()
projects = keystone.list_projects('Dataporten')
for project in projects:
msg = MIMEText(body_content)
msg['Subject'] = subject
if not options.dry_run:
notify.send_mail(project, msg)
print '\nProject: %s' % project
import torch
from pytorch_transformers import BertModel
from sklearn import metrics
from torch.utils.data import DataLoader
from data_utils import Tokenizer4Bert, ABSADataset
from models.aen import AEN_BERT
from utils import get_options
opt = get_options()
bert = BertModel.from_pretrained(opt.pretrained_bert_name)
model_path = 'state_dict/aen_bert_laptop_val_acc0.7821'
model = AEN_BERT(bert, opt)
model.load_state_dict(torch.load(model_path, map_location=torch.device('cpu')))
model.eval()
tokenizer = Tokenizer4Bert(opt.max_seq_len, opt.pretrained_bert_name)
test_set = ABSADataset(opt.dataset_file['test'], tokenizer)
data_loader = DataLoader(dataset=test_set, batch_size=1, shuffle=False)
n_correct, n_total = 0, 0
t_targets_all, t_outputs_all = None, None
with torch.no_grad():
for t_batch, t_sample_batched in enumerate(data_loader):
t_inputs = [
t_sample_batched[col].to(opt.device) for col in opt.inputs_cols
]
print("input: ", t_inputs)
t_targets = t_sample_batched['polarity'].to(opt.device)
length = len(node.code.string.split('\n'))
except:
assert False, f'pre node {node} has no text'
if length > utils.LENGTH:
entry = {'length': length, 'line': node.sourceline}
if 'class' in node:
entry['class'] = node['class']
if 'title' in node:
entry['title'] = node['title']
result.append(entry)
return result
def report(options, problems):
'''Report any problems.'''
print('- long chunks')
for filename in sorted(problems.keys()):
print(f' - {filename}')
for p in problems[filename]:
print(f' - line: {p["line"]}')
print(f' length: {p["length"]}')
if 'class' in p:
print(f' class: {p["class"]}')
if 'title' in p:
print(f' title: {p["title"]}')
if __name__ == '__main__':
options = utils.get_options(['--sources', True, 'List of input files'])
check_chunk_length(options)
def export_xls_full(raw):
"""
Выгрузка полной (с внутренними номерами) статистики звонков
:param raw:
:return:
"""
wb = xlwt.Workbook()
ws = wb.add_sheet('Подробный список')
# Заголовок
ws.write(0, 0, 'Гор. номер')
ws.write_merge(0, 0, 1, 5, 'Вх.')
ws.write_merge(0, 0, 6, 10, 'Исх.')
path = get_options('main', 'xls_path_full', True)
if not path:
log.critical('Ошибка чтения конфигурационного файла, см. ошибки выше')
return
line = 1
for kc in sorted(raw):
ws.write(line, 0, kc)
ki = raw[kc]['inc']
ko = raw[kc]['out']
kiu = ki['users']
kou = ko['users']
ws.write(line, 2, format_time(ki['duration']))
ws.write(line, 3, ki['count'])
ws.write(line, 4, format_time(ki['billsec']))
ws.write(line, 5, ki['answer'])
ws.write(line, 7, format_time(ko['duration']))
ws.write(line, 8, ko['count'])
ws.write(line, 9, format_time(ko['billsec']))
ws.write(line, 10, ko['answer'])
inc_line = 0
for inc in sorted(kiu):
inc_line += 1
ws.write(line + inc_line, 1, inc)
ws.write(line + inc_line, 4, format_time(kiu[inc]['billsec']))
ws.write(line + inc_line, 5, kiu[inc]['answer'])
out_line = 0
for out in sorted(kou):
out_line += 1
ws.write(line + out_line, 6, out)
ws.write(line + out_line, 7, format_time(kou[out]['duration']))
ws.write(line + out_line, 8, kou[out]['count'])
ws.write(line + out_line, 9, format_time(kou[out]['billsec']))
ws.write(line + out_line, 10, kou[out]['answer'])
line += max([inc_line, out_line]) + 1
try:
wb.save(path)
except PermissionError as e:
log.error('Недостаточно прав для сохранения файла: %s' % e.filename)
return
except FileNotFoundError as e:
log.error('Неверный путь или имя файла: %s' % e.filename)
return
return True
utils.report('links', referenced=referenced, defined=defined)
check_duplicates(options)
def get_keys(config):
'''Create set of chapter slugs found in configuration.'''
links = yaml.safe_load(config['kramdown']['link_defs'])
return set(links.keys())
def check_duplicates(options):
all_duplicates = {}
for filename in options.sources:
duplicates = []
matches = utils.get_matches(LINK_REF, filename, duplicates=duplicates)
if duplicates:
all_duplicates[filename] = duplicates
if all_duplicates:
print('- duplicate links')
for filename in sorted(all_duplicates.keys()):
print(f' - {filename}')
for key in sorted(all_duplicates[filename]):
print(f' - {key}')
if __name__ == '__main__':
options = utils.get_options(
['--config', False, 'Path to YAML configuration file'],
['--sources', True, 'List of input files'])
check_links(options)
#!/usr/bin/env python
""" Setup dataporten openid mapping """
import utils
from himlarcli.keystone import Keystone
from himlarcli import utils as himutils
options = utils.get_options('Setup dataporten openid mapping',
hosts=0, dry_run=True)
ksclient = Keystone(options.config, debug=options.debug)
ksclient.set_domain('dataporten')
# Domain should be create from hieradata
domain = ksclient.get_domain_id()
rules = [{
"local": [{
"user": { "name": "{0}", "id": "{0}" },
"group": { "name": "{0}-group", "domain": { "id": domain } } }],
"remote": [{ "type": "OIDC-email" }, { "type": "OIDC-name" }]
}, {
"local": [{
"group": { "name": "nologin", "domain": { "id": domain } } }],
"remote": [{ "type": "OIDC-email" }, { "type": "OIDC-name" }]
}]
# Crate nologin group
desc = 'All authenticated users are mapped to nologin which has no role grants'
ksclient.create_group('nologin', desc, 'dataporten')
# Create provider, mapping and container to connect them
ksclient.set_mapping('dataporten_personal', rules)
ksclient.set_protocol('openid', 'dataporten', 'dataporten_personal')
for info in utils.get_entry_info(config):
find_terms(info['slug'], info['file'], terms)
report(terms)
def find_terms(slug, filename, terms):
'''Collect index terms from file.'''
with open(filename, 'r') as reader:
text = reader.read()
for match in utils.INDEX_REF.finditer(text):
entries = [e.strip() for e in utils.WHITESPACE.sub(' ', match.group(1)).split(';')]
for entry in entries:
if entry not in terms:
terms[entry] = []
terms[entry].append(slug)
def report(terms):
'''Show where terms are used.'''
print('- index')
for key in sorted(terms):
sites = ', '.join(terms[key])
print(f' - {key}: {sites}')
if __name__ == '__main__':
options = utils.get_options(
['--config', False, 'Path to YAML configuration file']
)
show_index(options)
'/etc/conntrackd/conntrackd.conf',
'/etc/conntrackd/conntrackd.conf.back')
utils.ssh(himn, username,
'cp',
CONNTRACK_CONF_SAMPLE,
'/etc/conntrackd/conntrackd.conf')
# Restart conntrackd service
utils.ssh(himn, username, 'service', 'conntrackd', 'restart')
if __name__ == '__main__':
install_xenapi_sdk()
astute = utils.get_astute()
if astute:
username, password, install_xapi = utils.get_options(astute)
endpoints = get_endpoints(astute)
himn_eth, himn_local = utils.init_eth()
public_ip = utils.astute_get(
astute, ('network_metadata', 'vips', 'public', 'ipaddr'))
services_ssl = utils.astute_get(
astute, ('public_ssl', 'services'))
if username and password and endpoints and himn_local:
route_to_compute(endpoints, HIMN_IP, himn_local, username)
if install_xapi:
install_suppack(HIMN_IP, username, XS_PLUGIN_ISO)
enable_linux_bridge(HIMN_IP, username)
forward_from_himn(himn_eth)
def assistance_bot(self):
assist_bot, user_option_vals = train_bot.train_assistance_bot()
print("Our assistence bot is available now.")
options = utils.get_options(user_option_vals)
generate_response(assist_bot, "shopping assistance", options,
constants.USER_OPTIONS_COUNT)
dates = [d for d in dates if is_nice_date(d)]
#print(' ', dates)
if dates:
years = [get_year(d) for d in dates]
for year in years:
if EARLIEST_DOC < year < LATEST_DOC:
#print(' ', year)
return year
return None
def is_nice_date(date):
"""Something is a nice date if it has a month in it."""
for t in date.split():
if t.lower() in MONTHS:
return True
return False
def get_year(date):
for t in date.split():
if t.isdigit() and len(t) == 4:
return int(t)
return 0
if __name__ == '__main__':
data_dir, filelist, start, end, crash = get_options()
process_list(data_dir, filelist, start, end, crash, generate_metadata)
