def co_savings(cfg: Dict[str, Any]) -> Iterator[None]:
"""
Saves vsz, exports images and saves hdf5 log
Corutine must receive:
veusze: Veusz embedded object
log: dict with parameters: 'out_name' - log's index, 'out_vsz_full' - vsz file name to save
log parameters will be saved to pandas dataframe end then to hdf5 log cfg['program']['log'])[0]+'.h5'
"""
with pd.HDFStore(Path(cfg['program']['log']).with_suffix('.h5'),
mode='a') as storeLog:
veusze = None
if __name__ != '__main__':
path_prev = os_getcwd()
os_chdir(cfg['out']['path'].parent)
print('Saving to {}'.format(cfg['out']['path'].parent.absolute()))
try:
while True:
veusze, log = yield ()
if not cfg['out']['b_images_only']:
veusze.Save(str(log['out_vsz_full']))
# Save vsz modification date
log['fileChangeTime'] = datetime.fromtimestamp(
Path(log['out_vsz_full']).stat().st_mtime),
dfLog = pd.DataFrame.from_records(
log,
exclude=['out_name', 'out_vsz_full'],
index=[log['out_name']])
storeLog.append(Path(cfg['out']['path']).name,
dfLog,
data_columns=True,
expectedrows=cfg['in']['nfiles'],
index=False,
min_itemsize={'index': 30})
if cfg['async']['loop']:
try: # yield from asyncio.ensure_future(
# asyncio.wait_for(, cfg['async']['export_timeout_s'], loop=cfg['async']['loop'])
b = cfg['async']['loop'].run_until_complete(
export_images_timed(veusze, cfg,
'#' + log['out_name']))
except asyncio.TimeoutError:
l.warning('can not export in time')
else:
export_images(veusze, cfg['out'], '#' + log['out_name'])
except GeneratorExit:
print('Ok>')
finally:
if __name__ != '__main__':
os_chdir(path_prev)
if veusze and cfg['program']['return'] != '<embedded_object>':
veusze.Close()
l.info('closing Veusz embedded object')
def chdir(target):
"""Context manager for executing some code within a different
directory after which the current working directory will be set
back to what it was before.
Args:
target (str): path to the directory to change into.
"""
current = getcwd()
try:
os_chdir(target)
yield target
finally:
os_chdir(current)
def chdir(path=None):
print(f'===== path_ops/chdir =====')
if (path == None):
raise ValueError("No path is received")
else:
# Parse string to path object
path = dir_check(path, dir_create=False, return_type='Path')
print(f'target_dir = {path}')
path = pathlib.PurePath(path)
os_chdir(path)
cwd = pathlib.Path.cwd()
print(f'cwd = {cwd}')
def cleStart():
## base
from json import load as j_loads
v_lst = ("p2p_packet_info.txt", "packet_info.txt", "profile.txt", "stdout",
"syslog", "syserr", "usage.txt", "VERSION.txt", "DEV_LOG.log",
"mob_count", "*.core")
szPWD = os_getcwd()
## clear files from alog
with open("clear.list", "r") as fList:
mList = j_loads(fList)
for dic1 in mList:
# goto alog path
# print dic1["path"]
os_chdir(dic1["path"])
# clean files
fShell("cat /dev/null > PTS")
fShell("rm -rf log/* cores/*")
# goto base again
os_chdir(szPWD)
## clean other logs
with open("start.list", "r") as fList:
mList = j_loads(fList)
for dic1 in mList:
# goto alog path
# print dic1["path"]
os_chdir(dic1["path"])
fShell("echo --- delete inside '%s' ---" % dic1["path"])
fShell("rm -fv %s" % " ".join(v_lst))
# goto base again
os_chdir(szPWD)
def chdir(target):
"""Context manager for executing code within a different directory.
After the execution, the current working directory will be set back to its initial value.
Args:
target (str): path to the directory to change into.
"""
# from os import chdir, #getcwd
current = getcwd()
try:
os_chdir(target)
yield target
finally:
os_chdir(current)
# -*- coding: utf-8 -*-
"""
Created on Tue Mar 30 18:13:47 2021
@author: Meghana
"""
from os import path as os_path, chdir as os_chdir
os_chdir(os_path.dirname(os_path.abspath(__file__)))
from sys import path as sys_path
# insert at 1, 0 is the script path (or '' in REPL)
sys_path.insert(1, './functions_py3/')
from jaccard_coeff import jaccard_coeff
from numpy import percentile as np_percentile
from read_complexes import preprocess_complexes
#from pickle import load as pickle_load
import networkx
def NA_threshold(set1, set2):
ls1 = len(set1)
ls2 = len(set2)
if ls1 == 0 and ls2 == 0:
return 1
inter = float(len(set1.intersection(set2)))
a = inter / ls1
b = inter / ls2
return max(a, b)
def FromFile(cls, filePath):
if (not filePath.exists()): raise PyIpxactException("File '{0!s}' not found.".format(filePath)) from FileNotFoundError(str(filePath))
try:
with filePath.open(encoding="utf-8") as fileHandle:
content = fileHandle.read()
content = bytes(bytearray(content, encoding='utf-8'))
except OSError as ex:
raise PyIpxactException("Couldn't open '{0!s}'.".format(filePath)) from ex
os_chdir("lib/schema")
schemaPath = Path("index.xsd")
try:
with schemaPath.open(encoding="utf-8") as fileHandle:
schema = fileHandle.read()
schema = bytes(bytearray(schema, encoding='utf-8'))
except OSError as ex:
raise PyIpxactException("Couldn't open '{0!s}'.".format(schemaPath)) from ex
xmlParser = etree.XMLParser(remove_blank_text=True, encoding="utf-8")
schemaRoot = etree.XML(schema, xmlParser)
schemaTree = etree.ElementTree(schemaRoot)
xmlschema = etree.XMLSchema(schemaTree)
root = etree.XML(content, xmlParser)
rootTag = etree.QName(root.tag)
if (not xmlschema.validate(root)):
raise PyIpxactException("The input IP-XACT file is not valid.")
if (rootTag.namespace not in __URI_MAP__):
raise PyIpxactException("The input IP-XACT file uses an unsupported namespace: '{0}'.".format(rootTag.namespace))
if (rootTag.localname != "catalog"):
raise PyIpxactException("The input IP-XACT file is not a catalog file.")
print("==" * 20)
items = []
for rootElements in root:
element = etree.QName(rootElements)
if (element.localname == "vendor"):
vendor = rootElements.text
elif (element.localname == "library"):
library = rootElements.text
elif (element.localname == "name"):
name = rootElements.text
elif (element.localname == "version"):
version = rootElements.text
elif (element.localname == "description"):
description = rootElements.text
elif (element.localname == "catalogs"):
for ipxactFileElement in rootElements:
items.append(IpxactFile.FromXml(ipxactFileElement))
else:
raise PyIpxactException("Unsupported tag '{0}' at root-level.".format(element.localname))
print("==" * 20)
vlnv = Vlnv(vendor=vendor, library=library, name=name, version=version)
catalog = cls(vlnv, description=description)
for item in items:
catalog.AddItem(item)
return catalog
f'/{device}/logRuns', # str: name of log table - table with intervals:
'pattern_path': path_cruise / device / '~pattern~.vsz'
}
f_row = lambda r: [
'{Index:%y%m%d_%H%M}-{DateEnd:%H%M}.vsz'.format_map(r),
bytes(
"time_range = ['{:%Y-%m-%dT%H:%M:%S}', '{:%Y-%m-%dT%H:%M:%S}']".
format(r['Index'], r['DateEnd'] + pd.Timedelta(300, "s")), 'utf-8')
]
pattern_code = cfg_in['pattern_path'].read_bytes() #encoding='utf-8'
from os import getcwd as os_getcwd, chdir as os_chdir
path_prev = os_getcwd()
argv_prev = sys.argv
os_chdir(cfg_in['pattern_path'].parent)
for filename, str_expr in h5log_names_gen(cfg_in, f_row):
path_vsz = cfg_in['pattern_path'].with_name(filename)
path_vsz.write_bytes(
re.sub(rb'^([^\n]+)', str_expr, pattern_code, count=1))
veuszPropagate.main([
'ini/veuszPropagate.ini',
'--path',
str(cfg_in['pattern_path'].with_name(
'??????_????-????.vsz')), #path_db),
'--pattern_path',
f"{cfg_in['pattern_path']}_", # here used to auto get export dir only. must not be not existed file path
#'--table_log', f'/{device}/logRuns',
#'--add_custom_list', f'{device_veusz_prefix}USE_time_search_runs', # 'i3_USE_timeRange',
# '--add_custom_expressions',
def main(new_arg=None, veusze=None, **kwargs):
"""
Initialise configuration and runs or returns routines
cfg:
['program']['log'],
'out'
'in'
'async'
globals:
load_vsz
l
:param new_arg:
:param veusze: used to reuse veusz embedded object (thus to not leak memory)
:return:
"""
global l, load_vsz
cfg = cfg_from_args(my_argparser(), new_arg, **kwargs)
if not cfg or not cfg['program'].get('return'):
print('Can not initialise')
return cfg
elif cfg['program']['return'] == '<cfg_from_args>': # to help testing
return cfg
l = init_logging(logging, None, cfg['program']['log'],
cfg['program']['verbose'])
cfg['program']['log'] = l.root.handlers[
0].baseFilename # sinchronize obtained absolute file name
print('\n' + this_prog_basename(__file__), 'started', end=' ')
__name__ = '__main__' # indicate to other functions that they are called from main
if cfg['out'].get('paths'):
if not cfg['out']['b_images_only']:
raise NotImplementedError(
'Provided out in not "b_images_only" mode!')
cfg['out']['nfiles'] = len(cfg['out']['paths'])
cfg['out']['path'] = cfg['out']['paths'][0]
print(
end=f"\n- {cfg['out']['nfiles']} output files to export images...")
pass
else:
if cfg['out']['b_images_only']:
print(
'in images only mode. Output pattern: ') # todo Export path: '
else:
print('. Output pattern and Data: ')
try:
# Using cfg['out'] to store pattern information
if not Path(cfg['in']['pattern_path']).is_absolute():
cfg['in']['pattern_path'] = Path(cfg['in']['path']).with_name(
str(cfg['in']['pattern_path']))
cfg['out']['path'] = cfg['in']['pattern_path']
cfg['out']['paths'], cfg['out']['nfiles'], cfg['out'][
'path'] = init_file_names(**cfg['out'], b_interact=False)
except Ex_nothing_done as e:
if not cfg['out']['b_images_only']:
l.warning(
f'{e.message} - no pattern. Specify it or use "b_images_only" mode!'
)
return # or raise FileNotFoundError?
if (cfg['out']['b_images_only'] and cfg['out']['paths']):
cfg['in']['paths'] = cfg['out']['paths'] # have all we need to export
else:
try:
cfg['in']['paths'], cfg['in']['nfiles'], cfg['in'][
'path'] = init_file_names(**cfg['in'],
b_interact=cfg['program']
['b_interact'])
except Ex_nothing_done as e:
print(e.message)
return # or raise FileNotFoundError?
except TypeError: # expected str, bytes or os.PathLike object, not NoneType
# cfg['in']['path'] is None. May be it is not need
cfg['in']['paths'] = [cfg['in']['pattern_path']
] # dummy for compatibility
cfg['in']['nfiles'] = 1
cfg['out']['export_dir'] = dir_from_cfg(cfg['out']['path'].parent,
cfg['out']['export_dir'])
if 'restore_config' in cfg['program']['before_next']:
cfg['in_saved'] = cfg['in'].copy()
# Next is commented because reloading is Ok: not need to Close()
# if cfg['out']['b_images_only'] and not 'Close()' in cfg['program']['before_next']:
# cfg['program']['before_next'].append(
# 'Close()') # usually we need to load new file for export (not only modify previous file)
if cfg['program']['export_timeout_s'] and export_images_timed:
cfg['async'] = {
'loop': asyncio.get_event_loop(),
'export_timeout_s': cfg['program']['export_timeout_s']
}
else:
cfg['async'] = {'loop': None}
load_vsz = load_vsz_closure(cfg['program']['veusz_path'],
cfg['program']['load_timeout_s'],
cfg['program']['b_execute_vsz'])
cfg['load_vsz'] = load_vsz
cfg['co'] = {}
if cfg['in']['table_log'] and cfg['in']['path'].suffix == '.h5' and not (
cfg['out']['b_images_only'] and len(cfg['in']['paths']) > 1):
# load data by ranges from table log rows
cfg['in']['db_path'] = cfg['in']['path']
in_fulls = h5log_names_gen(cfg['in'])
elif cfg['in']['tables']:
# tables instead files
in_fulls = ge_names_from_hdf5_paths(cfg)
else: # switch to use found vsz as source if need only export images (even with database source)
in_fulls = ge_names(cfg)
cor_savings = co_savings(cfg)
cor_savings.send(None)
nfiles = 0
try: # if True:
path_prev = os_getcwd()
os_chdir(cfg['out']['path'].parent)
if cfg['program']['return'] == '<corutines_in_cfg>':
cfg['co']['savings'] = cor_savings
cfg['co']['gen_veusz_and_logs'] = load_to_veusz(in_fulls, cfg)
cfg['co']['send_data'] = co_send_data(load_to_veusz, cfg,
cor_savings)
return cfg # return with link to generator function
elif cfg['in'].get('data_yield_prefix'):
# Cycle with obtaining Veusz data
cfgin_update = None
while True: # for vsz_data, log in cor_send_data.send(cfgin_update):
try:
vsz_data, log = co_send_data.send(cfgin_update)
nfiles += 1
except (GeneratorExit, StopIteration, Ex_nothing_done):
break
if 'f_custom_in_cycle' in cfg['program']:
cfgin_update = cfg['program']['f_custom_in_cycle'](
vsz_data, log)
else:
# Cycle without obtaining Veusz data (or implemented by user's cfg['program']['f_custom_in_cycle'])
for veusze, log in load_to_veusz(in_fulls, cfg, veusze):
file_name_r = Path(log['out_vsz_full']).relative_to(
cfg['out']['path'].parent)
if cfg['program'].get('f_custom_in_cycle'):
cfgin_update = cfg['program']['f_custom_in_cycle'](veusze,
log)
veusze_commands(veusze, cfgin_update, file_name_r)
cor_savings.send((veusze, log))
nfiles += 1
cor_savings.close()
if cfg['program']['return'] != '<embedded_object>':
veusze = None # to note that it is closed in cor_savings.close()
print(f'{nfiles} processed. ok>')
pass
except Exception as e:
l.exception('Not good')
return # or raise FileNotFoundError?
finally:
if cfg['async']['loop']:
cfg['async']['loop'].close()
os_chdir(path_prev)
if veusze and cfg['program']['return'] == '<end>':
veusze.Close()
veusze.WaitForClose()
veusze = None
elif cfg['program']['return'] == '<embedded_object>':
cfg['veusze'] = veusze
return cfg
def load_vsz(
vsz: Union[str, PurePath, None] = None,
veusze: Optional[str] = None,
prefix: Optional[str] = None,
suffix_prior: Optional[str] = '_fbot'
) -> Tuple[veusz.Embedded, Optional[Dict[str, Any]]]:
"""
Load (create) specifid data from '*.vsz' files
:param vsz: full name of vsz or None. If not None and such file not found then create it
:param veusze: veusz.Embedded object or None - will be created if None else reused
:param prefix: only data started with this prefix will be loaded
:param suffix_prior: high priority names suffix, removes other version of data if starts same but with no such suffix (see veusz_data())
:return: (veusze, vsz_data):
- veusze - veusz.Embedded object
- vsz_data - loaded data if prefix is not None else None
"""
if vsz is None:
file_exists = False
if veusze is None:
title = 'empty'
l.debug('new embedded window')
else:
l.debug('keep same embedded window')
else: # isinstance(vsz, (str, PurePath)):
vsz = Path(vsz)
file_exists = vsz.is_file()
if file_exists:
l.debug(f'loading found vsz: {vsz}')
title = f'{vsz} - was found'
else:
l.debug(f'creatig vsz: {vsz}')
title = f'{vsz} - was created'
if veusze is None:
# Veusz embedded window construction
# Save right path in veusz.Embedded (closure)
if __name__ != '__main__': # if this haven't done in main()
path_prev = os_getcwd() # to recover
os_chdir(
vsz.parent
) # allows veusze.Load(path) to work if _path_ is relative or relative paths is used in vsz
veusze = veusz.Embedded(title) # , hidden=True
# veusze.EnableToolbar()
# veusze.Zoom('page')
if __name__ != '__main__':
os_chdir(path_prev) # recover
if file_exists:
if not b_execute_vsz:
if load_timeout_s:
# veusze.Load(str(vsz.name)) with timeout: # not tried veusze.serv_socket.settimeout(60)
SingletonTimeOut.run(partial(veusze.Load, str(vsz.name)),
load_timeout_s)
sleep(1)
else:
veusze.Load(vsz.name)
else:
def load_by_exec(vsz, veusze):
"""
Unsafe replasement for veusze.Load(vsz) to add variable argv
Runs any python commands before 1st Title command of
:param vsz:
:return:
"""
with vsz.open(encoding='utf-8') as v:
# comine pure python lines
lines = []
for line in v:
if line[:2].istitle():
break
lines.append(line)
# dangerous for unknown vsz but we allow 1 time at beginning of file: to use for known vsz
loc_exclude = locals().copy()
del loc_exclude['veusze']
loc = {
'argv': ['veusz.exe', str(vsz)],
'BASENAME': (lambda: vsz.stem)
}
# match = re.match
exec('\n'.join(lines), {}, loc)
loc.update(locals().copy())
for k in loc_exclude.keys():
del loc[k]
basename_result = "'{}'".format(loc['BASENAME']())
# eval Veusz commands
eval(f"""veusze.{line}""")
for line in v:
if 'BASENAME()' in line:
line = line.replace(
'BASENAME()', basename_result
) # only this helps in Custom Definitions expressions
# cmd, params = line.split('(', maxsplit=1)
eval(f"""veusze.{line}""", {},
loc) # , {"__builtins__": {}}
# from ast import literal_eval
# params_dict = literal_eval(params.rsplit(')', maxsplit=1)[0])
# getattr(veusze, cmd)(**params_dict)
return
load_by_exec(vsz, veusze)
if prefix is None:
return veusze, None
return veusze, veusz_data(veusze, prefix, suffix_prior)
def main(new_arg=None, **kwargs):
"""
:param new_arg: list of strings, command line arguments
:kwargs: dicts of dictcts (for each ini section): specified values overwrites ini values
"""
# global l
cfg = cfg_from_args(my_argparser(), new_arg, **kwargs)
if not cfg['program']:
return # usually error of unrecognized arguments displayed
cfg['in']['db_coefs'] = Path(cfg['in']['db_coefs'])
for path_field in ['db_coefs', 'path_cruise']:
if not cfg['in'][path_field].is_absolute():
cfg['in'][path_field] = (
cfg['in']['cfgFile'].parent / cfg['in'][path_field]
).resolve().absolute() # cfg['in']['cfgFile'].parent /
def constant_factory(val):
def default_val():
return val
return default_val
for lim in ('min_date', 'max_date'):
# convert keys to int because they must be comparable to probes_int_list (for command line arguments keys are allways strings, in yaml you can set string or int)
_ = {int(k): v for k, v in cfg['filter'][lim].items()}
cfg['filter'][lim] = defaultdict(constant_factory(_.get(0)), _)
l = init_logging(logging, None, None, 'INFO')
#l = init_logging(logging, None, cfg['program']['log'], cfg['program']['verbose'])
if True: # False. Experimental speedup but takes memory
from dask.cache import Cache
cache = Cache(2e9) # Leverage two gigabytes of memory
cache.register() # Turn cache on globally
#if __debug__:
# # because there was errors on debug when default scheduler used
# cfg['program']['dask_scheduler'] = 'synchronous'
if cfg['program']['dask_scheduler']:
if cfg['program']['dask_scheduler'] == 'distributed':
from dask.distributed import Client
# cluster = dask.distributed.LocalCluster(n_workers=2, threads_per_worker=1, memory_limit="5.5Gb")
client = Client(processes=False)
# navigate to http://localhost:8787/status to see the diagnostic dashboard if you have Bokeh installed
# processes=False: avoide inter-worker communication for computations releases the GIL (numpy, da.array) # without is error
else:
if cfg['program']['dask_scheduler'] == 'synchronous':
l.warning('using "synchronous" scheduler for debugging')
import dask
dask.config.set(scheduler=cfg['program']['dask_scheduler'])
# Run steps :
st.start = cfg['program']['step_start']
st.end = cfg['program']['step_end']
st.go = True
if not cfg['out'][
'db_name']: # set name by 'path_cruise' name or parent if it has digits at start. priority for name is "*inclinometer*"
for p in (lambda p: [p, p.parent])(cfg['in']['path_cruise']):
m = re.match('(^[\d_]*).*', p.name)
if m:
break
cfg['out']['db_name'] = f"{m.group(1).strip('_')}incl.h5"
dir_incl = next((d for d in cfg['in']['path_cruise'].glob('*inclinometer*')
if d.is_dir()), cfg['in']['path_cruise'])
db_path = dir_incl / '_raw' / cfg['out']['db_name']
# ---------------------------------------------------------------------------------------------
# def fs(probe, name):
# if 'w' in name.lower(): # Baranov's wavegauge electronic
# return 10 # 5
# return 5
# if probe < 20 or probe in [23, 29, 30, 32, 33]: # 30 [4, 11, 5, 12] + [1, 7, 13, 30]
# return 5
# if probe in [21, 25, 26] + list(range(28, 35)):
# return 8.2
# return 4.8
def datetime64_str(time_str: Optional[str] = None) -> np.ndarray:
"""
Reformat time_str to ISO 8601 or to 'NaT'. Used here for input in funcs that converts str to numpy.datetime64
:param time_str: May be 'NaT'
:return: ndarray of strings (tested for 1 element only) formatted by numpy.
"""
return np.datetime_as_string(np.datetime64(time_str, 's'))
probes = cfg['in']['probes'] or range(
1, 41) # sets default range, specify your values before line ---
raw_root, probe_is_incl = re.subn('INCL_?', 'INKL_',
cfg['in']['probes_prefix'].upper())
# some parameters that depends of probe type (indicated by probes_prefix)
p_type = defaultdict(
# baranov's format
constant_factory({
'correct_fun':
partial(correct_txt,
mod_file_name=mod_incl_name,
sub_str_list=[
b'^\r?(?P<use>20\d{2}(\t\d{1,2}){5}(\t\d{5}){8}).*',
b'^.+'
]),
'fs':
10,
'format':
'Baranov',
}),
{
(lambda x: x if x.startswith('incl') else 'incl')(cfg['in']['probes_prefix']):
{
'correct_fun':
partial(
correct_txt,
mod_file_name=mod_incl_name,
sub_str_list=[
b'^(?P<use>20\d{2}(,\d{1,2}){5}(,\-?\d{1,6}){6}(,\d{1,2}\.\d{2})(,\-?\d{1,3}\.\d{2})).*',
b'^.+'
]),
'fs':
5,
'format':
'Kondrashov',
},
'voln': {
'correct_fun':
partial(
correct_txt,
mod_file_name=mod_incl_name,
sub_str_list=[
b'^(?P<use>20\d{2}(,\d{1,2}){5}(,\-?\d{1,8})(,\-?\d{1,2}\.\d{2}){2}).*',
b'^.+'
]),
'fs':
5,
#'tbl_prefix': 'w',
'format':
'Kondrashov',
}
})
if st(1, 'Save inclinometer or wavegage data from ASCII to HDF5'):
# Note: Can not find additional not corrected files for same probe if already have any corrected in search path (move them out if need)
i_proc_probe = 0 # counter of processed probes
i_proc_file = 0 # counter of processed files
# patten to identify only _probe_'s raw data files that need to correct '*INKL*{:0>2}*.[tT][xX][tT]':
raw_parent = dir_incl / '_raw' # raw_parent /=
if cfg['in']['raw_subdir'] is None:
cfg['in']['raw_subdir'] = ''
dir_out = raw_parent / re.sub(r'[.\\/ *?]', '_',
cfg['in']['raw_subdir'])
# sub replaces multilevel subdirs to 1 level that correct_fun() can only make
def dt_from_utc_2000(probe):
""" Correct time of probes started without time setting. Raw date must start from 2000-01-01T00:00"""
return (
datetime(year=2000, month=1, day=1) -
cfg['in']['time_start_utc'][probe]
) if cfg['in']['time_start_utc'].get(probe) else timedelta(0)
# convert cfg['in']['dt_from_utc'] keys to int
cfg['in']['dt_from_utc'] = {
int(p): v
for p, v in cfg['in']['dt_from_utc'].items()
}
# convert cfg['in']['t_start_utc'] to cfg['in']['dt_from_utc'] and keys to int
cfg['in']['dt_from_utc'].update( # overwriting the 'time_start_utc' where already exist
{int(p): dt_from_utc_2000(p) for p, v in cfg['in']['time_start_utc'].items()}
)
# make cfg['in']['dt_from_utc'][0] be default value
cfg['in']['dt_from_utc'] = defaultdict(
constant_factory(cfg['in']['dt_from_utc'].pop(0, timedelta(0))),
cfg['in']['dt_from_utc'])
for probe in probes:
raw_found = []
raw_pattern_file = str(
Path(glob.escape(cfg['in']['raw_subdir'])) /
cfg['in']['raw_pattern'].format(prefix=raw_root, number=probe))
correct_fun = p_type[cfg['in']['probes_prefix']]['correct_fun']
# if not archive:
if (not re.match(r'.*(\.zip|\.rar)$', cfg['in']['raw_subdir'],
re.IGNORECASE)) and raw_parent.is_dir():
raw_found = list(raw_parent.glob(raw_pattern_file))
if not raw_found:
# Check if already have corrected files for probe generated by correct_txt(). If so then just use them
raw_found = list(
dir_out.glob(
f"{cfg['in']['probes_prefix']}{probe:0>2}.txt"))
if raw_found:
print('corrected csv file', [r.name for r in raw_found],
'found')
correct_fun = lambda x, dir_out: x
elif not cfg['in']['raw_subdir']:
continue
for file_in in (raw_found or open_csv_or_archive_of_them(
raw_parent, binary_mode=False, pattern=raw_pattern_file)):
file_in = correct_fun(file_in, dir_out=dir_out)
if not file_in:
continue
tbl = file_in.stem # f"{cfg['in']['probes_prefix']}{probe:0>2}"
# tbl = re.sub('^((?P<i>inkl)|w)_0', lambda m: 'incl' if m.group('i') else 'w', # correct name
# re.sub('^[\d_]*|\*', '', file_in.stem).lower()), # remove date-prefix if in name
csv2h5(
[
str(
Path(__file__).parent / 'ini' /
f"csv_{'inclin' if probe_is_incl else 'wavegage'}_{p_type[cfg['in']['probes_prefix']]['format']}.ini"
),
'--path',
str(file_in),
'--blocksize_int',
'50_000_000', # 50Mbt
'--table',
tbl,
'--db_path',
str(db_path),
# '--log', str(scripts_path / 'log/csv2h5_inclin_Kondrashov.log'),
# '--b_raise_on_err', '0', # ?
'--b_interact',
'0',
'--fs_float',
str(p_type[cfg['in']['probes_prefix']]
['fs']), #f'{fs(probe, file_in.stem)}',
'--dt_from_utc_seconds',
str(cfg['in']['dt_from_utc'][probe].total_seconds()),
'--b_del_temp_db',
'1',
] +
(['--csv_specific_param_dict', 'invert_magnitometr: True']
if probe_is_incl else []),
**{
'filter': {
'min_date':
cfg['filter']['min_date'].get(
probe, np.datetime64(0, 'ns')),
'max_date':
cfg['filter']['max_date'].get(
probe, np.datetime64('now', 'ns')
), # simple 'now' works in sinchronious mode
}
})
# Get coefs:
l.info(
f"Adding coefficients to {db_path}/{tbl} from {cfg['in']['db_coefs']}"
)
try:
h5copy_coef(cfg['in']['db_coefs'], db_path, tbl)
except KeyError as e: # Unable to open object (component not found)
l.warning(
'No coefs to copy?'
) # write some dummy coefficients to can load Veusz patterns:
h5copy_coef(None,
db_path,
tbl,
dict_matrices=dict_matrices_for_h5(tbl=tbl))
except OSError as e:
l.warning(
'Not found DB with coefs?'
) # write some dummy coefficients to can load Veusz patterns:
h5copy_coef(None,
db_path,
tbl,
dict_matrices=dict_matrices_for_h5(tbl=tbl))
i_proc_file += 1
else:
print('no', raw_pattern_file, end=', ')
i_proc_probe += 1
print('Ok:', i_proc_probe, 'probes,', i_proc_file, 'files processed.')
if st(2, 'Calculate physical parameters and average'):
kwarg = {
'in': {
'min_date': cfg['filter']['min_date'][0],
'max_date': cfg['filter']['max_date'][0],
'time_range_zeroing': cfg['in']['time_range_zeroing']
},
'proc': {}
}
# if aggregate_period_s is None then not average and write to *_proc_noAvg.h5 else loading from that h5 and writing to _proc.h5
if not cfg['out']['aggregate_period_s']:
cfg['out']['aggregate_period_s'] = [
None, 2, 600, 7200 if probe_is_incl else 3600
]
if cfg['in']['azimuth_add']:
if 'Lat' in cfg['in']['azimuth_add']:
# add magnetic declination,° for used coordinates
# todo: get time
kwarg['proc']['azimuth_add'] = mag_dec(
cfg['in']['azimuth_add']['Lat'],
cfg['in']['azimuth_add']['Lon'],
datetime(2020, 9, 10),
depth=-1)
else:
kwarg['proc']['azimuth_add'] = 0
if 'constant' in cfg['in']['azimuth_add']:
# and add constant. For example, subtruct declination at the calibration place if it was applied
kwarg['proc']['azimuth_add'] += cfg['in']['azimuth_add'][
'constant'] # add -6.656 to account for calibration in Kaliningrad (mag deg = 6.656°)
for aggregate_period_s in cfg['out']['aggregate_period_s']:
if aggregate_period_s is None:
db_path_in = db_path
db_path_out = dir_incl / f'{db_path.stem}_proc_noAvg.h5'
else:
db_path_in = dir_incl / f'{db_path.stem}_proc_noAvg.h5'
db_path_out = dir_incl / f'{db_path.stem}_proc.h5' # or separately: '_proc{aggregate_period_s}.h5'
# 'incl.*|w\d*' inclinometers or wavegauges w\d\d # 'incl09':
tables_list_regex = f"{cfg['in']['probes_prefix'].replace('voln', 'w')}.*"
if cfg['in']['probes']:
tables_list_regex += "(?:{})".format('|'.join(
'{:0>2}'.format(p) for p in cfg['in']['probes']))
args = [
'../../empty.yml', # all settings are here, so to not print 'using default configuration' we use some existed empty file
'--db_path',
str(db_path_in),
'--tables_list',
tables_list_regex,
'--aggregate_period',
f'{aggregate_period_s}S' if aggregate_period_s else '',
'--out.db_path',
str(db_path_out),
'--table',
f'V_incl_bin{aggregate_period_s}'
if aggregate_period_s else 'V_incl',
'--verbose',
'INFO', #'DEBUG' get many numba messages
'--b_del_temp_db',
'1',
# '--calc_version', 'polynom(force)', # depreshiated
# '--chunksize', '20000',
# '--not_joined_h5_path', f'{db_path.stem}_proc.h5',
]
if aggregate_period_s is None: # proc. parameters (if we have saved proc. data then when aggregating we are not processing)
# Note: for Baranov's prog 4096 is not suited:
args += ([
'--max_dict',
'M[xyz]:4096',
# '--time_range_zeroing_dict', "incl19: '2019-11-10T13:00:00', '2019-11-10T14:00:00'\n," # not works - use kwarg
# '--time_range_zeroing_list', '2019-08-26T04:00:00, 2019-08-26T05:00:00'
'--split_period',
'1D'
] if probe_is_incl else [
'--bad_p_at_bursts_starts_peroiod',
'1H',
])
# csv splitted by 1day (default for no avg) else csv is monolith
if aggregate_period_s not in cfg['out'][
'aggregate_period_s_not_to_text']: # , 300, 600]:
args += ['--text_path', str(dir_incl / 'text_output')]
# If need all data to be combined one after one:
# set_field_if_no(kwarg, 'in', {})
# kwarg['in'].update({
#
# 'tables': [f'incl{i:0>2}' for i in min_date.keys() if i!=0],
# 'dates_min': min_date.values(), # in table list order
# 'dates_max': max_date.values(), #
# })
# set_field_if_no(kwarg, 'out', {})
# kwarg['out'].update({'b_all_to_one_col': 'True'})
incl_h5clc.main(args, **kwarg)
if st(3, 'Calculate spectrograms'): # Can be done at any time after step 1
min_Pressure = 7
# add dict dates_min like {probe: parameter} of incl_clc to can specify param to each probe
def raise_ni():
raise NotImplementedError(
'Can not proc probes having different fs in one run: you need to do it separately'
)
args = [
Path(incl_h5clc.__file__).with_name(
f'incl_h5spectrum{db_path.stem}.yaml'),
# if no such file all settings are here
'--db_path',
str(dir_incl / f'{db_path.stem}_proc_noAvg.h5'),
'--tables_list',
f"{cfg['in']['probes_prefix']}.*", # inclinometers or wavegauges w\d\d ## 'w02', 'incl.*',
# '--aggregate_period', f'{aggregate_period_s}S' if aggregate_period_s else '',
'--min_date',
datetime64_str(cfg['filter']['min_date'][0]),
'--max_date',
datetime64_str(cfg['filter']['max_date']
[0]), # '2019-09-09T16:31:00', #17:00:00
'--min_Pressure',
f'{min_Pressure}',
# '--max_dict', 'M[xyz]:4096', # use if db_path is not ends with _proc_noAvg.h5 i.e. need calc velocity
'--out.db_path',
f"{db_path.stem.replace('incl', cfg['in']['probes_prefix'])}_proc_psd.h5",
# '--table', f'psd{aggregate_period_s}' if aggregate_period_s else 'psd',
'--fs_float',
str(p_type[cfg['in']['probes_prefix']]
['fs']), # f"{fs(probes[0], cfg['in']['probes_prefix'])}",
# (lambda x: x == x[0])(np.vectorize(fs)(probes, prefix))).all() else raise_ni()
#
# '--time_range_zeroing_list', '2019-08-26T04:00:00, 2019-08-26T05:00:00'
# '--verbose', 'DEBUG',
# '--chunksize', '20000',
'--b_interact',
'0',
]
if probe_is_incl:
args += [
'--split_period',
'2H',
'--fmin',
'0.0004', #0.0004
'--fmax',
'1.05'
]
else:
args += [
'--split_period',
'1H',
'--dt_interval_minutes',
'15', # set this if burst mode to the burst interval
'--fmin',
'0.0001',
'--fmax',
'4',
#'--min_Pressure', '-1e15', # to not load NaNs
]
incl_h5spectrum.main(args)
if st(4, 'Draw in Veusz'):
pattern_path = dir_incl / r'processed_h5,vsz/201202-210326incl_proc#28.vsz'
# r'\201202_1445incl_proc#03_pattern.vsz' #'
# db_path.parent / r'vsz_5min\191119_0000_5m_incl19.vsz' # r'vsz_5min\191126_0000_5m_w02.vsz'
b_images_only = False
# importing in vsz index slices replacing:
pattern_str_slice_old = None
# Length of not adjacent intervals, s (set None to not allow)
# pandas interval in string or tuple representation '1D' of period between intervals and interval to draw
period_str = '0s' # '1D' # dt
dt_str = '0s' # '5m'
file_intervals = None
period = to_offset(period_str).delta
dt = to_offset(dt_str).delta # timedelta(0) # 60 * 5
if file_intervals and period and dt:
# Load starts and assign ends
t_intervals_start = pd.read_csv(
cfg['in']['path_cruise'] /
r'vsz+h5_proc\intervals_selected.txt',
converters={
'time_start': lambda x: np.datetime64(x, 'ns')
},
index_col=0).index
edges = (pd.DatetimeIndex(t_intervals_start),
pd.DatetimeIndex(t_intervals_start + dt_custom_s)
) # np.zeros_like()
elif period and dt:
# Generate periodic intervals
t_interval_start, t_intervals_end = intervals_from_period(
datetime_range=np.array(
[
cfg['filter']['min_date']['0'],
cfg['filter']['max_date']['0']
],
# ['2018-08-11T18:00:00', '2018-09-06T00:00:00'],
# ['2019-02-11T13:05:00', '2019-03-07T11:30:00'],
# ['2018-11-16T15:19', '2018-12-14T14:35'],
# ['2018-10-22T12:30', '2018-10-27T06:30:00'],
'datetime64[s]'),
period=period)
edges = (pd.DatetimeIndex([t_interval_start
]).append(t_intervals_end[:-1]),
pd.DatetimeIndex(t_intervals_end))
else: # [min, max] edges for each probe
edges_dict = {
pr:
[cfg['filter']['min_date'][pr], cfg['filter']['max_date'][pr]]
for pr in probes
}
cfg_vp = {'veusze': None}
for i, probe in enumerate(probes):
# cfg_vp = {'veusze': None}
if edges_dict: # custom edges for each probe
edges = [pd.DatetimeIndex([t]) for t in edges_dict[probe]]
# substr in file to rerplace probe_name_in_pattern (see below).
probe_name = f"_{cfg['in']['probes_prefix'].replace('incl', 'i')}{probe:02}"
tbl = None # f"/{cfg['in']['probes_prefix']}{probe:02}" # to check probe data exist in db else will not check
l.info('Draw %s in Veusz: %d intervals...', probe_name,
edges[0].size)
# for i_interval, (t_interval_start, t_interval_end) in enumerate(zip(pd.DatetimeIndex([t_interval_start]).append(t_intervals_end[:-1]), t_intervals_end), start=1):
for i_interval, (t_interval_start,
t_interval_end) in enumerate(zip(*edges),
start=1):
# if i_interval < 23: #<= 0: # TEMPORARY Skip this number of intervals
# continue
if period and period != dt:
t_interval_start = t_interval_end - pd.Timedelta(
dt_custom_s, 's')
if tbl:
try: # skipping absent probes
start_end = h5q_interval2coord(
db_path=str(db_path),
table=tbl,
t_interval=(t_interval_start, t_interval_end))
if not len(start_end):
break # no data
except KeyError:
break # device name not in specified range, go to next name
pattern_path_new = pattern_path.with_name(''.join([
f'{t_interval_start:%y%m%d_%H%M}',
f'_{dt_str}' if dt else '', f'{probe_name}.vsz'
]))
# Modify pattern file
if not b_images_only:
pattern_type, pattern_number = re.match(
r'.*(incl|w)_proc?#?(\d*).*',
pattern_path.name).groups()
probe_name_in_pattern = f"_{pattern_type.replace('incl', 'i')}{pattern_number}"
def f_replace(line):
"""
Replace in file
1. probe name
2. slice
"""
# if i_interval == 1:
line, ok = re.subn(probe_name_in_pattern, probe_name,
line)
if ok and pattern_str_slice_old: # can be only in same line
str_slice = '(({:d}, {:d}, None),)'.format(
*(start_end +
np.int32([-1, 1]))) # bytes(, 'ascii')
line = re.sub(pattern_str_slice_old, str_slice,
line)
return line
if not rep_in_file(pattern_path,
pattern_path_new,
f_replace=f_replace,
binary_mode=False):
l.warning('Veusz pattern not changed!'
) # may be ok if we need draw pattern
# break
elif cfg_vp['veusze']:
cfg_vp['veusze'].Load(str(pattern_path_new))
elif cfg_vp['veusze']:
cfg_vp['veusze'].Load(str(pattern_path_new))
txt_time_range = \
"""
"[['{:%Y-%m-%dT%H:%M}', '{:%Y-%m-%dT%H:%M}']]" \
""".format(t_interval_start, t_interval_end)
print(f'{i_interval}. {txt_time_range}', end=' ')
cfg_vp = veuszPropagate.main(
[
Path(veuszPropagate.__file__).parent.with_name(
'veuszPropagate.ini'),
# '--data_yield_prefix', '-',
# '--path', str(db_path), # if custom loading from db and some source is required
'--tables_list',
'', # switches to search vsz-files only # f'/{probe_name}', # 181022inclinometers/ \d*
'--pattern_path',
str(pattern_path_new),
# fr'd:\workData\BalticSea\190801inclinometer_Schuka\{probe_name}_190807_1D.vsz',
# str(dir_incl / f'{probe_name}_190211.vsz'), #warning: create file with small name
# '--before_next', 'restore_config',
# '--add_to_filename', f"_{t_interval_start:%y%m%d_%H%M}_{dt}",
'--filename_fun',
f'lambda tbl: "{pattern_path_new.name}"',
'--add_custom_list',
f'USEtime__', # f'USEtime{probe_name}', nAveragePrefer',
'--add_custom_expressions_list',
txt_time_range,
# + """
# ", 5"
# """,
'--b_update_existed',
'True',
'--export_pages_int_list',
'0', # 0 for all '6, 7, 8', #'1, 2, 3'
# '--export_dpi_int', '200',
'--export_format',
'jpg', #'emf',
'--b_interact',
'0',
'--b_images_only',
f'{b_images_only}',
'--return',
'<embedded_object>', # reuse to not bloat memory
'--b_execute_vsz',
'True',
'--before_next',
'Close()' # Close() need if b_execute_vsz many files
],
veusze=cfg_vp['veusze'])
if st(40, f'Draw in Veusz by loader-drawer.vsz method'):
# save all vsz files that uses separate code
from os import chdir as os_chdir
dt_s = 300
cfg['in'][
'pattern_path'] = db_path.parent / f'vsz_{dt_s:d}s' / '~pattern~.vsz'
time_starts = pd.read_csv(
db_path.parent / r'processed_h5,vsz' / 'intervals_selected.txt',
index_col=0,
parse_dates=True,
date_parser=lambda x: pd.to_datetime(x, format='%Y-%m-%dT%H:%M:%S'
)).index
pattern_code = cfg['in']['pattern_path'].read_bytes(
) # encoding='utf-8'
path_vsz_all = []
for i, probe in enumerate(probes):
probe_name = f"{cfg['in']['probes_prefix']}{probe:02}" # table name in db
l.info('Draw %s in Veusz: %d intervals...', probe_name,
time_starts.size)
for i_interval, time_start in enumerate(time_starts, start=1):
path_vsz = cfg['in']['pattern_path'].with_name(
f"{time_start:%y%m%d_%H%M}_{probe_name.replace('incl','i')}.vsz"
)
# copy file to path_vsz
path_vsz.write_bytes(pattern_code) # replaces 1st row
path_vsz_all.append(path_vsz)
os_chdir(cfg['in']['pattern_path'].parent)
veuszPropagate.main(
[
'ini/veuszPropagate.ini',
'--path',
str(cfg['in']['pattern_path'].with_name(
'??????_????_*.vsz')), # db_path),
'--pattern_path',
f"{cfg['in']['pattern_path']}_",
# here used to auto get export dir only. may not be _not existed file path_ if ['out']['paths'] is provided
# '--table_log', f'/{device}/logRuns',
# '--add_custom_list', f'{device_veusz_prefix}USE_time_search_runs', # 'i3_USE_timeRange',
# '--add_custom_expressions',
# """'[["{log_row[Index]:%Y-%m-%dT%H:%M:%S}", "{log_row[DateEnd]:%Y-%m-%dT%H:%M:%S}"]]'""",
# '--export_pages_int_list', '1', #'--b_images_only', 'True'
'--b_interact',
'0',
'--b_update_existed',
'True', # todo: delete_overlapped
'--b_images_only',
'True',
'--load_timeout_s_float',
str(cfg['program']['load_timeout_s'])
# '--min_time', '2020-07-08T03:35:00',
],
**{'out': {
'paths': path_vsz_all
}})
if st(50, 'Export from existed Veusz files in dir'):
pattern_parent = db_path.parent # r'vsz_5min\191126_0000_5m_w02.vsz''
pattern_path = str(pattern_parent / r'processed_h5,vsz' /
'??????incl_proc#[1-9][0-9].vsz') # [0-2,6-9]
veuszPropagate.main([
'ini/veuszPropagate.ini',
'--path',
pattern_path,
'--pattern_path',
pattern_path,
# '--export_pages_int_list', '1', #'--b_images_only', 'True'
'--b_interact',
'0',
'--b_update_existed',
'True', # todo: delete_overlapped
'--b_images_only',
'True',
'--load_timeout_s_float',
str(cfg['program']['load_timeout_s']),
'--b_execute_vsz',
'True',
'--before_next',
'Close()' # Close() need if b_execute_vsz many files
])
from os import chdir as os_chdir
from os import system as os_system
from time import sleep as delay
from uuid import getnode as get_mac
macs = {'Macbook':'105773427819682',
'MacBookPro':'117637351435',
'MBP2':'220083054034723'}
this_mac = get_mac()
if str(this_mac)==macs['Macbook']: # MB
os_chdir('/Users/sethchase/Dropbox/BD_Scripts/django/Dropbox/aprinto')
elif str(this_mac)==macs['MacBookPro']: # MBP
os_chdir('/Users/admin/django/Dropbox/aprinto')
print '\n\tRe-Building Content and Testing...\n'
os_system('ENV/bin/python manage.py reset_db --noinput')
os_system('ENV/bin/python manage.py syncdb --noinput')
print '\n\tStarting Development Server ...'
os_system('ENV/bin/python manage.py runserver 0.0.0.0:8080 &')
delay(6)
# os_system('ENV/bin/python management/initialize.py')
os_system('ENV/bin/python management/tests.py "dev"') # Run Tests
os_system('pkill -f "python manage.py runserver"')
def chdir(path):
from os import chdir as os_chdir
print('entering directory: ' + path)
os_chdir(path)
#Cache Compatibilty Layer's Code
g = open('cache/build.ytmp', mode='w')
#g = open('cache/build.py', mode='w')
for j in compatiblity:
g.write(j)
g.close()
#Convert to Byte Code
name = 'example-compiled.pyc.yt'
try:
os_remove('cache/__pycache__/' + name)
except (FileNotFoundError):
pass
py_compile.compile('cache/build.ytmp')
os_rename('cache/__pycache__/build.cpython-37.pyc',
str('cache/__pycache__/' + name))
package_name = 'example-package.zip'
#Zip file creation
os_chdir('cache/__pycache__/')
package = zipfile.ZipFile(package_name, mode='w')
package.write(name)
package.close()
os_chdir('..')
os_chdir('..')
os_rename('cache/__pycache__/' + package_name, str('output/' + package_name))
#Wait
print("Press any key to close...")
a = input()
def _main(
inventory,
operations,
verbosity,
chdir,
ssh_user,
ssh_port,
ssh_key,
ssh_key_password,
ssh_password,
winrm_username,
winrm_password,
winrm_port,
winrm_transport,
shell_executable,
sudo,
sudo_user,
use_sudo_password,
su_user,
parallel,
fail_percent,
data,
group_data,
config_filename,
dry,
limit,
no_wait,
serial,
quiet,
debug,
debug_facts,
debug_operations,
support=None,
):
# Setup working directory
#
if chdir:
os_chdir(chdir)
# Setup logging
#
if not debug and not sys.warnoptions:
warnings.simplefilter("ignore")
log_level = logging.INFO
if debug:
log_level = logging.DEBUG
elif quiet:
log_level = logging.WARNING
setup_logging(log_level)
# Bootstrap any virtualenv
init_virtualenv()
# Check operations are valid and setup command
#
# Make a copy before we overwrite
original_operations = operations
# Debug (print) inventory + group data
if operations[0] == "debug-inventory":
command = "debug-inventory"
# Get one or more facts
elif operations[0] == "fact":
command = "fact"
operations = get_facts_and_args(operations[1:])
# Execute a raw command with server.shell
elif operations[0] == "exec":
command = "exec"
operations = operations[1:]
# Execute one or more deploy files
elif all(cmd.endswith(".py") for cmd in operations):
command = "deploy"
filenames = []
for filename in operations[0:]:
if path.exists(filename):
filenames.append(filename)
continue
if chdir and filename.startswith(chdir):
correct_filename = path.relpath(filename, chdir)
logger.warning(
(
"Fixing deploy filename under `--chdir` argument: "
f"{filename} -> {correct_filename}"
),
)
filenames.append(correct_filename)
continue
raise CliError(
"No deploy file: {0}".format(
path.join(chdir, filename) if chdir else filename,
),
)
operations = filenames
# Operation w/optional args (<module>.<op> ARG1 ARG2 ...)
elif len(operations[0].split(".")) == 2:
command = "op"
operations = get_operation_and_args(operations)
else:
raise CliError(
"""Invalid operations: {0}
Operation usage:
pyinfra INVENTORY deploy_web.py [deploy_db.py]...
pyinfra INVENTORY server.user pyinfra home=/home/pyinfra
pyinfra INVENTORY exec -- echo "hello world"
pyinfra INVENTORY fact os [users]...""".format(
operations,
),
)
# Setup state, config & inventory
#
cwd = getcwd()
if cwd not in sys.path: # ensure cwd is present in sys.path
sys.path.append(cwd)
state = State()
state.cwd = cwd
ctx_state.set(state)
if verbosity > 0:
state.print_fact_info = True
state.print_noop_info = True
if verbosity > 1:
state.print_input = state.print_fact_input = True
if verbosity > 2:
state.print_output = state.print_fact_output = True
if not quiet:
click.echo("--> Loading config...", err=True)
config = Config()
ctx_config.set(config)
# Load up any config.py from the filesystem
config_filename = path.join(state.cwd, config_filename)
if path.exists(config_filename):
exec_file(config_filename)
# Lock the current config, this allows us to restore this version after
# executing deploy files that may alter them.
config.lock_current_state()
# Arg based config overrides
if sudo:
config.SUDO = True
if sudo_user:
config.SUDO_USER = sudo_user
if use_sudo_password:
config.USE_SUDO_PASSWORD = use_sudo_password
if su_user:
config.SU_USER = su_user
if parallel:
config.PARALLEL = parallel
if shell_executable:
config.SHELL = None if shell_executable in ("None", "null") else shell_executable
if fail_percent is not None:
config.FAIL_PERCENT = fail_percent
if not quiet:
click.echo("--> Loading inventory...", err=True)
override_data = {}
for arg in data:
key, value = arg.split("=", 1)
override_data[key] = value
override_data = {key: parse_cli_arg(value) for key, value in override_data.items()}
for key, value in (
("ssh_user", ssh_user),
("ssh_key", ssh_key),
("ssh_key_password", ssh_key_password),
("ssh_port", ssh_port),
("ssh_password", ssh_password),
("winrm_username", winrm_username),
("winrm_password", winrm_password),
("winrm_port", winrm_port),
("winrm_transport", winrm_transport),
):
if value:
override_data[key] = value
# Load up the inventory from the filesystem
inventory, inventory_group = make_inventory(
inventory,
cwd=state.cwd,
override_data=override_data,
group_data_directories=group_data,
)
ctx_inventory.set(inventory)
# Now that we have inventory, apply --limit config override
initial_limit = None
if limit:
all_limit_hosts = []
for limiter in limit:
try:
limit_hosts = inventory.get_group(limiter)
except NoGroupError:
limit_hosts = [host for host in inventory if fnmatch(host.name, limiter)]
if not limit_hosts:
logger.warning("No host matches found for --limit pattern: {0}".format(limiter))
all_limit_hosts.extend(limit_hosts)
initial_limit = list(set(all_limit_hosts))
# Initialise the state
state.init(inventory, config, initial_limit=initial_limit)
if command == "debug-inventory":
print_inventory(state)
_exit()
# Connect to the hosts & start handling the user commands
#
if not quiet:
click.echo(err=True)
click.echo("--> Connecting to hosts...", err=True)
connect_all(state)
if command == "fact":
if not quiet:
click.echo(err=True)
click.echo("--> Gathering facts...", err=True)
state.print_fact_info = True
fact_data = {}
for i, command in enumerate(operations):
fact_cls, args, kwargs = command
fact_key = fact_cls.name
if args or kwargs:
fact_key = "{0}{1}{2}".format(
fact_cls.name,
args or "",
" ({0})".format(get_kwargs_str(kwargs)) if kwargs else "",
)
try:
fact_data[fact_key] = get_facts(
state,
fact_cls,
args=args,
kwargs=kwargs,
apply_failed_hosts=False,
)
except PyinfraError:
pass
print_facts(fact_data)
_exit()
if command == "exec":
state.print_output = True
add_op(
state,
server.shell,
" ".join(operations),
_allow_cli_mode=True,
)
elif command == "deploy":
if not quiet:
click.echo(err=True)
click.echo("--> Preparing operations...", err=True)
# Number of "steps" to make = number of files * number of hosts
for i, filename in enumerate(operations):
logger.info("Loading: {0}".format(click.style(filename, bold=True)))
state.current_op_file_number = i
load_deploy_file(state, filename)
# Remove any config changes introduced by the deploy file & any includes
config.reset_locked_state()
elif command == "op":
if not quiet:
click.echo(err=True)
click.echo("--> Preparing operation...", err=True)
op, args = operations
args, kwargs = args
kwargs["_allow_cli_mode"] = True
def print_host_ready(host):
logger.info(
"{0}{1} {2}".format(
host.print_prefix,
click.style("Ready:", "green"),
click.style(original_operations[0], bold=True),
),
)
kwargs["_after_host_callback"] = print_host_ready
add_op(state, op, *args, **kwargs)
# Print proposed changes, execute unless --dry, and exit
#
if not quiet:
click.echo(err=True)
click.echo("--> Proposed changes:", err=True)
print_meta(state)
# If --debug-facts or --debug-operations, print and exit
if debug_facts or debug_operations:
if debug_facts:
print_state_facts(state)
if debug_operations:
print_state_operations(state)
_exit()
if dry:
_exit()
if not quiet:
click.echo(err=True)
if not quiet:
click.echo("--> Beginning operation run...", err=True)
run_ops(state, serial=serial, no_wait=no_wait)
if not quiet:
click.echo("--> Results:", err=True)
print_results(state)
_exit()
