/
basics.py
192 lines (133 loc) · 5.13 KB
/
basics.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
#!/usr/bin/env ipython
#!/usr/bin/env python
# coding: utf-8
# In[94]:
#get_ipython().magic(u'load_ext autoreload')
#get_ipython().magic(u'autoreload 2')
#get_ipython().magic(u'alias nbconvert nbconvert basics.ipynb')
# In[95]:
#get_ipython().magic(u'nbconvert')
# In[75]:
from pathlib import Path
import logging
import logging.config
import configuration
import os
# In[40]:
def resource_path(relative_path):
'''
Get the absolute path for a given relative path
'''
return Path('.').absolute() / Path(relative_path)
# In[46]:
def fileToList(inputfile, stripWhitespace=True):
'''
Creates a list from a text file and optionally strips out whitespace
this is useful for converting a text file into a list for printingd
'''
logger = logging.getLogger(__name__)
logger.debug('inputfile = {}'.format(inputfile))
# super elegant solution as seen below
# https://stackoverflow.com/questions/4842057/easiest-way-to-ignore-blank-lines-when-reading-a-file-in-python
try:
with open(inputfile, 'r') as fhandle:
if stripWhitespace:
lines = [_f for _f in (line.strip() for line in fhandle) if _f]
else:
lines = [line.strip() for line in fhandle]
except IOError as e:
logger.debug(e)
return(lines)
# In[61]:
def setup_logging(
default_config=None,
default_level=logging.INFO,
output_path='~/',
env_key='LOG_CFG'):
"""Setup logging configuration
borrowed from: https://fangpenlin.com/posts/2012/08/26/good-logging-practice-in-python/
uses logging configuration json file
"""
path = default_config
value = os.getenv(env_key, None)
config = None
if value:
path = value
if os.path.exists(path):
try:
with open(path, 'rt') as f:
config = json.load(f)
except Exception as e:
print('failed to read logging configuration')
return(None)
# set the specific path to store log files
if config:
if output_path:
for handler in config['handlers']:
if 'filename' in config['handlers'][handler]:
logFile = os.path.basename(config['handlers'][handler]['filename'])
logPath = os.path.expanduser(output_path+'/')
if not os.path.isdir(logPath):
try:
os.makedirs(logPath)
except OSError as e:
logging.error('failed to make log file directory: {}'.format(e))
logging.error('using {} instead'.format(config['handlers'][handler]['filename']))
break
config['handlers'][handler]['filename'] = logPath+logFile
logging.config.dictConfig(config)
logging.getLogger().setLevel(default_level)
return(config)
else:
return(None)
else:
logging.basicConfig(level=default_level)
return(None)
# In[90]:
def getConfiguration(cfgfile=None, config_required={'Main': {'key1': 'value1', 'key2': 'value2'}}):
'''
read an ini configuration file and return a dictionary of key/value pairs
update configuration file if missing any sections
accepts:
cfgfile - path to configuration file
config_required - nested dictionary in the following format:
{'Section1':
{'key1': 'value1', 'key2': 'value2'},
'Section 2':
{'key1': 'value1'}
}
'''
if not cfgfile:
raise ValueError('no configuration file specified')
# required configuraiton options
# Section: {'option': 'default value'}
logger = logging.getLogger(__name__)
logger.debug('getting configuration from file: {}'.format(cfgfile))
cfgpath = os.path.dirname(cfgfile)
# config_required = {
# 'Main': {'credentials': os.path.join(cfgpath, 'credentials/'),
# },
# }
config = configuration.get_config(cfgfile)
update_config = False
logger.debug('checking sections')
for section, values in list(config_required.items()):
if not config.has_section(section):
logger.warning('section: {} not found in {}'.format(section, cfgfile))
logger.debug('adding section {}'.format(section))
config.add_section(section)
update_config = True
for option, value in list(values.items()):
if not config.has_option(section, option):
logger.warning('option: {} not found in {}'.format(option, cfgfile))
logger.debug('adding option {}: {}'.format(option, value))
config.set(section, option, value)
update_config = True
# for section, options in config_required.items():
if update_config:
try:
logger.debug('updating configuration file at: {}'.format(cfgfile))
configuration.create_config(cfgfile, config)
except Exception as e:
logger.error(e)
return(config)