-
Notifications
You must be signed in to change notification settings - Fork 0
/
common.py
executable file
·312 lines (254 loc) · 9.13 KB
/
common.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
import sys
import os
import subprocess
import argparse
import datetime
import warnings
import re
import unittest
import email.utils
import time
import os
import shlex
tracelevel = 4
log_handle = None
def trace(level, *args):
def fix_linendings(s: str) -> str:
if os.linesep == '\n':
return s
p = 0
while True:
p = s.find('\n', p+1)
if p<0: break
if p>0 and s[p-1] != '\r':
s = s[:p] + '\r' + s[p:]
return s
def mystr(thing):
if isinstance(thing, (list, tuple)):
msg = []
prefix = ''
if len(thing) <= 4:
separator = ', '
else:
separator = (os.linesep+" ")
prefix = separator
for s in thing:
msg += [mystr(s)]
return prefix + separator.join(msg)
elif isinstance(thing, datetime.datetime):
return thing.strftime("%Y-%m-%d %H:%M:%S")
elif isinstance(thing, datetime.date):
return thing.strftime("%Y-%m-%d")
#elif isinstance(thing, bytes):
# return bytes.decode('utf-8')
else:
try:
if isinstance(thing, bytes):
s = thing.decode('utf-8')
else:
s = str(thing)
s = fix_linendings(s)
return s
except UnicodeEncodeError:
return str(thing).encode('ascii', 'ignore')
except Exception as ex:
return 'Failed to format thing as string caught ' + str(ex)
#if tracelevel < level:
# return
msg = datetime.datetime.now().strftime("%H:%M:%S: ")
for thing in args:
msg += mystr(thing)
msg = msg.rstrip()
handle = sys.stderr if log_handle is None else log_handle
try:
print(msg, file=handle)
except UnicodeEncodeError:
print(msg.encode('cp850', errors='replace'), file=handle)
def pretty(value,htchar="\t",lfchar="\n",indent=0):
if type(value) in [dict]:
return "{%s%s%s}"%(",".join(["%s%s%s: %s"%(lfchar,htchar*(indent+1),repr(key),pretty(value[key],htchar,lfchar,indent+1))for key in value]),lfchar,(htchar*indent))
elif type(value) in [list,tuple]:
return (type(value)is list and"[%s%s%s]"or"(%s%s%s)")%(",".join(["%s%s%s"%(lfchar,htchar*(indent+1),pretty(item,htchar,lfchar,indent+1))for item in value]),lfchar,(htchar*indent))
else:
return repr(value)
def run_child_process(cmd, alt_path=None, get_stdout=True, get_stderr=True):
"""Start a child-process. Run it passing stdout-data to the main stdout."""
trace(6, 'Calling ', cmd, " alt_path:", alt_path)
if isinstance(cmd, str):
cmd = shlex.split(cmd)
try:
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout_data, stderr_data = p.communicate()
except OSError as e:
if e.errno != 2 or alt_path is None:
raise e
cmd[0] = os.path.join(alt_path, os.path.basename(cmd[0]))
return run_child_process(cmd, None)
if p.returncode == 0:
trace(6, 'process terminated successfully')
else:
trace(4, 'process failed ' + str(p.returncode) + "\r\n" + stdout_data + stderr_data)
trace(7, 'stdout:\r\n' + stdout_data)
trace(7, 'stderr:\r\n' + stderr_data)
data = ''
if get_stdout: data += stdout_data
if get_stderr: data += stderr_data
return (p.returncode, data)
with warnings.catch_warnings():
warnings.simplefilter("ignore")
swe_weekdays = {
'måndag':0, 'mån':0, 'måndag':0, 'mån':0,
'tisdag':1, 'tis':1,
'onsdag':2, 'ons':2,
'torsdag':3, 'tor':3,
'fredag':4, 'fre':4,
'l#246;ndag':5, 'l#214;rdag':5, 'lördag':5, 'lör':5, 'l#246;n':5, 'l#214;r':5,
's#246;ndag':6, 's#214;ndag':6, 'söndag':6, 's#246;n':6, 's#214;n':6, 'sön':6,
}
# make sure januari is month 1
swe_months = [
('januari',1),
('februari',2),
('mars',3),
('april',4),
('maj',5),
('juni',6),
('juli',7),
('augusti',8),
('september',9),
('oktober',10),
('november',11),
('december',12),
]
def is_swe_month(x: str):
return parse_swe_month(x) >= 0
def is_swe_weekday(x: str):
return parse_swe_weekday(x) >= 0
def parse_swe_month(x: str):
x = x.lower()
for key, value in swe_months:
if key.startswith(x) :
return value
return -1
def parse_swe_weekday(x: str):
return swe_weekdays.get(x.lower(), -1)
def unescape_html(html: str):
res = ''
last = 0
while True:
pos = html.find('&', last)
if pos < 0:
res += html[last:]
break
res += html[last:pos]
sc = html.find(';', pos)
code = html[pos+1: sc]
# print 'code: ' + code
if code == '#229' or code == 'aring':
res += 'a'
elif code == '#197' or code.lower() == 'aring' and code[0].isupper():
res += 'A'
elif code == '#228' or code == 'auml':
res += 'a'
elif code == '#196' or code.lower() == 'auml' and code[0].isupper():
res += 'A'
elif code == '#246' or code == 'ouml':
res += 'o'
elif code == '#214' or code.lower() == 'ouml' and code[0].isupper():
res += 'O'
elif code == '#233': # é
res += 'e'
elif code == '#225': # á
res += 'a'
elif code == '#237': # í
res += 'i'
elif code == '#39': # '
res += ' '
elif code == '#233': # é
res += 'e'
else:
res += '_'
last = sc+1
return res.replace(':', ' ').replace(' ', ' ').replace(' ', ' ').replace(' ', ' ').replace(' ', ' ')
def combine_http_path(x, y):
return x.rstrip('/') + '/' + y.lstrip('/')
"""
Parses a datetime like 2000-01-01T23:45:00
Timezone is ignored
2016-05-25T09:08:00+02:00
"""
def parse_datetime(s: str):
def parse_without_timezone(s):
try:
if len(s) > 9 and ( s[8] == 'T' or s[10] == 'T'):
return datetime.datetime.strptime(s, "%Y-%m-%dT%H:%M:%S")
except ValueError:
pass
return datetime.datetime.strptime(s, "%Y-%m-%d %H:%M:%S")
trace(9, 'parse_datetime(' + s + ')')
if re.match(r'.*[\+\-]\d\d\:\d\d$', s):
raise ValueError('common.parse_datetime doesnt support timezones')
return parse_without_timezone(s)
def parse_datetime_to_rfc822(s: str):
if re.match(r'.*[\+\-]\d\d\:\d\d$', s):
tz = s[-6:-3] + s[-2:] # timezone, without colon
s = s[:-6]
elif s[-1] == 'Z': # "2011-08-12T20:17:46Z" is Zulu-time
tz = 'GMT'
s = s[:-1]
else:
tz = 'GMT'
dt = parse_datetime(s)
#return dt
rfc822 = email.utils.format_datetime(dt)
rfc822 = rfc822[:-5] + tz # replace -0000 with timezone
trace(9, 'parse_datetime_to_rfc822("' + s + '") -> ' + str(rfc822))
return rfc822
def format_datetime(dt: datetime):
return dt.strftime('%Y-%m-%dT%H:%M:%S%z')
def get_el(root, name, ns=None):
if ns is None:
key = name
else:
if ns[0]!='{': ns = '{' + ns + '}'
key = ns + name
for child in root:
if child.tag == key:
return child
return None
def is_none_or_empty(s):
return s is None or isinstance(s, str) and s == ""
if __name__ == '__main__':
print(sys.argv)
print('hello world')
time.sleep(4)
tracelevel = 8
class TestCommon(unittest.TestCase):
def test_parse_datetime(self):
trace(4, 'test_parse_datetime testing...')
t = parse_datetime('2000-01-01T23:45:00')
self.assertEqual(datetime.datetime(2000,1,1,23,45,0), t)
t = parse_datetime('2000-01-01 23:45:00')
self.assertEqual(datetime.datetime(2000,1,1,23,45,0), t)
#t = parse_datetime('2000-01-01T23:45:00+02:00')
#self.assertEqual(datetime.datetime(2000,1,1,23,45,0), t)
#t = parse_datetime('2000-01-01 23:45:00+02:00')
#self.assertEqual(datetime.datetime(2000,1,1,23,45,0), t)
trace(4, 'test_parse_datetime passed')
def test_parse_datetime_rfc822(self):
trace(4, 'test_parse_datetime_rfc822 testing...')
t = parse_datetime_to_rfc822('2000-01-01T23:45:00')
self.assertEqual('Sat, 01 Jan 2000 23:45:00 GMT', t)
t = parse_datetime_to_rfc822('2000-01-01 23:45:00')
self.assertEqual('Sat, 01 Jan 2000 23:45:00 GMT', t)
t = parse_datetime_to_rfc822('2000-01-01T23:45:00+02:00')
self.assertEqual('Sat, 01 Jan 2000 23:45:00 +0200', t)
t = parse_datetime_to_rfc822('2000-01-01 23:45:00+02:00')
self.assertEqual('Sat, 01 Jan 2000 23:45:00 +0200', t)
t = parse_datetime_to_rfc822('2016-09-17T12:47:41Z')
self.assertEqual('Sat, 17 Sep 2016 12:47:41 GMT', t)
trace(4, 'test_parse_datetime_rfc822 passed')
pass
trace(4, 'Running common-unitttests')
sys.exit(unittest.main(argv=['-v']))