forked from bitcoinembassy/morphis
-
Notifications
You must be signed in to change notification settings - Fork 0
/
client_engine.py
347 lines (270 loc) · 11.1 KB
/
client_engine.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
import llog
import asyncio
import logging
import os
import time
from sqlalchemy.orm import joinedload
import base58
import chord
from db import DmailAddress
import dhgroup14
import dmail
import mbase32
import multipart
import rsakey
import sshtype
log = logging.getLogger(__name__)
class ClientEngine(object):
def __init__(self, engine, db):
assert type(engine) is chord.ChordEngine
self.engine = engine
self.db = db
self.loop = engine.loop
self.latest_version_number = None
self.latest_version_data = None
self.auto_publish_enabled = True
self.auto_scan_enabled = True
self.csrf_token = base58.encode(os.urandom(64))
self._dmail_engine = None
self._running = False
self._data_key =\
mbase32.decode("sp1nara3xhndtgswh7fznt414we4mi3y6kdwbkz4jmt8ocb6x"\
"4w1faqjotjkcrefta11swe3h53dt6oru3r13t667pr7cpe3ocxeuma")
self._path = b"latest_version"
self._dmail_autoscan_processes = {}
@property
def update_test(self):
raise Exception()
@update_test.setter
def update_test(self, value):
if value:
self._path = b"test_version"
@asyncio.coroutine
def start(self):
if self._running:
return
self._running = True
if not self._dmail_engine:
self._dmail_engine = dmail.DmailEngine(self.engine.tasks, self.db)
asyncio.async(self._start_version_poller(), loop=self.loop)
if self.auto_scan_enabled:
asyncio.async(self._start_dmail_autoscan(), loop=self.loop)
if self.auto_publish_enabled:
asyncio.async(self._start_dmail_auto_publish(), loop=self.loop)
@asyncio.coroutine
def stop(self):
if self._running:
self._running = False
for processor in self._dmail_autoscan_processes.values():
processor.stop()
@asyncio.coroutine
def _start_version_poller(self):
yield from self.engine.protocol_ready.wait()
while self._running:
data_rw = multipart.BufferingDataCallback()
r =\
yield from\
multipart.get_data(self.engine, self._data_key,\
data_callback=data_rw, path=self._path)
if data_rw.data:
if data_rw.version:
data = data_rw.data.decode()
p0 = data.find('<span id="version_number">')
p0 += 26
p1 = data.find("</span>", p0)
self.latest_version_number = data[p0:p1]
self.latest_version_data = data
if log.isEnabledFor(logging.INFO):
log.info("Found latest_version_number=[{}]"\
" (data_rw.version=[{}])."\
.format(\
self.latest_version_number,\
data_rw.version))
else:
if log.isEnabledFor(logging.INFO):
log.info("Found invalid latest_version record:"\
" data_rw.version=[{}], len(data)=[{}]."\
.format(data_rw.version, len(data_rw.data)))
delay = 5*60
else:
log.info("Couldn't find latest_version in network.")
delay = 60
yield from asyncio.sleep(delay, loop=self.loop)
@asyncio.coroutine
def _start_dmail_auto_publish(self):
yield from self.engine.protocol_ready.wait()
def dbcall():
with self.db.open_session(True) as sess:
q = sess.query(DmailAddress)\
.options(joinedload("keys"))
r = q.all()
sess.expunge_all()
return r
while self._running:
addrs = yield from self.loop.run_in_executor(None, dbcall)
for addr in addrs:
yield from self._dmail_auto_publish(addr)
log.info("Finished auto-publish scan, sleeping for now.")
yield from asyncio.sleep(60 * 60 * 24, loop=self.loop)
@asyncio.coroutine
def _dmail_auto_publish(self, dmail_address):
data_rw = yield from self.engine.tasks.send_get_data(\
dmail_address.site_key, retry_factor=100)
if data_rw.data:
if log.isEnabledFor(logging.DEBUG):
log.debug("Succeeded in fetching dmail site [{}]; won't"\
" auto-publish."\
.format(mbase32.encode(dmail_address.site_key)))
return
if log.isEnabledFor(logging.INFO):
log.info("Failed to fetch dmail site [{}]; republishing."\
.format(mbase32.encode(dmail_address.site_key)))
private_key = rsakey.RsaKey(privdata=dmail_address.site_privatekey)
dh = dhgroup14.DhGroup14()
dh.x = sshtype.parseMpint(dmail_address.keys[0].x)[1]
dh.generate_e()
dms = dmail.DmailSite()
root = dms.root
root["ssm"] = "mdh-v1"
root["sse"] = base58.encode(sshtype.encodeMpint(dh.e))
root["target"] =\
mbase32.encode(dmail_address.keys[0].target_key)
root["difficulty"] = int(dmail_address.keys[0].difficulty)
storing_nodes =\
yield from self._dmail_engine.publish_dmail_site(private_key, dms)
if log.isEnabledFor(logging.INFO):
log.info("Republished Dmail site with [{}] storing nodes."\
.format(storing_nodes))
@asyncio.coroutine
def _start_dmail_autoscan(self):
yield from self.engine.protocol_ready.wait()
def dbcall():
with self.db.open_session() as sess:
q = sess.query(DmailAddress)\
.options(joinedload("keys"))\
.filter(DmailAddress.scan_interval > 0)
r = q.all()
sess.expunge_all()
return r
addrs = yield from self.loop.run_in_executor(None, dbcall)
for addr in addrs:
self.update_dmail_autoscan(addr)
def update_dmail_autoscan(self, addr):
if not self.auto_scan_enabled:
return
if log.isEnabledFor(logging.INFO):
log.info(\
"Starting/Updating autoscan (scan_interval=[{}]) process for"\
" DmailAddress (id=[{}])."\
.format(addr.scan_interval, addr.id))
process = self._dmail_autoscan_processes.get(addr.id)
if not addr.scan_interval:
if process:
process.stop()
del self._dmail_autoscan_processes[addr.id]
else:
return
if process:
process.update_scan_interval(addr.scan_interval)
else:
process = DmailAutoscanProcess(self, addr, addr.scan_interval)
asyncio.async(process.run(), loop=self.loop)
self._dmail_autoscan_processes[addr.id] = process
def trigger_dmail_scan(self, addr):
if log.isEnabledFor(logging.INFO):
log.info("Ensuring scan of DmailAddress (id=[{}]) now."\
.format(addr.id))
process = self._dmail_autoscan_processes.get(addr.id)
if process:
process.scan_now()
else:
process = DmailAutoscanProcess(self, addr, 0)
asyncio.async(process.run(), loop=self.loop)
self._dmail_autoscan_processes[addr.id] = process
class DmailAutoscanProcess(object):
def __init__(self, client_engine, addr, interval):
self.client_engine = client_engine
self.loop = client_engine.loop
self.dmail_address = addr
self.scan_interval = interval
self._running = False
self._task = None
self._scan_now = False
def scan_now(self):
if self._task:
self._scan_now = True
self._task.cancel()
else:
if self._running:
log.info("Already scanning.")
return
asyncio.async(self.run(), loop=self.loop)
def update_scan_interval(self, interval):
if not interval:
self._running = False
if self._task:
self._task.cancel()
return
self.scan_interval = interval
if self._running:
if log.isEnabledFor(logging.INFO):
log.info("Notifying DmailAutoscanProcess (addr=[{}]) of"\
" interval change."\
.format(mbase32.encode(self.dmail_address.site_key)))
if self._task:
self._task.cancel()
else:
if log.isEnabledFor(logging.INFO):
log.info("Starting DmailAutoscanProcess (addr=[{}])."\
.format(mbase32.encode(self.dmail_address.site_key)))
asyncio.async(self.run(), loop=self.loop)
@asyncio.coroutine
def run(self):
self._running = True
if log.isEnabledFor(logging.INFO):
addr_enc = mbase32.encode(self.dmail_address.site_key)
log.info("DmailAutoscanProcess (addr=[{}]) running."\
.format(addr_enc))
while self._running:
new_cnt, old_cnt, err_cnt = yield from\
self.client_engine._dmail_engine.scan_and_save_new_dmails(\
self.dmail_address)
if log.isEnabledFor(logging.INFO):
log.info("Finished scanning Dmails for address [{}];"\
" new_cnt=[{}], old_cnt=[{}], err_cnt=[{}]."\
.format(addr_enc, new_cnt, old_cnt, err_cnt))
if not self.scan_interval:
self._running = False
if not self._running:
break
time_left = self.scan_interval
start = time.time()
while time_left > 0:
if log.isEnabledFor(logging.INFO):
log.info("Sleeping for [{}] seconds.".format(time_left))
self._task =\
asyncio.async(\
asyncio.sleep(time_left, loop=self.loop),\
loop=self.loop)
try:
yield from self._task
self._task = None
break
except asyncio.CancelledError:
self._task = None
if log.isEnabledFor(logging.INFO):
log.info("Woken from sleep for address [{}]."\
.format(\
mbase32.encode(self.dmail_address.site_key)))
if self._scan_now:
self._scan_now = False
break
time_left = self.scan_interval - (time.time() - start)
def stop(self):
if self._running:
if log.isEnabledFor(logging.INFO):
log.info("Stopping DmailAutoscanProcess (addr=[{}])."\
.format(mbase32.encode(self.dmail_address.site_key)))
self._running = False
if self._task:
self._task.cancel()