def get(self, key):
ret = self.cursor.execute('''SELECT value FROM d WHERE key=?''',
(key,)).fetchone()
if ret is None:
raise KeyError()
else:
return json.loads(ret[0])
def all_statuses(self):
if self.is_init():
all_ids = self.redis.smembers('all-pipelines')
pipe = self.redis.pipeline()
for _id in sorted(all_ids):
pipe.get(_id)
return [json.loads(sts.decode('ascii')) for sts in pipe.execute()]
return []
async def __anext__(self):
line = await self.f.readline()
if line == b'':
raise StopAsyncIteration
else:
line: str = line.decode('utf8')
s = line.find('{')
if s >= 0:
line = line[s:].strip()
try:
json.loads(line)
return line
except json.JSONDecodeError:
print('??', line)
pass
else:
print('>>', line)
def __getitem__(self, key):
conn = sqlite3.connect(self.filename)
cursor = conn.cursor()
result = cursor.execute('SELECT _value from d where _key=?',
(key, )).fetchone()
conn.close()
if result is not None:
return json.loads(result[0])
return None
async def download(request: web.Request):
loop = request.app.loop
uuid = request.match_info['id']
writer = None
csvf = None
fields = None
tasks = []
class encoder():
def __init__(self, out):
self.out = out
def write(self, i):
tasks.append(self.out.write(i.encode('utf-8')))
headers = copy(CORS_HEADERS)
headers.update({
'Content-Type': 'text/csv',
'Content-Disposition': 'attachment; filename=dppui.csv'
})
resp = web.StreamResponse(status=200, reason='OK', headers=headers)
await resp.prepare(request)
async with ProcessRunner(loop, uuid, True) as process:
print('downloading!', uuid)
async for line in LineReader(process.stderr):
if line is None:
continue
line = json.loads(line)
if 'e' in line:
if line['e'] == 'rs':
fields = dict((x['name'], x) for x in line['data'])
writer = csv.DictWriter(encoder(resp),
[x['name'] for x in line['data']])
writer.writeheader()
csvf = CSVFormat()
await asyncio.gather(*tasks)
tasks = []
await resp.drain()
elif line['e'] == 'r' and writer is not None:
csvf.write_row(writer, line['data'], fields)
# writer.writerow(line['data'])
await asyncio.gather(*tasks)
tasks = []
await resp.drain()
return resp
def get_status(self, pipeline_id):
if self.is_init():
status = self.redis.get(pipeline_id)
if status is not None:
status = json.loads(status.decode('ascii'))
return status
def iterate_values(db):
for k, v in db:
ret = json.loads(v)
assert ret is not None
yield ret
def get(db, key):
v = db.get(key.encode('utf8'))
if v is not None:
return json.loads(v.decode('ascii'))