def get(self, key): ret = self.cursor.execute('''SELECT value FROM d WHERE key=?''', (key,)).fetchone() if ret is None: raise KeyError() else: return json.loads(ret[0])
def all_statuses(self): if self.is_init(): all_ids = self.redis.smembers('all-pipelines') pipe = self.redis.pipeline() for _id in sorted(all_ids): pipe.get(_id) return [json.loads(sts.decode('ascii')) for sts in pipe.execute()] return []
async def __anext__(self): line = await self.f.readline() if line == b'': raise StopAsyncIteration else: line: str = line.decode('utf8') s = line.find('{') if s >= 0: line = line[s:].strip() try: json.loads(line) return line except json.JSONDecodeError: print('??', line) pass else: print('>>', line)
def __getitem__(self, key): conn = sqlite3.connect(self.filename) cursor = conn.cursor() result = cursor.execute('SELECT _value from d where _key=?', (key, )).fetchone() conn.close() if result is not None: return json.loads(result[0]) return None
async def download(request: web.Request): loop = request.app.loop uuid = request.match_info['id'] writer = None csvf = None fields = None tasks = [] class encoder(): def __init__(self, out): self.out = out def write(self, i): tasks.append(self.out.write(i.encode('utf-8'))) headers = copy(CORS_HEADERS) headers.update({ 'Content-Type': 'text/csv', 'Content-Disposition': 'attachment; filename=dppui.csv' }) resp = web.StreamResponse(status=200, reason='OK', headers=headers) await resp.prepare(request) async with ProcessRunner(loop, uuid, True) as process: print('downloading!', uuid) async for line in LineReader(process.stderr): if line is None: continue line = json.loads(line) if 'e' in line: if line['e'] == 'rs': fields = dict((x['name'], x) for x in line['data']) writer = csv.DictWriter(encoder(resp), [x['name'] for x in line['data']]) writer.writeheader() csvf = CSVFormat() await asyncio.gather(*tasks) tasks = [] await resp.drain() elif line['e'] == 'r' and writer is not None: csvf.write_row(writer, line['data'], fields) # writer.writerow(line['data']) await asyncio.gather(*tasks) tasks = [] await resp.drain() return resp
def get_status(self, pipeline_id): if self.is_init(): status = self.redis.get(pipeline_id) if status is not None: status = json.loads(status.decode('ascii')) return status
def iterate_values(db): for k, v in db: ret = json.loads(v) assert ret is not None yield ret
def get(db, key): v = db.get(key.encode('utf8')) if v is not None: return json.loads(v.decode('ascii'))