def test_request(self): # MAKE SOME DATA data = { "constant": "this is a test", "random-data": convert.bytes2base64(Random.bytes(100)) } client = Client(settings.url, unwrap(settings.hawk)) # unwrap() DUE TO BUG https://github.com/kumar303/mohawk/issues/21 link, id = client.send(data) Log.note("Success! Located at {{link}} id={{id}}", link=link, id=id) # FILL THE REST OF THE FILE Log.note("Add ing {{num}} more...", num=99-id) for i in range(id + 1, storage.BATCH_SIZE): l, k = client.send(data) if l != link: Log.error("Expecting rest of data to have same link") # TEST LINK HAS DATA raw_content = requests.get(link).content content = convert.zip2bytes(raw_content) for line in convert.utf82unicode(content).split("\n"): data = convert.json2value(line) if data.etl.id == id: Log.note("Data {{id}} found", id=id) break else: Log.error("Expecting to find data at link")
def read(self, key): source = self.get_meta(key) try: json = safe_size(source) except Exception as e: Log.error(READ_ERROR, e) if json == None: return None if source.key.endswith(".zip"): json = _unzip(json) elif source.key.endswith(".gz"): json = convert.zip2bytes(json) return json.decode("utf8")
def read(self, key): source = self.get_meta(key) try: json = safe_size(source) except Exception as e: Log.error(READ_ERROR, e) if json == None: return None if source.key.endswith(".zip"): json = _unzip(json) elif source.key.endswith(".gz"): json = convert.zip2bytes(json) return utf82unicode(json)
def get_json(url, **kwargs): """ ASSUME RESPONSE IN IN JSON """ response = get(url, **kwargs) try: c = response.all_content path = URL(url).path if path.endswith(".zip"): buff = StringIO(c) archive = zipfile.ZipFile(buff, mode='r') c = archive.read(archive.namelist()[0]) elif path.endswith(".gz"): c = convert.zip2bytes(c) return json2value(c.decode('utf8')) except Exception as e: if mo_math.round(response.status_code, decimal=-2) in [400, 500]: Log.error(u"Bad GET response: {{code}}", code=response.status_code) else: Log.error(u"Good GET requests, but bad JSON", cause=e)
def read(self, key): source = self.get_meta(key) try: json = safe_size(source) except Exception, e: Log.error(READ_ERROR, e) if json == None: return None if source.key.endswith(".zip"): json = _unzip(json) elif source.key.endswith(".gz"): json = convert.zip2bytes(json) return convert.utf82unicode(json) def read_bytes(self, key): source = self.get_meta(key) return safe_size(source) def read_lines(self, key): source = self.get_meta(key) if source is None: Log.error("{{key}} does not exist", key=key) if source.size < MAX_STRING_SIZE: if source.key.endswith(".gz"): return LazyLines(ibytes2ilines(scompressed2ibytes(source))) else:
def read(self, key): source = self.get_meta(key) try: json = safe_size(source) except Exception, e: Log.error(READ_ERROR, e) if json == None: return None if source.key.endswith(".zip"): json = _unzip(json) elif source.key.endswith(".gz"): json = convert.zip2bytes(json) return convert.utf82unicode(json) def read_bytes(self, key): source = self.get_meta(key) return safe_size(source) def read_lines(self, key): source = self.get_meta(key) if source is None: Log.error("{{key}} does not exist", key= key) if source.size < MAX_STRING_SIZE: if source.key.endswith(".gz"): return LazyLines(ibytes2ilines(scompressed2ibytes(source))) else: