def test_request(self):
# MAKE SOME DATA
data = {
"constant": "this is a test",
"random-data": convert.bytes2base64(Random.bytes(100))
}
client = Client(settings.url, unwrap(settings.hawk)) # unwrap() DUE TO BUG https://github.com/kumar303/mohawk/issues/21
link, id = client.send(data)
Log.note("Success! Located at {{link}} id={{id}}", link=link, id=id)
# FILL THE REST OF THE FILE
Log.note("Add ing {{num}} more...", num=99-id)
for i in range(id + 1, storage.BATCH_SIZE):
l, k = client.send(data)
if l != link:
Log.error("Expecting rest of data to have same link")
# TEST LINK HAS DATA
raw_content = requests.get(link).content
content = convert.zip2bytes(raw_content)
for line in convert.utf82unicode(content).split("\n"):
data = convert.json2value(line)
if data.etl.id == id:
Log.note("Data {{id}} found", id=id)
break
else:
Log.error("Expecting to find data at link")
def read(self, key):
source = self.get_meta(key)
try:
json = safe_size(source)
except Exception as e:
Log.error(READ_ERROR, e)
if json == None:
return None
if source.key.endswith(".zip"):
json = _unzip(json)
elif source.key.endswith(".gz"):
json = convert.zip2bytes(json)
return json.decode("utf8")
def read(self, key):
source = self.get_meta(key)
try:
json = safe_size(source)
except Exception as e:
Log.error(READ_ERROR, e)
if json == None:
return None
if source.key.endswith(".zip"):
json = _unzip(json)
elif source.key.endswith(".gz"):
json = convert.zip2bytes(json)
return utf82unicode(json)
def get_json(url, **kwargs):
"""
ASSUME RESPONSE IN IN JSON
"""
response = get(url, **kwargs)
try:
c = response.all_content
path = URL(url).path
if path.endswith(".zip"):
buff = StringIO(c)
archive = zipfile.ZipFile(buff, mode='r')
c = archive.read(archive.namelist()[0])
elif path.endswith(".gz"):
c = convert.zip2bytes(c)
return json2value(c.decode('utf8'))
except Exception as e:
if mo_math.round(response.status_code, decimal=-2) in [400, 500]:
Log.error(u"Bad GET response: {{code}}", code=response.status_code)
else:
Log.error(u"Good GET requests, but bad JSON", cause=e)
def read(self, key):
source = self.get_meta(key)
try:
json = safe_size(source)
except Exception, e:
Log.error(READ_ERROR, e)
if json == None:
return None
if source.key.endswith(".zip"):
json = _unzip(json)
elif source.key.endswith(".gz"):
json = convert.zip2bytes(json)
return convert.utf82unicode(json)
def read_bytes(self, key):
source = self.get_meta(key)
return safe_size(source)
def read_lines(self, key):
source = self.get_meta(key)
if source is None:
Log.error("{{key}} does not exist", key=key)
if source.size < MAX_STRING_SIZE:
if source.key.endswith(".gz"):
return LazyLines(ibytes2ilines(scompressed2ibytes(source)))
else:
def read(self, key):
source = self.get_meta(key)
try:
json = safe_size(source)
except Exception, e:
Log.error(READ_ERROR, e)
if json == None:
return None
if source.key.endswith(".zip"):
json = _unzip(json)
elif source.key.endswith(".gz"):
json = convert.zip2bytes(json)
return convert.utf82unicode(json)
def read_bytes(self, key):
source = self.get_meta(key)
return safe_size(source)
def read_lines(self, key):
source = self.get_meta(key)
if source is None:
Log.error("{{key}} does not exist", key= key)
if source.size < MAX_STRING_SIZE:
if source.key.endswith(".gz"):
return LazyLines(ibytes2ilines(scompressed2ibytes(source)))
else:
