def test_request(self):
        # MAKE SOME DATA
        data = {
            "constant": "this is a test",
            "random-data": convert.bytes2base64(Random.bytes(100))
        }

        client = Client(settings.url, unwrap(settings.hawk))  # unwrap() DUE TO BUG https://github.com/kumar303/mohawk/issues/21
        link, id = client.send(data)
        Log.note("Success!  Located at {{link}} id={{id}}", link=link, id=id)

        # FILL THE REST OF THE FILE
        Log.note("Add ing {{num}} more...", num=99-id)
        for i in range(id + 1, storage.BATCH_SIZE):
            l, k = client.send(data)
            if l != link:
                Log.error("Expecting rest of data to have same link")

        # TEST LINK HAS DATA
        raw_content = requests.get(link).content
        content = convert.zip2bytes(raw_content)
        for line in convert.utf82unicode(content).split("\n"):
            data = convert.json2value(line)
            if data.etl.id == id:
                Log.note("Data {{id}} found", id=id)
                break
        else:
            Log.error("Expecting to find data at link")
    def test_public_request(self):
        # MAKE SOME DATA
        data = {
            "a": {  # MATCHES SERVER PATTERN
                "b": "good",
                "c": [
                    {"k": "good", "m": 1},
                    {"k": 2, "m": 2}
                ]
            },
            "constant": "this is a test",
            "random-data": convert.bytes2base64(Random.bytes(100))
        }

        content = json.dumps(data)

        response = requests.post(
            url=settings.url,
            data=content,
            headers={
                'Content-Type': CONTENT_TYPE
            }
        )

        self.assertEqual(response.status_code, 200, "Expecting 200")

        about = json.loads(response.content)
        return about['link'], about['etl']['id']

        Log.note("Data located at {{link}} id={{id}}", link=link, id=id)
    def test_public_request_too_big(self):
        # MAKE SOME DATA
        data = {
            "a": {  # MATCHES SERVER PATTERN
                "b": "good",
                "c": [
                    {"k": "good", "m": 1},
                    {"k": 2, "m": 2}
                ]
            },
            "constant": "this is a test",
            "random-data": convert.bytes2base64(Random.bytes(500))
        }

        content = json.dumps(data)

        def poster():
            response = requests.post(
                url=settings.url,
                data=content,
                headers={
                    'Content-Type': CONTENT_TYPE
                }
            )

            self.assertEqual(response.status_code, 200, "Expecting 200")

        self.assertRaises(Exception, poster)
Esempio n. 4
0
def _deep_json_to_string(value, depth):
    """
    :param value: SOME STRUCTURE
    :param depth: THE MAX DEPTH OF PROPERTIES, DEEPER WILL BE STRING-IFIED
    :return: FLATTER STRUCTURE
    """
    if is_data(value):
        if depth == 0:
            return strings.limit(value2json(value), LOG_STRING_LENGTH)

        return {
            k: _deep_json_to_string(v, depth - 1)
            for k, v in value.items()
        }
    elif is_sequence(value):
        return strings.limit(value2json(value), LOG_STRING_LENGTH)
    elif isinstance(value, number_types):
        return value
    elif is_text(value):
        return strings.limit(value, LOG_STRING_LENGTH)
    elif is_binary(value):
        return strings.limit(bytes2base64(value), LOG_STRING_LENGTH)
    elif isinstance(value, (date, datetime)):
        return datetime2unix(value)
    else:
        return strings.limit(value2json(value), LOG_STRING_LENGTH)
    def test_missing_auth(self):
        # MAKE SOME DATA
        data = {
            "constant": "this is a test",
            "random-data": convert.bytes2base64(Random.bytes(100))
        }

        response = requests.post(settings.bad_url, data=convert.unicode2utf8(convert.value2json(data)))
        self.assertEqual(response.status_code, 403)
Esempio n. 6
0
    def save(self, query):
        query.meta = None
        json = convert.value2json(query)
        hash = convert.unicode2utf8(json)

        # TRY MANY HASHES AT ONCE
        hashes = [None] * HASH_BLOCK_SIZE
        for i in range(HASH_BLOCK_SIZE):
            hash = hashlib.sha1(hash).digest()
            hashes[i] = hash

        short_hashes = [
            convert.bytes2base64(h[0:6]).replace("/", "_") for h in hashes
        ]
        available = {h: True for h in short_hashes}

        existing = self.es.query({
            "from": {
                "type": "elasticsearch",
                "settings": self.es.settings
            },
            "where": {
                "terms": {
                    "hash": short_hashes
                }
            },
            "meta": {
                "timeout": "2second"
            }
        })

        for e in Cube(select=existing.select,
                      edges=existing.edges,
                      data=existing.data).values():
            if e.query == json:
                return e.hash
            available[e.hash] = False

        # THIS WILL THROW AN ERROR IF THERE ARE NONE, HOW UNLUCKY!
        best = [h for h in short_hashes if available[h]][0]

        self.queue.add({
            "id": best,
            "value": {
                "hash": best,
                "create_time": Date.now(),
                "last_used": Date.now(),
                "query": json
            }
        })

        Log.note("Saved query as {{hash}}", hash=best)

        return best
Esempio n. 7
0
def encrypt(text, _key, salt=None):
    """
    RETURN JSON OF ENCRYPTED DATA   {"salt":s, "length":l, "data":d}
    """
    from pyLibrary.queries import jx

    if not isinstance(text, unicode):
        Log.error("only unicode is encrypted")
    if _key is None:
        Log.error("Expecting a key")
    if isinstance(_key, str):
        _key = bytearray(_key)
    if salt is None:
        salt = Random.bytes(16)

    data = bytearray(text.encode("utf8"))

    # Initialize encryption using key and iv
    key_expander_256 = key_expander.KeyExpander(256)
    expanded_key = key_expander_256.expand(_key)
    aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
    aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
    aes_cbc_256.set_iv(salt)

    output = Data()
    output.type = "AES256"
    output.salt = convert.bytes2base64(salt)
    output.length = len(data)

    encrypted = bytearray()
    for _, d in jx.groupby(data, size=16):
        encrypted.extend(aes_cbc_256.encrypt_block(d))
    output.data = convert.bytes2base64(encrypted)
    json = convert.value2json(output)

    if DEBUG:
        test = decrypt(json, _key)
        if test != text:
            Log.error("problem with encryption")

    return json
Esempio n. 8
0
def encrypt(text, _key, salt=None):
    """
    RETURN JSON OF ENCRYPTED DATA   {"salt":s, "length":l, "data":d}
    """
    from pyLibrary.queries import jx

    if not isinstance(text, unicode):
        Log.error("only unicode is encrypted")
    if _key is None:
        Log.error("Expecting a key")
    if isinstance(_key, str):
        _key = bytearray(_key)
    if salt is None:
        salt = Random.bytes(16)

    data = bytearray(text.encode("utf8"))

    # Initialize encryption using key and iv
    key_expander_256 = key_expander.KeyExpander(256)
    expanded_key = key_expander_256.expand(_key)
    aes_cipher_256 = aes_cipher.AESCipher(expanded_key)
    aes_cbc_256 = cbc_mode.CBCMode(aes_cipher_256, 16)
    aes_cbc_256.set_iv(salt)

    output = Dict()
    output.type = "AES256"
    output.salt = convert.bytes2base64(salt)
    output.length = len(data)

    encrypted = bytearray()
    for _, d in jx.groupby(data, size=16):
        encrypted.extend(aes_cbc_256.encrypt_block(d))
    output.data = convert.bytes2base64(encrypted)
    json = convert.value2json(output)

    if DEBUG:
        test = decrypt(json, _key)
        if test != text:
            Log.error("problem with encryption")

    return json
Esempio n. 9
0
    def test_save_then_load(self):

        test = {
            "data": [{
                "a": "b"
            }],
            "query": {
                "meta": {
                    "save": True
                },
                "from": TEST_TABLE,
                "select": "a"
            },
            "expecting_list": {
                "meta": {
                    "format": "list"
                },
                "data": ["b"]
            }
        }

        settings = self.utils.fill_container(test)

        bytes = unicode2utf8(
            value2json({
                "from": settings.index,
                "select": "a",
                "format": "list"
            }))
        expected_hash = convert.bytes2base64(
            hashlib.sha1(bytes).digest()[0:6]).replace("/", "_")
        wrap(test).expecting_list.meta.saved_as = expected_hash

        self.utils.send_queries(test)

        # ENSURE THE QUERY HAS BEEN INDEXED
        Log.note("Flush saved query (with hash {{hash}})", hash=expected_hash)
        container = elasticsearch.Index(index="saved_queries",
                                        type=save_query.DATA_TYPE,
                                        kwargs=settings)
        container.flush(forced=True)
        with Timer("wait for 5 seconds"):
            Till(seconds=5).wait()

        url = URL(self.utils.testing.query)
        response = self.utils.try_till_response(url.scheme + "://" + url.host +
                                                ":" + text_type(url.port) +
                                                "/find/" + expected_hash,
                                                data=b'')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.all_content, bytes)
Esempio n. 10
0
    def test_save_then_load(self):

        test = {
            "data": [
                {"a": "b"}
            ],
            "query": {
                "meta": {"save": True},
                "from": TEST_TABLE,
                "select": "a"
            },
            "expecting_list": {
                "meta": {
                    "format": "list"
                },
                "data": ["b"]
            }
        }

        settings = self.utils.fill_container(test)

        bytes = convert.unicode2utf8(convert.value2json({
            "from": settings.index,
            "select": "a",
            "format": "list"
        }))
        expected_hash = convert.bytes2base64(hashlib.sha1(bytes).digest()[0:6]).replace("/", "_")
        wrap(test).expecting_list.meta.saved_as = expected_hash

        self.utils.send_queries(test)

        #ENSURE THE QUERY HAS BEEN INDEXED
        container = elasticsearch.Index(index="saved_queries", settings=settings)
        container.flush()
        Thread.sleep(seconds=5)

        url = URL(self.utils.service_url)

        response = self.utils.try_till_response(url.scheme+"://"+url.host+":"+unicode(url.port)+"/find/"+expected_hash, data=b'')

        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.all_content, bytes)
Esempio n. 11
0
    def test_recovery_of_empty_string(self):
        test = wrap({
            "data": [
                {"a": "bee"}
            ],
            "query": {
                "from": TEST_TABLE,
                "select": "a",
                "where": {"prefix": {"a": ""}},
                "format": "list"
            },
            "expecting_list": {
                "meta": {
                    "format": "list"
                },
                "data": ["bee"]
            }
        })

        settings = self.utils.fill_container(test)

        bytes = value2json(test.query).encode('utf8')
        expected_hash = convert.bytes2base64(hashlib.sha1(bytes).digest()[0:6]).replace("/", "_")
        test.expecting_list.meta.saved_as = expected_hash

        test.query.meta = {"save": True}
        self.utils.send_queries(test)

        # ENSURE THE QUERY HAS BEEN INDEXED
        Log.note("Flush saved query")
        container = elasticsearch.Index(index="saved_queries", kwargs=settings)
        container.flush(forced=True)
        with Timer("wait for 5 seconds"):
            Till(seconds=5).wait()

        url = URL(self.utils.testing.query)
        response = self.utils.try_till_response(url.scheme + "://" + url.host + ":" + text(url.port) + "/find/" + expected_hash, data=b'')
        self.assertEqual(response.status_code, 200)
        self.assertEqual(response.all_content, bytes)
def _deep_json_to_string(value, depth):
    """
    :param value: SOME STRUCTURE
    :param depth: THE MAX DEPTH OF PROPERTIES, DEEPER WILL BE STRING-IFIED
    :return: FLATTER STRUCTURE
    """
    if is_data(value):
        if depth == 0:
            return strings.limit(value2json(value), LOG_STRING_LENGTH)

        return {k: _deep_json_to_string(v, depth - 1) for k, v in value.items()}
    elif is_sequence(value):
        return strings.limit(value2json(value), LOG_STRING_LENGTH)
    elif isinstance(value, number_types):
        return value
    elif is_text(value):
        return strings.limit(value, LOG_STRING_LENGTH)
    elif is_binary(value):
        return strings.limit(bytes2base64(value), LOG_STRING_LENGTH)
    elif isinstance(value, (date, datetime)):
        return datetime2unix(value)
    else:
        return strings.limit(value2json(value), LOG_STRING_LENGTH)
Esempio n. 13
0
def _deep_json_to_string(value, depth):
    """
    :param value: SOME STRUCTURE
    :param depth: THE MAX DEPTH OF PROPERTIES, DEEPER WILL BE STRING-IFIED
    :return: FLATTER STRUCTURE
    """
    if isinstance(value, Mapping):
        if depth == 0:
            return strings.limit(value2json(value), LOG_STRING_LENGTH)

        return {
            k: _deep_json_to_string(v, depth - 1)
            for k, v in value.items()
        }
    elif isinstance(value, (list, FlatList)):
        return strings.limit(value2json(value), LOG_STRING_LENGTH)
    elif isinstance(value, number_types):
        return value
    elif isinstance(value, text_type):
        return strings.limit(value, LOG_STRING_LENGTH)
    elif isinstance(value, binary_type):
        return strings.limit(bytes2base64(value), LOG_STRING_LENGTH)
    else:
        return strings.limit(value2json(value), LOG_STRING_LENGTH)
import requests

from pyLibrary import convert, jsons
from pyLibrary.debugs.logs import Log
from pyLibrary.dot import unwrap
from pyLibrary.maths.randoms import Random
from modatasubmission import Client


settings = jsons.ref.get("file://~/MoDataSubmissionClient.json")


data={
    "constant": "this is a test",
    "random-data": convert.bytes2base64(Random.bytes(100))
}
link, id = Client(settings.url, unwrap(settings.hawk)).send(data)
Log.note("Success!  Located at {{link}} id={{id}}", link=link, id=id)


data = convert.unicode2utf8(convert.value2json(settings.example))

response = requests.post(
    settings.url,
    data=data,
    headers={
        'Content-Type': b'application/json'
    }
)
if response.status_code != 200:
Esempio n. 15
0
    def save(self, query):
        """
        SAVE query TO ES FOR LATER RECOVERY
        :param query:
        :return: HAS TO USE FOR RECOVERY
        """
        meta, query.meta = query.meta, None
        json = convert.value2json(query)
        query.meta = meta
        hash = json.encode("utf8")

        # TRY MANY HASHES AT ONCE
        hashes = [None] * HASH_BLOCK_SIZE
        for i in range(HASH_BLOCK_SIZE):
            hash = hashlib.sha1(hash).digest()
            hashes[i] = hash

        short_hashes = [convert.bytes2base64(h[0:6]).replace("/", "_") for h in hashes]
        available = {h: True for h in short_hashes}

        existing = self.es.query(
            {
                "from": "saved_queries",
                "where": {"terms": {"hash": short_hashes}},
                "meta": {"timeout": "2second"},
            }
        )

        for e in Cube(
            select=existing.select, edges=existing.edges, data=existing.data
        ).values():
            if e.query == json:
                return e.hash
            available[e.hash] = False

        # THIS WILL THROW AN ERROR IF THERE ARE NONE, HOW UNLUCKY!
        best = first(h for h in short_hashes if available[h])

        self.queue.add(
            {
                "id": best,
                "value": {
                    "hash": best,
                    "create_time": Date.now(),
                    "last_used": Date.now(),
                    "query": json,
                },
            }
        )

        if meta.testing:
            while True:
                verify = self.es.query(
                    {
                        "from": "saved_queries",
                        "where": {"terms": {"hash": best}},
                        "meta": {"timeout": "2second"},
                        "select": "query",
                        "format": "list",
                    }
                )
                if verify.data:
                    break
                Log.alert("wait for saved query")
                Till(seconds=1).wait()

        Log.note("Saved {{json}} query as {{hash}}", json=json, hash=best)
        return best