Beispiel #1
0
    def _send_request(
        self,
        method,
        url_path,
        query_string_args=None,
        body_deserialization=None,
    ):
        url = self._API_URL + url_path

        query_string_args = query_string_args or {}
        query_string_args = dict(query_string_args,
                                 auditId=self._change_source)

        request_headers = \
            {'content-type': 'application/json'} if body_deserialization else {}

        if body_deserialization:
            request_body_serialization = json_serialize(body_deserialization)
        else:
            request_body_serialization = None

        response = self._session.request(
            method,
            url,
            params=query_string_args,
            auth=self._authentication_handler,
            data=request_body_serialization,
            headers=request_headers,
        )

        response_body_deserialization = \
            self._deserialize_response_body(response)
        return response_body_deserialization
Beispiel #2
0
    def _check_request_sender(
        http_method_name,
        request_sender_name,
        include_request_body,
    ):
        connection = _MockPortalConnection()

        body_deserialization = {'foo': 'bar'} if include_request_body else None

        request_sender = getattr(connection, request_sender_name)
        request_sender_kwargs = {}
        if include_request_body:
            request_sender_kwargs[
                'body_deserialization'] = body_deserialization
        request_sender(_STUB_URL_PATH, **request_sender_kwargs)

        eq_(1, len(connection.prepared_requests))

        prepared_request = connection.prepared_requests[0]
        eq_(http_method_name, prepared_request.method)

        if include_request_body:
            assert_in('content-type', prepared_request.headers)
            assert_equal('application/json',
                         prepared_request.headers['content-type'])

        requested_url_path = _get_path_from_api_url(prepared_request.url)
        eq_(_STUB_URL_PATH, requested_url_path)

        if include_request_body:
            body_serialization = json_serialize(body_deserialization)
            eq_(body_serialization, prepared_request.body)
        else:
            assert_false(prepared_request.body)
    def _check_request_sender(
        http_method_name,
        request_sender_name,
        include_request_body,
        ):
        connection = _MockPortalConnection()

        body_deserialization = {'foo': 'bar'} if include_request_body else None

        request_sender = getattr(connection, request_sender_name)
        request_sender_kwargs = {}
        if include_request_body:
            request_sender_kwargs['body_deserialization'] = body_deserialization
        request_sender(_STUB_URL_PATH, **request_sender_kwargs)

        eq_(1, len(connection.prepared_requests))

        prepared_request = connection.prepared_requests[0]
        eq_(http_method_name, prepared_request.method)

        if include_request_body:
            assert_in('content-type', prepared_request.headers)
            assert_equal(
                'application/json',
                prepared_request.headers['content-type']
            )

        requested_url_path = _get_path_from_api_url(prepared_request.url)
        eq_(_STUB_URL_PATH, requested_url_path)

        if include_request_body:
            body_serialization = json_serialize(body_deserialization)
            eq_(body_serialization, prepared_request.body)
        else:
            assert_false(prepared_request.body)
    def _send_request(
        self,
        method,
        url_path,
        query_string_args=None,
        body_deserialization=None,
        ):
        url = self._API_URL + url_path

        query_string_args = query_string_args or {}
        query_string_args = dict(query_string_args, auditId=self._change_source)

        request_headers = \
            {'content-type': 'application/json'} if body_deserialization else {}

        if body_deserialization:
            request_body_serialization = json_serialize(body_deserialization)
        else:
            request_body_serialization = None

        response = self._session.request(
            method,
            url,
            params=query_string_args,
            auth=self._authentication_handler,
            data=request_body_serialization,
            headers=request_headers,
            )

        response_body_deserialization = \
            self._deserialize_response_body(response)
        return response_body_deserialization
Beispiel #5
0
    def _get_property_value(property_name, contact_properties):
        property_value = contact_properties[property_name]
        if isinstance(property_value, bool):
            property_value = json_serialize(property_value)
        elif isinstance(property_value, date):
            property_value = convert_date_to_timestamp_in_milliseconds(property_value)

        property_value = unicode(property_value)
        return property_value
Beispiel #6
0
    def _get_property_value(property_name, contact_properties):
        property_value = contact_properties[property_name]
        if isinstance(property_value, bool):
            property_value = json_serialize(property_value)
        elif isinstance(property_value, date):
            property_value = \
                convert_date_to_timestamp_in_milliseconds(property_value)

        property_value = text_type(property_value)
        return property_value
Beispiel #7
0
def _invoke(label, argument_dict, is_map=False):
    """low-level invoking with map ability"""
    # TODO: We actually have to json_serialize this.. somehows
    
    conn = _getcloudnetconnection()
    
    map_limit = conn.map_job_limit
    map_count = 1
    
    extracted_argdict = {}
    for name, arglist in argument_dict.items():
        extracted = []
        
        if not hasattr(arglist, '__iter__'):
            raise TypeError('%s must map to an iterable' % name)
            
        argiter = iter(arglist)
        
        iterated = False
        for arg in argiter:
            iterated = True
            if not hasattr(arg, 'read'): # encode anything but file objects
                try:
                    arg = json_serialize(arg)
                except (TypeError, UnicodeDecodeError):
                    raise TypeError('%s is not json encodable' % name)
            extracted.append(arg)
        
        if not iterated:
            raise ValueError('%s cannot be bound to an empty list' % name)
        
        extracted_len = len(extracted)
        if map_count == 1:
            if not is_map and extracted_len > 1:
                raise ValueError('%s can only have 1 item when allow_map is False' % name)
            
            if extracted_len > map_limit:
                raise ValueError('%s has %s items. Maximum is %s', name, extracted_len, map_limit)        
                        
            map_count = extracted_len
        elif extracted_len > 1 and extracted_len != map_count:
            raise ValueError('%s has %s items. Expected %s to match with other arguments' % (name, extracted_len, map_count))        
    
        extracted_argdict[name] = extracted    
    
    if is_map:        
        resp = conn.send_request(_invoke_map_query % label, extracted_argdict)        
        jid_res = resp['jids']
        rstart, rend = jid_res.split('-')
        return xrange(int(rstart), int(rend))
            
        
    else:
        resp = conn.send_request(_invoke_query % label, extracted_argdict)        
        return resp['jid']
Beispiel #8
0
    def __call__(self, request):
        response = super(_ResponseMaker, self).__call__(request)

        if self._content_type:
            content_type_header_value = \
                '{}; charset=UTF-8'.format(self._content_type)
            response.headers['Content-Type'] = content_type_header_value

        if self._status_code != 204 and self._body_deserialization is not None:
            response._content = \
                bytes(json_serialize(self._body_deserialization), 'UTF-8')

        return response
Beispiel #9
0
def rest_publish(command, label, return_file=None, 
                ignore_exit_status=False, **kwargs):
    """Publish shell *command* to PiCloud so it can be invoked through the PiCloud REST API
    The published function will be managed in the future by a unique (URL encoded) *label*.
    Returns url of published function. See cloud.rest.publish
    
    See cloud.shell.execute for description other arguments    
    See cloud.rest.publish for description of **kwargs
    """

    if not label:
        raise ValueError('label must be provided')
    m = re.match(r'^[A-Z0-9a-z_+-.]+$', label)
    if not m:
        raise TypeError('Label can only consist of valid URI characters (alphanumeric or from set(_+-.$)')
    try:
        label = label.decode('ascii').encode('ascii')
    except (UnicodeDecodeError, UnicodeEncodeError): #should not be possible
        raise TypeError('label must be an ASCII string')
    
    cloud, params = _get_cloud_and_params(command, kwargs,
                                          ignore=['_label', '_depends_on', '_depends_on_errors'] )
    
    # shell argspecs are dictionaries
    cmd_params = template.extract_vars(command)
    argspec = {'prms' : cmd_params,
               'cmd' : command}
    argspec_serialized = json_serialize(argspec)
    if len(argspec_serialized) >= 255: #won't fit in db - clear command
        del argspec['command']
        argspec_serialized = json_serialize(argspec)
        if len(argspec_serialized) >= 255: #commands too large; cannot type check
            argspec_serialized = json_serialize({})
    params['argspec'] = argspec_serialized
    
    return _low_level_publish(_wrap_execute_program(command, return_file, ignore_exit_status), 
                       label, 'raw', 'actiondct',
                       params, func_desc='command invoked in shell')['uri']
    def __call__(self, request):
        response = RequestsResponse()

        response.status_code = self._status_code
        response.reason = 'Reason'

        if self._content_type:
            content_type_header_value = \
                '{}; charset=UTF-8'.format(self._content_type)
            response.headers['Content-Type'] = content_type_header_value

        if self._status_code != 204 and self._body_deserialization is not None:
            response._content = json_serialize(self._body_deserialization)

        return response
Beispiel #11
0
    def __call__(self, request):
        response = RequestsResponse()

        response.status_code = self._status_code
        response.reason = 'Reason'

        if self._content_type:
            content_type_header_value = \
                '{}; charset=UTF-8'.format(self._content_type)
            response.headers['Content-Type'] = content_type_header_value

        if self._status_code != 204 and self._body_deserialization is not None:
            response._content = bytes(
                json_serialize(self._body_deserialization), 'UTF-8')

        return response
Beispiel #12
0
    def __deliver_message(self, message):
        ''' The processing step to deliver new messages to the clients

        .. note::
        
           We deliver the message first so we can mangle the
           message reference without having to do a deep copy.

        :param message: The message to process
        '''
        client_count = len(ShellProcessor.connections)
        if client_count > 0: # don't waste time if there are no clients
            _logger.debug("delivering message to %d clients" % client_count)
            message = json_serialize(message) + "\r\n"
            for queue in ShellProcessor.connections:
                queue.put_nowait(message)
Beispiel #13
0
    def _send_request(
        self,
        method,
        url,
        query_string_args=None,
        body_deserialization=None,
        ):
        if url.startswith(self._api_url):
            url = url
        else:
            url = self._api_url + url

        query_string_args = query_string_args or {}

        request_headers = \
            {'content-type': 'application/json'} if body_deserialization else {}

        if body_deserialization:
            request_body_serialization = json_serialize(body_deserialization)
        else:
            request_body_serialization = None

        response = self._session.request(
            method,
            url,
            params=query_string_args,
            auth=self._authentication_handler,
            data=request_body_serialization,
            headers=request_headers,
            timeout=self._timeout,
            )

        self._require_successful_response(response)
        self._require_deserializable_response_body(response)

        return response
Beispiel #14
0
def _json_serialize_to_boolean(value):
    value_boolean = bool(value)
    value_serialized = json_serialize(value_boolean)
    return value_serialized
Beispiel #15
0
def create_tinfoil_index(index_to_write: dict,
                         out_path: Path,
                         compression_flag: int,
                         rsa_pub_key_path: Path = None,
                         vm_path: Path = None):
    to_compress_buffer = b""

    if vm_path is not None and vm_path.is_file():
        to_compress_buffer += b"\x13\x37\xB0\x0B"
        vm_buffer = b""

        with open(vm_path, "rb") as vm_stream:
            vm_buffer += vm_stream.read()

        to_compress_buffer += len(vm_buffer).to_bytes(4, "little")
        to_compress_buffer += vm_buffer

    to_compress_buffer += bytes(json_serialize(index_to_write).encode())

    to_write_buffer = b""
    session_key = b""

    if compression_flag == CompressionFlag.ZSTD_COMPRESSION:
        to_write_buffer += ZstdCompressor(
            level=22).compress(to_compress_buffer)

    elif compression_flag == CompressionFlag.ZLIB_COMPRESSION:
        to_write_buffer += zlib_compress(to_compress_buffer, 9)

    elif compression_flag == CompressionFlag.NO_COMPRESSION:
        to_write_buffer += to_compress_buffer

    else:
        raise NotImplementedError(
            "Compression method supplied is not implemented yet.")

    data_size = len(to_write_buffer)
    flag = None
    to_write_buffer += (b"\x00" * (0x10 - (data_size % 0x10)))

    if rsa_pub_key_path is not None and rsa_pub_key_path.is_file():

        def rand_aes_key_generator() -> bytes:
            return randint(0, 0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF).to_bytes(
                0x10, byteorder="big")

        rsa_pub_key = import_rsa_key(open(rsa_pub_key_path).read())
        rand_aes_key = rand_aes_key_generator()

        pkcs1_oaep_ctx = new_pkcs1_oaep_ctx(rsa_pub_key,
                                            hashAlgo=SHA256,
                                            label=b"")
        aes_ctx = new_aes_ctx(rand_aes_key, MODE_ECB)

        session_key += pkcs1_oaep_ctx.encrypt(rand_aes_key)
        to_write_buffer = aes_ctx.encrypt(to_write_buffer)
        flag = compression_flag | EncryptionFlag.ENCRYPT
    else:
        session_key += b"\x00" * 0x100
        flag = compression_flag | EncryptionFlag.NO_ENCRYPT

    Path(out_path.parent).mkdir(parents=True, exist_ok=True)

    with open(out_path, "wb") as out_stream:
        out_stream.write(b"TINFOIL")
        out_stream.write(flag.to_bytes(1, byteorder="little"))
        out_stream.write(session_key)
        out_stream.write(data_size.to_bytes(8, "little"))
        out_stream.write(to_write_buffer)
Beispiel #16
0
def publish(func, label, out_encoding='json', **kwargs):
    """       
    Publish *func* (a callable) to PiCloud so it can be invoked through the PiCloud REST API
    
    The published function will be managed in the future by a unique (URL encoded) *label*. 
    
    *out_encoding* specifies the format that the return value should be in when retrieving the result
    via the REST API. Valid values are "json" for a JSON-encoded object and "raw", where the return value
    must be an str (but can contain any characters).
    
    The return value is the URL which can be HTTP POSTed to to invoke *func*. 
    See http://docs.picloud.com/rest.html for information about PiCloud's REST API    
    
    Certain special *kwargs* associated with cloud.call can be attached to the periodic jobs: 
        
    * _cores:
        Set number of cores your job will utilize. See http://docs.picloud.com/primer.html#choose-a-core-type/
        In addition to having access to more CPU cores, the amount of RAM available will grow linearly.
        Possible values for ``_cores`` depend on what ``_type`` you choose:
        
        * c1: 1                    
        * c2: 1, 2, 4, 8
        * f2: 1, 2, 4, 8, 16                                    
        * m1: 1, 2, 4, 8
        * s1: 1        
    * _env:
        A string specifying a custom environment you wish to run your job within.
        See environments overview at http://docs.picloud.com/environment.html                
    * _fast_serialization:
        This keyword can be used to speed up serialization, at the cost of some functionality.
        This affects the serialization of the spawned jobs' return value.
        The stored function will always be serialized by the enhanced serializer, with debugging features.
        Possible values keyword are:
                    
        0. default -- use cloud module's enhanced serialization and debugging info            
        1. no debug -- Disable all debugging features for result            
        2. use cPickle -- Use python's fast serializer, possibly causing PicklingErrors
    * _max_runtime:
        Specify the maximum amount of time (in integer minutes) jobs can run. If the job runs beyond 
        this time, it will be killed.                    
    * _os_env_vars:
        List of operating system environment variables that should be copied to PiCloud from your system
        Alternatively a dictionary mapping the environment variables to the desired values.                                                                            
    * _priority: 
        A positive integer denoting the job's priority. PiCloud tries to run jobs 
        with lower priority numbers before jobs with higher priority numbers.            
    * _profile:
        Set this to True to enable profiling of your code. Profiling information is 
        valuable for debugging, but may slow down your jobs.
    * _restartable:
        In the very rare event of hardware failure, this flag indicates that a spawned 
        job can be restarted if the failure happened in the middle of it.
        By default, this is true. This should be unset if the function has external state
        (e.g. it modifies a database entry)
    * _type:
        Choose the type of core to use, specified as a string:
        
        * c1: 1 compute unit, 300 MB ram, low I/O (default)                    
        * c2: 2.5 compute units, 800 MB ram, medium I/O
        * f2: 5.5 compute units, 3.75 GB ram, high I/O, hyperthreaded core                                    
        * m1: 3.25 compute units, 8 GB ram, high I/O
        * s1: Up to 2 compute units (variable), 300 MB ram, low I/O, 1 IP per core
                       
        See http://www.picloud.com/pricing/ for pricing information
    * _vol:
        A string or list of strings specifying a volume(s) you wish your jobs to have access to.

    """
    
    if not callable(func):
        raise TypeError( 'cloud.rest.publish first argument (%s) is not callable'  % (str(func) ))        
    
    m = re.match(r'^[A-Z0-9a-z_+-.]+$', label)
    if not m:
        raise TypeError('Label can only consist of valid URI characters (alphanumeric or from set(_+-.$)')
        
    #ASCII label:
    try:
        label = label.decode('ascii').encode('ascii')
    except (UnicodeDecodeError, UnicodeEncodeError): #should not be possible
        raise TypeError('label must be an ASCII string')
    
    try:
        docstring = '' if (func.__doc__ is None) else func.__doc__
        func_desc = (docstring).encode('utf8')
    except (UnicodeDecodeError, UnicodeEncodeError):
        raise TypeError('function docstring must be an UTF8 compatible unicode string')
    
    if not isinstance(out_encoding, str):
        raise TypeError('out_encoding must be an ASCII string')
    
    cloud = _getcloud()
    params = cloud._getJobParameters(func, kwargs, 
                                             ignore=['_label', '_depends_on', '_depends_on_errors'])
    
    #argument specification for error checking and visibility
    argspec = getargspec(func)    
    argspec_serialized = json_serialize(argspec,default=str)
    if len(argspec_serialized) >= 255: #won't fit in db - clear defaults
        argspec[4] = {}
        argspec_serialized = json_serialize(argspec,default=str)        
    params['argspec'] = argspec_serialized
    
    resp = _low_level_publish(func, label, out_encoding, 'raw', params, func_desc)
    
    return resp['uri']
def _json_serialize_to_boolean(value):
    value_boolean = bool(value)
    value_serialized = json_serialize(value_boolean)
    return value_serialized