예제 #1
2
    def shareload(identifier):
        chatname = None
        if 'chatname' in session:
            chatname = session['chatname']
        conn = sqlite3.connect(FROLIC_DB)
        c = conn.cursor()
        c.execute("SELECT places, swaps, waypoints FROM routes WHERE identifier=?", [identifier])
        try:
            places, swaps, waypoints = c.fetchone()
        except:
            return redirect(url_for('/'), 302)
        places = jsonpickle.decode(places)
        swaps = jsonpickle.decode(swaps)

        if not 'cache' in session:
            session['cache'] = {}
        session['cache']['places'] = jsonpickle.encode(places)
        session['cache']['swaps'] = jsonpickle.encode(deque(swaps))
        session['cache']['waypoints'] = waypoints
        conn.commit()
        conn.close()
        return render_template('places.html',
                               places=places,
                               swaps=swaps,
                               waypoints_list=waypoints,
                               mobile=_is_mobile(request),
                               chatname=chatname,
                               is_share=True)
예제 #2
0
    def test_republish(self):
        # create a build request
        build_string = '{"py/object": "pybit.models.BuildRequest",\
        "timestamp": null, "job": {"py/object": "pybit.models.Job",\
        "packageinstance": {"py/object": "pybit.models.PackageInstance",\
        "format": {"py/object": "pybit.models.Format", "id": 5,\
        "name": "completezip"}, "package": {"py/object": "pybit.models.Package",\
        "version": "1.2", "id": 2, "name": "col10642"}, "id": 1,\
        "master": true, "suite": {"py/object": "pybit.models.Suite",\
        "id": 3, "name": "latex"}, "distribution": {"py/object": \
        "pybit.models.Dist", "id": 2, "name": "cnx"}, "arch": \
        {"py/object": "pybit.models.Arch", "id": 9, "name": "desktop"},\
        "build_env": null}, "id": 36, "buildclient": null}, "transport": \
        {"py/object": "pybit.models.Transport", "uri": "http://cnx.org/", \
        "id": null, "vcs_id": "", "method": ""}, "web_host": "localhost:8080"}'
        build_request = jsonpickle.decode(build_string)

        # republish
        queue = 'cnx_desktop_latex_completezip'
        self.channel.queue_declare(queue=queue)
        coyote.republish(build_request, queue, self.channel)
        
        # get it off the queue and test it
        tag = self.channel.basic_get(queue=queue, no_ack=True)
        build_request = jsonpickle.decode(tag[-1])
        job_id = build_request.get_job_id()
        self.assertEquals(job_id, 36)
        package = build_request.get_package()
        self.assertEquals(package, 'col10642')
        version = build_request.get_version()
        self.assertEquals(version, '1.2')
예제 #3
0
def fromFile(filename, no_practice=False):
    """
    Read experiment log from file.
    :param filename:
    :param no_practice:
    :return:
    """
    from os import path
    if str.startswith(path.split(filename)[-1], "P2P"):
        return fromFile_p2p(filename)
    lines = open(filename).readlines()
    lines = [refactor_old_references(line) for line in lines]
    images = jsonpickle.decode(lines[0])
    responses = recursive_decode(lines[1])
    test_results = jsonpickle.decode(lines[2])
    test_results = _expandTestResults(test_results, images)
    responses = _expandResponsesNew(responses, images)

    if not no_practice:
        responses_practice = recursive_decode(lines[3])
        test_results_practice = jsonpickle.decode(lines[4])
        test_results_practice = _expandTestResults(test_results_practice, images)
        responses_practice = _expandResponsesNew(responses_practice, images)
        return responses, test_results, responses_practice, test_results_practice, images

    return responses, test_results, None, None, images
예제 #4
0
  def test_new_mitigate_post(self):
    method = 'test_new_mitigate_post'
    url = '/api/responses'
    self.logger.info('[%s] URL: %s', method, url)
    new_response_body = self.prepare_json()

    self.app.delete('/api/responses/name/%s?session_id=test' % quote(self.prepare_new_response().theName))
    rv = self.app.post(url, content_type='application/json', data=new_response_body)
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    self.logger.debug('[%s] Response data: %s', method, responseData)
    json_resp = jsonpickle.decode(responseData)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    msg = json_resp.get('message', None)
    self.assertIsNotNone(msg, 'No message returned')
    self.logger.info('[%s] Message: %s\n', method, msg)

    rv = self.app.post('/api/responses/name/%s/generate_goal?session_id=test' % quote(self.prepare_new_response().theName))
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    self.logger.debug('[%s] Response data: %s', method, responseData)
    json_resp = jsonpickle.decode(responseData)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    ackMsg = json_resp.get('message', None)
    self.assertEqual(ackMsg, 'Goal successfully generated')
    rv = self.app.delete('/api/responses/name/%s?session_id=test' % quote(self.prepare_new_response().theName))
    def test_get_existing_status_with_error_in_process(self, http_client, http_server,
                                 base_url, monkeypatch, stub_isdir):

        message_from_process = {"msg": "My message",
                                "stdout": "Stdout stuff",
                                "stderr": "This was some error from stderr"}

        def my_get_with_error(self, pid, wrapper_type):
            return ProcessInfo(runfolder="foo", host="bar",
                               state=State.ERROR,
                               proc=None, msg=message_from_process, pid=pid)

        monkeypatch.setattr("siswrap.wrapper_services.ProcessService.get_status",
                            my_get_with_error)

        resp = yield http_client.fetch(base_url + API_URL +
                                       "/report/status/123")
        assert resp.code == 200
        payload = jsonpickle.decode(resp.body)
        print payload
        assert payload["pid"] == 123
        assert payload["state"] == State.ERROR
        assert payload["msg"] == message_from_process

        resp = yield http_client.fetch(base_url + API_URL + "/qc/status/321")
        assert resp.code == 200
        payload = jsonpickle.decode(resp.body)
        assert payload["pid"] == 321
예제 #6
0
  def test_show_databases(self):
    method = 'test_show_databases'
    url = '/api/settings/database/testshowdb/create?session_id=test'
    rv = self.app.post(url)
    self.assertIsNotNone(rv.data, 'No response')
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    json_dict = jsonpickle.decode(responseData)
    message = str(json_dict['message'])
    self.assertGreater(message.find('successfully'), -1, 'Failed to create testshowdb')

    url = '/api/settings/database/default/open?session_id=test'
    rv = self.app.post(url)
    self.assertIsNotNone(rv.data, 'No response')
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    json_dict = jsonpickle.decode(responseData)
    message = str(json_dict['message'])
    self.assertGreater(message.find('successfully'), -1, 'Failed to open default databases')

    rv = self.app.get('/api/settings/databases?session_id=test')
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    dbs = jsonpickle.decode(responseData)
    self.assertIsNotNone(dbs, 'No results after deserialization')
    self.assertIsInstance(dbs, list, 'The result is not a list as expected')
    self.assertGreater(len(dbs), 0, 'No databases in the list')
    self.assertEqual('testshowdb' in dbs, True)
예제 #7
0
    def deserialize(self):
        cache_loaded = False
        if os.path.exists(self.server_fname()) and not os.path.isdir(self.backup):
            try:
                self.memcache = { "server" : {},
                                  "users" : {} }
                
                with open(self.server_fname()) as backupfile:
                    print ("Attempting to reload cache")
                    self.memcache['server'] = jsonpickle.decode(backupfile.read())

                print ("Server cache loaded", json.dumps(self.memcache, indent=4))
                for user in self.memcache['server']['user_list']:
                    # Try to load as much user data as possible
                    if os.path.exists(self.user_fname(user)):
                        print ("found path for user", user)
                        with open(self.user_fname(user)) as userfile:
                            user_data = jsonpickle.decode(userfile.read())
                        self.memcache['users'][user] = user_data
                cache_loaded = True
            except Exception as e:
                print ("Cache file corrupted...")
                raise e
        if not cache_loaded:
            print ("Cache could not be loaded")
            pass
        else:
            print ("CACHE LOADED SUCCESSFULLY!")
    def load_server_settings(root_dir, config):
        assert(hasattr(config, 'registry'))
        assert(hasattr(config.registry, 'settings'))
        log = logging.getLogger(__name__)
        for root, dirs, files in os.walk(root_dir):
            root = os.path.abspath(root)
            if '.settings.json' in files:
                lastfolder = os.path.abspath(root + '/..')
                last_settings = dict()
                if lastfolder in config.registry.settings['directory_settings']:
                    last_settings = config.registry.settings['directory_settings'][lastfolder]

                config.registry.settings['directory_settings'][root] = last_settings

                filename = root + '/.settings.json'
                with open(os.path.join(filename), "r") as myfile:
                    data = myfile.read()
                    settings_struct = jsonpickle.decode(data)
                    if not isinstance(settings_struct, dict):
                        settings_struct = jsonpickle.decode(settings_struct)

                try:
                    settings_struct.update(config.registry.settings['directory_settings'][root])
                    config.registry.settings['directory_settings'][root] = settings_struct
                    config.registry.settings['directory_settings'][root]['reload'] = config.registry.settings[
                        'reload_templates']
                    config.registry.settings['directory_settings'][root]['path'] = filename
                except Exception as e:
                    log.error(e.message)
            else:
                path = os.path.abspath(root + '/..')
                if path in config.registry.settings['directory_settings']:
                    config.registry.settings['directory_settings'][root] = \
                        config.registry.settings['directory_settings'][path]
예제 #9
0
    def test_x_put(self):
        method = 'test_x_put'
        url = '/api/requirements'
        rv = self.app.get('/api/requirements?session_id=test')
        reqs = jsonpickle.decode(rv.data)
        requirement = reqs.get(self.new_requirement.theDescription)

        upd_requirement = self.new_requirement
        upd_requirement.theName = 'Test2'
        upd_requirement.theId = requirement['theId']
        upd_requirement_dict = self.new_requirement_dict
        upd_requirement_dict['object'] = upd_requirement
        upd_requirement_body = jsonpickle.encode(upd_requirement_dict)
        self.logger.info('[%s] JSON data: %s', method, upd_requirement_body)

        rv = self.app.put(url, content_type='application/json', data=upd_requirement_body)
        self.logger.debug('[%s] Response data: %s', method, rv.data)
        json_resp = jsonpickle.decode(rv.data)
        self.assertIsNotNone(json_resp, 'No results after deserialization')
        message = json_resp.get('message', None)
        self.assertIsNotNone(message, 'No message returned')
        self.logger.info('Message: %s', message)

        rv = self.app.get('/api/requirements?session_id=test')
        self.logger.debug('[%s] Response data: %s', method, rv.data)
        requirements = jsonpickle.decode(rv.data)
        requirement = requirements.get(upd_requirement.theDescription, None)
        self.assertIsNotNone(requirement, 'Requirement not updated as expected')
        self.logger.info('[%s] Requirement: %s [%d]\n', method, requirement['theName'], requirement['theId'])
예제 #10
0
  def test_get_trace_dimensions(self):
    method = 'test_get_trace_dimensions'
    url = '/api/traces/dimensions/requirement/is_from/1?session_id=test'
    self.logger.info('[%s] URL: %s', method, url)
    rv = self.app.get(url)
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    fromDims = jsonpickle.decode(responseData)
    self.assertIsNotNone(fromDims, 'No results after deserialization')
    self.logger.info('[%s] Traces found: %d', method, len(fromDims))
    self.assertEqual(len(fromDims),6)

    url = '/api/traces/dimensions/requirement/is_from/0?session_id=test'
    self.logger.info('[%s] URL: %s', method, url)
    rv = self.app.get(url)
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    toDims = jsonpickle.decode(responseData)
    self.assertIsNotNone(toDims, 'No results after deserialization')
    self.logger.info('[%s] Traces found: %d', method, len(toDims))
    self.assertEqual(len(toDims),2)
예제 #11
0
  def test_settings_put(self):
    url = '/api/settings?session_id=test'
    method = 'test_settings_put'
    rv = self.app.get(url)
    json_dict = jsonpickle.decode(rv.data)
    self.logger.info('[%s] Current project name: %s', method, json_dict['projectName'])

    settings = self.convert_to_obj(json_dict)
    settings.projectName = 'A new project name'
    new_json_dict = { 'session_id': 'test', 'object': settings }
    json_body = jsonpickle.encode(new_json_dict)
    rv = self.app.put(url, data=json_body, content_type='application/json')
    self.assertIsNotNone(rv.data, 'No response')
    new_json_dict = jsonpickle.decode(rv.data)
    self.assertIsInstance(new_json_dict, dict, 'Response is not a valid JSON dictionary')
    message = new_json_dict.get('message', None)
    self.assertIsNotNone(message)
    self.logger.info('[%s] Message: %s', method, message)

    rv = self.app.get(url)
    new_json_dict = jsonpickle.decode(rv.data)
    self.logger.info('[%s] New project name: %s\n', method, new_json_dict['projectName'])

    new_json_dict = { 'session_id': 'test', 'object': json_dict }
    json_body = jsonpickle.encode(new_json_dict)
    rv = self.app.put(url, data=json_body, content_type='application/json')
예제 #12
0
  def test_create_new_project(self):
    url = '/api/settings/create?session_id=test'
    import_url = '/api/import/file/type/all'
    method = 'test_create_new_project'
    rv = self.app.post(url)
    self.assertIsNotNone(rv.data, 'No response')
    json_dict = jsonpickle.decode(rv.data)
    self.assertIsInstance(json_dict, dict, 'Response is not a valid JSON dictionary')
    self.assertTrue(json_dict.has_key('message'), 'No message in reponse')
    message = str(json_dict['message'])
    self.logger.info('[%s] Message: %s', method, message)
    self.assertGreater(message.find('successfully'), -1, 'Failed to create new project')

    fs_xmlfile = open(self.xmlfile, 'rb')
    file_contents = fs_xmlfile.read()
    data = {
      'session_id': 'test',
      'file': (StringIO(file_contents), 'import.xml')
    }
    rv = self.app.post(import_url, data=data, content_type='multipart/form-data')
    self.assertIsNotNone(rv.data, 'No response after reimporting model')
    json_dict = jsonpickle.decode(rv.data)
    self.assertIsInstance(json_dict, dict, 'Response is not a valid JSON dictionary')
    assert isinstance(json_dict, dict)
    message = json_dict.get('message', None)
    self.assertIsNotNone(message, 'No message in response')
    self.assertGreater(message.find('0'), -1, 'Failed to import any data')
    self.logger.info('[%s] Successfully created new project and restored the example project\n', method)
    def job_archive():
        """
        Returns the job archive from the database
        :return:
        """
        jobs_stmt = DBSession.query(Job).order_by(Job.updatetime.desc()).all()
        jobs_dict = []
        for jobs in jobs_stmt:
            job_outputs = DBSession.query(JobOutput).filter(JobOutput.jobid == jobs.id).all()
            jobs = jobs.__dict__

            if 'jobinfo' in jobs and jobs['jobinfo'] is not None:
                obj = jsonpickle.decode(jobs['jobinfo'])
                try:
                    jobs.update(obj)
                except BaseException as e:
                    print(str(e))

            if 'jobdetails' in jobs and jobs['jobdetails'] is not None:
                obj = jsonpickle.decode(jobs['jobdetails'])
                try:
                    jobs.update(obj)
                except BaseException as e:
                    print(str(e))
            jobs['number_of_joboutputs'] = len(job_outputs)
            jobs['joboutputs'] = []
            for outputs in job_outputs:
                jobs['joboutputs'].append(outputs.__dict__)
            jobs_dict.append(jobs)
        return jobs_dict
예제 #14
0
  def test_get_risk_threat_level_by_environment(self):
    method = 'test_get_risk_level_by_environment'
    url = '/api/risk_level/asset/threat_type/ICT%20Application/Enumeration/environment/Day?session_id=test'
    self.logger.info('[%s] URL: %s', method, url)
    rv = self.app.get(url)
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    level = jsonpickle.decode(responseData)
    self.assertIsNotNone(level, 'No results after deserialization')
    self.assertIsInstance(level, int, 'The result is not an integer as expected')
    self.assertEqual(level, 9)

    url = '/api/risk_level/asset/threat_type/ICT%20Application/Enumeration/environment/Night?session_id=test'
    self.logger.info('[%s] URL: %s', method, url)
    rv = self.app.get(url)
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    level = jsonpickle.decode(responseData)
    self.assertIsNotNone(level, 'No results after deserialization')
    self.assertIsInstance(level, int, 'The result is not an integer as expected')
    self.assertEqual(level, 0)
예제 #15
0
def new_recursive_decode(string, verbose=False):
    def print_(stuff):
        if verbose:
            print stuff

    obj = string
    if (isinstance(string, str) or isinstance(string, unicode)) \
            and "py/object" in string:
        print_("Decoding string: {}".format(string))
        obj = jsonpickle.decode(string)

    if isinstance(obj, dict):
        if ("py/object" in obj):
            print_("Detected a dict with py/object as key: {}".format(obj))
            new_obj = {}
            for k, v in obj.items():
                try:
                    new_obj[k] = jsonpickle.decode(v)
                except:
                    new_obj[k] = new_recursive_decode(v)

    if isinstance(obj, list):
        obj = [new_recursive_decode(o) for o in obj]
    if verbose:
        print_("Returning object: {}".format(obj))
    return obj
예제 #16
0
  def test_x_put(self):
    method = 'test_x_put'

    url = '/api/requirements?asset=%s' % quote(self.existing_asset_name)
    rv = self.app.post(url, content_type='application/json', data=self.new_requirement_body)

    upd_requirement = self.new_requirement
    upd_requirement['theName'] = 'Test2'
    upd_requirement_dict = self.new_requirement_dict
    upd_requirement_dict['object'] = upd_requirement
    upd_requirement_body = json.dumps(upd_requirement_dict)
    self.logger.info('[%s] JSON data: %s', method, upd_requirement_body)

    rv = self.app.put(url, content_type='application/json', data=upd_requirement_body)
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    self.logger.debug('[%s] Response data: %s', method, responseData)
    json_resp = jsonpickle.decode(responseData)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message returned')
    self.logger.info('Message: %s', message)

    rv = self.app.get('/api/requirements?session_id=test')
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    self.logger.debug('[%s] Response data: %s', method, responseData)
    requirements = jsonpickle.decode(responseData)
    requirement = requirements[0]
    self.assertIsNotNone(requirement, 'Requirement not updated as expected')
    self.logger.info('[%s] Requirement: %s\n', method, requirement['theName'])
예제 #17
0
파일: EnaRest.py 프로젝트: alexgarciac/COPO
def save_experiment(request):
    # method to save ENA experiment object(s). One experiment object is created for each panel on the
    # front-end (although as far as the users are concerned, multiple panels can belong to the same experiment

    # certain attributes are shared between the different experiments generated by the front end
    common = jsonpickle.decode(request.POST.get('common'))
    # others are particular to the individual object
    per_panel = jsonpickle.decode(request.POST.get('per_panel'))

    if (per_panel['experiment_id'] == ''):
        # if we are dealing with a new experiment (i.e. no id has been supplied)
        # then create a new object
        experiment_id = EnaCollection().add_experiment_to_study(per_panel, common, request.session["study_id"])
    else:
        # else retrieve the existing object
        experiment_id = EnaCollection().update_experiment_in_study(per_panel, common, request.session["study_id"])

    # here we need to loop through per_file.files adding object to exp files list
    for k in range(0, len(per_panel['files'])):
        c = ChunkedUpload.objects.get(id=int(per_panel['files'][k]))
        if len(per_panel['hashes']) > k:
            hash = per_panel['hashes'][k]
        else:
            hash = ''
        EnaCollection().add_file_to_study(request.session['study_id'], experiment_id, c.id, hash)
    out = {'experiment_id': experiment_id}

    return HttpResponse(jsonpickle.encode(experiment_id), content_type='text/plain')
예제 #18
0
def broker_owner(gameid, placeid):
    """
    GET: gets a owner
    PUT: trade the place by changing the owner
    POST: buy the place in question. it will fail if it is not for sale
    """
    if request.method == 'GET':
        estate = get_place(gameid, placeid)
        if estate:
            return make_response(estate.owner, 200)
    elif request.method == 'PUT':
        player = jsonpickle.decode(request.data)
        estate = get_place(gameid, placeid)
        estate.owner = player['id']
        return make_response('', 200)
    elif request.method == 'POST':
        player = jsonpickle.decode(request.data)
        estate = get_place(gameid, placeid)
        visit = Broker.get(gameid).visits.get(player['id'], None)
        if (estate and
                estate.owner == '' and
                visit and
                visit == estate.place):
            if buy_estate(gameid, player, estate):
                return make_response('', 200)
        return make_response('''The place is not for sale - either not buyable
                                or already sold (Conflict)''', 409)
예제 #19
0
파일: dworker.py 프로젝트: bennojoy/distrib
def process_task(body):
    new_stdin = sys.stdin
    try:
        fileno = sys.stdin.fileno()
        if fileno is not None:
            try:
                new_stdin = os.fdopen(os.dup(fileno))
            except OSError, e:
                # couldn't dupe stdin, most likely because it's
                # not a valid file descriptor, so we just rely on
                # using the one that was passed in
                pass
    except ValueError:
        # couldn't get stdin's fileno, so we just carry on
        pass
    shared_loader_obj = SharedPluginLoaderObj()
    loader = DataLoader(vault_password=None)
    task_json = json.loads(body)
    loader.set_basedir(task_json['base_dir'])
    host = jsonpickle.decode(task_json['host'])
    task = jsonpickle.decode(task_json['task'])
    job_vars = jsonpickle.decode(task_json['task_vars'])
    connection_info  = jsonpickle.decode(task_json['conn_info'])
    task.set_loader(loader)
    new_connection_info = connection_info.set_task_and_host_override(task=task, host=host)
    executor_result = TaskExecutor(host, task, job_vars, new_connection_info, new_stdin, loader, shared_loader_obj).run()
    task_result = TaskResult(host, task, executor_result)
    task_pickled = jsonpickle.encode(task_result)
    return(json.dumps(task_pickled))
예제 #20
0
    def arrange(index):
        try:
            places = jsonpickle.decode(session['cache']['places'])
            swaps = deque(jsonpickle.decode(session['cache']['swaps']))
        except KeyError:
            return redirect('/swap/{}'.format(index), 302)

        try:
            index = int(index)
            places_copy = []
            places_copy.append(places[index])
            if not places[index] == places[-1]:
                for place in places[index+1:]:
                    places_copy.append(place)
            for place in places[:index]:
                places_copy.append(place)
            places = places_copy
            _process_ratings(places)
            session['cache']['places'] = jsonpickle.encode(places)
            waypoints = _stringify([unicode(u' '.join([place.street, place.city, place.country]).replace("'", "\\'")) for place in places])
            session['cache']['waypoints'] = waypoints
        except Exception as e:
            traceback.print_exc()
            return render_template('places.html',
                                   places=places,
                                   swaps=swaps,
                                   waypoints_list=waypoints,
                                   mobile=_is_mobile(request))
        return render_template('places.html',
                               places=places,
                               swaps=swaps,
                               waypoints_list=waypoints,
                               mobile=_is_mobile(request))
예제 #21
0
  def test_put(self):
    method = 'test_put'
    url = '/api/domainproperties'
    self.logger.info('[%s] URL: %s', method, url)
    new_domainproperty_body = self.prepare_json()

    rv = self.app.post(url, content_type='application/json', data=new_domainproperty_body)
    self.logger.debug('[%s] Response data: %s', method, rv.data)
    json_resp = jsonpickle.decode(rv.data)
    self.assertIsNotNone(json_resp, 'No results after deserialization')

    domainproperty_to_update = self.prepare_new_domainproperty()
    domainproperty_to_update.theName = 'Edited test domainproperty'
    upd_env_body = self.prepare_json(domainproperty=domainproperty_to_update)
    rv = self.app.put('/api/domainproperties/name/%s?session_id=test' % quote(self.prepare_new_domainproperty().name()), data=upd_env_body, content_type='application/json')
    self.assertIsNotNone(rv.data, 'No response')
    json_resp = jsonpickle.decode(rv.data)
    self.assertIsNotNone(json_resp)
    self.assertIsInstance(json_resp, dict)
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message in response')
    self.logger.info('[%s] Message: %s', method, message)
    self.assertGreater(message.find('successfully updated'), -1, 'The domainproperty was not successfully updated')

    rv = self.app.get('/api/domainproperties/name/%s?session_id=test' % quote(domainproperty_to_update.name()))
    upd_domainproperty = jsonpickle.decode(rv.data)
    self.assertIsNotNone(upd_domainproperty, 'Unable to decode JSON data')
    self.logger.debug('[%s] Response data: %s', method, rv.data)
    self.logger.info('[%s] Domain Property: %s [%d]\n', method, upd_domainproperty['theName'], upd_domainproperty['theId'])

    rv = self.app.delete('/api/domainproperties/name/%s?session_id=test' % quote(domainproperty_to_update.theName))
예제 #22
0
def test_muffin_redis_cache(loop, app, client):
    assert app.ps.redis_cache
    assert app.ps.redis_cache.conn

    response = client.get('/cached')
    assert response.status_code == 200
    assert 'key' in response.json

    response = client.get('/cached_keyprefix')
    assert response.status_code == 200
    assert 'firstname' in response.json

    @asyncio.coroutine
    def exist_key_in_redis():
        return (yield from app.ps.redis_cache.get('view/cached'))

    result = loop.run_until_complete(exist_key_in_redis())
    assert jsonpickle.decode(result) == {'key': 'value'}

    @asyncio.coroutine
    def exist_key_in_redis():
        return (yield from app.ps.redis_cache.get('custom_keyprefix'))

    result = loop.run_until_complete(exist_key_in_redis())
    assert jsonpickle.decode(result) == {'firstname': 'Mike', 'gender': 'male'}
예제 #23
0
  def test_new_mitigate_post(self):
    method = 'test_new_mitigate_post'
    url = '/api/responses'
    self.logger.info('[%s] URL: %s', method, url)
    new_response_body = self.prepare_json()

    self.app.delete('/api/responses/name/%s?session_id=test' % quote(self.prepare_new_response().theName))
    rv = self.app.post(url, content_type='application/json', data=new_response_body)
    self.logger.debug('[%s] Response data: %s', method, rv.data)
    json_resp = jsonpickle.decode(rv.data)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    env_id = json_resp.get('response_id', None)
    self.assertIsNotNone(env_id, 'No response ID returned')
    self.assertGreater(env_id, 0, 'Invalid response ID returned [%d]' % env_id)
    self.logger.info('[%s] Response ID: %d\n', method, env_id)

    rv = self.app.post('/api/responses/name/%s/generate_goal?session_id=test' % quote(self.prepare_new_response().theName))
    self.logger.debug('[%s] Response data: %s', method, rv.data)
    json_resp = jsonpickle.decode(rv.data)
    self.assertIsNotNone(json_resp, 'No results after deserialization')
    ackMsg = json_resp.get('message', None)
    self.assertEqual(ackMsg, 'Goal successfully generated')
   

    rv = self.app.delete('/api/responses/name/%s?session_id=test' % quote(self.prepare_new_response().theName))
예제 #24
0
  def test_put(self):
    method = 'test_put'
    rv = self.app.post('/api/trust_boundaries?session_id=test', content_type='application/json', data=self.prepare_json())
    url = '/api/trust_boundaries/name/Shibboleth'
    self.logger.info('[%s] URL: %s', method, url)

    upd_body = self.prepare_json(trust_boundary=self.prepare_updated_trust_boundary())
    rv = self.app.put('/api/trust_boundaries/name/Shibboleth?session_id=test', data=upd_body, content_type='application/json')
    self.assertIsNotNone(rv.data, 'No response')
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    json_resp = jsonpickle.decode(responseData)
    self.assertIsNotNone(json_resp)
    self.assertEqual(json_resp['message'],'TrustBoundary successfully updated')

    rv = self.app.get('/api/trust_boundaries/name/Shibboleth?session_id=test')
    self.assertIsNotNone(rv.data, 'No response')
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    self.logger.debug('[%s] Response data: %s', method, responseData)
    upd_tb = jsonpickle.decode(responseData)
    self.assertIsNotNone(upd_tb, 'No results after deserialization')
    self.assertEqual(upd_tb['theName'],'Shibboleth')
    self.assertEqual(upd_tb['theDescription'],'Identity provider')
    self.assertEqual(upd_tb['theEnvironmentProperties'][0]['theComponents'][0]['theName'],'Authenticate Researcher')
    self.assertEqual(upd_tb['theEnvironmentProperties'][0]['theComponents'][0]['theType'],'process')
    rv = self.app.delete('/api/trust_boundaries/name/Shibboleth?session_id=test')
예제 #25
0
 def test_generate_obstacle_from_exception(self):
   method = 'test_generate_obstacle_from_exception'
   url = '/api/usecases/name/%s?session_id=test' % quote(self.existing_usecase_name)
   rv = self.app.get(url)
   if (sys.version_info > (3,)):
     responseData = rv.data.decode('utf-8')
   else:
     responseData = rv.data
   uc = jsonpickle.decode(responseData)
   url = '/api/usecases/environment/Psychosis/step/' + quote('Researcher does something') + '/exception/anException/generate_obstacle?session_id=test' 
   existing_uc_dict = {
     'session_id': 'test',
     'object': uc
   }
   rv = self.app.post(url, content_type='application/json', data=jsonpickle.encode(existing_uc_dict))
   self.assertIsNotNone(rv.data, 'No response')
   if (sys.version_info > (3,)):
     responseData = rv.data.decode('utf-8')
   else:
     responseData = rv.data
   json_resp = jsonpickle.decode(responseData)
   self.assertIsNotNone(json_resp)
   self.assertIsInstance(json_resp, dict)
   message = json_resp.get('message', None)
   self.assertIsNotNone(message, 'No message in response')
   self.logger.info('[%s] Message: %s', method, message)
   self.assertGreater(message.find('generated from exception'), -1, 'The obstacle was not generated')
예제 #26
0
  def test_delete(self):
    method = 'test_delete'

    rv = self.app.get('/api/persona_characteristics/name/Managers%20delegate%20security%20decisions?session_id=test')
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    pc = jsonpickle.decode(responseData)
    pc['theCharacteristicSynopsis'] = {"theActor" : "Claire", "theActorType" : "persona", "theSynopsis" : "Security delegated", "theDimension" : "goal"}
    pcDict = {'session_id' : 'test','object' : pc}
    rv = self.app.put('/api/persona_characteristics/name/Managers%20delegate%20security%20decisions?session_id=test', content_type='application/json', data=jsonpickle.encode(pcDict))
    url = '/api/usecases/name/%s?session_id=test' % quote(self.prepare_new_usecase().name())
    new_usecase_body = self.prepare_json()

    self.app.delete(url)
    self.logger.info('[%s] Object to delete: %s', method, new_usecase_body)
    self.app.post('/api/usecases', content_type='application/json', data=new_usecase_body)
    self.logger.info('[%s] URL: %s', method, url)
    rv = self.app.delete(url)
    if (sys.version_info > (3,)):
      responseData = rv.data.decode('utf-8')
    else:
      responseData = rv.data
    self.logger.info('[%s] Response data: %s', method, responseData)
    self.assertIsNotNone(responseData, 'No response')
    json_resp = jsonpickle.decode(responseData)
    self.assertIsInstance(json_resp, dict, 'The response cannot be converted to a dictionary')
    message = json_resp.get('message', None)
    self.assertIsNotNone(message, 'No message in response')
    self.logger.info('[%s] Message: %s\n', method, message)
예제 #27
0
	def get_context_data(self, **kwargs):
		user = self.request.session['user']
		context = super(LoginRequired, self).get_context_data(**kwargs)
		context['user'] = jsonpickle.decode(user)
		context['baiguullaga'] = jsonpickle.decode(user).user_id.position_id.dep_id.baiguullaga
		context['tasag'] = Tasag.objects.filter(baiguullaga = context['baiguullaga'])
		context['albantushaal'] = AlbanTushaal.objects.filter(dep_id = context['tasag'])
		context['ajiltan'] = Ajiltan.objects.filter(position_id = context['albantushaal'])
		return context
예제 #28
0
파일: server.py 프로젝트: shbroster/pypool
def _add_booking(my_request):
    logger.info("Attempt to make a booking")
    required_kit = jsonpickle.decode(my_request.form["required_kit"])
    #todo fail early
    user = jsonpickle.decode(my_request.form["user"])
    duration = jsonpickle.decode(my_request.form["duration"])
    logger.info("'%s' requested %s for %ss", user, required_kit, duration)
    bookings = manager.add_booking(required_kit, user, duration)
    return bookings
예제 #29
0
    def test_json(self):
        expect = self.obj
        pickle = jsonpickle.encode(self.obj)
        actual = jsonpickle.decode(pickle)
        self.assertEqual(actual.name, expect.name)
        self.assertEqual(actual.child, expect.child)

        actual = jsonpickle.decode(self.expected_json)
        self.assertEqual(self.obj.name, actual.name)
        self.assertEqual(type(self.obj), type(actual))
예제 #30
0
    def assertEncodeDecode(self, json_input):
        expect = SAMPLE_DATA
        actual = jsonpickle.decode(json_input)
        self.assertEqual(expect['things'][0].name, actual['things'][0].name)
        self.assertEqual(expect['things'][0].child, actual['things'][0].child)

        pickled = jsonpickle.encode(SAMPLE_DATA)
        actual = jsonpickle.decode(pickled)
        self.assertEqual(expect['things'][0].name, actual['things'][0].name)
        self.assertEqual(expect['things'][0].child, actual['things'][0].child)
예제 #31
0
파일: test.py 프로젝트: BenLin0/deepparser
import json, jsonpickle

a1 = """{ "score ":-0.9719291925430298, "answer ": "可以关注微信公众号‘’京东家电小秘书‘’点击页面底部菜单选项;一点无忧‘自助办理、或点击召唤小秘书联系家电专属京东客服进行人工咨询哦", "question ": "京东客服在哪里 ", "sourceList ":[ "cluster "], "optional ":{ "qwords ": "京东 客服 在 哪里 ", "awords ": "亲 可以 关注 微信 公众 号 ‘ ’ 京东 家电 小 秘书 ‘ ’ 点击 页面 底部 菜单 选项 ; 一点 无忧 ‘ 自助 办理 、 或 点击 召唤 小 秘书 联系 家电 专属 京东 客服 进行 人工 咨询 哦 ` , "}}"""
b1 = jsonpickle.decode(a1)
print(b1["answer "])

a2 = """{ "score ":-0.9719291925430298, "answer ": "可以关注微信公众号‘’京东家电小秘书‘’点击页面底部菜单选项;一点无忧‘自助办理、或点击召唤小秘书联系家电专属京东客服进行人工咨询哦", ", "question ": "京东客服在哪里 ", "sourceList ":[ "cluster "], "optional ":{ "qwords ": "京东 客服 在 哪里 ", "awords ": "亲 可以 关注 微信 公众 号 ‘ ’ 京东 家电 小 秘书 ‘ ’ 点击 页面 底部 菜单 选项 ; 一点 无忧 ‘ 自助 办理 、 或 点击 召唤 小 秘书 联系 家电 专属 京东 客服 进行 人工 咨询 哦 ` , "}}"""
b1 = jsonpickle.decode(a2)
a2 = json.dumps(a2)
b2 = jsonpickle.decode(a2)
print(b2["answer "])
예제 #32
0
def scanSingleHost(user, last_host, last_open_ports, last_secured_ports,
                   last_settings):

    ScanStatus().save()
    user = jp.decode(user)
    last_host = jp.decode(last_host)
    last_open_ports = jp.decode(last_open_ports)
    last_secured_ports = jp.decode(last_secured_ports)
    last_settings = jp.decode(last_settings)

    secure_proxy = last_host.secure_proxy_ip.split(":")[0]
    secure_port = int(last_host.secure_proxy_ip.split(":")[1])
    unsecure_proxy = last_host.unsecure_proxy_ip.split(":")[0]
    unsecure_port = int(last_host.unsecure_proxy_ip.split(":")[1])

    if len(last_open_ports.unsecured_ports) > 0:
        open_ports = [
            int(x.strip()) for x in last_open_ports.unsecured_ports.split(",")
        ]
    else:
        open_ports = []

    mulScan_unsecuredPorts = MultiScan(targets=[last_host.ip],
                                       ports=open_ports,
                                       threads=last_settings.threads,
                                       timeout=last_settings.timeout,
                                       proxy_ip=[secure_proxy, unsecure_proxy],
                                       proxy_port=[secure_port, unsecure_port])

    stat = ScanStatus.objects.filter().last()

    open_port_res = dict(mulScan_unsecuredPorts.run_proxy_scan(False))
    open_res_write = OpenPortResult(
        added_by=User.objects.get(username=user.username),
        scanned_on=datetime.now(),
        host=Host.objects.get(host_id=last_host.host_id),
        open_ports=", ".join([
            str(x) for x in open_port_res[unsecure_proxy + "::" +
                                          last_host.ip]["Opened Ports"]
        ]),
        closed_ports=", ".join([
            str(x) for x in open_port_res[unsecure_proxy + "::" +
                                          last_host.ip]["Closed Ports"]
        ]),
        runtime=open_port_res[unsecure_proxy + "::" + last_host.ip]["Runtime"])

    open_res_write.save()

    ScanStatus.objects.update_or_create(status_id=stat.status_id,
                                        defaults={'open_scan_status': True})

    if len(last_secured_ports.secured_ports) > 0:
        secured_ports = [
            int(x.strip()) for x in last_secured_ports.secured_ports.split(",")
        ]
    else:
        secure_port = []

    mulScan_securedPorts = MultiScan(targets=[last_host.ip],
                                     ports=secured_ports,
                                     threads=last_settings.threads,
                                     timeout=last_settings.timeout,
                                     proxy_ip=[secure_proxy, unsecure_proxy],
                                     proxy_port=[secure_port, unsecure_port])

    secure_port_res = dict(mulScan_securedPorts.run_proxy_scan(True))
    unsecure_port_res = dict(mulScan_securedPorts.run_proxy_scan(False))

    secure_res_write = SecurePortResult(
        added_by=User.objects.get(username=user.username),
        scanned_on=datetime.now(),
        host=Host.objects.get(host_id=last_host.host_id),
        secure_open_ports=", ".join([
            str(x) for x in secure_port_res[secure_proxy + "::" +
                                            last_host.ip]["Opened Ports"]
        ]),
        secure_closed_ports=", ".join([
            str(x) for x in secure_port_res[secure_proxy + "::" +
                                            last_host.ip]["Closed Ports"]
        ]),
        secure_scan_runtime=secure_port_res[secure_proxy + "::" +
                                            last_host.ip]["Runtime"],
        unsecure_open_ports=", ".join([
            str(x) for x in unsecure_port_res[unsecure_proxy + "::" +
                                              last_host.ip]["Opened Ports"]
        ]),
        unsecure_closed_ports=", ".join([
            str(x) for x in unsecure_port_res[unsecure_proxy + "::" +
                                              last_host.ip]["Closed Ports"]
        ]),
        unsecure_scan_runtime=unsecure_port_res[unsecure_proxy + "::" +
                                                last_host.ip]["Runtime"])

    secure_res_write.save()

    ScanStatus.objects.update_or_create(status_id=stat.status_id,
                                        defaults={'secure_scan_status': True})
예제 #33
0
def _read_json(environment, call):
    """Undecode the json data."""
    fixture = load_fixture(f"plugwise/{environment}/{call}.json")
    return jsonpickle.decode(fixture)
예제 #34
0
 def test_reduce_listitems_append(self):
     'Test reduce with listitems set (as a generator), yielding single items'
     instance = PickleProtocol2ReduceListitemsAppend()
     encoded = jsonpickle.encode(instance)
     decoded = jsonpickle.decode(encoded)
     self.assertEqual(decoded.inner, ['foo', 'bar'])
예제 #35
0
 def test_reduce_complex_zero(self):
     instance = 0j
     encoded = jsonpickle.encode(instance)
     decoded = jsonpickle.decode(encoded)
     self.assertEqual(decoded, instance)
예제 #36
0
 def test_string_key_not_requiring_escape_dict_keys_with_keys_enabled(self):
     """test that string keys that do not require escaping are not escaped"""
     str_dict = {'name': [1, 2]}
     pickled = jsonpickle.encode(str_dict, keys=True)
     unpickled = jsonpickle.decode(pickled)
     self.assertTrue('name' in unpickled)
예제 #37
0
 def test_list_roundtrip(self):
     data = [1, 2, 3]
     newdata = jsonpickle.decode(jsonpickle.encode(data))
     self.assertEqual(data, newdata)
예제 #38
0
 def load(self):
     with open("settings.json", 'r') as setting_file:
         obj = jsonpickle.decode(setting_file.read())
     self.__dict__.update(obj)
     return
예제 #39
0
    def test_put(self):
        method = 'test_put'
        url = '/api/dataflows'
        self.logger.info('[%s] URL: %s', method, url)
        new_dataflow_body = self.prepare_json()

        rv = self.app.post(url,
                           content_type='application/json',
                           data=new_dataflow_body)
        if (sys.version_info > (3, )):
            responseData = rv.data.decode('utf-8')
        else:
            responseData = rv.data
        self.logger.debug('[%s] Response data: %s', method, responseData)
        json_resp = jsonpickle.decode(responseData)
        self.assertIsNotNone(json_resp, 'No results after deserialization')

        dataflow_to_update = self.prepare_new_dataflow()
        dataflow_to_update.theName = 'Edited test dataflow'
        upd_env_body = self.prepare_json(dataflow=dataflow_to_update)
        rv = self.app.put(
            '/api/dataflows/name/acknowledge/environment/Psychosis?session_id=test',
            data=upd_env_body,
            content_type='application/json')
        self.assertIsNotNone(rv.data, 'No response')
        if (sys.version_info > (3, )):
            responseData = rv.data.decode('utf-8')
        else:
            responseData = rv.data
        json_resp = jsonpickle.decode(responseData)
        self.assertIsNotNone(json_resp)
        self.assertEqual(json_resp['message'], 'Edited test dataflow updated')

        rv = self.app.get(
            '/api/dataflows/name/Edited%20test%20dataflow/environment/Psychosis?session_id=test'
        )
        if (sys.version_info > (3, )):
            responseData = rv.data.decode('utf-8')
        else:
            responseData = rv.data
        upd_dataflow = jsonpickle.decode(responseData)
        self.assertIsNotNone(upd_dataflow, 'Unable to decode JSON data')

        self.assertEqual(upd_dataflow['theName'], dataflow_to_update.name())
        self.assertEqual(upd_dataflow['theEnvironmentName'],
                         dataflow_to_update.environment())
        self.assertEqual(upd_dataflow['theFromName'],
                         dataflow_to_update.fromName())
        self.assertEqual(upd_dataflow['theFromType'],
                         dataflow_to_update.fromType())
        self.assertEqual(upd_dataflow['theToName'],
                         dataflow_to_update.toName())
        self.assertEqual(upd_dataflow['theToType'],
                         dataflow_to_update.toType())

        rv = self.app.delete(
            '/api/dataflows/name/Edited%20test%20dataflow/environment/Psychosis?session_id=test'
        )
        self.assertIsNotNone(rv.data, 'No response')
        if (sys.version_info > (3, )):
            responseData = rv.data.decode('utf-8')
        else:
            responseData = rv.data
        json_resp = jsonpickle.decode(responseData)
        self.assertIsNotNone(json_resp)
        self.assertEqual(json_resp['message'], 'Edited test dataflow deleted')
예제 #40
0
def find_repeat_detections(inputFilename, outputFilename, options=None):

    ##%% Input handling

    if options is None:
        options = RepeatDetectionOptions()

    toReturn = RepeatDetectionResults()

    ##%% Load file

    detectionResults, otherFields = load_api_results(
        inputFilename,
        normalize_paths=True,
        filename_replacements=options.filenameReplacements)
    toReturn.detectionResults = detectionResults
    toReturn.otherFields = otherFields

    ##%% Separate files into directories

    # This will be a map from a directory name to smaller data frames
    rowsByDirectory = {}

    # This is a mapping back into the rows of the original table
    filenameToRow = {}

    # TODO: in the case where we're loading an existing set of FPs after manual filtering,
    # we should load these data frames too, rather than re-building them from the input.

    print('Separating files into directories...')

    # iRow = 0; row = detectionResults.iloc[0]
    for iRow, row in detectionResults.iterrows():
        relativePath = row['file']
        dirName = os.path.dirname(relativePath)

        if options.nDirLevelsFromLeaf > 0:
            iLevel = 0
            while (iLevel < options.nDirLevelsFromLeaf):
                iLevel += 1
                dirName = os.path.dirname(dirName)
        assert len(dirName) > 0

        if not dirName in rowsByDirectory:
            # Create a new DataFrame with just this row
            # rowsByDirectory[dirName] = pd.DataFrame(row)
            rowsByDirectory[dirName] = []

        rowsByDirectory[dirName].append(row)

        assert relativePath not in filenameToRow
        filenameToRow[relativePath] = iRow

    # Convert lists of rows to proper DataFrames
    dirs = list(rowsByDirectory.keys())
    for d in dirs:
        rowsByDirectory[d] = pd.DataFrame(rowsByDirectory[d])

    toReturn.rowsByDirectory = rowsByDirectory
    toReturn.filenameToRow = filenameToRow

    print('Finished separating {} files into {} directories'.format(
        len(detectionResults), len(rowsByDirectory)))

    ##% Look for matches (or load them from file)

    dirsToSearch = list(rowsByDirectory.keys())[0:options.debugMaxDir]

    # length-nDirs list of lists of DetectionLocation objects
    suspiciousDetections = [None] * len(dirsToSearch)

    # Are we actually looking for matches, or just loading from a file?
    if len(options.filterFileToLoad) == 0:

        # We're actually looking for matches...
        print('Finding similar detections...')

        allCandidateDetections = [None] * len(dirsToSearch)

        if not options.bParallelizeComparisons:

            options.pbar = None
            # iDir = 0; dirName = dirsToSearch[iDir]
            for iDir, dirName in enumerate(tqdm(dirsToSearch)):
                allCandidateDetections[iDir] = find_matches_in_directory(
                    dirName, options, rowsByDirectory)

        else:

            options.pbar = tqdm(total=len(dirsToSearch))
            allCandidateDetections = Parallel(
                n_jobs=options.nWorkers,
                prefer='threads')(delayed(find_matches_in_directory)(
                    dirName, options, rowsByDirectory)
                                  for dirName in tqdm(dirsToSearch))

        print('\nFinished looking for similar bounding boxes')

        ##%% Find suspicious locations based on match results

        print('Filtering out repeat detections...')

        nImagesWithSuspiciousDetections = 0
        nSuspiciousDetections = 0

        # For each directory
        #
        # iDir = 51
        for iDir in range(len(dirsToSearch)):

            # A list of DetectionLocation objects
            suspiciousDetectionsThisDir = []

            # A list of DetectionLocation objects
            candidateDetectionsThisDir = allCandidateDetections[iDir]

            for iLocation, candidateLocation in enumerate(
                    candidateDetectionsThisDir):

                # occurrenceList is a list of file/detection pairs
                nOccurrences = len(candidateLocation.instances)

                if nOccurrences < options.occurrenceThreshold:
                    continue

                nImagesWithSuspiciousDetections += nOccurrences
                nSuspiciousDetections += 1

                suspiciousDetectionsThisDir.append(candidateLocation)
                # Find the images corresponding to this bounding box, render boxes

            suspiciousDetections[iDir] = suspiciousDetectionsThisDir

        print(
            'Finished searching for repeat detections\nFound {} unique detections on {} images that are suspicious'
            .format(nSuspiciousDetections, nImagesWithSuspiciousDetections))

    else:

        print('Bypassing detection-finding, loading from {}'.format(
            options.filterFileToLoad))

        # Load the filtering file
        detectionIndexFileName = options.filterFileToLoad
        sIn = open(detectionIndexFileName, 'r').read()
        suspiciousDetections = jsonpickle.decode(sIn)
        filteringBaseDir = os.path.dirname(options.filterFileToLoad)
        assert len(suspiciousDetections) == len(dirsToSearch)

        nDetectionsRemoved = 0
        nDetectionsLoaded = 0

        # We're skipping detection-finding, but to see which images are actually legit false
        # positives, we may be looking for physical files or loading from a text file.
        fileList = None
        if options.filteredFileListToLoad is not None:
            with open(options.filteredFileListToLoad) as f:
                fileList = f.readlines()
                fileList = [x.strip() for x in fileList]
            nSuspiciousDetections = sum([len(x) for x in suspiciousDetections])
            print(
                'Loaded false positive list from file, will remove {} of {} suspicious detections'
                .format(len(fileList), nSuspiciousDetections))

        # For each directory
        # iDir = 0; detections = suspiciousDetections[0]
        for iDir, detections in enumerate(suspiciousDetections):

            bValidDetection = [True] * len(detections)
            nDetectionsLoaded += len(detections)

            # For each detection that was present before filtering
            # iDetection = 0; detection = detections[iDetection]
            for iDetection, detection in enumerate(detections):

                # Are we checking the directory to see whether detections were actually false positives,
                # or reading from a list?
                if fileList is None:

                    # Is the image still there?
                    imageFullPath = os.path.join(
                        filteringBaseDir,
                        detection.sampleImageRelativeFileName)

                    # If not, remove this from the list of suspicious detections
                    if not os.path.isfile(imageFullPath):
                        nDetectionsRemoved += 1
                        bValidDetection[iDetection] = False

                else:

                    if detection.sampleImageRelativeFileName not in fileList:
                        nDetectionsRemoved += 1
                        bValidDetection[iDetection] = False

            # ...for each detection

            nRemovedThisDir = len(bValidDetection) - sum(bValidDetection)
            if nRemovedThisDir > 0:
                print('Removed {} of {} detections from directory {}'.format(
                    nRemovedThisDir, len(detections), iDir))

            detectionsFiltered = list(compress(detections, bValidDetection))
            suspiciousDetections[iDir] = detectionsFiltered

        # ...for each directory

        print('Removed {} of {} total detections via manual filtering'.format(
            nDetectionsRemoved, nDetectionsLoaded))

    # ...if we are/aren't finding detections (vs. loading from file)

    toReturn.suspiciousDetections = suspiciousDetections

    if options.bRenderHtml:

        # Render problematic locations with html (loop)

        print('Rendering html')

        nDirs = len(dirsToSearch)
        directoryHtmlFiles = [None] * nDirs

        if options.bParallelizeRendering:

            # options.pbar = tqdm(total=nDirs)
            options.pbar = None

            directoryHtmlFiles = Parallel(
                n_jobs=options.nWorkers,
                prefer='threads')(delayed(render_images_for_directory)(
                    iDir, directoryHtmlFiles, suspiciousDetections, options)
                                  for iDir in tqdm(range(nDirs)))

        else:

            options.pbar = None

            # For each directory
            # iDir = 51
            for iDir in range(nDirs):
                # Add this directory to the master list of html files
                directoryHtmlFiles[iDir] = render_images_for_directory(
                    iDir, directoryHtmlFiles, suspiciousDetections, options)

            # ...for each directory

        # Write master html file

        masterHtmlFile = os.path.join(options.outputBase, 'index.html')
        os.makedirs(options.outputBase, exist_ok=True)
        toReturn.masterHtmlFile = masterHtmlFile

        with open(masterHtmlFile, 'w') as fHtml:

            fHtml.write('<html><body>\n')
            fHtml.write(
                '<h2><b>Repeat detections by directory</b></h2></br>\n')

            for iDir, dirHtmlFile in enumerate(directoryHtmlFiles):

                if dirHtmlFile is None:
                    continue

                relPath = os.path.relpath(dirHtmlFile, options.outputBase)
                dirName = dirsToSearch[iDir]

                # Remove unicode characters before formatting
                relPath = relPath.encode('ascii', 'ignore').decode('ascii')
                dirName = dirName.encode('ascii', 'ignore').decode('ascii')

                fHtml.write('<a href={}>{}</a><br/>\n'.format(
                    relPath, dirName))

            fHtml.write('</body></html>\n')

    # ...if we're rendering html

    toReturn.allRowsFiltered = update_detection_table(toReturn, options,
                                                      outputFilename)

    # Create filtering directory
    if options.bWriteFilteringFolder:

        print('Creating filtering folder...')

        dateString = datetime.now().strftime('%Y.%m.%d.%H.%M.%S')
        filteringDir = os.path.join(options.outputBase,
                                    'filtering_' + dateString)
        os.makedirs(filteringDir, exist_ok=True)

        # iDir = 0; suspiciousDetectionsThisDir = suspiciousDetections[iDir]
        for iDir, suspiciousDetectionsThisDir in enumerate(
                tqdm(suspiciousDetections)):

            # suspiciousDetectionsThisDir is a list of DetectionLocation objects
            # iDetection = 0; detection = suspiciousDetectionsThisDir[0]
            for iDetection, detection in enumerate(
                    suspiciousDetectionsThisDir):

                instance = detection.instances[0]
                relativePath = instance.filename
                inputFullPath = os.path.join(options.imageBase, relativePath)
                assert (os.path.isfile(inputFullPath)
                        ), 'Not a file: {}'.format(inputFullPath)
                outputRelativePath = 'dir{:0>4d}_det{:0>4d}.jpg'.format(
                    iDir, iDetection)
                outputFullPath = os.path.join(filteringDir, outputRelativePath)
                render_bounding_box(detection, inputFullPath, outputFullPath,
                                    15)
                detection.sampleImageRelativeFileName = outputRelativePath

        # Write out the detection index
        detectionIndexFileName = os.path.join(filteringDir,
                                              'detectionIndex.json')
        jsonpickle.set_encoder_options('json', sort_keys=True, indent=4)
        s = jsonpickle.encode(suspiciousDetections)
        with open(detectionIndexFileName, 'w') as f:
            f.write(s)
        toReturn.filterFile = detectionIndexFileName

        print('Done')

    # ...if we're writing filtering info

    return toReturn
예제 #41
0
 def load_from_file(cls, filepath):
     with open(filepath, 'r') as f:
         return jsonpickle.decode(f.read())
예제 #42
0
def load_from_json(file):
    with open(
            os.path.join(os.path.dirname(os.path.abspath(__file__)),
                         "data/%s.json" % file)) as f:
        return jsonpickle.decode(f.read())
예제 #43
0
        from_experiment = Experiment.objects.get(name=from_experiment_name)
    except:
        print "From experiment not found"
        sys.exit(0)

    try:
        to_experiment = Experiment.objects.get(name=to_experiment_name)
    except:
        print "To experiment not found"
        sys.exit(0)

    print "Found experiments"
    from_documents = Document.objects.filter(experiment=from_experiment)
    print "Extracted {} documents from from_experiment".format(
        len(from_documents))
    n_found = 0
    for document in from_documents:
        to_document = Document.objects.filter(experiment=to_experiment,
                                              name=document.name)
        if len(to_document) == 1:
            print "Found match for {}".format(document.name)
            n_found += 1
            to_document = to_document[0]
            to_metadata = {}
            from_metadata = jsonpickle.decode(document.metadata)
            for key, value in from_metadata.items():
                to_metadata[key] = value
            to_document.metadata = jsonpickle.encode(to_metadata)
            to_document.save()
    print "Found {} matches".format(n_found)
예제 #44
0
 def test_string_key_requiring_escape_dict_keys_with_keys_enabled(self):
     json_key_dict = {tags.JSON_KEY + '6': [1, 2]}
     pickled = jsonpickle.encode(json_key_dict, keys=True)
     unpickled = jsonpickle.decode(pickled, keys=True)
     self.assertEqual(unpickled[tags.JSON_KEY + '6'], [1, 2])
예제 #45
0
 def test_builtin_error(self):
     expect = AssertionError
     json = jsonpickle.encode(expect)
     actual = jsonpickle.decode(json)
     self.assertEqual(expect, actual)
     self.assertTrue(expect is actual)
예제 #46
0
 def test_int_dict_keys_with_keys_enabled(self):
     int_dict = {1000: [1, 2]}
     pickle = jsonpickle.encode(int_dict, keys=True)
     actual = jsonpickle.decode(pickle, keys=True)
     self.assertEqual(actual[1000], [1, 2])
예제 #47
0
 def test_tuple_roundtrip(self):
     data = (1, 2, 3)
     newdata = jsonpickle.decode(jsonpickle.encode(data))
     self.assertEqual(data, newdata)
예제 #48
0
 def test_int_dict_keys_defaults(self):
     int_dict = {1000: [1, 2]}
     pickle = jsonpickle.encode(int_dict)
     actual = jsonpickle.decode(pickle)
     self.assertEqual(actual['1000'], [1, 2])
예제 #49
0
 def test_reduce_dictitems(self):
     'Test reduce with dictitems set (as a generator)'
     instance = PickleProtocol2ReduceDictitems()
     encoded = jsonpickle.encode(instance)
     decoded = jsonpickle.decode(encoded)
     self.assertEqual(decoded.inner, {'foo': 'foo', 'bar': 'bar'})
예제 #50
0
 def test_None_dict_key_with_keys_enabled(self):
     expect = {None: None}
     obj = {None: None}
     pickle = jsonpickle.encode(obj, keys=True)
     actual = jsonpickle.decode(pickle, keys=True)
     self.assertEqual(expect, actual)
예제 #51
0
def load_config_from_file(file):
    config_file = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                               file)
    with open(config_file) as cf:
        configuration = jsonpickle.decode(cf.read())
    return configuration
예제 #52
0
 def test_unicode_dict_keys(self):
     uni = unichr(0x1234)
     pickle = jsonpickle.encode({uni: uni})
     actual = jsonpickle.decode(pickle)
     self.assertTrue(uni in actual)
     self.assertEqual(actual[uni], uni)
예제 #53
0
 def from_json_str(self, s):
     return jsonpickle.decode(s)
예제 #54
0
 def test_decode(self):
     actual = jsonpickle.decode(self.expected_json)
     self.assertEqual(self.obj.name, actual.name)
     self.assertEqual(type(self.obj), type(actual))
예제 #55
0
파일: sql.py 프로젝트: sim1/slackpro
 def value(self) -> Any:
     return decode(self._value)
예제 #56
0
 def test_encode_notunpicklable(self):
     expect = {'name': 'A name', 'child': None}
     pickle = jsonpickle.encode(self.obj, unpicklable=False)
     actual = jsonpickle.decode(pickle)
     self.assertEqual(expect['name'], actual['name'])
예제 #57
0
    jsonpickle_numpy.register_handlers()
    stopwords = utils_.GitMineUtils.STOPWORDS \
                + list(set(java_reserved + c_reserved + cpp_reserved + javascript_reserved + python_reserved))
    min_tok_len = 3
    net_size_in_days = 14
    multiplicity = 1000
    repo_locations = ['../data/dev_set/%s.json' % s for s in [
        'PhilJay_MPAndroidChart',
        'ReactiveX_RxJava',
        'palantir_plottable',
        'tensorflow_tensorflow',
    ]]

    for repo_loc in repo_locations:
        with open(repo_loc) as f:
            repo = jsonpickle.decode(f.read())
        with open(repo_loc[:-5] + '_truth.json') as f:
            truth = jsonpickle.decode(f.read())
        model, dictionary, cache = generate_tfidf(repo, stopwords, min_tok_len)
        similarity_config = {
            'dict': dictionary,
            'model': model,
            'min_len': min_tok_len,
            'stopwords': stopwords,
        }

        fingerprint = generate_dev_fingerprint(repo)
        temporal_config = {
            'fingerprint': fingerprint,
            'net_size_in_days': net_size_in_days,
        }
예제 #58
0
 def test_encode(self):
     expect = self.obj
     pickle = jsonpickle.encode(self.obj)
     actual = jsonpickle.decode(pickle)
     self.assertEqual(expect.name, actual.name)
     self.assertEqual(expect.child, actual.child)
예제 #59
0
 def from_json(cls, json):
     return jp.decode(json)
예제 #60
0
from src.broadcast import Broadcast
from src.config import *

from argparse import ArgumentParser
import requests
import asyncio
import jsonpickle

parser = ArgumentParser()
parser.add_argument('-n',
                    '--nodes',
                    default=5,
                    type=int,
                    help='number of nodes running')
args = parser.parse_args()
n = args.nodes

url = f'http://{BOOTSTRAP}/get_nodes'
headers = {'Content-type': 'application/json', 'Accept': 'text/plain'}
response = requests.get(url, headers)

hosts = jsonpickle.decode(response.json())['data']

b = Broadcast('test')
for host in hosts:
    b.add_peer(host)

print("Starting the stress test...")
responses = asyncio.run(b.broadcast('stress_test', n, 'POST'))