def setup(self, argv):
        """
        Setup the parameters for the simulation. Create result folder and logger file.
        :param argv: Dictionary list of the parameters
        """

        self.config_name = argv[
            "config_name"] if "config_name" in argv else self.root + "/configs/default_dog_vert_simulation_config.json"
        log_file = argv[
            "logfile"] if "logfile" in argv else os.path.expanduser("~").replace("\\", "/") + "/.log/locomotionSim.log"

        if "filename" in argv:
            self.save_file = argv["filename"]
        else:
            dirname = self.root + "/save"
            filename = "sim_" + datetime.datetime.now().strftime("%Y_%m_%d_%H_%M_%S_%f") + ".qsm"
            if not os.path.exists(dirname):
                os.makedirs(dirname)
            self.save_file = dirname + "/" + filename

        self.genome = eval(argv["genome"]) if "genome" in argv else False

        FileUtils.create_file(log_file)

        logging.config.fileConfig(self.root + "/etc/logging.conf",
                                  defaults={'logfilename': log_file, 'simLevel': "DEBUG"})
Example #2
0
    def test_get_empty_file(self):
        returned_f = FileUtils.get_empty_file()

        self.assertEqual(returned_f, {
            'version': FileUtils.LAST_FILE_VERSION,
            'content': '',
            'currentVideo': None,
            'videos': {},
            'syncNotesVideo': True
        })
    def get_file(self, file_id):
        service = self.CreateDrive()
        if service is None:
            return

        # Requests are expected to pass the file_id query parameter.
        logging.info('Get file %s', file_id)

        if file_id:
            # Fetch the file metadata by making the service.files().get method of
            # the Drive API.
            f = service.files().get(fileId=file_id).execute()
            downloadUrl = f.get('downloadUrl')
            # If a download URL is provided in the file metadata, use it to make an
            # authorized request to fetch the file ontent. Set this content in the
            # data to return as the 'content' field. If there is no downloadUrl,
            # just set empty content.
            if downloadUrl:
                logging.debug('Downloading the file from %s', downloadUrl)
                resp, raw_content = service._http.request(downloadUrl)
                logging.debug('Response status : %s', resp.status)
                logging.debug('Raw content : %s', raw_content)
                if resp and resp.status == int(200) and raw_content:
                    try:
                        json_content = json.loads(raw_content)
                        f.update(json_content)
                    except ValueError:
                        logging.info("ValueError when decoding raw content in JSON")
                        f.update(FileUtils.get_empty_file())
                else:
                    logging.debug("No content or error response")
                    f.update(FileUtils.get_empty_file())
            else:
                logging.debug('No download URL')
                f.update(FileUtils.get_empty_file())
        else:
            f = None
            # Generate a JSON response with the file data and return to the client.

        return f
    def _extract_values_set(self):
        '''
        Loads the node data from the ground truth network file and
        fetches the values
        '''
        bayesian_network_filename = FileUtils.get_network_filename(self._network_name)
        nodes_data = None
        with open(bayesian_network_filename) as file:
            data = file.read()
            nodes_data = loads(data)

        for node in nodes_data['Vdata']:
            self._variables_values_set[node] = nodes_data['Vdata'][node]['vals']
Example #5
0
    def test_transformations_v1_to_v2(self):
        v1_f = {
            'content': '',
            'video': 'test',
            'syncNotesVideo': {
                'enabled': True,
                '1': 10.00
            }
        }

        v2_f = FileUtils.transformation_v1_to_v2(v1_f)

        self.assertEqual(v2_f['version'], 2)
        self.assertNotIn('video', v2_f)
        self.assertIsNotNone(v2_f['videos']['test'])
        self.assertEqual(v2_f['videos']['test']['1'], {'time':10.00})
 def _extract_data(self, format):
     '''
     Parses the generated data file and populates the data into a dictionary.
     '''
     data_filename = FileUtils.get_observed_data_filename(self._network_name)
     with open(data_filename) as file:
         self.get_csv_data(file)
         if format == 'json':
             #  Now we convert the data to json format
             keys = self._data_vectors.keys()
             data_length = len(self._data_vectors[keys[0]])
             data_dict_list = []
             for i in xrange(data_length):
                 data_dict = {}
                 for j in xrange(len(keys)):
                     data_dict[keys[j]] = self._data_vectors[keys[j]][i]
                 data_dict_list.append(data_dict)
             self._data_vectors = data_dict_list
Example #7
0
    def test_flatten_sync(self):
        input_sync = {
            'video1': {
                2: {
                    'time': 0,
                },
                1: {
                    'time': 0,
                }
            },
            'video2': {
                0: {
                    'time': 0,
                },
                3: {
                    'time': 0,
                }
            }
        }

        expected_output = {
            0: {
                'url': 'video2',
                'time': 0,
            },
            1: {
                'url': 'video1',
                'time': 0,
            },
            2: {
                'url': 'video1',
                'time': 0,
            },
            3: {
                'url': 'video2',
                'time': 0,
            }
        }

        expected_output = collections.OrderedDict(sorted(expected_output.items()))
        effective_output = FileUtils.flatten_sync(input_sync)

        self.assertDictEqual(expected_output, effective_output)
Example #8
0
    def test_to_enml_with_screenshots(self):
        file = {
            'id': 'test',
            'content': 'test' + '\n' + '<{0}>'.format(FileUtils.SNAPSHOT_KEY),
            'videos': {
                'video1': {
                    2: {
                        'time': 0,
                        'snapshot': None
                    },
                    1: {
                        'time': 0,
                        'snapshot': 'snapshot'
                    }
                },
                'video2': {
                    0: {
                        'time': 0,
                        'snapshot': None
                    },
                    3: {
                        'time': 0,
                        'snapshot': None
                    }
                }
            }
        }

        base_url = 'http://test.videonot.es/edit/' + file['id']

        expected_enml = [
            '<a href="{0}?l=1">+</a> test'.format(base_url),
            '<br></br>',
            '<br></br>',
            '<img src="{0}"></img>'.format('snapshot'),
            '<br></br>',
            '<a href="{0}">{0}</a>'.format('video1'),
            '<br></br><br></br>'
        ]

        content_enml = FileUtils.to_ENML(file, base_url)
        self.assertEqual(expected_enml, content_enml)
Example #9
0
    def get(self):
        """Called when HTTP GET requests are received by the web application.

        Use the query parameter file_id to fetch the required file's metadata then
        content and return it as a JSON object.

        Since DrEdit deals with text files, it is safe to dump the content directly
        into JSON, but this is not the case with binary files, where something like
        Base64 encoding is more appropriate.
        """
        try:
            f = self.get_file(self.request.get('file_id'))
            f = FileUtils.transformations(f)

            self.RespondJSON(f)
        except AccessTokenRefreshError:
            # Catch AccessTokenRefreshError which occurs when the API client library
            # fails to refresh a token. This occurs, for example, when a refresh token
            # is revoked. When this happens the user is redirected to the
            # Authorization URL.
            logging.info('AccessTokenRefreshError')
            return self.abort(401)
Example #10
0
    def test_to_enml_without_screenshots(self):
        file = {
            'id': 'test',
            'content': 'test' + '\n' + 'test2',
            'videos': {
                'video1': {
                    2: {
                        'time': 0,
                        'snapshot': None
                    },
                    1: {
                        'time': 0,
                        'snapshot': None
                    }
                },
                'video2': {
                    0: {
                        'time': 0,
                        'snapshot': None
                    },
                    3: {
                        'time': 0,
                        'snapshot': None
                    }
                }
            }
        }

        base_url = 'http://test.videonot.es/edit/' + file['id']

        expected_enml = [
            '<a href="{0}?l=1">+</a> test'.format(base_url),
            '<br></br>',
            '<a href="{0}?l=2">+</a> test2'.format(base_url),
            '<br></br>',
        ]

        content_enml = FileUtils.to_ENML(file, base_url)
        self.assertEqual(expected_enml, content_enml)
Example #11
0
def analyze(filename, known_words_filepath, not_known_words_filepath, print_example):
    known_words = set(FileUtils.read(known_words_filepath).split())
    not_known_words = set(FileUtils.read(not_known_words_filepath).split())
    tmp_filepath = FileUtils.random_path()
    output_filepath = tmp_filepath + '.xml'
    FileUtils.copy(filename, tmp_filepath)

    croncob_word_list = os.path.join('data', 'corncob_lowercase.txt')
    word_filter = WordFilterFactory.create_word_filter(croncob_word_list)

    cmd = ['java',
           '-cp',
           'stanford-corenlp-full/stanford-corenlp-3.3.1.jar:stanford-corenlp-full/stanford-corenlp-3.3.1-models.jar:stanford-corenlp-full/xom.jar:stanford-corenlp-full/joda-time.jar:stanford-corenlp-full/jollyday.jar:stanford-corenlp-full/ejml-0.23.jar',
           '-Xmx2g',
           'edu.stanford.nlp.pipeline.StanfordCoreNLP',
           '-annotators',
           'tokenize,ssplit,pos,lemma',
           '-file',
           tmp_filepath,
           '-outputDirectory',
           '/tmp/'
    ]
    subprocess.call(cmd)
    raw_output = FileUtils.read(output_filepath)
    d = xmltodict.parse(raw_output)
    sentences = d['root']['document']['sentences']['sentence']

    candidate_words = defaultdict(dict)

    def word_filter_fun(word, lemma, tag):
        del word
        del tag
        return word_filter.isok(lemma)

    def adjective_filter_fun(word, lemma, tag):
        del word
        del lemma
        if tag in ['JJR', 'JJS']:
            return False
        else:
            return True

    filters = [
            word_filter_fun,
            adjective_filter_fun
    ]

    for sentence_dict in sentences:
        tokens = sentence_dict['tokens']['token']
        if not isinstance(tokens, list):
            continue

        last_offset = int(tokens[0]['CharacterOffsetBegin'])
        sentence_raw = ''
        for token in tokens:
            word = token['word']
            begin_offset = int(token['CharacterOffsetBegin'])
            sentence_raw += (begin_offset - last_offset) * ' '
            sentence_raw += word
            last_offset = int(token['CharacterOffsetEnd'])

        for token in tokens:
            word = token['word']
            lemma = token['lemma']
            tag = token['POS']

            if tag in TAG_TO_PART_OF_SPEECH:
                ok = True
                for filter_fun in filters:
                    if not filter_fun(word, lemma, tag):
                        ok = False
                        break
                if ok:
                    candidate_words[(lemma, TAG_TO_PART_OF_SPEECH[tag])] = {
                        'example_sentence': sentence_raw,
                        'word': word
                    }

    not_known = []
    for ((lemma, part_of_speech), d) in candidate_words.iteritems():
        if lemma not in known_words and lemma not in not_known_words:
            not_known.append((lemma, part_of_speech, d))

    for (lemma, part_of_speech, d) in not_known:
        word = d['word']
        example_sentence = d['example_sentence']
        out = '(%s.) %s' % (
            convert_part_of_speech(part_of_speech),
            lemma
            )

        if print_example:
            line = utils.fill_suffix(out, 22, ' ') + ' # ' + example_sentence
            match_pos = re.search(word, example_sentence).start()
            print line.encode('utf-8')
            print ((match_pos + 25) * ' ') + (len(word) * '^')
        else:
            print out.encode('utf-8')
Example #12
0
    def put(self):
        """
        Called when HTTP PUT requests are received by the web application.

        The PUT body is JSON which is deserialized and used as values to update
        a file in Drive. The authorization access token for this action is
        retreived from the data store.
        """

        # Create a Drive service
        service = self.CreateDrive()
        if service is None:
            return

        # Load the data that has been posted as JSON
        logging.debug('Get JSON data')
        data = self.RequestJSON()
        logging.debug('JSON data retrieved %s', json.dumps(data))

        logging.info('Updating file %s', data['id'])

        # Create a new file data structure.
        content = FileUtils.get_content_from_data(data)
        #data['indexableText'] = {'text': content['content']}

        max_try = 5
        for n in range(0, max_try):
            try:
                if content is not None:
                    # Make an update request to update the file. A MediaInMemoryUpload
                    # instance is used to upload the file body. Because of a limitation, this
                    # request must be made in two parts, the first to update the metadata, and
                    # the second to update the body.
                    resource = service.files().update(
                        fileId=data['id'],
                        newRevision=self.request.get('newRevision', False),
                        body=data,
                        media_body=MediaInMemoryUpload(
                            content, data['mimeType'], resumable=True)
                    ).execute()
                else:
                    # Only update the metadata, a patch request is prefered but not yet
                    # supported on Google App Engine; see
                    # http://code.google.com/p/googleappengine/issues/detail?id=6316.
                    resource = service.files().update(
                        fileId=data['id'],
                        newRevision=self.request.get('newRevision', False),
                        body=data).execute()
                    # Respond with the new file id as JSON.
                return self.RespondJSON({'id': resource['id']})
            except HttpError, http_error:
                logging.getLogger("error").exception("Try #%d: Exception occurred when updating file", n)
                # HTTP status code 403 indicates that the app is not authorized to save the file (third-party app disabled, user without access, etc.)
                # Don't need to try several times
                if http_error.resp.status == 403:
                    return self.abort(403)
                else:
                    time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
            except HTTPException:
                logging.getLogger("error").exception("Try #%d: Exception occurred when updating file", n)
                time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
Example #13
0
    def post(self):
        """
        Called when HTTP POST requests are received by the web application.

        The POST body is JSON which is deserialized and used as values to create a
        new file in Drive. The authorization access token for this action is
        retrieved from the data store.
        """

        # Create a Drive service
        service = self.CreateDrive()
        if service is None:
            return

        # Load the data that has been posted as JSON
        logging.debug('Get JSON data')
        data = self.RequestJSON()
        logging.debug('JSON data retrieved %s', json.dumps(data))

        content = FileUtils.get_content_from_data(data)

        max_try = 5
        for n in range(0, max_try):
            try:
                if 'templateId' in data:
                    body = {'title': 'Your notes'}
                    resource = service.files().copy(fileId=data['templateId'], body=body).execute()
                else:
                    # Create a new file data structure.
                    resource = {
                        'title': data['title'],
                        'description': data['description'],
                        'mimeType': data['mimeType'],
                    }

                    if 'parent' in data and data['parent']:
                        logging.debug('Creating from a parent folder %s', data['parent'])
                        default_folder_id = data['parent']
                    else:
                        if 'defaultFolderId' in self.session and self.session['defaultFolderId']:
                            default_folder_id = self.session['defaultFolderId']
                        else:
                            default_folder_list = service.files().list(q='title="VideoNot.es"').execute()
                            if default_folder_list and 'items' in default_folder_list and len(default_folder_list['items']):
                                default_folder_id = default_folder_list['items'][0]['id']
                                self.session['defaultFolderId'] = default_folder_id
                            else:
                                folder_ressource = {
                                    'title': 'VideoNot.es',
                                    'mimeType': 'application/vnd.google-apps.folder'
                                }
                                default_folder = service.files().insert(body=folder_ressource).execute()
                                default_folder_id = default_folder['id']
                                self.session['defaultFolderId'] = default_folder_id
                    resource['parents'] = [{'id':default_folder_id}]

                    # Make an insert request to create a new file. A MediaInMemoryUpload
                    # instance is used to upload the file body.
                    logging.debug('Calling Drive API with content %s', str(content))
                    resource = service.files().insert(
                        body=resource,
                        media_body=MediaInMemoryUpload(
                            content,
                            data['mimeType'],
                            resumable=True)
                    ).execute()

                    if BaseHandler.is_production():
                        # clement_permission = {
                        #     'value': '[email protected]',
                        #     'type': 'user',
                        #     'role': 'reader'
                        # }

                        anyone_permission = {
                            'type': 'anyone',
                            'role': 'reader',
                            'withLink': True
                        }

                        # try:
                        #     logging.info('Add Clement as a reader')
                        #     service.permissions().insert(fileId=resource['id'], body=clement_permission).execute()
                        # except HttpError:
                        #     logging.info('Error when adding Clement as a reader')

                        try:
                            logging.info('Add anyone as a reader')
                            service.permissions().insert(fileId=resource['id'], body=anyone_permission).execute()
                        except HttpError:
                            logging.info('Error when adding anyone as a reader')

                # Respond with the new file id as JSON.
                logging.debug('New ID created %s', resource['id'])
                return self.RespondJSON({'id': resource['id']})
            except AccessTokenRefreshError:
                # In cases where the access token has expired and cannot be refreshed
                # (e.g. manual token revoking) redirect the user to the authorization page
                # to authorize.
                logging.info('AccessTokenRefreshError')
                return self.abort(401)
            except HttpError, http_error:
                logging.getLogger("error").exception("Try #%d: Exception occurred when creating file", n)
                # HTTP status code 403 indicates that the app is not authorized to save the file (third-party app disabled, user without access, etc.)
                # Don't need to try several times
                if http_error.resp.status == 403:
                    return self.abort(403)
                else:
                    time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))
            except HTTPException:
                logging.getLogger("error").exception("Try #%d: Exception occurred when creating file", n)
                time.sleep((2 ** n) + (random.randint(0, 1000) / 1000))