Ejemplo n.º 1
0
 def setUp(self):
     self.resizeimages = activity_ResizeImages(settings_mock, None, None, None, None)
     helpers.create_folder('tests/test_cdn')
     helpers.create_folder(self.resize_images_start_folder + testdata.session_example['expanded_folder'])
     shutil.copy('tests/files_source/elife-00353-fig1-v1.tif', self.resize_images_start_folder + testdata.session_example['expanded_folder'] + '/elife-00353-fig1-v1.tif')
     shutil.copy('tests/files_source/elife-00353-v1.pdf', self.resize_images_start_folder + testdata.session_example['expanded_folder'] + '/elife-00353-v1.pdf')
     shutil.copy('tests/files_source/elife-00353-v1.xml', self.resize_images_start_folder + testdata.session_example['expanded_folder'] + '/elife-00353-v1.xml')
    def store_predictions(self):
        """ dump predictions_dictionary in file """
        for model_name in self.predictions_dictionary.keys():
            folder_name = './CV/' + model_name
            create_folder(folder_name)

            for j in self.predictions_dictionary[model_name].keys():
                file_name = folder_name + '/' + str(j) + '.csv'
                self.predictions_dictionary[model_name][j].to_csv(file_name, index=False)
Ejemplo n.º 3
0
def create_basic_structure(config):
    """Create basic folders
    """

    DEBUG("Creating basic structure")

    root = config["target"]["path"] = create_folder(**config["target"])

    for folder in constants.BASE_FOLDERS:
        create_folder(os.path.join(root, folder))

    INFO("Basic structure successfully created")
    def store_indices(self):
        """ dump indices_dictionary in file """
        folder_name = './CV/' + 'indices'
        create_folder(folder_name)

        for i in self.indices_dictionary.keys():
            file_name = folder_name + '/' + str(i)
            with open(file_name + '_train.csv', 'w') as file:
                for item in self.indices_dictionary[i]['train']:
                    file.write("%s\n" % item)

            with open(file_name + '_test.csv', 'w') as file:
                for item in self.indices_dictionary[i]['test']:
                    file.write("%s\n" % item)
 def models_floor(self, model_array):
     """ Create the floor of the model predictions """
     model_var = [] 
     for idx, name in enumerate(model_array):
         print("Floor Variation for model %s"%name)
         n = elements_in_folder('./CV/' + name + '/')
         folder_name = './CV/' + name + '_floor/'
         create_folder(folder_name)
         model_var.append(name + '_floor')                                       
         for i in range(n):
             file_ = './CV/' + name + '/' + str(i) + '.csv'
             df = pd.read_csv(file_)
             ratings = np.array(df.Rating)
             ratings = np.floor(ratings)
             df.Rating = ratings
             df.to_csv(folder_name + str(i) + '.csv', index=False)  
     return model_var   
    def test_rewrite_xml_file_no_changes(self, mock_path_join):
        #given
        helpers.create_folder('tests/files_dest_ApplyVersionNumber')
        shutil.copy(u'tests/files_source/ApplyVersionNumber/elife-15224-v1-rewritten.xml', u'tests/files_dest_ApplyVersionNumber/elife-15224-v1.xml')
        mock_path_join.return_value = u'tests/files_dest_ApplyVersionNumber/elife-15224-v1.xml'

        #when
        self.applyversionnumber.rewrite_xml_file(u'elife-15224-v1.xml', example_file_name_map)

        #then
        with open(u'tests/files_dest_ApplyVersionNumber/elife-15224-v1.xml', 'r') as result_file:
            result_file_content = result_file.read()
        with open(u'tests/files_source/ApplyVersionNumber/elife-15224-v1-rewritten.xml', 'r') as expected_file:
            expected_file_content = expected_file.read()
        self.assertEqual(result_file_content, expected_file_content)

        helpers.delete_folder('tests/files_dest_ApplyVersionNumber', True)
Ejemplo n.º 7
0
    async def find_usages(self):
        """
        Try to find examples with usages of typed class/function on github
        """
        result_report = RESULT_FILE + get_extensions(self.language)[0]

        # remove old report
        with suppress(IOError):
            os.mkdir(TMP_FOLDER)
            os.remove(result_report)

        # get repositories for the module
        response = await github_search(self.module)
        repos = [(get_download_url(repo['full_name']), repo['full_name'])
                 for repo in response['items']]

        with create_folder(TMP_FOLDER), create_folder(EXTRACT_DESTINATION):
            async with aiohttp.ClientSession(loop=loop) as session:

                # start scrapping asynchronously
                tasks = [
                    self.scrap_repo(session, url, repo) for url, repo in repos
                ]
                data_available = True
                while data_available:

                    # извлеки выполненные задачи
                    done, pending = await asyncio.wait(
                        tasks, return_when=asyncio.FIRST_COMPLETED)

                    # запусти поиск на них
                    for found, report in [task.result() for task in done]:
                        self.examples_found += found
                        append_report(result_report, report)
                        if self.examples_found >= self.examples_limit:
                            for task in pending:
                                task.cancel()
                            data_available = False

                    if not pending:
                        break

                    tasks = pending
    def k_fold_predictions_and_store(self, df, model, model_name, override, **kwargs):
        """ produce k-fold predictions AND store the prediction in files """
       
        # check folder or create
        folder_name = './CV/' + model_name       
        
        compute = True
        if not override:
            if os.path.isdir(folder_name):
                compute = False
            
        if compute:
            create_folder(folder_name)

            pred_dict = self.k_fold_predictions(df, model, model_name, **kwargs)

            for i in pred_dict.keys():
                file_name = folder_name + '/' + str(i) + '.csv'
                pred_dict[i].to_csv(file_name, index=False)
Ejemplo n.º 9
0
 def setUp(self):
     self.resizeimages = activity_ResizeImages(settings_mock, None, None,
                                               None, None)
     helpers.create_folder('tests/test_cdn')
     helpers.create_folder(self.resize_images_start_folder +
                           testdata.session_example['expanded_folder'])
     shutil.copy(
         'tests/files_source/elife-00353-fig1-v1.tif',
         self.resize_images_start_folder +
         testdata.session_example['expanded_folder'] +
         '/elife-00353-fig1-v1.tif')
     shutil.copy(
         'tests/files_source/elife-00353-v1.pdf',
         self.resize_images_start_folder +
         testdata.session_example['expanded_folder'] +
         '/elife-00353-v1.pdf')
     shutil.copy(
         'tests/files_source/elife-00353-v1.xml',
         self.resize_images_start_folder +
         testdata.session_example['expanded_folder'] +
         '/elife-00353-v1.xml')
Ejemplo n.º 10
0
def create_basic_structure(config):
    """Create basic folders
    """
    DEBUG("Creating basic structure")

    root = config["target"]["path"] = create_folder(**config["target"])

    if config["parse_all"]:
        for folder in constants.BASE_FOLDERS:
            create_folder(os.path.join(root, folder))

    else:
        if config["parse"]["databases"]:
            create_folder(os.path.join(root, constants.DATABASES_FOLDER))

        if config["parse"]["libraries"]:
            create_folder(os.path.join(root, constants.LIBRARIES_FOLDER))

        if config["parse"]["pages"]:
            create_folder(os.path.join(root, constants.PAGES_FOLDER))

        if config["parse"]["resources"]:
            create_folder(os.path.join(root, constants.RESOURCES_FOLDER))

        if config["parse"]["security"]:
            create_folder(os.path.join(root, constants.SECURITY_FOLDER))

        if config["parse"]["app_actions"]:
            create_folder(os.path.join(root, constants.APP_ACTIONS_FOLDER))

    INFO("Basic structure successfully created")
Ejemplo n.º 11
0
 def setUp(self):
     self.applyversionnumber = activity_ApplyVersionNumber(settings_mock, None, None, None, None)
     self.test_dest_folder = 'tests/files_dest_ApplyVersionNumber'
     helpers.create_folder(self.test_dest_folder)