def setUp(self):
self.resizeimages = activity_ResizeImages(settings_mock, None, None, None, None)
helpers.create_folder('tests/test_cdn')
helpers.create_folder(self.resize_images_start_folder + testdata.session_example['expanded_folder'])
shutil.copy('tests/files_source/elife-00353-fig1-v1.tif', self.resize_images_start_folder + testdata.session_example['expanded_folder'] + '/elife-00353-fig1-v1.tif')
shutil.copy('tests/files_source/elife-00353-v1.pdf', self.resize_images_start_folder + testdata.session_example['expanded_folder'] + '/elife-00353-v1.pdf')
shutil.copy('tests/files_source/elife-00353-v1.xml', self.resize_images_start_folder + testdata.session_example['expanded_folder'] + '/elife-00353-v1.xml')
def store_predictions(self):
""" dump predictions_dictionary in file """
for model_name in self.predictions_dictionary.keys():
folder_name = './CV/' + model_name
create_folder(folder_name)
for j in self.predictions_dictionary[model_name].keys():
file_name = folder_name + '/' + str(j) + '.csv'
self.predictions_dictionary[model_name][j].to_csv(file_name, index=False)
def create_basic_structure(config):
"""Create basic folders
"""
DEBUG("Creating basic structure")
root = config["target"]["path"] = create_folder(**config["target"])
for folder in constants.BASE_FOLDERS:
create_folder(os.path.join(root, folder))
INFO("Basic structure successfully created")
def store_indices(self):
""" dump indices_dictionary in file """
folder_name = './CV/' + 'indices'
create_folder(folder_name)
for i in self.indices_dictionary.keys():
file_name = folder_name + '/' + str(i)
with open(file_name + '_train.csv', 'w') as file:
for item in self.indices_dictionary[i]['train']:
file.write("%s\n" % item)
with open(file_name + '_test.csv', 'w') as file:
for item in self.indices_dictionary[i]['test']:
file.write("%s\n" % item)
def models_floor(self, model_array):
""" Create the floor of the model predictions """
model_var = []
for idx, name in enumerate(model_array):
print("Floor Variation for model %s"%name)
n = elements_in_folder('./CV/' + name + '/')
folder_name = './CV/' + name + '_floor/'
create_folder(folder_name)
model_var.append(name + '_floor')
for i in range(n):
file_ = './CV/' + name + '/' + str(i) + '.csv'
df = pd.read_csv(file_)
ratings = np.array(df.Rating)
ratings = np.floor(ratings)
df.Rating = ratings
df.to_csv(folder_name + str(i) + '.csv', index=False)
return model_var
def test_rewrite_xml_file_no_changes(self, mock_path_join):
#given
helpers.create_folder('tests/files_dest_ApplyVersionNumber')
shutil.copy(u'tests/files_source/ApplyVersionNumber/elife-15224-v1-rewritten.xml', u'tests/files_dest_ApplyVersionNumber/elife-15224-v1.xml')
mock_path_join.return_value = u'tests/files_dest_ApplyVersionNumber/elife-15224-v1.xml'
#when
self.applyversionnumber.rewrite_xml_file(u'elife-15224-v1.xml', example_file_name_map)
#then
with open(u'tests/files_dest_ApplyVersionNumber/elife-15224-v1.xml', 'r') as result_file:
result_file_content = result_file.read()
with open(u'tests/files_source/ApplyVersionNumber/elife-15224-v1-rewritten.xml', 'r') as expected_file:
expected_file_content = expected_file.read()
self.assertEqual(result_file_content, expected_file_content)
helpers.delete_folder('tests/files_dest_ApplyVersionNumber', True)
async def find_usages(self):
"""
Try to find examples with usages of typed class/function on github
"""
result_report = RESULT_FILE + get_extensions(self.language)[0]
# remove old report
with suppress(IOError):
os.mkdir(TMP_FOLDER)
os.remove(result_report)
# get repositories for the module
response = await github_search(self.module)
repos = [(get_download_url(repo['full_name']), repo['full_name'])
for repo in response['items']]
with create_folder(TMP_FOLDER), create_folder(EXTRACT_DESTINATION):
async with aiohttp.ClientSession(loop=loop) as session:
# start scrapping asynchronously
tasks = [
self.scrap_repo(session, url, repo) for url, repo in repos
]
data_available = True
while data_available:
# извлеки выполненные задачи
done, pending = await asyncio.wait(
tasks, return_when=asyncio.FIRST_COMPLETED)
# запусти поиск на них
for found, report in [task.result() for task in done]:
self.examples_found += found
append_report(result_report, report)
if self.examples_found >= self.examples_limit:
for task in pending:
task.cancel()
data_available = False
if not pending:
break
tasks = pending
def k_fold_predictions_and_store(self, df, model, model_name, override, **kwargs):
""" produce k-fold predictions AND store the prediction in files """
# check folder or create
folder_name = './CV/' + model_name
compute = True
if not override:
if os.path.isdir(folder_name):
compute = False
if compute:
create_folder(folder_name)
pred_dict = self.k_fold_predictions(df, model, model_name, **kwargs)
for i in pred_dict.keys():
file_name = folder_name + '/' + str(i) + '.csv'
pred_dict[i].to_csv(file_name, index=False)
def setUp(self):
self.resizeimages = activity_ResizeImages(settings_mock, None, None,
None, None)
helpers.create_folder('tests/test_cdn')
helpers.create_folder(self.resize_images_start_folder +
testdata.session_example['expanded_folder'])
shutil.copy(
'tests/files_source/elife-00353-fig1-v1.tif',
self.resize_images_start_folder +
testdata.session_example['expanded_folder'] +
'/elife-00353-fig1-v1.tif')
shutil.copy(
'tests/files_source/elife-00353-v1.pdf',
self.resize_images_start_folder +
testdata.session_example['expanded_folder'] +
'/elife-00353-v1.pdf')
shutil.copy(
'tests/files_source/elife-00353-v1.xml',
self.resize_images_start_folder +
testdata.session_example['expanded_folder'] +
'/elife-00353-v1.xml')
def create_basic_structure(config):
"""Create basic folders
"""
DEBUG("Creating basic structure")
root = config["target"]["path"] = create_folder(**config["target"])
if config["parse_all"]:
for folder in constants.BASE_FOLDERS:
create_folder(os.path.join(root, folder))
else:
if config["parse"]["databases"]:
create_folder(os.path.join(root, constants.DATABASES_FOLDER))
if config["parse"]["libraries"]:
create_folder(os.path.join(root, constants.LIBRARIES_FOLDER))
if config["parse"]["pages"]:
create_folder(os.path.join(root, constants.PAGES_FOLDER))
if config["parse"]["resources"]:
create_folder(os.path.join(root, constants.RESOURCES_FOLDER))
if config["parse"]["security"]:
create_folder(os.path.join(root, constants.SECURITY_FOLDER))
if config["parse"]["app_actions"]:
create_folder(os.path.join(root, constants.APP_ACTIONS_FOLDER))
INFO("Basic structure successfully created")
def setUp(self):
self.applyversionnumber = activity_ApplyVersionNumber(settings_mock, None, None, None, None)
self.test_dest_folder = 'tests/files_dest_ApplyVersionNumber'
helpers.create_folder(self.test_dest_folder)