def prepare_dataset_6_eng(input_file_path: str, lang: str, type: str):
"""
Reads the data from dataset_6. Converts it into format Text,Label(0-Appearance,1-Intelligence,2-Political,3-Racial,4-Sextual) and writes the contents to file.
"""
class_text_number_map = {
'Appearance Data': 0,
'Intelligence Data': 1,
'Political Data': 2,
'Racial Data': 3,
'Sextual Data': 4
}
directory = os.path.join(rootpath.detect(), 'data', 'source_data',
lang, type, 'dataset_6', 'tweets_dataset')
comments = []
for filename in os.listdir(directory):
labels, data = utilities.read_file_contents(
os.path.join(directory, filename), ',', 'unicode_escape')
class_text = filename.split(".")[0]
comments_file = [[row[0], class_text_number_map[class_text]]
for row in data if row[1].lower() == "yes"]
comments.extend(comments_file)
comments.insert(0, ['Text', 'Label'])
comments = utilities.strip_and_replace_new_lines(comments)
output_path = os.path.join(rootpath.detect(), 'data',
'structured_data', lang, type, 'dataset_6',
'data.csv')
utilities.write_to_file(comments, output_path)
logger.info('Eng Dataset_6 has been prepared')
def combine_multiclass_datasets(lang = 'eng'):
"""
Parameters
----------
lang
Returns
-------
"""
base_data_path = os.path.join(rootpath.detect(), 'data', 'structured_data', lang, 'multiclass')
output_data_path = os.path.join(rootpath.detect(), 'data', 'final_data', lang, 'multiclass', 'data.csv')
if os.path.exists(base_data_path):
dataset_numbers = [int(name.split('_')[1]) for name in os.listdir(base_data_path)]
dataset_lower_bound, dataset_upper_bound = min(dataset_numbers), max(dataset_numbers)
if lang == "eng":
dataset_5,dataset_6 = read_combine_datasets(base_data_path, (dataset_lower_bound, dataset_upper_bound), concatenate=False)
dataset_5 = normalize_eng_dataset_5(dataset_5)
dataset_6 = normalize_eng_dataset_6(dataset_6)
dataset = pd.concat([dataset_5,dataset_6], axis=0, ignore_index=True)
if lang == "slo":
dataset_1,dataset_2 = read_combine_datasets(base_data_path, (dataset_lower_bound, dataset_upper_bound), concatenate=False)
dataset_2 = normalize_slo_dataset_2(dataset_2)
dataset = pd.concat([dataset_1,dataset_2], axis=0, ignore_index=True)
dataset.dropna(inplace=True)
dataset.drop_duplicates(subset=['Text', 'Label'], keep='first',inplace=True)
dataset.reset_index(inplace=True, drop=True)
utils.utilities.write_to_file_pd(dataset, output_data_path)
def test_rootpath_detect_base(self):
root_path = rootpath.detect()
self.assertEqual(root_path, ROOT_PATH)
root_path = rootpath.detect(helper.fixture_path('projects/null'))
self.assertEqual(root_path, ROOT_PATH)
def test_rootpath_detect_entry_pattern(self):
foo_root_path = helper.fixture_path('projects/py-foo')
root_path = rootpath.detect(helper.fixture_path('projects/py-foo'),
'not_a_file')
self.assertNotEqual(root_path, foo_root_path)
root_path = rootpath.detect(helper.fixture_path('projects/py-foo/'),
'not_a_file')
self.assertNotEqual(root_path, foo_root_path)
root_path = rootpath.detect(helper.fixture_path('projects/py-foo/foo'),
'not_a_file')
self.assertNotEqual(root_path, foo_root_path)
root_path = rootpath.detect(
helper.fixture_path('projects/py-foo/foo/'), 'not_a_file')
self.assertNotEqual(root_path, foo_root_path)
root_path = rootpath.detect(helper.fixture_path('projects/py-foo'),
'not_a_file')
self.assertNotEqual(root_path, foo_root_path)
root_path = rootpath.detect(helper.fixture_path('projects/py-foo/'),
'not_a_file')
self.assertNotEqual(root_path, foo_root_path)
root_path = rootpath.detect(helper.fixture_path('projects/py-foo/foo'),
'not_a_file')
self.assertNotEqual(root_path, foo_root_path)
root_path = rootpath.detect(
helper.fixture_path('projects/py-foo/foo/'), 'not_a_file')
self.assertNotEqual(root_path, foo_root_path)
root_path = rootpath.detect(
helper.fixture_path('projects/py-foo/foo/utils'), 'not_a_file')
self.assertNotEqual(root_path, foo_root_path)
root_path = rootpath.detect(
helper.fixture_path('projects/py-foo/foo/utils/'), 'not_a_file')
self.assertNotEqual(root_path, foo_root_path)
def get_element(self, cached=True):
if config.element_caching() and self._cached_element is not None \
and self._cached_element.driver is self._driver and cached:
return self._cached_element.element
try:
self._cached_element = CachedElement(self._driver, self.__find())
if self.__is_xpath() and not self._is_xpath_warning and config.xpath_warning():
attribute_name = "id"
attribute_value = self._cached_element.element.get_attribute(attribute_name)
if not attribute_value:
attribute_name = "name"
attribute_value = self._cached_element.element.get_attribute("name")
if attribute_value:
path = rootpath.detect()
stack = [line for line in traceback.format_stack() if
path in line and "core\\" not in line]
logging.warning(
"Your defined locator: '%s' has %s attribute with value '%s'. Please use it instead of xpath." +
"\n\rTraceback: \n\r%s",
self._locator, attribute_name, attribute_value, "\n".join(stack))
self._is_xpath_warning = True
return self._cached_element.element
except NoSuchElementException:
return None
def append(current_path = None, pattern = None):
"""
Automatically adds current file's package root to Python load path (i.e. `sys.path`) unless already added.
This makes it possible to always ensure module imports behave same no matter how the file is loaded.
Examples:
rootpath.append()
rootpath.append(__file__)
rootpath.append('./src')
"""
project_root_path = rootpath.detect(current_path, pattern)
try:
if project_root_path != current_path:
try:
sys.path.remove(current_path)
except:
pass
sys.path.index(project_root_path)
return False, project_root_path
except ValueError as error:
sys.path.append(project_root_path)
return True, project_root_path
class LocalModels(StackLayout):
"""docstring for Run."""
models = ListProperty()
bundle_dir = rootpath.detect(
pattern='main.py') # Obtain the dir of main.py
Builder.load_file(bundle_dir + os.sep + 'ui' + os.sep + 'local_models.kv')
def __init__(self):
super(LocalModels, self).__init__()
self.refresh()
def collect_models(self):
model_zoo_dir = os.sep.join(
[self.bundle_dir, 'plugins', 'processing', 'model_zoo'])
if not os.path.isdir(model_zoo_dir + os.sep + 'models'):
return
self.model_names = os.listdir(model_zoo_dir + os.sep + 'models')
self.models = [name for name in self.model_names if name[:2] != '__']
def render_model_cards(self):
for model_id in self.models:
self.add_widget(Factory.LocalModelCard(model_id))
def refresh(self, *args):
self.clear_widgets()
self.collect_models()
self.render_model_cards()
def run():
widget_handler=WidgetHandler()
plugin_handler=PluginHandler()
widget_handler.switch_screens('processing','files')
bundle_dir=rootpath.detect(pattern='main.py')
plugin_handler.set_plugin_attr(
'files','path',os.sep.join([bundle_dir,'img','face.jpg']))
def get_webdriver_instance(self):
"""
Get WebDriver Instance based on the browser configuration
Returns:
'WebDriver Instance'
"""
if self.browser == "iexplorer":
# Set ie driver
driver = webdriver.Ie()
elif self.browser == "firefox":
driver = webdriver.Firefox()
else: # self.browser == "chrome"
# Set chrome driver
opt = webdriver.ChromeOptions()
opt.add_argument(
"user-data-dir=/home/alexey/.config/google-chrome/Default")
driver = webdriver.Chrome(options=opt,
executable_path=rootpath.detect() +
"/SeleniumWebDriverCourse/chromedriver")
# Setting Driver Implicit Time out for An Element
driver.implicitly_wait(5)
# Maximize the window
driver.maximize_window()
# Loading browser with App URL
return driver
def test_correct_return_types_and_detect_code_smells(self):
import subprocess, rootpath, os
for cl_path in self.class_paths:
root_dir = rootpath.detect()
path = os.path.join(root_dir, cl_path)
process = subprocess.Popen(
[
"mypy", path, "--ignore-missing-imports",
"--no-site-packages"
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
stdout, stderr = process.communicate()
stdout = stdout.decode("utf-8")
stderr = stderr.decode("utf-8")
output = str(stdout).split("\n")
errors = []
for line in output:
if line.__contains__("error:"):
errors.append(line)
self.assertTrue(
errors == [],
"Following errors were produced by "
"MyPy: \n{}".format("\n".join(errors)),
)
self.assertTrue(
stderr == "",
"Following error were produced by MyPy: {}".format(stderr))
def prepare_dataset_2_slo(input_file_path: str, lang: str, type: str):
data = utilities.read_file_contents_pd(input_file_path)
non_hate = data.loc[data['Class'].isin([0, 1])]
del non_hate['Type']
non_hate.columns = ['Text', 'Label']
non_hate['Label'] = 0
hate = data.loc[data['Class'].isin([2, 3])]
del hate['Class']
hate.columns = ['Text', 'Label']
hate['Label'] = hate['Label'].fillna(12)
non_hate['Label'] = non_hate['Label'].astype(int)
hate['Label'] = hate['Label'].astype(int)
comments = []
comments.extend(non_hate.values)
comments.extend(hate.values)
comments.insert(0, ['Text', 'Label'])
comments = utilities.strip_and_replace_new_lines(comments)
output_path = os.path.join(rootpath.detect(), 'data',
'structured_data', lang, type, 'dataset_2',
'data.csv')
utilities.write_to_file(comments, output_path)
logger.info('Slo Dataset_2 has been prepared')
def settings(self):
SettingsObject = {}
ssettings = configparser.ConfigParser()
path = rootpath.detect()
print(path)
readconfigs = ssettings.read(
path + "/Documents/nivya/weathershopper/shopperProps.properties")
print(readconfigs)
SettingsObject['url'] = ssettings.get(
"defaults", "pythonanywhere.weathershopper.base.url")
SettingsObject['tempcutoff'] = ssettings.get(
"defaults", "pythonanywhere.weathershopper.temperatureCutOff")
SettingsObject['browsers'] = ssettings.get(
"defaults", "pythonanywhere.weathershopper.app.browser.run")
SettingsObject['email'] = ssettings.get(
"defaults", "pythonanywhere.weathershopper.user.email")
SettingsObject['expiry'] = ssettings.get(
"sandboxcarddetails",
"pythonanywhere.weathershopper.sandboc.card.expiry")
SettingsObject['card'] = ssettings.get(
"sandboxcarddetails",
"pythonanywhere.weathershopper.sandbox.card.number")
SettingsObject['cvv'] = ssettings.get(
"sandboxcarddetails",
"pythonanywhere.weathershopper.sandboc.card.cvv")
import random
SettingsObject['email'] = "smailxcdvfserfsfs" + str(
random.randint(10000, 1000000)) + "@gmail.com"
return SettingsObject
class DemoCard(BoxLayout):
title = StringProperty()
id = StringProperty()
tags = StringProperty()
abstract = StringProperty()
bundle_dir = rootpath.detect(
pattern='main.py') # Obtain the dir of main.py
def __init__(self, meta_info):
super(DemoCard, self).__init__()
self.title = meta_info['title']
self.id = meta_info['id']
self.abstract = meta_info['abstract']
self.meta_info = meta_info
self.register_event_type('on_install')
def install(self, *args):
installer = Installer()
self.ids.installer.clear_widgets()
self.ids.installer.add_widget(installer)
installer.bind(on_success=lambda x: self.dispatch('on_install'))
installer.install_demo_full(self.id, self.meta_info)
def on_install(self, *args):
pass
def _scraping(self):
root = rootpath.detect()
df_krx = fdr.StockListing('KRX')
codes = list(df_krx['Symbol'])
keywords = list(df_krx['Name'])
for code, keyword in zip(codes, keywords):
if self.threadactive:
try:
if not os.path.exists(root + '/stock/{0}/{0}.csv'.format(keyword)):
continue
sc = Scraping(code, keyword)
data = pd.read_csv(root + '/stock/{0}/{0}.csv'.format(keyword))
res = sc.get_current_data(code, data)
if res.empty:
self.process.emit('{0} is already updated!'.format(keyword))
self.msleep(500)
continue
res = res.loc[~res.index.duplicated(keep='first')]
res.to_csv(sc.make_csv(keyword))
self.process.emit('{0} is completed!'.format(keyword))
self.msleep(500)
except Exception as e:
print('SCRAPING ERROR')
error = '[{0}] {1} Error. {2} \n'.format(datetime.datetime.now(), keyword, e)
report_error(error, 'scraping.err')
continue
class ModelCollector(object):
"""docstring for Run."""
bundle_dir = rootpath.detect(pattern='plugins')
def __init__(self):
super(ModelCollector, self).__init__()
self.models = {}
self.scan_model_zoo()
def scan_model_zoo(self):
self.model_names = os.listdir(self.bundle_dir + os.sep + 'model_zoo')
self.model_names = [
name for name in self.model_names if name[:2] != '__'
]
for name in self.model_names:
module_name = '.'.join(['model_zoo', name, 'api'])
try:
module = importlib.import_module(module_name)
except:
continue
self.models[name] = getattr(module, 'Api')()
self.models[name].config_list = get_file_list(self.bundle_dir +
os.sep +
'model_zoo' +
os.sep + name +
os.sep + 'configs')
self.models[name].weight_list = get_file_list(self.bundle_dir +
os.sep +
'model_zoo' +
os.sep + name +
os.sep + 'weights')
self.models[name].train_notebooks = get_file_list(
self.bundle_dir + os.sep + 'model_zoo' + os.sep + name +
os.sep + 'training',
formats=['ipynb'])
def init_alembic_env():
dst = rootpath.detect() + "/alembic/"
if os.path.exists(dst):
src = ICECREAM_PATH + "/migration_tool/env.py"
copy(src, dst)
else:
logging.info("please alembic init alembic , before makealembic")
class Files(BoxLayout):
data = ObjectProperty(lambda: None)
id = 'select_path_panel'
bundle_dir = rootpath.detect(pattern='plugins')
Builder.load_file(bundle_dir + os.sep + 'ui' + os.sep + 'files.kv')
def open_file(self):
path = select_file()
self.data['file_list'] = [path.replace('/', os.sep)]
self.add_to_tree()
def open_folder(self):
path = select_folder()
if path == '':
return
self.add_to_tree(path=path)
def add_to_tree(self, path=''):
if path != '':
self.data.file_list = get_file_list(
path, formats=['jpg', 'jpeg', 'bmp', 'png', 'tiff', 'tif'])
tree = {'node_id': 'resources', 'children': [], 'type': 'root'}
for file_path in self.data.file_list:
tree['children'].append({
'node_id': file_path.split(os.sep)[-1],
'type': 'file_path',
'content': file_path,
'display': 'image_viewer',
'children': []
})
self.data.tree = tree
self.property('data').dispatch(self)
print(path)
class Config(object):
CONFIG_FILE = rootpath.detect() + "/config.ini"
@staticmethod
def get_config():
cfg = ConfigParser()
cfg.read(Config.CONFIG_FILE)
return cfg
@staticmethod
def get_config_value(section, key):
cfg = ConfigParser()
cfg.read(Config.CONFIG_FILE)
if cfg.has_section(section):
if cfg.has_option(section, key):
return cfg.get(section, key)
else:
return None
else:
return None
@staticmethod
def write_config(section, key, value):
config = Config.get_config()
if not config.has_section(section):
config.add_section(section)
config.set(section, key, value)
with open(Config.CONFIG_FILE, 'w') as configfile:
config.write(configfile)
def setUp(self):
self.mockCMDObject = CMD()
self.path = rootpath.detect()
self.test_target_directory = f"{self.path}\\tests\\test_target"
self.test_file_path = f"{self.test_target_directory}\\test.txt"
with open(self.test_file_path, "w"):
pass
class Hotkey(BoxLayout):
"""docstring for Hotkey."""
bundle_dir = rootpath.detect(pattern='plugins')
# Builder.load_file(bundle_dir +os.sep+'ui'+os.sep+'gallery.kv')
data=DictProperty()
def __init__(self,**kwargs):
super(Hotkey, self).__init__(**kwargs)
self.super = []
text = StringProperty()
keyboard = Window.request_keyboard(self._keyboard_released, self)
keyboard.bind(on_key_down=self._keyboard_on_key_down, on_key_up=self._keyboard_released)
########################################
#end def __init__
def _keyboard_released(self, window=None, keycode=None):
self.super = []
def _keyboard_on_key_down(self, window, keycode, text, super):
if 'lctrl' in self.super and keycode[1] == 's':
app=App.get_running_app()
app.plugin_manager.open()
Logger.info("Item saved, {}".format(self.super))
self.super = []
return False
elif 'lctrl' not in self.super and keycode[1] in ["lctrl"]:
self.super.append(keycode[1])
return False
else:
Logger.info("key {} pressed.".format(keycode))
return False
def run():
widget_handler=WidgetHandler()
plugin_handler=PluginHandler()
widget_handler.switch_screens('processing','rotation')
bundle_dir=rootpath.detect(pattern='main.py')
plugin_handler.set_plugin_attr(
'files','path','D:\onedrive\program/for_liuyuan_rotation\data/test.tiff')
class Debugger(BoxLayout):
"""docstring for Debugger."""
data = ObjectProperty()
debug_packages = ListProperty()
bundle_dir = rootpath.detect(
pattern='main.py') # Obtain the dir of main.py
# Builder.load_file(bundle_dir +os.sep+'ui'+os.sep+'demo.kv')
def __init__(self):
super(Debugger, self).__init__()
self.collect_debug_packages()
self.run_debug_packages()
def collect_debug_packages(self):
for importer, modname, ispkg in pkgutil.walk_packages(
path=[
os.sep.join(
[self.bundle_dir, 'plugins', 'system', 'debugger'])
],
prefix='plugins.system.debugger.',
onerror=lambda x: None):
if len(modname.split('.')) > 4 and '__' not in modname:
self.debug_packages.append(modname)
def run_debug_packages(self):
for modname in self.debug_packages:
try:
module = importlib.import_module(modname)
except Exception as e:
logging.warning('Fail to load debug script <{}>: {}'.format(
modname, e))
def load_multiclass_datasets(lang='eng')-> pd.DataFrame:
input_base_path = os.path.join(rootpath.detect(), 'data', 'final_data', lang, 'multiclass', 'data.csv')
try:
df = pd.read_csv(input_base_path)
except (pd.errors.EmptyDataError, FileNotFoundError) as e:
df = pd.DataFrame()
return df
class TableViewer(BoxLayout):
"""docstring for TableViewer."""
data=DictProperty()
bundle_dir = rootpath.detect(pattern='main.py') # Obtain the dir of main.py
Builder.load_file(bundle_dir +os.sep+'ui'+os.sep+'table_viewer.kv')
def __init__(self):
super(TableViewer, self).__init__()
self.is_selected_=False
self.bind(data=self.refresh)
def on_click_save(self,*args):
out_file=select_save_path()
self.save(out_file)
def refresh(self,*args):
table=Table(self.data['content'])
self.ids.table.clear_widgets()
self.ids.table.add_widget(table)
def save(self,out_file):
if out_file is None:
return
import csv
with open(out_file.name, 'w', newline='') as resultFile:
wr = csv.writer(resultFile, dialect='excel')
for item in self.data['content']:
wr.writerow(item)
class OnlineDemos(ModalView):
"""docstring for OnlineDemos."""
data = ObjectProperty()
local_demos = ListProperty()
online_demos = ListProperty()
bundle_dir = rootpath.detect(
pattern='main.py') # Obtain the dir of main.py
Builder.load_file(bundle_dir + os.sep + 'ui' + os.sep + 'online_demos.kv')
def __init__(self):
super(OnlineDemos, self).__init__()
self.collect_local_demos()
self.collect_online_demos()
self.bind(online_demos=self.insert_model_cards)
self.register_event_type('on_install')
def collect_local_demos(self):
pass
def collect_online_demos(self):
req = UrlRequest(
'http://www.deepdiy.net/model_zoo/api/demo_list.json',
lambda req, result: setattr(self, 'online_demos', result))
def insert_model_cards(self, *args):
for meta_info in self.online_demos:
card = Factory.DemoCard(meta_info)
self.ids.model_album.add_widget(card)
card.bind(on_install=lambda x: self.dispatch('on_install'))
def on_install(self, *args):
pass
class Demo(BoxLayout):
"""docstring for Demo."""
url = StringProperty()
filename = StringProperty()
percent = NumericProperty()
bundle_dir = rootpath.detect(
pattern='main.py') # Obtain the dir of main.py
# Builder.load_file(bundle_dir +os.sep+'ui'+os.sep+'gallery.kv')
data = DictProperty()
def __init__(self):
super(Demo, self).__init__()
self.add_widget(Button(text='download', on_release=self.download))
def download(self, *args):
with open(self.filename, 'wb') as f:
response = requests.get(self.url, stream=True)
total = response.headers.get('content-length')
if total is None:
f.write(response.content)
else:
downloaded = 0
total = int(total)
for data in response.iter_content(
chunk_size=max(int(total / 1000), 1024 * 1024)):
downloaded += len(data)
f.write(data)
self.percent = int(100 * downloaded / total)
print(self.percent)
def __init__(self): super(Test, self).__init__() bundle_dir = rootpath.detect() dataset=Dataset() dataset.img_dir=bundle_dir+os.sep+'datasets/dog' dataset.destination_dir=bundle_dir+os.sep+'temp' dataset.annotation_path=bundle_dir+os.sep+'datasets/dog/via_region_data.json' dataset.run()
def make_csv(self, keyword):
root = rootpath.detect()
outname = '{keyword}.csv'.format(keyword=keyword)
outdir = str(root) + '/stock/{keyword}/'.format(keyword=keyword)
if not os.path.exists(outdir):
os.makedirs(outdir)
fullname = os.path.join(outdir, outname)
return fullname
def get_commands_path(self, command_name: str) -> t.Iterator[commands.Command] or False:
"""Yield all commands for all cogs in all extensions."""
for module in self.walk_modules():
for cog in self.walk_cogs(module):
for cmd in self.walk_commands(cog):
if str(cmd).lower() == command_name.lower():
return module.__file__.replace(rootpath.detect(), "", 1)
return False
def test_rootpath_detect_entry(self):
foo_root_path = helper.fixture_path('projects/py-foo')
root_path = rootpath.detect(helper.fixture_path('projects/py-foo'))
self.assertEqual(root_path, foo_root_path)
root_path = rootpath.detect(helper.fixture_path('projects/py-foo/'))
self.assertEqual(root_path, foo_root_path)
root_path = rootpath.detect(helper.fixture_path('projects/py-foo/foo'))
self.assertEqual(root_path, foo_root_path)
root_path = rootpath.detect(
helper.fixture_path('projects/py-foo/foo/'))
self.assertEqual(root_path, foo_root_path)
root_path = rootpath.detect(helper.fixture_path('projects/py-foo'))
self.assertEqual(root_path, foo_root_path)
root_path = rootpath.detect(helper.fixture_path('projects/py-foo/'))
self.assertEqual(root_path, foo_root_path)
root_path = rootpath.detect(helper.fixture_path('projects/py-foo/foo'))
self.assertEqual(root_path, foo_root_path)
root_path = rootpath.detect(
helper.fixture_path('projects/py-foo/foo/'))
self.assertEqual(root_path, foo_root_path)
root_path = rootpath.detect(
helper.fixture_path('projects/py-foo/foo/utils'))
self.assertEqual(root_path, foo_root_path)
root_path = rootpath.detect(
helper.fixture_path('projects/py-foo/foo/utils/'))
self.assertEqual(root_path, foo_root_path)