def __init__(self, rel_dir="../FourmiCrawler/sources"): """ The initiation of a SourceLoader, selects and indexes a directory for usable sources. Also loads a configuration file for Sources and passes the arguments in the named section to the source :param rel_dir: A relative path to a directory. """ path = os.path.dirname(os.path.abspath(__file__)) path += "/" + rel_dir known_parser = set() config = Configurator.read_sourceconfiguration() for py in [ f[:-3] for f in os.listdir(path) if f.endswith('.py') and f != '__init__.py' ]: mod = __import__('.'.join( [rel_dir.replace("../", "").replace("/", "."), py]), fromlist=[py]) classes = [ getattr(mod, x) for x in dir(mod) if inspect.isclass(getattr(mod, x)) ] for cls in classes: if issubclass(cls, Source) and cls not in known_parser: sourcecfg = Configurator.get_section(config, cls.__name__) self.sources.append(cls(sourcecfg)) known_parser.add(cls)
def set_mode(mode, config): configurator = Configurator(config) config = configurator.from_yaml() if mode not in AVAILABLE_MODES: return 'Wrong mode name' config['mode'] = mode configurator.dump_yaml(config) return 'I change mode to ' + mode
def run(external_processor=None): start_time = time.time() log = logging.getLogger('bot') config = load_config() if not SerialController().serial: SerialController().run_serial(config['serial']) mode = config['mode'] if mode == 'enhance': enhancer = Enhancer(config['enhancer']) enhancer.process() if mode == 'buff': buff_config = Configurator(config['buffer']).from_yaml() processor = ProcessInitializer(Buffer(buff_config), config['serial']) processor.handle() if mode == 'make': external_processor.process() if mode == 'combination': combinate = Combinator(config['enhancer']) combinate.process() if mode == 'taming': Taming().run() if mode == 'farming': Farming().run() if mode == 'test': print('nothings') exec_time = (time.time() - start_time) final_message = "Finished work, time: {0} (sec), {1} (min)".format( exec_time, exec_time / 60) log.info(final_message) return exec_time
def search(docopt_arguments, source_loader): """ The function that facilitates the search for a specific compound. :param docopt_arguments: A dictionary generated by docopt containing all CLI arguments. :param source_loader: An initiated SourceLoader object pointed at the directory with the sources. """ conf = Configurator() conf.set_logging(docopt_arguments["--log"], docopt_arguments["-v"]) conf.set_output(docopt_arguments["--output"], docopt_arguments["--format"], docopt_arguments["<compound>"]) setup_crawler( docopt_arguments["<compound>"], conf.scrapy_settings, source_loader, docopt_arguments["--attributes"].split(",") ) if conf.scrapy_settings.getbool("LOG_ENABLED"): log.start( conf.scrapy_settings.get("LOG_FILE"), conf.scrapy_settings.get("LOG_LEVEL"), conf.scrapy_settings.get("LOG_STDOUT"), ) reactor.run()
def initialize_configs(self, config): import jobs.helpers.configs as markers import sys config = Configurator(config).from_yaml() for c in config['templates']: setattr(self, c['name'], getattr(sys.modules['jobs.helpers.configs'], c['name'])) for field, value in c.items(): if field is not 'name': setattr(getattr(self, c['name']), field, value)
class CubesScreen(Screen): CELLS_ROW = 12 CELLS_COL = 10 def __init__(self, message, bot): super().__init__(message, bot) self.title = 'Select cube position:' self.load_config() self.generate_cubes() def generate_cubes(self): for i in range(1, self.CELLS_ROW): for j in range(1, self.CELLS_COL): attr_name = 'cube_{0}_{1}'.format(i, j) def cube(call, state): self.set_cube_config(call) es = state['EnhanceScreen'] es.render(call=call) return 'EnhanceScreen', es setattr(self, attr_name, cube) return None def set_cube_config(self, call): data = call.data.split('_')[1:] data = data[::-1] self.config['enhancement']['cube'] = data self.configfile.dump_yaml(self.config) self.bot.answer_callback_query(call.id, 'Set cube {0}'.format(data)) def render(self, call=None): self.markup.keyboard = [] self.markup.row_width = self.CELLS_COL for i in range(1, self.CELLS_ROW): line = [] for j in range(1, self.CELLS_COL): title = '{i}:{j}'.format(i=i, j=j) callback = '{name}.cube_{i}_{j}'.format(name=self.name, i=str(i), j=str(j)) line.append( self.InlineKeyboardButton(title, callback_data=callback)) self.markup.row(*line) if call is None: self.send() else: self.edit(call) def load_config(self): self.configfile = Configurator(self.config['enhancer']) self.config = self.configfile.from_yaml()
def __init__(self, rel_dir="../FourmiCrawler/sources"): """ The initiation of a SourceLoader, selects and indexes a directory for usable sources. Also loads a configuration file for Sources and passes the arguments in the named section to the source :param rel_dir: A relative path to a directory. """ path = os.path.dirname(os.path.abspath(__file__)) path += "/" + rel_dir known_parser = set() config = Configurator.read_sourceconfiguration() for py in [f[:-3] for f in os.listdir(path) if f.endswith(".py") and f != "__init__.py"]: mod = __import__(".".join([rel_dir.replace("../", "").replace("/", "."), py]), fromlist=[py]) classes = [getattr(mod, x) for x in dir(mod) if inspect.isclass(getattr(mod, x))] for cls in classes: if issubclass(cls, Source) and cls not in known_parser: sourcecfg = Configurator.get_section(config, cls.__name__) self.sources.append(cls(sourcecfg)) known_parser.add(cls)
def __init__(self): self.config = Configurator('config.yml').from_yaml()['window'] self.width, self.height = self.config['width'], self.config['height'] self.locate_window() if self.windowHead is None: log.error('Window has not initiated!') return self.x = self.windowHead[0] self.y = self.windowHead[1] + self.windowHead[3] self.screen = ui.screenshot(region=(self.x, self.y, self.width, self.height)) self.rect = (self.x, self.y, self.width, self.height)
def quests(self, call, state): self.bot.answer_callback_query(call.id, 'Get circus quests') buff_cfg = Configurator(self.config['buffer']).from_yaml() buff_cfg['spawn'] = True buff_cfg['logout'] = True buff = Buffer(buff_cfg) for i in range(8): CircusHandler().get_quest() buff.process_flow() return None, None
def get_quests(config): from utils.serial_controller import SerialController from jobs.helpers.circus_handler import CircusHandler from processes.wait import Wait from jobs.buffer import Buffer from utils.config import Config Config().initialize_configs(config['navigator']) if not SerialController().serial: SerialController().run_serial(config['serial']) buff_cfg = Configurator(config['buffer']).from_yaml() buff = Buffer(buff_cfg) for i in range(8): CircusHandler().get_quest() Wait(2).delay() buff.process_flow()
class BuffScreen(Screen): # another option for future buttons = ['Buff', 'Logout', 'Spawn', 'Back'] def __init__(self, message, bot): super().__init__(message, bot) self.title = 'Buffer menu:' self.load_config() # Try to move this method do Screen class def load_config(self): self.configfile = Configurator(self.config['buffer']) self.config = self.configfile.from_yaml() def back(self, call, state): screen = 'StartScreen' ss = state[screen] ss.render(call=call) return 'StartScreen', ss def buff(self, call, state): self.config['refresh'] = True self.config['logout'] = True self.config['spawn'] = False self.configfile.dump_yaml(self.config) return self.run_action(call) def logout(self, call, state): self.config['spawn'] = True self.config['logout'] = True self.configfile.dump_yaml(self.config) return self.run_action(call) def spawn(self, call, state): self.config['spawn'] = True self.config['logout'] = False self.configfile.dump_yaml(self.config) return self.run_action(call) def run_action(self, call): self.bot.answer_callback_query(call.id, 'Start ' + call.data) Buffer(self.config).process() return self.name, self
def search(docopt_arguments, source_loader): """ The function that facilitates the search for a specific compound. :param docopt_arguments: A dictionary generated by docopt containing all CLI arguments. :param source_loader: An initiated SourceLoader object pointed at the directory with the sources. """ conf = Configurator() conf.set_logging(docopt_arguments["--log"], docopt_arguments["-v"]) conf.set_output(docopt_arguments["--output"], docopt_arguments["--format"], docopt_arguments["<compound>"]) setup_crawler(docopt_arguments["<compound>"], conf.scrapy_settings, source_loader, docopt_arguments["--attributes"].split(',')) if conf.scrapy_settings.getbool("LOG_ENABLED"): log.start(conf.scrapy_settings.get("LOG_FILE"), conf.scrapy_settings.get("LOG_LEVEL"), conf.scrapy_settings.get("LOG_STDOUT")) reactor.run()
def __init__(self, config): self.config = Configurator(config) self.config = self.config.from_yaml() self.combination_cfg = self.config['combination'] self.config = self.config['enhance']
class Enhancer: def __init__(self, config): self.config = Configurator(config) self.config = self.config.from_yaml() self.combination_cfg = self.config['combination'] self.config = self.config['enhance'] # enhancement _enhance_move for enhancing, _open_move - for open items def enhancement(self): # firstly get enhancement matrix matrix = self._slice_matrix_by_config() current_item_enhance = [] for line in matrix: for cell in line: current_item_enhance += self._enhance_move(cell).get_flow() return current_item_enhance def _slice_matrix_by_config(self): scope = self.config['scope'] max_x = self.config['max']['horizontal'] first_line = scope['start'] last_line = scope['end'] matrix = self.get_matrix() result = [] first_line_points = matrix[first_line[1] - 1][first_line[0] - 1:max_x] # get first line result.append(first_line_points) body = matrix[first_line[1]:last_line[1] - 1] for b in body: result.append(b) last_line_points = matrix[last_line[1] - 1][:last_line[0]] result.append(last_line_points) return result def _enhance_move(self, cell): return EnhanceFlow(cell, self.config) def enhance(self, flow): loops = [] for i in range(self.config['cycles']): loops += flow() return loops def _get_click(self, _x, _y): values = self.get_matrix() _x -= 1 _y -= 1 return Click(values[_y][_x][0], values[_y][_x][1], process='dclick') def get_matrix(self): values = [] for y in range(self.config['max']['vertical']): values.append([]) for x in range(self.config['max']['horizontal']): delta = self.config['delta'] start_point = self.config['points']['start'] calc_x = delta * x + start_point[0] calc_y = delta * y + start_point[1] values[y].append((calc_x, calc_y)) return values def combination(self): matrix = self.get_matrix() config = self.combination_cfg cell0 = config['cell_0'] cell1 = config['cell_1'] cells = [[cell0[0], cell0[1]], [cell1[0], cell1[1]]] return CombinationFlow(cells, self.combination_cfg).get_flow()
import logging from logging.handlers import RotatingFileHandler if __name__ == '__main__': sys.path.append(os.getcwd()) # comment out the following 2 lines to print debug info on stdout logging.basicConfig(format='%(asctime)s %(message)s', datefmt='%m/%d/%Y %I:%M:%S %p', level=logging.INFO, filename='maskednet.log') logging.getLogger().addHandler( RotatingFileHandler('maskednet.log', maxBytes=1024, backupCount=0)) sys.path[0] = os.path.dirname(os.path.realpath(__file__)) from server.srv import Server from client.client import Client from bridge.bridge import Bridge from utils.configurator import Configurator configurator = Configurator() server = Server(address=configurator.get('Server', 'address'), model_port=configurator.get('Server', 'model_port'), bridge_port=configurator.get('Server', 'bridge_port')) client = Client(server_uri=configurator.get('Client', 'server_uri')) bridge = Bridge(server_uri=configurator.get('Bridge', 'server_uri')) server.start() client.start() bridge.start()
def load_config(self, config): if not hasattr(self, 'config'): print('load config') self.config = Configurator(config).from_yaml()
def set_cube(params, config): configurator = Configurator(config['enhancer']) config = configurator.from_yaml() config['enhancement']['cube'] = [params[0], params[1]] configurator.dump_yaml(config)
from torchvision import transforms from configparser import ConfigParser from PIL import Image import torch import json import numpy as np import math import os, sys if __name__ == '__main__': sys.path.append(os.getcwd()) sys.path[0] = os.path.dirname(os.path.realpath(__file__)) from utils.configurator import Configurator configurator = Configurator() DATASET_ROOT_PATH = configurator.get('Dataset', 'dataset_root_path') CORRECT_DATASET_ROOT_PATH = configurator.get('Dataset', 'correct_dataset_root_path') INCORRECT_DATASET_ROOT_PATH = configurator.get('Dataset', 'incorrect_dataset_root_path') CORRECT_SAMPLES_NUMBER = 67048 # but 67049 officially declared INCORRECT_SAMPLES_NUMBER = 66734 VGG_IMG_SIZE = 224 class LazyDataset(Dataset): def __init__(self, mode='train'):
class FarmScreen(Screen): def __init__(self, message, bot): super().__init__(message, bot) self.title = 'Famring options:' self.load_config() self._init_buttons() # Try to move this method do Screen class def load_config(self): self.configfile = Configurator(self.config['farming']) self.config = self.configfile.from_yaml() # print(self.config) def back(self, call, state): screen = 'StartScreen' ss = state[screen] ss.render(call=call) return 'StartScreen', ss def stop(self, call, state): Config().disable() return None, None def _init_buttons(self): for b in self.config['presets']: name = 'farm_{0}'.format(b['name']) def handle(call, state): data = call.data.split('_')[1] preset = self._findPreset(data) a = preset['actions'] t = preset['timings'] print('call key', data, a, t) Farming(a, t).run() return None, None setattr(self, name, handle) return None def render(self, call=None): self.markup.keyboard = [] for i in self.config['presets']: title = '{0}'.format(i['name']) callback = '{name}.farm_{executor}'.format(name=self.name, executor=str(title)) self.markup.add( self.InlineKeyboardButton(title, callback_data=callback)) self.markup.add( self.InlineKeyboardButton('STOP', callback_data='{0}.{1}'.format( self.name, 'stop'))) self.markup.add( self.InlineKeyboardButton('BACK', callback_data='{0}.{1}'.format( self.name, 'back'))) if call is None: self.send() else: self.edit(call) def _findPreset(self, name): for p in self.config['presets']: if p['name'] == name: return p return []
def setUp(self): self.conf = Configurator()
def set_logout(config): configurator = Configurator(config['buffer']) config = configurator.from_yaml() config['spawn'] = True config['logout'] = True configurator.dump_yaml(config)
def set_buff(params, config): configurator = Configurator(config['buffer']) config = configurator.from_yaml() config['refresh'] = True config['spawn'] = False configurator.dump_yaml(config)
def set_enhance_mode(mode, config): configurator = Configurator(config['enhancer']) config = configurator.from_yaml() config['mode'] = mode configurator.dump_yaml(config)
def set_cycles(params, config): configurator = Configurator(config['enhancer']) config = configurator.from_yaml() config['enhancement']['cycles'] = params configurator.dump_yaml(config)
def load_config(self): self.config = Configurator(self.config['enhancer']).from_yaml()
def load_config(self): self.configfile = Configurator(self.config['farming']) self.config = self.configfile.from_yaml()
def load_config(self): self.configfile = Configurator(self.config['enhancer']) self.config = self.configfile.from_yaml()
from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import sessionmaker import os, sys if __name__ == '__main__': sys.path.append(os.getcwd()) sys.path[0] = os.path.dirname(os.path.realpath(__file__)) from utils.configurator import Configurator configurator = Configurator() engine = create_engine(configurator.get('Database', 'connection_string')) Session = sessionmaker(bind=engine) Base = declarative_base()
class TestConfigurator(unittest.TestCase): def setUp(self): self.conf = Configurator() def test_set_output(self): self.conf.set_output(filename="test.txt", fileformat="csv", compound="test") self.assertEqual(self.conf.scrapy_settings["FEED_URI"], "test.txt") self.assertEqual(self.conf.scrapy_settings["FEED_FORMAT"], "csv") self.conf.set_output("<compound>.*format*", "jsonlines", "test") self.assertEqual(self.conf.scrapy_settings["FEED_URI"], "test.json") self.assertEqual(self.conf.scrapy_settings["FEED_FORMAT"], "jsonlines") self.conf.set_output("<compound>.*format*", "csv", "test") self.assertEqual(self.conf.scrapy_settings["FEED_URI"], "test.csv") self.assertEqual(self.conf.scrapy_settings["FEED_FORMAT"], "csv") def test_start_log(self): for i in range(0, 3): self.conf.set_logging("TEST", i) self.assertEqual(self.conf.scrapy_settings.get("LOG_FILE"), "TEST") if i > 0: self.assertEqual(self.conf.scrapy_settings.get("LOG_ENABLED"), True) if i > 1: self.assertEqual(self.conf.scrapy_settings.get("LOG_STDOUT"), False) else: self.assertEqual(self.conf.scrapy_settings.get("LOG_STDOUT"), True) else: self.assertEqual(self.conf.scrapy_settings.get("LOG_ENABLED"), False) self.assertEqual(self.conf.scrapy_settings.get("LOG_STDOUT"), True) if i == 1: self.assertEqual(self.conf.scrapy_settings.get("LOG_LEVEL"), "WARNING") elif i == 2: self.assertEqual(self.conf.scrapy_settings.get("LOG_LEVEL"), "INFO") elif i == 3: self.assertEqual(self.conf.scrapy_settings.get("LOG_LEVEL"), "DEBUG") self.conf.set_logging(verbose=i) self.assertEqual(self.conf.scrapy_settings.get("LOG_FILE"), None) def test_read_sourceconfiguration(self): config = self.conf.read_sourceconfiguration() self.assertIsInstance(config, ConfigParser.ConfigParser) def test_get_section(self): config = ConfigParser.ConfigParser() section = self.conf.get_section(config, 'test') self.assertIn('reliability', section) self.assertEquals(section['reliability'], '') config.set('DEFAULT', 'reliability', 'Low') section = self.conf.get_section(config, 'test') self.assertEquals(section['reliability'], 'Low') config.add_section('test') config.set('test', 'var', 'Maybe') section = self.conf.get_section(config, 'test') self.assertEquals(section['reliability'], 'Low') self.assertEqual(section['var'], 'Maybe')
class EnhanceScreen(Screen): buttons = ['Cube', 'Binary', 'Combination', 'Disassamble', 'Back'] def __init__(self, message, bot): super().__init__(message, bot) self.title = 'Enhance menu:' self.load_config() self.generate_cycles() def generate_cycles(self): for i in range(4): cycle_id = i+1 attr_name = 'cycle_{0}'.format(cycle_id) def cycle(call, state): data = call.data.split('_')[1] self.config['mode'] = 'single' self.config['enhancement']['cycles'] = data self.config['enhancement']['cube'] = state['CubesScreen'].config['enhancement']['cube'] self.configfile.dump_yaml(self.config) self.bot.answer_callback_query(call.id, 'Start {0} cycles'.format(data)) Enhancer(self.config).process() return self.name, self setattr(self, attr_name, cycle) return None def render(self, call=None): self.markup.keyboard = [] cycles = [] for i in range(1, 5): title = 'Сycles ' + str(i) callback = '{name}.cycle_{i}'.format(name=self.name, i=str(i)) cycles.append(self.InlineKeyboardButton(title, callback_data=callback)) self.markup.row_width = 4 self.markup.add(*cycles) for b in self.buttons: self.markup.add(self.InlineKeyboardButton(b, callback_data='{name}.{action}'.format(name=self.name, action=b.lower()))) if call is None: self.send() else: self.edit(call) def cube(self, call, state): cs = CubesScreen(self.message, self.bot) cs.render(call=call) return cs.name, cs def binary(self, call, state): self.config['enhancement']['cube'] = state['CubesScreen'].config['enhancement']['cube'] self.config['mode'] = 'binary' self.configfile.dump_yaml(self.config) Enhancer(self.config).process() return self.name, self def combination(self, call, state): cs = CombinationScreen(self.message, self.bot) cs.render(call=call) return cs.name, cs def disassamble(self, call, state): self.config['enhancement']['cube'] = state['CubesScreen'].config['enhancement']['cube'] self.config['mode'] = 'disassamble' Enhancer(self.config).process() return self.name, self def back(self, call, state): screen = 'StartScreen' ss = state[screen] ss.render(call=call) return 'StartScreen', ss def load_config(self): self.configfile = Configurator(self.config['enhancer']) self.config = self.configfile.from_yaml()
class TestConfigurator(unittest.TestCase): def setUp(self): self.conf = Configurator() def test_set_output(self): self.conf.set_output(filename="test.txt", fileformat="csv", compound="test") self.assertEqual(self.conf.scrapy_settings["FEED_URI"], "test.txt") self.assertEqual(self.conf.scrapy_settings["FEED_FORMAT"], "csv") self.conf.set_output("<compound>.*format*", "jsonlines", "test") self.assertEqual(self.conf.scrapy_settings["FEED_URI"], "test.json") self.assertEqual(self.conf.scrapy_settings["FEED_FORMAT"], "jsonlines") self.conf.set_output("<compound>.*format*", "csv", "test") self.assertEqual(self.conf.scrapy_settings["FEED_URI"], "test.csv") self.assertEqual(self.conf.scrapy_settings["FEED_FORMAT"], "csv") def test_start_log(self): for i in range(0, 3): self.conf.set_logging("TEST", i) self.assertEqual(self.conf.scrapy_settings.get("LOG_FILE"), "TEST") if i > 0: self.assertEqual(self.conf.scrapy_settings.get("LOG_ENABLED"), True) if i > 1: self.assertEqual( self.conf.scrapy_settings.get("LOG_STDOUT"), False) else: self.assertEqual( self.conf.scrapy_settings.get("LOG_STDOUT"), True) else: self.assertEqual(self.conf.scrapy_settings.get("LOG_ENABLED"), False) self.assertEqual(self.conf.scrapy_settings.get("LOG_STDOUT"), True) if i == 1: self.assertEqual(self.conf.scrapy_settings.get("LOG_LEVEL"), "WARNING") elif i == 2: self.assertEqual(self.conf.scrapy_settings.get("LOG_LEVEL"), "INFO") elif i == 3: self.assertEqual(self.conf.scrapy_settings.get("LOG_LEVEL"), "DEBUG") self.conf.set_logging(verbose=i) self.assertEqual(self.conf.scrapy_settings.get("LOG_FILE"), None) def test_read_sourceconfiguration(self): config = self.conf.read_sourceconfiguration() self.assertIsInstance(config, ConfigParser.ConfigParser) def test_get_section(self): config = ConfigParser.ConfigParser() section = self.conf.get_section(config, 'test') self.assertIn('reliability', section) self.assertEquals(section['reliability'], '') config.set('DEFAULT', 'reliability', 'Low') section = self.conf.get_section(config, 'test') self.assertEquals(section['reliability'], 'Low') config.add_section('test') config.set('test', 'var', 'Maybe') section = self.conf.get_section(config, 'test') self.assertEquals(section['reliability'], 'Low') self.assertEqual(section['var'], 'Maybe')
def set_combination_mode(mode, config): configurator = Configurator(config['enhancer']) config = configurator.from_yaml() config['combination']['mode'] = mode configurator.dump_yaml(config)
from processes.process import Process from utils.configurator import Configurator from processes.click import Click from processes.wait import Wait from processes.key import Key import pyautogui as ui config = Configurator('config.yml').from_yaml() serial = Process(config['serial']).run_serial() for i in range(0, 6): count = Click(536, 450, process='dclick') count.make_click(serial) Wait(0.5).delay() on = Click(464, 451, process='dclick') on.make_click(serial) Wait(0.5).delay() one = Key('1') one.press(serial) Wait(0.5).delay() enter = Key('E') enter.press(serial) Wait(0.5).delay() make = Click(352, 492) make.make_click(serial) Wait(1).delay()