class CommonTest(object): """ Common class tests """ db = None def setup_class(self): open(wrpath + '/proxies.list', 'w').close() config = configparser.ConfigParser() config.read(os.getcwd() + '/' + 'config.ini') try: db = mysql.connector.connect( host=config['db']['host'], user=config['db']['user'], password=config['db']['pass'], database=config['db']['database'] ) db.autocommit = True except mysql.connector.errors.ProgrammingError as e: print " ERROR: Can`t connect to MySQL server! ({0})".format(str(e)) exit(0) try: mc = MongoClient(host=config['mongo']['host'], port=int(config['mongo']['port'])) mongo_collection = getattr(mc, config['mongo']['collection']) except pymongo.errors.ConnectionFailure as e: print " ERROR: Can`t connect to MongoDB server! ({0})".format(str(e)) exit(0) R = Registry() R.set('config', config) R.set('db', db) R.set('mongo', mongo_collection) R.set('wr_path', wrpath) R.set('data_path', wrpath + '/data/') R.set('ndb', Database(config['db']['host'], config['db']['user'], config['db']['pass'], config['db']['database'])) self.db = Registry().get('ndb') def setup(self): self.clear_db() def clear_db(self): """ Clearing db """ self.db.q("TRUNCATE TABLE `hosts`") self.db.q("TRUNCATE TABLE `ips`") self.db.q("TRUNCATE TABLE `projects`") self.db.q("TRUNCATE TABLE `urls`") self.db.q("TRUNCATE TABLE `requests`") self.db.q("TRUNCATE TABLE `hosts_info`") self.db.q("TRUNCATE TABLE `urls_base`") self.db.q("TRUNCATE TABLE `urls_base_params`") self.db.q("TRUNCATE TABLE `cms`") self.db.q("TRUNCATE TABLE `cms_paths`") self.db.q("TRUNCATE TABLE `cms_paths_hashes`") def output_errors(self, out): """ Check output for errors :param out: output :return: """ assert out.find("Traceback") == -1 assert "killed by time" not in out def _replace_config(self, name): """ Change work config :param name: config name (*.ini) :return: """ shutil.copyfile("{0}/config.ini".format(wrpath), "{0}/config.ini.bak".format(wrpath)) shutil.copyfile("{0}/configs/{1}.ini".format(testpath, name), "{0}/config.ini".format(wrpath)) def _restore_config(self): """ Restore old original config """ shutil.move("{0}/config.ini.bak".format(wrpath), "{0}/config.ini".format(wrpath)) def _run(self, config_name, run_params): """ Run WS :param config_name: name of config for test :param run_params: params for run WS-process :return: output of WS """ self._replace_config(config_name) os.chdir(wrpath) out = subprocess.check_output(run_params) self._restore_config() self.output_errors(out) return out
class UrlsModel(object): """ Class for work with urls table """ _db = None def __init__(self): self._db = Registry().get('ndb') def add( self, pid, host_id, url, referer='', response_code=0, response_time=0, who_add='human', spidered=0, size=0, descr='' ): """ Add url to table """ try: return self._db.insert( "urls", { "project_id": pid, "host_id": host_id, "hash": md5(url), "url": url, "referer": referer, "response_code": response_code, "response_time": response_time, "when_add": int(time.time()), "who_add": who_add, "spidered": spidered, "size": size, "descr": descr }, 1 ) except BaseException as e: if Registry().isset('logger'): Registry().get('logger').ex(e) else: print "Can`t add url: " + str(e) def add_mass(self, pid, host_id, data): """ Add many urls at once in table """ to_insert = [] for row in data: for field in ['url', 'referer', 'response_code', 'response_time', 'who_add', 'spidered', 'size', 'descr']: if field not in row.keys(): if field in ['referer', 'response_code', 'response_time', 'descr']: row[field] = '' elif field in ['spidered', 'size']: row[field] = 0 elif field == 'who_add': row[field] = 'human' elif field == 'url': raise WSException("URL row must have a 'url' key") for k in row.keys(): if k not in [ 'url', 'referer', 'response_code', 'response_time', 'who_add', 'spidered', 'size', 'descr' ]: raise WSException("Key '{0}' must not be in url data".format(k)) to_insert.append({ 'project_id': pid, "host_id": host_id, "hash": md5(row['url']), "url": row['url'], "referer": row['referer'], "response_code": row['response_code'], "response_time": row['response_time'], "when_add": int(time.time()), "who_add": row['who_add'], "spidered": row['spidered'], "size": row['size'], "descr": row['descr'] }) if len(to_insert)%50 == 0: self._db.insert_mass("urls", to_insert, 1) to_insert = [] if len(to_insert): self._db.insert_mass("urls", to_insert, 1) return True def list_by_host_name(self, project_id, host, like=""): """ Get urls list by host name and project_id """ host_id = HostsModel().get_id_by_name(project_id, host) like_expr = "" \ if not len(like.strip()) \ else " AND url LIKE '%{0}%' ".format(self._db.escape(like.strip())) return self._db.fetch_all( "SELECT url, response_code as code, response_time as time, when_add, who_add, descr, size FROM urls " "WHERE project_id = {0} AND host_id = {1} ".format(project_id, host_id) + like_expr + "ORDER BY url" ) def list_by_host_name_for_spider(self, project_id, host): """ Get urls list by host name and project_id, but in special format for spider """ host_id = HostsModel().get_id_by_name(project_id, host) return self._db.fetch_all( "SELECT url, response_code as code, response_time as time, when_add, who_add, descr FROM urls " "WHERE project_id = {0} AND host_id = {1} AND !spidered " "ORDER BY url".format(project_id, host_id) ) def exists(self, project_id, host, url): """ Is url exists? """ host_id = HostsModel().get_id_by_name(project_id, host) return self._db.fetch_one( "SELECT 1 FROM urls WHERE project_id = {0} AND host_id={1} AND hash = '{2}'" .format(project_id, host_id, md5(url)) ) def delete(self, project_id, host, url): """ Delete url from table """ host_id = HostsModel().get_id_by_name(project_id, host) self._db.q( "DELETE FROM urls WHERE project_id = {0} AND host_id = {1} AND hash = {2} " .format(project_id, host_id, self._db.quote(md5(url))) ) def update_url_field(self, project_id, host, url, field, value): """ Update custom field of current url """ host_id = HostsModel().get_id_by_name(project_id, host) return self._db.update( "urls", {field: value}, "hash = '{0}' AND project_id = {1} AND host_id = {2}".format(md5(url), project_id, host_id) ) def update_url_field_mass(self, project_id, host, field, data): """ Mass update custom field of many urls """ host_id = HostsModel().get_id_by_name(project_id, host) update = {} for row in data: case = "host_id = '{0}' AND `hash` = '{1}' ".format(host_id, md5(row['url'])) update[case] = row['value'] self._db.update_mass("urls", field, update)
class UrlsModel(object): """ Class for work with urls table """ _db = None def __init__(self): self._db = Registry().get('ndb') def add(self, pid, host_id, url, referer='', response_code=0, response_time=0, who_add='human', spidered=0, size=0, descr=''): """ Add url to table """ try: return self._db.insert( "urls", { "project_id": pid, "host_id": host_id, "hash": md5(url), "url": url, "referer": referer, "response_code": response_code, "response_time": response_time, "when_add": int(time.time()), "who_add": who_add, "spidered": spidered, "size": size, "descr": descr }, 1) except BaseException as e: if Registry().isset('logger'): Registry().get('logger').ex(e) else: print "Can`t add url: " + str(e) def add_mass(self, pid, host_id, data): """ Add many urls at once in table """ to_insert = [] for row in data: for field in [ 'url', 'referer', 'response_code', 'response_time', 'who_add', 'spidered', 'size', 'descr' ]: if field not in row.keys(): if field in [ 'referer', 'response_code', 'response_time', 'descr' ]: row[field] = '' elif field in ['spidered', 'size']: row[field] = 0 elif field == 'who_add': row[field] = 'human' elif field == 'url': raise WSException("URL row must have a 'url' key") for k in row.keys(): if k not in [ 'url', 'referer', 'response_code', 'response_time', 'who_add', 'spidered', 'size', 'descr' ]: raise WSException( "Key '{0}' must not be in url data".format(k)) to_insert.append({ 'project_id': pid, "host_id": host_id, "hash": md5(row['url']), "url": row['url'], "referer": row['referer'], "response_code": row['response_code'], "response_time": row['response_time'], "when_add": int(time.time()), "who_add": row['who_add'], "spidered": row['spidered'], "size": row['size'], "descr": row['descr'] }) if len(to_insert) % 50 == 0: self._db.insert_mass("urls", to_insert, 1) to_insert = [] if len(to_insert): self._db.insert_mass("urls", to_insert, 1) return True def list_by_host_name(self, project_id, host, like=""): """ Get urls list by host name and project_id """ host_id = HostsModel().get_id_by_name(project_id, host) like_expr = "" \ if not len(like.strip()) \ else " AND url LIKE '%{0}%' ".format(self._db.escape(like.strip())) return self._db.fetch_all( "SELECT url, response_code as code, response_time as time, when_add, who_add, descr, size FROM urls " "WHERE project_id = {0} AND host_id = {1} ".format( project_id, host_id) + like_expr + "ORDER BY url") def list_by_host_name_for_spider(self, project_id, host): """ Get urls list by host name and project_id, but in special format for spider """ host_id = HostsModel().get_id_by_name(project_id, host) return self._db.fetch_all( "SELECT url, response_code as code, response_time as time, when_add, who_add, descr FROM urls " "WHERE project_id = {0} AND host_id = {1} AND !spidered " "ORDER BY url".format(project_id, host_id)) def exists(self, project_id, host, url): """ Is url exists? """ host_id = HostsModel().get_id_by_name(project_id, host) return self._db.fetch_one( "SELECT 1 FROM urls WHERE project_id = {0} AND host_id={1} AND hash = '{2}'" .format(project_id, host_id, md5(url))) def delete(self, project_id, host, url): """ Delete url from table """ host_id = HostsModel().get_id_by_name(project_id, host) self._db.q( "DELETE FROM urls WHERE project_id = {0} AND host_id = {1} AND hash = {2} " .format(project_id, host_id, self._db.quote(md5(url)))) def update_url_field(self, project_id, host, url, field, value): """ Update custom field of current url """ host_id = HostsModel().get_id_by_name(project_id, host) return self._db.update( "urls", {field: value}, "hash = '{0}' AND project_id = {1} AND host_id = {2}".format( md5(url), project_id, host_id)) def update_url_field_mass(self, project_id, host, field, data): """ Mass update custom field of many urls """ host_id = HostsModel().get_id_by_name(project_id, host) update = {} for row in data: case = "host_id = '{0}' AND `hash` = '{1}' ".format( host_id, md5(row['url'])) update[case] = row['value'] self._db.update_mass("urls", field, update)
class CommonUnit(object): """ Common class for unit tests """ db = None def setup_class(self): """ Prepare class for run tests """ CURPATH = os.path.dirname(__file__) + "/" config = configparser.ConfigParser() config.read(CURPATH + 'config.ini') Registry().set('config', config) Registry().set('logger', LoggerMock()) db = Database( config['main']['mysql_host'], config['main']['mysql_user'], config['main']['mysql_pass'], config['main']['mysql_dbname'], ) Registry().set('db', db) self.db = Registry().get('db') # type: Database def _clean_db(self): """ Clean tables for tests """ self.db.q("TRUNCATE TABLE dicts") self.db.q("TRUNCATE TABLE dicts_groups") self.db.q("TRUNCATE TABLE hashes") self.db.q("TRUNCATE TABLE hashlists") self.db.q("TRUNCATE TABLE rules") self.db.q("TRUNCATE TABLE tasks") self.db.q("TRUNCATE TABLE tasks_groups") self.db.q("TRUNCATE TABLE task_works") self.db.q("TRUNCATE TABLE logs") self.db.update("algs", {'finder_insidepro_allowed': 0}, "id") def _add_hashlist(self, id=1, name='test', alg_id=3, have_salts=0, status='ready', common_by_alg=0, parsed=1, last_finder_checked=0): """ Add hashlist record """ self.db.insert( "hashlists", { 'id': id, 'name': name, 'alg_id': alg_id, 'have_salts': have_salts, 'delimiter': '', 'cracked': 0, 'uncracked': 0, 'errors': '', 'parsed': parsed, 'tmp_path': '', 'status': status, 'when_loaded': 0, 'common_by_alg': common_by_alg, 'last_finder_checked': last_finder_checked } ) def _add_hash(self, hashlist_id=1, hash='', salt='', summ='', password='', cracked=0, id=None): """ Add hash record """ self.db.insert( "hashes", { 'id': id, 'hashlist_id': hashlist_id, 'hash': hash, 'salt': salt, 'password': password, 'cracked': cracked, 'summ': summ } ) def _add_work_task(self, id=1, hashlist_id=1, task_id=1, status='wait', priority=0): """ Add work task record """ self.db.insert( "task_works", { 'id': id, 'hashlist_id': hashlist_id, 'task_id': task_id, 'status': status, 'priority': priority, } ) def _add_task(self, id=1, name='task', group_id=1, type='dict', source=1): """ Add task record """ self.db.insert( "tasks", { 'id': id, 'name': name, 'group_id': group_id, 'type': type, 'source': source, } ) def _add_dict(self, id=1, group_id=1, name='dict', hash='1'): """ Add dict record """ self.db.insert( "dicts", { 'id': id, 'name': name, 'group_id': group_id, 'hash': hash, } ) def _add_dict_group(self, id=1, name='group'): """ Add dict group record """ self.db.insert( "dicts_groups", { 'id': id, 'name': name, } ) def _add_rule(self, id=1, name='rule', hash='1.rule', count=1): """ Add rule record """ self.db.insert( "rules", { 'id': id, 'name': name, 'hash': hash, 'count': count } )
class CommonIntegration(object): """ Common class for integration tests """ db = None def setup_class(self): """ Prepare class for run tests """ CURPATH = os.path.dirname(__file__) + "/" config = configparser.ConfigParser() config.read(CURPATH + 'config.ini') Registry().set('config', config) Registry().set('logger', LoggerMock()) db = Database( config['main']['mysql_host'], config['main']['mysql_user'], config['main']['mysql_pass'], config['main']['mysql_dbname'], ) Registry().set('db', db) self.db = Registry().get('db') # type: Database def _clean_db(self): """ Clean tables for tests """ self.db.q("TRUNCATE TABLE dicts") self.db.q("TRUNCATE TABLE dicts_groups") self.db.q("TRUNCATE TABLE hashes") self.db.q("TRUNCATE TABLE hashlists") self.db.q("TRUNCATE TABLE rules") self.db.q("TRUNCATE TABLE tasks") self.db.q("TRUNCATE TABLE tasks_groups") self.db.q("TRUNCATE TABLE task_works") self.db.q("TRUNCATE TABLE logs") self.db.update("algs", {'finder_insidepro_allowed': 0}, "id") def _add_hashlist(self, id=1, name='test', alg_id=3, have_salts=0, status='ready', common_by_alg=0, parsed=1, tmp_path='', last_finder_checked=0, uncracked=0): """ Add hashlist record """ self.db.insert( "hashlists", { 'id': id, 'name': name, 'alg_id': alg_id, 'have_salts': have_salts, 'delimiter': 'UNIQUEDELIMITER' if have_salts else '', 'cracked': 0, 'uncracked': uncracked, 'errors': '', 'parsed': parsed, 'tmp_path': tmp_path, 'status': status, 'when_loaded': 0, 'common_by_alg': common_by_alg, 'last_finder_checked': last_finder_checked, }) def _add_hash(self, hashlist_id=1, hash='', salt='', summ='', password='', cracked=0, id=None): """ Add hash record """ self.db.insert( "hashes", { 'id': id, 'hashlist_id': hashlist_id, 'hash': hash, 'salt': salt, 'password': password, 'cracked': cracked, 'summ': summ }) def _add_work_task(self, id=1, hashlist_id=1, task_id=1, status='wait', priority=0, out_file=''): """ Add work task record """ self.db.insert( "task_works", { 'id': id, 'hashlist_id': hashlist_id, 'task_id': task_id, 'status': status, 'priority': priority, 'out_file': out_file, }) def _add_task(self, id=1, name='task', group_id=1, type='dict', source=1): """ Add task record """ self.db.insert( "tasks", { 'id': id, 'name': name, 'group_id': group_id, 'type': type, 'source': source, }) def _add_dict(self, id=1, group_id=1, name='dict', hash='1'): """ Add dict record """ self.db.insert("dicts", { 'id': id, 'name': name, 'group_id': group_id, 'hash': hash, }) def _add_dict_group(self, id=1, name='group'): """ Add dict group record """ self.db.insert("dicts_groups", { 'id': id, 'name': name, }) def _add_rule(self, id=1, name='rule', hash='1.rule', count=1): """ Add rule record """ self.db.insert("rules", { 'id': id, 'name': name, 'hash': hash, 'count': count })