def suggest(self, criteria): # Argument "criteria" is the keywords typed in by user criteria = criteria.lower() logger.log(0, 'User wants a suggestion about ' + criteria + ' via API.', 'api_logfile.txt', self.SESSION_ID) suggest.api(criteria)
def error_except(self): try: loop = True while loop == True: quit = None ########## LABEL FOR SSF ONLY ########## ask = input( CB + FB + FI + 'Do you want to keep Shadow Suite running? (y/n)> ' + CW + FR) ask = ask.lower() if ask == 'y': loop = False quit = False elif ask == 'n': loop = False quit = True else: loop = True return quit except KeyboardInterrupt: print(error.errorCodes().ERROR0002) logger.log(2, "SystemExit raised with error code 2.", 'logfile.txt') sys.exit(2)
def get_meta(): # Set Flag, Get Host, Get Realm, Get Date global con global crawler_id b_id = None url = None realm = None last_crawl = None last_post = None succ_flag = False try: cursor = con.cursor() cursor.execute("select * from get_noncrawled_meta( %s )", [crawler_id]) record = cursor.fetchone() if record: b_id, url, realm, last_crawl, last_post = record succ_flag = True con.commit() else: con.rollback() except Exception, e: logger.log("##DBERR get meta error :", e.message) if con: con.rollback()
def resolve(self, var_id, data_type): condition = {} if self._state.is_tracking(var_id): var = self._state.get_var_ref(var_id) if var['type'] == 'unknown': self._state.update_guessed_type(var_id, data_type) condition = {'type': 'base_var', 'id': var_id} elif self.is_tracking(var_id): transform = self._transforms[var_id] function_name = transform['function'] if function_name in self._resolver: condition = self._resolver[function_name].process( data_type, transform['args']) if 'args' in condition: for i in range(len(condition['args'])): if condition['args'][i]['type'] == 'symbolic': condition['args'][i] = self.resolve( condition['args'][i]['id'], data_type) else: msg = 'Not processing %s (no resolver)' % function_name logger.log(msg, '', Logger.DEBUG) raise Exception(msg) else: raise Exception('Cannot resolve value for untracked id \'%s\'.' % var_id) return condition
def export_conf(self, config_file, config_dict): # Exports the current configuration to a file. logger.log( 3, self.current_user + " is exporting settings to " + config_file + " via API.", 'api_logfile.txt', self.SESSION_ID) result = misc.programFunctions().export_conf(config_file, config_dict) return result
def unzip(file_name) -> bool: """ Faz o un-zip do arquivo final Peguei de: https://stackoverflow.com/questions/17913952/monitoring-zip-file-extraction-display-percentage-in-python :return: bool """ with ZipFile(file_name, 'r') as infile: for member_info in infile.infolist(): filename = member_info.filename file_size = member_info.file_size with open("{}".format(filename), 'wb') as outfile: member_fd = infile.open(filename) total_bytes = 0 while 1: x = member_fd.read(Zip.chunk_size) if not x: break total_bytes += outfile.write(x) log("{0}% extraido".format(100 * total_bytes / file_size)) return True
def input_on(db_type, log_type): account = input('\033[1;34m账号:\033[0m') pwd = input('\033[1;34m密码:\033[0m') user_data = enroll(account, pwd, db_type) if user_data != None: logger.log('登录成功', log_type, user_data['id']) return user_data
def facebook_login(web_driver: RemoteWebDriver, login: str, password: str) -> bool: """ Funcao para realizar os passos necessarios para o login, e retornar se foi bem sucedida ou nao :param web_driver: RemoteWebDriver instancia do driver :param login: str login text-plain :param password: str senha text-plain :return: bool login realizado com sucesso? """ # Salvamos a instancia Login.set_web_driver(web_driver) # Salvamos o login Login.set_login(login) # Salvamos a senha Login.set_password(password) # Acessamos a pagina de login Login.__access() # Informamos o Handler log('Handler: [{}]'.format(web_driver.current_window_handle)) # Processos para o login is_logged = Login.__input_login().__input_password().__submit( ).__check_save_device().__is_logged() if is_logged: log("O login foi realizado com sucesso!") return is_logged
def withdraw(acc): """ 取款 :param acc: 用户信息字典 :return: None """ while True: money = input("请输入提款金额: ") if money.isdigit(): if acc['balance'] >= int(money): fee = 0.05 * int(money) # 手续费5% if fee < 1: # 不足一元按一元收手续费 fee = 1 acc['balance'] -= int(money) save_check(acc, WITHDRAW, money, acc['balance']) save_db(acc) print("\033[32;1m成功提现%d元(手续费%d元),当前余额为%s元\033[0m" % ((int(money) - fee), fee, acc['balance'])) logger.log("transactions").info( log_msg % (acc['id'], 'withdraw', int(money))) return else: print("\033[31;1m余额不足\033[0m") logger.log("transactions").error( log_msg % (acc['id'], 'withdraw', int(money)) + " error: balance not enough") else: print("\033[31;1m请输入数字\033[0m")
def action(self, query, args=None): if query is None: return sql_result = None attempt = 0 while attempt < 5: try: if args is None: logger.log(u'{name}: {query}'.format(name=self.filename, query=query), logger.DB) sql_result = self.connection.execute(query) else: logger.log(u'{name}: {query} with args {args}'.format (name=self.filename, query=query, args=args), logger.DB) sql_result = self.connection.execute(query, args) self.connection.commit() # get out of the connection attempt loop since we were successful break except sqlite3.OperationalError as error: if 'unable to open database file' in error.args[0] or 'database is locked' in error.args[0]: logger.log(u'DB error: {msg}'.format(msg=error), logger.WARNING) attempt += 1 time.sleep(1) else: logger.log(u'DB error: {msg}'.format(msg=error), logger.ERROR) raise except sqlite3.DatabaseError as error: logger.log(u'Fatal error executing query: {msg}'.format(msg=error), logger.ERROR) raise return sql_result
def run(self): options = self.options # List mods if options.listmods: Logger.header("List of mods", True) mods_dir = "core/mods/" for dir in os.listdir(mods_dir): if os.path.isdir(os.path.join(mods_dir, dir)): Logger.log(dir, True) sys.exit(2) # Train if options.train: captcha = options.train self.train(captcha) # Crack if options.crack: captcha = options.crack self.crack(captcha) # Batch train with percentage if options.batch_train: self.batch_train(options.batch_train) # Batch crack with percentage if options.batch_crack: self.batch_crack(options.batch_crack)
def __get_base_data(): """ Obtem dados base do Post :return: Post """ driver = Post.get_web_driver() log(u"Obtendo dados base do post.") element = PostElements.get_post_data(driver) data_ft = "" if element: data_ft = element.get_attribute('data-ft') Post.set_data_ft_file(Post.__save_path + '/data-ft.json') if data_ft: with codecs.open(Post.get_data_ft_file(), mode='w', encoding="utf-8") as f: f.write(data_ft) log(u"Dados base salvos em: [{}]".format(Post.get_data_ft_file())) return Post
def _check_github_for_update(self): """ Uses git commands to check if there is a newer version that the provided commit hash. If there is a newer version it sets _num_commits_behind. """ self._newest_commit_hash = None self._num_commits_behind = 0 self._num_commits_ahead = 0 # get all new info from github output, err, exit_status = self._run_git(self._git_path, 'fetch origin') if not exit_status == 0: logger.log(u"Unable to contact github, can't check for update", logger.ERROR) return # get latest commit_hash from remote output, err, exit_status = self._run_git( self._git_path, 'rev-parse --verify --quiet "@{upstream}"') if exit_status == 0 and output: cur_commit_hash = output.strip() if not re.match('^[a-z0-9]+$', cur_commit_hash): logger.log(u"Output doesn't look like a hash, not using it", logger.DEBUG) return else: self._newest_commit_hash = cur_commit_hash else: logger.log(u"git didn't return newest commit hash", logger.DEBUG) return # get number of commits behind and ahead (option --count not supported git < 1.7.2) output, err, exit_status = self._run_git( self._git_path, 'rev-list --left-right "@{upstream}"...HEAD') if exit_status == 0 and output: try: self._num_commits_behind = int(output.count("<")) self._num_commits_ahead = int(output.count(">")) except: logger.log( u"git didn't return numbers for behind and ahead, not using it", logger.DEBUG) return logger.log( u"cur_commit = {current} % (newest_commit)= {new}, " u"num_commits_behind = {x}, num_commits_ahead = {y}".format( current=self._cur_commit_hash, new=self._newest_commit_hash, x=self._num_commits_behind, y=self._num_commits_ahead), logger.DEBUG)
def get_url(self) -> str: """ Neste metodo vamos obter a url de download dos drivers :return: str Url de Download """ so = SystemArchiteture.get_operational_system() bits = SystemArchiteture.get_bits() if self.__package == 'geckodriver': # Versao do Gecko vers = consts.GECKO_DRIVER_VERSION # Vamos trabalhar apenas com Windows e Linux plat = 'win' if so == 'WindowsPE' else 'linux' log(u"Estamoso buscando um driver para seu sistema {0}".format( plat)) url = FindDriverDownload.get_url(self.__package, vers, plat, bits) log(u"A URL encontrada foi: [{0}]".format(url)) return url
def save_article(b_id, data): global con result = False try: cursor = con.cursor() cursor.execute("SELECT article_id, b_id, post_id from blog_data where b_id = %s and post_id = %s", [b_id, data["post_id"]]) row = cursor.fetchone() if row is None: cursor.execute("INSERT INTO blog_data(b_id, post_id, author, title, crawl_date, post_date, contents) values(%s, %s, %s, %s, %s, %s, %s)", [b_id, data["post_id"], data["name"], data["title"], datetime.datetime.now(), data["date"], data["content"]]) con.commit() result = True image_list = [[b_id, data["post_id"], url, datetime.datetime.now()] for url in data["images"]] for image in image_list: cursor.execute("INSERT INTO blog_image(b_id, post_id, image_url, crawl_date) VALUES(%s, %s, %s, %s)", image) con.commit() else: a_id, b_id, p_id = row cursor.execute("UPDATE blog_data set contents = %s , crawl_date = %s, post_date = %s where article_id = %s", [data["content"], datetime.datetime.now(), data["date"], a_id]) con.commit() result = True except Exception, e: logger.log("##DBERR save error : %s " % e.message) con.rollback()
def action(self, query, args=None): if query == None: return sqlResult = None attempt = 0 while attempt < 5: try: if args == None: logger.log(self.filename + ": " + query, logger.DB) sqlResult = self.connection.execute(query) else: logger.log( self.filename + ": " + query + " with args " + str(args), logger.DB) sqlResult = self.connection.execute(query, args) self.connection.commit() # get out of the connection attempt loop since we were successful break except sqlite3.OperationalError, e: if "unable to open database file" in e.args[ 0] or "database is locked" in e.args[0]: logger.log(u"DB error: " + str(e), logger.WARNING) attempt += 1 time.sleep(1) else: logger.log(u"DB error: " + str(e), logger.ERROR) raise except sqlite3.DatabaseError, e: logger.log(u"Fatal error executing query: " + str(e), logger.ERROR) raise
def fetch(self, query, args=None): if query == None: return sqlResult = None attempt = 0 while attempt < 5: try: if args == None: logger.log(self.filename + ": " + query, logger.DB) cursor = self.connection.cursor() cursor.execute(query) sqlResult = cursor.fetchone()[0] else: logger.log(self.filename + ": " + query + " with args " + str(args), logger.DB) cursor = self.connection.cursor() cursor.execute(query, args) sqlResult = cursor.fetchone()[0] # get out of the connection attempt loop since we were successful break except sqlite3.OperationalError, e: if "unable to open database file" in e.args[0] or "database is locked" in e.args[0]: logger.log(u"DB error: " + str(e), logger.WARNING) attempt += 1 time.sleep(1) else: logger.log(u"DB error: " + str(e), logger.ERROR) raise except sqlite3.DatabaseError, e: logger.log(u"Fatal error executing query: " + str(e), logger.ERROR) raise
def trans_money(acc, obj_acc): """ 转账 :param acc: 转账发起用户的信息字典 :param obj_acc: 转账对象的用户字典 :return: None """ while True: money = input("请输入转账金额: ") if money.isdigit(): if acc['balance'] >= int(money): acc['balance'] -= int(money) obj_acc['balance'] += int(money) save_check(acc, TRANS, money, acc['balance']) save_check(obj_acc, GET, money, obj_acc['balance']) save_db(acc) save_db(obj_acc) print("\033[32;1m转账成功\033[0m") logger.log("transactions").info( log_msg % (acc['id'], 'transfer money', int(money))) return else: print("\033[31;1m余额不足\033[0m") logger.log("transactions").error( log_msg % (acc['id'], 'transfer money', int(money)) + " error: balance not enough") else: print("\033[31;1m请输入数字\033[0m")
def solve_rec(self, condition): op_name = condition['type'] if op_name in self._adapters: return self._adapters[op_name].process(condition) else: logger.log('Not processing %s (no adapter)' % op_name, '', Logger.DEBUG) raise Exception('Not processing %s (no adapter)' % op_name)
def need_update(self): self._find_installed_version() try: self._check_github_for_update() except Exception, e: logger.log(u"Unable to contact github, can't check for update: " + repr(e), logger.ERROR) return False
def extract(self): log(u"Extraindo compartilhamentos...") from modules.social_media.facebook.packages.facebook_api.post_sharedposts import PostSharedPosts self.__shared_posts = PostSharedPosts( access_token=PostShares.get_access_token(), page_id=PostShares.get_page_id(), post_id=PostShares.get_post_id()) return self
def uncompress(self): log("Descompactando...") import tarfile if tarfile.is_tarfile(self.__file_name): tar = tarfile.open(self.__file_name) tar.extractall() tar.close() else: Zip.unzip(self.__file_name)
def extract(self): log(u"Extraindo comentários...") from modules.social_media.facebook.packages.facebook_api.post_comments import PostComments as Comments self.__comments = Comments( access_token=PostComments.get_access_token(), page_id=PostComments.get_page_id(), post_id=PostComments.get_post_id()) return self
def set_newest_text(self): if self._num_commits_ahead: logger.log(u"Local branch is ahead of {branch}. Automatic update not possible.".format (branch=self.branch), logger.ERROR) elif self._num_commits_behind: logger.log(u"There is a newer version available (you're {x} commit{s} behind)".format (x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE) else: return
def set_login(self, login): """ Login que sera usado :param login: str :return: PostExploit """ log('Setado login: {}'.format(login)) self.__login = login return self
def saveCharacters(self, characters): characters_hashes = [] # Get characters' hashes for character in characters: m = hashlib.md5() # Get image as a string of 0 and 1 (for 255) and n (for new line) ones_array = map(lambda lst: map(lambda x: "0" if x == 0 else "1", lst), numpy.array(character)) ones_string = "n".join(map(lambda lst: "".join(lst), ones_array)) m.update(ones_string) characters_hashes.append(m.hexdigest()) saveAsCategorized = False if self.options.auto_train: # Parse file name and find code basename = os.path.basename(self._captcha) code = self.getCodeFromString(basename) if len(code) != len(characters): saveAsCategorized = False Logger.error( "Error! Training found {0} chars while in file name are specified {1} chars. Real code is: {2}".format( len(characters), len(code), basename ), 1, ) else: saveAsCategorized = True if self.options.verbose: if saveAsCategorized: Logger.subheader("Saving characters into categorized folders") else: Logger.log("Saving characters into output folder") i = 0 for character in characters: character_hash = characters_hashes[i] if saveAsCategorized: character_symbol = code[i] dst = os.path.join(self.options.mod_dir, "char", character_symbol.lower(), character_hash + ".gif") else: dst = os.path.join(self.options.output_char_dir, character_hash + ".gif") character.save(dst) if saveAsCategorized: Logger.log("Saving {0} into mod folder".format(character_symbol)) i += 1
def get_reactions_list(self) -> list: from modules.social_media.facebook.helper.likes_file_reader import LikesFileReader likes_reader = LikesFileReader(self.get_likes_file()) likes = likes_reader.get_likes() log(u"{0} reações contabilizadas.".format(len(likes))) del likes_reader return likes
def rchmod(path, mod): logger.log("Changing file mode of {0} to {1}".format(path, oct(mod))) os.chmod(path, mod) if not os.path.isdir(path): return # Skip files for root, dirs, files in os.walk(path): for d in dirs: os.chmod(os.path.join(root, d), mod) for f in files: os.chmod(os.path.join(root, f), mod)
def need_update(self): self._find_installed_version() try: self._check_github_for_update() except Exception, e: logger.log( u"Unable to contact github, can't check for update: " + repr(e), logger.ERROR) return False
def create_options(self, args=None): self.options = CaptchonkaOptions().get_options(args) Logger.header('Options') for key, value in self.options.__dict__.items(): Logger.log(' ' + key + ': ' + str(value)) project_folder = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir)) self.options.output_dir = os.path.join(project_folder, 'output') if self.options.mod: self.options.mod_dir = os.path.join(project_folder, 'core', 'mods', self.options.mod)
def set_newest_text(self): if self._num_commits_ahead: logger.log(u"Local branch is ahead of " + self.branch + ". Automatic update not possible.", logger.ERROR) elif self._num_commits_behind > 0: newest_text = 'There is a newer version available ' newest_text += " (you're " + str(self._num_commits_behind) + " commit" if self._num_commits_behind > 1: newest_text += 's' newest_text += ' behind)' logger.log(newest_text, logger.MESSAGE) else: return
def __insert_like(reaction: str, username: str, user_url: str): data = { # "id": Profile.getId(user_url), "name": username, "profile_url": user_url, "reaction": reaction } log("Identificado: ") log(json.dumps(data, indent=4)) Post.__likes_list.append(data)
def set_password(self, password): """ Senha que sera usada :param password: str :return: PostExploit """ log('Setado senha: {}'.format(password)) self.__password = password return self
def set_newest_text(self): # if we're up to date then don't set this core.NEWEST_VERSION_STRING = None if not self._cur_commit_hash: logger.log(u"Unknown current version number, don't know if we should update or not", logger.ERROR) elif self._num_commits_behind > 0: logger.log(u"There is a newer version available (you're {x} commit{s} behind)".format (x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE) else: return
def find_service(self, service): # Argument "service" is the target service to view the info. service_name = self.SERVICES_PATH + service try: service_name = service_name.replace('/', '.') logger.log( 3, 'User shows info about ' + service_name + ' service via API.', 'logfile.txt', global_variables['SESSION_ID']) service_importlib(service_name, 'info') except Exception as service_err: print(error.errorCodes().ERROR0020(str(service_err)))
def set_newest_text(self): if self._num_commits_ahead: logger.log( u"Local branch is ahead of {branch}. Automatic update not possible." .format(branch=self.branch), logger.ERROR) elif self._num_commits_behind: logger.log( u"There is a newer version available (you're {x} commit{s} behind)" .format(x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE) else: return
def use_module(self, module): # Argument "module" is the target module to run. module = module.lower() logger.log( 0, 'User used use_module method to use ' + module + ' via API.', 'api_logfile.txt', self.SESSION_ID) try: module_path = self.MODULE_PATH.replace('/', '.') + module module_import = importlib.import_module(module_path) module_import.main() except ModuleNotFoundError as modulenotfounderror_msg: print("[i] " + str(modulenotfounderror_msg))
def flag(b_id, sw): global con try: cursor = con.cursor() cursor.execute("UPDATE blog_meta set crawler_id = 0, last_crawl = %s where b_id = %s", [datetime.datetime.now(), b_id]) con.commit() except: logger.log("##DBERR set flag error") if con: con.rollback()
def _find_installed_version(self): version_file = os.path.join(core.PROGRAM_DIR, u'version.txt') if not os.path.isfile(version_file): self._cur_commit_hash = None return try: with open(version_file, 'r') as fp: self._cur_commit_hash = fp.read().strip(' \n\r') except EnvironmentError, e: logger.log(u"Unable to open 'version.txt': " + str(e), logger.DEBUG)
def flag_rollback(b_id): global con try: cursor = con.cursor() cursor.execute("UPDATE blog_meta set crawler_id = 0 where b_id = %s", [b_id]) con.commit() except: logger.log("##DBERR rollback flag error") if con: con.rollback()
def __get_reactions(in_extended: bool = False): if not in_extended: log(u"Obtendo reações extendidas...") Post.get_web_driver().get(PATTERN_PAGE_POST_REACTIONS.format(Post.get_post_id())) lines = PostElements.get_post_reactions_lines(Post.get_web_driver()) lines_count = len(lines) if not in_extended and lines_count > 0: log(u"Lendo reações...") more = str(lines[lines_count - 1].find_element_by_tag_name("a").get_attribute("href")).replace( "limit=10", "limit=1000000") Post.get_web_driver().get(more) Post.__get_reactions(True) if lines_count > 0 and in_extended: log(u"Salvando reações") Post.set_likes_file(Post.__save_path + '/likes.html') file_path = Post.get_likes_file() with codecs.open(file_path, "w", encoding="utf-8") as infile: page_source = Post.get_web_driver().page_source infile.write(page_source) log(u"Reações salvas em: [{0}]".format(file_path))
def _check_github_for_update(self): """ Uses git commands to check if there is a newer version that the provided commit hash. If there is a newer version it sets _num_commits_behind. """ self._newest_commit_hash = None self._num_commits_behind = 0 self._num_commits_ahead = 0 # get all new info from github output, err, exit_status = self._run_git(self._git_path, 'fetch origin') if not exit_status == 0: logger.log(u"Unable to contact github, can't check for update", logger.ERROR) return # get latest commit_hash from remote output, err, exit_status = self._run_git(self._git_path, 'rev-parse --verify --quiet "@{upstream}"') if exit_status == 0 and output: cur_commit_hash = output.strip() if not re.match('^[a-z0-9]+$', cur_commit_hash): logger.log(u"Output doesn't look like a hash, not using it", logger.DEBUG) return else: self._newest_commit_hash = cur_commit_hash else: logger.log(u"git didn't return newest commit hash", logger.DEBUG) return # get number of commits behind and ahead (option --count not supported git < 1.7.2) output, err, exit_status = self._run_git(self._git_path, 'rev-list --left-right "@{upstream}"...HEAD') if exit_status == 0 and output: try: self._num_commits_behind = int(output.count("<")) self._num_commits_ahead = int(output.count(">")) except: logger.log(u"git didn't return numbers for behind and ahead, not using it", logger.DEBUG) return logger.log(u"cur_commit = {current} % (newest_commit)= {new}, " u"num_commits_behind = {x}, num_commits_ahead = {y}".format (current=self._cur_commit_hash, new=self._newest_commit_hash, x=self._num_commits_behind, y=self._num_commits_ahead), logger.DEBUG)
def _process_upgrade(connection, upgrade_class): instance = upgrade_class(connection) logger.log( u'Checking {name} database upgrade'.format( name=pretty_name(upgrade_class.__name__)), logger.DEBUG) if not instance.test(): logger.log( u'Database upgrade required: {name}'.format( name=pretty_name(upgrade_class.__name__)), logger.MESSAGE) try: instance.execute() except sqlite3.DatabaseError as error: print(u'Error in {name}: {msg}'.format(name=upgrade_class.__name__, msg=error)) raise logger.log( u'{name} upgrade completed'.format(name=upgrade_class.__name__), logger.DEBUG) else: logger.log( u'{name} upgrade not required'.format(name=upgrade_class.__name__), logger.DEBUG) for upgradeSubClass in upgrade_class.__subclasses__(): _process_upgrade(connection, upgradeSubClass)
def need_update(self): self._find_installed_version() try: self._check_github_for_update() except Exception as error: logger.log(u"Unable to contact github, can't check for update: {msg!r}".format(msg=error), logger.ERROR) return False if not self._cur_commit_hash or self._num_commits_behind > 0: return True return False
def train(self): Logger.log("Train on " + self._captcha) processed = self.getImage() processed = self.cleanImage(processed) # Ensure that image is in black and white processed = self.blackAndWhite(processed) # List of images characters = self.getCharacters(processed) # Save characters on hard disc self.saveCharacters(characters) return self.isTrainingSuccessful(characters)
def set_newest_text(self): if self._num_commits_ahead: logger.log( u"Local branch is ahead of " + self.branch + ". Automatic update not possible.", logger.ERROR) elif self._num_commits_behind > 0: newest_text = 'There is a newer version available ' newest_text += " (you're " + str( self._num_commits_behind) + " commit" if self._num_commits_behind > 1: newest_text += 's' newest_text += ' behind)' logger.log(newest_text, logger.MESSAGE) else: return
def set_newest_text(self): # if we're up to date then don't set this core.NEWEST_VERSION_STRING = None if not self._cur_commit_hash: logger.log(u"Unknown current version number, don't know if we should update or not", logger.ERROR) elif self._num_commits_behind > 0: newest_text = 'There is a newer version available' newest_text += " (you're " + str(self._num_commits_behind) + " commit" if self._num_commits_behind > 1: newest_text += "s" newest_text += " behind)" logger.log(newest_text, logger.MESSAGE) else: return
def need_update(self): if not self._find_installed_version(): logger.error("Unable to determine installed version via git, please check your logs!") return False if not self._cur_commit_hash: return True else: try: self._check_github_for_update() except Exception, e: logger.log(u"Unable to contact github, can't check for update: " + repr(e), logger.ERROR) return False if self._num_commits_behind > 0: return True
def restart(): install_type = versionCheck.CheckVersion().install_type status = 0 popen_list = [] if install_type in ('git', 'source'): popen_list = [sys.executable, APP_FILENAME] if popen_list: popen_list += SYS_ARGV logger.log(u"Restarting nzbToMedia with {args}".format(args=popen_list)) logger.close() p = subprocess.Popen(popen_list, cwd=os.getcwd()) p.wait() status = p.returncode os._exit(status)
def _find_installed_version(self): version_file = os.path.join(core.PROGRAM_DIR, u'version.txt') if not os.path.isfile(version_file): self._cur_commit_hash = None return try: with open(version_file, 'r') as fp: self._cur_commit_hash = fp.read().strip(' \n\r') except EnvironmentError as error: logger.log(u"Unable to open 'version.txt': {msg}".format(msg=error), logger.DEBUG) if not self._cur_commit_hash: self._cur_commit_hash = None else: core.NZBTOMEDIA_VERSION = self._cur_commit_hash
def need_update(self): if not self._find_installed_version(): logger.error('Unable to determine installed version via git, please check your logs!') return False if not self._cur_commit_hash: return True else: try: self._check_github_for_update() except Exception as error: logger.log(u'Unable to contact github, can\'t check for update: {msg!r}'.format(msg=error), logger.ERROR) return False if self._num_commits_behind > 0: return True return False
def _check_github_for_update(self): """ Uses pygithub to ask github if there is a newer version that the provided commit hash. If there is a newer version it sets Sick Beard's version text. commit_hash: hash that we're checking against """ self._num_commits_behind = 0 self._newest_commit_hash = None gh = github.GitHub(self.github_repo_user, self.github_repo, self.branch) # try to get newest commit hash and commits behind directly by comparing branch and current commit if self._cur_commit_hash: branch_compared = gh.compare(base=self.branch, head=self._cur_commit_hash) if 'base_commit' in branch_compared: self._newest_commit_hash = branch_compared['base_commit']['sha'] if 'behind_by' in branch_compared: self._num_commits_behind = int(branch_compared['behind_by']) # fall back and iterate over last 100 (items per page in gh_api) commits if not self._newest_commit_hash: for curCommit in gh.commits(): if not self._newest_commit_hash: self._newest_commit_hash = curCommit['sha'] if not self._cur_commit_hash: break if curCommit['sha'] == self._cur_commit_hash: break # when _cur_commit_hash doesn't match anything _num_commits_behind == 100 self._num_commits_behind += 1 logger.log(u"cur_commit = {current} % (newest_commit)= {new}, num_commits_behind = {x}".format (current=self._cur_commit_hash, new=self._newest_commit_hash, x=self._num_commits_behind), logger.DEBUG)
def initDirs(self): options = self.options if not os.path.exists(options.output_dir): os.makedirs(options.output_dir) if not os.path.exists(options.output_char_dir): os.makedirs(options.output_char_dir) # Clean output dir from previous preview files for the_file in os.listdir(options.output_dir): if re.match("^[0-9]\-", the_file): file_path = os.path.join(options.output_dir, the_file) try: if os.path.isfile(file_path): os.unlink(file_path) Logger.log("Removed preview file {}".format(the_file)) except Exception, e: print e
def _find_installed_version(self): """ Attempts to find the currently installed version of Sick Beard. Uses git show to get commit version. Returns: True for success or False for failure """ output, err, exit_status = self._run_git(self._git_path, 'rev-parse HEAD') # @UnusedVariable if exit_status == 0 and output: cur_commit_hash = output.strip() if not re.match('^[a-z0-9]+$', cur_commit_hash): logger.log(u"Output doesn't look like a hash, not using it", logger.ERROR) return False self._cur_commit_hash = cur_commit_hash if self._cur_commit_hash: core.NZBTOMEDIA_VERSION = self._cur_commit_hash return True else: return False
def _processUpgrade(connection, upgradeClass): instance = upgradeClass(connection) logger.log(u"Checking {name} database upgrade".format(name=prettyName(upgradeClass.__name__)), logger.DEBUG) if not instance.test(): logger.log(u"Database upgrade required: {name}".format(name=prettyName(upgradeClass.__name__)), logger.MESSAGE) try: instance.execute() except sqlite3.DatabaseError as error: print(u"Error in {name}: {msg}".format(name=upgradeClass.__name__, msg=error)) raise logger.log(u"{name} upgrade completed".format(name=upgradeClass.__name__), logger.DEBUG) else: logger.log(u"{name} upgrade not required".format(name=upgradeClass.__name__), logger.DEBUG) for upgradeSubClass in upgradeClass.__subclasses__(): _processUpgrade(connection, upgradeSubClass)
def fetch(self, query, args=None): if query is None: return sqlResult = None attempt = 0 while attempt < 5: try: if args is None: logger.log("{name}: {query}".format(name=self.filename, query=query), logger.DB) cursor = self.connection.cursor() cursor.execute(query) sqlResult = cursor.fetchone()[0] else: logger.log( "{name}: {query} with args {args}".format(name=self.filename, query=query, args=args), logger.DB ) cursor = self.connection.cursor() cursor.execute(query, args) sqlResult = cursor.fetchone()[0] # get out of the connection attempt loop since we were successful break except sqlite3.OperationalError as error: if "unable to open database file" in error.args[0] or "database is locked" in error.args[0]: logger.log(u"DB error: {msg}".format(msg=error), logger.WARNING) attempt += 1 time.sleep(1) else: logger.log(u"DB error: {msg}".format(msg=error), logger.ERROR) raise except sqlite3.DatabaseError as error: logger.log(u"Fatal error executing query: {msg}".format(msg=error), logger.ERROR) raise return sqlResult
def main(): while True: if isKilled: break try: b_id, host, realm, last_crawl, last_post, succeed = database.get_meta() if not succeed: logger.log("Getting blog entity is failed") return try: article_list = get_article_list(host, realm, last_post) except Exception, e: logger.log("##ERROR:get_list", host, e.message) database.flag_rollback(b_id) continue logger.log(host, " [", len(article_list), "]") success_count = 0 for article in article_list: try: data = get_article(article, realm) except Exception, e: logger.log("##ERROR:get_article", article, e.message) continue if len(data) == 0: continue if database.save_article(b_id, data): success_count += 1 logger.log(success_count, "accepted") database.flag(b_id, 0)
def _processUpgrade(connection, upgradeClass): instance = upgradeClass(connection) logger.log(u"Checking " + prettyName(upgradeClass.__name__) + " database upgrade", logger.DEBUG) if not instance.test(): logger.log(u"Database upgrade required: " + prettyName(upgradeClass.__name__), logger.MESSAGE) try: instance.execute() except sqlite3.DatabaseError, e: print "Error in " + str(upgradeClass.__name__) + ": " + str(e) raise logger.log(upgradeClass.__name__ + " upgrade completed", logger.DEBUG)