def suggest(self, criteria):
# Argument "criteria" is the keywords typed in by user
criteria = criteria.lower()
logger.log(0,
'User wants a suggestion about ' + criteria + ' via API.',
'api_logfile.txt', self.SESSION_ID)
suggest.api(criteria)
def error_except(self):
try:
loop = True
while loop == True:
quit = None
########## LABEL FOR SSF ONLY ##########
ask = input(
CB + FB + FI +
'Do you want to keep Shadow Suite running? (y/n)> ' + CW +
FR)
ask = ask.lower()
if ask == 'y':
loop = False
quit = False
elif ask == 'n':
loop = False
quit = True
else:
loop = True
return quit
except KeyboardInterrupt:
print(error.errorCodes().ERROR0002)
logger.log(2, "SystemExit raised with error code 2.",
'logfile.txt')
sys.exit(2)
def get_meta(): # Set Flag, Get Host, Get Realm, Get Date
global con
global crawler_id
b_id = None
url = None
realm = None
last_crawl = None
last_post = None
succ_flag = False
try:
cursor = con.cursor()
cursor.execute("select * from get_noncrawled_meta( %s )", [crawler_id])
record = cursor.fetchone()
if record:
b_id, url, realm, last_crawl, last_post = record
succ_flag = True
con.commit()
else:
con.rollback()
except Exception, e:
logger.log("##DBERR get meta error :", e.message)
if con:
con.rollback()
def resolve(self, var_id, data_type):
condition = {}
if self._state.is_tracking(var_id):
var = self._state.get_var_ref(var_id)
if var['type'] == 'unknown':
self._state.update_guessed_type(var_id, data_type)
condition = {'type': 'base_var', 'id': var_id}
elif self.is_tracking(var_id):
transform = self._transforms[var_id]
function_name = transform['function']
if function_name in self._resolver:
condition = self._resolver[function_name].process(
data_type, transform['args'])
if 'args' in condition:
for i in range(len(condition['args'])):
if condition['args'][i]['type'] == 'symbolic':
condition['args'][i] = self.resolve(
condition['args'][i]['id'], data_type)
else:
msg = 'Not processing %s (no resolver)' % function_name
logger.log(msg, '', Logger.DEBUG)
raise Exception(msg)
else:
raise Exception('Cannot resolve value for untracked id \'%s\'.' %
var_id)
return condition
def export_conf(self, config_file, config_dict):
# Exports the current configuration to a file.
logger.log(
3, self.current_user + " is exporting settings to " + config_file +
" via API.", 'api_logfile.txt', self.SESSION_ID)
result = misc.programFunctions().export_conf(config_file, config_dict)
return result
def unzip(file_name) -> bool:
"""
Faz o un-zip do arquivo final
Peguei de: https://stackoverflow.com/questions/17913952/monitoring-zip-file-extraction-display-percentage-in-python
:return: bool
"""
with ZipFile(file_name, 'r') as infile:
for member_info in infile.infolist():
filename = member_info.filename
file_size = member_info.file_size
with open("{}".format(filename), 'wb') as outfile:
member_fd = infile.open(filename)
total_bytes = 0
while 1:
x = member_fd.read(Zip.chunk_size)
if not x:
break
total_bytes += outfile.write(x)
log("{0}% extraido".format(100 * total_bytes /
file_size))
return True
def input_on(db_type, log_type):
account = input('\033[1;34m账号:\033[0m')
pwd = input('\033[1;34m密码:\033[0m')
user_data = enroll(account, pwd, db_type)
if user_data != None:
logger.log('登录成功', log_type, user_data['id'])
return user_data
def facebook_login(web_driver: RemoteWebDriver, login: str,
password: str) -> bool:
"""
Funcao para realizar os passos necessarios para o login, e retornar se foi bem sucedida ou nao
:param web_driver: RemoteWebDriver instancia do driver
:param login: str login text-plain
:param password: str senha text-plain
:return: bool login realizado com sucesso?
"""
# Salvamos a instancia
Login.set_web_driver(web_driver)
# Salvamos o login
Login.set_login(login)
# Salvamos a senha
Login.set_password(password)
# Acessamos a pagina de login
Login.__access()
# Informamos o Handler
log('Handler: [{}]'.format(web_driver.current_window_handle))
# Processos para o login
is_logged = Login.__input_login().__input_password().__submit(
).__check_save_device().__is_logged()
if is_logged:
log("O login foi realizado com sucesso!")
return is_logged
def withdraw(acc):
"""
取款
:param acc: 用户信息字典
:return: None
"""
while True:
money = input("请输入提款金额: ")
if money.isdigit():
if acc['balance'] >= int(money):
fee = 0.05 * int(money) # 手续费5%
if fee < 1: # 不足一元按一元收手续费
fee = 1
acc['balance'] -= int(money)
save_check(acc, WITHDRAW, money, acc['balance'])
save_db(acc)
print("\033[32;1m成功提现%d元(手续费%d元),当前余额为%s元\033[0m" %
((int(money) - fee), fee, acc['balance']))
logger.log("transactions").info(
log_msg % (acc['id'], 'withdraw', int(money)))
return
else:
print("\033[31;1m余额不足\033[0m")
logger.log("transactions").error(
log_msg % (acc['id'], 'withdraw', int(money)) +
" error: balance not enough")
else:
print("\033[31;1m请输入数字\033[0m")
def action(self, query, args=None):
if query is None:
return
sql_result = None
attempt = 0
while attempt < 5:
try:
if args is None:
logger.log(u'{name}: {query}'.format(name=self.filename, query=query), logger.DB)
sql_result = self.connection.execute(query)
else:
logger.log(u'{name}: {query} with args {args}'.format
(name=self.filename, query=query, args=args), logger.DB)
sql_result = self.connection.execute(query, args)
self.connection.commit()
# get out of the connection attempt loop since we were successful
break
except sqlite3.OperationalError as error:
if 'unable to open database file' in error.args[0] or 'database is locked' in error.args[0]:
logger.log(u'DB error: {msg}'.format(msg=error), logger.WARNING)
attempt += 1
time.sleep(1)
else:
logger.log(u'DB error: {msg}'.format(msg=error), logger.ERROR)
raise
except sqlite3.DatabaseError as error:
logger.log(u'Fatal error executing query: {msg}'.format(msg=error), logger.ERROR)
raise
return sql_result
def run(self):
options = self.options
# List mods
if options.listmods:
Logger.header("List of mods", True)
mods_dir = "core/mods/"
for dir in os.listdir(mods_dir):
if os.path.isdir(os.path.join(mods_dir, dir)):
Logger.log(dir, True)
sys.exit(2)
# Train
if options.train:
captcha = options.train
self.train(captcha)
# Crack
if options.crack:
captcha = options.crack
self.crack(captcha)
# Batch train with percentage
if options.batch_train:
self.batch_train(options.batch_train)
# Batch crack with percentage
if options.batch_crack:
self.batch_crack(options.batch_crack)
def __get_base_data():
"""
Obtem dados base do Post
:return: Post
"""
driver = Post.get_web_driver()
log(u"Obtendo dados base do post.")
element = PostElements.get_post_data(driver)
data_ft = ""
if element:
data_ft = element.get_attribute('data-ft')
Post.set_data_ft_file(Post.__save_path + '/data-ft.json')
if data_ft:
with codecs.open(Post.get_data_ft_file(), mode='w', encoding="utf-8") as f:
f.write(data_ft)
log(u"Dados base salvos em: [{}]".format(Post.get_data_ft_file()))
return Post
def _check_github_for_update(self):
"""
Uses git commands to check if there is a newer version that the provided
commit hash. If there is a newer version it sets _num_commits_behind.
"""
self._newest_commit_hash = None
self._num_commits_behind = 0
self._num_commits_ahead = 0
# get all new info from github
output, err, exit_status = self._run_git(self._git_path,
'fetch origin')
if not exit_status == 0:
logger.log(u"Unable to contact github, can't check for update",
logger.ERROR)
return
# get latest commit_hash from remote
output, err, exit_status = self._run_git(
self._git_path, 'rev-parse --verify --quiet "@{upstream}"')
if exit_status == 0 and output:
cur_commit_hash = output.strip()
if not re.match('^[a-z0-9]+$', cur_commit_hash):
logger.log(u"Output doesn't look like a hash, not using it",
logger.DEBUG)
return
else:
self._newest_commit_hash = cur_commit_hash
else:
logger.log(u"git didn't return newest commit hash", logger.DEBUG)
return
# get number of commits behind and ahead (option --count not supported git < 1.7.2)
output, err, exit_status = self._run_git(
self._git_path, 'rev-list --left-right "@{upstream}"...HEAD')
if exit_status == 0 and output:
try:
self._num_commits_behind = int(output.count("<"))
self._num_commits_ahead = int(output.count(">"))
except:
logger.log(
u"git didn't return numbers for behind and ahead, not using it",
logger.DEBUG)
return
logger.log(
u"cur_commit = {current} % (newest_commit)= {new}, "
u"num_commits_behind = {x}, num_commits_ahead = {y}".format(
current=self._cur_commit_hash,
new=self._newest_commit_hash,
x=self._num_commits_behind,
y=self._num_commits_ahead), logger.DEBUG)
def get_url(self) -> str:
"""
Neste metodo vamos obter a url de download dos drivers
:return: str Url de Download
"""
so = SystemArchiteture.get_operational_system()
bits = SystemArchiteture.get_bits()
if self.__package == 'geckodriver':
# Versao do Gecko
vers = consts.GECKO_DRIVER_VERSION
# Vamos trabalhar apenas com Windows e Linux
plat = 'win' if so == 'WindowsPE' else 'linux'
log(u"Estamoso buscando um driver para seu sistema {0}".format(
plat))
url = FindDriverDownload.get_url(self.__package, vers, plat, bits)
log(u"A URL encontrada foi: [{0}]".format(url))
return url
def save_article(b_id, data):
global con
result = False
try:
cursor = con.cursor()
cursor.execute("SELECT article_id, b_id, post_id from blog_data where b_id = %s and post_id = %s", [b_id, data["post_id"]])
row = cursor.fetchone()
if row is None:
cursor.execute("INSERT INTO blog_data(b_id, post_id, author, title, crawl_date, post_date, contents) values(%s, %s, %s, %s, %s, %s, %s)", [b_id, data["post_id"], data["name"], data["title"], datetime.datetime.now(), data["date"], data["content"]])
con.commit()
result = True
image_list = [[b_id, data["post_id"], url, datetime.datetime.now()] for url in data["images"]]
for image in image_list:
cursor.execute("INSERT INTO blog_image(b_id, post_id, image_url, crawl_date) VALUES(%s, %s, %s, %s)", image)
con.commit()
else:
a_id, b_id, p_id = row
cursor.execute("UPDATE blog_data set contents = %s , crawl_date = %s, post_date = %s where article_id = %s", [data["content"], datetime.datetime.now(), data["date"], a_id])
con.commit()
result = True
except Exception, e:
logger.log("##DBERR save error : %s " % e.message)
con.rollback()
def action(self, query, args=None):
if query == None:
return
sqlResult = None
attempt = 0
while attempt < 5:
try:
if args == None:
logger.log(self.filename + ": " + query, logger.DB)
sqlResult = self.connection.execute(query)
else:
logger.log(
self.filename + ": " + query + " with args " +
str(args), logger.DB)
sqlResult = self.connection.execute(query, args)
self.connection.commit()
# get out of the connection attempt loop since we were successful
break
except sqlite3.OperationalError, e:
if "unable to open database file" in e.args[
0] or "database is locked" in e.args[0]:
logger.log(u"DB error: " + str(e), logger.WARNING)
attempt += 1
time.sleep(1)
else:
logger.log(u"DB error: " + str(e), logger.ERROR)
raise
except sqlite3.DatabaseError, e:
logger.log(u"Fatal error executing query: " + str(e),
logger.ERROR)
raise
def fetch(self, query, args=None):
if query == None:
return
sqlResult = None
attempt = 0
while attempt < 5:
try:
if args == None:
logger.log(self.filename + ": " + query, logger.DB)
cursor = self.connection.cursor()
cursor.execute(query)
sqlResult = cursor.fetchone()[0]
else:
logger.log(self.filename + ": " + query + " with args " + str(args), logger.DB)
cursor = self.connection.cursor()
cursor.execute(query, args)
sqlResult = cursor.fetchone()[0]
# get out of the connection attempt loop since we were successful
break
except sqlite3.OperationalError, e:
if "unable to open database file" in e.args[0] or "database is locked" in e.args[0]:
logger.log(u"DB error: " + str(e), logger.WARNING)
attempt += 1
time.sleep(1)
else:
logger.log(u"DB error: " + str(e), logger.ERROR)
raise
except sqlite3.DatabaseError, e:
logger.log(u"Fatal error executing query: " + str(e), logger.ERROR)
raise
def trans_money(acc, obj_acc):
"""
转账
:param acc: 转账发起用户的信息字典
:param obj_acc: 转账对象的用户字典
:return: None
"""
while True:
money = input("请输入转账金额: ")
if money.isdigit():
if acc['balance'] >= int(money):
acc['balance'] -= int(money)
obj_acc['balance'] += int(money)
save_check(acc, TRANS, money, acc['balance'])
save_check(obj_acc, GET, money, obj_acc['balance'])
save_db(acc)
save_db(obj_acc)
print("\033[32;1m转账成功\033[0m")
logger.log("transactions").info(
log_msg % (acc['id'], 'transfer money', int(money)))
return
else:
print("\033[31;1m余额不足\033[0m")
logger.log("transactions").error(
log_msg % (acc['id'], 'transfer money', int(money)) +
" error: balance not enough")
else:
print("\033[31;1m请输入数字\033[0m")
def solve_rec(self, condition):
op_name = condition['type']
if op_name in self._adapters:
return self._adapters[op_name].process(condition)
else:
logger.log('Not processing %s (no adapter)' % op_name, '',
Logger.DEBUG)
raise Exception('Not processing %s (no adapter)' % op_name)
def need_update(self):
self._find_installed_version()
try:
self._check_github_for_update()
except Exception, e:
logger.log(u"Unable to contact github, can't check for update: " + repr(e), logger.ERROR)
return False
def extract(self):
log(u"Extraindo compartilhamentos...")
from modules.social_media.facebook.packages.facebook_api.post_sharedposts import PostSharedPosts
self.__shared_posts = PostSharedPosts(
access_token=PostShares.get_access_token(),
page_id=PostShares.get_page_id(),
post_id=PostShares.get_post_id())
return self
def uncompress(self):
log("Descompactando...")
import tarfile
if tarfile.is_tarfile(self.__file_name):
tar = tarfile.open(self.__file_name)
tar.extractall()
tar.close()
else:
Zip.unzip(self.__file_name)
def extract(self):
log(u"Extraindo comentários...")
from modules.social_media.facebook.packages.facebook_api.post_comments import PostComments as Comments
self.__comments = Comments(
access_token=PostComments.get_access_token(),
page_id=PostComments.get_page_id(),
post_id=PostComments.get_post_id())
return self
def set_newest_text(self):
if self._num_commits_ahead:
logger.log(u"Local branch is ahead of {branch}. Automatic update not possible.".format
(branch=self.branch), logger.ERROR)
elif self._num_commits_behind:
logger.log(u"There is a newer version available (you're {x} commit{s} behind)".format
(x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE)
else:
return
def set_login(self, login):
"""
Login que sera usado
:param login: str
:return: PostExploit
"""
log('Setado login: {}'.format(login))
self.__login = login
return self
def saveCharacters(self, characters):
characters_hashes = []
# Get characters' hashes
for character in characters:
m = hashlib.md5()
# Get image as a string of 0 and 1 (for 255) and n (for new line)
ones_array = map(lambda lst: map(lambda x: "0" if x == 0 else "1", lst), numpy.array(character))
ones_string = "n".join(map(lambda lst: "".join(lst), ones_array))
m.update(ones_string)
characters_hashes.append(m.hexdigest())
saveAsCategorized = False
if self.options.auto_train:
# Parse file name and find code
basename = os.path.basename(self._captcha)
code = self.getCodeFromString(basename)
if len(code) != len(characters):
saveAsCategorized = False
Logger.error(
"Error! Training found {0} chars while in file name are specified {1} chars. Real code is: {2}".format(
len(characters), len(code), basename
),
1,
)
else:
saveAsCategorized = True
if self.options.verbose:
if saveAsCategorized:
Logger.subheader("Saving characters into categorized folders")
else:
Logger.log("Saving characters into output folder")
i = 0
for character in characters:
character_hash = characters_hashes[i]
if saveAsCategorized:
character_symbol = code[i]
dst = os.path.join(self.options.mod_dir, "char", character_symbol.lower(), character_hash + ".gif")
else:
dst = os.path.join(self.options.output_char_dir, character_hash + ".gif")
character.save(dst)
if saveAsCategorized:
Logger.log("Saving {0} into mod folder".format(character_symbol))
i += 1
def get_reactions_list(self) -> list:
from modules.social_media.facebook.helper.likes_file_reader import LikesFileReader
likes_reader = LikesFileReader(self.get_likes_file())
likes = likes_reader.get_likes()
log(u"{0} reações contabilizadas.".format(len(likes)))
del likes_reader
return likes
def rchmod(path, mod):
logger.log("Changing file mode of {0} to {1}".format(path, oct(mod)))
os.chmod(path, mod)
if not os.path.isdir(path):
return # Skip files
for root, dirs, files in os.walk(path):
for d in dirs:
os.chmod(os.path.join(root, d), mod)
for f in files:
os.chmod(os.path.join(root, f), mod)
def need_update(self):
self._find_installed_version()
try:
self._check_github_for_update()
except Exception, e:
logger.log(
u"Unable to contact github, can't check for update: " +
repr(e), logger.ERROR)
return False
def create_options(self, args=None):
self.options = CaptchonkaOptions().get_options(args)
Logger.header('Options')
for key, value in self.options.__dict__.items():
Logger.log(' ' + key + ': ' + str(value))
project_folder = os.path.abspath(os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir))
self.options.output_dir = os.path.join(project_folder, 'output')
if self.options.mod:
self.options.mod_dir = os.path.join(project_folder, 'core', 'mods', self.options.mod)
def set_newest_text(self):
if self._num_commits_ahead:
logger.log(u"Local branch is ahead of " + self.branch + ". Automatic update not possible.", logger.ERROR)
elif self._num_commits_behind > 0:
newest_text = 'There is a newer version available '
newest_text += " (you're " + str(self._num_commits_behind) + " commit"
if self._num_commits_behind > 1:
newest_text += 's'
newest_text += ' behind)'
logger.log(newest_text, logger.MESSAGE)
else:
return
def __insert_like(reaction: str, username: str, user_url: str):
data = {
# "id": Profile.getId(user_url),
"name": username,
"profile_url": user_url,
"reaction": reaction
}
log("Identificado: ")
log(json.dumps(data, indent=4))
Post.__likes_list.append(data)
def set_password(self, password):
"""
Senha que sera usada
:param password: str
:return: PostExploit
"""
log('Setado senha: {}'.format(password))
self.__password = password
return self
def set_newest_text(self):
# if we're up to date then don't set this
core.NEWEST_VERSION_STRING = None
if not self._cur_commit_hash:
logger.log(u"Unknown current version number, don't know if we should update or not", logger.ERROR)
elif self._num_commits_behind > 0:
logger.log(u"There is a newer version available (you're {x} commit{s} behind)".format
(x=self._num_commits_behind, s=u's' if self._num_commits_behind > 1 else u''), logger.MESSAGE)
else:
return
def find_service(self, service):
# Argument "service" is the target service to view the info.
service_name = self.SERVICES_PATH + service
try:
service_name = service_name.replace('/', '.')
logger.log(
3,
'User shows info about ' + service_name + ' service via API.',
'logfile.txt', global_variables['SESSION_ID'])
service_importlib(service_name, 'info')
except Exception as service_err:
print(error.errorCodes().ERROR0020(str(service_err)))
def set_newest_text(self):
if self._num_commits_ahead:
logger.log(
u"Local branch is ahead of {branch}. Automatic update not possible."
.format(branch=self.branch), logger.ERROR)
elif self._num_commits_behind:
logger.log(
u"There is a newer version available (you're {x} commit{s} behind)"
.format(x=self._num_commits_behind,
s=u's' if self._num_commits_behind > 1 else u''),
logger.MESSAGE)
else:
return
def use_module(self, module):
# Argument "module" is the target module to run.
module = module.lower()
logger.log(
0, 'User used use_module method to use ' + module + ' via API.',
'api_logfile.txt', self.SESSION_ID)
try:
module_path = self.MODULE_PATH.replace('/', '.') + module
module_import = importlib.import_module(module_path)
module_import.main()
except ModuleNotFoundError as modulenotfounderror_msg:
print("[i] " + str(modulenotfounderror_msg))
def flag(b_id, sw):
global con
try:
cursor = con.cursor()
cursor.execute("UPDATE blog_meta set crawler_id = 0, last_crawl = %s where b_id = %s", [datetime.datetime.now(), b_id])
con.commit()
except:
logger.log("##DBERR set flag error")
if con:
con.rollback()
def _find_installed_version(self):
version_file = os.path.join(core.PROGRAM_DIR, u'version.txt')
if not os.path.isfile(version_file):
self._cur_commit_hash = None
return
try:
with open(version_file, 'r') as fp:
self._cur_commit_hash = fp.read().strip(' \n\r')
except EnvironmentError, e:
logger.log(u"Unable to open 'version.txt': " + str(e), logger.DEBUG)
def flag_rollback(b_id):
global con
try:
cursor = con.cursor()
cursor.execute("UPDATE blog_meta set crawler_id = 0 where b_id = %s", [b_id])
con.commit()
except:
logger.log("##DBERR rollback flag error")
if con:
con.rollback()
def _find_installed_version(self):
version_file = os.path.join(core.PROGRAM_DIR, u'version.txt')
if not os.path.isfile(version_file):
self._cur_commit_hash = None
return
try:
with open(version_file, 'r') as fp:
self._cur_commit_hash = fp.read().strip(' \n\r')
except EnvironmentError, e:
logger.log(u"Unable to open 'version.txt': " + str(e),
logger.DEBUG)
def __get_reactions(in_extended: bool = False):
if not in_extended:
log(u"Obtendo reações extendidas...")
Post.get_web_driver().get(PATTERN_PAGE_POST_REACTIONS.format(Post.get_post_id()))
lines = PostElements.get_post_reactions_lines(Post.get_web_driver())
lines_count = len(lines)
if not in_extended and lines_count > 0:
log(u"Lendo reações...")
more = str(lines[lines_count - 1].find_element_by_tag_name("a").get_attribute("href")).replace(
"limit=10", "limit=1000000")
Post.get_web_driver().get(more)
Post.__get_reactions(True)
if lines_count > 0 and in_extended:
log(u"Salvando reações")
Post.set_likes_file(Post.__save_path + '/likes.html')
file_path = Post.get_likes_file()
with codecs.open(file_path, "w", encoding="utf-8") as infile:
page_source = Post.get_web_driver().page_source
infile.write(page_source)
log(u"Reações salvas em: [{0}]".format(file_path))
def _check_github_for_update(self):
"""
Uses git commands to check if there is a newer version that the provided
commit hash. If there is a newer version it sets _num_commits_behind.
"""
self._newest_commit_hash = None
self._num_commits_behind = 0
self._num_commits_ahead = 0
# get all new info from github
output, err, exit_status = self._run_git(self._git_path, 'fetch origin')
if not exit_status == 0:
logger.log(u"Unable to contact github, can't check for update", logger.ERROR)
return
# get latest commit_hash from remote
output, err, exit_status = self._run_git(self._git_path, 'rev-parse --verify --quiet "@{upstream}"')
if exit_status == 0 and output:
cur_commit_hash = output.strip()
if not re.match('^[a-z0-9]+$', cur_commit_hash):
logger.log(u"Output doesn't look like a hash, not using it", logger.DEBUG)
return
else:
self._newest_commit_hash = cur_commit_hash
else:
logger.log(u"git didn't return newest commit hash", logger.DEBUG)
return
# get number of commits behind and ahead (option --count not supported git < 1.7.2)
output, err, exit_status = self._run_git(self._git_path, 'rev-list --left-right "@{upstream}"...HEAD')
if exit_status == 0 and output:
try:
self._num_commits_behind = int(output.count("<"))
self._num_commits_ahead = int(output.count(">"))
except:
logger.log(u"git didn't return numbers for behind and ahead, not using it", logger.DEBUG)
return
logger.log(u"cur_commit = {current} % (newest_commit)= {new}, "
u"num_commits_behind = {x}, num_commits_ahead = {y}".format
(current=self._cur_commit_hash, new=self._newest_commit_hash,
x=self._num_commits_behind, y=self._num_commits_ahead), logger.DEBUG)
def _process_upgrade(connection, upgrade_class):
instance = upgrade_class(connection)
logger.log(
u'Checking {name} database upgrade'.format(
name=pretty_name(upgrade_class.__name__)), logger.DEBUG)
if not instance.test():
logger.log(
u'Database upgrade required: {name}'.format(
name=pretty_name(upgrade_class.__name__)), logger.MESSAGE)
try:
instance.execute()
except sqlite3.DatabaseError as error:
print(u'Error in {name}: {msg}'.format(name=upgrade_class.__name__,
msg=error))
raise
logger.log(
u'{name} upgrade completed'.format(name=upgrade_class.__name__),
logger.DEBUG)
else:
logger.log(
u'{name} upgrade not required'.format(name=upgrade_class.__name__),
logger.DEBUG)
for upgradeSubClass in upgrade_class.__subclasses__():
_process_upgrade(connection, upgradeSubClass)
def need_update(self):
self._find_installed_version()
try:
self._check_github_for_update()
except Exception as error:
logger.log(u"Unable to contact github, can't check for update: {msg!r}".format(msg=error), logger.ERROR)
return False
if not self._cur_commit_hash or self._num_commits_behind > 0:
return True
return False
def train(self):
Logger.log("Train on " + self._captcha)
processed = self.getImage()
processed = self.cleanImage(processed)
# Ensure that image is in black and white
processed = self.blackAndWhite(processed)
# List of images
characters = self.getCharacters(processed)
# Save characters on hard disc
self.saveCharacters(characters)
return self.isTrainingSuccessful(characters)
def set_newest_text(self):
if self._num_commits_ahead:
logger.log(
u"Local branch is ahead of " + self.branch +
". Automatic update not possible.", logger.ERROR)
elif self._num_commits_behind > 0:
newest_text = 'There is a newer version available '
newest_text += " (you're " + str(
self._num_commits_behind) + " commit"
if self._num_commits_behind > 1:
newest_text += 's'
newest_text += ' behind)'
logger.log(newest_text, logger.MESSAGE)
else:
return
def set_newest_text(self):
# if we're up to date then don't set this
core.NEWEST_VERSION_STRING = None
if not self._cur_commit_hash:
logger.log(u"Unknown current version number, don't know if we should update or not", logger.ERROR)
elif self._num_commits_behind > 0:
newest_text = 'There is a newer version available'
newest_text += " (you're " + str(self._num_commits_behind) + " commit"
if self._num_commits_behind > 1:
newest_text += "s"
newest_text += " behind)"
logger.log(newest_text, logger.MESSAGE)
else:
return
def need_update(self):
if not self._find_installed_version():
logger.error("Unable to determine installed version via git, please check your logs!")
return False
if not self._cur_commit_hash:
return True
else:
try:
self._check_github_for_update()
except Exception, e:
logger.log(u"Unable to contact github, can't check for update: " + repr(e), logger.ERROR)
return False
if self._num_commits_behind > 0:
return True
def restart():
install_type = versionCheck.CheckVersion().install_type
status = 0
popen_list = []
if install_type in ('git', 'source'):
popen_list = [sys.executable, APP_FILENAME]
if popen_list:
popen_list += SYS_ARGV
logger.log(u"Restarting nzbToMedia with {args}".format(args=popen_list))
logger.close()
p = subprocess.Popen(popen_list, cwd=os.getcwd())
p.wait()
status = p.returncode
os._exit(status)
def _find_installed_version(self):
version_file = os.path.join(core.PROGRAM_DIR, u'version.txt')
if not os.path.isfile(version_file):
self._cur_commit_hash = None
return
try:
with open(version_file, 'r') as fp:
self._cur_commit_hash = fp.read().strip(' \n\r')
except EnvironmentError as error:
logger.log(u"Unable to open 'version.txt': {msg}".format(msg=error), logger.DEBUG)
if not self._cur_commit_hash:
self._cur_commit_hash = None
else:
core.NZBTOMEDIA_VERSION = self._cur_commit_hash
def need_update(self):
if not self._find_installed_version():
logger.error('Unable to determine installed version via git, please check your logs!')
return False
if not self._cur_commit_hash:
return True
else:
try:
self._check_github_for_update()
except Exception as error:
logger.log(u'Unable to contact github, can\'t check for update: {msg!r}'.format(msg=error), logger.ERROR)
return False
if self._num_commits_behind > 0:
return True
return False
def _check_github_for_update(self):
"""
Uses pygithub to ask github if there is a newer version that the provided
commit hash. If there is a newer version it sets Sick Beard's version text.
commit_hash: hash that we're checking against
"""
self._num_commits_behind = 0
self._newest_commit_hash = None
gh = github.GitHub(self.github_repo_user, self.github_repo, self.branch)
# try to get newest commit hash and commits behind directly by comparing branch and current commit
if self._cur_commit_hash:
branch_compared = gh.compare(base=self.branch, head=self._cur_commit_hash)
if 'base_commit' in branch_compared:
self._newest_commit_hash = branch_compared['base_commit']['sha']
if 'behind_by' in branch_compared:
self._num_commits_behind = int(branch_compared['behind_by'])
# fall back and iterate over last 100 (items per page in gh_api) commits
if not self._newest_commit_hash:
for curCommit in gh.commits():
if not self._newest_commit_hash:
self._newest_commit_hash = curCommit['sha']
if not self._cur_commit_hash:
break
if curCommit['sha'] == self._cur_commit_hash:
break
# when _cur_commit_hash doesn't match anything _num_commits_behind == 100
self._num_commits_behind += 1
logger.log(u"cur_commit = {current} % (newest_commit)= {new}, num_commits_behind = {x}".format
(current=self._cur_commit_hash, new=self._newest_commit_hash, x=self._num_commits_behind), logger.DEBUG)
def initDirs(self):
options = self.options
if not os.path.exists(options.output_dir):
os.makedirs(options.output_dir)
if not os.path.exists(options.output_char_dir):
os.makedirs(options.output_char_dir)
# Clean output dir from previous preview files
for the_file in os.listdir(options.output_dir):
if re.match("^[0-9]\-", the_file):
file_path = os.path.join(options.output_dir, the_file)
try:
if os.path.isfile(file_path):
os.unlink(file_path)
Logger.log("Removed preview file {}".format(the_file))
except Exception, e:
print e
def _find_installed_version(self):
"""
Attempts to find the currently installed version of Sick Beard.
Uses git show to get commit version.
Returns: True for success or False for failure
"""
output, err, exit_status = self._run_git(self._git_path, 'rev-parse HEAD') # @UnusedVariable
if exit_status == 0 and output:
cur_commit_hash = output.strip()
if not re.match('^[a-z0-9]+$', cur_commit_hash):
logger.log(u"Output doesn't look like a hash, not using it", logger.ERROR)
return False
self._cur_commit_hash = cur_commit_hash
if self._cur_commit_hash:
core.NZBTOMEDIA_VERSION = self._cur_commit_hash
return True
else:
return False
def _processUpgrade(connection, upgradeClass):
instance = upgradeClass(connection)
logger.log(u"Checking {name} database upgrade".format(name=prettyName(upgradeClass.__name__)), logger.DEBUG)
if not instance.test():
logger.log(u"Database upgrade required: {name}".format(name=prettyName(upgradeClass.__name__)), logger.MESSAGE)
try:
instance.execute()
except sqlite3.DatabaseError as error:
print(u"Error in {name}: {msg}".format(name=upgradeClass.__name__, msg=error))
raise
logger.log(u"{name} upgrade completed".format(name=upgradeClass.__name__), logger.DEBUG)
else:
logger.log(u"{name} upgrade not required".format(name=upgradeClass.__name__), logger.DEBUG)
for upgradeSubClass in upgradeClass.__subclasses__():
_processUpgrade(connection, upgradeSubClass)
def fetch(self, query, args=None):
if query is None:
return
sqlResult = None
attempt = 0
while attempt < 5:
try:
if args is None:
logger.log("{name}: {query}".format(name=self.filename, query=query), logger.DB)
cursor = self.connection.cursor()
cursor.execute(query)
sqlResult = cursor.fetchone()[0]
else:
logger.log(
"{name}: {query} with args {args}".format(name=self.filename, query=query, args=args), logger.DB
)
cursor = self.connection.cursor()
cursor.execute(query, args)
sqlResult = cursor.fetchone()[0]
# get out of the connection attempt loop since we were successful
break
except sqlite3.OperationalError as error:
if "unable to open database file" in error.args[0] or "database is locked" in error.args[0]:
logger.log(u"DB error: {msg}".format(msg=error), logger.WARNING)
attempt += 1
time.sleep(1)
else:
logger.log(u"DB error: {msg}".format(msg=error), logger.ERROR)
raise
except sqlite3.DatabaseError as error:
logger.log(u"Fatal error executing query: {msg}".format(msg=error), logger.ERROR)
raise
return sqlResult
def main():
while True:
if isKilled:
break
try:
b_id, host, realm, last_crawl, last_post, succeed = database.get_meta()
if not succeed:
logger.log("Getting blog entity is failed")
return
try:
article_list = get_article_list(host, realm, last_post)
except Exception, e:
logger.log("##ERROR:get_list", host, e.message)
database.flag_rollback(b_id)
continue
logger.log(host, " [", len(article_list), "]")
success_count = 0
for article in article_list:
try:
data = get_article(article, realm)
except Exception, e:
logger.log("##ERROR:get_article", article, e.message)
continue
if len(data) == 0:
continue
if database.save_article(b_id, data):
success_count += 1
logger.log(success_count, "accepted")
database.flag(b_id, 0)
def _processUpgrade(connection, upgradeClass):
instance = upgradeClass(connection)
logger.log(u"Checking " + prettyName(upgradeClass.__name__) + " database upgrade", logger.DEBUG)
if not instance.test():
logger.log(u"Database upgrade required: " + prettyName(upgradeClass.__name__), logger.MESSAGE)
try:
instance.execute()
except sqlite3.DatabaseError, e:
print "Error in " + str(upgradeClass.__name__) + ": " + str(e)
raise
logger.log(upgradeClass.__name__ + " upgrade completed", logger.DEBUG)
