def run(self):
""" create new socket for communicate with client, receive header data contain request information, process it and
send accept, then listen for receive body data
"""
self.tcpSocket = QtNetwork.QTcpSocket()
if not self.tcpSocket.setSocketDescriptor(self.socketDescriptor):
raise Exception(self.tcpSocket.errorString())
self.tcpSocket.waitForReadyRead()
buffer = self.tcpSocket.readAll()
try:
try:
self.size, self.priority, self.timeout, self.route, self.module_version, self.call_back, self.token = self.extract(
buffer)
except ValueError:
self.size, self.priority, self.timeout, self.route, self.call_back, self.token = self.extract(buffer)
self.call_back = base64.b64decode(self.call_back).decode()
self.token = utils.decrypt_rsa(self.token, utils.get_rsa())
self.route_name = self.route_to_name(self.route)
ApiLogging.info('route is: ' + self.route_name)
if not self.module_version:
self.module_version = 'v_1_0'
except Exception as e:
print("rejected", e)
self.tcpSocket.write('{"status":"reject"}'.encode())
self.__close()
else:
self.size = int(self.size)
self.tcpSocket.write('{"status":"accept"}'.encode())
self.tcpSocket.waitForBytesWritten()
self.tcpSocket.readyRead.connect(self.__read_data)
self.tcpSocket.waitForDisconnected(
30 * 60 * 1000) # FIXME: change to better solution for big data timeout and handle incompleted data
def put(self, item):
"""
receive item and serialized it, then put on queue
:param item: the item that should be put on queue
:return: return True if put item on queue successfully
"""
try:
serialized_item = dill.dumps(item)
except PicklingError:
raise SerializedError('can not serialized request')
attempt = 0
# attempt two time to put serialized item on queue
while attempt < 2:
try:
# use direct exchange to put on queue (address queue by name)
self.channel.basic_publish(
exchange='',
routing_key=self.name,
body=serialized_item,
properties=pika.BasicProperties(
delivery_mode=2, # make message persistent
))
ApiLogging.info('add to queue')
return True
# if something on connection goes wrong, try again
except pika.exceptions.AMQPConnectionError:
self.connection, self.channel = self.__reconnect()
attempt += 1
continue
# after two attempt, raise error
raise RabbitmqConnectionError('can not access to MQ')
def download_by_process_id(data):
params = Command.__get_data(data)
base_data_path = BASE_APP_PATH + '/modules/storage'
if params:
try:
full_data_path = base_data_path + '/' + params['process_id']
ApiLogging.info("download path: " + str(full_data_path))
if not isdir(full_data_path):
return {'data': 'process_id Not Found', 'status': 'error'}
shutil.make_archive(full_data_path, 'zip', full_data_path)
with open(base_data_path + '/' + params['process_id'] + '.zip',
"rb") as f:
bytes = f.read()
encoded = base64.b64encode(bytes)
return {'data': encoded, 'status': 'success'}
except Exception as e:
print(e)
return {
'data': 'internal exception',
'status': constants.STATUS_ERROR
}
return {
'data': 'process_id format is wrong',
'status': constants.STATUS_ERROR
}
def set_attribute(self, attr, value=True):
""" Sets a custom attribute for our Webkit instance. Possible attributes are (deprecated) :
* ``auto_load_images``
* ``dns_prefetch_enabled``
* ``plugins_enabled``
* ``private_browsing_enabled``
* ``javascript_can_open_windows``
* ``javascript_can_access_clipboard``
* ``offline_storage_database_enabled``
* ``offline_web_application_cache_enabled``
* ``local_storage_enabled``
* ``local_storage_database_enabled``
* ``local_content_can_access_remote_urls``
* ``local_content_can_access_file_urls``
* ``accelerated_compositing_enabled``
* ``site_specific_quirks_enabled``
For all those options, ``value`` must be a boolean. You can find more
information about these options `in the QT docs
<http://developer.qt.nokia.com/doc/qt-4.8/qwebsettings.html#WebAttribute-enum>`_.
"""
# value = "true" if value else "false"
# self.conn.issue_command("SetAttribute",
# self._normalize_attr(attr),
# value)
ApiLogging.info("%s method is deprecated" % self._normalize_attr(attr))
def download(data):
"""
:param bytearray data: contain file path
:return: file content if exists
:rtype: dict
"""
params = Command.__get_data(data)
if params:
try:
bp = os.path.realpath('.')
path = bp + '/' + params
ApiLogging.info("download path: " + str(path))
is_exists = os.path.exists(path)
if is_exists:
file = open(path, 'r')
raw_data = file.read()
file.close()
else:
return {'data': 'File Not Found', 'status': 'error'}
return {'data': raw_data, 'status': 'success'}
except:
return {
'data': 'internal exception',
'status': constants.STATUS_ERROR
}
return {
'data': 'path format is wrong',
'status': constants.STATUS_ERROR
}
def stop(process_name=None):
if process_name and __get_process(process_name) is not None:
process = __get_process(process_name)
pids = find_pid(process.get('token'))
if not pids:
ApiLogging.warning(process_name + ' is not running!', True)
else:
for pid in pids:
ApiLogging.info(process_name + ' stopped successful!', True)
proc = psutil.Process(pid)
if proc.is_running():
proc.kill()
else:
for process in constants.APP_PROCESSES:
pids = find_pid(process.get('token'))
if not pids:
ApiLogging.warning(
process.get('name') + ' is not running!', True)
else:
for pid in pids:
ApiLogging.info(
process.get('name') + ' stopped successful!', True)
proc = psutil.Process(pid)
if proc.is_running():
proc.kill()
try:
__cleanup()
except Exception as e:
# print("cleanup exception: ", e)
pass
def __run(process_name, action_name):
process = __get_process(process_name)
if process:
ApiLogging.info(process.get('name') + ' running successful!', True)
with open(process.get('log'), 'a+') as err:
subprocess.Popen(['python3', process.get('path') + '/' + process.get('name'), process.get('token')],
close_fds=True, stderr=err,bufsize=1, preexec_fn=__set_limit)
def status():
for process in constants.APP_PROCESSES:
pids = find_pid(process.get('token'))
if not pids:
ApiLogging.error([process.get('name'), pids], True)
else:
ApiLogging.info([process.get('name'), pids], True)
def check_result(self, result):
if result.get_text() == 'Verify code error!':
return VERIFY
elif 'payment' in result.get_text():
ApiLogging.info('found payment')
return PAYMENT
elif 'Not Found' in result.get_text():
return NOT_FOUND
else:
return None
def __init__(self):
if not FetchResult.TLDS:
path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
ApiLogging.info("open TLDS file for read...")
file = open(path + '/vendor/data/tlds.txt')
lines = file.readlines()
for line in lines:
linee = line.strip()
if len(linee) < 1:
continue
FetchResult.TLDS.append(linee)
file.close()
self.internettlds = FetchResult.TLDS
def __init__(self):
# define shared operations between upgrade and downgrade
ApiLogging.info('update to version 1.0')
# define migration and models dir
path = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
migration_path = path + '/migrations/v_1_0'
model_path = path + '/models'
self.manager = DatabaseManager(Database.db,
directory=path + '/migrations/v_1_0')
self.mg = Migrate()
self.mg.create_migration(self.manager,
model_path,
migration_path,
ignore=['base_model'])
def send_signal(process_names):
try:
for name in process_names:
pids = []
for process_name in constants.APP_PROCESSES:
if process_name.get('name') == name:
pids = find_pid(process_name.get('token'))
if len(pids) > 1:
ApiLogging.warning('Too many ' + str(name) +
' process running')
elif len(pids) == 1:
p = psutil.Process(pids[0])
ApiLogging.info('process name: ' + str(pids[0]))
p.send_signal(signal.SIGUSR1)
except Exception as e:
ApiLogging.critical('broadcast signal exception: ' + str(e))
def fill_captcha_if_needed():
captcha_field = robot.find_by_css('#ctl00_ContentPlaceHolder1_TextBoxCode')
if captcha_field is not None:
ApiLogging.info ('captcha needed')
robot.set_viewport_size(1280,800)
img = robot.find_by_css("#Image1")
rect = img.get_position()
box = (int(rect['left']), int(rect['top']), int(rect['right']), int(rect['bottom']))
filename = tempfile.mktemp('.png')
robot.save_as_png(filename, 1280, 800)
image = Image.open(filename)
os.unlink(filename)
captcha_image = image.crop(box)
captcha_image.save('%s.png' % unique_time, 'png')
captcha_field.set_value(resolve_captcha('%s.png' % unique_time))
os.remove('%s.png' % unique_time)
def __load_module(self):
"""
load module if not loaded
:return: module instance
:rtype: BaseModule
"""
module_path = "modules." + self.task_model.route + '.' + self.task_model.module_version + ".module"
try:
ApiLogging.info("import")
app_module = importlib.import_module(module_path)
return app_module.Module(self.task_model)
except Exception as e:
# TODO: save exception same as run method exception part
ApiLogging.error("import exception " + str(e))
return None
def __init__(self, parent=None):
self.parent = parent # type: BaseModule
if not SimilarDomain.TLDS:
ApiLogging.info("open TLDS file for read...")
path = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
file = open(path + '/vendor/data/tlds.txt')
lines = file.readlines()
for line in lines:
linee = line.strip()
if len(linee) < 1:
continue
SimilarDomain.TLDS.append(linee)
file.close()
self.internettlds = SimilarDomain.TLDS
self.results = list()
def _get_public_suffix_list():
"""Return a set containing all Public Suffixes.
If the env variable PUBLIC_SUFFIX_LIST does not point to a local copy of the
public suffix list it is downloaded into memory each time urltools is
imported.
"""
global PSL
if not PSL:
ApiLogging.info('open PLS ...')
with codecs.open(
os.path.realpath(os.path.dirname(__file__) + '/data/tlds.txt'),
'r', 'utf-8') as f:
psl_raw = f.readlines()
psl = set()
for line in psl_raw:
item = line.strip()
if item != '' and not item.startswith('//'):
psl.add(item)
PSL = psl
def decode(hash_type,hash_code):
if login():
hash_field = robot.find_by_css('#ctl00_ContentPlaceHolder1_TextBoxInput')
if hash_field is not None:
type_field = robot.find_by_css('#ctl00_ContentPlaceHolder1_InputHashType')
ApiLogging.info('hash: ' + str(hash) +' type: ' + str(type) + ' hashcode: '+ hash_code )
return None
hash_field.set_value(hash)
type_field.set_value(type)
fill_captcha_if_needed()
submit_button = robot.find_by_css("#ctl00_ContentPlaceHolder1_Button1")
submit_button.click()
result = robot.find_by_css('#ctl00_ContentPlaceHolder1_LabelAnswer')
ApiLogging.info ("result: %s"%result.get_text().split('\n')[0])
if result.get_text() == 'Verify code error!':
decode(hash_type,hash_code)
elif 'payment' in result.get_text():
pr = robot.find_by_contain_text('a', 'Purchase')
if pr:
pr.click()
result = robot.find_by_css('#ctl00_ContentPlaceHolder1_LabelAnswer')
ApiLogging.info("result: %s" % result.get_text().split('\n')[0])
elif 'Not Found' in result.get_text():
ApiLogging.warning('Not Found')
else:
log(result.get_text().split('\n')[0])
else:
ApiLogging.warning ('login fail')
def put(self, item, tag, priority=None):
"""
receive RequestObject as item and put in PGQ
:param item: the item that should be put on queue
:param tag: use tag as previewing inside of each item, it must be unique
:param priority: use priority for ordering item
:return: return True if put item on queue successfully
"""
try:
model = QueueModel()
model.name = self.name
model.data = item
model.tag = tag
if priority:
model.priority = priority
model.save()
ApiLogging.info('add to queue')
return True
except IntegrityError:
raise DatabaseError('tag is repetitive')
except Exception:
raise DatabaseError('can not access database')
def run(self):
try:
""" run module and save result in database """
ApiLogging.info("module " + str(self.task_model.route))
self.instance_module.prepare()
self.instance_module.run()
self.task_model.status = constants.STATUS_SUCCESS
if not self.instance_module.result:
error = {'code': ResultNotSetError.get_code(), 'message': 'module not set any result before return!'}
self.task_model.response_data = {
'data': to_json(self.instance_module.result),
'error': to_json(error),
'status': constants.STATUS_ERROR,
'token': self.task_model.token,
'process_id': self.task_model.process_id
}
except Exception as e:
LogHandler.save(sys.exc_info(), process_id=self.task_model.process_id)
ApiLogging.error('result exception ' + str(e))
self.task_model.status = constants.STATUS_FAILED
old_exception = e.__str__()
try:
error = {'code': e.get_code(), 'message': e.args[0]}
except Exception as e:
error = {'code': 0, 'message': old_exception}
self.task_model.response_data = {
'data': to_json(self.instance_module.result),
'error': to_json(error),
'status': constants.STATUS_ERROR,
'token': self.task_model.token,
'process_id': self.task_model.process_id
}
finally:
self.task_model.save()
ApiLogging.info('emit finish signal')
self.instance_module.task_finished.emit(self.task_model.process_id)
def run(self):
ApiLogging.info('Start Consuming From ' + str(self.queue_name))
while 1:
# # check if there is new request on queue
status = self.channel.queue_declare(queue=self.queue_name, durable=True)
if int(status.method.message_count) == 0:
continue
else:
ApiLogging.info(status.method.message_count)
# check if still we're able to process request, then get new message
if int(self.process_limit) == 3:
(_, _, message) = self.__get_message()
ApiLogging.info(message)
def login():
if is_logeed_in():
ApiLogging.info('cookie login')
return True
else:
ApiLogging.info ('captcha login')
robot.go_to('/login.aspx')
email_field = robot.find_by_css('#ctl00_ContentPlaceHolder1_TextBoxCmd5_E')
password_field = robot.find_by_css('#ctl00_ContentPlaceHolder1_TextBoxCmd5_P')
email_field.set_value(email)
password_field.set_value(password)
fill_captcha_if_needed()
submit_button = robot.find_by_css("#ctl00_ContentPlaceHolder1_Button1")
submit_button.click()
robot.save_cookies_to_file(robot.get_cookies())
if is_logeed_in():
ApiLogging.info ('logged in')
return True
return False
def decode(self, hash_type, hash_code):
if self.login():
hash_field = self.robot.find_by_css(
'#ctl00_ContentPlaceHolder1_TextBoxInput')
if hash_field is not None:
type_field = self.robot.find_by_css(
'#ctl00_ContentPlaceHolder1_InputHashType')
hash_field.set_value(hash_code)
type_field.set_value(hash_type)
self.fill_captcha_if_needed()
submit_button = self.robot.find_by_css(
"#ctl00_ContentPlaceHolder1_Button1")
submit_button.click()
result = self.robot.find_by_css(
'#ctl00_ContentPlaceHolder1_LabelAnswer')
ApiLogging.info("result in hash: %s" % result.get_text())
ApiLogging.info('type: ' + str(hash_type) + ' code: ' +
str(hash_code))
chk_result = self.check_result(result)
if chk_result == VERIFY:
self.decode(hash_type, hash_code)
elif chk_result == PAYMENT:
pr = self.robot.find_by_contain_text('a', 'Purchase')
ApiLogging.info('click payment' + str(pr.get_text()))
if pr:
pr.click()
result = self.robot.find_by_css(
'#ctl00_ContentPlaceHolder1_LabelAnswer')
chk_result = self.check_result(result)
if chk_result is None:
return result.get_text()
elif chk_result == NOT_FOUND:
return None
else:
return result.get_text().split('\n')[0]
else:
ApiLogging.warning('login fail')
import os, sys
import subprocess
from components.utils import ApiLogging
ps = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.append(ps)
from core import constants
ApiLogging.info('base path', constants.BASE_APP_PATH)
ApiLogging.info('path: ',
os.path.dirname(os.path.abspath(os.path.dirname(__file__))))
full_command = "./receive.py"
subprocess.Popen(['/usr/bin/python3', full_command], close_fds=True)
current_version = 'v_' + str(database_version).replace('.', '_')
for dirc in os.listdir(update_path):
if dirc > current_version:
updates.append(dirc)
updates = sorted(updates)
for update in updates:
# create object from Update class and run upgrade
import_file = importlib.import_module('updates.' + update + '.update')
up = import_file.Update()
temp_version = update.replace('v_', '')
# convert dir name format to database record format
database_new_version = temp_version.replace('_', '.')
up.upgrade()
Setting.put('api_version', database_new_version, force=True)
elif float(database_version) > float(current_version):
# Todo: downgrade
pass
else: # database version is equal to system current version
ApiLogging.info('system is updated', True)
if patch:
try:
import_file = importlib.import_module('updates.patch.patch_' + num)
up = import_file.Update()
up.upgrade()
ApiLogging.info('patch ' + str(num) + ' applied!', True)
except Exception as e:
ApiLogging.error("patch exception: " + str(e), True)
size = default_size * 1000
if os.path.isfile(file):
if os.path.getsize(file) >= size:
with open(file, 'r') as f:
lines = f.readlines()
if len(lines) < remove_lines:
ApiLogging.critical(
"the specific number of lines is greater than number of desired {0} file lines".format(file))
with open(file, 'w') as f:
f.writelines(lines[remove_lines:])
else:
ApiLogging.critical("the {0} is not existed".format(file))
if __name__ == '__main__':
if sys.argv.pop(-1) != "0X105070":
try:
ApiLogging.info(type(sys.argv.pop(-1)))
except IndexError:
ApiLogging.critical('missing token parameter')
sys.exit(0)
ApiLogging.critical('wrong token parameter')
sys.exit(0)
# app = QCoreApplication(sys.argv)
while True:
time.sleep(30)
for file in os.listdir(BASE_APP_PATH + '/logs'):
p = ProcessLog()
p.run(BASE_APP_PATH + '/logs/' + file, default_size=DEFAULT_SIZE_LOG, remove_lines=REMOVE_LINES_LOG)
# sys.exit(app.exec_())
def resolve_captcha(self, file):
api_key = "2632143214b9b24e9dc7590396f1dd22"
captcha_object = CaptchaUpload(key=api_key, waittime=3)
captcha = captcha_object.solve(file)
ApiLogging.info('finded capcha: ' + str(captcha))
return captcha
def signal_handler(self, signal, frame):
ApiLogging.info('You pressed Ctrl+C!')
self.name = 'step2'
sleep(5)
ApiLogging.info('cont')
def __task_finished(self, task_id):
ApiLogging.info('task finished')
del self.running_tasks[task_id]
self.done_tasks.append(task_id)
self.send_signal()
def __signal_handler(self, signal, frame):
ApiLogging.info('base process signal received')
# TODO: update limit
self.check_pending_actions()
