def get_result_by_api(api_key, email, hash_code):
url = 'https://www.cmd5.org/api.ashx?email=' + email + '&key=' + api_key + '&hash=' + hash_code
result = Qhttp.get(url)
if result.status_code == 200:
if ':' in result.content.decode():
error_code = result.content.decode().split(':')[-1]
if error_code == '-1':
raise InvalidInputError(' invalid input ')
if error_code == '-2':
raise InsufficientCredit('InsufficientCredit')
if error_code == '-3':
raise NetworkError('server failed on cmd5.org')
if error_code == '-4':
raise InvalidInputError('unknown sipher text')
if error_code == '-7':
raise InvalidInputError('hash type not supported')
if error_code == '-999':
raise NetworkError('some thing wrong with cmd5.org')
try:
return_result = {'results': result.json()}
return return_result
except Exception:
ResultNotFoundError(' unknown result format ')
else:
raise NetworkError(result.status_code)
def profile(self, user_id):
"""
this function show the user's post
:param user_id: user_id or username as string, set for profile request
:return Returns a json of user's account info
"""
if not self.isLoggedIn:
self.login()
self.update_progressbar('request submitted: check profile', 10)
if user_id.isdigit(): # create url base on username or pk received
url = Instagram.profile_url_user_id.format(user_id)
else:
url = Instagram.profile_url_username.format(user_id)
self.update_progressbar('set request to instagram api', 20)
try: # set request to insta
r = self.session.get(url=url,
data=Instagram.generate_signature(
self, data=self.data),
headers=self.headers)
except Exception:
raise NetworkError(' can not access api')
if r.status_code == 200:
result = r.json()
if self.parent:
self.parent.check_point()
self.update_progressbar('parse_result', 80)
result = parse_profile(result, user_id, self.process_id)
return result
else:
if r.status_code == 404:
raise ResultNotFoundError('not found profile')
else:
raise NetworkError(r.status_code)
def get_result(self, search_term, api_key_value, cse_id_value, pages, **kwargs):
"""
:param search_term: query for search
:param api_key_value: google api key
:param cse_id_value: custom search engine id
:param pages: number of google pages wants to search
:param kwargs:
:return: list of result from custom google search api
"""
return_result = {}
# build a google custom search v1
try:
service = build("customsearch", "v1", developerKey=api_key_value,
cache_discovery=False)
except httplib2.ServerNotFoundError:
raise NetworkError('Unable to find the server at www.googleapis.com')
result_list = []
if pages:
self.pages = pages
# iterate on number of excepted result, save one page on each iteration
for i in range(pages):
start_point_of_search_result = (i * 10) + 1
self.parent.check_point()
try:
# dictionary which is produced by google custom search api
result_dict_temp = (
service.cse().list(q=search_term, cx=cse_id_value, num=10, start=start_point_of_search_result,
**kwargs).execute())
# extract items from dictionary and reformat it
if 'items' in result_dict_temp:
temp_parsed_result = self.parse_result(
result_dict_temp['items'])
for element in temp_parsed_result:
result_list.append(element)
# update progressbar value
self.update_progressbar(" Pages has been searched: " + str(i + 1),
(100 * (i + 1) / self.pages))
# invalid api_key or cse_id
except HttpError as e:
if "Invalid Value" in str(e):
raise WrongCseKeyError("wrong cse_id number")
elif "Bad Request" in str(e):
raise WrongApiKeyError("wrong api_key number")
elif 'Forbidden' in str(e):
raise WrongCseKeyError('wrong api_key or wrong cse_key')
else:
raise NetworkError(e)
except Exception as e:
raise NetworkError(e)
# no result found
if len(result_list) == 0:
raise ResultNotFoundError(' can not find result for your query')
return_result['results'] = result_list
return return_result
def __init__(self, email, password, proccess_id, parent=None):
super().__init__()
try:
print('TrueCaller: ', int(QThread.currentThreadId()))
self.email = email
self.password = password
self.proccess_id = proccess_id
# save cookies for preventing repetitive login from one client
cookies_dir = BASE_APP_PATH + "/modules/truecaller/v_1_0/cookies/"
cookie_hashname = str(hashlib.md5(self.email.encode('utf-8')).hexdigest()) + str(
hashlib.md5(self.password.encode('utf-8')).hexdigest()) + '.pkl'
self.cookie_path = cookies_dir + cookie_hashname
os.makedirs(cookies_dir, exist_ok=True)
# profile name
# profile_name = str(hashlib.md5(self.email.encode('utf-8')).hexdigest()) + str(hashlib.md5(self.password.encode('utf-8')).hexdigest())
# self.robot = Hrobot(gui=True, private=True, profile_name=profile_name)
self.robot = Hrobot(gui=True, private=True)
print('before')
self.set_cookie(self.cookie_path)
self.robot.go_to("https://www.truecaller.com")
print('after ')
except Exception as e: # can not go to mbasic.facebook.com
raise NetworkError(str(e))
def get_csrf_token(self):
"""
this function generate csrf_token
:return Returns csrf_token as string
"""
try:
url = self.csrfUrl.format(self.generate_uuid(True))
r = Qhttp.get(url, headers=self.headers)
if json.loads(r.text)['status'] == 'ok':
csrf_token = r.cookies['csrftoken']
return csrf_token
else:
raise NetworkError(r.text)
except Exception as e:
raise NetworkError(e.__str__())
def whois_history(self, domain, api_key):
"""
:param domain: whois_history domain requested
:param api_key: optional, for whoxy.com api
return whois info of domain
"""
if not (validators.domain(domain)):
raise InvalidInputError('invalid domain')
try:
url_request = "http://api.whoxy.com/?key=%s&history=%s" % (api_key,
domain)
with urllib.request.urlopen(url_request) as url:
data = json.loads(url.read().decode())
if 'status_reason' in data:
if data['status_reason'] == 'Incorrect API Key':
raise WrongApiKeyError('wrong or invalid api key')
else:
raise InvalidInputError(data['status_reason'])
elif 'status' in data:
if data['status'] == 0:
raise InternalModuleError(
' something goes wrong in api')
if 'whois_records' not in data:
raise ResultNotFoundError(
'cant find any result for your request')
except URLError:
raise NetworkError('can not access whoxy.com')
if self.parent:
self.parent.check_point()
result = {'results': self.parse_history(data['whois_records'])}
return result
def login(self):
"""
this function login into Instagram account
:return Returns boolean, If the user login correctly
return True also it keeps logged user information
"""
if not self.isLoggedIn:
data = {
'phone_id': self.generate_uuid(False),
'_csrftoken': self.get_csrf_token(),
'username': self.username,
'password': self.password,
'guid': self.generate_uuid(False),
'device_id': self.device_id,
'login_attempt_count': 0
}
json_data = json.dumps(data)
try:
r = self.session.post(url=Instagram.loginUrl,
data=self.generate_signature(json_data),
headers=self.headers)
except Exception as e:
raise NetworkError(e)
result = r.json()
self.login_result = result
if r.status_code == 200:
if 'status' in r.text:
if result['status'] == 'ok':
self.isLoggedIn = True
self.logged_in_user = result["logged_in_user"]
self.token = r.cookies['csrftoken']
self.username_id = result["logged_in_user"]["pk"] # pk
self.rank_token = "{}_{}".format(
self.username_id, self.generate_uuid(False))
return True
else:
if 'message' in result:
raise LoginError(result['message'])
else:
raise LoginError('can not login')
else:
if 'message' in result:
raise LoginError(result['message'])
else:
raise LoginError('can not login')
else:
if 'error_type' in result:
if result['error_type'] == 'rate_limit_error':
raise InstagramApiConstraint(
'rate limit error, Please wait a few minutes before you try again'
)
if 'message' in result:
raise LoginError(result['message'])
else:
raise LoginError('can not login')
else:
return True
def get_whois_by_domain(self, domain):
"""
:param domain: whois domain requested
return whois info of domain
"""
if not (validators.domain(domain)):
raise InvalidInputError('invalid domain')
# use manual whois module for .ir domains
try: # using old version of whois
whois = Whois(parent=self)
parser = Parser()
whois_res, dns = whois.get_whois_with_zone(domain)
except gaierror: # raise when can not access network
raise NetworkError('Unable to find the server')
has_dns = True
if not dns:
has_dns = False
if whois_res == -1:
raise IncorrectDataError('whois server not found for your domain ')
unstructured_res = parser.get_main_srtucture(whois_res,
has_dns=has_dns)
email_res = parser.get_emails(whois_res)
phone_res = parser.get_phones(whois_res)
name_res = parser.get_names(whois_res)
result_list = []
properties_list = []
special_properties_list = []
result = {'results': []}
# create structure data from result
if unstructured_res:
result_list.append(unstructured_res['results'])
special_properties_list = (unstructured_res['special_properties'])
properties_list = (unstructured_res['properties'])
# saving emails, phone, names in list of results
for email in email_res:
if email:
result_list.append(email)
for phone in phone_res:
if phone:
result_list.append(phone)
for name in name_res:
if name:
result_list.append(name)
# if our output is empty, create empty skeleton
if len(special_properties_list) == 0:
result['special_properties'] = ''
else:
result['special_properties'] = special_properties_list
if len(properties_list) == 0:
result['properties'] = ''
else:
result['properties'] = properties_list
result['results'] = result_list
return result
def friends_list(self, user_id, count=None):
"""
:param user_id: string, use as a user_id or screen_name search
:param count: number of result
:return: search result
"""
self.update_progressbar(
'request submitted successfully, request to twitter: friends_list',
20)
try: # set request to twitter api, using different link if use id or screen_name and use count or not
int(user_id)
if count is not None:
if count > 200:
count = 200
timeline_endpoint = "https://api.twitter.com/1.1/friends/list.json?user_id=%s&count=%s" % (
str(user_id), str(count))
else:
timeline_endpoint = "https://api.twitter.com/1.1/friends/list.json?user_id=%s&count=%s" % (
str(user_id), str(self.default_number))
except ValueError:
if count is not None:
if count > 200:
count = 200
timeline_endpoint = "https://api.twitter.com/1.1/friends/list.json?screen_name=%s&count=%s" % (
str(user_id), str(count))
else:
timeline_endpoint = "https://api.twitter.com/1.1/friends/list.json?screen_name=%s&count=%s" % (
str(user_id), str(self.default_number))
try: # set request to twitter api
if self.parent:
self.parent.check_point()
response, data = self.client.request(timeline_endpoint)
except Exception:
raise NetworkError('can not connect to twitter')
self.update_progressbar('received data from api', 40)
if self.parent:
self.parent.check_point()
if 'status' not in response: # somethings goes wrong
raise InvalidInputError("status not found")
if response['status'] != '200': # something goes wrong
data_loads = json.loads(str(bytes.decode(data)))
if data_loads['errors'][-1]['message'] == 'Rate limit exceeded':
raise TwitterApiConstraint(
'Twitter rejected 3 consecutive attempts to access its API under your '
'Twitter account ')
else:
raise InvalidInputError(data_loads['errors'][-1]['message'])
# parse result
data_loads = json.loads(str(bytes.decode(data)))
self.update_progressbar('parse_result', 75)
return_result = Twitter.__parse_search(self, data_loads['users'])[0]
return_result['has_media'] = True
return return_result
def generate_rank_token(self):
"""
this function generate rank token
:return Returns rank token as string
"""
try:
account_id = self.login_result["logged_in_user"]["pk"]
account_uuid = self.generate_uuid(True)
return str(account_id) + '_' + str(account_uuid)
except Exception as e:
raise NetworkError(e.__str__())
def search(self, query, count):
"""
:param query: string, use as a query search
:param count: string, use as a number of search result
:return: search result
"""
self.update_progressbar(
'request submitted successfully, request to twitter: search query',
20)
if count is not None:
if count > 200:
count = 200
timeline_endpoint = "https://api.twitter.com/1.1/users/search.json?q=%s&count=%s" % (
query, str(count))
else:
timeline_endpoint = "https://api.twitter.com/1.1/users/search.json?q=%s&count=%s" % \
(query, str(self.default_number))
try: # set request to twitter api, using different link if use id or screen_name
if self.parent:
self.parent.check_point()
response, data = self.client.request(timeline_endpoint)
except Exception as e: # can not access twitter
raise NetworkError(e)
self.update_progressbar('received data from api', 40)
if self.parent:
self.parent.check_point()
if 'status' not in response: # some thing goes wrong
raise InvalidInputError("status not found")
if response['status'] != '200': # something goes wrong
data_loads = json.loads(str(bytes.decode(data)))
if data_loads['errors'][-1]['message'] == 'Rate limit exceeded':
raise TwitterApiConstraint(
'Twitter rejected 3 consecutive attempts to access its API under your '
'Twitter account ')
else:
raise InvalidInputError(data_loads['errors'][-1]['message'])
data_loads = json.loads(str(bytes.decode(data)))
self.update_progressbar('parse_result', 75)
# parse results
return_result = Twitter.__parse_search(self, data_loads)[0]
return_result['has_mdia'] = True
return return_result
def search(self, query, pages=None):
"""
:param query: this parameter show searching term
:param pages: the number of requested page for result of searching term
:return:list of searching result in form of entity_property
"""
final_result = []
return_result = {}
if pages:
self.page = pages
try:
# open bing.com and set query value
try:
self.robot.go_to('https://www.bing.com')
except InvalidResponseError:
raise NetworkError('Unable to find the server at www.bing.com')
query_field = self.robot.find_by_xpath('//*[@id="sb_form_q"]')
if query_field:
query_field.set_value(query)
search_button = self.robot.find_by_xpath('//*[@id="sb_form_go"]')
search_button.click()
# iterate on number of excepted result, save one page on each iteration
for i in range(self.page):
self.parent.check_point()
result = self.robot.find_by_xpath('/html/body/div[1]')
res = result.find_all_by_css('li[class="b_algo"]')
final_result.extend(self.parse_result(res))
pagination = self.robot.find_by_xpath("//a[@title='Next page']")
if pagination:
pagination.click()
# update progressbar
self.update_progressbar(" Pages has been searched: " + str(i + 1),
(100 * (i + 1) / self.page))
# no result found
if len(final_result) == 0:
return_result["results"] = [
{"data": " ", "properties": [{'title': '', 'type': 0}, {'description': '', 'type': 0}],
"type": 1}]
return return_result
return_result["results"] = final_result
return return_result
finally:
self.robot.cleanup()
def __init__(self, queue_name, default_limit):
super().__init__()
self.running_tasks = {}
self.done_tasks = []
self.default_limit = default_limit
self.queue = PGQ(queue_name)
self.queue_name = queue_name
self.pool = QThreadPool(self)
self.pool.setMaxThreadCount(5000)
signal.signal(signal.SIGUSR1, self.__signal_handler)
self.items = []
try:
__items = TaskModel.select().where((TaskModel.status == constants.STATUS_RUNNING) & (TaskModel.queue_name == self.queue_name))
for __item in __items:
self.items.append(__item)
except OperationalError:
raise NetworkError('Network is unreachable')
def get_ip_info_balance(self):
"""
this function return the get_ip_info balance
:return Returns api balance as a dictionary
"""
api_key = self.api_key
self.update_progressbar('set request to get ip_info balance', 50)
if self.parent:
self.parent.check_point()
try:
r = requests.get('https://api.ip2location.com/?key=' + api_key +
'&check=1')
except (requests.exceptions.RequestException,
requests.exceptions.ConnectionError):
raise NetworkError('can not access api')
if r.text == 'INVALID ACCOUNT':
raise WrongApiKeyError(r.text)
else:
return {'ip_info_balance': r.text, 'type': 0}
def get_whois_by_name(name, api_key):
"""
:param name: whois name requested
:param api_key: optional, for whoxy.com api
return whois info of name
"""
try:
if ' ' in name:
name = name.replace(' ', '+')
url_request = "http://api.whoxy.com/?key=%s&reverse=whois&name=%s" % (
api_key, name)
with urllib.request.urlopen(url_request) as url:
data = json.loads(url.read().decode())
if 'status_reason' in data:
if data['status_reason'] == 'Incorrect API Key':
raise WrongApiKeyError('wrong or invalid api key')
else:
raise InvalidInputError(data['status_reason'])
except URLError:
raise NetworkError('can not access whoxy.com')
result = {'results': parse_reverse(data)}
return result
def get_whois_by_email(email, api_key):
"""
:param email: whois email requested
:param api_key: optional, for whoxy.com api
return whois info of domain
"""
pattern = re.compile("[\w\.-]+@[\w\.-]+\.\w+")
if (pattern.match(email)) == None:
raise InvalidInputError('email format is invalid')
try:
url_request = "http://api.whoxy.com/?key=%s&reverse=whois&email=%s" % (
api_key, email)
with urllib.request.urlopen(url_request) as url:
data = json.loads(url.read().decode())
if 'status_reason' in data:
if data['status_reason'] == 'Incorrect API Key':
raise WrongApiKeyError('wrong or invalid api key')
else:
raise InvalidInputError(data['status_reason'])
except URLError:
raise NetworkError('can not access whoxy.com')
result = {'results': parse_reverse(data)}
return result
def check_balance(api_key):
try:
balance = ''
key = api_key
url_request = "http://api.whoxy.com/?key=%s&account=balance" % (key)
with urllib.request.urlopen(
url_request) as url:
data = json.loads(url.read().decode())
if 'status_reason' in data:
if data['status_reason'] == 'Incorrect API Key':
raise WrongApiKeyError('wrong or invalid api key')
else:
raise InvalidInputError(data['status_reason'])
if 'live_whois_balance' in data:
balance = data['live_whois_balance']
if balance:
return balance
else:
return 'can not get balance'
except URLError:
raise NetworkError('can not access whoxy.com')
def get_ip_is_proxy_balance(self):
"""
this function return the ip_is_proxy balance
:return Returns api balance as a dictionary
"""
api_key = self.api_key
self.update_progressbar('set request to get ip_proxy balance', 50)
if self.parent:
self.parent.check_point()
try:
r = requests.get('https://api.ip2proxy.com/?key=' + api_key +
'&check=1')
except (requests.exceptions.RequestException,
requests.exceptions.ConnectionError):
raise NetworkError(' can not access api')
if 'response' in json.loads(r.text):
if json.loads(r.text)['response'] == "INVALID ACCOUNT":
raise WrongApiKeyError(json.loads(r.text)['response'])
else:
return {
'ip_is_proxy_balance': json.loads(r.text)['response'],
'type': 0
}
def search(self, query, pages=None):
"""
:param query: this parameter show searching term
:param pages:the number of requested page for result of searching term
:return: list of searching result in form of entity_property
"""
result_list = []
return_result = {}
if pages:
self.page = pages
try:
self.parent.check_point()
# open google.com and set query value
try:
self.robot.go_to('https://www.google.com/ncr')
except InvalidResponseError:
raise NetworkError(
'Unable to find the server at www.google.com')
query_field = self.robot.find_by_css("input[name='q']")
if query_field is not None:
query_field.set_value(query)
query_field.get_form().submit()
# iterate on number of excepted result, save one page on each iteration
for i in range(self.page):
self.parent.check_point()
result1 = self.robot.find_by_xpath('//div[@id="search"]')
result = result1.find_all_by_css('div[class="g"]')
pagination = self.robot.find_by_xpath(
"//*[contains(text(), 'Next')]")
result_list.extend(self.parse_result(result))
# update progressbar value
self.update_progressbar(
'Pages has been searched:' + str(i + 1),
(100 * (i + 1) / self.page))
if pagination is not None:
pagination.click()
else:
break
if len(result_list) == 0:
# if the following condition become true, we are faced captcha in search progress
captcha_field1 = self.robot.find_by_xpath(
'/html/body/div[1]/form/input[3]')
captcha_field2 = self.robot.find_by_css(
'#ctl00_ContentPlaceHolder1_TextBoxCode')
if captcha_field1 or captcha_field2:
raise CaptchaNeededError(
"it is needed to resolve captcha")
else:
# no result found
return_result["results"] = [{
"data":
" ",
"properties": [{
'title': '',
'type': 0
}, {
'description': '',
'type': 0
}],
"type":
1
}]
return return_result
return_result['results'] = result_list
return return_result
finally:
self.robot.cleanup()
def get_ip_info(self, ip):
"""
this function get the given ip information
:return Returns ip information in json format
"""
api_key = self.api_key
if self.parent:
self.parent.check_point()
self.update_progressbar('set request to get ip info', 20)
try:
r = requests.get('https://api.ip2location.com/?ip=' + ip +
'&key=' + api_key + '&package=WS24&format=json')
except (requests.exceptions.RequestException,
requests.exceptions.ConnectionError):
raise NetworkError(' can no access api')
if r.status_code == 200:
result = json.loads(r.content.decode())
temp = list()
self.update_progressbar('parsing result', 60)
if self.parent:
self.parent.check_point()
if 'response' in result:
if result['response'] == 'INSUFFICIENT CREDIT':
raise InsufficientCredit(result['response'])
elif result['response'] == 'INVALID ACCOUNT':
raise WrongApiKeyError(result['response'])
else:
raise InternalModuleError(result['response'])
else:
longitude = 'unknown'
latitude = 'unknown'
if result.items():
for k, v in result.items():
# parsing result
if k in [
'country_name', 'region_name', 'city_name',
'isp', 'mobile_brand'
]:
temp.append({
self.get_ip_info_keys[k]: v,
"type": 11
})
elif k in ['domain']:
temp.append({
self.get_ip_info_keys[k]: v,
"type": 12
})
elif k in ['usage_type']:
for each in v.split('/'):
try:
temp.append({
self.get_ip_info_keys[k]:
self.usage_and_proxy_type[each],
"type":
0
})
except KeyError:
pass
elif k in [
'latitude', 'zip_code', 'time_zone',
'idd_code', 'area_code', 'elevation', 'mcc',
'mnc', 'net_speed', 'longitude'
]:
temp.append({k: v, "type": 0})
if k == 'longitude':
longitude = v
if k == 'latitude':
latitude = v
else:
pass # new key is founded
geo_coordinates = str(latitude) + ',' + str(longitude)
geo_record = {
'geo_coordinates': geo_coordinates,
'type': 14
}
final = {
"properties": [],
"special_properties": temp,
"results": [geo_record]
}
return final
else:
raise ResultNotFoundError(
'no result found for your request')
else:
if r.status_code == 404:
raise ResultNotFoundError('no result found for your request')
NetworkError('status code: ', r.status_code)
def get_result(self):
"""
:return: whois info of domain
"""
# build a whois request
try:
key = self.api_key
domain = self.domain
url_request = "http://api.whoxy.com/?key=%s&whois=%s" % (key, domain)
with urllib.request.urlopen(
url_request) as url:
data = json.loads(url.read().decode())
if 'status_reason' in data:
if data['status_reason'] == 'Incorrect API Key':
raise WrongApiKeyError(data['status_reason'])
else:
raise InvalidInputError(data['status_reason'])
except URLError:
raise NetworkError('can not access whoxy.com')
self.parent.check_point()
self.update_progressbar('parsing result', 80)
unstructured_res = WhoisApi.parse_unstructure(data)
email_res = WhoisApi.get_emails(data)
phone_res = WhoisApi.parse_phone(data)
name_res = WhoisApi.get_names(data)
result_list = []
properties_list = []
special_properties_list = []
result = {'results': [], 'special_properties': [], 'properties': []}
# create structure data from result
if unstructured_res:
result_list.append(unstructured_res['results'])
special_properties_list = (unstructured_res['special_properties'])
properties_list = (unstructured_res['properties'])
# saving emails, phone, names in list of results
for email in email_res:
if email:
result_list.append(email)
for phone in phone_res:
if phone:
result_list.append(phone)
for name in name_res:
if name:
result_list.append(name)
if len(special_properties_list) == 0:
result['special_properties'] = ''
else:
result['special_properties'] = special_properties_list
if len(properties_list) == 0:
result['properties'] = ''
else:
result['properties'] = properties_list
result['results'] = result_list
return result
def reverse_ip_lookup(self, host, api_key, domain_count_flag, max_results):
""""
this functions takes a domain or IP address as host parameter and quickly shows all other domains hosted
from the same server
:param host: the domain or IP address to find all hosted domains on type as string
:param api_key: api key of this API type as string
:param max_results: maximum number of results that its default value is 200 type as Integer
:param domain_count_flag: just return number of domains
:returns if output_type is json, return {'query': {'tool': 'reverseip_PRO', 'host': ''},
'response': {'domain_count': '', 'domains': [{'name': '', 'last_resolved': ''}] }}
:raises if got request status code except 200, return {'error': 'Bad request'}
:raises if got exception {'error': 'error message'}
"""
page = 1
domains = []
self.update_progressbar('setting request to server', 20)
if domain_count_flag:
payload = {
'host': host,
'apikey': api_key,
'output': 'json',
'page': page
}
try:
self.parent.check_point()
r = Qhttp.get(url=Whois.Reverse_IP_Lookup_url,
url_query=payload)
if r.status_code == 200:
result = r.json()
response = result.get('response')
if response:
domain_count = response.get('domain_count')
if domain_count:
return {
'results': [{
'domain_count': int(domain_count),
'type': 0
}]
}
else:
raise NetworkError(r.text)
else:
raise BadRequest(r.status_code)
except Exception as e:
raise NetworkError(e.__str__())
while int(max_results) > len(domains):
payload = {
'host': host,
'apikey': api_key,
'output': 'json',
'page': page
}
try:
self.parent.check_point()
r = Qhttp.get(url=Whois.Reverse_IP_Lookup_url,
url_query=payload)
if r.status_code == 200:
result = r.json()
response = result.get('response')
if response:
domain_count = response.get('domain_count')
domains_results = response.get('domains')
if domains_results and domain_count:
domains.extend(domains_results)
if int(domain_count) <= len(domains):
# reaching end of result number
break
else:
# continue to next request if something bad happening
continue
else:
raise NetworkError(r.text)
else:
raise BadRequest(r.status_code)
except Exception as e:
raise NetworkError(e.__str__())
page += 1
self.update_progressbar('parsing_results', 80)
parse_result = {'results': []}
for item in domains:
if len(parse_result['results']) >= int(max_results):
break
try:
parse_result['results'].append({
'data':
item['name'],
'type':
12,
'properties': [{
'type': 0,
'last_resolved': item['last_resolved']
}]
})
except Exception:
continue
if len(parse_result['results']) == 0:
raise ResultNotFoundError
return parse_result
def ip_is_proxy(self, ip):
"""
this function specifies whether the given ip is proxy or not
:return Returns ip information as a dictionary
"""
api_key = self.api_key
self.update_progressbar('set request to is it proxy', 20)
if self.parent:
self.parent.check_point()
try:
r = requests.get('http://api.ip2proxy.com/?ip=' + ip + '&key=' +
api_key + '&package=PX4&format=json')
except (requests.exceptions.RequestException,
requests.exceptions.ConnectionError):
raise NetworkError(' can not access api')
if r.status_code == 200:
result = json.loads(r.content.decode())
self.update_progressbar('parsing result', 60)
if self.parent:
self.parent.check_point()
if 'response' in result:
temp = list()
if result['response'] == 'OK':
if result.items():
# parsing result
for k, v in result.items():
if k in [
'countryName', 'regionName', 'cityName',
'isp'
]:
temp.append({
self.ip_is_proxy_keys[k]: v,
"type": 11
})
elif k in ['proxyType']:
for each in v.split('/'):
try:
temp.append({
self.ip_is_proxy_keys[k]:
self.usage_and_proxy_type[each],
"type":
0
})
except KeyError:
temp.append({
self.ip_is_proxy_keys[k]: v,
"type": 0
})
elif k in ['isProxy']:
temp.append({
self.ip_is_proxy_keys[k]: v,
"type": 0
})
else:
pass
final = {
"properties": [],
"special_properties": temp,
"results": []
}
return final
else:
raise ResultNotFoundError(
'no result found for your request')
else:
if result['response'] == 'INSUFFICIENT CREDIT':
raise InsufficientCredit(result['response'])
elif result['response'] == 'INVALID ACCOUNT':
raise WrongApiKeyError(result['response'])
else:
raise InternalModuleError(result['response'])
else:
raise NetworkError(r.status_code)
else:
if r.status_code == 404:
raise ResultNotFoundError('no result found for your request')
else:
raise NetworkError('status_code ' + str(r.status_code))
def show_user(self, user_id):
"""
:param user_id: string, use as a user_id or screen_name search
:return: search result
"""
return_result = {'results': [], 'has_media': True}
self.update_progressbar(
'request submitted successfully, request to twitter: show user',
20)
try: # set request to twitter api, using different link if use id or screen_name
int(user_id)
timeline_endpoint = "https://api.twitter.com/1.1/users/show.json?user_id=%s" % (
str(user_id))
except ValueError:
timeline_endpoint = "https://api.twitter.com/1.1/users/show.json?screen_name=%s" % (
str(user_id))
try:
if self.parent:
self.parent.check_point()
response, data = self.client.request(timeline_endpoint)
except Exception:
raise NetworkError('can not connect to twitter')
self.update_progressbar('received data from api', 40)
if self.parent:
self.parent.check_point()
if 'status' not in response: # something goes wrong
raise InvalidInputError("status not found")
if response['status'] != '200': # something goes wrong
data_loads = json.loads(str(bytes.decode(data)))
if data_loads['errors'][-1]['message'] == 'Rate limit exceeded':
raise TwitterApiConstraint(
'Twitter rejected 3 consecutive attempts to access its API under your '
'Twitter account ')
else:
raise InvalidInputError(data_loads['errors'][-1]['message'])
data_loads = json.loads(str(bytes.decode(data)))
info = [data_loads]
# extract json demand format from parse result
self.update_progressbar('parse_result', 75)
return_value = Twitter.__parse_search(self, info)
result = return_value[0]
entity_list = return_value[1]
return_result['special_properties'] = result['results'][-1][
'special_properties']
return_result['properties'] = result['results'][-1]['properties']
for item in entity_list:
data = {'data': '', 'type': ''}
ref = {
'ref': {
'task': 'twitter_profile',
'twitter_account': user_id,
'section': ''
}
}
if 'name' in item.keys():
data['data'] = item['name']
data['type'] = 11
ref['ref']['section'] = 'name'
if 'location' in item.keys():
ref['ref']['section'] = 'location'
data['data'] = item['location']
data['type'] = 8
if 'screen_name' in item.keys():
ref['ref']['section'] = 'screen_name'
data['data'] = item['screen_name']
data['type'] = 5
data.update(ref)
return_result['results'].append(data)
return return_result
def get_follower_following(self, user_id, request_type):
"""
this function show the user's follower/following list
:param user_id: user_id or username as string
:param request_type: follower/ following
:return Returns a json of user's account info
"""
if not self.isLoggedIn:
self.login()
base_url = ''
user = []
self.update_progressbar('request submitted: get follower', 10)
# preparing urls:
if request_type == 'follower':
if user_id.isdigit(): # create url base on username or pk received
base_url = Instagram.get_follower_url.format(user_id)
else:
pk = self.profile(user_id)[1]['pk']
base_url = Instagram.get_follower_url.format(pk)
elif request_type == 'following':
if user_id.isdigit(): # create url base on username or pk received
base_url = Instagram.get_following_url.format(user_id)
else:
pk = self.profile(user_id)[1]['pk']
base_url = Instagram.get_following_url.format(pk)
self.update_progressbar('set request to Instagram api', 20)
if self.parent:
self.parent.check_point()
try: # set request to insta
r = self.session.get(url=base_url,
data=Instagram.generate_signature(
self, data=self.data),
headers=self.headers)
except Exception as e:
raise NetworkError(' can not access api' + e.__str__())
if r.status_code == 200:
result = r.json()
if self.parent:
self.parent.check_point()
user.extend(result['users'])
else:
if r.status_code == 404:
raise ResultNotFoundError('not found profile')
else:
raise NetworkError(r.status_code, r.text)
while True: # continue to extract person til reach request number or follower/following list finish
if self.parent:
self.parent.check_point()
if len(user) <= self.request_number:
if 'next_max_id' in result.keys():
max_id = result[
'next_max_id'] # extract person from continues list
try: # set request to insta
url = base_url + '?max_id=' + max_id
r = self.session.get(url=url,
data=Instagram.generate_signature(
self, data=self.data),
headers=self.headers)
except Exception:
break # in case of that s.t happening and can not extract more person
try:
if r.status_code == 200:
result = r.json()
if self.parent:
self.parent.check_point()
# parsing results
user.extend(
result['users']
) # adding profiles tp result till reach max_result number
else:
break # in case of that s.t happening and can not extract more person
except Exception:
break # in case of that s.t happening and can not extract more person
else:
break # in case of reaching end of follower/following list
else:
break # in case of reaching request number
# parsing result from extract info
if self.parent:
self.parent.check_point()
self.update_progressbar('parse_result', 80)
result = parse_follower_following(user, user_id, self.process_id,
self.request_number)
return result