def validate_proxy(proxy_id):
proxy = mongo.db.proxies.find_one({'_id': ObjectId(proxy_id)})
util.abort_if_doesnt_exist(proxy)
cfg = mongo.db.configurations.find_one({'status': True})
if not cfg:
cfg = {
'syncInterval': 10,
'maxAge': 1,
'downloadDelay': 1,
'proxyTestTimeout': 1
}
testurls = mongo.db.testurls.find()
proxy_checker = HttpProxyChecker()
for testurl in testurls:
try:
result = proxy_checker.validate_proxy(f"{proxy['ip']}:{proxy['port']}", testurl['url'], cfg['proxyTestTimeout'])
proxytesturl = {
'proxyId': proxy['_id'],
'testurlId': testurl['_id'],
'validationTestDate': datetime.datetime.now(),
'validationTestStatus': result['status']
}
mongo.db.proxytesturls.update_one(
{'proxyId': proxy['_id'], 'testurlId': testurl['_id']},
{'$set': proxytesturl},
upsert=True
)
except:
print('error: validate_proxy')
continue
def get(self, args, id):
util.abort_if_invalid_id_format(id)
if args['embed']:
pipeline = [
{
'$match': {
'_id': ObjectId(id)
}
},
{
'$lookup': {
'from': 'proxies',
'localField': '_id',
'foreignField': 'providerId',
'as': 'proxies'
}
},
]
docs = list(mongo.db.providers.aggregate(
pipeline)) # pymongo aggregate returns list
util.abort_if_doesnt_exist(docs)
return provider_schema_embedded.dump(docs[0])
doc = mongo.db.providers.find_one({'_id': ObjectId(id)})
util.abort_if_doesnt_exist(doc)
return provider_schema.dump(doc)
def sync_proxy(proxy_id):
proxy = mongo.db.proxies.find_one({'_id': ObjectId(proxy_id)})
util.abort_if_doesnt_exist(proxy)
cfg = mongo.db.configurations.find_one({'status': True})
if not cfg:
cfg = {
'syncInterval': 10,
'maxAge': 1,
'downloadDelay': 1,
'proxyTestTimeout': 1
}
proxy_checker = HttpProxyChecker()
result = proxy_checker.check_proxy(f"{proxy['ip']}:{proxy['port']}", cfg['proxyTestTimeout'])
if result['status'] == 'online':
proxy['anonymity'] = result['anonymity']
proxy['funcTestDate'] = datetime.datetime.now()
try:
mongo.db.proxies.update_one({'_id': proxy['_id']}, {'$set': proxy})
return True
except:
abort(500, message='Error occurred while syncing', status="error")
else:
try:
mongo.db.proxytesturls.delete_many({'proxyId': proxy['_id']})
mongo.db.proxies.delete_one({'_id': proxy['_id']})
except:
abort(500, message='Error occurred while syncing', status="error")
return False
def patch(self, id):
util.abort_if_invalid_id_format(id)
doc = mongo.db.providers.find_one({'_id': ObjectId(id)})
util.abort_if_doesnt_exist(doc)
# Synchronising Proxy List
syncProxies(id, 2)
return '', 204
def put(self, proxy_new, id):
abort_if_provider_doesnt_exist(proxy_new['providerId'])
util.abort_if_invalid_id_format(id)
proxy = mongo.db.proxies.find_one({'_id': ObjectId(id)})
util.abort_if_doesnt_exist(proxy)
for k, v in proxy_new.items():
proxy[k] = v
mongo.db.proxies.update_one({'_id': ObjectId(id)}, {'$set': proxy})
return '', 204
def put(self, testurl_new, params, id):
util.abort_if_invalid_id_format(id)
testurl = mongo.db.testurls.find_one({'_id': ObjectId(id)})
util.abort_if_doesnt_exist(testurl)
for k, v in testurl_new.items():
testurl[k] = v
mongo.db.testurls.update_one({'_id': ObjectId(id)}, {'$set': testurl})
if params['validate']:
validate_testurl(testurl['_id'])
return '', 204
def put(self, provider_new, params, id):
util.abort_if_invalid_id_format(id)
provider = mongo.db.providers.find_one({'_id': ObjectId(id)})
util.abort_if_doesnt_exist(provider)
for k, v in provider_new.items():
provider[k] = v
mongo.db.providers.update_one({'_id': ObjectId(id)},
{'$set': provider})
# Synchronising Proxy List based on syncTest param
syncProxies(id, params['syncTest'])
return '', 204
def delete(self, id):
util.abort_if_invalid_id_format(id)
provider = mongo.db.providers.find_one({'_id': ObjectId(id)})
util.abort_if_doesnt_exist(provider)
#Clean proxytesturls
proxies = mongo.db.proxies.find({'providerId': provider['_id']})
for proxy in proxies:
mongo.db.proxytesturls.delete_many({'proxyId': proxy['_id']})
# Clean proxies
mongo.db.proxies.delete_many({'providerId': provider['_id']})
# Delete provider
mongo.db.providers.delete_one({'_id': provider['_id']})
return '', 204
def put(self, configuration_new):
configuration = mongo.db.configurations.find_one({'status': True})
util.abort_if_doesnt_exist(configuration)
for k, v in configuration_new.items():
configuration[k] = v
result = mongo.db.configurations.update_one(
{'_id': configuration['_id']}, {'$set': configuration})
if result.modified_count != 0 and configuration['status']:
mongo.db.configurations.update_many(
{'_id': {
'$ne': configuration['_id']
}}, {'$set': {
'status': False
}})
return '', 204
def get(self, args, id):
util.abort_if_invalid_id_format(id)
if args['embed']:
docs = mongo.db.proxies.aggregate([
{'$match': {'_id': ObjectId(id)}},
{'$lookup': {'from': 'providers', 'localField': 'providerId', 'foreignField': '_id', 'as': 'provider'}},
{'$unwind': '$provider'},
{
'$lookup': {
'from': 'proxytesturls',
'let': {'proxy_id': '$_id'},
'pipeline': [
{'$match': {'$expr': {'$eq': ['$proxyId', '$$proxy_id']}}},
{
'$lookup': {
'from': 'testurls',
'let': {'testurl_id': '$testurlId'},
'pipeline': [{'$match': {'$expr': {'$eq': ['$_id', '$$testurl_id']}}}],
'as': 'testurl'
}
},
{'$unwind': '$testurl'},
{'$project': {'_id': '$testurl._id', 'url': '$testurl.url',
'validationAttempt': '$testurl.validationAttempt', 'validationTestDate': '$validationTestDate', 'validationTestStatus': '$validationTestStatus'}}
],
'as': 'testurls'
}
}
])
docs = list(docs)
util.abort_if_doesnt_exist(docs)
return proxy_schema_embedded.dump(docs[0])
doc = mongo.db.proxies.find_one({'_id': ObjectId(id)})
util.abort_if_doesnt_exist(doc)
return proxy_schema.dump(doc)
def syncProxies(providerId, syncTest):
provider = mongo.db.providers.find_one({'_id': ObjectId(providerId)})
util.abort_if_doesnt_exist(provider)
if provider["fetcher"] == 'None':
return
# Checking update interval
cfg = mongo.db.configurations.find_one({'status': True})
if not cfg:
cfg = {
'syncInterval': 10,
'maxAge': 1,
'downloadDelay': 1,
'proxyTestTimeout': 1
}
if 'updateAttempt' in provider:
duration = datetime.datetime.now(
) - provider['updateAttempt']['attemptDate']
syncInterval = datetime.timedelta(0, 0, 0, 0, cfg['syncInterval'], 0,
0)
if duration < syncInterval:
abort(
429,
message=
"Too many requests,. You can change the interval from settings section",
status="error")
proxies = []
#print(provider)
# Attempting to fetch
if syncTest in [1, 2]: # 0->save only, 1-> fetch only, 2-> fetch+test
# Delete stale proxies #############################################
stale_expr_1 = {
'providerId': provider['_id'],
'funcTestDate': {
'$exists': False
}
}
stale_expr_2 = {
'providerId': provider['_id'],
'funcTestDate': {
'$exists': True
},
'funcTestDate': {
'$lte': (datetime.datetime.now() -
datetime.timedelta(weeks=cfg['maxAge']))
}
}
stale_expr_final = {'$or': [stale_expr_1, stale_expr_2]}
#print(stale_expr_final)
try:
stale_proxies = list(mongo.db.proxies.find(stale_expr_final))
#print(len(stale_proxies))
for sp in stale_proxies:
mongo.db.proxytesturls.delete_many({'proxyId': sp['_id']})
mongo.db.proxies.delete_many(stale_expr_final)
#print(len(list(mongo.db.proxies.find(stale_expr_final))))
#return
except Exception as err:
print(err)
pass
###################################################################
try:
mdl = import_module(f'core.proxy_fetchers.{provider["fetcher"]}')
fetch = getattr(mdl, 'fetch')
fetchedProxies = fetch(config=cfg)
proxies = fetchedProxies # updating with fetched proxies
provider['updateAttempt'] = {
'type': 'fetch',
'status': 'success',
'proxyCount': len(fetchedProxies),
'attemptDate': datetime.datetime.now()
}
except Exception as err:
print(err)
provider['updateAttempt'] = {
'type': 'fetch',
'status': 'failed',
'message': 'Error occurred while fetching',
'attemptDate': datetime.datetime.now()
}
if provider['updateAttempt']['status'] == 'success':
# Attempting to test
if syncTest == 2:
proxyChecker = HttpProxyChecker()
checkedProxies = []
try:
for proxy in fetchedProxies:
checkResult = proxyChecker.check_proxy(
f"{proxy['ip']}:{proxy['port']}",
cfg['proxyTestTimeout'])
if checkResult['status'] == 'online':
proxy['anonymity'] = checkResult['anonymity']
#proxy['country'] = checkResult['country']
#proxy['timeout'] = checkResult['timeout']
proxy['funcTestDate'] = datetime.datetime.now()
checkedProxies.append(proxy)
proxies = checkedProxies # updating with online proxies
provider['updateAttempt'] = {
'type': 'funcTest',
'status': 'success',
'proxyCount': len(checkedProxies),
'attemptDate': datetime.datetime.now()
}
except Exception as err:
print(err)
provider['updateAttempt'] = {
'type': 'funcTest',
'status': 'failed',
'message':
'Error occurred while testing basic functionality',
'attemptDate': datetime.datetime.now()
}
#util.var_dump(len(proxies))
for proxy in proxies:
proxy['providerId'] = provider['_id']
proxy['lastFoundDate'] = datetime.datetime.now()
try:
result = mongo.db.proxies.update_one(
{
'ip': proxy['ip'],
'port': proxy['port']
}, {'$set': proxy})
# print('modified count: ', result.modified_count)
if result.modified_count == 0:
raise Exception("Could not update")
except:
try:
proxy['discoveredDate'] = datetime.datetime.now()
result = mongo.db.proxies.insert_one(proxy)
# print('inserted')
except:
continue
proxyCount = mongo.db.proxies.count(
{'providerId': provider['_id']})
#util.var_dump(proxyCount)
if proxyCount == 0:
provider['updateAttempt'] = {
'type': f'syncDB_{provider["updateAttempt"]["type"]}',
'status': 'failed',
'message':
'No proxy is functional or error occurred while updating database',
'attemptDate': datetime.datetime.now()
}
elif proxyCount == len(proxies):
provider['updateAttempt'] = {
'type': f'syncDB_{provider["updateAttempt"]["type"]}',
'status': 'success',
'proxyCount': proxyCount,
'attemptDate': datetime.datetime.now()
}
else:
provider['updateAttempt'] = {
'type': f'syncDB_{provider["updateAttempt"]["type"]}',
'status': 'partial',
'proxyCount': proxyCount,
'message':
'Some proxies ar not functional or duplicate found',
'attemptDate': datetime.datetime.now()
}
mongo.db.providers.update_one({'_id': provider['_id']},
{'$set': provider})
def get(self, args, id):
util.abort_if_invalid_id_format(id)
if args['embed']:
docs = mongo.db.testurls.aggregate([{
'$match': {
'_id': ObjectId(id)
}
}, {
"$lookup": {
"from":
"proxytesturls",
"let": {
"testurl_id": "$_id"
},
"pipeline": [{
"$match": {
"$expr": {
"$eq": ["$testurlId", "$$testurl_id"]
}
}
}, {
"$lookup": {
"from":
"proxies",
"let": {
"proxy_id": "$proxyId"
},
"pipeline": [{
"$match": {
"$expr": {
"$eq": ["$_id", "$$proxy_id"]
}
}
}],
"as":
"proxy"
}
}, {
"$unwind": "$proxy"
}, {
"$project": {
"_id": "$proxy._id",
"providerId": "$proxy.providerId",
"ip": "$proxy.ip",
"port": "$proxy.port",
"funcTestDate": "$proxy.funcTestDate",
"lastFoundDate": "$proxy.lastFoundDate",
"discoveredDate": "$proxy.discoveredDate",
"anonymity": "$proxy.anonymity",
"validationTestDate": "$validationTestDate",
"validationTestStatus": "$validationTestStatus"
}
}],
"as":
"proxies"
}
}])
docs = list(docs)
util.abort_if_doesnt_exist(docs)
return testurl_schema_embedded.dump(docs[0])
doc = mongo.db.testurls.find_one({'_id': ObjectId(id)})
util.abort_if_doesnt_exist(doc)
return testurl_schema.dump(doc)
def validate_testurl(testurl_id):
testurl = mongo.db.testurls.find_one({'_id': ObjectId(testurl_id)})
util.abort_if_doesnt_exist(testurl)
cfg = mongo.db.configurations.find_one({'status': True})
if not cfg:
cfg = {
'syncInterval': 10,
'maxAge': 1,
'downloadDelay': 1,
'proxyTestTimeout': 1
}
proxies = list(mongo.db.proxies.find())
proxy_checker = HttpProxyChecker()
valid_proxy_count = 0
for proxy in proxies:
try:
proxy_address = f"{proxy['ip']}:{proxy['port']}"
# Checking if online
result = proxy_checker.check_proxy(proxy_address,
cfg['proxyTestTimeout'])
if result['status'] == 'online':
proxy['anonymity'] = result['anonymity']
proxy['funcTestDate'] = datetime.datetime.now()
try:
mongo.db.proxies.update_one({'_id': proxy['_id']},
{'$set': proxy})
except:
print('error: while syncing proxy')
continue
else:
try:
mongo.db.proxytesturls.delete_many(
{'proxyId': proxy['_id']})
mongo.db.proxies.delete_one({'_id': proxy['_id']})
continue
except:
print('error: while deleting proxy')
continue
###############
# Validating
result = proxy_checker.validate_proxy(proxy_address,
testurl['url'],
cfg['proxyTestTimeout'])
proxytesturl = {
'proxyId': proxy['_id'],
'testurlId': testurl['_id'],
'validationTestDate': datetime.datetime.now(),
'validationTestStatus': result['status']
}
mongo.db.proxytesturls.update_one(
{
'proxyId': proxy['_id'],
'testurlId': testurl['_id']
}, {'$set': proxytesturl},
upsert=True)
if result['status'] == 'success':
valid_proxy_count += 1
except:
print('error: validate_testurl')
continue
try:
testurl['validationAttempt'] = {
'attemptDate':
datetime.datetime.now(),
'validProxyCount':
valid_proxy_count,
'status':
'success' if
(valid_proxy_count != 0 and valid_proxy_count == len(proxies)) else
('partial' if valid_proxy_count > 0 else 'failed')
}
mongo.db.testurls.update_one({'_id': testurl['_id']},
{'$set': testurl})
return True
except:
return False
def get(self):
doc = mongo.db.configurations.find_one({'status': True})
util.abort_if_doesnt_exist(doc)
return configuration_schema.dump(doc)
def get(self, id):
util.abort_if_invalid_id_format(id)
doc = mongo.db.configurations.find_one({'_id': ObjectId(id)})
util.abort_if_doesnt_exist(doc)
return configuration_schema.dump(doc)