def validate_proxy(proxy_id): proxy = mongo.db.proxies.find_one({'_id': ObjectId(proxy_id)}) util.abort_if_doesnt_exist(proxy) cfg = mongo.db.configurations.find_one({'status': True}) if not cfg: cfg = { 'syncInterval': 10, 'maxAge': 1, 'downloadDelay': 1, 'proxyTestTimeout': 1 } testurls = mongo.db.testurls.find() proxy_checker = HttpProxyChecker() for testurl in testurls: try: result = proxy_checker.validate_proxy(f"{proxy['ip']}:{proxy['port']}", testurl['url'], cfg['proxyTestTimeout']) proxytesturl = { 'proxyId': proxy['_id'], 'testurlId': testurl['_id'], 'validationTestDate': datetime.datetime.now(), 'validationTestStatus': result['status'] } mongo.db.proxytesturls.update_one( {'proxyId': proxy['_id'], 'testurlId': testurl['_id']}, {'$set': proxytesturl}, upsert=True ) except: print('error: validate_proxy') continue
def get(self, args, id): util.abort_if_invalid_id_format(id) if args['embed']: pipeline = [ { '$match': { '_id': ObjectId(id) } }, { '$lookup': { 'from': 'proxies', 'localField': '_id', 'foreignField': 'providerId', 'as': 'proxies' } }, ] docs = list(mongo.db.providers.aggregate( pipeline)) # pymongo aggregate returns list util.abort_if_doesnt_exist(docs) return provider_schema_embedded.dump(docs[0]) doc = mongo.db.providers.find_one({'_id': ObjectId(id)}) util.abort_if_doesnt_exist(doc) return provider_schema.dump(doc)
def sync_proxy(proxy_id): proxy = mongo.db.proxies.find_one({'_id': ObjectId(proxy_id)}) util.abort_if_doesnt_exist(proxy) cfg = mongo.db.configurations.find_one({'status': True}) if not cfg: cfg = { 'syncInterval': 10, 'maxAge': 1, 'downloadDelay': 1, 'proxyTestTimeout': 1 } proxy_checker = HttpProxyChecker() result = proxy_checker.check_proxy(f"{proxy['ip']}:{proxy['port']}", cfg['proxyTestTimeout']) if result['status'] == 'online': proxy['anonymity'] = result['anonymity'] proxy['funcTestDate'] = datetime.datetime.now() try: mongo.db.proxies.update_one({'_id': proxy['_id']}, {'$set': proxy}) return True except: abort(500, message='Error occurred while syncing', status="error") else: try: mongo.db.proxytesturls.delete_many({'proxyId': proxy['_id']}) mongo.db.proxies.delete_one({'_id': proxy['_id']}) except: abort(500, message='Error occurred while syncing', status="error") return False
def patch(self, id): util.abort_if_invalid_id_format(id) doc = mongo.db.providers.find_one({'_id': ObjectId(id)}) util.abort_if_doesnt_exist(doc) # Synchronising Proxy List syncProxies(id, 2) return '', 204
def put(self, proxy_new, id): abort_if_provider_doesnt_exist(proxy_new['providerId']) util.abort_if_invalid_id_format(id) proxy = mongo.db.proxies.find_one({'_id': ObjectId(id)}) util.abort_if_doesnt_exist(proxy) for k, v in proxy_new.items(): proxy[k] = v mongo.db.proxies.update_one({'_id': ObjectId(id)}, {'$set': proxy}) return '', 204
def put(self, testurl_new, params, id): util.abort_if_invalid_id_format(id) testurl = mongo.db.testurls.find_one({'_id': ObjectId(id)}) util.abort_if_doesnt_exist(testurl) for k, v in testurl_new.items(): testurl[k] = v mongo.db.testurls.update_one({'_id': ObjectId(id)}, {'$set': testurl}) if params['validate']: validate_testurl(testurl['_id']) return '', 204
def put(self, provider_new, params, id): util.abort_if_invalid_id_format(id) provider = mongo.db.providers.find_one({'_id': ObjectId(id)}) util.abort_if_doesnt_exist(provider) for k, v in provider_new.items(): provider[k] = v mongo.db.providers.update_one({'_id': ObjectId(id)}, {'$set': provider}) # Synchronising Proxy List based on syncTest param syncProxies(id, params['syncTest']) return '', 204
def delete(self, id): util.abort_if_invalid_id_format(id) provider = mongo.db.providers.find_one({'_id': ObjectId(id)}) util.abort_if_doesnt_exist(provider) #Clean proxytesturls proxies = mongo.db.proxies.find({'providerId': provider['_id']}) for proxy in proxies: mongo.db.proxytesturls.delete_many({'proxyId': proxy['_id']}) # Clean proxies mongo.db.proxies.delete_many({'providerId': provider['_id']}) # Delete provider mongo.db.providers.delete_one({'_id': provider['_id']}) return '', 204
def put(self, configuration_new): configuration = mongo.db.configurations.find_one({'status': True}) util.abort_if_doesnt_exist(configuration) for k, v in configuration_new.items(): configuration[k] = v result = mongo.db.configurations.update_one( {'_id': configuration['_id']}, {'$set': configuration}) if result.modified_count != 0 and configuration['status']: mongo.db.configurations.update_many( {'_id': { '$ne': configuration['_id'] }}, {'$set': { 'status': False }}) return '', 204
def get(self, args, id): util.abort_if_invalid_id_format(id) if args['embed']: docs = mongo.db.proxies.aggregate([ {'$match': {'_id': ObjectId(id)}}, {'$lookup': {'from': 'providers', 'localField': 'providerId', 'foreignField': '_id', 'as': 'provider'}}, {'$unwind': '$provider'}, { '$lookup': { 'from': 'proxytesturls', 'let': {'proxy_id': '$_id'}, 'pipeline': [ {'$match': {'$expr': {'$eq': ['$proxyId', '$$proxy_id']}}}, { '$lookup': { 'from': 'testurls', 'let': {'testurl_id': '$testurlId'}, 'pipeline': [{'$match': {'$expr': {'$eq': ['$_id', '$$testurl_id']}}}], 'as': 'testurl' } }, {'$unwind': '$testurl'}, {'$project': {'_id': '$testurl._id', 'url': '$testurl.url', 'validationAttempt': '$testurl.validationAttempt', 'validationTestDate': '$validationTestDate', 'validationTestStatus': '$validationTestStatus'}} ], 'as': 'testurls' } } ]) docs = list(docs) util.abort_if_doesnt_exist(docs) return proxy_schema_embedded.dump(docs[0]) doc = mongo.db.proxies.find_one({'_id': ObjectId(id)}) util.abort_if_doesnt_exist(doc) return proxy_schema.dump(doc)
def syncProxies(providerId, syncTest): provider = mongo.db.providers.find_one({'_id': ObjectId(providerId)}) util.abort_if_doesnt_exist(provider) if provider["fetcher"] == 'None': return # Checking update interval cfg = mongo.db.configurations.find_one({'status': True}) if not cfg: cfg = { 'syncInterval': 10, 'maxAge': 1, 'downloadDelay': 1, 'proxyTestTimeout': 1 } if 'updateAttempt' in provider: duration = datetime.datetime.now( ) - provider['updateAttempt']['attemptDate'] syncInterval = datetime.timedelta(0, 0, 0, 0, cfg['syncInterval'], 0, 0) if duration < syncInterval: abort( 429, message= "Too many requests,. You can change the interval from settings section", status="error") proxies = [] #print(provider) # Attempting to fetch if syncTest in [1, 2]: # 0->save only, 1-> fetch only, 2-> fetch+test # Delete stale proxies ############################################# stale_expr_1 = { 'providerId': provider['_id'], 'funcTestDate': { '$exists': False } } stale_expr_2 = { 'providerId': provider['_id'], 'funcTestDate': { '$exists': True }, 'funcTestDate': { '$lte': (datetime.datetime.now() - datetime.timedelta(weeks=cfg['maxAge'])) } } stale_expr_final = {'$or': [stale_expr_1, stale_expr_2]} #print(stale_expr_final) try: stale_proxies = list(mongo.db.proxies.find(stale_expr_final)) #print(len(stale_proxies)) for sp in stale_proxies: mongo.db.proxytesturls.delete_many({'proxyId': sp['_id']}) mongo.db.proxies.delete_many(stale_expr_final) #print(len(list(mongo.db.proxies.find(stale_expr_final)))) #return except Exception as err: print(err) pass ################################################################### try: mdl = import_module(f'core.proxy_fetchers.{provider["fetcher"]}') fetch = getattr(mdl, 'fetch') fetchedProxies = fetch(config=cfg) proxies = fetchedProxies # updating with fetched proxies provider['updateAttempt'] = { 'type': 'fetch', 'status': 'success', 'proxyCount': len(fetchedProxies), 'attemptDate': datetime.datetime.now() } except Exception as err: print(err) provider['updateAttempt'] = { 'type': 'fetch', 'status': 'failed', 'message': 'Error occurred while fetching', 'attemptDate': datetime.datetime.now() } if provider['updateAttempt']['status'] == 'success': # Attempting to test if syncTest == 2: proxyChecker = HttpProxyChecker() checkedProxies = [] try: for proxy in fetchedProxies: checkResult = proxyChecker.check_proxy( f"{proxy['ip']}:{proxy['port']}", cfg['proxyTestTimeout']) if checkResult['status'] == 'online': proxy['anonymity'] = checkResult['anonymity'] #proxy['country'] = checkResult['country'] #proxy['timeout'] = checkResult['timeout'] proxy['funcTestDate'] = datetime.datetime.now() checkedProxies.append(proxy) proxies = checkedProxies # updating with online proxies provider['updateAttempt'] = { 'type': 'funcTest', 'status': 'success', 'proxyCount': len(checkedProxies), 'attemptDate': datetime.datetime.now() } except Exception as err: print(err) provider['updateAttempt'] = { 'type': 'funcTest', 'status': 'failed', 'message': 'Error occurred while testing basic functionality', 'attemptDate': datetime.datetime.now() } #util.var_dump(len(proxies)) for proxy in proxies: proxy['providerId'] = provider['_id'] proxy['lastFoundDate'] = datetime.datetime.now() try: result = mongo.db.proxies.update_one( { 'ip': proxy['ip'], 'port': proxy['port'] }, {'$set': proxy}) # print('modified count: ', result.modified_count) if result.modified_count == 0: raise Exception("Could not update") except: try: proxy['discoveredDate'] = datetime.datetime.now() result = mongo.db.proxies.insert_one(proxy) # print('inserted') except: continue proxyCount = mongo.db.proxies.count( {'providerId': provider['_id']}) #util.var_dump(proxyCount) if proxyCount == 0: provider['updateAttempt'] = { 'type': f'syncDB_{provider["updateAttempt"]["type"]}', 'status': 'failed', 'message': 'No proxy is functional or error occurred while updating database', 'attemptDate': datetime.datetime.now() } elif proxyCount == len(proxies): provider['updateAttempt'] = { 'type': f'syncDB_{provider["updateAttempt"]["type"]}', 'status': 'success', 'proxyCount': proxyCount, 'attemptDate': datetime.datetime.now() } else: provider['updateAttempt'] = { 'type': f'syncDB_{provider["updateAttempt"]["type"]}', 'status': 'partial', 'proxyCount': proxyCount, 'message': 'Some proxies ar not functional or duplicate found', 'attemptDate': datetime.datetime.now() } mongo.db.providers.update_one({'_id': provider['_id']}, {'$set': provider})
def get(self, args, id): util.abort_if_invalid_id_format(id) if args['embed']: docs = mongo.db.testurls.aggregate([{ '$match': { '_id': ObjectId(id) } }, { "$lookup": { "from": "proxytesturls", "let": { "testurl_id": "$_id" }, "pipeline": [{ "$match": { "$expr": { "$eq": ["$testurlId", "$$testurl_id"] } } }, { "$lookup": { "from": "proxies", "let": { "proxy_id": "$proxyId" }, "pipeline": [{ "$match": { "$expr": { "$eq": ["$_id", "$$proxy_id"] } } }], "as": "proxy" } }, { "$unwind": "$proxy" }, { "$project": { "_id": "$proxy._id", "providerId": "$proxy.providerId", "ip": "$proxy.ip", "port": "$proxy.port", "funcTestDate": "$proxy.funcTestDate", "lastFoundDate": "$proxy.lastFoundDate", "discoveredDate": "$proxy.discoveredDate", "anonymity": "$proxy.anonymity", "validationTestDate": "$validationTestDate", "validationTestStatus": "$validationTestStatus" } }], "as": "proxies" } }]) docs = list(docs) util.abort_if_doesnt_exist(docs) return testurl_schema_embedded.dump(docs[0]) doc = mongo.db.testurls.find_one({'_id': ObjectId(id)}) util.abort_if_doesnt_exist(doc) return testurl_schema.dump(doc)
def validate_testurl(testurl_id): testurl = mongo.db.testurls.find_one({'_id': ObjectId(testurl_id)}) util.abort_if_doesnt_exist(testurl) cfg = mongo.db.configurations.find_one({'status': True}) if not cfg: cfg = { 'syncInterval': 10, 'maxAge': 1, 'downloadDelay': 1, 'proxyTestTimeout': 1 } proxies = list(mongo.db.proxies.find()) proxy_checker = HttpProxyChecker() valid_proxy_count = 0 for proxy in proxies: try: proxy_address = f"{proxy['ip']}:{proxy['port']}" # Checking if online result = proxy_checker.check_proxy(proxy_address, cfg['proxyTestTimeout']) if result['status'] == 'online': proxy['anonymity'] = result['anonymity'] proxy['funcTestDate'] = datetime.datetime.now() try: mongo.db.proxies.update_one({'_id': proxy['_id']}, {'$set': proxy}) except: print('error: while syncing proxy') continue else: try: mongo.db.proxytesturls.delete_many( {'proxyId': proxy['_id']}) mongo.db.proxies.delete_one({'_id': proxy['_id']}) continue except: print('error: while deleting proxy') continue ############### # Validating result = proxy_checker.validate_proxy(proxy_address, testurl['url'], cfg['proxyTestTimeout']) proxytesturl = { 'proxyId': proxy['_id'], 'testurlId': testurl['_id'], 'validationTestDate': datetime.datetime.now(), 'validationTestStatus': result['status'] } mongo.db.proxytesturls.update_one( { 'proxyId': proxy['_id'], 'testurlId': testurl['_id'] }, {'$set': proxytesturl}, upsert=True) if result['status'] == 'success': valid_proxy_count += 1 except: print('error: validate_testurl') continue try: testurl['validationAttempt'] = { 'attemptDate': datetime.datetime.now(), 'validProxyCount': valid_proxy_count, 'status': 'success' if (valid_proxy_count != 0 and valid_proxy_count == len(proxies)) else ('partial' if valid_proxy_count > 0 else 'failed') } mongo.db.testurls.update_one({'_id': testurl['_id']}, {'$set': testurl}) return True except: return False
def get(self): doc = mongo.db.configurations.find_one({'status': True}) util.abort_if_doesnt_exist(doc) return configuration_schema.dump(doc)
def get(self, id): util.abort_if_invalid_id_format(id) doc = mongo.db.configurations.find_one({'_id': ObjectId(id)}) util.abort_if_doesnt_exist(doc) return configuration_schema.dump(doc)