def pSend(self, args):
if app['debug']: print "RPC - sending cmd :", args
r = rpcClient.rpcClient(self.conf['host'], int(self.conf['port']))
error, data = r.sendJson(args)
return error, data
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.connect((self.conf['host'], int(self.conf['port'])))
s.sendall(json.dumps(args))
data = s.recv(4096)
s.close()
if app['debug']: print 'RPC - received data : ', repr(data)
data = json.loads(data)
if app['debug'] and 'result' in data and 'prints' in data['result']:
print data['result']['prints']
if data['error'] is None:
return data['result']['reply']
else:
return "ERROR: " + data['result']
except Exception, e:
print "ERROR: unable to send request. Is program started ?"
print e
s.close()
return False
def pSend(self, args):
if app['debug']: print "RPC - sending cmd :", args
r = rpcClient.rpcClient(self.conf['host'], int(self.conf['port']))
error, data = r.sendJson(args)
return error, data
try:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
s.connect((self.conf['host'], int(self.conf['port'])))
s.sendall(json.dumps(args))
data = s.recv(4096)
s.close()
if app['debug']: print 'RPC - received data : ', repr(data)
data = json.loads(data)
if app['debug'] and 'result' in data and 'prints' in data['result']:
print data['result']['prints']
if data['error'] is None:
return data['result']['reply']
else:
return "ERROR: " + data['result']
except Exception, e:
print "ERROR: unable to send request. Is program started ?"
print e
s.close()
return False
#!/usr/bin/python2
import json
import rpcClient
nmcontrolSession = rpcClient.rpcClient("127.0.0.1", 9000)
ip = json.loads(nmcontrolSession.sendSimple(["dns", "getIp4", "dot-bit.bit"]))
print "IPv4 address of dot-bit.bit is", ip
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'utility'))
sys.path.append(os.path.join(os.path.dirname(__file__), '..', 'config'))
from CloudAMQPClient import CloudAMQPClient
import config
import MongodbClient
from rpcClient import rpcClient
CLOUDAMQPURL = config.AMQP_URL_MONITOR_TO_SCRAPPER
CLOUDAMQPQUEUE = config.MONITOR_TO_SCRAPPER_QUEUE_NAME
NEWS_COLLECTION_NAME = config.NEWS_COLLECTION_NAME
cloudamqp_client = CloudAMQPClient(CLOUDAMQPURL, CLOUDAMQPQUEUE)
collection = MongodbClient.get_collection(NEWS_COLLECTION_NAME)
rpc = rpcClient('http://localhost:4040')
try:
nltk.data.find("corpora/stopwords")
except LookupError:
print("stopwords not found on the machine, download it now")
nltk.download("stopwords")
def check_possible_dup_docs(doc, dt):
start = dt - timedelta(hours=12)
end = dt + timedelta(hours=12)
news_list = collection.find({'publishedAt': {'$gte': start, '$lt': end}})
for news in news_list:
if check_dup(doc, [news['text']]):
return True
