def write_body(file, limit):
canceled = False
todo = done = 0
filename = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
for feed in Feed.iter(filename):
todo += 1
try:
ok, result = Feed.read(feed, limit)
if not ok:
Qtrac.report(result, True)
elif result is not None:
Qtrac.report("read {} at {}".format(feed.title, feed.url))
for item in result:
file.write(item)
done += 1
except KeyboardInterrupt:
Qtrac.report("canceling...")
canceled = True
break
return todo, done, canceled
def worker(limit, jobs, results):
while True:
try:
feed = jobs.get()
ok, result = Feed.read(feed, limit)
if not ok:
Qtrac.report(result, True)
elif result is not None:
Qtrac.report("read {}".format(result[0][4:-6]))
results.put(result)
finally:
jobs.task_done()
def worker(limit, jobs, results):
while True:
try:
feed = jobs.get()
ok, result = Feed.read(feed, limit)
if not ok:
Qtrac.report(result, True)
elif result is not None:
Qtrac.report("read {}".format(result[0][4:-6]))
results.put(result)
finally:
jobs.task_done()
def reader(receiver, sink, limit, me):
while True:
feed, who = (yield)
if who == me:
ok, result = Feed.read(feed, limit)
if not ok:
Qtrac.report(result, True)
result = None
else:
Qtrac.report("read {} at {}".format(feed.title, feed.url))
sink.send(result)
elif receiver is not None:
receiver.send((feed, who))
def reader(receiver, sink, limit, me):
while True:
feed, who = (yield)
if who == me:
ok, result = Feed.read(feed, limit)
if not ok:
Qtrac.report(result, True)
result = None
else:
Qtrac.report("read {} at {}".format(feed.title, feed.url))
sink.send(result)
elif receiver is not None:
receiver.send((feed, who))
def setUp(self):
CD = ContractDeployer.Contract_Deployer("http://127.0.0.1:8545")
w3 = CD.web3Connection()
self.w3 = w3
self.NULL_ENCODING = b'\x00' * 32
w3.eth.defaultAccount = w3.eth.accounts[0]
self.account = w3.eth.defaultAccount
w3.eth.defaultAccount = w3.eth.accounts[1]
self.account1 = w3.eth.defaultAccount
w3.eth.defaultAccount = w3.eth.accounts[2]
self.account2 = w3.eth.defaultAccount
w3.eth.defaultAccount = w3.eth.accounts[3]
self.account3 = w3.eth.defaultAccount
w3.eth.defaultAccount = w3.eth.accounts[4]
self.account4 = w3.eth.defaultAccount
w3.eth.defaultAccount = w3.eth.accounts[5]
self.account5 = w3.eth.defaultAccount
self.metadata = b'Test'
#First deploy the Instance Registry Contract.
self.Erasure_Posts_Contract = CD.deployContract(
"../contracts/Erasure_Posts.sol", "Erasure_Posts",
"Erasure_Posts.sol", self.account)
self.Erasure_Posts = Erasure_Posts.Erasure_Posts(
self.Erasure_Posts_Contract, self.account)
self.Registry = self.Erasure_Posts.Registry()
#Deploy the contract template that needs to be used.
self.Feed_contract = CD.deployContract("../contracts/Feed.sol", "Feed",
"Feed.sol", self.account)
self.Feed = Feed.Feed(self.Feed_contract, self.Feed_contract)
#Call the contract deployer before deployment of contract, deploy contract manually with specified paramters.
self.Feed_Factory_Contract = CD.deployContractConstructor(
"../contracts/Feed_Factory.sol", "Feed_Factory",
"Feed_Factory.sol", self.account,
self.Erasure_Posts_Contract.address, self.Feed_contract.address)
#InstanceRegistry is needed and so is
self.Feed_Factory = Feed_Factory.Feed_Factory(
self.Feed_Factory_Contract, self.account)
self.Factory = self.Feed_Factory.Factory()
self.Registry.addFactory(self.Feed_Factory_Contract.address,
self.metadata)
def testNext(self):
uq = UpdateQueue()
f1 = Feed('feed1')
f2 = Feed('feed2')
f3 = Feed('feed3')
f1.lastUpdated = 0
f2.lastUpdated = time.time() - 5
f3.lastUpdated = time.time()
f2.updateInterval = 10
f3.updateInterval = 6
uq.add(f1)
uq.add(f2)
uq.add(f3)
self.assertEqual(uq.next(), f1 )
self.assertEqual(uq.next(), f2 )
self.assertTrue(time.time() > (f2.lastUpdated + f2.updateInterval))
self.assertEqual(uq.next(), f3 )
self.assertTrue(time.time() > (f3.lastUpdated + f3.updateInterval))
self.assertTrue((time.time() - (f3.lastUpdated + f3.updateInterval)) < 1)
def main():
limit, concurrency = handle_commandline()
Qtrac.report("starting...")
filename = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
futures = set()
with concurrent.futures.ProcessPoolExecutor(
max_workers=concurrency) as executor:
for feed in Feed.iter(filename):
future = executor.submit(Feed.read, feed, limit)
futures.add(future)
done, filename, canceled = process(futures)
if canceled:
executor.shutdown()
Qtrac.report("read {}/{} feeds using {} processes{}".format(
done, len(futures), concurrency, " [canceled]" if canceled else ""))
print()
if not canceled:
webbrowser.open(filename)
def main():
limit, concurrency = handle_commandline()
Qtrac.report("starting...")
filename = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
futures = set()
with concurrent.futures.ThreadPoolExecutor(
max_workers=concurrency) as executor:
for feed in Feed.iter(filename):
future = executor.submit(Feed.read, feed, limit)
futures.add(future)
done, filename, canceled = process(futures)
if canceled:
executor.shutdown()
Qtrac.report("read {}/{} feeds using {} threads{}".format(done,
len(futures), concurrency, " [canceled]" if canceled else ""))
print()
if not canceled:
webbrowser.open(filename)
def main():
limit, concurrency = handle_commandline()
Qtrac.report("starting...")
datafile = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
filename = os.path.join(tempfile.gettempdir(), "whatsnew.html")
canceled = False
with open(filename, "wt", encoding="utf-8") as file:
write_header(file)
pipeline = create_pipeline(limit, concurrency, file)
try:
for i, feed in enumerate(Feed.iter(datafile)):
pipeline.send((feed, i % concurrency))
except KeyboardInterrupt:
Qtrac.report("canceling...")
canceled = True
write_footer(file, results.ok, results.todo, canceled, concurrency)
if not canceled:
webbrowser.open(filename)
def main():
limit, concurrency = handle_commandline()
Qtrac.report("starting...")
datafile = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
filename = os.path.join(tempfile.gettempdir(), "whatsnew.html")
canceled = False
with open(filename, "wt", encoding="utf-8") as file:
write_header(file)
pipeline = create_pipeline(limit, concurrency, file)
try:
for i, feed in enumerate(Feed.iter(datafile)):
pipeline.send((feed, i % concurrency))
except KeyboardInterrupt:
Qtrac.report("canceling...")
canceled = True
write_footer(file, results.ok, results.todo, canceled,
concurrency)
if not canceled:
webbrowser.open(filename)
def main():
limit, concurrency = handle_commandline()
Qtrac.report("starting...")
filename = os.path.join(os.path.dirname(__file__), "whatsnew.dat")
futures = set() # instances set of futures
with concurrent.futures.ThreadPoolExecutor(
max_workers=concurrency) as executor:
for feed in Feed.iter(filename): # a generator of Feed(title, url)
future = executor.submit(
Feed.read, feed,
limit) # Feed.read is fn; feed and limit are fn's parameters
futures.add(future) # add future instance into futures set
done, filename, canceled = process(futures)
if canceled:
executor.shutdown()
Qtrac.report("read {}/{} feeds using {} threads{}".format(
done, len(futures), concurrency, " [canceled]" if canceled else ""))
print()
if not canceled:
webbrowser.open(filename)
class Login(object):
def print_json(j, prefix=''):
for key, value in j.items():
if isinstance(value, dict):
print '%s%s' % (prefix, key)
else:
print '%s%s:%s' % (prefix, key, value)
username = '******'
password = '******'
service = 'NEXTAPI'
URL = 'api.test.nordnet.se'
API_VERSION = '2'
timestamp = int(round(time.time() * 1000))
timestamp = str(timestamp)
buf = base64.b64encode(username) + ':' + base64.b64encode(password) + ':' + base64.b64encode(timestamp)
rsa = RSA.load_pub_key('NEXTAPI_TEST_public.pem')
encrypted_hash = rsa.public_encrypt(buf, RSA.pkcs1_padding)
hash = base64.b64encode(encrypted_hash)
headers = {"Accept": "application/json"}
conn = httplib.HTTPSConnection(URL)
# GET server status
conn.request('GET', '/next/' + API_VERSION + '/', '', headers)
r = conn.getresponse()
response = r.read()
j = json.loads(response)
print_json(j)
# POST login
params = urllib.urlencode({'service': 'NEXTAPI', 'auth': hash})
conn.request('POST', '/next/' + API_VERSION + '/login', params, headers)
r = conn.getresponse()
response = r.read()
j = json.loads(response)
print_json(j)
session_key = j["session_key"]
Feed.GetRequests().create_market_list(session_key, headers)
Feed.GetRequests().instrument_name(session_key, headers, 'FINGerprint')
#################################################################################
# Create SSL-wrapped socket
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ssl_socket = ssl.wrap_socket(s)
# Connect to socket
ssl_socket.connect(("pub.api.test.nordnet.se", 443))
print repr(ssl_socket.getpeername)
print ssl_socket.cipher()
# Send session key
cmd = {"cmd": "login", "args": {"session_key": session_key, "service": "NEXTAPI"}}
num_bytes = ssl_socket.write(json.dumps(cmd) + "\n")
print "Session key sent (%d bytes)" % num_bytes
# Get account information
# Subscribe to a stock
market = 11
instruments = ["101", "4870"] # ERIC B and FING B
for i in range(0, len(instruments)):
instrument = instruments[i]
cmd = {"cmd": "subscribe", "args": {"t": "price", "m": 11, "i": instrument}}
numBytes = ssl_socket.send(json.dumps(cmd) + "\n")
print("Subscription request sent for market = %d and instrument = %s (%d bytes)" % (11, instrument, numBytes))
# Read stream
print "Reading stream"
for i in range(10):
output = ssl_socket.read(1024)
print (output)
time.sleep(1)
print ("-")
output = ssl_socket.recv()
print (output)
# print "Closing socket connection..."
del ssl_socket
s.close()
def get_feeds(self, load_entries):
return Feed.get_user_feeds(self, load_entries)
def add_jobs(filename, jobs):
for todo, feed in enumerate(Feed.iter(filename), start=1):
jobs.put(feed)
return todo
def add_jobs(filename, jobs):
for todo, feed in enumerate(Feed.iter(filename), start=1):
jobs.put(feed)
return todo
#!/usr/bin/python
import Feed
for f in Feed.getNewFeeds():
f.reload()
Feed.printFeed(f)
f.save()
for f in Feed.getActiveFeeds():
f.reload()
Feed.printFeed(f)
f.save()
# !!! Once a day or so we should poll inactive feeds to make sure
# nothing has changed
for f in Feed.getInactiveFeeds():
f.reload()
Feed.printFeed(f)
f.save()
def testAdd(self):
uq = UpdateQueue()
f1 = Feed('feed2')
f2 = Feed('feed4')
f3 = Feed('feed5')
f4 = Feed('feed1')
f5 = Feed('feed3')
f1.lastUpdated = 20
f2.lastUpdated = 40
f3.lastUpdated = 50
f4.lastUpdated = 10
f5.lastUpdated = 30
f6 = feed('feed1')
f5.updateInterval = 30 * 60
uq.add(f1)
uq.add(f2)
uq.add(f3)
uq.add(f4)
uq.add(f5)
uq.add(f6)
self.assertEqual( uq.next(), f4 )
self.assertEqual( uq.next(), f1 )
self.assertEqual( uq.next(), f2 )
self.assertEqual( uq.next(), f3 )
self.assertEqual( uq.next(), f5 )
#!/usr/bin/python
import Feed
feeds = Feed.get_all_feeds()
for b in feeds:
b.reload()
b.save()
def add_jobs(filename, jobs):
# Feed.iter() yields a Feed instance (name, rss_url)
for todo, feed in enumerate(Feed.iter(filename), start=1):
jobs.put(feed) # add the feed to the input queue
return todo # number of feeds added to jobs queue
def testNext(self):
uq = UpdateQueue()
f1 = Feed('feed1')
f2 = Feed('feed2')
f3 = Feed('feed3')
f1.lastUpdated = 0
f2.lastUpdated = time.time() - 5
f3.lastUpdated = time.time()
f2.updateInterval = 10
f3.updateInterval = 6
uq.add(f1)
uq.add(f2)
uq.add(f3)
self.assertEqual(uq.next(), f1)
self.assertEqual(uq.next(), f2)
self.assertTrue(time.time() > (f2.lastUpdated + f2.updateInterval))
self.assertEqual(uq.next(), f3)
self.assertTrue(time.time() > (f3.lastUpdated + f3.updateInterval))
self.assertTrue((time.time() -
(f3.lastUpdated + f3.updateInterval)) < 1)
'entryID': {
'type': 'string'
},
'feedID': {
'type': 'string'
},
'link': {
'type': 'string'
},
'mongoID': {
'type': 'string'
},
'publishedDate': {
'type': 'date'
},
'summary': {
'type': 'string'
},
'title': {
'type': 'string'
}
}
}
}
es.indices.create(index='reader', ignore=400, body=feed_entry_mapping)
for f in Feed.get_all_feeds():
f.loadEntries()
f.save()
def SyncFeed():
feedInstace = Feed()
feedInstace.feedParser()
def testAdd(self):
uq = UpdateQueue()
f1 = Feed('feed2')
f2 = Feed('feed4')
f3 = Feed('feed5')
f4 = Feed('feed1')
f5 = Feed('feed3')
f1.lastUpdated = 20
f2.lastUpdated = 40
f3.lastUpdated = 50
f4.lastUpdated = 10
f5.lastUpdated = 30
f6 = feed('feed1')
f5.updateInterval = 30 * 60
uq.add(f1)
uq.add(f2)
uq.add(f3)
uq.add(f4)
uq.add(f5)
uq.add(f6)
self.assertEqual(uq.next(), f4)
self.assertEqual(uq.next(), f1)
self.assertEqual(uq.next(), f2)
self.assertEqual(uq.next(), f3)
self.assertEqual(uq.next(), f5)
def add_jobs(filename, jobs):
for todo, feed in enumerate(Feed.iter(filename), start=1): # _todo is 1, 2, 3, 4... feed is Feed(title ,url) gen
jobs.put(feed) # load Feed's generator into jobs' queue
return todo # return the title count of mission
