class TestAsyncClient(TestCase):
def setUp(self):
TestCase.setUp(self)
self.client = Client("testuser", "testapikey", async=True)
def test_can_return_futures(self):
f = self.client.usage()
assert(isinstance(f, concurrent.futures.Future))
def test_can_return_futures_from_prefixed_endpoints(self):
f = self.client.pylon.analyze(1, 2, 3, service="service")
assert(isinstance(f, concurrent.futures.Future))
def test_futures_have_a_process_method(self):
f = self.client.pylon.analyze(1, 2, 3, service="service")
assert(hasattr(f, "process"))
from __future__ import print_function
from datasift import Client
datasift = Client("your username", "your API key")
print('Creating a managed source')
parameters = {'likes': True, 'posts_by_others': True, 'comments': True}
resources = [{
'parameters': {
'url': 'http://www.facebook.com/theguardian',
'title': 'Some news page',
'id': 'theguardian'
}
}]
auth = [{
'parameters': {
'value':
'CAAIUKbXn8xsBAC9uxyezVl4J5xHkKwg9fqhZB1sPJI9LvZAQd6UwHFDJZAkjxSjfEnXzAHzhOVFrHZBKwhOGYghTRMUCZA5iamBN9xK9Yg4oZAxgvzv9j40DAkEsF9XZBbrwcvpJ5ZAL5byL3Ba9oRIYod4AZBfR7nSoQfXbAF7ql94nCBUpzSPCe4DGS40jDys0ZD'
}
}]
source = datasift.managed_sources.create('facebook_page', 'My managed source',
resources, auth, parameters)
print(source)
source_id = source['id']
print('Starting delivery for my private source')
print(datasift.managed_sources.start(source_id))
def setUp(self):
TestCase.setUp(self)
self.client = Client("testuser", "testapikey", async=True)
# -*- coding: utf8 -*-
from __future__ import print_function
from datasift import Client
client = Client("your username", "your API key")
csdl = 'interaction.content contains "python"'
if client.is_valid(csdl):
response = client.compile(csdl)
stream = response['hash']
print('Stream %s created' % stream)
print('It takes %s DPUs' % client.dpu(stream)['dpu'])
print('Usage INFO \n %s' % client.usage())
print('Account balance is %s ' % client.balance())
else:
print('Could not validate CSDL')
class TestMockedClient(TestCase):
def setUp(self):
TestCase.setUp(self)
self.client = Client("testuser", "testapikey")
def test_creation_of_client(self):
self.assertTrue(self.client)
def test_handling_of_authorization_failed(self):
with HTTMock(authorization_failed):
self.assertRaises(AuthException, self.client.balance)
def test_output_of_balance(self):
mock, expected = mock_output_of(self.client.balance)
with HTTMock(mock):
runs = 0
for expecteditem in expected:
runs += 1
results = self.client.balance()
assert_dict_structure(self, results, expecteditem)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_rate_limit_exception(self):
with HTTMock(rate_limited):
self.assertRaises(RateLimitException, self.client.usage)
def test_ratelimit_headers(self):
with HTTMock(rate_limit_headers):
result = self.client.usage()
ratelimits = result.ratelimits
self.assertNotEqual(len(ratelimits), 0, "ensure that we got some rate limit headers")
self.assertEqual(ratelimits["cost"], 25)
def test_compile_raw_output(self):
mock, expected = mock_output_of(self.client.compile)
with HTTMock(mock):
runs = 0
for item in expected:
runs += 1
assert_dict_structure(self, item, self.client.compile("dummy csdl that is valid").raw)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_compile_python_friendly_output(self):
mock, expected = mock_output_of(self.client.compile)
with HTTMock(mock):
runs = 0
for item in expected:
runs += 1
result = self.client.compile("dummy csdl that is valid")
assert_dict_structure(self, item, result.raw)
assert_dict_structure(self, item, result)
self.assertEqual(result["created_at"], datetime.strptime(result.raw["created_at"], "%Y-%m-%d %H:%M:%S"))
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_compile_with_valid_output(self):
mock, expected = mock_output_of(self.client.compile)
with HTTMock(mock):
runs = 0
for item in expected:
runs += 1
assert_dict_structure(self, item, self.client.compile("dummy csdl that is valid"))
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_compile_invalid_csdl(self):
with HTTMock(failed_compilation_of_csdl):
self.assertRaises(DataSiftApiException, self.client.compile, ("dummy csdl which is bad"))
def test_is_valid_csdl_with_bad_data(self):
with HTTMock(failed_compilation_of_csdl):
self.assertFalse(self.client.is_valid("dummy csdl which is bad"))
def test_is_valid_csdl_with_good_data(self):
mock, expected = mock_output_of(self.client.validate)
with HTTMock(mock):
runs = 0
for item in expected:
runs+=1
r = self.client.is_valid("dummy csdl which is valid")
self.assertTrue(r)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_is_valid_csdl_cause_exception(self):
with HTTMock(internal_server_error_with_json):
self.assertRaises(DataSiftApiException, self.client.is_valid, ("csdl which turns into a teapot"))
def test_error_handling_of_internal_server_errors(self):
with HTTMock(internal_server_error):
self.assertRaises(DataSiftApiFailure, self.client.balance)
def test_error_handling_of_weird_errors(self):
with HTTMock(weird_error):
self.assertRaises(HTTPError, self.client.validate, ("csdl which turns into a teapot"))
def test_client_usage(self):
mock, expected = mock_output_of(self.client.usage)
with HTTMock(mock):
runs = 0
for expected_output in expected:
runs += 1
results = self.client.usage()
self.assertDictEqual(results.headers, {'content-type': 'application/json'})
assert_dict_structure(self, results, expected_output)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_client_usage_with_parameter(self):
mock, expected = mock_output_of(self.client.usage)
with HTTMock(mock):
runs = 0
for expected_output in expected:
runs += 1
results = self.client.usage(period="day")
assert_dict_structure(self, results, expected_output)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_client_dpu(self):
mock, expected = mock_output_of(self.client.dpu)
with HTTMock(mock):
runs = 0
for expected_output in expected:
runs += 1
results = self.client.dpu("valid stream id")
assert_dict_structure(self, results, expected_output)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
@unittest.skipIf(sys.version_info >= (3,0), "Mocking requests does not work correctly on py3")
def test_client_pull(self):
mock, expected = normal_pull_output()
with HTTMock(mock):
print("expected", expected)
results = self.client.pull("dummy valid subscription id", size=2048, cursor=512)
print("got", results)
self.assertEquals(results.status_code, 200)
self.assertEqual(len(results), len(expected), msg="get the same number of interactions out")
self.assertEqual(len(results), len(results.raw), msg="number of messages mangled by conversion")
self.assertDictEqual(results.headers, {})
self.assertTrue(results.status_code == 200)
for output, expected in zip(results, expected):
assert_dict_structure(self, output, expected)
def test_client_pull_json_meta(self):
with HTTMock(pull_json_meta):
result = self.client.pull("dummy valid subscription id")
self.assertIn("interactions", result, msg="ensure that there is an interactions section in the response")
self.assertEqual(len(result["interactions"]), 2, msg="check that both interactions made it")
@unittest.skipIf(sys.version_info >= (3,0), "Mocking requests does not work correctly on py3")
def test_client_pull_json_array(self):
with HTTMock(pull_json_array):
result = self.client.pull("dummy valid subscription id")
self.assertEqual(len(result), 2, msg="check that both interactions made it")
def test_live_streaming_exceptions_warn_on_bad_starts(self):
self.assertRaises(StreamSubscriberNotStarted, self.client.subscribe, ("hash"))
self.client._stream_process_started = True
func = self.client.subscribe("hash")
self.assertRaises(DeleteRequired, func, ("hash"))
def test_live_streaming_client_setup(self):
mock, expected = mock_output_of(self.client.compile)
with HTTMock(mock):
@self.client.on_delete
def on_delete(interaction):
print( 'Deleted interaction %s ' % interaction)
@self.client.on_open
def on_open():
print( 'Streaming ready, can start subscribing')
csdl = 'interaction.content contains "music"'
stream = self.client.compile(csdl)['hash']
@self.client.subscribe(stream)
def subscribe_to_hash(msg):
print(msg)
@self.client.on_closed
def on_close(wasClean, code, reason):
print('Streaming connection closed')
@self.client.on_ds_message
def on_ds_message(msg):
print('DS Message %s' % msg)
self.client._stream_process_started = True
self.client.start_stream_subscriber()
class DataCollector(object):
def __init__(self, ds_username, ds_api_key, csdl, data_path):
try:
logger.debug("Connecting to Datasift")
self.ds_client = Client(ds_username, ds_api_key)
self.data_path = data_path
self.current_file_for_received_tweets = None
logger.info("Connected to Datasift as %s" % ds_username)
logger.debug("Compiling query")
self.stream = self.ds_client.compile(csdl)["hash"]
logger.debug("query compiled")
except Exception as e:
logger.exception("exception in DataCollector.__init__%s" % str(e))
self.current_file_for_received_tweets.close()
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
raise e
@self.ds_client.on_delete
def on_delete(msg):
try:
logger.debug("deleted tweet")
f = open("%s/%s.deleted" % (self.data_path, datetime.datetime.utcnow().strftime("%Y-%m-%dT%H")), "a")
logger.debug("Opening %s" % f.name)
f.write(json.dumps(msg, default=json_date_handler) + "\n")
f.close()
except Exception as e:
logger.exception("exception in on_delete: %s" % str(e))
self.current_file_for_received_tweets.close()
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
pid = os.getpid()
logger.critical("Killing collecting process: %d" % pid)
os.kill(pid, 1)
@self.ds_client.on_open
def on_open():
try:
logger.info("Opening stream")
logger.debug("Waiting for tweets")
except Exception as e:
logger.exception("exception in on_open: %s" % str(e))
self.current_file_for_received_tweets.close()
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
pid = os.getpid()
logger.critical("Killing collecting process: %d" % pid)
os.kill(pid, 1)
@self.ds_client.subscribe(self.stream)
def on_interaction(msg):
try:
# logger.debug("tweet received")
creation_date = msg["twitter"]["created_at"]
if self.current_file_for_received_tweets is None or self.current_file_for_received_tweets.closed:
self.current_file_for_received_tweets = open(
"%s/%s.data" % (self.data_path, creation_date.strftime("%Y-%m-%dT%H")), "a"
)
logger.debug("opening %s" % self.current_file_for_received_tweets.name)
if self.current_file_for_received_tweets.name != "%s/%s.data" % (
self.data_path,
creation_date.strftime("%Y-%m-%dT%H"),
):
logger.debug("File switching")
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
self.current_file_for_received_tweets.close()
self.current_file_for_received_tweets = open(
"%s/%s.data" % (self.data_path, creation_date.strftime("%Y-%m-%dT%H")), "a"
)
logger.debug("opening %s" % self.current_file_for_received_tweets.name)
self.current_file_for_received_tweets.write(json.dumps(msg, default=json_date_handler) + "\n")
# self.current_file_for_received_tweets.close()
except Exception as e:
logger.exception("exception in on_interaction: %s" % str(e))
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
self.current_file_for_received_tweets.close()
pid = os.getpid()
logger.critical("Killing collecting process: %d" % pid)
os.kill(pid, 1)
# self.stopCollection()
# raise e
@self.ds_client.on_closed
def on_close(wasClean, code, reason):
try:
logger.info("Closing stream : %s" % reason)
self.current_file_for_received_tweets.close()
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
except Exception as e:
logger.exception("exception in onClose: %s" % str(e))
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
pid = os.getpid()
logger.critical("Killing collecting process: %d" % pid)
os.kill(pid, 1)
def startCollection(self):
try:
logger.debug("starting datasift client")
self.ds_client.start_stream_subscriber()
logger.info("datasift client started")
except Exception as e:
logger.exception("exception in DataCollector.startCollection%s" % str(e))
self.stopCollection()
raise e
def stopCollection(self):
try:
logger.debug("stoping datasift client")
# self.current_file_for_received_tweets.close() # cannot be closed here as it was opened in a different process
if self.ds_client._stream_process.is_alive():
logger.debug("stream process is alive, trying to terminate")
logger.debug(
"self.ds_client._stream_process is %s" % "None"
if self.ds_client._stream_process is None
else "not None: " + str(self.ds_client._stream_process)
)
self.ds_client._stream_process.terminate()
logger.info("datasift client stoped")
except Exception as e:
logger.exception("exception in DataCollector.stopCollection%s" % str(e))
raise e
def isCollecting(self): # maybe rather use self.ds_client._stream_process.join()
return self.ds_client._stream_process.is_alive()
def nTweetsReceivedInPast24h(self):
try:
return self.ds_client.usage("day")["streams"][self.stream]["licenses"]["twitter"]
except Exception as e:
logger.exception("exception %s" % str(e))
raise e
def __init__(self, ds_username, ds_api_key, csdl, data_path):
try:
logger.debug("Connecting to Datasift")
self.ds_client = Client(ds_username, ds_api_key)
self.data_path = data_path
self.current_file_for_received_tweets = None
logger.info("Connected to Datasift as %s" % ds_username)
logger.debug("Compiling query")
self.stream = self.ds_client.compile(csdl)["hash"]
logger.debug("query compiled")
except Exception as e:
logger.exception("exception in DataCollector.__init__%s" % str(e))
self.current_file_for_received_tweets.close()
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
raise e
@self.ds_client.on_delete
def on_delete(msg):
try:
logger.debug("deleted tweet")
f = open("%s/%s.deleted" % (self.data_path, datetime.datetime.utcnow().strftime("%Y-%m-%dT%H")), "a")
logger.debug("Opening %s" % f.name)
f.write(json.dumps(msg, default=json_date_handler) + "\n")
f.close()
except Exception as e:
logger.exception("exception in on_delete: %s" % str(e))
self.current_file_for_received_tweets.close()
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
pid = os.getpid()
logger.critical("Killing collecting process: %d" % pid)
os.kill(pid, 1)
@self.ds_client.on_open
def on_open():
try:
logger.info("Opening stream")
logger.debug("Waiting for tweets")
except Exception as e:
logger.exception("exception in on_open: %s" % str(e))
self.current_file_for_received_tweets.close()
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
pid = os.getpid()
logger.critical("Killing collecting process: %d" % pid)
os.kill(pid, 1)
@self.ds_client.subscribe(self.stream)
def on_interaction(msg):
try:
# logger.debug("tweet received")
creation_date = msg["twitter"]["created_at"]
if self.current_file_for_received_tweets is None or self.current_file_for_received_tweets.closed:
self.current_file_for_received_tweets = open(
"%s/%s.data" % (self.data_path, creation_date.strftime("%Y-%m-%dT%H")), "a"
)
logger.debug("opening %s" % self.current_file_for_received_tweets.name)
if self.current_file_for_received_tweets.name != "%s/%s.data" % (
self.data_path,
creation_date.strftime("%Y-%m-%dT%H"),
):
logger.debug("File switching")
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
self.current_file_for_received_tweets.close()
self.current_file_for_received_tweets = open(
"%s/%s.data" % (self.data_path, creation_date.strftime("%Y-%m-%dT%H")), "a"
)
logger.debug("opening %s" % self.current_file_for_received_tweets.name)
self.current_file_for_received_tweets.write(json.dumps(msg, default=json_date_handler) + "\n")
# self.current_file_for_received_tweets.close()
except Exception as e:
logger.exception("exception in on_interaction: %s" % str(e))
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
self.current_file_for_received_tweets.close()
pid = os.getpid()
logger.critical("Killing collecting process: %d" % pid)
os.kill(pid, 1)
# self.stopCollection()
# raise e
@self.ds_client.on_closed
def on_close(wasClean, code, reason):
try:
logger.info("Closing stream : %s" % reason)
self.current_file_for_received_tweets.close()
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
except Exception as e:
logger.exception("exception in onClose: %s" % str(e))
logger.debug("closing %s" % self.current_file_for_received_tweets.name)
pid = os.getpid()
logger.critical("Killing collecting process: %d" % pid)
os.kill(pid, 1)
# -*- coding: utf8 -*-
from __future__ import print_function
import time
from datasift import Client
datasift = Client("your username", "your API key")
stream = datasift.compile('interaction.content contains "datasift"')
stream = stream['hash']
start = time.time() - (7200 * 48) # 48hrs ago
end_time = start + 3600
print('Check the data coverage for the last two hours')
print(datasift.historics.status(start, end_time))
print('Preparing')
historics = datasift.historics.prepare(stream, start, end_time,
'My python historics', ['tumblr'])
print(historics)
historics_id = historics['id']
print('Creating push subscription for historics')
create_params = {
'name': 'My awesome push subscription',
'initial_status': 'active',
'playback_id': stream,
'start': time.time(),
'end': time.time() + 320
# -*- coding: utf8 -*-
from __future__ import print_function
from datasift import Client
ds = Client("your username", "your API key")
@ds.on_delete
def on_delete(interaction):
print( 'Deleted interaction %s ' % interaction)
@ds.on_open
def on_open():
print( 'Streaming ready, can start subscribing')
csdl = 'interaction.content contains "music"'
stream = ds.compile(csdl)['hash']
@ds.subscribe(stream)
def subscribe_to_hash(msg):
print( msg)
@ds.on_closed
def on_close(wasClean, code, reason):
print( 'Streaming connection closed')
@ds.on_ds_message
def on_ds_message(msg):
print( 'DS Message %s' % msg)
def setUp(self):
TestCase.setUp(self)
self.client = Client("testuser", "testapikey")
class TestMockedClient(TestCase):
def setUp(self):
TestCase.setUp(self)
self.client = Client("testuser", "testapikey")
def test_creation_of_client(self):
self.assertTrue(self.client)
def test_handling_of_authorization_failed(self):
with HTTMock(authorization_failed):
self.assertRaises(AuthException, self.client.balance)
def test_output_of_balance(self):
mock, expected = mock_output_of(self.client.balance)
with HTTMock(mock):
runs = 0
for expecteditem in expected:
runs += 1
results = self.client.balance()
assert_dict_structure(self, results, expecteditem)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_rate_limit_exception(self):
with HTTMock(rate_limited):
self.assertRaises(RateLimitException, self.client.usage)
def test_ratelimit_headers(self):
with HTTMock(rate_limit_headers):
result = self.client.usage()
ratelimits = result.ratelimits
self.assertNotEqual(len(ratelimits), 0, "ensure that we got some rate limit headers")
self.assertEqual(ratelimits["cost"], 25)
def test_compile_raw_output(self):
mock, expected = mock_output_of(self.client.compile)
with HTTMock(mock):
runs = 0
for item in expected:
runs += 1
assert_dict_structure(self, item, self.client.compile("dummy csdl that is valid").raw)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_compile_python_friendly_output(self):
mock, expected = mock_output_of(self.client.compile)
with HTTMock(mock):
runs = 0
for item in expected:
runs += 1
result = self.client.compile("dummy csdl that is valid")
assert_dict_structure(self, item, result.raw)
assert_dict_structure(self, item, result)
self.assertEqual(result["created_at"], datetime.strptime(result.raw["created_at"], "%Y-%m-%d %H:%M:%S"))
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_compile_with_valid_output(self):
mock, expected = mock_output_of(self.client.compile)
with HTTMock(mock):
runs = 0
for item in expected:
runs += 1
assert_dict_structure(self, item, self.client.compile("dummy csdl that is valid"))
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_compile_invalid_csdl(self):
with HTTMock(failed_compilation_of_csdl):
self.assertRaises(DataSiftApiException, self.client.compile, ("dummy csdl which is bad"))
def test_is_valid_csdl_with_bad_data(self):
with HTTMock(failed_compilation_of_csdl):
self.assertFalse(self.client.is_valid("dummy csdl which is bad"))
def test_is_valid_csdl_with_good_data(self):
mock, expected = mock_output_of(self.client.validate)
with HTTMock(mock):
runs = 0
for item in expected:
runs+=1
r = self.client.is_valid("dummy csdl which is valid")
self.assertTrue(r)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_is_valid_csdl_cause_exception(self):
with HTTMock(internal_server_error_with_json):
self.assertRaises(DataSiftApiException, self.client.is_valid, ("csdl which turns into a teapot"))
def test_error_handling_of_internal_server_errors(self):
with HTTMock(internal_server_error):
self.assertRaises(DataSiftApiFailure, self.client.balance)
def test_error_handling_of_weird_errors(self):
with HTTMock(weird_error):
self.assertRaises(HTTPError, self.client.validate, ("csdl which turns into a teapot"))
def test_client_usage(self):
mock, expected = mock_output_of(self.client.usage)
with HTTMock(mock):
runs = 0
for expected_output in expected:
runs += 1
results = self.client.usage()
self.assertDictEqual(results.headers, {'content-type': 'application/json'})
assert_dict_structure(self, results, expected_output)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_client_usage_with_parameter(self):
mock, expected = mock_output_of(self.client.usage)
with HTTMock(mock):
runs = 0
for expected_output in expected:
runs += 1
results = self.client.usage(period="day")
assert_dict_structure(self, results, expected_output)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
def test_client_dpu(self):
mock, expected = mock_output_of(self.client.dpu)
with HTTMock(mock):
runs = 0
for expected_output in expected:
runs += 1
results = self.client.dpu("valid stream id")
assert_dict_structure(self, results, expected_output)
self.assertNotEqual(runs, 0, "ensure that at least one case was tested")
@unittest.skipIf(sys.version_info >= (3,0), "Mocking requests does not work correctly on py3")
def test_client_pull(self):
mock, expected = normal_pull_output()
with HTTMock(mock):
print("expected", expected)
results = self.client.pull("dummy valid subscription id", size=2048, cursor=512)
print("got", results)
self.assertEquals(results.status_code, 200)
self.assertEqual(len(results), len(expected), msg="get the same number of interactions out")
self.assertEqual(len(results), len(results.raw), msg="number of messages mangled by conversion")
self.assertDictEqual(results.headers, {})
self.assertTrue(results.status_code == 200)
for output, expected in zip(results, expected):
assert_dict_structure(self, output, expected)
def test_live_streaming_exceptions_warn_on_bad_starts(self):
self.assertRaises(StreamSubscriberNotStarted, self.client.subscribe, ("hash"))
self.client._stream_process_started = True
func = self.client.subscribe("hash")
self.assertRaises(DeleteRequired, func, ("hash"))
def test_live_streaming_client_setup(self):
mock, expected = mock_output_of(self.client.compile)
with HTTMock(mock):
@self.client.on_delete
def on_delete(interaction):
print( 'Deleted interaction %s ' % interaction)
@self.client.on_open
def on_open():
print( 'Streaming ready, can start subscribing')
csdl = 'interaction.content contains "music"'
stream = self.client.compile(csdl)['hash']
@self.client.subscribe(stream)
def subscribe_to_hash(msg):
print(msg)
@self.client.on_closed
def on_close(wasClean, code, reason):
print('Streaming connection closed')
@self.client.on_ds_message
def on_ds_message(msg):
print('DS Message %s' % msg)
self.client._stream_process_started = True
self.client.start_stream_subscriber()
# -*- coding: utf8 -*-
from __future__ import print_function
import time
from datasift import Client
datasift = Client("your username", "your API key")
csdl = 'interaction.content contains "python"'
s3_params = {
'bucket': 'apitests',
'directory': 'ruby',
'acl': 'private',
'auth': {
'access_key': 'AKIAIINK5C4FH75RSWNA',
'secret_key': 'F9mLnLoFFGuCNgbMUhdhHmm5YCcNAt/OG32SUhPy'
},
'delivery_frequency': 0,
'max_size': 10485760,
'file_prefix': 'DataSift',
}
res = datasift.push.validate('s3', s3_params)
print(res)
stream = datasift.compile(csdl)
print('Creating a pull subscription from stream ==> %s' % stream['hash'])
print(stream)
subscription = datasift.push.create_from_hash(stream['hash'], 'My Python Pull subscription', 'pull', {})
print(subscription)
# -*- coding: utf8 -*-
from __future__ import print_function
from datasift import Client
from datasift.exceptions import DataSiftApiException
from concurrent.futures import as_completed
client = Client("yourusername", "yourkey", async=True, max_workers=5)
csdl = [
'interaction.content contains "python"',
'this is invalid CSDL',
'language.tag == "en"',
'interaction.content any "foo, bar, baz"',
'interaction.type != "facebook"'
]
# build ourselves a dict of future -> input so we know which one failed
results = dict([(client.validate(x), x) for x in csdl])
# add callbacks
for result in as_completed(results.keys()):
try:
result.process()
print("'%s' was valid CSDL" % results[result])
except DataSiftApiException:
print("'%s' was invalid CSDL" % results[result])
from __future__ import print_function
import time
from datasift import Client
datasift = Client("your username", "your API key")
stream = datasift.compile('interaction.content contains "datasift"')
stream = stream['hash']
start = time.time() - (7200 * 48) # 48hrs ago
end_time = start + 3600
print('Check the data coverage for the last two hours')
print(datasift.historics.status(start, end_time))
print('Preparing')
historics = datasift.historics.prepare(stream, start, end_time, 'My python historics', ['twitter'])
print(historics)
historics_id = historics['id']
print('Creating push subscription for historics')
create_params = {
'name': 'My awesome push subscription',
'initial_status': 'active',
'playback_id': stream,
'start': time.time(),
'end': time.time() + 320
}
