class TestAsyncClient(TestCase): def setUp(self): TestCase.setUp(self) self.client = Client("testuser", "testapikey", async=True) def test_can_return_futures(self): f = self.client.usage() assert(isinstance(f, concurrent.futures.Future)) def test_can_return_futures_from_prefixed_endpoints(self): f = self.client.pylon.analyze(1, 2, 3, service="service") assert(isinstance(f, concurrent.futures.Future)) def test_futures_have_a_process_method(self): f = self.client.pylon.analyze(1, 2, 3, service="service") assert(hasattr(f, "process"))
from __future__ import print_function from datasift import Client datasift = Client("your username", "your API key") print('Creating a managed source') parameters = {'likes': True, 'posts_by_others': True, 'comments': True} resources = [{ 'parameters': { 'url': 'http://www.facebook.com/theguardian', 'title': 'Some news page', 'id': 'theguardian' } }] auth = [{ 'parameters': { 'value': 'CAAIUKbXn8xsBAC9uxyezVl4J5xHkKwg9fqhZB1sPJI9LvZAQd6UwHFDJZAkjxSjfEnXzAHzhOVFrHZBKwhOGYghTRMUCZA5iamBN9xK9Yg4oZAxgvzv9j40DAkEsF9XZBbrwcvpJ5ZAL5byL3Ba9oRIYod4AZBfR7nSoQfXbAF7ql94nCBUpzSPCe4DGS40jDys0ZD' } }] source = datasift.managed_sources.create('facebook_page', 'My managed source', resources, auth, parameters) print(source) source_id = source['id'] print('Starting delivery for my private source') print(datasift.managed_sources.start(source_id))
def setUp(self): TestCase.setUp(self) self.client = Client("testuser", "testapikey", async=True)
# -*- coding: utf8 -*- from __future__ import print_function from datasift import Client client = Client("your username", "your API key") csdl = 'interaction.content contains "python"' if client.is_valid(csdl): response = client.compile(csdl) stream = response['hash'] print('Stream %s created' % stream) print('It takes %s DPUs' % client.dpu(stream)['dpu']) print('Usage INFO \n %s' % client.usage()) print('Account balance is %s ' % client.balance()) else: print('Could not validate CSDL')
class TestMockedClient(TestCase): def setUp(self): TestCase.setUp(self) self.client = Client("testuser", "testapikey") def test_creation_of_client(self): self.assertTrue(self.client) def test_handling_of_authorization_failed(self): with HTTMock(authorization_failed): self.assertRaises(AuthException, self.client.balance) def test_output_of_balance(self): mock, expected = mock_output_of(self.client.balance) with HTTMock(mock): runs = 0 for expecteditem in expected: runs += 1 results = self.client.balance() assert_dict_structure(self, results, expecteditem) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_rate_limit_exception(self): with HTTMock(rate_limited): self.assertRaises(RateLimitException, self.client.usage) def test_ratelimit_headers(self): with HTTMock(rate_limit_headers): result = self.client.usage() ratelimits = result.ratelimits self.assertNotEqual(len(ratelimits), 0, "ensure that we got some rate limit headers") self.assertEqual(ratelimits["cost"], 25) def test_compile_raw_output(self): mock, expected = mock_output_of(self.client.compile) with HTTMock(mock): runs = 0 for item in expected: runs += 1 assert_dict_structure(self, item, self.client.compile("dummy csdl that is valid").raw) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_compile_python_friendly_output(self): mock, expected = mock_output_of(self.client.compile) with HTTMock(mock): runs = 0 for item in expected: runs += 1 result = self.client.compile("dummy csdl that is valid") assert_dict_structure(self, item, result.raw) assert_dict_structure(self, item, result) self.assertEqual(result["created_at"], datetime.strptime(result.raw["created_at"], "%Y-%m-%d %H:%M:%S")) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_compile_with_valid_output(self): mock, expected = mock_output_of(self.client.compile) with HTTMock(mock): runs = 0 for item in expected: runs += 1 assert_dict_structure(self, item, self.client.compile("dummy csdl that is valid")) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_compile_invalid_csdl(self): with HTTMock(failed_compilation_of_csdl): self.assertRaises(DataSiftApiException, self.client.compile, ("dummy csdl which is bad")) def test_is_valid_csdl_with_bad_data(self): with HTTMock(failed_compilation_of_csdl): self.assertFalse(self.client.is_valid("dummy csdl which is bad")) def test_is_valid_csdl_with_good_data(self): mock, expected = mock_output_of(self.client.validate) with HTTMock(mock): runs = 0 for item in expected: runs+=1 r = self.client.is_valid("dummy csdl which is valid") self.assertTrue(r) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_is_valid_csdl_cause_exception(self): with HTTMock(internal_server_error_with_json): self.assertRaises(DataSiftApiException, self.client.is_valid, ("csdl which turns into a teapot")) def test_error_handling_of_internal_server_errors(self): with HTTMock(internal_server_error): self.assertRaises(DataSiftApiFailure, self.client.balance) def test_error_handling_of_weird_errors(self): with HTTMock(weird_error): self.assertRaises(HTTPError, self.client.validate, ("csdl which turns into a teapot")) def test_client_usage(self): mock, expected = mock_output_of(self.client.usage) with HTTMock(mock): runs = 0 for expected_output in expected: runs += 1 results = self.client.usage() self.assertDictEqual(results.headers, {'content-type': 'application/json'}) assert_dict_structure(self, results, expected_output) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_client_usage_with_parameter(self): mock, expected = mock_output_of(self.client.usage) with HTTMock(mock): runs = 0 for expected_output in expected: runs += 1 results = self.client.usage(period="day") assert_dict_structure(self, results, expected_output) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_client_dpu(self): mock, expected = mock_output_of(self.client.dpu) with HTTMock(mock): runs = 0 for expected_output in expected: runs += 1 results = self.client.dpu("valid stream id") assert_dict_structure(self, results, expected_output) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") @unittest.skipIf(sys.version_info >= (3,0), "Mocking requests does not work correctly on py3") def test_client_pull(self): mock, expected = normal_pull_output() with HTTMock(mock): print("expected", expected) results = self.client.pull("dummy valid subscription id", size=2048, cursor=512) print("got", results) self.assertEquals(results.status_code, 200) self.assertEqual(len(results), len(expected), msg="get the same number of interactions out") self.assertEqual(len(results), len(results.raw), msg="number of messages mangled by conversion") self.assertDictEqual(results.headers, {}) self.assertTrue(results.status_code == 200) for output, expected in zip(results, expected): assert_dict_structure(self, output, expected) def test_client_pull_json_meta(self): with HTTMock(pull_json_meta): result = self.client.pull("dummy valid subscription id") self.assertIn("interactions", result, msg="ensure that there is an interactions section in the response") self.assertEqual(len(result["interactions"]), 2, msg="check that both interactions made it") @unittest.skipIf(sys.version_info >= (3,0), "Mocking requests does not work correctly on py3") def test_client_pull_json_array(self): with HTTMock(pull_json_array): result = self.client.pull("dummy valid subscription id") self.assertEqual(len(result), 2, msg="check that both interactions made it") def test_live_streaming_exceptions_warn_on_bad_starts(self): self.assertRaises(StreamSubscriberNotStarted, self.client.subscribe, ("hash")) self.client._stream_process_started = True func = self.client.subscribe("hash") self.assertRaises(DeleteRequired, func, ("hash")) def test_live_streaming_client_setup(self): mock, expected = mock_output_of(self.client.compile) with HTTMock(mock): @self.client.on_delete def on_delete(interaction): print( 'Deleted interaction %s ' % interaction) @self.client.on_open def on_open(): print( 'Streaming ready, can start subscribing') csdl = 'interaction.content contains "music"' stream = self.client.compile(csdl)['hash'] @self.client.subscribe(stream) def subscribe_to_hash(msg): print(msg) @self.client.on_closed def on_close(wasClean, code, reason): print('Streaming connection closed') @self.client.on_ds_message def on_ds_message(msg): print('DS Message %s' % msg) self.client._stream_process_started = True self.client.start_stream_subscriber()
class DataCollector(object): def __init__(self, ds_username, ds_api_key, csdl, data_path): try: logger.debug("Connecting to Datasift") self.ds_client = Client(ds_username, ds_api_key) self.data_path = data_path self.current_file_for_received_tweets = None logger.info("Connected to Datasift as %s" % ds_username) logger.debug("Compiling query") self.stream = self.ds_client.compile(csdl)["hash"] logger.debug("query compiled") except Exception as e: logger.exception("exception in DataCollector.__init__%s" % str(e)) self.current_file_for_received_tweets.close() logger.debug("closing %s" % self.current_file_for_received_tweets.name) raise e @self.ds_client.on_delete def on_delete(msg): try: logger.debug("deleted tweet") f = open("%s/%s.deleted" % (self.data_path, datetime.datetime.utcnow().strftime("%Y-%m-%dT%H")), "a") logger.debug("Opening %s" % f.name) f.write(json.dumps(msg, default=json_date_handler) + "\n") f.close() except Exception as e: logger.exception("exception in on_delete: %s" % str(e)) self.current_file_for_received_tweets.close() logger.debug("closing %s" % self.current_file_for_received_tweets.name) pid = os.getpid() logger.critical("Killing collecting process: %d" % pid) os.kill(pid, 1) @self.ds_client.on_open def on_open(): try: logger.info("Opening stream") logger.debug("Waiting for tweets") except Exception as e: logger.exception("exception in on_open: %s" % str(e)) self.current_file_for_received_tweets.close() logger.debug("closing %s" % self.current_file_for_received_tweets.name) pid = os.getpid() logger.critical("Killing collecting process: %d" % pid) os.kill(pid, 1) @self.ds_client.subscribe(self.stream) def on_interaction(msg): try: # logger.debug("tweet received") creation_date = msg["twitter"]["created_at"] if self.current_file_for_received_tweets is None or self.current_file_for_received_tweets.closed: self.current_file_for_received_tweets = open( "%s/%s.data" % (self.data_path, creation_date.strftime("%Y-%m-%dT%H")), "a" ) logger.debug("opening %s" % self.current_file_for_received_tweets.name) if self.current_file_for_received_tweets.name != "%s/%s.data" % ( self.data_path, creation_date.strftime("%Y-%m-%dT%H"), ): logger.debug("File switching") logger.debug("closing %s" % self.current_file_for_received_tweets.name) self.current_file_for_received_tweets.close() self.current_file_for_received_tweets = open( "%s/%s.data" % (self.data_path, creation_date.strftime("%Y-%m-%dT%H")), "a" ) logger.debug("opening %s" % self.current_file_for_received_tweets.name) self.current_file_for_received_tweets.write(json.dumps(msg, default=json_date_handler) + "\n") # self.current_file_for_received_tweets.close() except Exception as e: logger.exception("exception in on_interaction: %s" % str(e)) logger.debug("closing %s" % self.current_file_for_received_tweets.name) self.current_file_for_received_tweets.close() pid = os.getpid() logger.critical("Killing collecting process: %d" % pid) os.kill(pid, 1) # self.stopCollection() # raise e @self.ds_client.on_closed def on_close(wasClean, code, reason): try: logger.info("Closing stream : %s" % reason) self.current_file_for_received_tweets.close() logger.debug("closing %s" % self.current_file_for_received_tweets.name) except Exception as e: logger.exception("exception in onClose: %s" % str(e)) logger.debug("closing %s" % self.current_file_for_received_tweets.name) pid = os.getpid() logger.critical("Killing collecting process: %d" % pid) os.kill(pid, 1) def startCollection(self): try: logger.debug("starting datasift client") self.ds_client.start_stream_subscriber() logger.info("datasift client started") except Exception as e: logger.exception("exception in DataCollector.startCollection%s" % str(e)) self.stopCollection() raise e def stopCollection(self): try: logger.debug("stoping datasift client") # self.current_file_for_received_tweets.close() # cannot be closed here as it was opened in a different process if self.ds_client._stream_process.is_alive(): logger.debug("stream process is alive, trying to terminate") logger.debug( "self.ds_client._stream_process is %s" % "None" if self.ds_client._stream_process is None else "not None: " + str(self.ds_client._stream_process) ) self.ds_client._stream_process.terminate() logger.info("datasift client stoped") except Exception as e: logger.exception("exception in DataCollector.stopCollection%s" % str(e)) raise e def isCollecting(self): # maybe rather use self.ds_client._stream_process.join() return self.ds_client._stream_process.is_alive() def nTweetsReceivedInPast24h(self): try: return self.ds_client.usage("day")["streams"][self.stream]["licenses"]["twitter"] except Exception as e: logger.exception("exception %s" % str(e)) raise e
def __init__(self, ds_username, ds_api_key, csdl, data_path): try: logger.debug("Connecting to Datasift") self.ds_client = Client(ds_username, ds_api_key) self.data_path = data_path self.current_file_for_received_tweets = None logger.info("Connected to Datasift as %s" % ds_username) logger.debug("Compiling query") self.stream = self.ds_client.compile(csdl)["hash"] logger.debug("query compiled") except Exception as e: logger.exception("exception in DataCollector.__init__%s" % str(e)) self.current_file_for_received_tweets.close() logger.debug("closing %s" % self.current_file_for_received_tweets.name) raise e @self.ds_client.on_delete def on_delete(msg): try: logger.debug("deleted tweet") f = open("%s/%s.deleted" % (self.data_path, datetime.datetime.utcnow().strftime("%Y-%m-%dT%H")), "a") logger.debug("Opening %s" % f.name) f.write(json.dumps(msg, default=json_date_handler) + "\n") f.close() except Exception as e: logger.exception("exception in on_delete: %s" % str(e)) self.current_file_for_received_tweets.close() logger.debug("closing %s" % self.current_file_for_received_tweets.name) pid = os.getpid() logger.critical("Killing collecting process: %d" % pid) os.kill(pid, 1) @self.ds_client.on_open def on_open(): try: logger.info("Opening stream") logger.debug("Waiting for tweets") except Exception as e: logger.exception("exception in on_open: %s" % str(e)) self.current_file_for_received_tweets.close() logger.debug("closing %s" % self.current_file_for_received_tweets.name) pid = os.getpid() logger.critical("Killing collecting process: %d" % pid) os.kill(pid, 1) @self.ds_client.subscribe(self.stream) def on_interaction(msg): try: # logger.debug("tweet received") creation_date = msg["twitter"]["created_at"] if self.current_file_for_received_tweets is None or self.current_file_for_received_tweets.closed: self.current_file_for_received_tweets = open( "%s/%s.data" % (self.data_path, creation_date.strftime("%Y-%m-%dT%H")), "a" ) logger.debug("opening %s" % self.current_file_for_received_tweets.name) if self.current_file_for_received_tweets.name != "%s/%s.data" % ( self.data_path, creation_date.strftime("%Y-%m-%dT%H"), ): logger.debug("File switching") logger.debug("closing %s" % self.current_file_for_received_tweets.name) self.current_file_for_received_tweets.close() self.current_file_for_received_tweets = open( "%s/%s.data" % (self.data_path, creation_date.strftime("%Y-%m-%dT%H")), "a" ) logger.debug("opening %s" % self.current_file_for_received_tweets.name) self.current_file_for_received_tweets.write(json.dumps(msg, default=json_date_handler) + "\n") # self.current_file_for_received_tweets.close() except Exception as e: logger.exception("exception in on_interaction: %s" % str(e)) logger.debug("closing %s" % self.current_file_for_received_tweets.name) self.current_file_for_received_tweets.close() pid = os.getpid() logger.critical("Killing collecting process: %d" % pid) os.kill(pid, 1) # self.stopCollection() # raise e @self.ds_client.on_closed def on_close(wasClean, code, reason): try: logger.info("Closing stream : %s" % reason) self.current_file_for_received_tweets.close() logger.debug("closing %s" % self.current_file_for_received_tweets.name) except Exception as e: logger.exception("exception in onClose: %s" % str(e)) logger.debug("closing %s" % self.current_file_for_received_tweets.name) pid = os.getpid() logger.critical("Killing collecting process: %d" % pid) os.kill(pid, 1)
# -*- coding: utf8 -*- from __future__ import print_function import time from datasift import Client datasift = Client("your username", "your API key") stream = datasift.compile('interaction.content contains "datasift"') stream = stream['hash'] start = time.time() - (7200 * 48) # 48hrs ago end_time = start + 3600 print('Check the data coverage for the last two hours') print(datasift.historics.status(start, end_time)) print('Preparing') historics = datasift.historics.prepare(stream, start, end_time, 'My python historics', ['tumblr']) print(historics) historics_id = historics['id'] print('Creating push subscription for historics') create_params = { 'name': 'My awesome push subscription', 'initial_status': 'active', 'playback_id': stream, 'start': time.time(), 'end': time.time() + 320
# -*- coding: utf8 -*- from __future__ import print_function from datasift import Client ds = Client("your username", "your API key") @ds.on_delete def on_delete(interaction): print( 'Deleted interaction %s ' % interaction) @ds.on_open def on_open(): print( 'Streaming ready, can start subscribing') csdl = 'interaction.content contains "music"' stream = ds.compile(csdl)['hash'] @ds.subscribe(stream) def subscribe_to_hash(msg): print( msg) @ds.on_closed def on_close(wasClean, code, reason): print( 'Streaming connection closed') @ds.on_ds_message def on_ds_message(msg): print( 'DS Message %s' % msg)
def setUp(self): TestCase.setUp(self) self.client = Client("testuser", "testapikey")
class TestMockedClient(TestCase): def setUp(self): TestCase.setUp(self) self.client = Client("testuser", "testapikey") def test_creation_of_client(self): self.assertTrue(self.client) def test_handling_of_authorization_failed(self): with HTTMock(authorization_failed): self.assertRaises(AuthException, self.client.balance) def test_output_of_balance(self): mock, expected = mock_output_of(self.client.balance) with HTTMock(mock): runs = 0 for expecteditem in expected: runs += 1 results = self.client.balance() assert_dict_structure(self, results, expecteditem) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_rate_limit_exception(self): with HTTMock(rate_limited): self.assertRaises(RateLimitException, self.client.usage) def test_ratelimit_headers(self): with HTTMock(rate_limit_headers): result = self.client.usage() ratelimits = result.ratelimits self.assertNotEqual(len(ratelimits), 0, "ensure that we got some rate limit headers") self.assertEqual(ratelimits["cost"], 25) def test_compile_raw_output(self): mock, expected = mock_output_of(self.client.compile) with HTTMock(mock): runs = 0 for item in expected: runs += 1 assert_dict_structure(self, item, self.client.compile("dummy csdl that is valid").raw) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_compile_python_friendly_output(self): mock, expected = mock_output_of(self.client.compile) with HTTMock(mock): runs = 0 for item in expected: runs += 1 result = self.client.compile("dummy csdl that is valid") assert_dict_structure(self, item, result.raw) assert_dict_structure(self, item, result) self.assertEqual(result["created_at"], datetime.strptime(result.raw["created_at"], "%Y-%m-%d %H:%M:%S")) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_compile_with_valid_output(self): mock, expected = mock_output_of(self.client.compile) with HTTMock(mock): runs = 0 for item in expected: runs += 1 assert_dict_structure(self, item, self.client.compile("dummy csdl that is valid")) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_compile_invalid_csdl(self): with HTTMock(failed_compilation_of_csdl): self.assertRaises(DataSiftApiException, self.client.compile, ("dummy csdl which is bad")) def test_is_valid_csdl_with_bad_data(self): with HTTMock(failed_compilation_of_csdl): self.assertFalse(self.client.is_valid("dummy csdl which is bad")) def test_is_valid_csdl_with_good_data(self): mock, expected = mock_output_of(self.client.validate) with HTTMock(mock): runs = 0 for item in expected: runs+=1 r = self.client.is_valid("dummy csdl which is valid") self.assertTrue(r) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_is_valid_csdl_cause_exception(self): with HTTMock(internal_server_error_with_json): self.assertRaises(DataSiftApiException, self.client.is_valid, ("csdl which turns into a teapot")) def test_error_handling_of_internal_server_errors(self): with HTTMock(internal_server_error): self.assertRaises(DataSiftApiFailure, self.client.balance) def test_error_handling_of_weird_errors(self): with HTTMock(weird_error): self.assertRaises(HTTPError, self.client.validate, ("csdl which turns into a teapot")) def test_client_usage(self): mock, expected = mock_output_of(self.client.usage) with HTTMock(mock): runs = 0 for expected_output in expected: runs += 1 results = self.client.usage() self.assertDictEqual(results.headers, {'content-type': 'application/json'}) assert_dict_structure(self, results, expected_output) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_client_usage_with_parameter(self): mock, expected = mock_output_of(self.client.usage) with HTTMock(mock): runs = 0 for expected_output in expected: runs += 1 results = self.client.usage(period="day") assert_dict_structure(self, results, expected_output) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") def test_client_dpu(self): mock, expected = mock_output_of(self.client.dpu) with HTTMock(mock): runs = 0 for expected_output in expected: runs += 1 results = self.client.dpu("valid stream id") assert_dict_structure(self, results, expected_output) self.assertNotEqual(runs, 0, "ensure that at least one case was tested") @unittest.skipIf(sys.version_info >= (3,0), "Mocking requests does not work correctly on py3") def test_client_pull(self): mock, expected = normal_pull_output() with HTTMock(mock): print("expected", expected) results = self.client.pull("dummy valid subscription id", size=2048, cursor=512) print("got", results) self.assertEquals(results.status_code, 200) self.assertEqual(len(results), len(expected), msg="get the same number of interactions out") self.assertEqual(len(results), len(results.raw), msg="number of messages mangled by conversion") self.assertDictEqual(results.headers, {}) self.assertTrue(results.status_code == 200) for output, expected in zip(results, expected): assert_dict_structure(self, output, expected) def test_live_streaming_exceptions_warn_on_bad_starts(self): self.assertRaises(StreamSubscriberNotStarted, self.client.subscribe, ("hash")) self.client._stream_process_started = True func = self.client.subscribe("hash") self.assertRaises(DeleteRequired, func, ("hash")) def test_live_streaming_client_setup(self): mock, expected = mock_output_of(self.client.compile) with HTTMock(mock): @self.client.on_delete def on_delete(interaction): print( 'Deleted interaction %s ' % interaction) @self.client.on_open def on_open(): print( 'Streaming ready, can start subscribing') csdl = 'interaction.content contains "music"' stream = self.client.compile(csdl)['hash'] @self.client.subscribe(stream) def subscribe_to_hash(msg): print(msg) @self.client.on_closed def on_close(wasClean, code, reason): print('Streaming connection closed') @self.client.on_ds_message def on_ds_message(msg): print('DS Message %s' % msg) self.client._stream_process_started = True self.client.start_stream_subscriber()
# -*- coding: utf8 -*- from __future__ import print_function import time from datasift import Client datasift = Client("your username", "your API key") csdl = 'interaction.content contains "python"' s3_params = { 'bucket': 'apitests', 'directory': 'ruby', 'acl': 'private', 'auth': { 'access_key': 'AKIAIINK5C4FH75RSWNA', 'secret_key': 'F9mLnLoFFGuCNgbMUhdhHmm5YCcNAt/OG32SUhPy' }, 'delivery_frequency': 0, 'max_size': 10485760, 'file_prefix': 'DataSift', } res = datasift.push.validate('s3', s3_params) print(res) stream = datasift.compile(csdl) print('Creating a pull subscription from stream ==> %s' % stream['hash']) print(stream) subscription = datasift.push.create_from_hash(stream['hash'], 'My Python Pull subscription', 'pull', {}) print(subscription)
# -*- coding: utf8 -*- from __future__ import print_function from datasift import Client from datasift.exceptions import DataSiftApiException from concurrent.futures import as_completed client = Client("yourusername", "yourkey", async=True, max_workers=5) csdl = [ 'interaction.content contains "python"', 'this is invalid CSDL', 'language.tag == "en"', 'interaction.content any "foo, bar, baz"', 'interaction.type != "facebook"' ] # build ourselves a dict of future -> input so we know which one failed results = dict([(client.validate(x), x) for x in csdl]) # add callbacks for result in as_completed(results.keys()): try: result.process() print("'%s' was valid CSDL" % results[result]) except DataSiftApiException: print("'%s' was invalid CSDL" % results[result])
from __future__ import print_function import time from datasift import Client datasift = Client("your username", "your API key") stream = datasift.compile('interaction.content contains "datasift"') stream = stream['hash'] start = time.time() - (7200 * 48) # 48hrs ago end_time = start + 3600 print('Check the data coverage for the last two hours') print(datasift.historics.status(start, end_time)) print('Preparing') historics = datasift.historics.prepare(stream, start, end_time, 'My python historics', ['twitter']) print(historics) historics_id = historics['id'] print('Creating push subscription for historics') create_params = { 'name': 'My awesome push subscription', 'initial_status': 'active', 'playback_id': stream, 'start': time.time(), 'end': time.time() + 320 }