Exemplo n.º 1
0
 def get(self):
     
     table = self.request.get("table", None)
     
     response = rules.get_rules(**GNIP_RULES_PARAMS)
     
     FORMAT = "%Y%m%d%H%M"
     end = datetime.now()
     
     # scope by table
     if table:
         
         (project, dataset, table) = Utils.parse_bqid(table)
         tag = Utils.make_tag(dataset, table)
         response = [r for r in response if r['tag'] == tag]
         
         table = Utils.get_bq().tables().get(projectId=project, datasetId=dataset, tableId=table).execute()
         
         end = float(table['creationTime'])
         end = Utils.millis_to_date(end)
         
     start = end - timedelta(days=SEARCH_DAYS)
                     
     for r in response:
         tag = r['tag']
         r['hpt'] = json.dumps({
         "publisher": "twitter", 
         "streamType": "track", 
         "dataFormat": "activity-streams", 
         "fromDate": start.strftime(FORMAT), 
         "toDate": end.strftime(FORMAT),
         "rules": [{"tag": tag, "value": r['value']}], "title": tag})
         
     self.response.headers['Content-Type'] = 'application/json'   
     self.response.out.write(json.dumps(response))
Exemplo n.º 2
0
 def test_get_rules_requests_get_exception(self):
     """ Get rules with exception thrown. """
     try:
         r = rules.get_rules(config_file_path=test_utils.test_config_path)
     except RulesGetFailedException:
         return
     self.fail("rules.get() was supposed to throw a RulesGetFailedException")
Exemplo n.º 3
0
 def test_get_rules_bad_json(self):
     try:
         r = rules.get_rules(config_file_path=test_utils.test_config_path)
     except RulesGetFailedException as e:
         self.assertTrue("GNIP API returned malformed JSON" in str(e))
         return
     self.fail("rules.get() was supposed to throw a RulesGetFailedException")
Exemplo n.º 4
0
 def test_get_rules_no_rules_field_json(self):
     try:
         r = rules.get_rules(config_file_path=test_utils.test_config_path)
     except RulesGetFailedException as e:
         self.assertTrue("GNIP API response did not return a rules object" in str(e))
         return
     self.fail("rules.get() was supposed to throw a RulesGetFailedException")
Exemplo n.º 5
0
 def test_get_rules_bad_status_code(self):
     try:
         r = rules.get_rules(config_file_path=test_utils.test_config_path)
     except RulesGetFailedException as e:
         self.assertTrue("HTTP Status Code" in str(e))
         return
     self.fail("rules.get() was supposed to throw a RulesGetFailedException")
Exemplo n.º 6
0
 def get(self):
     
     table = self.request.get("table", None)
     
     response = rules.get_rules(**GNIP_RULES_PARAMS)
     
     FORMAT = "%Y%m%d%H%M"
     end = datetime.now()
     
     # scope by table
     if table:
         
         (project, dataset, table) = Utils.parse_bqid(table)
         tag = Utils.make_tag(dataset, table)
         response = [r for r in response if r['tag'] == tag]
         
         table = Utils.get_bq().tables().get(projectId=project, datasetId=dataset, tableId=table).execute()
         
         end = float(table['creationTime'])
         end = Utils.millis_to_date(end)
         
     start = end - timedelta(days=SEARCH_DAYS)
                     
     for r in response:
         tag = r['tag']
         r['hpt'] = json.dumps({
         "publisher": "twitter", 
         "streamType": "track", 
         "dataFormat": "activity-streams", 
         "fromDate": start.strftime(FORMAT), 
         "toDate": end.strftime(FORMAT),
         "rules": [{"tag": tag, "value": r['value']}], "title": tag})
         
     self.response.headers['Content-Type'] = 'application/json'   
     self.response.out.write(json.dumps(response))
Exemplo n.º 7
0
 def test_get_rules_bad_json(self):
     try:
         r = rules.get_rules(config_file_path=test_utils.test_config_path)
     except RulesGetFailedException as e:
         self.assertTrue("GNIP API returned malformed JSON" in str(e))
         return
     self.fail(
         "rules.get() was supposed to throw a RulesGetFailedException")
Exemplo n.º 8
0
 def test_get_rules_bad_status_code(self):
     try:
         r = rules.get_rules(config_file_path=test_utils.test_config_path)
     except RulesGetFailedException as e:
         self.assertTrue("HTTP Status Code" in str(e))
         return
     self.fail(
         "rules.get() was supposed to throw a RulesGetFailedException")
Exemplo n.º 9
0
 def test_get_rules_requests_get_exception(self):
     """ Get rules with exception thrown. """
     try:
         r = rules.get_rules(config_file_path=test_utils.test_config_path)
     except RulesGetFailedException:
         return
     self.fail(
         "rules.get() was supposed to throw a RulesGetFailedException")
Exemplo n.º 10
0
 def test_get_rules_no_rules_field_json(self):
     try:
         r = rules.get_rules(config_file_path=test_utils.test_config_path)
     except RulesGetFailedException as e:
         self.assertTrue(
             "GNIP API response did not return a rules object" in str(e))
         return
     self.fail(
         "rules.get() was supposed to throw a RulesGetFailedException")
Exemplo n.º 11
0
 def get(self):
     
     response = None
     
     try:
         response = rules.get_rules(url=GNIP_URL, auth=(GNIP_USERNAME, GNIP_PASSWORD))
     except RulesGetFailedException:
         pass # uh oh
     
     self.response.headers['Content-Type'] = 'application/json'   
     self.response.out.write(json.dumps(response))
Exemplo n.º 12
0
    def get(self):

        tables = []

        if TABLE_CACHE.get("cache", None) == None:

            datasets = Utils.get_bq().datasets().list(
                projectId=config.PROJECT_ID).execute()
            datasets = datasets.get("datasets", None)

            for d in datasets:
                ref = d.get("datasetReference", None)

                bq_tables = Utils.get_bq().tables().list(
                    projectId=ref.get("projectId"),
                    datasetId=ref.get("datasetId")).execute()
                if bq_tables['totalItems'] > 0:
                    for t in bq_tables.get("tables", None):
                        id = t.get("id")
                        ref = t.get("tableReference", None)
                        dataset = ref.get("datasetId", None)
                        table = ref.get("tableId", None)
                        tables.append({
                            "id":
                            id,
                            "projectId":
                            ref.get("projectId", None),
                            "datasetId":
                            dataset,
                            "tableId":
                            table,
                            "type":
                            "Twitter" if "twitter" in dataset else "Gnip"
                        })

            rules_list = rules.get_rules(**GNIP_RULES_PARAMS)
            for t in tables:
                tag = Utils.make_tag(t['datasetId'], t['tableId'])
                rs = [r['value'] for r in rules_list if r['tag'] == tag]
                t['rules'] = rs

            TABLE_CACHE["cache"] = tables

        else:

            tables = TABLE_CACHE.get("cache")

        self.response.headers['Content-Type'] = 'application/json'
        self.response.out.write(json.dumps(tables))
Exemplo n.º 13
0
    def get(self, id):
        
        (project, dataset, table) = Utils.parse_bqid(id)
        
        try:
            response = Utils.get_bq().tables().delete(projectId=project, datasetId=dataset, tableId=table).execute()
            TABLE_CACHE.clear()
        except:
            # OK to ignore here if it's already deleted; continue onto deleting rules
            pass
        
        tag = dataset + "." + table

        rules_list = rules.get_rules(**GNIP_RULES_PARAMS)
        rules_list = [r for r in rules_list if r['tag'] == tag]
        response = rules.delete_rules(rules_list, **GNIP_RULES_PARAMS)
        TABLE_CACHE.clear()

        self.response.headers['Content-Type'] = 'application/json'   
        self.response.out.write(json.dumps(response))
Exemplo n.º 14
0
    def get(self, id):
        
        (project, dataset, table) = Utils.parse_bqid(id)
        
        try:
            response = Utils.get_bq().tables().delete(projectId=project, datasetId=dataset, tableId=table).execute()
            TABLE_CACHE.clear()
        except:
            # OK to ignore here if it's already deleted; continue onto deleting rules
            pass
        
        tag = dataset + "." + table

        rules_list = rules.get_rules(**GNIP_RULES_PARAMS)
        rules_list = [r for r in rules_list if r['tag'] == tag]
        response = rules.delete_rules(rules_list, **GNIP_RULES_PARAMS)
        TABLE_CACHE.clear()

        self.response.headers['Content-Type'] = 'application/json'   
        self.response.out.write(json.dumps(response))
Exemplo n.º 15
0
    def get(self):

        tables = []
    
        if TABLE_CACHE.get("cache", None) == None:
    
            datasets = Utils.get_bq().datasets().list(projectId=config.PROJECT_ID).execute()
            datasets = datasets.get("datasets", None)
            
            for d in datasets:
                ref = d.get("datasetReference", None)
                
                bq_tables = Utils.get_bq().tables().list(projectId=ref.get("projectId"), datasetId=ref.get("datasetId")).execute()
                if bq_tables['totalItems'] > 0:
                    for t in bq_tables.get("tables", None):
                        id = t.get("id")
                        ref = t.get("tableReference", None)
                        dataset = ref.get("datasetId", None)
                        table = ref.get("tableId", None)
                        tables.append({
                            "id": id,
                            "projectId": ref.get("projectId", None),
                            "datasetId": dataset,
                            "tableId": table,
                            "type": "Twitter" if "twitter" in dataset else "Gnip"
                        })
                    
            rules_list = rules.get_rules(**GNIP_RULES_PARAMS)
            for t in tables:
                tag = Utils.make_tag(t['datasetId'], t['tableId'])
                rs = [r['value'] for r in rules_list if r['tag'] == tag]
                t['rules'] = rs
                
            TABLE_CACHE["cache"] = tables
            
        else:
            
            tables = TABLE_CACHE.get("cache")
                
        self.response.headers['Content-Type'] = 'application/json'   
        self.response.out.write(json.dumps(tables))   
Exemplo n.º 16
0
    def get(self):
        
        rule_index = int(self.request.get("index"))
        
        response = None
        
        try:
            rules_list = rules.get_rules(url=GNIP_URL, auth=(GNIP_USERNAME, GNIP_PASSWORD))
            rule_delete = rules_list[rule_index]
            
            print rules_list
            print rule_delete
            
            r = rule_delete
            
            if not isinstance(r, dict):
                print "1"
    
            if "value" not in r:
                print "2"
    
            if not isinstance(r['value'], basestring):
                print "3"
    
            if "tag" in r and not isinstance(r['tag'], basestring):
                print "4"
    
#             for k in r:
#                 if k not in expected:
#                     fail()
            
            response = rules.delete_rule(rule_delete, url=GNIP_URL, auth=(GNIP_USERNAME, GNIP_PASSWORD))
        except RulesGetFailedException:
            pass # uh oh
        
        self.response.headers['Content-Type'] = 'application/json'   
        self.response.out.write(json.dumps(response))
Exemplo n.º 17
0
 def test_get_rules_success_one_rule(self):
     """ Get one rule. """
     r = rules.get_rules(config_file_path=test_utils.test_config_path)
     self.assertEqual(1, len(r))
Exemplo n.º 18
0
 def test_get_rules_success_no_rules(self):
     """ Get rules with empty response. """
     r = rules.get_rules(config_file_path=test_utils.test_config_path)
     self.assertEqual(0, len(r))
Exemplo n.º 19
0
 def test_get_rules_success_no_rules(self):
     r = rules.get_rules(config_file_path=test_utils.test_config_path)
     self.assertEqual(0, len(r))
Exemplo n.º 20
0
 def test_get_rules_success_no_rules(self):
     """ Get rules with empty response. """
     r = rules.get_rules(config_file_path=test_utils.test_config_path)
     self.assertEqual(0, len(r))
Exemplo n.º 21
0
 def test_get_rules_success_one_rule(self):
     r = rules.get_rules(config_file_path=test_utils.test_config_path)
     self.assertEqual(1, len(r))