def get(self):
table = self.request.get("table", None)
response = rules.get_rules(**GNIP_RULES_PARAMS)
FORMAT = "%Y%m%d%H%M"
end = datetime.now()
# scope by table
if table:
(project, dataset, table) = Utils.parse_bqid(table)
tag = Utils.make_tag(dataset, table)
response = [r for r in response if r['tag'] == tag]
table = Utils.get_bq().tables().get(projectId=project, datasetId=dataset, tableId=table).execute()
end = float(table['creationTime'])
end = Utils.millis_to_date(end)
start = end - timedelta(days=SEARCH_DAYS)
for r in response:
tag = r['tag']
r['hpt'] = json.dumps({
"publisher": "twitter",
"streamType": "track",
"dataFormat": "activity-streams",
"fromDate": start.strftime(FORMAT),
"toDate": end.strftime(FORMAT),
"rules": [{"tag": tag, "value": r['value']}], "title": tag})
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response))
def test_get_rules_requests_get_exception(self):
""" Get rules with exception thrown. """
try:
r = rules.get_rules(config_file_path=test_utils.test_config_path)
except RulesGetFailedException:
return
self.fail("rules.get() was supposed to throw a RulesGetFailedException")
def test_get_rules_bad_json(self):
try:
r = rules.get_rules(config_file_path=test_utils.test_config_path)
except RulesGetFailedException as e:
self.assertTrue("GNIP API returned malformed JSON" in str(e))
return
self.fail("rules.get() was supposed to throw a RulesGetFailedException")
def test_get_rules_no_rules_field_json(self):
try:
r = rules.get_rules(config_file_path=test_utils.test_config_path)
except RulesGetFailedException as e:
self.assertTrue("GNIP API response did not return a rules object" in str(e))
return
self.fail("rules.get() was supposed to throw a RulesGetFailedException")
def test_get_rules_bad_status_code(self):
try:
r = rules.get_rules(config_file_path=test_utils.test_config_path)
except RulesGetFailedException as e:
self.assertTrue("HTTP Status Code" in str(e))
return
self.fail("rules.get() was supposed to throw a RulesGetFailedException")
def get(self):
table = self.request.get("table", None)
response = rules.get_rules(**GNIP_RULES_PARAMS)
FORMAT = "%Y%m%d%H%M"
end = datetime.now()
# scope by table
if table:
(project, dataset, table) = Utils.parse_bqid(table)
tag = Utils.make_tag(dataset, table)
response = [r for r in response if r['tag'] == tag]
table = Utils.get_bq().tables().get(projectId=project, datasetId=dataset, tableId=table).execute()
end = float(table['creationTime'])
end = Utils.millis_to_date(end)
start = end - timedelta(days=SEARCH_DAYS)
for r in response:
tag = r['tag']
r['hpt'] = json.dumps({
"publisher": "twitter",
"streamType": "track",
"dataFormat": "activity-streams",
"fromDate": start.strftime(FORMAT),
"toDate": end.strftime(FORMAT),
"rules": [{"tag": tag, "value": r['value']}], "title": tag})
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response))
def test_get_rules_bad_json(self):
try:
r = rules.get_rules(config_file_path=test_utils.test_config_path)
except RulesGetFailedException as e:
self.assertTrue("GNIP API returned malformed JSON" in str(e))
return
self.fail(
"rules.get() was supposed to throw a RulesGetFailedException")
def test_get_rules_bad_status_code(self):
try:
r = rules.get_rules(config_file_path=test_utils.test_config_path)
except RulesGetFailedException as e:
self.assertTrue("HTTP Status Code" in str(e))
return
self.fail(
"rules.get() was supposed to throw a RulesGetFailedException")
def test_get_rules_requests_get_exception(self):
""" Get rules with exception thrown. """
try:
r = rules.get_rules(config_file_path=test_utils.test_config_path)
except RulesGetFailedException:
return
self.fail(
"rules.get() was supposed to throw a RulesGetFailedException")
def test_get_rules_no_rules_field_json(self):
try:
r = rules.get_rules(config_file_path=test_utils.test_config_path)
except RulesGetFailedException as e:
self.assertTrue(
"GNIP API response did not return a rules object" in str(e))
return
self.fail(
"rules.get() was supposed to throw a RulesGetFailedException")
def get(self):
response = None
try:
response = rules.get_rules(url=GNIP_URL, auth=(GNIP_USERNAME, GNIP_PASSWORD))
except RulesGetFailedException:
pass # uh oh
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response))
def get(self):
tables = []
if TABLE_CACHE.get("cache", None) == None:
datasets = Utils.get_bq().datasets().list(
projectId=config.PROJECT_ID).execute()
datasets = datasets.get("datasets", None)
for d in datasets:
ref = d.get("datasetReference", None)
bq_tables = Utils.get_bq().tables().list(
projectId=ref.get("projectId"),
datasetId=ref.get("datasetId")).execute()
if bq_tables['totalItems'] > 0:
for t in bq_tables.get("tables", None):
id = t.get("id")
ref = t.get("tableReference", None)
dataset = ref.get("datasetId", None)
table = ref.get("tableId", None)
tables.append({
"id":
id,
"projectId":
ref.get("projectId", None),
"datasetId":
dataset,
"tableId":
table,
"type":
"Twitter" if "twitter" in dataset else "Gnip"
})
rules_list = rules.get_rules(**GNIP_RULES_PARAMS)
for t in tables:
tag = Utils.make_tag(t['datasetId'], t['tableId'])
rs = [r['value'] for r in rules_list if r['tag'] == tag]
t['rules'] = rs
TABLE_CACHE["cache"] = tables
else:
tables = TABLE_CACHE.get("cache")
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(tables))
def get(self, id):
(project, dataset, table) = Utils.parse_bqid(id)
try:
response = Utils.get_bq().tables().delete(projectId=project, datasetId=dataset, tableId=table).execute()
TABLE_CACHE.clear()
except:
# OK to ignore here if it's already deleted; continue onto deleting rules
pass
tag = dataset + "." + table
rules_list = rules.get_rules(**GNIP_RULES_PARAMS)
rules_list = [r for r in rules_list if r['tag'] == tag]
response = rules.delete_rules(rules_list, **GNIP_RULES_PARAMS)
TABLE_CACHE.clear()
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response))
def get(self, id):
(project, dataset, table) = Utils.parse_bqid(id)
try:
response = Utils.get_bq().tables().delete(projectId=project, datasetId=dataset, tableId=table).execute()
TABLE_CACHE.clear()
except:
# OK to ignore here if it's already deleted; continue onto deleting rules
pass
tag = dataset + "." + table
rules_list = rules.get_rules(**GNIP_RULES_PARAMS)
rules_list = [r for r in rules_list if r['tag'] == tag]
response = rules.delete_rules(rules_list, **GNIP_RULES_PARAMS)
TABLE_CACHE.clear()
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response))
def get(self):
tables = []
if TABLE_CACHE.get("cache", None) == None:
datasets = Utils.get_bq().datasets().list(projectId=config.PROJECT_ID).execute()
datasets = datasets.get("datasets", None)
for d in datasets:
ref = d.get("datasetReference", None)
bq_tables = Utils.get_bq().tables().list(projectId=ref.get("projectId"), datasetId=ref.get("datasetId")).execute()
if bq_tables['totalItems'] > 0:
for t in bq_tables.get("tables", None):
id = t.get("id")
ref = t.get("tableReference", None)
dataset = ref.get("datasetId", None)
table = ref.get("tableId", None)
tables.append({
"id": id,
"projectId": ref.get("projectId", None),
"datasetId": dataset,
"tableId": table,
"type": "Twitter" if "twitter" in dataset else "Gnip"
})
rules_list = rules.get_rules(**GNIP_RULES_PARAMS)
for t in tables:
tag = Utils.make_tag(t['datasetId'], t['tableId'])
rs = [r['value'] for r in rules_list if r['tag'] == tag]
t['rules'] = rs
TABLE_CACHE["cache"] = tables
else:
tables = TABLE_CACHE.get("cache")
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(tables))
def get(self):
rule_index = int(self.request.get("index"))
response = None
try:
rules_list = rules.get_rules(url=GNIP_URL, auth=(GNIP_USERNAME, GNIP_PASSWORD))
rule_delete = rules_list[rule_index]
print rules_list
print rule_delete
r = rule_delete
if not isinstance(r, dict):
print "1"
if "value" not in r:
print "2"
if not isinstance(r['value'], basestring):
print "3"
if "tag" in r and not isinstance(r['tag'], basestring):
print "4"
# for k in r:
# if k not in expected:
# fail()
response = rules.delete_rule(rule_delete, url=GNIP_URL, auth=(GNIP_USERNAME, GNIP_PASSWORD))
except RulesGetFailedException:
pass # uh oh
self.response.headers['Content-Type'] = 'application/json'
self.response.out.write(json.dumps(response))
def test_get_rules_success_one_rule(self):
""" Get one rule. """
r = rules.get_rules(config_file_path=test_utils.test_config_path)
self.assertEqual(1, len(r))
def test_get_rules_success_no_rules(self):
""" Get rules with empty response. """
r = rules.get_rules(config_file_path=test_utils.test_config_path)
self.assertEqual(0, len(r))
def test_get_rules_success_no_rules(self):
r = rules.get_rules(config_file_path=test_utils.test_config_path)
self.assertEqual(0, len(r))
def test_get_rules_success_no_rules(self):
""" Get rules with empty response. """
r = rules.get_rules(config_file_path=test_utils.test_config_path)
self.assertEqual(0, len(r))
def test_get_rules_success_one_rule(self):
r = rules.get_rules(config_file_path=test_utils.test_config_path)
self.assertEqual(1, len(r))
