def test_report_current(mocker, config_dir):
mocker.patch('arrow.utcnow', return_value=arrow.get(5000))
watson = Watson(
current={'project': 'foo', 'start': 4000},
config_dir=config_dir
)
for _ in range(2):
report = watson.report(
arrow.utcnow(), arrow.utcnow(), current=True, projects=['foo']
)
assert len(report['projects']) == 1
assert report['projects'][0]['name'] == 'foo'
assert report['projects'][0]['time'] == pytest.approx(1000)
report = watson.report(
arrow.utcnow(), arrow.utcnow(), current=False, projects=['foo']
)
assert len(report['projects']) == 0
report = watson.report(
arrow.utcnow(), arrow.utcnow(), projects=['foo']
)
assert len(report['projects']) == 0
def test_save_frames_no_change(config_dir, mocker, json_mock):
watson = Watson(frames=[[4000, 4010, 'foo', None]], config_dir=config_dir)
mocker.patch('builtins.open', mocker.mock_open())
watson.save()
assert not json_mock.called
def test_save_frames_no_change(config_dir, mock):
watson = Watson(frames=[['abcdefg', 'foo', 0, 10]], config_dir=config_dir)
mock.patch('%s.open' % builtins, mock.mock_open())
with mock.patch('json.dump') as json_mock:
watson.save()
assert not json_mock.called
def test_save_frames_no_change(config_dir):
watson = Watson(frames=[[0, 10, "foo", None]], config_dir=config_dir)
with mock.patch("%s.open" % builtins, mock.mock_open()):
with mock.patch("json.dump") as json_mock:
watson.save()
assert not json_mock.called
def test_save_frames_no_change(config_dir, mock):
watson = Watson(frames=[[0, 10, 'foo', None]], config_dir=config_dir)
mock.patch('%s.open' % builtins, mock.mock_open())
json_mock = mock.patch('json.dump')
watson.save()
assert not json_mock.called
def test_save_empty_last_sync(config_dir, mock, json_mock):
watson = Watson(last_sync=arrow.now(), config_dir=config_dir)
watson.last_sync = None
mock.patch('%s.open' % builtins, mock.mock_open())
watson.save()
assert json_mock.call_count == 1
assert json_mock.call_args[0][0] == 0
def test_save_frames_no_change(config_dir, mock):
watson = Watson(frames=[[0, 10, 'foo', None]],
config_dir=config_dir)
mock.patch('%s.open' % builtins, mock.mock_open())
json_mock = mock.patch('json.dump')
watson.save()
assert not json_mock.called
def test_save_empty_last_sync(config_dir):
watson = Watson(last_sync=arrow.now(), config_dir=config_dir)
watson.last_sync = None
with mock.patch('%s.open' % builtins, mock.mock_open()):
with mock.patch('json.dump') as json_mock:
watson.save()
assert json_mock.call_count == 1
assert json_mock.call_args[0][0] == 0
def test_save_empty_current(config_dir):
watson = Watson(current={"project": "foo", "start": 0}, config_dir=config_dir)
watson.current = {}
with mock.patch("%s.open" % builtins, mock.mock_open()):
with mock.patch("json.dump") as json_mock:
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert result == {}
def test_save_empty_current(config_dir):
watson = Watson(current={'project': 'foo', 'start': 0},
config_dir=config_dir)
watson.current = {}
with mock.patch('%s.open' % builtins, mock.mock_open()):
with mock.patch('json.dump') as json_mock:
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert result == {}
def test_save_empty_current(config_dir):
watson = Watson(current={'project': 'foo', 'start': 0},
config_dir=config_dir)
watson.current = {}
with mock.patch('%s.open' % builtins, mock.mock_open()):
with mock.patch('json.dump') as json_mock:
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert result == {}
def test_save_changed_frame(config_dir):
watson = Watson(frames=[[0, 10, "foo", None, ["A"]]], config_dir=config_dir)
watson.frames[0] = ("bar", 0, 10, ["A", "B"])
with mock.patch("%s.open" % builtins, mock.mock_open()):
with mock.patch("json.dump") as json_mock:
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert len(result) == 1
assert result[0][2] == "bar"
assert result[0][4] == ["A", "B"]
def test_save_added_frame(config_dir, mock):
watson = Watson(frames=[['abcdefg', 'foo', 0, 10]], config_dir=config_dir)
watson.frames.add('bar', 10, 20, tags=['A'])
mock.patch('%s.open' % builtins, mock.mock_open())
json_mock = mock.patch('json.dump')
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert len(result) == 2
assert result[0][2] == 'foo'
assert result[0][4] == []
assert result[1][2] == 'bar'
assert result[1][4] == ['A']
def test_save_changed_frame(config_dir):
watson = Watson(frames=[[0, 10, 'foo', None, ['A']]],
config_dir=config_dir)
watson.frames[0] = ('bar', 0, 10, ['A', 'B'])
with mock.patch('%s.open' % builtins, mock.mock_open()):
with mock.patch('json.dump') as json_mock:
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert len(result) == 1
assert result[0][2] == 'bar'
assert result[0][4] == ['A', 'B']
def test_save_changed_frame(config_dir):
watson = Watson(frames=[[0, 10, 'foo', None, ['A']]],
config_dir=config_dir)
watson.frames[0] = ('bar', 0, 10, ['A', 'B'])
with mock.patch('%s.open' % builtins, mock.mock_open()):
with mock.patch('json.dump') as json_mock:
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert len(result) == 1
assert result[0][2] == 'bar'
assert result[0][4] == ['A', 'B']
def test_save_added_frame(config_dir, mock, json_mock):
watson = Watson(frames=[[4000, 4010, 'foo', None]], config_dir=config_dir)
watson.frames.add('bar', 4010, 4020, ['A'])
mock.patch('%s.open' % builtins, mock.mock_open())
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert len(result) == 2
assert result[0][2] == 'foo'
assert result[0][4] == []
assert result[1][2] == 'bar'
assert result[1][4] == ['A']
def test_save_added_frame(config_dir, mock):
watson = Watson(frames=[[0, 10, 'foo', None]], config_dir=config_dir)
watson.frames.add('bar', 10, 20, ['A'])
mock.patch('%s.open' % builtins, mock.mock_open())
json_mock = mock.patch('json.dump')
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert len(result) == 2
assert result[0][2] == 'foo'
assert result[0][4] == []
assert result[1][2] == 'bar'
assert result[1][4] == ['A']
def test_current_with_given_state(config_dir, mocker):
content = json.dumps({'project': 'foo', 'start': 4000})
watson = Watson(current={'project': 'bar', 'start': 4000},
config_dir=config_dir)
mocker.patch('builtins.open', mocker.mock_open(read_data=content))
assert watson.current['project'] == 'bar'
def test_save_changed_frame(config_dir, mock, json_mock):
watson = Watson(frames=[[4000, 4010, 'foo', None, ['A']]],
config_dir=config_dir)
watson.frames[0] = ('bar', 4000, 4010, ['A', 'B'])
mock.patch('%s.open' % builtins, mock.mock_open())
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert len(result) == 1
assert result[0][2] == 'bar'
assert result[0][4] == ['A', 'B']
dump_args = json_mock.call_args[1]
assert dump_args['ensure_ascii'] is False
def test_last_sync_with_given_state(config_dir, mock):
content = json.dumps(123)
now = arrow.now()
watson = Watson(last_sync=now, config_dir=config_dir)
mock.patch('%s.open' % builtins, mock.mock_open(read_data=content))
assert watson.last_sync == now
def test_frames_with_empty_given_state(config_dir, mock):
content = json.dumps([['abcdefg', 'foo', 0, 10, ['A']]])
watson = Watson(frames=[], config_dir=config_dir)
mock.patch('%s.open' % builtins, mock.mock_open(read_data=content))
assert len(watson.frames) == 0
def test_current_with_given_state(config_dir):
content = json.dumps({'project': 'foo', 'start': 0})
watson = Watson(current={'project': 'bar', 'start': 0},
config_dir=config_dir)
with mock.patch('%s.open' % builtins, mock.mock_open(read_data=content)):
assert watson.current['project'] == 'bar'
def __init__(self, api, file_name):
self.api = api
self.file_name = file_name
self.tweet = {}
self.idSelf = 0
self.tweets = []
self.watson = Watson()
self.columns = [
'Keyword',
'Sentiment',
'Score'
]
self.loop_counter = 0
self.client = MongoClient()
self.db = self.client['phase3']
self.db_collection = self.file_name
self.db_data = []
def test_save_changed_frame(config_dir, mock):
watson = Watson(frames=[['abcdefg', 'foo', 0, 10, ['A']]],
config_dir=config_dir)
watson.frames['abcdefg'] = ('bar', 0, 10, ['A', 'B'])
mock.patch('%s.open' % builtins, mock.mock_open())
json_mock = mock.patch('json.dump')
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert len(result) == 1
assert result[0][0] == 'abcdefg'
assert result[0][1] == 'bar'
assert result[0][2] == 0
assert result[0][3] == 10
assert result[0][4] == ['A', 'B']
def test_save_changed_frame(config_dir, mock):
watson = Watson(frames=[[0, 10, 'foo', None, ['A']]],
config_dir=config_dir)
watson.frames[0] = ('bar', 0, 10, ['A', 'B'])
mock.patch('%s.open' % builtins, mock.mock_open())
json_mock = mock.patch('json.dump')
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert len(result) == 1
assert result[0][2] == 'bar'
assert result[0][4] == ['A', 'B']
dump_args = json_mock.call_args[1]
assert dump_args['ensure_ascii'] is False
def __init__(self, input_file=None, output_file=None):
# hhhhhhhhhhhhhhhh
# print "BBBBBBBb"
self.alchemy_options = ['main', 'sentiment',
'emotion'] # ['emotion', 'sentiment', 'main']
self.input_file = input_file
self.output_file = output_file
if input_file:
self.data = self.json_file_to_obj(input_file)
self.alchemy = Alchemy()
self.oxford = Oxford()
self.watson = Watson()
# print "BASE"
logging.info("Ready to extract")
def start_watson(task):
watson = Watson()
if watson.is_started and watson.config.getboolean('options', 'stop_on_start'):
watson.stop()
watson.start(
"@".join([
task.get("description", "").strip(),
task.get("project", "None").strip()]),
[t.strip() for t in task.get("tags") or [] if t.strip() != '' ])
watson.save()
def test_given_frames(config_dir, mock):
content = json.dumps([[0, 10, 'foo', None, ['A']]])
watson = Watson(frames=[[0, 10, 'bar', None, ['A', 'B']]],
config_dir=config_dir)
mock.patch('%s.open' % builtins, mock.mock_open(read_data=content))
assert len(watson.frames) == 1
assert watson.frames[0].project == 'bar'
assert watson.frames[0].tags == ['A', 'B']
def test_given_frames(config_dir, mock):
content = json.dumps([['abcdefg', 'foo', 0, 10, ['A']]])
watson = Watson(frames=[['abcdefg', 'bar', 0, 10, ['A', 'B']]],
config_dir=config_dir)
mock.patch('%s.open' % builtins, mock.mock_open(read_data=content))
assert len(watson.frames) == 1
frame = watson.frames.get_by_index(0)
assert frame.project == 'bar'
assert frame.tags == ['A', 'B']
def __init__(self, input_file=None, output_file=None):
self.alchemy_options = ['main', 'sentiment', 'emotion'] # ['emotion', 'sentiment', 'main']
self.input_file = input_file
self.output_file = output_file
if input_file:
self.data = self.json_file_to_obj(input_file)
self.alchemy = Alchemy()
self.oxford = Oxford()
self.watson = Watson()
logging.info("Ready to extract")
def test_save_empty_current(config_dir, mock, json_mock):
watson = Watson(current={}, config_dir=config_dir)
mock.patch('%s.open' % builtins, mock.mock_open())
watson.current = {'project': 'foo', 'start': 4000}
watson.save()
assert json_mock.call_count == 1
result = json_mock.call_args[0][0]
assert result == {'project': 'foo', 'start': 4000, 'tags': []}
watson.current = {}
watson.save()
assert json_mock.call_count == 2
result = json_mock.call_args[0][0]
assert result == {}
def test_report_current(config_dir):
watson = Watson(current={
'project': 'foo',
'start': arrow.now().shift(hours=-1)
},
config_dir=config_dir)
_ = watson.report(arrow.now(), arrow.now(), current=True, projects=['foo'])
report = watson.report(arrow.now(),
arrow.now(),
current=True,
projects=['foo'])
assert len(report['projects']) == 1
assert report['projects'][0]['name'] == 'foo'
assert report['projects'][0]['time'] == pytest.approx(3600, rel=1e-2)
report = watson.report(arrow.now(),
arrow.now(),
current=False,
projects=['foo'])
assert len(report['projects']) == 0
report = watson.report(arrow.now(), arrow.now(), projects=['foo'])
assert len(report['projects']) == 0
class Extractor:
def __init__(self, input_file=None, output_file=None):
self.alchemy_options = ['main', 'sentiment', 'emotion'] # ['emotion', 'sentiment', 'main']
self.input_file = input_file
self.output_file = output_file
if input_file:
self.data = self.json_file_to_obj(input_file)
self.alchemy = Alchemy()
self.oxford = Oxford()
self.watson = Watson()
logging.info("Ready to extract")
def json_file_to_obj(self, json_file):
try:
# preprocess
json_data = self.json_file_to_string(json_file)
# load
obj = json.loads(json_data)
return obj
except SyntaxError:
logging.error("JSON input has invalid format! Bye.")
sys.exit(0)
return None
def json_file_to_string(self, file_name):
with open(file_name) as json_file:
data = "[" + ",".join([line.strip() for line in json_file]) + "]"
return data
def remove_from_input(self):
'''
Remove the first line of the input file.
This is horrible implementation. But hey, it's a hackathon.
'''
with open(self.input_file, 'r') as fin:
data = fin.read().splitlines(True)
with open(self.input_file, 'w') as fout:
fout.writelines(data[1:])
def write_to_json(self, article):
json_article = json.dumps(article, sort_keys=True)
f = open(self.output_file, "a")
f.write(json_article + '\n')
f.close()
def user_extract(self, payload):
if not payload['article_link'] or not payload['image_link']:
raise Exception('Missing data')
new_payload = [{
"image_link": payload['image_link'],
"article_link": payload['article_link'],
"link_name": None,
"love": -1,
"likes": -1,
"wow": -1,
"angry": -1,
"haha": -1,
"sad": -1
}]
extracted = self.extract(data=new_payload)
return extracted['articles'][0]
def extract(self, data=None, write=False):
if data is None:
data = self.data
if data is None:
raise Exception('No data to extract')
result_obj = {
"articles": [],
}
for idx, payload in enumerate(data):
logging.info("(" + str(idx + 1) + "/" + str(len(data)) + ") Extracting: " + payload['article_link'])
# get content and real url using Alchemy API
article = {}
text, url = self.extract_alchemy_text(payload['article_link'])
# assign calues to the article object
article['title'] = payload['link_name']
article['text'] = text
article['real_url'] = url
article['image_link'] = payload['image_link']
article['alchemy'] = self.extract_alchemy_data(text)
article['oxford'] = self.extract_oxford_vision_data(payload['image_link'])
article['watson'] = self.extract_watson_tone_data(text)
article['targets'] = {}
# store targets
reactions = ['angry', 'haha', 'likes', 'love', 'sad', 'wow']
for r in reactions:
article['targets'][r] = payload[r]
result_obj['articles'].append(article)
if write:
logging.info('Saving process...')
self.remove_from_input()
self.write_to_json(article)
logging.info('Done')
return result_obj
def extract_alchemy_text(self, url):
result = self.alchemy.run(url, target='text')
return result['text'], result['url']
def extract_alchemy_data(self, text):
alchemy_result = self.alchemy.run(text, target='combined', options=self.alchemy_options)
return self.convert_to_alchemy_template(alchemy_result)
def convert_to_alchemy_template(self, combined):
obj = {}
for n in ['keywords', 'taxonomy', 'concepts', 'entities', 'docEmotions', 'docSentiment']:
if n in combined:
obj[n] = combined[n]
return obj
def extract_oxford_vision_data(self, url):
return self.oxford.run(url, target='emotion')
def extract_watson_tone_data(self, text):
return self.watson.run(text, target='tone_analyzer')
def test_empty_config_dir():
watson = Watson()
assert watson._dir == get_app_dir('watson')
#!/usr/bin/env python
import arrow
import random
import os
import sys
from watson import Watson
if not os.environ.get('WATSON_DIR'):
sys.exit(
"This script will corrupt Watson's data, please set the WATSON_DIR "
"environment variable to safely use it for development purpose.")
watson = Watson(config_dir=os.environ.get('WATSON_DIR'),
frames=None,
current=None)
projects = [
("apollo11", ["reactor", "module", "wheels", "steering", "brakes"]),
("hubble", ["lens", "camera", "transmission"]),
("voyager1", ["probe", "generators", "sensors", "antenna"]),
("voyager2", ["probe", "generators", "sensors", "antenna"]),
]
now = arrow.now()
for date in arrow.Arrow.range('day', now.shift(months=-1), now):
if date.weekday() in (5, 6):
# Weekend \o/
continue
import arrow
import random
import os
import sys
from watson import Watson
if not os.environ.get('WATSON_DIR'):
sys.exit(
"This script will corrupt Watson's data, please set the WATSON_DIR "
"environment variable to safely use it for development purpose."
)
watson = Watson(config_dir=os.environ.get('WATSON_DIR'),
frames=None,
current=None)
projects = [
("apollo11", ["reactor", "module", "wheels", "steering", "brakes"]),
("hubble", ["lens", "camera", "transmission"]),
("voyager1", ["probe", "generators", "sensors", "antenna"]),
("voyager2", ["probe", "generators", "sensors", "antenna"]),
]
now = arrow.now()
for date in arrow.Arrow.range('day', now.replace(months=-1), now):
if date.weekday() in (5, 6):
# Weekend \o/
continue
import arrow
import random
from watson import Watson
watson = Watson(frames=None, current=None)
projects = [
("apollo11", ["reactor", "module", "wheels", "steering", "brakes"]),
("hubble", ["lens", "camera", "transmission"]),
("voyager1", ["probe", "generators", "sensors", "antenna"]),
("voyager2", ["probe", "generators", "sensors", "antenna"]),
]
now = arrow.now()
for date in arrow.Arrow.range('day', now.replace(months=-1), now):
if date.weekday() in (5, 6):
# Weekend \o/
continue
start = date.replace(hour=9,
minute=random.randint(0, 59),
seconds=random.randint(0, 59))
while start.hour < random.randint(16, 19):
project, tags = random.choice(projects)
frame = watson.frames.add(
project,
start,
start.replace(seconds=random.randint(60, 4 * 60 * 60)),
def test_last_sync_with_empty_given_state(config_dir):
content = json.dumps(123)
watson = Watson(last_sync=None, config_dir=config_dir)
with mock.patch('%s.open' % builtins, mock.mock_open(read_data=content)):
assert watson.last_sync == arrow.get(0)
def test_current_with_empty_given_state(config_dir, mock):
content = json.dumps({'project': 'foo', 'start': 0})
watson = Watson(current=[], config_dir=config_dir)
mock.patch('%s.open' % builtins, mock.mock_open(read_data=content))
assert watson.current == {}
def stop_watson():
watson = Watson()
if watson.is_started:
watson.stop()
watson.save()
from youtube import CustomYouTube
from kensho_tests import Kensho
from watson import Watson
from timeline_types import timeline_types
import requests
from transcribe import Transcribe
import json
import re
from string import punctuation
import ast
app = Flask(__name__)
CORS(app)
api = Api(app)
wat = Watson()
Kensho = Kensho()
transcribe = Transcribe()
class Res(Resource):
def get(self):
pass
def post(self):
def parse(response, type):
str = response.text
str = str.replace('{', '[')
str = str.replace('}', ']')
def watson(config_dir):
return Watson(config_dir=config_dir)
