def test_report_current(mocker, config_dir): mocker.patch('arrow.utcnow', return_value=arrow.get(5000)) watson = Watson( current={'project': 'foo', 'start': 4000}, config_dir=config_dir ) for _ in range(2): report = watson.report( arrow.utcnow(), arrow.utcnow(), current=True, projects=['foo'] ) assert len(report['projects']) == 1 assert report['projects'][0]['name'] == 'foo' assert report['projects'][0]['time'] == pytest.approx(1000) report = watson.report( arrow.utcnow(), arrow.utcnow(), current=False, projects=['foo'] ) assert len(report['projects']) == 0 report = watson.report( arrow.utcnow(), arrow.utcnow(), projects=['foo'] ) assert len(report['projects']) == 0
def test_save_frames_no_change(config_dir, mocker, json_mock): watson = Watson(frames=[[4000, 4010, 'foo', None]], config_dir=config_dir) mocker.patch('builtins.open', mocker.mock_open()) watson.save() assert not json_mock.called
def test_save_frames_no_change(config_dir, mock): watson = Watson(frames=[['abcdefg', 'foo', 0, 10]], config_dir=config_dir) mock.patch('%s.open' % builtins, mock.mock_open()) with mock.patch('json.dump') as json_mock: watson.save() assert not json_mock.called
def test_save_frames_no_change(config_dir): watson = Watson(frames=[[0, 10, "foo", None]], config_dir=config_dir) with mock.patch("%s.open" % builtins, mock.mock_open()): with mock.patch("json.dump") as json_mock: watson.save() assert not json_mock.called
def test_save_frames_no_change(config_dir, mock): watson = Watson(frames=[[0, 10, 'foo', None]], config_dir=config_dir) mock.patch('%s.open' % builtins, mock.mock_open()) json_mock = mock.patch('json.dump') watson.save() assert not json_mock.called
def test_save_empty_last_sync(config_dir, mock, json_mock): watson = Watson(last_sync=arrow.now(), config_dir=config_dir) watson.last_sync = None mock.patch('%s.open' % builtins, mock.mock_open()) watson.save() assert json_mock.call_count == 1 assert json_mock.call_args[0][0] == 0
def test_save_empty_last_sync(config_dir): watson = Watson(last_sync=arrow.now(), config_dir=config_dir) watson.last_sync = None with mock.patch('%s.open' % builtins, mock.mock_open()): with mock.patch('json.dump') as json_mock: watson.save() assert json_mock.call_count == 1 assert json_mock.call_args[0][0] == 0
def test_save_empty_current(config_dir): watson = Watson(current={"project": "foo", "start": 0}, config_dir=config_dir) watson.current = {} with mock.patch("%s.open" % builtins, mock.mock_open()): with mock.patch("json.dump") as json_mock: watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert result == {}
def test_save_empty_current(config_dir): watson = Watson(current={'project': 'foo', 'start': 0}, config_dir=config_dir) watson.current = {} with mock.patch('%s.open' % builtins, mock.mock_open()): with mock.patch('json.dump') as json_mock: watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert result == {}
def test_save_changed_frame(config_dir): watson = Watson(frames=[[0, 10, "foo", None, ["A"]]], config_dir=config_dir) watson.frames[0] = ("bar", 0, 10, ["A", "B"]) with mock.patch("%s.open" % builtins, mock.mock_open()): with mock.patch("json.dump") as json_mock: watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert len(result) == 1 assert result[0][2] == "bar" assert result[0][4] == ["A", "B"]
def test_save_added_frame(config_dir, mock): watson = Watson(frames=[['abcdefg', 'foo', 0, 10]], config_dir=config_dir) watson.frames.add('bar', 10, 20, tags=['A']) mock.patch('%s.open' % builtins, mock.mock_open()) json_mock = mock.patch('json.dump') watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert len(result) == 2 assert result[0][2] == 'foo' assert result[0][4] == [] assert result[1][2] == 'bar' assert result[1][4] == ['A']
def test_save_changed_frame(config_dir): watson = Watson(frames=[[0, 10, 'foo', None, ['A']]], config_dir=config_dir) watson.frames[0] = ('bar', 0, 10, ['A', 'B']) with mock.patch('%s.open' % builtins, mock.mock_open()): with mock.patch('json.dump') as json_mock: watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert len(result) == 1 assert result[0][2] == 'bar' assert result[0][4] == ['A', 'B']
def test_save_added_frame(config_dir, mock, json_mock): watson = Watson(frames=[[4000, 4010, 'foo', None]], config_dir=config_dir) watson.frames.add('bar', 4010, 4020, ['A']) mock.patch('%s.open' % builtins, mock.mock_open()) watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert len(result) == 2 assert result[0][2] == 'foo' assert result[0][4] == [] assert result[1][2] == 'bar' assert result[1][4] == ['A']
def test_save_added_frame(config_dir, mock): watson = Watson(frames=[[0, 10, 'foo', None]], config_dir=config_dir) watson.frames.add('bar', 10, 20, ['A']) mock.patch('%s.open' % builtins, mock.mock_open()) json_mock = mock.patch('json.dump') watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert len(result) == 2 assert result[0][2] == 'foo' assert result[0][4] == [] assert result[1][2] == 'bar' assert result[1][4] == ['A']
def test_current_with_given_state(config_dir, mocker): content = json.dumps({'project': 'foo', 'start': 4000}) watson = Watson(current={'project': 'bar', 'start': 4000}, config_dir=config_dir) mocker.patch('builtins.open', mocker.mock_open(read_data=content)) assert watson.current['project'] == 'bar'
def test_save_changed_frame(config_dir, mock, json_mock): watson = Watson(frames=[[4000, 4010, 'foo', None, ['A']]], config_dir=config_dir) watson.frames[0] = ('bar', 4000, 4010, ['A', 'B']) mock.patch('%s.open' % builtins, mock.mock_open()) watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert len(result) == 1 assert result[0][2] == 'bar' assert result[0][4] == ['A', 'B'] dump_args = json_mock.call_args[1] assert dump_args['ensure_ascii'] is False
def test_last_sync_with_given_state(config_dir, mock): content = json.dumps(123) now = arrow.now() watson = Watson(last_sync=now, config_dir=config_dir) mock.patch('%s.open' % builtins, mock.mock_open(read_data=content)) assert watson.last_sync == now
def test_frames_with_empty_given_state(config_dir, mock): content = json.dumps([['abcdefg', 'foo', 0, 10, ['A']]]) watson = Watson(frames=[], config_dir=config_dir) mock.patch('%s.open' % builtins, mock.mock_open(read_data=content)) assert len(watson.frames) == 0
def test_current_with_given_state(config_dir): content = json.dumps({'project': 'foo', 'start': 0}) watson = Watson(current={'project': 'bar', 'start': 0}, config_dir=config_dir) with mock.patch('%s.open' % builtins, mock.mock_open(read_data=content)): assert watson.current['project'] == 'bar'
def __init__(self, api, file_name): self.api = api self.file_name = file_name self.tweet = {} self.idSelf = 0 self.tweets = [] self.watson = Watson() self.columns = [ 'Keyword', 'Sentiment', 'Score' ] self.loop_counter = 0 self.client = MongoClient() self.db = self.client['phase3'] self.db_collection = self.file_name self.db_data = []
def test_save_changed_frame(config_dir, mock): watson = Watson(frames=[['abcdefg', 'foo', 0, 10, ['A']]], config_dir=config_dir) watson.frames['abcdefg'] = ('bar', 0, 10, ['A', 'B']) mock.patch('%s.open' % builtins, mock.mock_open()) json_mock = mock.patch('json.dump') watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert len(result) == 1 assert result[0][0] == 'abcdefg' assert result[0][1] == 'bar' assert result[0][2] == 0 assert result[0][3] == 10 assert result[0][4] == ['A', 'B']
def test_save_changed_frame(config_dir, mock): watson = Watson(frames=[[0, 10, 'foo', None, ['A']]], config_dir=config_dir) watson.frames[0] = ('bar', 0, 10, ['A', 'B']) mock.patch('%s.open' % builtins, mock.mock_open()) json_mock = mock.patch('json.dump') watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert len(result) == 1 assert result[0][2] == 'bar' assert result[0][4] == ['A', 'B'] dump_args = json_mock.call_args[1] assert dump_args['ensure_ascii'] is False
def __init__(self, input_file=None, output_file=None): # hhhhhhhhhhhhhhhh # print "BBBBBBBb" self.alchemy_options = ['main', 'sentiment', 'emotion'] # ['emotion', 'sentiment', 'main'] self.input_file = input_file self.output_file = output_file if input_file: self.data = self.json_file_to_obj(input_file) self.alchemy = Alchemy() self.oxford = Oxford() self.watson = Watson() # print "BASE" logging.info("Ready to extract")
def start_watson(task): watson = Watson() if watson.is_started and watson.config.getboolean('options', 'stop_on_start'): watson.stop() watson.start( "@".join([ task.get("description", "").strip(), task.get("project", "None").strip()]), [t.strip() for t in task.get("tags") or [] if t.strip() != '' ]) watson.save()
def test_given_frames(config_dir, mock): content = json.dumps([[0, 10, 'foo', None, ['A']]]) watson = Watson(frames=[[0, 10, 'bar', None, ['A', 'B']]], config_dir=config_dir) mock.patch('%s.open' % builtins, mock.mock_open(read_data=content)) assert len(watson.frames) == 1 assert watson.frames[0].project == 'bar' assert watson.frames[0].tags == ['A', 'B']
def test_given_frames(config_dir, mock): content = json.dumps([['abcdefg', 'foo', 0, 10, ['A']]]) watson = Watson(frames=[['abcdefg', 'bar', 0, 10, ['A', 'B']]], config_dir=config_dir) mock.patch('%s.open' % builtins, mock.mock_open(read_data=content)) assert len(watson.frames) == 1 frame = watson.frames.get_by_index(0) assert frame.project == 'bar' assert frame.tags == ['A', 'B']
def __init__(self, input_file=None, output_file=None): self.alchemy_options = ['main', 'sentiment', 'emotion'] # ['emotion', 'sentiment', 'main'] self.input_file = input_file self.output_file = output_file if input_file: self.data = self.json_file_to_obj(input_file) self.alchemy = Alchemy() self.oxford = Oxford() self.watson = Watson() logging.info("Ready to extract")
def test_save_empty_current(config_dir, mock, json_mock): watson = Watson(current={}, config_dir=config_dir) mock.patch('%s.open' % builtins, mock.mock_open()) watson.current = {'project': 'foo', 'start': 4000} watson.save() assert json_mock.call_count == 1 result = json_mock.call_args[0][0] assert result == {'project': 'foo', 'start': 4000, 'tags': []} watson.current = {} watson.save() assert json_mock.call_count == 2 result = json_mock.call_args[0][0] assert result == {}
def test_report_current(config_dir): watson = Watson(current={ 'project': 'foo', 'start': arrow.now().shift(hours=-1) }, config_dir=config_dir) _ = watson.report(arrow.now(), arrow.now(), current=True, projects=['foo']) report = watson.report(arrow.now(), arrow.now(), current=True, projects=['foo']) assert len(report['projects']) == 1 assert report['projects'][0]['name'] == 'foo' assert report['projects'][0]['time'] == pytest.approx(3600, rel=1e-2) report = watson.report(arrow.now(), arrow.now(), current=False, projects=['foo']) assert len(report['projects']) == 0 report = watson.report(arrow.now(), arrow.now(), projects=['foo']) assert len(report['projects']) == 0
class Extractor: def __init__(self, input_file=None, output_file=None): self.alchemy_options = ['main', 'sentiment', 'emotion'] # ['emotion', 'sentiment', 'main'] self.input_file = input_file self.output_file = output_file if input_file: self.data = self.json_file_to_obj(input_file) self.alchemy = Alchemy() self.oxford = Oxford() self.watson = Watson() logging.info("Ready to extract") def json_file_to_obj(self, json_file): try: # preprocess json_data = self.json_file_to_string(json_file) # load obj = json.loads(json_data) return obj except SyntaxError: logging.error("JSON input has invalid format! Bye.") sys.exit(0) return None def json_file_to_string(self, file_name): with open(file_name) as json_file: data = "[" + ",".join([line.strip() for line in json_file]) + "]" return data def remove_from_input(self): ''' Remove the first line of the input file. This is horrible implementation. But hey, it's a hackathon. ''' with open(self.input_file, 'r') as fin: data = fin.read().splitlines(True) with open(self.input_file, 'w') as fout: fout.writelines(data[1:]) def write_to_json(self, article): json_article = json.dumps(article, sort_keys=True) f = open(self.output_file, "a") f.write(json_article + '\n') f.close() def user_extract(self, payload): if not payload['article_link'] or not payload['image_link']: raise Exception('Missing data') new_payload = [{ "image_link": payload['image_link'], "article_link": payload['article_link'], "link_name": None, "love": -1, "likes": -1, "wow": -1, "angry": -1, "haha": -1, "sad": -1 }] extracted = self.extract(data=new_payload) return extracted['articles'][0] def extract(self, data=None, write=False): if data is None: data = self.data if data is None: raise Exception('No data to extract') result_obj = { "articles": [], } for idx, payload in enumerate(data): logging.info("(" + str(idx + 1) + "/" + str(len(data)) + ") Extracting: " + payload['article_link']) # get content and real url using Alchemy API article = {} text, url = self.extract_alchemy_text(payload['article_link']) # assign calues to the article object article['title'] = payload['link_name'] article['text'] = text article['real_url'] = url article['image_link'] = payload['image_link'] article['alchemy'] = self.extract_alchemy_data(text) article['oxford'] = self.extract_oxford_vision_data(payload['image_link']) article['watson'] = self.extract_watson_tone_data(text) article['targets'] = {} # store targets reactions = ['angry', 'haha', 'likes', 'love', 'sad', 'wow'] for r in reactions: article['targets'][r] = payload[r] result_obj['articles'].append(article) if write: logging.info('Saving process...') self.remove_from_input() self.write_to_json(article) logging.info('Done') return result_obj def extract_alchemy_text(self, url): result = self.alchemy.run(url, target='text') return result['text'], result['url'] def extract_alchemy_data(self, text): alchemy_result = self.alchemy.run(text, target='combined', options=self.alchemy_options) return self.convert_to_alchemy_template(alchemy_result) def convert_to_alchemy_template(self, combined): obj = {} for n in ['keywords', 'taxonomy', 'concepts', 'entities', 'docEmotions', 'docSentiment']: if n in combined: obj[n] = combined[n] return obj def extract_oxford_vision_data(self, url): return self.oxford.run(url, target='emotion') def extract_watson_tone_data(self, text): return self.watson.run(text, target='tone_analyzer')
def test_empty_config_dir(): watson = Watson() assert watson._dir == get_app_dir('watson')
#!/usr/bin/env python import arrow import random import os import sys from watson import Watson if not os.environ.get('WATSON_DIR'): sys.exit( "This script will corrupt Watson's data, please set the WATSON_DIR " "environment variable to safely use it for development purpose.") watson = Watson(config_dir=os.environ.get('WATSON_DIR'), frames=None, current=None) projects = [ ("apollo11", ["reactor", "module", "wheels", "steering", "brakes"]), ("hubble", ["lens", "camera", "transmission"]), ("voyager1", ["probe", "generators", "sensors", "antenna"]), ("voyager2", ["probe", "generators", "sensors", "antenna"]), ] now = arrow.now() for date in arrow.Arrow.range('day', now.shift(months=-1), now): if date.weekday() in (5, 6): # Weekend \o/ continue
import arrow import random import os import sys from watson import Watson if not os.environ.get('WATSON_DIR'): sys.exit( "This script will corrupt Watson's data, please set the WATSON_DIR " "environment variable to safely use it for development purpose." ) watson = Watson(config_dir=os.environ.get('WATSON_DIR'), frames=None, current=None) projects = [ ("apollo11", ["reactor", "module", "wheels", "steering", "brakes"]), ("hubble", ["lens", "camera", "transmission"]), ("voyager1", ["probe", "generators", "sensors", "antenna"]), ("voyager2", ["probe", "generators", "sensors", "antenna"]), ] now = arrow.now() for date in arrow.Arrow.range('day', now.replace(months=-1), now): if date.weekday() in (5, 6): # Weekend \o/ continue
import arrow import random from watson import Watson watson = Watson(frames=None, current=None) projects = [ ("apollo11", ["reactor", "module", "wheels", "steering", "brakes"]), ("hubble", ["lens", "camera", "transmission"]), ("voyager1", ["probe", "generators", "sensors", "antenna"]), ("voyager2", ["probe", "generators", "sensors", "antenna"]), ] now = arrow.now() for date in arrow.Arrow.range('day', now.replace(months=-1), now): if date.weekday() in (5, 6): # Weekend \o/ continue start = date.replace(hour=9, minute=random.randint(0, 59), seconds=random.randint(0, 59)) while start.hour < random.randint(16, 19): project, tags = random.choice(projects) frame = watson.frames.add( project, start, start.replace(seconds=random.randint(60, 4 * 60 * 60)),
def test_last_sync_with_empty_given_state(config_dir): content = json.dumps(123) watson = Watson(last_sync=None, config_dir=config_dir) with mock.patch('%s.open' % builtins, mock.mock_open(read_data=content)): assert watson.last_sync == arrow.get(0)
def test_current_with_empty_given_state(config_dir, mock): content = json.dumps({'project': 'foo', 'start': 0}) watson = Watson(current=[], config_dir=config_dir) mock.patch('%s.open' % builtins, mock.mock_open(read_data=content)) assert watson.current == {}
def stop_watson(): watson = Watson() if watson.is_started: watson.stop() watson.save()
from youtube import CustomYouTube from kensho_tests import Kensho from watson import Watson from timeline_types import timeline_types import requests from transcribe import Transcribe import json import re from string import punctuation import ast app = Flask(__name__) CORS(app) api = Api(app) wat = Watson() Kensho = Kensho() transcribe = Transcribe() class Res(Resource): def get(self): pass def post(self): def parse(response, type): str = response.text str = str.replace('{', '[') str = str.replace('}', ']')
def watson(config_dir): return Watson(config_dir=config_dir)