class TestCheckResponseCode(unittest.TestCase):
def setUp(self):
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/ok',
status_code=200,
)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/fail',
status_code=404,
)
arguments = parse_arguments([
'-p', 'harvestman', '-d', 'http://192.168.124.30', '-P', '6800',
'-s', '10'
])
self.overwatch = Overwatch(arguments)
def test_check_response_code_true(self):
self.overwatch.response = self.session.get('mock://0.0.0.1:6800/ok')
self.assertEqual(self.overwatch.check_response_code(), True)
def test_check_response_code_false(self):
self.overwatch.response = self.session.get('mock://0.0.0.1:6800/fail')
self.assertEqual(self.overwatch.check_response_code(), False)
def setUp(self):
arguments = parse_arguments([
'-p', 'harvestman', '-d', 'http://192.168.124.30', '-P', '6800',
'-s', '10'
])
self.overwatch = Overwatch(arguments)
self.data_file_path = os.path.join(os.getcwd(), 'test_data')
self.json_file_name = 'scrapyd_list_jobs_for_loop_json.json'
self.json_path = self.create_file_path(self.json_file_name)
self.scrapyd_json = open(self.json_path, 'rb').read()
json_dict = json.loads(self.scrapyd_json)
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/listjobs.json?project=harvestman',
json=json_dict,
status_code=200,
)
self.overwatch.response = self.session.get(
'mock://0.0.0.1:6800/listjobs.json?project=harvestman')
self.outliers = self.overwatch.gather_crawl_outliers()
def get_overwatch_wrapper(self):
"""
Instantiate the Overwatch Wrapper
"""
self.overwatch = Overwatch(
api_secret=os.environ['OVERWATCH_API_SECRET'],
name=self.name,
exchange=self.exchange)
def get_dva(battletag):
""" Gather Dva Self-Destruct Stats"""
dva = Overwatch(battletag=battletag, mode='quickplay', hero='dva',
filter='miscellaneous')
results = dva.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def get_hook(battletag):
""" Gathers those mad hook stats"""
roadhog = Overwatch(battletag=battletag, mode='quickplay', hero='roadhog',
filter='hero specific')
results = roadhog.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def most_played_comp(battletag):
""" Gather the data for heroes most played in comp """
play = Overwatch(battletag=battletag, mode='competitive', hero='all',
filter='played')
results = play.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def get_average_stats(battletag):
"""Gather average Overwatch Stats for Bastion Bot"""
average = Overwatch(battletag=battletag, mode='quickplay',
filter='featured')
results = average.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def get_most_comp(battletag):
""" Gather most in game/best competitive Overwatch stats """
most = Overwatch(battletag=battletag, mode='competitive', hero='all',
filter='best')
results = most.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def get_average_comp(battletag):
""" Gather average competitive Overwatch stats """
average = Overwatch(battletag=battletag, mode='competitive',
filter='featured')
results = average.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def get_average_comp(battletag):
""" Gather average competitive Overwatch stats """
average = Overwatch(battletag=battletag,
mode='competitive',
filter='featured')
results = average.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def get_average_stats(battletag):
"""Gather average Overwatch Stats for Bastion Bot"""
average = Overwatch(battletag=battletag,
mode='quickplay',
filter='featured')
results = average.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def get_dva(battletag):
""" Gather Dva Self-Destruct Stats"""
dva = Overwatch(battletag=battletag,
mode='quickplay',
hero='dva',
filter='miscellaneous')
results = dva.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def get_hook(battletag):
""" Gathers those mad hook stats"""
roadhog = Overwatch(battletag=battletag,
mode='quickplay',
hero='roadhog',
filter='hero specific')
results = roadhog.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def get_most_comp(battletag):
""" Gather most in game/best competitive Overwatch stats """
most = Overwatch(battletag=battletag,
mode='competitive',
hero='all',
filter='best')
results = most.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def most_played_comp(battletag):
""" Gather the data for heroes most played in comp """
play = Overwatch(battletag=battletag,
mode='competitive',
hero='all',
filter='played')
results = play.get_results()
descr, stats = parse(results)
for i in range(len(descr)):
yield descr[i], stats[i]
def setUp(self):
arguments = parse_arguments(['-p',
'harvestman',
'-d',
'http://192.168.124.30',
'-P',
'6800',
'-s',
'10'])
self.overwatch = Overwatch(arguments)
self.data_file_path = os.path.join(os.getcwd(), 'test_data')
self.json_file_name = 'scrapyd_list_jobs_for_loop_json.json'
self.json_path = self.create_file_path(self.json_file_name)
self.scrapyd_json = open(self.json_path, 'rb').read()
json_dict = json.loads(self.scrapyd_json)
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/listjobs.json?project=harvestman',
json=json_dict,
status_code=200,
)
self.overwatch.response = self.session.get(
'mock://0.0.0.1:6800/listjobs.json?project=harvestman')
self.outliers = self.overwatch.gather_crawl_outliers()
def setUp(self):
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/ok',
status_code=200,
)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/fail',
status_code=404,
)
arguments = parse_arguments(['-p',
'harvestman',
'-d',
'http://192.168.124.30',
'-P',
'6800',
'-s',
'10'])
self.overwatch = Overwatch(arguments)
def link_account(request, game, api_required_info):
if request.user.is_authenticated:
data = {}
if game == overwatch:
try:
profile = Overwatch(battletag=api_required_info["battletag"])
except:
print("invalid credentials")
return redirect('/games')
return render(request, 'games/{}.html'.format(game))
return redirect('/accounts/login')
def setUp(self):
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/ok',
status_code=200,
)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/fail',
status_code=404,
)
arguments = parse_arguments([
'-p', 'harvestman', '-d', 'http://192.168.124.30', '-P', '6800',
'-s', '10'
])
self.overwatch = Overwatch(arguments)
def setUp(self):
today = datetime.datetime.today().strftime('%d-%m-%Y')
arguments = parse_arguments([
'-p', 'harvestman', '-d', 'http://192.168.124.30', '-P', '6800',
'-s', '50'
])
self.overwatch = Overwatch(arguments)
self.exp_url = 'http://192.168.124.30:6800/listjobs.json?project=harvestman'
filename = '{}_{}.csv'.format(today,
self.overwatch.arguments.project_name[0])
self.output_file = os.path.join(settings.OUTPUT_PATH, filename)
self.temp_dir = tempfile.mkdtemp(prefix='temp_test_dir')
self.data_file_path = os.path.join(os.getcwd(), 'test_data')
self.json_outliers_file_name = 'scrapyd_list_jobs_outliers_json.json'
self.scrapyd_outliers_json_path = self.create_file_path(
self.json_outliers_file_name)
self.scrapyd_outliers_json = open(self.scrapyd_outliers_json_path,
'rb').read()
outliers_json_dict = json.loads(self.scrapyd_outliers_json)
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/listjobs.json?project=harvestman',
json=outliers_json_dict,
status_code=200,
)
self.overwatch.response = self.session.get(
'mock://0.0.0.1:6800/listjobs.json?project=harvestman')
def main():
overwatch = Overwatch(battletag="Serb#11472")
print(getInfo('Lucio', overwatch))
db_file = r'''C:\Users\Liam Edelman\Documents\Personal\OverwatchDSApp\DB.accdb'''
user = '******'
password = ''
odbc_conn_str = 'DRIVER={Microsoft Access Driver (*.mdb, *.accdb)};DBQ=%s;UID=%s;PWD=%s' %\
(db_file, user, password)
print('connecting to db...')
conn = pyodbc.connect(odbc_conn_str)
crsr = conn.cursor()
for table_info in crsr.tables():
print(table_info.table_name)
print('closing db...')
conn.close()
class TestCheckResponseCode(unittest.TestCase):
def setUp(self):
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/ok',
status_code=200,
)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/fail',
status_code=404,
)
arguments = parse_arguments(['-p',
'harvestman',
'-d',
'http://192.168.124.30',
'-P',
'6800',
'-s',
'10'])
self.overwatch = Overwatch(arguments)
def test_check_response_code_true(self):
self.overwatch.response = self.session.get('mock://0.0.0.1:6800/ok')
self.assertEqual(self.overwatch.check_response_code(), True)
def test_check_response_code_false(self):
self.overwatch.response = self.session.get('mock://0.0.0.1:6800/fail')
self.assertEqual(self.overwatch.check_response_code(), False)
class TestGatherCrawlOutliers(unittest.TestCase):
def create_file_path(self, file_name):
path = os.path.join(self.data_file_path, file_name)
return path
def setUp(self):
arguments = parse_arguments(['-p',
'harvestman',
'-d',
'http://192.168.124.30',
'-P',
'6800',
'-s',
'10'])
self.overwatch = Overwatch(arguments)
self.data_file_path = os.path.join(os.getcwd(), 'test_data')
self.json_file_name = 'scrapyd_list_jobs_for_loop_json.json'
self.json_path = self.create_file_path(self.json_file_name)
self.scrapyd_json = open(self.json_path, 'rb').read()
json_dict = json.loads(self.scrapyd_json)
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/listjobs.json?project=harvestman',
json=json_dict,
status_code=200,
)
self.overwatch.response = self.session.get(
'mock://0.0.0.1:6800/listjobs.json?project=harvestman')
self.outliers = self.overwatch.gather_crawl_outliers()
def test_gather_crawl_outliers(self):
expted = {'strt': datetime.datetime(2016, 4, 1, 1, 1, 59, 999999),
'end': datetime.datetime(2016, 4, 1, 23, 59, 59, 999999)}
self.assertEqual(self.outliers, expted)
class TestGatherCrawlOutliers(unittest.TestCase):
def create_file_path(self, file_name):
path = os.path.join(self.data_file_path, file_name)
return path
def setUp(self):
arguments = parse_arguments([
'-p', 'harvestman', '-d', 'http://192.168.124.30', '-P', '6800',
'-s', '10'
])
self.overwatch = Overwatch(arguments)
self.data_file_path = os.path.join(os.getcwd(), 'test_data')
self.json_file_name = 'scrapyd_list_jobs_for_loop_json.json'
self.json_path = self.create_file_path(self.json_file_name)
self.scrapyd_json = open(self.json_path, 'rb').read()
json_dict = json.loads(self.scrapyd_json)
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/listjobs.json?project=harvestman',
json=json_dict,
status_code=200,
)
self.overwatch.response = self.session.get(
'mock://0.0.0.1:6800/listjobs.json?project=harvestman')
self.outliers = self.overwatch.gather_crawl_outliers()
def test_gather_crawl_outliers(self):
expted = {
'strt': datetime.datetime(2016, 4, 1, 1, 1, 59, 999999),
'end': datetime.datetime(2016, 4, 1, 23, 59, 59, 999999)
}
self.assertEqual(self.outliers, expted)
def setUp(self):
today = datetime.datetime.today().strftime('%d-%m-%Y')
arguments = parse_arguments(['-p',
'harvestman',
'-d',
'http://192.168.124.30',
'-P',
'6800',
'-s',
'50'])
self.overwatch = Overwatch(arguments)
self.exp_url = 'http://192.168.124.30:6800/listjobs.json?project=harvestman'
filename = '{}_{}.csv'.format(today,
self.overwatch.arguments.project_name[0])
self.output_file = os.path.join(settings.OUTPUT_PATH, filename)
self.temp_dir = tempfile.mkdtemp(prefix='temp_test_dir')
self.data_file_path = os.path.join(os.getcwd(), 'test_data')
self.json_outliers_file_name = 'scrapyd_list_jobs_outliers_json.json'
self.scrapyd_outliers_json_path = self.create_file_path(
self.json_outliers_file_name)
self.scrapyd_outliers_json = open(self.scrapyd_outliers_json_path,
'rb').read()
outliers_json_dict = json.loads(self.scrapyd_outliers_json)
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/listjobs.json?project=harvestman',
json=outliers_json_dict,
status_code=200,
)
self.overwatch.response = self.session.get(
'mock://0.0.0.1:6800/listjobs.json?project=harvestman')
class TestOverwatch(unittest.TestCase):
def create_file_path(self, file_name):
path = os.path.join(self.data_file_path, file_name)
return path
def setUp(self):
today = datetime.datetime.today().strftime('%d-%m-%Y')
arguments = parse_arguments([
'-p', 'harvestman', '-d', 'http://192.168.124.30', '-P', '6800',
'-s', '50'
])
self.overwatch = Overwatch(arguments)
self.exp_url = 'http://192.168.124.30:6800/listjobs.json?project=harvestman'
filename = '{}_{}.csv'.format(today,
self.overwatch.arguments.project_name[0])
self.output_file = os.path.join(settings.OUTPUT_PATH, filename)
self.temp_dir = tempfile.mkdtemp(prefix='temp_test_dir')
self.data_file_path = os.path.join(os.getcwd(), 'test_data')
self.json_outliers_file_name = 'scrapyd_list_jobs_outliers_json.json'
self.scrapyd_outliers_json_path = self.create_file_path(
self.json_outliers_file_name)
self.scrapyd_outliers_json = open(self.scrapyd_outliers_json_path,
'rb').read()
outliers_json_dict = json.loads(self.scrapyd_outliers_json)
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/listjobs.json?project=harvestman',
json=outliers_json_dict,
status_code=200,
)
self.overwatch.response = self.session.get(
'mock://0.0.0.1:6800/listjobs.json?project=harvestman')
def test__init__(self):
self.assertEqual(self.overwatch.query_url, self.exp_url)
self.assertEqual(self.overwatch.con_spiders, 50)
self.assertEqual(self.overwatch.output_file, self.output_file)
def test_gather_crawl_outliers(self):
expected = {
'strt': datetime.datetime(2016, 4, 29, 10, 28, 8, 4732),
'end': datetime.datetime(2016, 4, 29, 10, 33, 51, 420786)
}
self.assertEqual(self.overwatch.gather_crawl_outliers(), expected)
def test_gather_completed_crawl_count(self):
self.assertEqual(self.overwatch.gather_completed_crawl_count(), 3)
def test_calculate_total_duration(self):
expected = 343.416054
self.assertEqual(self.overwatch.calculate_total_duration(), expected)
def test_gather_crawl_durations(self):
expected = [241.547793, 233.489018, 258.652448]
self.assertEqual(self.overwatch.gather_crawl_durations(), expected)
def test_calculate_av_crawl_duration(self):
expected = 244.56
self.assertEqual(self.overwatch.calculate_av_crawl_duration(),
expected)
def test_calculate_single_crawls_per_hour(self):
expected = 14.72
self.assertEqual(self.overwatch.calculate_single_crawls_per_hour(),
expected)
def test_calculate_est_total_crawls_per_hour(self):
expected = 736
self.assertEqual(self.overwatch.calculate_est_total_crawls_per_hour(),
expected)
def test_calculate_single_crawls_per_day(self):
expected = 353.28
self.assertEqual(self.overwatch.calculate_single_crawls_per_day(),
expected)
def test_calculate_est_total_crawls_per_day(self):
expected = 17664
self.assertEqual(self.overwatch.calculate_est_total_crawls_per_day(),
expected)
def test_calculate_single_crawls_per_week(self):
expected = 2472.96
self.assertEqual(self.overwatch.calculate_single_crawls_per_week(),
expected)
def test_calculate_est_total_crawls_per_week(self):
expected = 123648
self.assertEqual(self.overwatch.calculate_est_total_crawls_per_week(),
expected)
def test_gather_scrapy_metrics(self):
expected = {
'Av CR (S)': 244.56,
'Longest CR (S)': 258.652448,
'Shortest CR (S)': 233.489018,
'Total Duration': 343.416054,
'Single CR p/h': 14.72,
'Max CR p/h': 736,
'Single CR p/d': 353.28,
'Max CR p/d': 17664,
'Single CR p/7d': 2472.96,
'Max CR p/7d': 123648,
'Completed crawls': 3
}
self.assertEqual(self.overwatch.gather_scrapy_metrics(), expected)
def test_write_to_csv(self):
filename = 'test_ouput.csv'
self.overwatch.output_file = os.path.join(self.temp_dir, filename)
os.mknod(self.overwatch.output_file)
self.overwatch.write_to_csv()
with open(self.overwatch.output_file, 'rb') as csvfile:
reader = csv.DictReader(csvfile)
csv_data = [row for row in reader]
self.assertEqual(csv_data[0]['Total Duration'], '343.416054')
self.assertEqual(csv_data[0]['Completed crawls'], '3')
class TestOverwatch(unittest.TestCase):
def create_file_path(self, file_name):
path = os.path.join(self.data_file_path, file_name)
return path
def setUp(self):
today = datetime.datetime.today().strftime('%d-%m-%Y')
arguments = parse_arguments(['-p',
'harvestman',
'-d',
'http://192.168.124.30',
'-P',
'6800',
'-s',
'50'])
self.overwatch = Overwatch(arguments)
self.exp_url = 'http://192.168.124.30:6800/listjobs.json?project=harvestman'
filename = '{}_{}.csv'.format(today,
self.overwatch.arguments.project_name[0])
self.output_file = os.path.join(settings.OUTPUT_PATH, filename)
self.temp_dir = tempfile.mkdtemp(prefix='temp_test_dir')
self.data_file_path = os.path.join(os.getcwd(), 'test_data')
self.json_outliers_file_name = 'scrapyd_list_jobs_outliers_json.json'
self.scrapyd_outliers_json_path = self.create_file_path(
self.json_outliers_file_name)
self.scrapyd_outliers_json = open(self.scrapyd_outliers_json_path,
'rb').read()
outliers_json_dict = json.loads(self.scrapyd_outliers_json)
self.session = requests.Session()
self.adapter = requests_mock.Adapter()
self.session.mount('mock', self.adapter)
self.adapter.register_uri(
'GET',
'mock://0.0.0.1:6800/listjobs.json?project=harvestman',
json=outliers_json_dict,
status_code=200,
)
self.overwatch.response = self.session.get(
'mock://0.0.0.1:6800/listjobs.json?project=harvestman')
def test__init__(self):
self.assertEqual(self.overwatch.query_url, self.exp_url)
self.assertEqual(self.overwatch.con_spiders, 50)
self.assertEqual(self.overwatch.output_file, self.output_file)
def test_gather_crawl_outliers(self):
expected = {'strt': datetime.datetime(2016,
4,
29,
10,
28,
8,
4732),
'end': datetime.datetime(2016,
4,
29,
10,
33,
51,
420786)
}
self.assertEqual(self.overwatch.gather_crawl_outliers(), expected)
def test_gather_completed_crawl_count(self):
self.assertEqual(self.overwatch.gather_completed_crawl_count(), 3)
def test_calculate_total_duration(self):
expected = 343.416054
self.assertEqual(self.overwatch.calculate_total_duration(), expected)
def test_gather_crawl_durations(self):
expected = [241.547793, 233.489018, 258.652448]
self.assertEqual(self.overwatch.gather_crawl_durations(), expected)
def test_calculate_av_crawl_duration(self):
expected = 244.56
self.assertEqual(self.overwatch.calculate_av_crawl_duration(),
expected)
def test_calculate_single_crawls_per_hour(self):
expected = 14.72
self.assertEqual(self.overwatch.calculate_single_crawls_per_hour(),
expected)
def test_calculate_est_total_crawls_per_hour(self):
expected = 736
self.assertEqual(self.overwatch.calculate_est_total_crawls_per_hour(),
expected)
def test_calculate_single_crawls_per_day(self):
expected = 353.28
self.assertEqual(self.overwatch.calculate_single_crawls_per_day(),
expected)
def test_calculate_est_total_crawls_per_day(self):
expected = 17664
self.assertEqual(self.overwatch.calculate_est_total_crawls_per_day(),
expected)
def test_calculate_single_crawls_per_week(self):
expected = 2472.96
self.assertEqual(self.overwatch.calculate_single_crawls_per_week(),
expected)
def test_calculate_est_total_crawls_per_week(self):
expected = 123648
self.assertEqual(self.overwatch.calculate_est_total_crawls_per_week(),
expected)
def test_gather_scrapy_metrics(self):
expected = {
'Av CR (S)': 244.56,
'Longest CR (S)': 258.652448,
'Shortest CR (S)': 233.489018,
'Total Duration': 343.416054,
'Single CR p/h': 14.72,
'Max CR p/h': 736,
'Single CR p/d': 353.28,
'Max CR p/d': 17664,
'Single CR p/7d': 2472.96,
'Max CR p/7d': 123648,
'Completed crawls': 3
}
self.assertEqual(self.overwatch.gather_scrapy_metrics(), expected)
def test_write_to_csv(self):
filename = 'test_ouput.csv'
self.overwatch.output_file = os.path.join(self.temp_dir,
filename)
os.mknod(self.overwatch.output_file)
self.overwatch.write_to_csv()
with open(self.overwatch.output_file, 'rb') as csvfile:
reader = csv.DictReader(csvfile)
csv_data = [row for row in reader]
self.assertEqual(csv_data[0]['Total Duration'], '343.416054')
self.assertEqual(csv_data[0]['Completed crawls'], '3')
class Bot(object):
def __init__(self, name, exchange):
# get a decent logger#
self.logger = self.setup_logging()
# set the bot name and exchange
self.name = name
self.exchange = exchange
# instantiate the Overwatch connection
self.overwatch = None
self.get_overwatch_wrapper()
# get the bot config from Overwatch
self.config = self.overwatch.get_config()
if not self.config:
self.logger.error('Failed to get Overwatch config')
sys.exit(1)
self.symbol = self.config.get('market')
# instantiate the ccxt wrapper for this bots exchange
self.wrapper = None
self.market = None
self.get_exchange_wrapper()
# set the sleep values
self.sleep_short = int(os.environ.get('SLEEP_SHORT', 2))
self.sleep_medium = int(os.environ.get('SLEEP_MEDIUM', 3))
self.sleep_long = int(os.environ.get('SLEEP_LONG', 5))
self.logger.info('Working on {}@{}'.format(self.symbol, self.exchange))
self.logger.info('{}'.format(datetime.datetime.now()))
# get the prices
self.price = 0
self.buy_price = 0
self.sell_price = 0
self.quote_price = None
self.base_price = None
self.peg_price = None
self.get_prices()
# calculate the limits in base currency
self.order_amount = 0
self.total_bid = 0
self.total_ask = 0
self.get_limits()
@staticmethod
def setup_logging():
logger = logging.getLogger()
for h in logger.handlers:
logger.removeHandler(h)
h = logging.StreamHandler(sys.stdout)
h.setFormatter(logging.Formatter('[%(levelname)s] %(message)s'))
logger.addHandler(h)
logger.setLevel(logging.INFO)
return logger
def get_exchange_wrapper(self):
"""
Instantiate the CCXT exchange wrapper
"""
wrapper_class = getattr(ccxt, self.exchange.lower())
self.wrapper = wrapper_class({
'apiKey': os.environ['API_KEY'],
'secret': os.environ['API_SECRET'],
'nonce': ccxt.Exchange.milliseconds
})
self.market = next(m for m in self.wrapper.fetch_markets()
if m.get('symbol') == self.symbol)
def get_overwatch_wrapper(self):
"""
Instantiate the Overwatch Wrapper
"""
self.overwatch = Overwatch(
api_secret=os.environ['OVERWATCH_API_SECRET'],
name=self.name,
exchange=self.exchange)
def get_prices(self):
"""
Get the current sell and buy prices
:return:
"""
self.logger.info('###########')
self.logger.info('Getting Price')
market_price = self.wrapper.fetch_ticker(self.symbol).get('last')
pm = PriceManager()
self.quote_price = pm.get_price(self.market.get('quote'),
self.config.get('quote_price_url'))
self.base_price = pm.get_price(self.market.get('base'),
self.config.get('base_price_url'))
price = None
known_config = False
if self.config.get('use_market_price'):
# just operate using the market price
self.logger.info('Using Market Price')
price = float(market_price)
known_config = True
else:
# otherwise there are 4 PEG options.
# 1. The peg currency is the quote currency: Price = 1/(Quote Price/Base Price)
# 2. The peg currency is the base currency: Price = Base Price / Quote Price
# 3. The peg currency is neither and the quote is being pegged to it: Price = Base Price / Peg Price
# 4. The peg currency is neither and the base is being pegged to it: Price = Peg Price / Quote Price
if self.config.get('peg_currency',
'peg').upper() == self.market.get(
'quote', 'quote').upper():
known_config = True
# this is option 1
if self.quote_price is not None and self.base_price is not None:
price = 1 / (self.quote_price / self.base_price)
if self.config.get('peg_currency',
'peg').upper() == self.market.get(
'base', 'base').upper():
known_config = True
# this is option 2
if self.quote_price is not None and self.base_price is not None:
price = self.base_price / self.quote_price
if (self.config.get('peg_currency', '').upper() != self.market.get(
'base', '').upper()
and self.config.get('peg_currency', '').upper() !=
self.market.get('quote', '').upper()):
self.peg_price = pm.get_price(
self.config.get('peg_currency').upper(),
self.config.get('peg_price_url'))
if self.config.get('peg_side', '').lower() == 'quote':
known_config = True
# this is option 3
if self.base_price is not None and self.peg_price is not None:
price = self.base_price / self.peg_price
self.quote_price = self.peg_price
if self.config.get('peg_side', '').lower() == 'base':
known_config = True
# this is option 4
if self.quote_price is not None and self.peg_price is not None:
price = self.peg_price / self.quote_price
self.base_price = self.peg_price
if not known_config:
self.logger.warning('Not a known Bot Config!')
self.cancel_all_orders()
self.report_balances()
return
if price is None:
self.logger.warning('No Price Available!')
self.cancel_all_orders()
self.report_balances()
return
self.price = price
self.buy_price = float(price) - (
float(self.config.get('fee') + self.config.get('bid_spread')) *
float(price))
self.sell_price = float(price) + (
float(self.config.get('fee') + self.config.get('ask_spread')) *
float(price))
self.logger.info('Buy Price set to {:.8f}'.format(self.buy_price))
self.logger.info('Sell Price set to {:.8f}'.format(self.sell_price))
# send prices to Overwatch
self.overwatch.record_price(price=price,
bid_price=self.buy_price,
ask_price=self.sell_price,
market_price=market_price,
base_price=self.base_price,
quote_price=self.quote_price)
def get_limits(self):
"""
Order amount, Total Ask and total Bid come to the Bot in USD. We need them in 'Base' currency
"""
if self.base_price is not None:
self.order_amount = self.config.get('order_amount',
0) / self.base_price
self.total_ask = self.config.get('total_ask', 0) / self.base_price
self.total_bid = self.config.get('total_bid', 0) / self.base_price
def get_open_orders(self):
"""
get the currently open orders
"""
return self.wrapper.fetch_open_orders(self.symbol)
def get_base(self) -> str:
"""
return the base currency based on if the bot is reversed or not
"""
return self.market.get('base')
def get_quote(self) -> str:
"""
return the quote currency based on if the bot is reversed or not
"""
return self.market.get('quote')
def get_jittered_price(self, price, order_type):
"""
Use the tolerance to calculate a random price +/- the given price
"""
jitter = random.triangular(0, self.config.get('tolerance')) * price
return (price + jitter) if order_type == 'sell' else (price - jitter)
def check_amount(self, amount):
amount_min = self.market.get('limits', {}).get('amount', {}).get('min')
amount_max = self.market.get('limits', {}).get('amount', {}).get('max')
if amount_min:
if amount < amount_min:
self.logger.warning(
'Minimum order amount not reached: {} < {}'.format(
amount, amount_min))
return False
if amount_max:
if amount > amount_max:
self.logger.warning(
'Maximum order amount breached: {} > {}'.format(
amount, amount_max))
return False
return True
def check_price(self, price):
price_min = self.market.get('limits', {}).get('price', {}).get('min')
price_max = self.market.get('limits', {}).get('price', {}).get('max')
if price_min:
if price < price_min:
self.logger.warning(
'Minimum order price not reached: {} < {}'.format(
price, price_min))
return False
if price_max:
if price > price_max:
self.logger.warning(
'Maximum order price breached: {} > {}'.format(
price, price_max))
return False
return True
def check_cost(self, amount, price):
cost = amount * price
cost_min = self.market.get('limits', {}).get('cost', {}).get('min')
cost_max = self.market.get('limits', {}).get('cost', {}).get('max')
if cost_min:
if cost < cost_min:
self.logger.warning(
'Minimum order cost not reached: {} < {}'.format(
cost, cost_min))
return False
if cost_max:
if cost > cost_max:
self.logger.warning(
'Maximum order cost breached: {} > {}'.format(
cost, cost_max))
return False
return True
def place_order(self, order_type, price, amount):
"""
place an order based on the order_type
"""
if not self.check_amount(amount):
return
if not self.check_price(price):
return
if not self.check_cost(amount, price):
return
if order_type == 'buy':
try:
self.logger.info('Placing Buy order of {} @ {}'.format(
amount, price))
place = self.wrapper.create_limit_buy_order(
self.symbol, amount, price)
except Exception as e:
self.logger.error(
'Placing limit buy order failed: {}'.format(e))
return
else:
try:
self.logger.info('Placing Sell order of {} @ {}'.format(
amount, price))
place = self.wrapper.create_limit_sell_order(
self.symbol, amount, price)
except Exception as e:
self.logger.error(
'Placing limit sell order failed: {}'.format(e))
return
if place:
self.overwatch.record_placed_order(self.market.get('base'),
self.market.get('quote'),
order_type, price,
self.order_amount)
self.logger.info('Order Placed: {}'.format(place.get('id')))
else:
# TODO: if order placing fails. alert to vigil
self.logger.error('Failed to place order')
def reset_order(self, order_id, order_type, price):
"""
Cancel the order given by order_id
Place an new order at price
"""
self.logger.info('Resetting order {}'.format(order_id))
# cancel the order
success = self.wrapper.cancel_order(order_id)
# check we've cancelled the order
if not success:
self.logger.error('Unable to cancel order {}'.format(order_id))
return
time.sleep(self.sleep_short)
jittered_price = self.get_jittered_price(price, order_type)
amount = self.order_amount
self.place_order(order_type, jittered_price, amount)
def get_order_total(self):
"""
Get orders from exchange wrapper.
Return the total amount on each side
"""
total = {'sell': 0, 'buy': 0}
for order in self.get_open_orders():
if order.get('side') == 'buy':
total['buy'] += order.get('amount')
else:
total['sell'] += order.get('amount')
return total
def get_available_balance(self, currency):
"""
Return the available balance for the given currency
"""
balances = self.wrapper.fetch_balance()
for cur in balances:
if cur == currency.upper():
return balances.get(cur).get('free', 0.0)
return 0.0
def check_existing_orders(self):
orders = self.get_open_orders()
self.check_existing_side_orders(orders, 'buy', self.buy_price)
self.check_existing_side_orders(orders, 'sell', self.sell_price)
def check_existing_side_orders(self, orders, order_type, price):
"""
Check that existing order prices lie within the pair tolerance
"""
self.logger.info('Checking Existing {} Orders'.format(
order_type.title()))
for order in orders:
if order.get('side') != order_type:
continue
self.logger.info('Checking {} order {}'.format(
order_type.title(), order.get('id')))
order_tolerance = (max(order.get('price'), price) -
min(order.get('price'), price)) / price
self.logger.info('Got an order tolerance of {} against {}'.format(
order_tolerance, self.config.get('tolerance')))
# if the order price is outside of the allowed tolerance
if order_tolerance > self.config.get('tolerance'):
self.reset_order(order.get('id'), order_type.lower(), price)
time.sleep(self.sleep_medium)
def report_balances(self):
"""
Calculate the balances available and on order and report them to Overwatch
"""
self.logger.info('Reporting Balances')
totals_on_order = self.get_order_total()
buy_balance = self.get_available_balance(self.get_quote())
sell_balance = self.get_available_balance(self.get_base())
self.overwatch.record_balances(unit=self.get_base(),
bid_available=buy_balance,
ask_available=sell_balance,
bid_on_order=totals_on_order['buy'],
ask_on_order=totals_on_order['sell'])
def check_total_is_reached(self):
self.check_side_total_is_reached('buy', self.total_ask, self.buy_price)
self.check_side_total_is_reached('sell', self.total_bid,
self.sell_price)
def check_side_total_is_reached(self, side, side_total, price):
"""
Check that the combined amount on order is equal to or more than the target
If the combined amount comes up short,
place a number of orders to reach the target.
(We don't worry too much if the target isn't reached completely,
the bot should place the missing order in 60 seconds time.
We also don't worry if the target is slightly over)
"""
self.logger.info('Checking {} Wall Height'.format(side.title()))
total = self.get_order_total()[side]
target = side_total
step = self.order_amount
self.logger.info('{}: Total = {}, Target = {}, step = {}'.format(
side.title(), total, target, step))
if total < target:
# calculate the difference between current total and the target
difference = target - total
# get the balance Available
check_currency = self.market.get(
'quote') if side == 'buy' else self.market.get('base')
balance = self.get_available_balance(check_currency)
self.logger.info('Got available balance of {} {}'.format(
balance, check_currency))
if side == "buy":
# if we are looking at the sell wall the balance will be in 'quote' currency
# we need to work out how many 'base' currency that is
balance = balance / self.price
self.logger.info('Got working balance of {} {}'.format(
balance, self.market.get('base')))
# we can only place orders up to the value of 'balance'
if balance < difference:
# then warn
self.logger.warning(
'Not enough funds available to reach target')
self.logger.warning('Need {} to reach target of {} '
'but only {:.4f} available'.format(
difference, target, balance))
vigil_alert(alert_channel_id=os.
environ['VIGIL_FUNDS_ALERT_CHANNEL_ID'],
data={
'bot_name':
self.name,
'currency':
self.market.get('base')
if side == 'buy' else self.market.get('quote'),
'exchange':
self.exchange.title(),
'target_amount':
target,
'amount_on_order':
total,
'amount_available':
balance
})
# set difference to == balance
difference = balance
# calculate the number of orders we need to make the total
# use balance instead of step amount if not enough is available
if balance < step:
self.logger.warning(
'Not enough funds to place a full order. Attempting to place available balance'
)
# setting step to balance exactly can cause api errors
step = balance * 0.9
number_of_orders = 0
if step > 0:
number_of_orders = math.ceil(difference / step)
# place the orders needed
if number_of_orders > 0:
self.logger.info(
'Placing {} orders to reach {} target {} from {}'.format(
number_of_orders, side, target, total))
for x in range(number_of_orders):
jittered_price = self.get_jittered_price(price, side)
self.place_order(side, jittered_price, step)
time.sleep(self.sleep_short)
def check_order_prices(self):
orders = self.get_open_orders()
self.check_order_side_prices(orders, 'buy', self.buy_price)
self.check_order_side_prices(orders, 'sell', self.sell_price)
def check_order_side_prices(self, orders, side, price):
"""
For each order, check that the price it is placed at is correct.
Cancel the order if it isn't
"""
self.logger.info(
'Checking Existing {} Order Prices Against {} price {:.8f}'.format(
side.title(), 'Sell' if side == 'sell' else 'Buy', price))
for order in orders:
if order.get('side') != side:
continue
if side == 'buy':
cancel = (float(order['price']) > price)
else:
cancel = (float(order['price']) < price)
if cancel:
self.logger.info('Cancelling {} Order {} @ {}'.format(
side.title(), order['id'], order['price']))
self.wrapper.cancel_order(order.get('id'))
def check_orders_over_target(self):
orders = self.get_open_orders()
self.check_orders_over_target_side(orders, 'buy', self.total_bid,
self.buy_price)
self.check_orders_over_target_side(orders, 'sell', self.total_ask,
self.sell_price)
def check_orders_over_target_side(self, orders, side, side_total, price):
"""
Cancel any orders that take the side total to greater than the target
"""
self.logger.info('Checking For {} Orders Over Target'.format(
side.title()))
total = self.get_order_total()[side]
target = side_total + self.order_amount
self.logger.info('{}: Total = {}, Target = {}'.format(
side.title(), total, target))
if total > target:
# total on side is too high so remove bottom orders
difference = total - target
num = math.floor(difference / self.order_amount)
self.logger.info('Got Diff of {}. Removing {} orders'.format(
difference, num))
remove_orders = sorted(
[o for o in orders if o.get('side') == side],
key=lambda x: x['price'],
reverse=(side == 'sell'))
self.logger.info('Remove {} Orders'.format(len(
remove_orders[:num])))
for order in remove_orders[:num]:
self.logger.info('Cancelling {} Order {}'.format(
side.title(), order.get('id')))
self.wrapper.cancel_order(order.get('id'))
def cancel_all_orders(self):
"""
In an emergency, cancel all the orders
"""
for order in self.wrapper.fetch_open_orders(self.symbol):
self.wrapper.cancel_order(order.get('id'))
time.sleep(self.sleep_short)
def get_trades(self):
"""
Get any new trades and report them to Overwatch
"""
if not self.wrapper.has['fetchMyTrades']:
self.logger.warning(
'fetchMyTrades is not implemented on this exchange')
return
self.logger.info('Getting Trades')
last_trade_id = self.overwatch.get_last_trade_id()
self.logger.info('Got Last trade id : {}'.format(last_trade_id))
for trade in sorted(self.wrapper.fetch_my_trades(self.symbol),
key=lambda x: x['datetime'],
reverse=True):
self.logger.info('Found Trade with ID: {}'.format(trade.get('id')))
if trade.get('id') == last_trade_id:
break
self.overwatch.record_trade(trade.get('datetime'), trade.get('id'),
trade.get('side'), trade.get('price'),
trade.get('amount'), trade.get('cost'),
0)
def run(self):
start_time = time.time()
if self.config.get('stop'):
# we should cancel all the orders
self.logger.warning('STOP SIGNAL RECEIVED')
self.cancel_all_orders()
return
# cancel orders with prices which overlap the calculated price
self.check_order_prices()
time.sleep(self.sleep_short)
# cancel orders placed over side target
self.check_orders_over_target()
time.sleep(self.sleep_short)
# check that existing orders are placed within the price tolerance
self.check_existing_orders()
time.sleep(self.sleep_short)
# check that side targets are reached
self.check_total_is_reached()
time.sleep(self.sleep_short)
# report balances to Overwatch
self.report_balances()
time.sleep(self.sleep_long)
# report new trades to Overwatch
self.get_trades()
self.logger.info('COMPLETE IN {} Seconds'.format(time.time() -
start_time))
def stats(name):
"""Retrieve the stats for Overwatch."""
info = Overwatch(battletag=name, hero='all')
stats = info()
translated = translate_stats(stats)
return simple_stats(translated)
from multiprocessing import JoinableQueue, Array
from libs.controller import Controller
from overwatch import Overwatch
if __name__ == "__main__":
t = time.time()
settings = {
'symbol': 'SP500_M1_TA',
'fraction': [1, 1e4],
'window_size': 100,
'num_workers': 20,
'buffer_size': int(1e3),
'buffer_batch_size': 1,
'normalization': False,
'skewed': True,
'training_epochs': 1,
'verbose': 1,
'start_time': t
}
cycles = 100
schedule = [10, 2]
val = Array('i', [1 for _ in range(settings['num_workers'])])
news_in_q = JoinableQueue()
overwatch = Overwatch(in_q=news_in_q, val=val, settings=settings).start()
controller = Controller(news_q=news_in_q, val=val, settings=settings)
controller.work(cycles, schedule)
controller.deinit()
news_in_q.close()
