def fetch_group_time_series(self, market_data_request_list):
logger = LoggerManager().getLogger(__name__)
data_frame_agg = None
thread_no = constants.market_thread_no['other']
if market_data_request_list[
0].data_source in constants.market_thread_no:
thread_no = constants.market_thread_no[
market_data_request_list[0].data_source]
if thread_no > 0:
pool = SwimPool().create_pool(
thread_technique=constants.market_thread_technique,
thread_no=thread_no)
# Open the market data downloads in their own threads and return the results
result = pool.map_async(self.fetch_single_time_series,
market_data_request_list)
data_frame_group = result.get()
pool.close()
pool.join()
else:
data_frame_group = []
for md_request in market_data_request_list:
data_frame_group.append(
self.fetch_single_time_series(md_request))
# Collect together all the time series
if data_frame_group is not None:
data_frame_group = [i for i in data_frame_group if i is not None]
# import itertools
# columns = list(itertools.chain.from_iterable([i.columns for i in data_frame_group if i is not None]))
# For debugging!
# import pickle
# import datetime
# pickle.dump(data_frame_group, open(str(datetime.datetime.now()).replace(':', '-').replace(' ', '-').replace(".", "-") + ".p", "wb"))
if data_frame_group is not None:
try:
data_frame_agg = self.calculations.join(data_frame_group,
how='outer')
# Force ordering to be the same!
# data_frame_agg = data_frame_agg[columns]
except Exception as e:
logger.warning(
'Possible overlap of columns? Have you specifed same ticker several times: '
+ str(e))
return data_frame_agg
def fetch_group_time_series(self, market_data_request_list):
logger = LoggerManager().getLogger(__name__)
df_agg = None
thread_no = constants.market_thread_no["other"]
if market_data_request_list[
0].data_source in constants.market_thread_no:
thread_no = constants.market_thread_no[
market_data_request_list[0].data_source]
if thread_no > 0:
pool = SwimPool().create_pool(
thread_technique=constants.market_thread_technique,
thread_no=thread_no)
# Open the market data downloads in their own threads and return
# the results
result = pool.map_async(self.fetch_single_time_series,
market_data_request_list)
df_group = result.get()
pool.close()
pool.join()
else:
df_group = []
for md_request in market_data_request_list:
df_group.append(
self.fetch_single_time_series(md_request))
# Collect together all the time series
if df_group is not None:
df_group = [i for i in df_group if i is not None]
if df_group is not None:
try:
df_agg = self._calculations.join(df_group,
how="outer")
# Force ordering to be the same!
# df_agg = df_agg[columns]
except Exception as e:
logger.warning(
"Possible overlap of columns? Have you specifed same "
"ticker several times: " + str(e))
return df_agg
def fetch_group_time_series(self, market_data_request_list):
data_frame_agg = None
thread_no = DataConstants().market_thread_no['other']
if market_data_request_list[0].data_source in DataConstants(
).market_thread_no:
thread_no = DataConstants().market_thread_no[
market_data_request_list[0].data_source]
if thread_no > 0:
pool = SwimPool().create_pool(
thread_technique=DataConstants().market_thread_technique,
thread_no=thread_no)
# open the market data downloads in their own threads and return the results
result = pool.map_async(self.fetch_single_time_series,
market_data_request_list)
data_frame_group = result.get()
pool.close()
pool.join()
else:
data_frame_group = []
for md_request in market_data_request_list:
data_frame_group.append(
self.fetch_single_time_series(md_request))
# collect together all the time series
if data_frame_group is not None:
data_frame_group = [i for i in data_frame_group if i is not None]
# for debugging!
# import pickle
# import datetime
# pickle.dump(data_frame_group, open(str(datetime.datetime.now()).replace(':', '-').replace(' ', '-').replace(".", "-") + ".p", "wb"))
if data_frame_group is not None:
try:
data_frame_agg = self.calculations.pandas_outer_join(
data_frame_group)
except Exception as e:
self.logger.warning(
'Possible overlap of columns? Have you specifed same ticker several times: '
+ str(e))
return data_frame_agg
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and limitations under the License.
#
# TODO: Needs fixing, given change in APIs
if __name__ == '__main__':
###### below line CRUCIAL when running Windows, otherwise multiprocessing doesn't work! (not necessary on Linux)
from findatapy.util import SwimPool
SwimPool()
from findatapy.market import Market, MarketDataRequest, MarketDataGenerator
market = Market(market_data_generator=MarketDataGenerator())
# Choose run_example (0 = all examples)
# Example 1: bitcoincharts
# Example 2: poloniex (needs to be fixed)
# Example 3: binance
# Example 4: bitfinex
# Example 5: gdax
# Example 6: kraken
#
# Check findatapy/conf/time_series_tickers_list.csv for all possible tickers
# Note we use XBT instead of BTC. Same for XET (ETH) and XLC (LTC).
#
# Copyright 2016-2020 Cuemacro
#
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the
# License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#
# See the License for the specific language governing permissions and limitations under the License.
#
if __name__ == '__main__':
###### below line CRUCIAL when running Windows, otherwise multiprocessing doesn't work! (not necessary on Linux)
from findatapy.util import SwimPool; SwimPool()
from findatapy.market import Market, MarketDataRequest, MarketDataGenerator
market = Market(market_data_generator=MarketDataGenerator())
# Get the first release for GDP and also print the release date of that
md_request = MarketDataRequest(
start_date="01 Jun 2000", # start date (download data over past decade)
data_source='alfred', # use ALFRED/FRED as data source
tickers=['US GDP'], # ticker
fields=['actual-release', 'release-date-time-full'], # which fields to download
vendor_tickers=['GDP'], # ticker (FRED)
vendor_fields=['actual-release', 'release-date-time-full']) # which FRED fields to download
df = market.fetch_market(md_request)
