def test_parse_dates(self):
input_date = "2006-01-15"
d = Date.strptime(input_date, "%Y-%m-%d")
flag = d == Date(2006, 1, 15)
self.assertTrue(
flag, "date parsing failed\n"
" input date: {0:s}\n"
" parsed: {1}".format(input_date, d))
input_date = "12/02/2012"
d = Date.strptime(input_date, "%m/%d/%Y")
flag = d == Date(2012, 12, 2)
self.assertTrue(
flag, "date parsing failed\n"
" input date: {0:s}\n"
" parsed: {1}".format(input_date, d))
d = Date.strptime(input_date, "%d/%m/%Y")
flag = d == Date(2012, 2, 12)
self.assertTrue(
flag, "date parsing failed\n"
" input date: {0:s}\n"
" parsed: {1}".format(input_date, d))
input_date = "20011002"
d = Date.strptime(input_date, "%Y%m%d")
flag = d == Date(2001, 10, 2)
self.assertTrue(
flag, "date parsing failed\n"
" input date: {0:s}\n"
" parsed: {1}".format(input_date, d))
class TimeProvider:
def __init__(self):
logging.debug("TimeProvider __init__")
def is_market_open(self, timezone):
"""
Return True if the market is open, false otherwise
- **timezone**: string representing the timezone
"""
tz = pytz.timezone(timezone)
now_time = datetime.now(tz=tz).strftime("%H:%M")
return BankHolidays().is_work_day(datetime.now(tz=tz)) and Utils.is_between(
str(now_time), ("07:55", "16:35")
)
def get_seconds_to_market_opening(self, from_time):
"""Return the amount of seconds from now to the next market opening,
taking into account UK bank holidays and weekends"""
today_opening = datetime(
year=from_time.year,
month=from_time.month,
day=from_time.day,
hour=8,
minute=0,
second=0,
microsecond=0,
)
if from_time < today_opening and BankHolidays().is_work_day(from_time.date()):
nextMarketOpening = today_opening
else:
# Get next working day
nextWorkDate = BankHolidays().get_next_work_day(date=from_time.date())
nextMarketOpening = datetime(
year=nextWorkDate.year,
month=nextWorkDate.month,
day=nextWorkDate.day,
hour=8,
minute=0,
second=0,
microsecond=0,
)
# Calculate the delta from from_time to the next market opening
return (nextMarketOpening - from_time).total_seconds()
def wait_for(self, time_amount_type, amount=-1):
"""Wait for the specified amount of time.
An TimeAmount type can be specified
"""
if time_amount_type is TimeAmount.NEXT_MARKET_OPENING:
amount = self.get_seconds_to_market_opening(datetime.now())
elif time_amount_type is TimeAmount.SECONDS:
if amount < 0:
raise ValueError("Invalid amount of time to wait for")
logging.info("Wait for {0:.2f} hours...".format(amount / 3600))
time.sleep(amount)
# -------------------------------------------------------------------------------------------------------------
# string to datetime -> calculate -> date string for d days ago
# today is August 13, 10 days ago means August 3
def days_ago(d, start_date=None):
if start_date==None:
date = datetime.datetime.today() - datetime.timedelta(days=d)
else:
date = str_to_date(start_date) - datetime.timedelta(days=d)
return date.strftime("%Y-%m-%d")
def str_to_date(dt):
year, month, day = (int(x) for x in dt.split('-'))
return datetime.date(year, month, day)
# transform to the valid data:
# value "09/2007" to date 2007-09-01.
# value "2006" to date 2016-01-01.
def parse_thisdate(text: str) -> datetime.date:
parts = text.split('/')
if len(parts) == 2:
return datetime.date(int(parts[1]), int(parts[0]), 1)
elif len(parts) == 1:
return datetime.date(int(parts[0]), 1, 1)
else:
assert False, 'Unknown date format'
# creating datetime object from zip
with zipfile.ZipFile(self.fname) as zf:
with zf.open(zf.namelist()[0]) as infile:
header = infile.readline()
datestr, record_count = header.split(b':')
self.month = int(datestr[2:4])
self.day = int(datestr[4:6])
self.year = int(datestr[6:10])
utc_base_time = datetime(self.year, self.month, self.day)
self.base_time = timezone('US/Eastern').localize(utc_base_time).timestamp()
# creating url which contains date
date = pd.datetime.today() - pd.offsets.BDay(1)
date=date.date()
datestr = date.strftime('%Y%m%d')
daystr=str(date.day)
monthstr=str(date.month)
yearstr=str(date.year)
url='http://www.netfonds.no/quotes/exchange.php?'
url=url+'exchange=%s'
url=url+'&at_day=' + daystr
url=url+'&at_month=' +monthstr
url=url+'&at_year=' +yearstr
url=url+'&format=csv'
# date to string to date
# date to string
current_date = Date(2015, 7, 24) # create date object
two_days_later = current_date + 2 # => Date(2015, 7, 26)
str(two_days_later) # => 2015-07-26
current_date + '1M' # => Date(2015, 8, 24)
current_date + Period('1M') # same with previous line # => Date(2015, 8, 24)
current_date.strftime("%Y%m%d") # => '20150724'
Date.strptime('20160115', '%Y%m%d') # => Date(2016, 1, 15)
Date.strptime('2016-01-15', '%Y-%m-%d') # => Date(2016, 1, 15)
Date.from_datetime(datetime.datetime(2015, 7, 24)) # => Date(2015, 7, 24)
# string to date
pd.to_datetime(pd.Series(["Jul 31, 2017","2010-10-01","2016/10/10","2014.06.10"]))
pd.to_datetime(pd.Series(["11 Jul 2018","13.04.2015","30/12/2011"]),dayfirst=True)
# providing a format could increase speed of conversion significantly
pd.to_datetime(pd.Series(["12-11-2010 01:56","11-01-2012 22:10","28-02-2013 14:59"]), format='%d-%m-%Y %H:%M')
import market_calendars as mcal
cal_sse = mcal.get_calendar('China.SSE') # create chinese shanghai stock exchange calendar
cal_sse.adjust_date('20130131') # => datetime.datetime(2013, 1, 31, 0, 0)
cal_sse.adjust_date('20130131', return_string=True) # => '2013-01-31'
cal_sse.adjust_date('2017/10/01') # => datetime.datetime(2017, 10, 9, 0, 0)
cal_sse.adjust_date('2017/10/01', convention=2) # => datetime.datetime(2017, 9, 29, 0, 0)
# reading time stamp
def read_hhmmss(field: str) -> int:
# """Read a HH:MM:SS field and return us since midnight.""
if field != "":
hour = int(field[0:2])
minute = int(field[3:5])
second = int(field[6:8])
return 1000000 * ((3600 * hour) + (60 * minute) + second)
else:
return 0
def read_hhmmssmil(field: str) -> int:
"""Read a HH:MM:SS:MILL field and return us since midnight."""
if field != "":
hour = int(field[0:2])
minute = int(field[3:5])
second = int(field[6:8])
msecs = int(field[9:])
return ((1000000 * ((3600 * hour) + (60 * minute) + second)) +
(1000 * msecs))
else:
return 0
def read_mmddccyy(field: str) -> np.datetime64:
"""Read a MM-DD-CCYY field and return a np.datetime64('D') type."""
if field != "":
month = int(field[0:2])
day = int(field[3:5])
year = int(field[6:10])
return np.datetime64(
datetime.date(year=year, month=month, day=day), 'D')
else:
return np.datetime64(datetime.date(year=1, month=1, day=1), 'D')
def read_ccyymmdd(field: str) -> np.datetime64:
"""Read a CCYYMMDD field and return a np.datetime64('D') type."""
if field != "":
year = int(field[0:4])
month = int(field[4:6])
day = int(field[6:8])
return np.datetime64(
datetime.date(year=year, month=month, day=day), 'D')
else:
return np.datetime64(datetime.date(year=1, month=1, day=1), 'D')
def read_timestamp_msg(dt_tm: str) -> Tuple[np.datetime64, int]:
"""Read a CCYYMMDD HH:MM:SS field."""
if dt_tm != "":
(date_str, time_str) = dt_tm.split(' ')
dt = read_ccyymmdd(date_str)
tm = read_hhmmss(time_str)
return dt, tm
else:
return np.datetime64(datetime.date(year=1, month=1, day=1), 'D'), 0
def read_hist_news_timestamp(dt_tm: str) -> Tuple[np.datetime64, int]:
"""Read a news story time"""
if dt_tm != "":
date_str = dt_tm[0:8]
time_str = dt_tm[8:14]
dt = read_ccyymmdd(date_str)
tm = read_hhmmss_no_colon(time_str)
return dt, tm
else:
return np.datetime64(datetime.date(year=1, month=1, day=1), 'D'), 0
# named granularity into seconds
def granularity_to_time(s):
"""convert a named granularity into seconds.
get value in seconds for named granularities: M1, M5 ... H1 etc.
>>> print(granularity_to_time("M5"))
300
"""
mfact = {
'S': 1,
'M': 60,
'H': 3600,
'D': 86400,
'W': 604800,
}
try:
f, n = re.match("(?P<f>[SMHDW])(?:(?P<n>\d+)|)", s).groups()
n = n if n else 1
return mfact[f] * int(n)
except Exception as e:
raise ValueError(e)
# -------------------------------------------------------------------------------------------------------------
# UTC convertion and timezone
from pytz import timezone
from pytz import utc
# We're using NYSE and NASDAQ, which are both in the easters timezone.
MARKET_TIMEZONE = timezone("US/Eastern")
def utc_to_market_time(self, timestamp):
"""Converts a UTC timestamp to local market time."""
utc_time = utc.localize(timestamp)
market_time = utc_time.astimezone(MARKET_TIMEZONE)
return market_time
def market_time_to_utc(self, timestamp):
"""Converts a timestamp in local market time to UTC."""
market_time = MARKET_TIMEZONE.localize(timestamp)
utc_time = market_time.astimezone(utc)
return utc_time
def as_market_time(self, year, month, day, hour=0, minute=0, second=0):
"""Creates a timestamp in market time."""
market_time = datetime(year, month, day, hour, minute, second)
return MARKET_TIMEZONE.localize(market_time)
# Remove timezone information.
def unlocalize(dateTime):
return dateTime.replace(tzinfo=None)
def localize(dateTime, timeZone):
"""Returns a datetime adjusted to a timezone:
* If dateTime is a naive datetime (datetime with no timezone information), timezone information is added but date
and time remains the same.
* If dateTime is not a naive datetime, a datetime object with new tzinfo attribute is returned, adjusting the date
and time data so the result is the same UTC time.
"""
if datetime_is_naive(dateTime):
ret = timeZone.localize(dateTime)
else:
ret = dateTime.astimezone(timeZone)
return ret
def as_utc(dateTime):
return localize(dateTime, pytz.utc)
def datetime_to_timestamp(dateTime):
""" Converts a datetime.datetime to a UTC timestamp."""
diff = as_utc(dateTime) - epoch_utc
return diff.total_seconds()
def timestamp_to_datetime(timeStamp, localized=True):
""" Converts a UTC timestamp to a datetime.datetime."""
ret = datetime.datetime.utcfromtimestamp(timeStamp)
if localized:
ret = localize(ret, pytz.utc)
return ret
epoch_utc = as_utc(datetime.datetime(1970, 1, 1))
def utc_to_local(self, utc_dt):
utc_dt = datetime.strptime(utc_dt, "%Y-%m-%d %H:%M:%S")
local = utc_dt.replace(tzinfo=timezone.utc).astimezone(tz=None)
return local.strftime("%Y-%m-%d %H:%M:%S")
def utc_to_local(utc_dt):
local = utc_dt.replace(tzinfo=timezone.utc).astimezone(tz=None)
return local.strftime("%Y-%m-%d %H:%M:%S")
utc_dt = datetime.strptime("2018-11-01 01:45:00", "%Y-%m-%d %H:%M:%S")
print(utc_to_local(utc_dt))
# -------------------------------------------------------------------------------------------------------------
# Adjust dates in csv according to the time zone
time_zone_difference = int(args[2])
input_filename = args[1]
output_filename = 'OUT_' + input_filename
with open(output_filename, 'w') as w:
with open(input_filename, 'r') as r:
reader = csv.reader(r, delimiter=';')
for row in reader:
print(row)
new_row = list(row)
ts = datetime.strptime(new_row[0], '%Y%m%d %H%M%S')
ts += timedelta(hours=time_zone_difference)
new_row[0] = ts.strftime('%Y%m%d %H%M%S')
w.write(';'.join(new_row) + '\n')