def testTzwinTimeOnlyUTCOffset(self):
# For zones with DST, .utcoffset() should return None
tw_est = tz.tzwin('Eastern Standard Time')
self.assertIs(dt_time(14, 10, tzinfo=tw_est).utcoffset(), None)
# This zone has no DST, so .utcoffset() returns standard offset
tw_sast = tz.tzwin('South Africa Standard Time')
self.assertEqual(dt_time(14, 10, tzinfo=tw_sast).utcoffset(),
timedelta(hours=2))
def testTzwinTimeOnlyDST(self):
# For zones with DST, .dst() should return None
tw_est = tz.tzwin('Eastern Standard Time')
self.assertIs(dt_time(14, 10, tzinfo=tw_est).dst(), None)
# This zone has no DST, so .dst() can return 0
tw_sast = tz.tzwin('South Africa Standard Time')
self.assertEqual(dt_time(14, 10, tzinfo=tw_sast).dst(),
timedelta(0))
def testTzwinLocalTimeOnlyDST(self):
# For zones with DST, .dst() should return None
with TZWinContext('Eastern Standard Time'):
twl = tz.tzwinlocal()
self.assertIs(dt_time(14, 10, tzinfo=twl).dst(), None)
# This zone has no DST, so .dst() can return 0
with TZWinContext('South Africa Standard Time'):
twl = tz.tzwinlocal()
self.assertEqual(dt_time(14, 10, tzinfo=twl).dst(), timedelta(0))
def testTzwinTimeOnlyTZName(self):
# For zones with DST, the name defaults to standard time
tw_est = tz.tzwin('Eastern Standard Time')
self.assertEqual(dt_time(14, 10, tzinfo=tw_est).tzname(),
'Eastern Standard Time')
# For zones with no DST, this should work normally.
tw_sast = tz.tzwin('South Africa Standard Time')
self.assertEqual(dt_time(14, 10, tzinfo=tw_sast).tzname(),
'South Africa Standard Time')
def testTzwinLocalTimeOnlyUTCOffset(self):
# For zones with DST, .utcoffset() should return None
with TZWinContext('Eastern Standard Time'):
twl = tz.tzwinlocal()
self.assertIs(dt_time(14, 10, tzinfo=twl).utcoffset(), None)
# This zone has no DST, so .utcoffset() returns standard offset
with TZWinContext('South Africa Standard Time'):
twl = tz.tzwinlocal()
self.assertEqual(dt_time(14, 10, tzinfo=twl).utcoffset(),
timedelta(hours=2))
def testTzwinLocalTimeOnlyTZName(self):
# For zones with DST, the name defaults to standard time
with TZWinContext('Eastern Standard Time'):
twl = tz.tzwinlocal()
self.assertEqual(dt_time(14, 10, tzinfo=twl).tzname(),
'Eastern Standard Time')
# For zones with no DST, this should work normally.
with TZWinContext('South Africa Standard Time'):
twl = tz.tzwinlocal()
self.assertEqual(dt_time(14, 10, tzinfo=twl).tzname(),
'South Africa Standard Time')
def parse_iso_time(value):
#NEEDS-TEST
match = ISO_TIME_RE.match(value)
if not match:
raise InvalidFormat('invalid ISO-8601 time: "{}"'.format(value))
# split out into time, secs, usecs, and tz
tmstr = match.group(1)
secs = match.group(3)
usecs = match.group(4)
tzstr = match.group(5)
try:
dt_args = time.strptime(tmstr, '%H:%M')[3:5]
except ValueError:
raise InvalidDate('invalid time: "{}"'.format(value))
# append seconds, usecs, and tz
dt_args += (int(secs) if secs else 0,)
dt_args += (int(usecs) if usecs else 0,)
dt_args += (parse_tz(tzstr),)
try:
return dt_time(*dt_args)
except ValueError:
raise InvalidDate('invalid date: "{}"'.format(value))
def setup_logging(log_file):
# TODO: more advanced filters, logging info like when rooms go live to console
# https://docs.python.org/3/library/logging.config.html#logging-config-dictschema
log_backup_time = dt_time(tzinfo=TOKYO_TZ)
log_filter = logging.Filter(name="showroom")
file_log_handler = TimedRotatingFileHandler(log_file, encoding='utf8',
when='midnight', atTime=log_backup_time)
file_log_formatter = logging.Formatter(fmt='%(asctime)s %(name)-12s %(levelname)-8s %(threadName)s:\n%(message)s',
datefmt='%m-%d %H:%M:%S')
file_log_handler.setFormatter(file_log_formatter)
# leave this in local time?
file_log_handler.addFilter(log_filter)
file_log_handler.setLevel(logging.DEBUG)
console_handler = logging.StreamHandler()
console_formatter = logging.Formatter(fmt='%(asctime)s %(message)s', datefmt=HHMM_FMT)
console_formatter.converter = tokyotime
console_handler.setLevel(logging.INFO)
console_handler.setFormatter(console_formatter)
console_handler.addFilter(log_filter)
logger = logging.getLogger('showroom')
logger.setLevel(logging.DEBUG)
logger.propagate = False
# at this moment, there shouldn't be any handlers in the showroom logger
# however, i can't preclude the possibility of there ever being such handlers
for handler in (file_log_handler, console_handler):
if handler not in logger.handlers:
logger.addHandler(handler)
def get_strongholds_state(self, callback, clan_id, fields = None):
"""
return data from WGCCFE backend using `stronghold state API method`_
.. _stronghold state API method: http://rtd.wargaming.net/docs/wgccfe/en/latest/rst/
strongholds.html#strongholds-state
"""
url = '/strongholds/state/'
get_params = {'clan_id': clan_id}
return self._request_data(callback, url, get_data=get_params, converters={'clan_id': int,
'defence_hour': lambda x: (dt_time(x, 0) if x >= 0 else None)})
def parse(datetime_string, localize=True):
_utc_to_local = utc_to_local if localize else lambda x: x
def _to_int(value):
if value is None:
return 0
return int(value)
# match time only '00:45:10'
time_only_match = __RE_MATCH_TIME_ONLY__.match(datetime_string)
if time_only_match:
return _utc_to_local(datetime.combine(date.today(),
dt_time(hour=_to_int(time_only_match.group('hour')),
minute=_to_int(time_only_match.group('minute')),
second=_to_int(time_only_match.group('second'))))
).time()
# match date only '2014-11-08'
date_only_match = __RE_MATCH_DATE_ONLY__.match(datetime_string)
if date_only_match:
return _utc_to_local(date(_to_int(date_only_match.group('year')),
_to_int(date_only_match.group('month')),
_to_int(date_only_match.group('day'))))
# full date time
date_time_match = __RE_MATCH_DATETIME__.match(datetime_string)
if date_time_match:
return _utc_to_local(datetime(_to_int(date_time_match.group('year')),
_to_int(date_time_match.group('month')),
_to_int(date_time_match.group('day')),
_to_int(date_time_match.group('hour')),
_to_int(date_time_match.group('minute')),
_to_int(date_time_match.group('second'))))
# period - at the moment we support only hours, minutes and seconds (e.g. videos and audio)
period_match = __RE_MATCH_PERIOD__.match(datetime_string)
if period_match:
return timedelta(hours=_to_int(period_match.group('hours')),
minutes=_to_int(period_match.group('minutes')),
seconds=_to_int(period_match.group('seconds')))
# abbreviated match
abbreviated_match = __RE_MATCH_ABBREVIATED__.match(datetime_string)
if abbreviated_match:
month = {'Jan': 1, 'Feb': 2, 'Mar': 3, 'Apr': 4, 'May': 5, 'June': 6, 'Jun': 6, 'July': 7, 'Jul': 7, 'Aug': 8,
'Sept': 9, 'Sep': 9, 'Oct': 10, 'Nov': 11, 'Dec': 12}
return _utc_to_local(datetime(year=_to_int(abbreviated_match.group('year')),
month=month[abbreviated_match.group('month')],
day=_to_int(abbreviated_match.group('day')),
hour=_to_int(abbreviated_match.group('hour')),
minute=_to_int(abbreviated_match.group('minute')),
second=_to_int(abbreviated_match.group('second'))))
raise KodionException("Could not parse iso 8601 timestamp '%s'" % datetime_string)
def determine_break_from_interview_time(time_period, interview_type):
if interview_type != models.InterviewType.ON_SITE:
return None
weekday = time_period.start_time.weekday()
if weekday != 4: # Friday
return None
tz_info = time_period.start_time.tzinfo
start_of_break = dt_time(12, 0, tzinfo=tz_info)
end_of_break = dt_time(13, 30, tzinfo=tz_info)
date = time_period.start_time.date()
start_of_break_dt = datetime.combine(date, start_of_break)
end_of_break_dt = datetime.combine(date, end_of_break)
break_time_period = TimePeriod(
start_of_break_dt,
end_of_break_dt,
)
if time_period.contains(break_time_period):
print "Adding break %s" % break_time_period
return break_time_period
return None
def cbNotificationTimeChanged(self, configElement=None):
print "[Birthday Reminder] Changing timer times..."
timeList = config.plugins.birthdayreminder.notificationTime.value
notifyTime = dt_time(timeList[0], timeList[1])
for timer in self.timer_list:
day = date.fromtimestamp(timer.begin)
newDateTime = datetime.combine(day, notifyTime)
timer.begin = int(mktime(newDateTime.timetuple()))
timer.end = timer.begin - 1
self.calcNextActivation()
def get_day_end(day, tzinfo=None):
"""Return the latest datetime for a given day.
:param day: A `date` or `datetime`.
:param tzinfo: The timezone to display the resulting datetime. Not valid for
non-naive `datetime` objects.
"""
if isinstance(day, datetime):
if day.tzinfo and tzinfo:
raise ValueError("datetime is not naive.")
tzinfo = day.tzinfo
day = day.date()
end_dt = datetime.combine(day, dt_time(23, 59))
return tzinfo.localize(end_dt) if tzinfo else end_dt
def parse_isotime(st):
"""
st - string or Unicode with ISO 8601 time
"""
m = TIME_PAT.match(st)
if not m:
return None
gd = m.groupdict('0')
#FIXME: does not handle time zones
t = dt_time(int(gd['hour']),
int(gd['minute']),
int(gd['second']),
int(float(u'.' + gd['fract_second'])*1000000),
)
return t
def get_signups_per_day_for_range(day_from, day_until, still_active=False):
"""
Return the signups per day for a range of days. Limited to accounts that
are still active, if "still_active" is True.
"""
noon = dt_time(12, 0)
d1 = datetime.combine(day_from, noon).replace(tzinfo=pytz.utc)
d2 = datetime.combine(day_until, noon).replace(tzinfo=pytz.utc)
qs = User.objects.filter(date_joined__gte=d1, date_joined__lte=d2)
if still_active:
qs = qs.filter(is_active=True, last_login__isnull=False)
qs = qs.extra({'day': "date(date_joined)"}).values('day')
qs = qs.annotate(Count('id')).order_by('day')
return qs
def _get_row(self, row, field, convert, token_type):
# id 'column' is expected first
id = row[0]
# and raw token 'lookup' second
raw = row[1]
if type(raw) is date:
# force convert dates into datetimes... otherwise mongo barfs
raw = datetime.combine(raw, dt_time()).replace(tzinfo=UTC)
# convert based on driver defined conversion method
# and cast to appropriate data type
if convert:
tokens = convert(self, raw)
else:
tokens = raw
tokens = type_cast(tokens, token_type)
return {'id': id, 'field': field, 'tokens': tokens}
def format_pretty_date(dt, locale=None, tzinfo=None):
"""Format a date in a pretty way using relative units if possible.
:param dt: a date or datetime object. if a date is provided, its
time is assumed to be midnight
:param locale: the locale to use for formatting
:param tzinfo: the timezone to use
"""
if not isinstance(dt, datetime):
dt = datetime.combine(dt, dt_time())
return _format_pretty_datetime(dt, locale, tzinfo, {
'last_day': _(u"'Yesterday'"),
'same_day': _(u"'Today'"),
'next_day': _(u"'Tomorrow'"),
'last_week': _(u"'Last' EEEE"),
'next_week': _(u"EEEE"),
'other': _(u"{date_fmt}")
})
def addTimer(self, entry, preremind=False):
if preremind:
print "[Birthday Reminder] Adding preremind timer for", entry[0]
else:
print "[Birthday Reminder] Adding birthday timer for", entry[0]
timeList = config.plugins.birthdayreminder.notificationTime.value
notifyTime = dt_time(timeList[0], timeList[1])
now = date.today()
bDay = entry[1]
if preremind:
numDays = int(config.plugins.birthdayreminder.preremind.getValue())
# set timer to feb 28th for birthdays on feb 29th
try:
dateThisYear = date(now.year, bDay.month, bDay.day) - timedelta(numDays)
except ValueError: # raised on feb 29th
dateThisYear = date(now.year, bDay.month, bDay.day - 1) - timedelta(numDays)
else:
# set timer to feb 28th for birthdays on feb 29th
try:
dateThisYear = date(now.year, bDay.month, bDay.day)
except ValueError: # raised on feb 29th
dateThisYear = date(now.year, bDay.month, bDay.day - 1)
dateTimeThisYear = datetime.combine(dateThisYear, notifyTime)
if dateThisYear >= now: # check if the birthday is in this year
begin = int(mktime(dateTimeThisYear.timetuple()))
else: # birthday is in the past, we need a timer for the next year
# set timer to feb 28th for birthdays on feb 29th
try:
bDayNextYear = dateTimeThisYear.replace(year=dateThisYear.year + 1)
except ValueError: # raised on feb 29th
bDayNextYear = dateTimeThisYear.replace(year=dateThisYear.year + 1, day=dateThisYear.day - 1)
begin = int(mktime(bDayNextYear.timetuple()))
end = begin - 1
timerEntry = BirthdayTimerEntry(begin, end, preremind)
timerEntry.bDay = entry
self.addTimerEntry(timerEntry)
def create_plane_journey_from_flightradar_data(airports, departure_date):
"""
We create a fake plane journey with only the approximate eqCO2 to be used in the computation in the front end
:param query:
:return: fake_journey
"""
day_of_week = departure_date.weekday()
hour_of_day = departure_date.hour
relevant_flights = _FLIGHTRADAR_DATA[
_FLIGHTRADAR_DATA.city_sky.isin(airports['departure'])
& _FLIGHTRADAR_DATA.city_sky_arr.isin(airports['arrival'])]
relevant_flights = relevant_flights[relevant_flights.day_of_week ==
day_of_week]
relevant_flights['hour_dep'] = relevant_flights.apply(
lambda x: dt.strptime(x.hour_dep, '%H:%M:%S') + timedelta(hours=1),
axis=1)
relevant_flights['hour_dep_int'] = relevant_flights.apply(
lambda x: x.hour_dep.hour, axis=1)
response_flights = pd.DataFrame()
for airport_dep in airports['departure']:
for airport_arr in airports['arrival']:
flights_df = relevant_flights[
(relevant_flights.city_sky == airport_dep)
& (relevant_flights.city_sky_arr == airport_arr) &
(relevant_flights.hour_dep_int >= hour_of_day)]
response_flights = response_flights.append(flights_df)
# distance_m = distance(geoloc_dep, geoloc_arrival).m
response_flights['local_range_km'] = response_flights.apply(
lambda x: get_range_km(x.distance_m), axis=1)
response_flights['local_emissions'] = response_flights.apply(
lambda x: calculate_co2_emissions(
constants.TYPE_PLANE, constants.DEFAULT_CITY, constants.
DEFAULT_FUEL, constants.NB_SEATS_TEST, x.local_range_km) *
constants.DEFAULT_NB_PASSENGERS * x.distance_m,
axis=1)
# merge global departure date and flight time to create flight actual departure datetime
response_flights['flight_departure_date'] = response_flights.apply(
lambda x: dt.combine(departure_date,
dt_time(x.hour_dep.hour, x.hour_dep.minute)),
axis=1)
response_flights['flight_arrival_date'] = response_flights.apply(
lambda x: x.flight_departure_date + timedelta(seconds=x.flight_time_s),
axis=1)
journey_list = list()
for index, flight in response_flights.iterrows():
lst_sections = list()
# We add a waiting period at the airport of x hours
step = tmw.Journey_step(
0,
_type=constants.TYPE_WAIT,
label=
f'Arrive at the airport {format_timespan(_AIRPORT_WAITING_PERIOD)} before departure',
distance_m=0,
duration_s=_AIRPORT_WAITING_PERIOD,
price_EUR=[],
gCO2=0,
departure_point=[flight.latitude, flight.longitude],
arrival_point=[flight.latitude, flight.longitude],
departure_date=flight.flight_departure_date -
timedelta(seconds=_AIRPORT_WAITING_PERIOD),
arrival_date=flight.flight_departure_date,
geojson=[],
)
lst_sections.append(step)
step = tmw.Journey_step(
1,
_type=constants.TYPE_PLANE,
label=f'Flight {flight.flight_number} to {flight.airport_to_code}',
distance_m=flight.distance_m,
duration_s=flight.flight_time_s,
price_EUR=[],
gCO2=flight.local_emissions,
departure_point=[flight.latitude, flight.longitude],
arrival_point=[flight.latitude_arr, flight.longitude_arr],
departure_stop_name=flight.airport_from,
arrival_stop_name=flight.airport_to_code,
departure_date=flight.flight_departure_date,
arrival_date=flight.flight_arrival_date,
trip_code=flight.flight_number,
geojson=[],
)
lst_sections.append(step)
departure_date_formated = dt.strptime(
str(lst_sections[0].departure_date)[0:10], '%Y-%m-%d')
departure_date_formated = str(departure_date_formated.year)[2:4]+\
('0'+str(departure_date_formated.month))[-2:]+\
('0'+str(departure_date_formated.day))[-2:]
journey_flightradar = tmw.Journey(
0,
steps=lst_sections,
departure_date=lst_sections[0].departure_date,
arrival_date=lst_sections[-1].arrival_date,
booking_link=
f'https://www.skyscanner.fr/transport/vols/{flight.airport_from}/{flight.airport_to_code}/{departure_date_formated}/'
)
journey_flightradar.category = [constants.TYPE_PLANE]
journey_flightradar.update()
journey_flightradar.is_real_journey = False
journey_list.append(journey_flightradar)
return journey_list
def __init__(self, interval, ticker, kwargs):
if (interval % 5 != 0) or (7200 % interval != 0):
raise ValueError(
'interval must be a multiple of 5 secs and can divide 2 hours')
# ugly!
tickercsv = ticker + '.csv'
if not tickercsv in listdir(kwargs['DATA_PATH'] +
(kwargs['TODAY'] -
timedelta(days=1)).strftime('%Y%m%d')):
raise Exception('no data for %s' % ticker)
dates = set(listdir(kwargs['DATA_PATH']))
self.TODAY = kwargs['TODAY']
# self.TODAY = datetime.strptime(today_for_test, "%Y-%m-%d") # Tracey to notice
self.T_START_TIME = kwargs['T_START_TIME']
self.T_START_SEC = time.mktime(
datetime.combine(
self.TODAY, dt_time(hour=9, minute=30, second=0,
microsecond=0)).timetuple())
# self.T_START_TIME = self.TODAY.replace(hour = 9, minute = 30, second = 0, microsecond = 0)
self.T_END_TIME = kwargs['T_END_TIME']
self.T_END_SECS = int(
(datetime.combine(self.TODAY, self.T_END_TIME) -
datetime.combine(self.TODAY, self.T_START_TIME)).total_seconds())
# self.T_END_TIME = self.TODAY.replace(hour = 15, minute = 00, second = 0, microsecond = 0)
self.LASSO_LAMBDA = kwargs['LASSO_LAMBDA']
self.N_TICK_THRESHOLD = kwargs['N_TICK_THRESHOLD'] # Tracey to notice
self.DATA_PATH = kwargs['DATA_PATH']
# self.DATA_PATH = './data_path/' # Tracey to notice
self._interval = interval
self._interval_timedelta = timedelta(seconds=self._interval)
self._am_n_interval = int(self.HALFTIME.total_seconds() /
self._interval_timedelta.total_seconds())
self._n_interval = int(self._am_n_interval + ceil((
(self.T_END_SECS - 60 * 60 * 3.5) / self._interval)))
self._features_to_train = np.ones((11, 3), dtype=float) # CA, M, L, A
self._histo_volume = np.full((10, self._n_interval), 0,
dtype=float) # historical trading volume
self._intraday_percentage = [
1 / self._n_interval
] * self._n_interval # notice .sum() =self._n_interval
# self._AR_pars = np.array([1,0],dtype =float) # (u and phi)
self._AR_pars = [0., 1.]
self._CA_today = 0
self._predicted_V = 0.
self._is_V_predicted = 0
self._last_update = 0
self._iter = 0
self._datetime_index = ([
str(dt) for dt in datetime_range(
self.T_START_TIME,
dt_time(hour=11, minute=30, second=0, microsecond=0),
timedelta(seconds=self._interval))
] + [
str(dt) for dt in datetime_range(
dt_time(hour=13, minute=0, second=0, microsecond=0),
self.T_END_TIME, timedelta(seconds=self._interval))
])
self._today_vol = [0.] * self._n_interval
self._p_per = [0.] * self._n_interval
self._p_vol = [0] * self._n_interval
self._cum_vol = 0
self._VWAP_log = {}
histo_date = self.TODAY
past_days = 0
x_output = np.concatenate(
(np.arange(0 + self._interval, 7200 + self._interval,
self._interval),
np.arange(12600 + self._interval, self.T_END_SECS,
self._interval), np.array([self.T_END_SECS])),
axis=0)
iter = 1
while iter < 11:
if not bool(dates):
raise Exception('Insufficient historical data')
histo_date = histo_date - timedelta(days=1)
past_days += 1
# if histo_date.weekday() in set([5,6]):
# continue
histo_date_str = histo_date.strftime("%Y%m%d")
if histo_date_str not in dates:
continue
dates.remove(histo_date_str)
try:
dat = pd.read_csv(self.DATA_PATH + histo_date_str + '/' +
tickercsv,
header=0)
except Exception:
print 'Error in reading %s for %s, go to the previous day.' % (
tickercsv, str(histo_date))
continue
if dat.shape[0] < self.N_TICK_THRESHOLD:
print '%s in %s has few data for prediction' % (
tickercsv, str(histo_date))
continue
if past_days > 20:
warnings.warn(
'Lack historical data. Time span of data for predicting intraday_volume of today has exceeded 20 days.'
'We are using data %d days from today' % past_days)
try:
dat.Nano = dat.Nano / 1e9 - time.mktime(
datetime.combine(
histo_date,
dt_time(hour=9, minute=30, second=0,
microsecond=0)).timetuple())
tmp_Volume = np.array(dat.Volume)
dat.Volume = list(
np.append(tmp_Volume[0], tmp_Volume[1:] - tmp_Volume[:-1]))
dat = dat.as_matrix(columns=['Nano', 'Volume'
]) # there will be Microsecond
datCA = dat[dat[:, 0] < 0][:, 1].sum() # Tracey to notice
if datCA < 1: # no data or no trade ?
continue
self._features_to_train[10 - iter, 0] = datCA
dat = dat[dat[:, 0] > 0]
# Tracey by reviewing the data from ctp finds it impossible
if any(7200 < t < 7230 for t in dat[:, 0]): # tracey_to_notice
dat = np.vstack((dat[dat[:, 0] < 7200], [
7200, dat[(dat[:, 0] >= 7200) * (dat[:, 0] < 7230),
1].sum()
], dat[dat[:, 0] > 7230]))
if any(t >= 198000 for t in dat[:, 0]):
dat = np.vstack((dat[dat[:, 0] < 19800],
[19800,
dat[dat[:, 0] >= 19800,
1].sum()])) # tracey to notice
x_input = np.append(0, dat[:, 0])
volume_cumsum = np.append(0, dat[:, 1].cumsum())
# volume_cumsum = np.append(0, dat[:, 1]) # tracey to notice
y_interp = scipy.interpolate.interp1d(
x_input, volume_cumsum) # ,interval)
intraday_volume = y_interp(x_output)
intraday_volume = np.append(
intraday_volume[0],
(intraday_volume[1:] - intraday_volume[:-1]))
self._histo_volume[10 - iter] = intraday_volume
except Exception:
print 'Error when read %s at %s, you may check its format' % (
ticker, histo_date_str)
continue
iter += 1
volume_sums = np.zeros(5, dtype=float)
while iter < 16:
if not bool(dates):
raise Exception('Insufficient historical data')
histo_date = histo_date - timedelta(days=1)
past_days += 1
# if histo_date.weekday() in set([5,6]):
# continue
histo_date_str = histo_date.strftime("%Y%m%d")
if histo_date_str not in dates:
continue
dates.remove(histo_date_str)
try:
dat = pd.read_csv(self.DATA_PATH + histo_date_str + '/' +
tickercsv,
header=0)
except Exception:
print 'Error in reading %s for %s, go to the previous day.' % (
tickercsv, str(histo_date))
continue
if dat.shape[0] < self.N_TICK_THRESHOLD:
print '%s in %s has few data for prediction' % (
tickercsv, str(histo_date))
continue
if past_days > 30:
warnings.warn(
'Lack efficacious historical data. Time span of data for predicting total trading volume of today has exceeded 30 days.'
)
try:
dat.Nano = dat.Nano / 1e9 - time.mktime(
datetime.combine(
histo_date,
dt_time(hour=9, minute=30, second=0,
microsecond=0)).timetuple())
tmp_Volume = np.array(dat.Volume)
dat.Volume = list(
np.append(tmp_Volume[0], tmp_Volume[1:] - tmp_Volume[:-1]))
dat = dat.as_matrix(columns=['Nano', 'Volume'
]) # there will be Microsecond
volume_sums[15 - iter] = dat[(dat[:, 0] > 0) *
(dat[:, 0] < self.T_END_SECS),
1].sum()
except Exception:
print 'Error when read %s at %s, you may check its format' % (
ticker, histo_date_str)
continue
iter += 1
# preparing sample for predicting today's total volume
self.volume_to_train = self._histo_volume.sum(axis=1)
volume_sums = np.append(volume_sums, self.volume_to_train)
self._features_to_train[:, 1] = rolling_mean(volume_sums)
self._features_to_train[:, 2] = rolling_linear(volume_sums)
# get intraday pattern and intialize intraday prediction
intraday_mean = self._histo_volume.mean(axis=0)
self._p_vol[0] = float(intraday_mean[0])
self._intraday_percentage = list(
np.divide(intraday_mean, intraday_mean.sum()) * self._n_interval)
tmp = np.divide(intraday_mean, intraday_mean.sum()) * self._n_interval
print tmp # Tracey to notice
if np.any(tmp < 0.1):
warnings.warn(
'adjust intraday trading volume pattern for irregular data')
tmp[tmp >= 0.1] = tmp[tmp > 0.1] * (
self._n_interval - 0.1 * len(tmp[tmp < 0.1])) / sum(
tmp[tmp >= 0.1])
tmp[tmp < 0.1] = 0.1
self._intraday_percentage = list(tmp)
self._p_per[0] = self._intraday_percentage[0] / self._n_interval
self._VWAP_log[self._datetime_index[0]] = get_log(
None, self._p_vol[0], self._p_per[0])
# compute AR
arma = ARMA(self._histo_volume[-1] / self._intraday_percentage,
order=(1, 0))
self._AR_pars = arma.fit().params.tolist()
class StagingDataAccessor(base.BaseDataAccessor):
"""
obtain data directly from stagings
It is not secure to use `StagingDataAccessor` in production.
both function with fetchurl signature and stagings configuration
should be passed during instantination
:Example:
>>> from client_request_lib.data_sources.fetcher import fetchURL
>>> staging_accessor = StagingDataAccessor(
... fetchURL, {'clans': 'http://wgccbe.ru.cwpp.iv/'})
>>> requester = Requester(staging_accessor)
>>> requester.login(str, 12312, 'sdfee23e2')
>>> def printer (*args, **kwargs):
pprint(args)
...
>>> requester.clans.get_account_applications_count_since(printer, 123)
(
{'total': 17},
200,
0
)
currently following backends are supported
- ratings
- clans
- spa
- exporter
- strongholds
- global_map
"""
requests_before_logout = -1
def __init__(self, url_fetcher, staging_hosts = {}, client_lang = None):
"""
url_fetcher is fetch_url method with following signature
staging_hosts is dict of staging hosts for example
:param url_fetcher: fetchURL callback with following signature
fetchURL(url, callback, headers={}, timeout=30, method='GET', postData='')
:param staging_hosts: stagings hosts with backend name as a key
:type url_fetcher: function
:type staging_hosts: dict
:Example:
>>> from client_request_lib.data_sources.fetcher import fetchURL
>>> staging_accessor = StagingDataAccessor(
... fetchURL,
... {
... 'ratings': 'http://wgrs.clan0101.wott.iv/',
... 'exporter': 'http://exp.clan0101.wott.iv/',
... 'global_map': 'https://wgcwx.clan0101.wott.iv/wgapi/',
... 'clans': 'http://wgccbe.clan0101.wott.iv/'
... 'spa': 'http://spa.clan0101.wott.iv/',
... 'strongholds': 'http://wgccfe.clan0101.wgnt.iv/clans/api/',
... }
... )
"""
self.client_lang = client_lang
self._account = None
self.url_fetcher = url_fetcher
self.staging_hosts = staging_hosts
return
def login(self, callback, account_id, spa_token):
self._account = account_id
result, status_code = ('ok', 200)
response_code = exceptions.ResponseCodes.NO_ERRORS
callback(result, status_code, response_code)
def logout(self, callback):
self._account = None
result, status_code = ('ok', 200)
response_code = exceptions.ResponseCodes.NO_ERRORS
callback(result, status_code, response_code)
return
def get_alive_status(self, callback):
result, status_code = {'status': 'I am alive!'}, 200
response_code = exceptions.ResponseCodes.NO_ERRORS
callback(result, status_code, response_code)
def _request_data(self, callback, service, url, method = 'GET', postData = None):
service_host = self.staging_hosts[service].strip('/')
url = '/'.join([service_host] + url.strip('/').split('/'))
if '?' not in url:
url = url + '/'
args = [None, 30.0, method]
if postData:
args.append(json.dumps(postData))
self.url_fetcher(url, callback, *args)
return
@mapped_fields({'efficiency': 'efficiency',
'clan_id': 'clan_id',
'battles_count_avg': 'battles_count_avg',
'wins_ratio_avg': 'wins_ratio_avg',
'xp_avg': 'xp_avg',
'gm_elo_rating_6': 'gm_elo_rating_6',
'gm_elo_rating_8': 'gm_elo_rating_8',
'gm_elo_rating_10': 'gm_elo_rating_10',
'gm_elo_rating_6_rank': 'gm_elo_rating_6_rank',
'gm_elo_rating_8_rank': 'gm_elo_rating_8_rank',
'gm_elo_rating_10_rank': 'gm_elo_rating_10_rank',
'fb_elo_rating_8': 'fb_elo_rating_8',
'fb_elo_rating_10': 'fb_elo_rating_10',
'fb_battles_count_10_28d': 'fb_battles_count_10_28d',
'fs_battles_count_10_28d': 'fs_battles_count_10_28d',
'gm_battles_count_28d': 'gm_battles_count_28d',
'fs_battles_count_28d': 'fs_battles_count_28d',
'fb_battles_count_28d': 'fb_battles_count_28d'})
def get_clans_ratings(self, callback, clan_ids, fields = None):
"""
return data from ratings backend using `bulks API method`_
.. _bulks API method: http://rtd.wargaming.net/docs/wgrs-api/en/latest/clans.html#bulks
"""
get_params = {'project': 'api',
'fields': ','.join(fields),
'ids': ','.join(map(str, clan_ids))}
url = 'api/wot/clans/bulks/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'ratings')
def inner_callback(data):
return data['data']
return self._request_data(inner_callback, 'ratings', url)
@convert_data({'created_at': from_iso})
@mapped_fields({'name': 'name',
'tag': 'tag',
'motto': 'motto',
'leader_id': 'leader_id',
'members_count': 'members_count',
'created_at': 'created_at',
'clan_id': 'id',
'treasury': 'treasury',
'accepts_join_requests': 'accepts_join_requests'})
def get_clans_info(self, callback, clan_ids, fields = None):
"""
return data from WGCCBE backend using `clans API method`_
.. _clans API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/api-common/clans.html
"""
get_params = {'ids': ','.join(map(str, clan_ids)),
'fields': ','.join(fields)}
url = '/clans/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'clans')
def inner_callback(data):
return data['items']
return self._request_data(inner_callback, 'clans', url)
@mapped_fields({'id': 'id',
'name': 'name'})
def get_accounts_names(self, callback, account_ids, fields = None):
"""
return data from SPA backend using `account id/name mappings API method`_
.. _account id/name mappings API method: https://confluence.wargaming.net/display/
WEBDEV/%5BWGNSPA%5D+-+SPA+HTTP+API+Examples#id-[WGNSPA]-SPAHTTPAPIExamples-Byids
"""
get_params = {'id': account_ids}
url = '/spa/accounts/names/?%s' % urlencode(get_params, doseq=True)
@preprocess_callback(callback, 'spa')
def inner_callback(data):
return [ {'id': k,
'name': v} for k, v in data.iteritems() ]
return self._request_data(inner_callback, 'spa', url)
@convert_data({'joined_at': from_iso})
@mapped_fields({'account_id': 'id',
'joined_at': 'joined_at',
'clan_id': 'clan_id',
'role_bw_flag': 'role.bw_flag',
'role_name': 'role.name'})
def get_clan_members(self, callback, clan_id, fields = None):
"""
return data from WGCCBE backend using `clan members API method`_
.. _clan members API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/api-common/
clans_id_members.html
"""
get_params = {'fields': ','.join(fields)}
url = '/clans/%s/members?%s' % (clan_id, urlencode(get_params))
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'favorite_primetime': lambda x: x and datetime.strptime(x, '%H:%M').time()})
@mapped_fields({'favorite_arena_6': 'favorite_arena_6',
'favorite_arena_8': 'favorite_arena_8',
'favorite_arena_10': 'favorite_arena_10',
'clan_id': 'clan_id',
'favorite_primetime': 'favorite_primetime'})
def get_clan_favorite_attributes(self, callback, clan_id, fields = None):
"""
return data from WGCCBE backend using `favorite_attributes API method`_
.. _favorite_attributes API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/statistics/
favorite_attributes.html
"""
url = '/gm/clans/%s/favorite_attributes' % clan_id
@preprocess_callback(callback, 'clans')
def inner_callback(backend_data):
result = {}
for field in ['clan_id', 'favorite_primetime']:
if field in backend_data:
result[field] = backend_data[field]
for data in backend_data.get('favorite_arenas', []):
if data.get('frontlevel') in (6, 8, 10) and 'arena' in data:
result['favorite_arena_{}'.format(data['frontlevel'])] = data['arena']
return result
return self._request_data(inner_callback, 'clans', url)
@convert_data({'joined_at': from_iso,
'in_clan_cooldown_till': from_iso})
@mapped_fields({'account_id': 'id',
'joined_at': 'joined_at',
'clan_id': 'clan_id',
'role_bw_flag': 'role.bw_flag',
'role_name': 'role.name',
'in_clan_cooldown_till': 'in_clan_cooldown_till'})
def get_accounts_clans(self, callback, account_ids, fields = None):
"""
return data from WGCCBE backend using `accounts API method`_
.. _accounts API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/api-common/accounts.html
"""
get_params = {'fields': ','.join(fields),
'ids': ','.join(map(str, account_ids))}
url = '/accounts/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'clans')
def inner_callback(data):
return data['items']
return self._request_data(inner_callback, 'clans', url)
@mapped_fields({'total': 'total'}, accept_fields_argument=False)
def get_account_applications_count_since(self, callback, account_id, since = None):
"""
return data from WGCCBE backend using `applications API method`_
.. _applications API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/applications.html
"""
get_params = {'fields': 'id',
'account_id': account_id,
'created_after': since.isoformat()}
url = '/applications/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@mapped_fields({'total': 'total'}, accept_fields_argument=False)
def get_clan_invites_count_since(self, callback, clan_id, since = None):
"""
return data from WGCCBE backend using `invites API method`_
.. _invites API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/invites.html
"""
get_params = {'fields': 'id',
'clan_id': clan_id,
'created_after': since.isoformat()}
url = '/invites/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'created_at': from_iso,
'updated_at': from_iso}, paginated=True)
@mapped_fields({'status': 'status',
'created_at': 'created_at',
'updated_at': 'updated_at',
'sender_id': 'sender_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id',
'comment': 'data.comment',
'status_changer_id': 'data.status_changer_id'}, paginated=True)
def get_account_applications(self, callback, fields = None, statuses = None, get_total_count = False, limit = 18, offset = 0):
"""
return data from WGCCBE backend using `applications API method`_
.. _applications API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/applications.html
"""
statuses = statuses or ['active',
'declined',
'accepted',
'expired',
'error',
'deleted']
get_params = {'fields': ','.join(fields),
'account_id': self._account,
'statuses': ','.join(statuses),
'limit': limit,
'offset': offset}
url = '/applications/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'created_at': from_iso,
'updated_at': from_iso}, paginated=True)
@mapped_fields({'status': 'status',
'created_at': 'created_at',
'updated_at': 'updated_at',
'sender_id': 'sender_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id',
'comment': 'data.comment',
'status_changer_id': 'data.status_changer_id'}, paginated=True)
def get_clan_applications(self, callback, clan_id, fields = None, statuses = None, get_total_count = False, limit = 18, offset = 0):
"""
return data from WGCCBE backend using `applications API method`_
.. _applications API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/applications.html
"""
statuses = statuses or ['active',
'declined',
'accepted',
'expired',
'error',
'deleted']
get_params = {'fields': ','.join(fields),
'clan_id': clan_id,
'statuses': ','.join(statuses),
'limit': limit,
'offset': offset}
url = '/applications/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@mapped_fields({'clan_id': 'clan_id',
'id': 'id',
'account_id': 'account_id'})
def create_applications(self, callback, clan_ids, comment, fields = None):
"""
create applications for accounts into clan using `create applications API method`_
.. _create applications API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/
applications.html
"""
url = '/applications/'
data = {'account_id': self._account,
'clan_ids': clan_ids,
'comment': comment}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
return data.values()
return self._request_data(inner_callback, 'clans', url, method='POST', postData=data)
@mapped_fields({'transaction_id': 'transaction_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id'})
def accept_application(self, callback, application_id, fields = None):
"""
accept application for accounts into clan using `accept applications API method`_
.. _accept applications API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/
applications_id.html
"""
url = '/applications/%s/' % application_id
data = {'initiator_id': self._account,
'status': 'accepted'}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
data = data or {}
data['account_id'] = data.pop('account_ids')[0]
data['id'] = application_id
return data
return self._request_data(inner_callback, 'clans', url, method='PATCH', postData=data)
@mapped_fields({'transaction_id': 'transaction_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id'})
def decline_application(self, callback, application_id, fields = None):
"""
decline application for accounts into clan using `decline applications API method`_
.. _decline applications API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/
applications_id.html
"""
url = '/applications/%s/' % application_id
data = {'initiator_id': self._account,
'status': 'declined'}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
data = data or {}
data['id'] = application_id
return data
return self._request_data(inner_callback, 'clans', url, method='PATCH', postData=data)
@mapped_fields({'clan_id': 'clan_id',
'id': 'id',
'account_id': 'account_id'})
def create_invites(self, callback, clan_id, account_ids, comment, fields = None):
"""
create applications for accounts into clan using `create invites API method`_
.. _create invites API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/invites.html
"""
url = '/invites/'
data = {'initiator_id': self._account,
'clan_id': clan_id,
'account_ids': account_ids,
'comment': comment}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
return data.values()
return self._request_data(inner_callback, 'clans', url, method='POST', postData=data)
@mapped_fields({'transaction_id': 'transaction_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id'})
def accept_invite(self, callback, invite_id, fields = None):
"""
accept application for accounts into clan using `accept invite API method`_
.. _accept invite API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/invites_id.html
"""
url = '/invites/%s/' % invite_id
data = {'initiator_id': self._account,
'status': 'accepted'}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
data = data or {}
data['account_id'] = data.pop('account_ids')[0]
data['id'] = invite_id
return data
return self._request_data(inner_callback, 'clans', url, method='PATCH', postData=data)
@mapped_fields({'transaction_id': 'transaction_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id'})
def decline_invite(self, callback, invite_id, fields = None):
"""
decline application for accounts into clan using `decline invites API method`_
.. _decline invites API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/invites_id.html
"""
url = '/invites/%s/' % invite_id
data = {'initiator_id': self._account,
'status': 'declined'}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
data = data or {}
data['id'] = invite_id
return data
return self._request_data(inner_callback, 'clans', url, method='PATCH', postData=data)
@mapped_fields({'id': 'id',
'clan_id': 'clan_id',
'account_id': 'account_id'})
def bulk_decline_invites(self, callback, invite_ids, fields = None):
"""
decline invites for clan using `decline invites API method`_
.. _decline invites API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wgcc/invites.html#patch
"""
url = '/invites/'
data = {'initiator_id': self._account,
'status': 'declined',
'ids': invite_ids}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
data = data and data['items'] or {}
return data
return self._request_data(inner_callback, 'clans', url, method='PATCH', postData=data)
@convert_data({'created_at': from_iso}, paginated=True)
@mapped_fields({'name': 'name',
'tag': 'tag',
'motto': 'motto',
'leader_id': 'leader_id',
'members_count': 'members_count',
'created_at': 'created_at',
'clan_id': 'id',
'treasury': 'treasury',
'accepts_join_requests': 'accepts_join_requests'}, paginated=True)
def search_clans(self, callback, search, get_total_count = False, fields = None, offset = 0, limit = 18):
"""
return data from WGCCBE backend using `clans API method`_
.. _clans API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/api-common/clans.html
"""
get_params = {'search': search,
'game': 'wot',
'fields': ','.join(fields),
'limit': limit,
'offset': offset}
url = '/clans/search/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'created_at': from_iso}, paginated=True)
@mapped_fields({'name': 'name',
'tag': 'tag',
'motto': 'motto',
'leader_id': 'leader_id',
'members_count': 'members_count',
'created_at': 'created_at',
'clan_id': 'id',
'treasury': 'treasury',
'accepts_join_requests': 'accepts_join_requests'}, paginated=True)
def get_recommended_clans(self, callback, get_total_count = False, fields = None, offset = 0, limit = 18):
"""
return data from WGCCBE backend using `clans API method`_
.. _clans API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/api-common/clans.html
"""
get_params = {'game': 'wot',
'fields': ','.join(fields),
'limit': limit,
'offset': offset}
url = '/clans/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'created_at': from_iso,
'updated_at': from_iso}, paginated=True)
@mapped_fields({'status': 'status',
'created_at': 'created_at',
'updated_at': 'updated_at',
'sender_id': 'sender_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id',
'comment': 'data.comment',
'status_changer_id': 'data.status_changer_id'}, paginated=True)
def get_clan_invites(self, callback, clan_id, fields = None, statuses = None, get_total_count = False, limit = 18, offset = 0):
"""
return data from WGCCBE backend using `invites API method`_
.. _invites API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/invites.html
"""
statuses = statuses or ['active',
'declined',
'accepted',
'expired',
'error',
'deleted']
get_params = {'fields': ','.join(fields),
'clan_id': clan_id,
'statuses': ','.join(statuses),
'limit': limit,
'offset': offset}
url = '/invites/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'created_at': from_iso,
'updated_at': from_iso}, paginated=True)
@mapped_fields({'status': 'status',
'created_at': 'created_at',
'updated_at': 'updated_at',
'sender_id': 'sender_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id',
'comment': 'data.comment',
'status_changer_id': 'data.status_changer_id'}, paginated=True)
def get_account_invites(self, callback, fields = None, statuses = None, get_total_count = False, limit = 18, offset = 0):
"""
return data from WGCCBE backend using `invites API method`_
.. _invites API method: http://rtd.wargaming.net/docs/wgccbe/en/latest/wotx/invites.html
"""
statuses = statuses or ['active',
'declined',
'accepted',
'expired',
'error',
'deleted']
get_params = {'fields': ','.join(fields),
'account_id': self._account,
'statuses': ','.join(statuses),
'limit': limit,
'offset': offset}
url = '/invites/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@mapped_fields({'global_rating': 'summary.global_rating',
'battle_avg_xp': 'summary.battle_avg_xp',
'battles_count': 'summary.battles_count',
'battle_avg_performance': 'summary.battle_avg_performance',
'xp_amount': 'summary.xp_amount',
'account_id': 'account_id'})
def get_accounts_info(self, callback, account_ids, fields = None):
"""
return data from exporter backend using `accounts detailed information`_
.. _accounts detailed information: http://rtd.wargaming.net/docs/exporter/en/latest/
api_wot.html#accounts-detailed-information
"""
fields = [ i.split('.', 1) for i in fields if i != 'account_id' ]
grouped = groupby(sorted(fields), key=lambda x: x[0])
sections = [ '%s[%s]' % (k, ','.join([ j[1] for j in v ])) for k, v in grouped ]
get_params = {'account_ids': ','.join(map(str, account_ids)),
'sections': ','.join(sections)}
url = '/wot/accounts/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'exporter')
def inner_callback(data):
new_data = []
for account_id, values in data.items():
values['account_id'] = account_id
new_data.append(values)
return new_data
return self._request_data(inner_callback, 'exporter', url)
@convert_data({'pillage_end_datetime': from_iso,
'prime_time': lambda x: x and datetime.strptime(x, '%H:%M').time()})
@mapped_fields({'front_name': 'frontname',
'province_id': 'province_id',
'front_name_localized': 'frontname_localized',
'province_id_localized': 'province_id_localized',
'revenue': 'daily_revenue',
'hq_connected': 'hq_connected',
'prime_time': 'primetime',
'game_map': 'game_map',
'periphery': 'periphery_id',
'turns_owned': 'turns_owned',
'pillage_cooldown': 'pillage_cooldown',
'pillage_end_datetime': 'pillage_end_datetime',
'arena_id': 'arena_id'})
def get_clan_provinces(self, callback, clan_id, fields = None):
"""
return data from WGCW backend using `clans provinces API method`_
.. _clans provinces API method: http://rtd.wargaming.net/docs/wgcw/en/latest/api/
wgapi.html?highlight=stats#clans-provinces
"""
get_params = {'clans': ','.join(map(str, [clan_id]))}
url = '/clans/provinces/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'global_map')
def inner_callback(data):
res = data['clans'] and data['clans'][0]['provinces']
for i in res:
i['frontname_localized'] = i['frontname']
i['province_id_localized'] = i['province_id']
return res
return self._request_data(inner_callback, 'global_map', url)
@mapped_fields({'battles_lost': 'battles_lost',
'battles_played': 'battles_played',
'battles_played_on_10_level': 'battles_played_on_10_level',
'battles_played_on_6_level': 'battles_played_on_6_level',
'battles_played_on_8_level': 'battles_played_on_8_level',
'battles_won': 'battles_won',
'battles_won_on_10_level': 'battles_won_on_10_level',
'battles_won_on_6_level': 'battles_won_on_6_level',
'battles_won_on_8_level': 'battles_won_on_8_level',
'influence_points': 'influence_points',
'provinces_captured': 'provinces_captured',
'provinces_count': 'provinces_count'})
def get_clan_globalmap_stats(self, callback, clan_id, fields = None):
"""
return data from WGCW backend using `clans stats API method`_
.. _clans stats API method: http://rtd.wargaming.net/docs/wgcw/en/latest/api/
wgapi.html?highlight=stats#clans-stats
"""
get_params = {'clans': ','.join(map(str, [clan_id]))}
url = '/clans/stats?%s' % urlencode(get_params)
@preprocess_callback(callback, 'global_map')
def inner_callback(data):
return data['clans'][0]['stats']
return self._request_data(inner_callback, 'global_map', url)
@mapped_fields({'front_name': 'id',
'front_name_localized': 'id_localized',
'min_vehicle_level': 'min_vehicle_level',
'max_vehicle_level': 'max_vehicle_level'})
def get_fronts_info(self, callback, front_names = None, fields = None):
"""
return data from WGCW backend using `fronts info API method`_
.. _fronts info API method: http://rtd.wargaming.net/docs/wgcw/en/latest/api/
wgapi.html?highlight=stats#id1
"""
url = '/fronts/'
@preprocess_callback(callback, 'global_map')
def inner_callback(data):
res = data['fronts']
for i in res:
i['id_localized'] = i['id']
return res
return self._request_data(inner_callback, 'global_map', url)
@convert_data({'defence_hour': lambda x: (dt_time(x, 0) if x >= 0 else None)})
@mapped_fields({'buildings.direction': 'buildings.direction',
'buildings.type': 'buildings.type',
'buildings.level': 'buildings.level',
'buildings.position': 'buildings.position',
'defence_attack_efficiency': 'defence_attack_efficiency',
'defence_battles_count': 'defence_battles_count',
'defence_capture_enemy_building_total_count': 'defence_capture_enemy_building_total_count',
'defence_combat_wins': 'defence_combat_wins',
'defence_defence_efficiency': 'defence_defence_efficiency',
'defence_enemy_base_capture_count': 'defence_enemy_base_capture_count',
'defence_loss_own_building_total_count': 'defence_loss_own_building_total_count',
'defence_resource_capture_count': 'defence_resource_capture_count',
'defence_resource_loss_count': 'defence_resource_loss_count',
'sortie_absolute_battles_count': 'sortie_absolute_battles_count',
'sortie_battles_count': 'sortie_battles_count',
'sortie_champion_battles_count': 'sortie_champion_battles_count',
'sortie_middle_battles_count': 'sortie_middle_battles_count',
'defence_attack_count': 'defence_attack_count',
'defence_defence_count': 'defence_defence_count',
'defence_success_attack_count': 'defence_success_attack_count',
'defence_success_defence_count': 'defence_success_defence_count',
'sortie_fort_resource_in_absolute': 'sortie_fort_resource_in_absolute',
'sortie_fort_resource_in_champion': 'sortie_fort_resource_in_champion',
'sortie_fort_resource_in_middle': 'sortie_fort_resource_in_middle',
'sortie_losses': 'sortie_losses',
'sortie_wins': 'sortie_wins',
'level': 'level',
'defence_hour': 'defence_hour',
'defence_mode_is_activated': 'defence_mode_is_activated',
'fb_battles_count_10': 'fb_battles_count_10',
'fb_battles_count_8': 'fb_battles_count_8',
'total_resource_amount': 'total_resource_amount'})
def get_stronghold_info(self, callback, clan_id, fields = None):
"""
return data from WGCCFE backend using `stronghold info API method`_
.. _stronghold info API method: http://rtd.wargaming.net/docs/wgccfe/en/latest/rst/
strongholds.html#strongholds-clan-id
"""
get_params = urlencode({'performer_id': self._account})
try:
clan_id = int(clan_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = 'api/strongholds/%s/' % clan_id
if self._account:
url = '?'.join([url, get_params])
@preprocess_callback(callback, 'strongholds')
def inner_callback(data):
return data['stronghold']
return self._request_data(inner_callback, 'strongholds', url)
@convert_data({'vacation_finish': timestamp_to_datetime,
'vacation_start': timestamp_to_datetime})
@mapped_fields({'buildings.type': 'buildings.type',
'buildings.hp': 'buildings.hp',
'buildings.direction': 'buildings.direction',
'buildings.position': 'buildings.position',
'buildings.storage': 'buildings.resource_amount',
'buildings.level': 'buildings.level',
'buildings_count': 'buildings_count',
'clan_id': 'clan_id',
'level': 'level',
'clan_name': 'clan_name',
'clan_tag': 'clan_tag',
'directions': 'directions',
'directions_count': 'directions_count',
'off_day': 'off_day',
'periphery_id': 'periphery_id',
'vacation_finish': 'vacation_finish',
'vacation_start': 'vacation_start',
'sortie_wins_period': 'sortie_wins_period',
'sortie_battles_wins_percentage_period': 'sortie_battles_wins_percentage_period',
'sortie_battles_count_period': 'sortie_battles_count_period',
'defence_battles_count_period': 'defence_battles_count_period'})
def get_strongholds_statistics(self, callback, clan_id, fields = None):
"""
return data from WGCCFE backend using `stronghold statistics API method`_
.. _stronghold statistics API method: http://rtd.wargaming.net/docs/wgccfe/en/
latest/rst/strongholds.html#strongholds-statistics-clan-id
"""
get_params = urlencode({'performer_id': self._account})
try:
clan_id = int(clan_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/api/strongholds/statistics/%s/' % clan_id
if self._account:
url = '?'.join([url, get_params])
@preprocess_callback(callback, 'strongholds')
def inner_callback(data):
return data[0]
return self._request_data(inner_callback, 'strongholds', url)
@convert_data({'defence_hour': lambda x: (dt_time(x, 0) if x >= 0 else None)})
@mapped_fields({'clan_id': 'clan_id',
'defence_hour': 'defence_hour'})
def get_strongholds_state(self, callback, clan_id, fields = None):
"""
return data from WGCCFE backend using `stronghold state API method`_
.. _stronghold state API method: http://rtd.wargaming.net/docs/wgccfe/en/latest/
rst/strongholds.html#strongholds-state
"""
get_params = {'clan_id': clan_id}
try:
clan_id = int(clan_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
if self._account:
get_params['performer_id'] = self._account
url = '/api/strongholds/state/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'strongholds')
def inner_callback(data):
return data and data[0] or {}
return self._request_data(inner_callback, 'strongholds', url)
import time
from datetime import date, datetime, time as dt_time, timedelta
UPDATE_TIME = dt_time(6, second=30) # noqa: WPS432
def day_from_iso(iso_date):
month_day = date.fromisoformat(iso_date).strftime('%B %d') # noqa: WPS323
month, day = month_day.split()
return '{0} {1}'.format(_(month), day)
def hm_from_seconds(secs):
return time.strftime('%Hh %Mm', time.gmtime(secs))
def is_before_update_time(): # noqa: N802,WPS114
return datetime.utcnow().time() < UPDATE_TIME
def seconds_for_next_update():
now = datetime.utcnow()
today_6GMT = datetime.combine(now.date(), UPDATE_TIME) # noqa: N806,WPS114
if now <= today_6GMT:
return (today_6GMT - now).seconds
tommorow_6GMT = datetime.combine( # noqa: N806,WPS114
now.date() + timedelta(days=1),
UPDATE_TIME,
)
return (tommorow_6GMT - now).seconds
def testTimeOnlyRange(self):
# tzrange returns None
tz_range = tz.tzrange('dflt')
self.assertIs(dt_time(13, 20, tzinfo=tz_range).utcoffset(), None)
class FaultsReviewNotice(models.Model):
UNREVIEWED = 0
NOTIFICATION_TYPES = ((UNREVIEWED,
_l("Notify about Faults awaiting review")), )
TIME_CHOICES = [(dt_time(x // 60, x % 60), "%02d:%02d" % (x // 60, x % 60))
for x in range(0, 24 * 60, 15)]
notification_type = models.IntegerField(
verbose_name=_l("Notification Type"),
choices=NOTIFICATION_TYPES,
default=UNREVIEWED,
)
send_empty = models.BooleanField(
verbose_name=_l("Send Empty Notices"),
help_text=_l(
"Check to send notices even if there's no unreviewed Service Events to currently notify about"
),
default=False,
)
recurrences = RecurrenceField(
verbose_name=_l("Recurrences"),
help_text=_l(
"Define the schedule this notification should be sent on."),
default="",
)
time = models.TimeField(
verbose_name=_l("Time of day"),
help_text=_l(
"Set the time of day this notice should be sent (00:00-23:59)."),
choices=TIME_CHOICES,
)
recipients = models.ForeignKey(
RecipientGroup,
verbose_name=_l("Recipients"),
help_text=_l(
"Choose the group of recipients who should receive these notifications"
),
on_delete=models.PROTECT,
)
units = models.ForeignKey(
UnitGroup,
verbose_name=_l("Unit Group filter"),
help_text=_l(
"Select which group of Units this notification should be limited to. Leave blank to include all units"
),
null=True,
blank=True,
on_delete=models.PROTECT,
)
last_sent = models.DateTimeField(null=True, editable=False)
class Meta:
verbose_name = _l("Fault Review Notice")
@property
def is_unreviewed(self):
return self.notification_type == self.UNREVIEWED
def faults(self):
"""Return faults relevant to this notice"""
faults = Fault.objects.unreviewed()
if self.units_id:
faults = faults.filter(unit__in=self.units.units.all())
return faults.order_by(
"unit__%s" % settings.ORDER_UNITS_BY,
"fault_types__code",
)
def faults_by_unit_fault_type(self):
faults = self.faults()
return faults.values(
"unit__name",
"fault_types__code",
).order_by(
"unit__name",
"fault_types__code",
).annotate(
Count("unit__name"),
Count("fault_types__code"),
)
def send_required(self):
return self.send_empty or self.faults().count() > 0
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
response.update_status('execution started at : %s ' % dt.now(), 5)
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 5)
resource = archiveextract(resource=rename_complexinputs(request.inputs['resource']))
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
# bbox = [-80, 20, 50, 70]
# TODO: Add checking for wrong cordinates and apply default if nesessary
#level = 500
level = request.inputs['level'][0].data
if (level == 500):
dummylevel = 1000 # dummy workaround for cdo sellevel
else:
dummylevel = 500
LOGGER.debug('LEVEL selected: %s hPa' % (level))
bbox=[]
bboxStr = request.inputs['BBox'][0].data
bboxStr = bboxStr.split(',')
#for i in bboxStr: bbox.append(int(i))
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
# if bbox_obj is not None:
# LOGGER.info("bbox_obj={0}".format(bbox_obj.coords))
# bbox = [bbox_obj.coords[0][0],
# bbox_obj.coords[0][1],
# bbox_obj.coords[1][0],
# bbox_obj.coords[1][1]]
# LOGGER.info("bbox={0}".format(bbox))
# else:
# bbox = None
# region = self.getInputValues(identifier='region')[0]
# bbox = [float(b) for b in region.split(',')]
# bbox_obj = self.BBox.getValue()
normalize = request.inputs['normalize'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
# model_var = request.inputs['reanalyses'][0].data
# model, var = model_var.split('_')
# experiment = self.getInputValues(identifier='experiment')[0]
# dataset, var = experiment.split('_')
# LOGGER.info('environment set')
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 5)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.error(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
try:
# refSt = dt.strptime(refSt[0], '%Y-%m-%d')
# refEn = dt.strptime(refEn[0], '%Y-%m-%d')
# dateSt = dt.strptime(dateSt[0], '%Y-%m-%d')
# dateEn = dt.strptime(dateEn[0], '%Y-%m-%d')
#not nesessary if fix ocgis_module.py
refSt = dt.combine(refSt,dt_time(12,0))
refEn = dt.combine(refEn,dt_time(12,0))
dateSt = dt.combine(dateSt,dt_time(12,0))
dateEn = dt.combine(dateEn,dt_time(12,0))
# refSt = refSt.replace(hour=12)
# refEn = refEn.replace(hour=12)
# dateSt = dateSt.replace(hour=12)
# dateEn = dateEn.replace(hour=12)
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.error('output format not valid')
start = min(refSt, dateSt)
end = max(refEn, dateEn)
# if bbox_obj is not None:
# LOGGER.info("bbox_obj={0}".format(bbox_obj.coords))
# bbox = [bbox_obj.coords[0][0],
# bbox_obj.coords[0][1],
# bbox_obj.coords[1][0],
# bbox_obj.coords[1][1]]
# LOGGER.info("bbox={0}".format(bbox))
# else:
# bbox = None
LOGGER.info('environment set')
except Exception as e:
msg = 'failed to set environment %s ' % e
LOGGER.error(msg)
raise Exception(msg)
LOGGER.debug("init took %s seconds.", time.time() - start_time)
response.update_status('Read in and convert the arguments', 5)
########################
# input data preperation
########################
# TODO: Check if files containing more than one dataset
response.update_status('Start preparing input data', 12)
start_time = time.time() # mesure data preperation ...
try:
# TODO: Add selection of the level. maybe bellow in call(..., level_range=[...,...])
if type(resource) == list:
#resource.sort()
resource = sorted(resource, key=lambda i: path.splitext(path.basename(i))[0])
else:
resource=[resource]
#===============================================================
# TODO: REMOVE resources which are out of interest from the list
# (years > and < than requested for calculation)
tmp_resource = []
for re in resource:
s,e = get_timerange(re)
tmpSt = dt.strptime(s,'%Y%m%d')
tmpEn = dt.strptime(e,'%Y%m%d')
if ((tmpSt <= end ) and (tmpEn >= start)):
tmp_resource.append(re)
LOGGER.debug('Selected file: %s ' % (re))
resource = tmp_resource
# ===============================================================
#================================================================
# Try to fix memory issue... (ocgis call for files like 20-30 gb... )
# IF 4D - select pressure level before domain cut
#
# resource properties
ds = Dataset(resource[0])
variable = get_variable(resource[0])
var = ds.variables[variable]
dims = list(var.dimensions)
dimlen = len(dims)
try:
model_id = ds.getncattr('model_id')
except AttributeError:
model_id = 'Unknown model'
LOGGER.debug('MODEL: %s ' % (model_id))
lev_units = 'hPa'
if (dimlen>3) :
lev = ds.variables[dims[1]]
# actually index [1] need to be detected... assuming zg(time, plev, lat, lon)
lev_units = lev.units
if (lev_units=='Pa'):
level = level*100
dummylevel=dummylevel*100
# TODO: OR check the NAME and units of vertical level and find 200 , 300, or 500 mbar in it
# Not just level = level * 100.
# Get Levels
from cdo import Cdo
cdo = Cdo()
lev_res=[]
if(dimlen>3):
for res_fn in resource:
tmp_f = 'lev_' + path.basename(res_fn)
comcdo = '%s,%s' % (level,dummylevel)
cdo.sellevel(comcdo, input=res_fn, output=tmp_f)
lev_res.append(tmp_f)
else:
lev_res = resource
# Get domain
regr_res=[]
for res_fn in lev_res:
tmp_f = 'dom_' + path.basename(res_fn)
comcdo = '%s,%s,%s,%s' % (bbox[0],bbox[2],bbox[1],bbox[3])
cdo.sellonlatbox(comcdo, input=res_fn, output=tmp_f)
regr_res.append(tmp_f)
#archive_tmp = call(resource=resource, time_range=[refSt, refEn], geom=bbox, spatial_wrapping='wrap')
#simulation_tmp = call(resource=resource, time_range=[dateSt, dateEn], geom=bbox, spatial_wrapping='wrap')
#============================
archive_tmp = call(resource=regr_res, time_range=[refSt, refEn], spatial_wrapping='wrap')
simulation_tmp = call(resource=regr_res, time_range=[dateSt, dateEn], spatial_wrapping='wrap')
#######################################################################################
# TEMORAL dirty workaround to get the level and it's units - will be func in utils.py
if (dimlen>3) :
archive = get_level(archive_tmp, level = level)
simulation = get_level(simulation_tmp,level = level)
variable = 'z%s' % level
# TODO: here should be modulated
else:
archive = archive_tmp
simulation = simulation_tmp
# 3D, move forward
#######################################################################################
if seacyc is True:
seasoncyc_base, seasoncyc_sim = analogs.seacyc(archive, simulation, method=normalize)
else:
seasoncyc_base = None
seasoncyc_sim = None
except Exception as e:
msg = 'failed to prepare archive and simulation files %s ' % e
LOGGER.debug(msg)
raise Exception(msg)
ip, output = mkstemp(dir='.', suffix='.txt')
output_file = path.abspath(output)
files = [path.abspath(archive), path.abspath(simulation), output_file]
LOGGER.debug("data preperation took %s seconds.", time.time() - start_time)
############################
# generating the config file
############################
# TODO: add MODEL name as argument
response.update_status('writing config file', 15)
start_time = time.time() # measure write config ...
try:
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=model_id,
sim_id=model_id,
timewin=timewin,
varname=variable,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')],
bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3]))
except Exception as e:
msg = 'failed to generate config file %s ' % e
LOGGER.debug(msg)
raise Exception(msg)
LOGGER.debug("write_config took %s seconds.", time.time() - start_time)
##############
# CASTf90 call
##############
import subprocess
import shlex
start_time = time.time() # measure call castf90
response.update_status('Start CASTf90 call', 20)
try:
# response.update_status('execution of CASTf90', 50)
cmd = 'analogue.out %s' % path.relpath(config_file)
# system(cmd)
args = shlex.split(cmd)
output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
LOGGER.info('analogue.out info:\n %s ' % output)
LOGGER.debug('analogue.out errors:\n %s ' % error)
response.update_status('**** CASTf90 suceeded', 70)
except Exception as e:
msg = 'CASTf90 failed %s ' % e
LOGGER.error(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
response.update_status('preparing output', 70)
response.outputs['config'].file = config_file #config_output_url # config_file )
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 80)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 90)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
return response
def _handler(self, request, response):
chdir(self.workdir)
init_process_logger('log.txt')
process_start_time = time.time() # measure process execution time ...
response.update_status('execution started at : %s ' % dt.now(), 5)
start_time = time.time() # measure init ...
resource = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
# Filter resource:
if type(resource) == list:
resource = sorted(resource,
key=lambda i: path.splitext(path.basename(i))[0])
else:
resource = [resource]
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
regrset = request.inputs['regrset'][0].data
direction = request.inputs['direction'][0].data
# Check if model has 360_day calendar:
try:
modcal, calunits = get_calendar(resource[0])
LOGGER.debug('CALENDAR: %s' % (modcal))
if '360_day' in modcal:
if direction == 're2mo':
if refSt.day == 31:
refSt = refSt.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (refSt))
if refEn.day == 31:
refEn = refEn.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (refEn))
else: # mo2re
if dateSt.day == 31:
dateSt = dateSt.replace(day=30)
LOGGER.debug('Date has been changed for: %s' %
(dateSt))
if dateEn.day == 31:
dateEn = dateEn.replace(day=30)
LOGGER.debug('Date has been changed for: %s' %
(dateEn))
except:
LOGGER.debug('Could not detect calendar')
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
bboxDef = '-20,40,30,70' # in general format
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or abs(float(bboxStr[1]) > 180)
or abs(float(bboxStr[2]) > 90) or abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug(
'BBOX is out of the range, using default instead: %s ' %
(bboxStr))
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
normalize = request.inputs['normalize'][0].data
plot = request.inputs['plot'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
try:
if direction == 're2mo':
anaSt = dt.combine(dateSt, dt_time(
0, 0)) # dt.strptime(dateSt[0], '%Y-%m-%d')
anaEn = dt.combine(dateEn, dt_time(
0, 0)) # dt.strptime(dateEn[0], '%Y-%m-%d')
refSt = dt.combine(refSt, dt_time(
12, 0)) # dt.strptime(refSt[0], '%Y-%m-%d')
refEn = dt.combine(refEn, dt_time(
12, 0)) # dt.strptime(refEn[0], '%Y-%m-%d')
r_time_range = [anaSt, anaEn]
m_time_range = [refSt, refEn]
elif direction == 'mo2re':
anaSt = dt.combine(dateSt, dt_time(
12, 0)) # dt.strptime(refSt[0], '%Y-%m-%d')
anaEn = dt.combine(dateEn, dt_time(
12, 0)) # dt.strptime(refEn[0], '%Y-%m-%d')
refSt = dt.combine(refSt, dt_time(
0, 0)) # dt.strptime(dateSt[0], '%Y-%m-%d')
refEn = dt.combine(refEn, dt_time(
0, 0)) # dt.strptime(dateEn[0], '%Y-%m-%d')
r_time_range = [refSt, refEn]
m_time_range = [anaSt, anaEn]
else:
LOGGER.exception(
'failed to find time periods for comparison direction')
except:
msg = 'failed to put simulation and reference time in order'
LOGGER.exception(msg)
raise Exception(msg)
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.exception('output format not valid')
try:
if model == 'NCEP':
# getlevel = True
getlevel = False
if 'z' in var:
level = var.strip('z')
variable = 'hgt'
# conform_units_to='hPa'
else:
variable = 'slp'
level = None
# conform_units_to='hPa'
elif '20CRV2' in model:
getlevel = False
if 'z' in var:
variable = 'hgt'
level = var.strip('z')
# conform_units_to=None
else:
variable = 'prmsl'
level = None
# conform_units_to='hPa'
else:
LOGGER.exception('Reanalyses model not known')
LOGGER.info('environment set')
except:
msg = 'failed to set environment'
LOGGER.exception(msg)
raise Exception(msg)
# LOGGER.exception("init took %s seconds.", time.time() - start_time)
response.update_status('Read in the arguments', 10)
#################
# get input data
#################
# TODO: do not forget to select years
start_time = time.time() # measure get_input_data ...
response.update_status('fetching input data', 20)
try:
if direction == 're2mo':
nc_reanalyses = reanalyses(start=anaSt.year,
end=anaEn.year,
variable=var,
dataset=model,
getlevel=getlevel)
else:
nc_reanalyses = reanalyses(start=refSt.year,
end=refEn.year,
variable=var,
dataset=model,
getlevel=getlevel)
if type(nc_reanalyses) == list:
nc_reanalyses = sorted(
nc_reanalyses,
key=lambda i: path.splitext(path.basename(i))[0])
else:
nc_reanalyses = [nc_reanalyses]
# For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb
# So it makes sense, to operate it step-by-step
# TODO: need to create dictionary for such datasets (for models as well)
# TODO: benchmark the method bellow for NCEP z500 for 60 years, may be use the same (!)
# TODO Now everything regrid to the reanalysis
# if ('20CRV2' in model) and ('z' in var):
if ('z' in var):
tmp_total = []
origvar = get_variable(nc_reanalyses[0])
for z in nc_reanalyses:
# tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z,
variable=origvar,
level_range=[int(level), int(level)],
geom=bbox,
spatial_wrapping='wrap',
prefix='levdom_' + path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(
tmp_total,
key=lambda i: path.splitext(path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total,
variable=origvar,
time_range=r_time_range)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level,
z_var.dtype,
dimensions=(dims[0], dims[2],
dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
nc_subset = call(inter_subset_tmp, variable='z%s' % level)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
system(tbr)
# for i in inter_subset_tmp
tbr = 'rm -f %s' % (inter_subset_tmp)
system(tbr)
else:
# TODO: ADD HERE serial as well as in analogs reanalysis process!!
nc_subset = call(
resource=nc_reanalyses,
variable=var,
geom=bbox,
spatial_wrapping='wrap',
time_range=r_time_range,
)
response.update_status('**** Input reanalyses data fetched', 30)
except:
msg = 'failed to fetch or subset input files'
LOGGER.exception(msg)
raise Exception(msg)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 40)
m_start = m_time_range[0]
m_end = m_time_range[1]
# ===============================================================
# REMOVE resources from the list which are out of interest from the list
# (years > and < than requested for calculation)
tmp_resource = []
for re in resource:
s, e = get_timerange(re)
tmpSt = dt.strptime(s, '%Y%m%d')
tmpEn = dt.strptime(e, '%Y%m%d')
if ((tmpSt <= m_end) and (tmpEn >= m_start)):
tmp_resource.append(re)
LOGGER.debug('Selected file: %s ' % (re))
resource = tmp_resource
start_time = time.time() # mesure data preperation ...
# TODO: Check the callendars ! for model vs reanalyses.
# TODO: Check the units! model vs reanalyses.
try:
m_total = []
modvar = get_variable(resource)
# resource properties
ds = Dataset(resource[0])
m_var = ds.variables[modvar]
dims = list(m_var.dimensions)
dimlen = len(dims)
try:
model_id = ds.getncattr('model_id')
except AttributeError:
model_id = 'Unknown model'
LOGGER.debug('MODEL: %s ' % (model_id))
lev_units = 'hPa'
if (dimlen > 3):
lev = ds.variables[dims[1]]
# TODO: actually index [1] need to be detected... assuming zg(time, plev, lat, lon)
lev_units = lev.units
if (lev_units == 'Pa'):
m_level = str(int(level) * 100)
else:
m_level = level
else:
m_level = None
if level == None:
level_range = None
else:
level_range = [int(m_level), int(m_level)]
ds.close()
for z in resource:
tmp_n = 'tmp_%s' % (uuid.uuid1())
# TODO: Important! if only 1 file - select time period from that first!
# select level and regrid
# \/\/ working version 19Feb2019
# b0 = call(resource=z, variable=modvar, level_range=level_range,
# spatial_wrapping='wrap', cdover='system',
# regrid_destination=nc_reanalyses[0], regrid_options='bil', prefix=tmp_n)
try:
b0 = call(resource=z,
variable=modvar,
level_range=level_range,
spatial_wrapping='wrap',
cdover='system',
regrid_destination=nc_subset,
regrid_options='bil',
prefix=tmp_n)
except:
b0 = call(resource=z,
variable=modvar,
level_range=level_range,
spatial_wrapping='wrap',
cdover='system',
regrid_destination=nc_reanalyses[0],
regrid_options='bil',
prefix=tmp_n)
# select domain (already selected in fact, if regrided to 'nc_subset')
b01 = call(resource=b0,
geom=bbox,
spatial_wrapping='wrap',
prefix='levregr_' + path.basename(z)[0:-3])
# TODO: REPLACE rm -f by os.remove() !
tbr = 'rm -f %s' % (b0)
system(tbr)
tbr = 'rm -f %s.nc' % (tmp_n)
system(tbr)
# get full resource
m_total.append(b01)
model_subset = call(m_total, time_range=m_time_range)
for i in m_total:
tbr = 'rm -f %s' % (i)
system(tbr)
if m_level is not None:
# Create new variable in model set
ds = Dataset(model_subset, mode='a')
mod_var = ds.variables.pop(modvar)
dims = mod_var.dimensions
new_modvar = ds.createVariable('z%s' % level,
mod_var.dtype,
dimensions=(dims[0], dims[2],
dims[3]))
new_modvar[:, :, :] = squeeze(mod_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
mod_subset = call(model_subset, variable='z%s' % level)
else:
mod_subset = model_subset
except:
msg = 'failed to subset simulation or reference data'
LOGGER.exception(msg)
raise Exception(msg)
# --------------------------------------------
try:
if direction == 'mo2re':
simulation = mod_subset
archive = nc_subset
base_id = model
sim_id = model_id
elif direction == 're2mo':
simulation = nc_subset
archive = mod_subset
base_id = model_id
sim_id = model
else:
LOGGER.exception('direction not valid: %s ' % direction)
except:
msg = 'failed to find comparison direction'
LOGGER.exception(msg)
raise Exception(msg)
try:
if level is not None:
out_var = 'z%s' % level
else:
var_archive = get_variable(archive)
var_simulation = get_variable(simulation)
if var_archive != var_simulation:
rename_variable(archive,
oldname=var_archive,
newname=var_simulation)
out_var = var_simulation
LOGGER.info('varname %s in netCDF renamed to %s' %
(var_archive, var_simulation))
except:
msg = 'failed to rename variable in target files'
LOGGER.exception(msg)
raise Exception(msg)
try:
if seacyc is True:
seasoncyc_base, seasoncyc_sim = analogs.seacyc(
archive, simulation, method=normalize)
else:
seasoncyc_base = None
seasoncyc_sim = None
except:
msg = 'failed to prepare seasonal cycle reference files'
LOGGER.exception(msg)
raise Exception(msg)
# ip, output = mkstemp(dir='.', suffix='.txt')
# output_file = path.abspath(output)
output_file = 'output.txt'
################################
# Prepare names for config.txt #
################################
# refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
# simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
# Fix < 1900 issue...
refDatesString = refSt.isoformat().strip().split(
"T")[0] + "_" + refEn.isoformat().strip().split("T")[0]
simDatesString = dateSt.isoformat().strip().split(
"T")[0] + "_" + dateEn.isoformat().strip().split("T")[0]
archiveNameString = "base_" + out_var + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3]) + '.nc'
simNameString = "sim_" + out_var + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3]) + '.nc'
move(archive, archiveNameString)
move(simulation, simNameString)
archive = archiveNameString
simulation = simNameString
files = [path.abspath(archive), path.abspath(simulation), output_file]
############################
# generating the config file
############################
response.update_status('writing config file', 50)
start_time = time.time() # measure write config ...
try:
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=base_id,
sim_id=sim_id,
timewin=timewin,
# varname=var,
varname=out_var,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
# period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')],
period=[
refSt.isoformat().strip().split("T")[0],
refEn.isoformat().strip().split("T")[0]
],
bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3]))
except:
msg = 'failed to generate config file'
LOGGER.exception(msg)
raise Exception(msg)
#######################
# CASTf90 call
#######################
import subprocess
import shlex
start_time = time.time() # measure call castf90
response.update_status('Start CASTf90 call', 60)
# -----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
environ['MKL_NUM_THREADS'] = str(nth)
environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
# -----------------------
# ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ###############
# ##### MUST be removed after castf90 recompiled with the latest hdf version
# ##### NOT safe
environ['HDF5_DISABLE_VERSION_CHECK'] = '1'
# hdflib = os.path.expanduser("~") + '/anaconda/lib'
# hdflib = os.getenv("HOME") + '/anaconda/lib'
import pwd
hdflib = pwd.getpwuid(getuid()).pw_dir + '/anaconda/lib'
environ['LD_LIBRARY_PATH'] = hdflib
# ################################################################################
try:
response.update_status('execution of CASTf90', 70)
cmd = 'analogue.out %s' % path.relpath(config_file)
# system(cmd)
args = shlex.split(cmd)
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
LOGGER.info('analogue.out info:\n %s ' % output)
LOGGER.exception('analogue.out errors:\n %s ' % error)
response.update_status('**** CASTf90 suceeded', 80)
except:
msg = 'CASTf90 failed'
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# TODO: Add try - except for pdfs
if plot == 'Yes':
analogs_pdf = analogs.plot_analogs(configfile=config_file)
else:
analogs_pdf = 'dummy_plot.pdf'
with open(analogs_pdf, 'a'):
utime(analogs_pdf, None)
response.update_status('preparting output', 90)
# Stopper to keep twitcher results, for debug
# dummy=dummy
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
response.outputs['target_netcdf'].file = archive
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
# response.update_status('reformatted analog file', 95)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 95)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
response.outputs['output_log'].file = 'log.txt'
return response
def parseEvent(self, event, fg, bg):
"""Method to turn google event into a format that we can use more
easily.
"""
# Set up a reference to UTC (all events need a timezone so we can
# sort them)
utc = pytz.UTC
# Have we got a "date" or "dateTime" event?
# Parse the start end end times as appropriate
if event["start"].get("date", False):
start = dateutil.parser.parse(event["start"]["date"])
end = dateutil.parser.parse(event["end"]["date"])
else:
start = dateutil.parser.parse(event["start"]["dateTime"])
end = dateutil.parser.parse(event["end"]["dateTime"])
# Change the end time to one second earlier (useful to check number
# of days of event)
false_end = end - timedelta(0, 1)
duration = false_end - start
# Empty list for our events
ev_list = []
# Split long events into daily events
for i in range(duration.days + 1):
# Create a new start time if the daily event start time isn't the
# same as the overall start time
new_date = start + timedelta(i)
if new_date.date() != start.date():
st = datetime.combine(new_date.date(),
dt_time(0, 0, tzinfo=start.tzinfo))
else:
st = start
# Create a new end time if the daily event end time isn't the same
# as the overall end time
if new_date.date() != false_end.date():
add_day = new_date.date() + timedelta(1)
en = datetime.combine(add_day,
dt_time(0, 0, tzinfo=start.tzinfo))
else:
en = end
# If there's no timezone set, then let's set one
if st.tzinfo is None:
st = utc.localize(st)
if en.tzinfo is None:
en = utc.localize(en)
# Create a dict of the info we need
ev = {
"fg": fg,
"bg": bg,
"summary": event.get("summary", ""),
"location": event.get("location", ""),
"start": st,
"end": en,
"stdate": st.date()
}
# Add to our list
ev_list.append(ev)
return ev_list
sys.exit(0)
if infoboks.has_param(commonargs['year']) and infoboks.has_param(commonargs['week']):
year = int(re.sub(ur'<\!--.+?-->', ur'', unicode(infoboks.parameters[commonargs['year']])).strip())
startweek = int(re.sub(ur'<\!--.+?-->', ur'', unicode(infoboks.parameters[commonargs['week']])).strip())
if infoboks.has_param(commonargs['week2']):
endweek = re.sub(ur'<\!--.+?-->', ur'', unicode(infoboks.parameters[commonargs['week2']])).strip()
if endweek == '':
endweek = startweek
else:
endweek = startweek
endweek = int(endweek)
startweek = Week(year, startweek)
endweek = Week(year, endweek)
start = wiki_tz.localize(datetime.combine(startweek.monday(), dt_time(0, 0, 0)))
end = wiki_tz.localize(datetime.combine(endweek.sunday(), dt_time(23, 59, 59)))
elif infoboks.has_param(ibcfg['start']) and infoboks.has_param(ibcfg['end']):
startdt = infoboks.parameters[ibcfg['start']].value
enddt = infoboks.parameters[ibcfg['end']].value
start = wiki_tz.localize(datetime.strptime(startdt + ' 00 00 00', '%Y-%m-%d %H %M %S'))
end = wiki_tz.localize(datetime.strptime(enddt + ' 23 59 59', '%Y-%m-%d %H %M %S'))
else:
log('!! fant ikke datoer')
sys.exit(0)
year = start.isocalendar()[0]
startweek = start.isocalendar()[1]
endweek = end.isocalendar()[1]
figname = config['plot']['figname'] % {'year': year, 'week': startweek}
from datetime import datetime, timedelta
from datetime import time as dt_time
from collections import defaultdict
from pathlib import Path, PurePath
from vnpy.trader.object import BarData, TradeData
from vnpy.trader.constant import Interval, Offset
from vnpy.trader.database import database_manager
from vnpy.trader.utility import extract_vt_symbol
from strategy.boll_channel_strategy import BollChannelStrategy
from strategy.turtle_signal_strategy import TurtleSignalStrategy
from backtesting import SegBacktestingEngine
from basic_data import future_basic_data, future_hot_start, dominant_data
DAY_START = dt_time(8)
DAY_END = dt_time(16)
NIGHT_A_START = dt_time(20)
NIGHT_A_END = dt_time(23, 59, 59)
NIGHT_B_START = dt_time(0)
NIGHT_B_END = dt_time(5)
strategy_dict = {'boll': BollChannelStrategy, 'turtle': TurtleSignalStrategy}
zh_to_en = {'多': 'long', '空': 'short', '开': 'open', '平': 'close'}
compare_items = [
'total_days', 'profit_days', 'max_ddpercent', 'max_drawdown_duration',
'total_return', 'return_std', 'daily_return', 'sharpe_ratio',
arrivals = directions_data[direction]
arrivals.append({'t': int(dt.timestamp()), 'v': vid})
if line_num % 1000 == 0:
print(f"{path}:{line_num}: {time_str}")
def time_key(arrival):
return arrival['t']
for d, routes_data in all_data.items():
next_day = d + incr
start_time = int(
tz.localize(datetime.combine(
d, dt_time(hour=start_hour))).timestamp())
end_time = int(
tz.localize(datetime.combine(
next_day, dt_time(hour=start_hour))).timestamp())
for route, stops_data in routes_data.items():
route_id = route.replace(' ', '_')
for stop, stop_data in stops_data.items():
for direction, arrivals in stop_data["arrivals"].items():
arrivals.sort(key=time_key)
history = arrival_history.ArrivalHistory(agency_id,
route_id,
start_time=start_time,
class StagingDataAccessor(base.BaseDataAccessor):
requests_before_logout = -1
def __init__(self, url_fetcher, staging_hosts=None, client_lang=None, user_agent=None):
self.client_lang = client_lang
self._account = None
self.url_fetcher = url_fetcher
self.staging_hosts = staging_hosts or {}
self.user_agent = user_agent
return
def login(self, callback, account_id, spa_token):
self._account = account_id
result, status_code = ('ok', 200)
response_code = exceptions.ResponseCodes.NO_ERRORS
callback(result, status_code, response_code)
def logout(self, callback):
self._account = None
result, status_code = ('ok', 200)
response_code = exceptions.ResponseCodes.NO_ERRORS
callback(result, status_code, response_code)
return
def get_alive_status(self, callback):
result, status_code = {'status': 'I am alive!'}, 200
response_code = exceptions.ResponseCodes.NO_ERRORS
callback(result, status_code, response_code)
def _request_data(self, callback, service, url, method='GET', postData=None):
service_host = self.staging_hosts[service].strip('/')
url = '/'.join([service_host] + url.strip('/').split('/'))
if '?' not in url:
url = url + '/'
args = [None, 30.0, method]
if postData:
args.append(json.dumps(postData))
self.url_fetcher(url, callback, *args)
return
@mapped_fields({'efficiency': 'efficiency',
'clan_id': 'clan_id',
'battles_count_avg': 'battles_count_avg',
'wins_ratio_avg': 'wins_ratio_avg',
'xp_avg': 'xp_avg',
'gm_elo_rating_6': 'gm_elo_rating_6',
'gm_elo_rating_8': 'gm_elo_rating_8',
'gm_elo_rating_10': 'gm_elo_rating_10',
'gm_elo_rating_6_rank': 'gm_elo_rating_6_rank',
'gm_elo_rating_8_rank': 'gm_elo_rating_8_rank',
'gm_elo_rating_10_rank': 'gm_elo_rating_10_rank',
'fb_elo_rating_8': 'fb_elo_rating_8',
'fb_elo_rating_10': 'fb_elo_rating_10',
'fb_battles_count_10_28d': 'fb_battles_count_10_28d',
'fs_battles_count_10_28d': 'fs_battles_count_10_28d',
'gm_battles_count_28d': 'gm_battles_count_28d',
'fs_battles_count_28d': 'fs_battles_count_28d',
'fb_battles_count_28d': 'fb_battles_count_28d'})
def get_clans_ratings(self, callback, clan_ids, fields=None):
get_params = {'project': 'api',
'fields': ','.join(fields),
'ids': ','.join(map(str, clan_ids))}
url = 'api/wot/clans/bulks/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'ratings')
def inner_callback(data):
return data['data']
return self._request_data(inner_callback, 'ratings', url)
@convert_data({'created_at': from_iso})
@mapped_fields({'name': 'name',
'tag': 'tag',
'motto': 'motto',
'leader_id': 'leader_id',
'members_count': 'members_count',
'created_at': 'created_at',
'clan_id': 'id',
'treasury': 'treasury',
'accepts_join_requests': 'accepts_join_requests'})
def get_clans_info(self, callback, clan_ids, fields=None):
get_params = {'ids': ','.join(map(str, clan_ids)),
'fields': ','.join(fields)}
url = '/clans/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'clans')
def inner_callback(data):
return data['items']
return self._request_data(inner_callback, 'clans', url)
@mapped_fields({'id': 'id',
'name': 'name'})
def get_accounts_names(self, callback, account_ids, fields=None):
get_params = {'id': account_ids}
url = '/spa/accounts/names/?%s' % urlencode(get_params, doseq=True)
@preprocess_callback(callback, 'spa')
def inner_callback(data):
return [ {'id': k,
'name': v} for k, v in data.iteritems() ]
return self._request_data(inner_callback, 'spa', url)
@convert_data({'joined_at': from_iso})
@mapped_fields({'account_id': 'id',
'joined_at': 'joined_at',
'clan_id': 'clan_id',
'role_bw_flag': 'role.bw_flag',
'role_name': 'role.name'})
def get_clan_members(self, callback, clan_id, fields=None):
get_params = {'fields': ','.join(fields)}
url = '/clans/%s/members?%s' % (clan_id, urlencode(get_params))
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'favorite_primetime': lambda x: x and datetime.strptime(x, '%H:%M').time()})
@mapped_fields({'favorite_arena_6': 'favorite_arena_6',
'favorite_arena_8': 'favorite_arena_8',
'favorite_arena_10': 'favorite_arena_10',
'clan_id': 'clan_id',
'favorite_primetime': 'favorite_primetime'})
def get_clan_favorite_attributes(self, callback, clan_id, fields=None):
url = '/gm/clans/%s/favorite_attributes' % clan_id
@preprocess_callback(callback, 'clans')
def inner_callback(backend_data):
result = {}
for field in ['clan_id', 'favorite_primetime']:
if field in backend_data:
result[field] = backend_data[field]
for data in backend_data.get('favorite_arenas', []):
if data.get('frontlevel') in (6, 8, 10) and 'arena' in data:
result['favorite_arena_{}'.format(data['frontlevel'])] = data['arena']
return result
return self._request_data(inner_callback, 'clans', url)
@convert_data({'joined_at': from_iso,
'in_clan_cooldown_till': from_iso})
@mapped_fields({'account_id': 'id',
'joined_at': 'joined_at',
'clan_id': 'clan_id',
'role_bw_flag': 'role.bw_flag',
'role_name': 'role.name',
'in_clan_cooldown_till': 'in_clan_cooldown_till'})
def get_accounts_clans(self, callback, account_ids, fields=None):
get_params = {'fields': ','.join(fields),
'ids': ','.join(map(str, account_ids))}
url = '/accounts/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'clans')
def inner_callback(data):
return data['items']
return self._request_data(inner_callback, 'clans', url)
@mapped_fields({'total': 'total'}, accept_fields_argument=False)
def get_account_applications_count_since(self, callback, account_id, since=None):
get_params = {'fields': 'id',
'account_id': account_id,
'created_after': since.isoformat()}
url = '/applications/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@mapped_fields({'total': 'total'}, accept_fields_argument=False)
def get_clan_invites_count_since(self, callback, clan_id, since=None):
get_params = {'fields': 'id',
'clan_id': clan_id,
'created_after': since.isoformat()}
url = '/invites/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'created_at': from_iso,
'updated_at': from_iso}, paginated=True)
@mapped_fields({'status': 'status',
'created_at': 'created_at',
'updated_at': 'updated_at',
'sender_id': 'sender_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id',
'comment': 'data.comment',
'status_changer_id': 'data.status_changer_id'}, paginated=True)
def get_account_applications(self, callback, fields=None, statuses=None, get_total_count=False, limit=18, offset=0):
statuses = statuses or ['active',
'declined',
'accepted',
'expired',
'error',
'deleted']
get_params = {'fields': ','.join(fields),
'account_id': self._account,
'statuses': ','.join(statuses),
'limit': limit,
'offset': offset}
url = '/applications/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'created_at': from_iso,
'updated_at': from_iso}, paginated=True)
@mapped_fields({'status': 'status',
'created_at': 'created_at',
'updated_at': 'updated_at',
'sender_id': 'sender_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id',
'comment': 'data.comment',
'status_changer_id': 'data.status_changer_id'}, paginated=True)
def get_clan_applications(self, callback, clan_id, fields=None, statuses=None, get_total_count=False, limit=18, offset=0):
statuses = statuses or ['active',
'declined',
'accepted',
'expired',
'error',
'deleted']
get_params = {'fields': ','.join(fields),
'clan_id': clan_id,
'statuses': ','.join(statuses),
'limit': limit,
'offset': offset}
url = '/applications/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@mapped_fields({'clan_id': 'clan_id',
'id': 'id',
'account_id': 'account_id'})
def create_applications(self, callback, clan_ids, comment, fields=None):
url = '/applications/'
data = {'account_id': self._account,
'clan_ids': clan_ids,
'comment': comment}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
return data.values()
return self._request_data(inner_callback, 'clans', url, method='POST', postData=data)
@mapped_fields({'transaction_id': 'transaction_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id'})
def accept_application(self, callback, application_id, fields=None):
url = '/applications/%s/' % application_id
data = {'initiator_id': self._account,
'status': 'accepted'}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
data = data or {}
data['account_id'] = data.pop('account_ids')[0]
data['id'] = application_id
return data
return self._request_data(inner_callback, 'clans', url, method='PATCH', postData=data)
@mapped_fields({'transaction_id': 'transaction_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id'})
def decline_application(self, callback, application_id, fields=None):
url = '/applications/%s/' % application_id
data = {'initiator_id': self._account,
'status': 'declined'}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
data = data or {}
data['id'] = application_id
return data
return self._request_data(inner_callback, 'clans', url, method='PATCH', postData=data)
@mapped_fields({'clan_id': 'clan_id',
'id': 'id',
'account_id': 'account_id'})
def create_invites(self, callback, clan_id, account_ids, comment, fields=None):
url = '/invites/'
data = {'initiator_id': self._account,
'clan_id': clan_id,
'account_ids': account_ids,
'comment': comment}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
return data.values()
return self._request_data(inner_callback, 'clans', url, method='POST', postData=data)
@mapped_fields({'transaction_id': 'transaction_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id'})
def accept_invite(self, callback, invite_id, fields=None):
url = '/invites/%s/' % invite_id
data = {'initiator_id': self._account,
'status': 'accepted'}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
data = data or {}
data['account_id'] = data.pop('account_ids')[0]
data['id'] = invite_id
return data
return self._request_data(inner_callback, 'clans', url, method='PATCH', postData=data)
@mapped_fields({'transaction_id': 'transaction_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id'})
def decline_invite(self, callback, invite_id, fields=None):
url = '/invites/%s/' % invite_id
data = {'initiator_id': self._account,
'status': 'declined'}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
data = data or {}
data['id'] = invite_id
return data
return self._request_data(inner_callback, 'clans', url, method='PATCH', postData=data)
@mapped_fields({'id': 'id',
'clan_id': 'clan_id',
'account_id': 'account_id'})
def bulk_decline_invites(self, callback, invite_ids, fields=None):
url = '/invites/'
data = {'initiator_id': self._account,
'status': 'declined',
'ids': invite_ids}
@preprocess_callback(callback, 'clans')
def inner_callback(data):
data = data and data['items'] or {}
return data
return self._request_data(inner_callback, 'clans', url, method='PATCH', postData=data)
@convert_data({'created_at': from_iso}, paginated=True)
@mapped_fields({'name': 'name',
'tag': 'tag',
'motto': 'motto',
'leader_id': 'leader_id',
'members_count': 'members_count',
'created_at': 'created_at',
'clan_id': 'id',
'treasury': 'treasury',
'accepts_join_requests': 'accepts_join_requests'}, paginated=True)
def search_clans(self, callback, search, get_total_count=False, fields=None, offset=0, limit=18):
get_params = {'search': search,
'game': 'wot',
'fields': ','.join(fields),
'limit': limit,
'offset': offset}
url = '/clans/search/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'created_at': from_iso}, paginated=True)
@mapped_fields({'name': 'name',
'tag': 'tag',
'motto': 'motto',
'leader_id': 'leader_id',
'members_count': 'members_count',
'created_at': 'created_at',
'clan_id': 'id',
'treasury': 'treasury',
'accepts_join_requests': 'accepts_join_requests'}, paginated=True)
def get_recommended_clans(self, callback, get_total_count=False, fields=None, offset=0, limit=18):
get_params = {'game': 'wot',
'fields': ','.join(fields),
'limit': limit,
'offset': offset}
url = '/clans/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'created_at': from_iso,
'updated_at': from_iso}, paginated=True)
@mapped_fields({'status': 'status',
'created_at': 'created_at',
'updated_at': 'updated_at',
'sender_id': 'sender_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id',
'comment': 'data.comment',
'status_changer_id': 'data.status_changer_id'}, paginated=True)
def get_clan_invites(self, callback, clan_id, fields=None, statuses=None, get_total_count=False, limit=18, offset=0):
statuses = statuses or ['active',
'declined',
'accepted',
'expired',
'error',
'deleted']
get_params = {'fields': ','.join(fields),
'clan_id': clan_id,
'statuses': ','.join(statuses),
'limit': limit,
'offset': offset}
url = '/invites/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@convert_data({'created_at': from_iso,
'updated_at': from_iso}, paginated=True)
@mapped_fields({'status': 'status',
'created_at': 'created_at',
'updated_at': 'updated_at',
'sender_id': 'sender_id',
'id': 'id',
'account_id': 'account_id',
'clan_id': 'clan_id',
'comment': 'data.comment',
'status_changer_id': 'data.status_changer_id'}, paginated=True)
def get_account_invites(self, callback, fields=None, statuses=None, get_total_count=False, limit=18, offset=0):
statuses = statuses or ['active',
'declined',
'accepted',
'expired',
'error',
'deleted']
get_params = {'fields': ','.join(fields),
'account_id': self._account,
'statuses': ','.join(statuses),
'limit': limit,
'offset': offset}
url = '/invites/?%s' % urlencode(get_params)
return self._request_data(preprocess_callback(callback, 'clans'), 'clans', url)
@mapped_fields({'global_rating': 'summary.global_rating',
'battle_avg_xp': 'summary.battle_avg_xp',
'battles_count': 'summary.battles_count',
'battle_avg_performance': 'summary.battle_avg_performance',
'xp_amount': 'summary.xp_amount',
'account_id': 'account_id'})
def get_accounts_info(self, callback, account_ids, fields=None):
fields = [ i.split('.', 1) for i in fields if i != 'account_id' ]
grouped = groupby(sorted(fields), key=lambda x: x[0])
sections = [ '%s[%s]' % (k, ','.join([ j[1] for j in v ])) for k, v in grouped ]
get_params = {'account_ids': ','.join(map(str, account_ids)),
'sections': ','.join(sections)}
url = '/wot/accounts/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'exporter')
def inner_callback(data):
new_data = []
for account_id, values in data.items():
values['account_id'] = account_id
new_data.append(values)
return new_data
return self._request_data(inner_callback, 'exporter', url)
@convert_data({'pillage_end_datetime': from_iso,
'prime_time': lambda x: x and datetime.strptime(x, '%H:%M').time()})
@mapped_fields({'front_name': 'frontname',
'province_id': 'province_id',
'front_name_localized': 'frontname_localized',
'province_id_localized': 'province_id_localized',
'revenue': 'daily_revenue',
'hq_connected': 'hq_connected',
'prime_time': 'primetime',
'game_map': 'game_map',
'periphery': 'periphery_id',
'turns_owned': 'turns_owned',
'pillage_cooldown': 'pillage_cooldown',
'pillage_end_datetime': 'pillage_end_datetime',
'arena_id': 'arena_id'})
def get_clan_provinces(self, callback, clan_id, fields=None):
get_params = {'clans': ','.join(map(str, [clan_id]))}
url = '/clans/provinces/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'global_map')
def inner_callback(data):
res = data['clans'] and data['clans'][0]['provinces']
for i in res:
i['frontname_localized'] = i['frontname']
i['province_id_localized'] = i['province_id']
return res
return self._request_data(inner_callback, 'global_map', url)
@mapped_fields({'battles_lost': 'battles_lost',
'battles_played': 'battles_played',
'battles_played_on_10_level': 'battles_played_on_10_level',
'battles_played_on_6_level': 'battles_played_on_6_level',
'battles_played_on_8_level': 'battles_played_on_8_level',
'battles_won': 'battles_won',
'battles_won_on_10_level': 'battles_won_on_10_level',
'battles_won_on_6_level': 'battles_won_on_6_level',
'battles_won_on_8_level': 'battles_won_on_8_level',
'influence_points': 'influence_points',
'provinces_captured': 'provinces_captured',
'provinces_count': 'provinces_count'})
def get_clan_globalmap_stats(self, callback, clan_id, fields=None):
get_params = {'clans': ','.join(map(str, [clan_id]))}
url = '/clans/stats?%s' % urlencode(get_params)
@preprocess_callback(callback, 'global_map')
def inner_callback(data):
return data['clans'][0]['stats']
return self._request_data(inner_callback, 'global_map', url)
@mapped_fields({'front_name': 'id',
'front_name_localized': 'id_localized',
'min_vehicle_level': 'min_vehicle_level',
'max_vehicle_level': 'max_vehicle_level'})
def get_fronts_info(self, callback, front_names=None, fields=None):
url = '/fronts/'
@preprocess_callback(callback, 'global_map')
def inner_callback(data):
res = data['fronts']
for i in res:
i['id_localized'] = i['id']
return res
return self._request_data(inner_callback, 'global_map', url)
@convert_data({'defence_hour': lambda x: dt_time(x, 0) if x >= 0 else None})
@mapped_fields({'buildings.direction': 'buildings.direction',
'buildings.type': 'buildings.type',
'buildings.level': 'buildings.level',
'buildings.position': 'buildings.position',
'defence_attack_efficiency': 'defence_attack_efficiency',
'defence_battles_count': 'defence_battles_count',
'defence_capture_enemy_building_total_count': 'defence_capture_enemy_building_total_count',
'defence_combat_wins': 'defence_combat_wins',
'defence_defence_efficiency': 'defence_defence_efficiency',
'defence_enemy_base_capture_count': 'defence_enemy_base_capture_count',
'defence_loss_own_building_total_count': 'defence_loss_own_building_total_count',
'defence_resource_capture_count': 'defence_resource_capture_count',
'defence_resource_loss_count': 'defence_resource_loss_count',
'sortie_absolute_battles_count': 'sortie_absolute_battles_count',
'sortie_battles_count': 'sortie_battles_count',
'sortie_champion_battles_count': 'sortie_champion_battles_count',
'sortie_middle_battles_count': 'sortie_middle_battles_count',
'defence_attack_count': 'defence_attack_count',
'defence_defence_count': 'defence_defence_count',
'defence_success_attack_count': 'defence_success_attack_count',
'defence_success_defence_count': 'defence_success_defence_count',
'sortie_fort_resource_in_absolute': 'sortie_fort_resource_in_absolute',
'sortie_fort_resource_in_champion': 'sortie_fort_resource_in_champion',
'sortie_fort_resource_in_middle': 'sortie_fort_resource_in_middle',
'sortie_losses': 'sortie_losses',
'sortie_wins': 'sortie_wins',
'level': 'level',
'defence_hour': 'defence_hour',
'defence_mode_is_activated': 'defence_mode_is_activated',
'fb_battles_count_10': 'fb_battles_count_10',
'fb_battles_count_8': 'fb_battles_count_8',
'total_resource_amount': 'total_resource_amount'})
def get_stronghold_info(self, callback, clan_id=None, fields=None):
get_params = urlencode({'performer_id': self._account})
try:
clan_id = int(clan_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = 'api/strongholds/%s/' % clan_id
if self._account:
url = '?'.join([url, get_params])
@preprocess_callback(callback, 'strongholds')
def inner_callback(data):
return data['stronghold']
return self._request_data(inner_callback, 'strongholds', url)
@convert_data({'vacation_finish': timestamp_to_datetime,
'vacation_start': timestamp_to_datetime})
@mapped_fields({'buildings.type': 'buildings.type',
'buildings.hp': 'buildings.hp',
'buildings.direction': 'buildings.direction',
'buildings.position': 'buildings.position',
'buildings.storage': 'buildings.resource_amount',
'buildings.level': 'buildings.level',
'buildings_count': 'buildings_count',
'clan_id': 'clan_id',
'level': 'level',
'clan_name': 'clan_name',
'clan_tag': 'clan_tag',
'directions': 'directions',
'directions_count': 'directions_count',
'off_day': 'off_day',
'periphery_id': 'periphery_id',
'vacation_finish': 'vacation_finish',
'vacation_start': 'vacation_start',
'sortie_wins_period': 'sortie_wins_period',
'sortie_battles_wins_percentage_period': 'sortie_battles_wins_percentage_period',
'sortie_battles_count_period': 'sortie_battles_count_period',
'defence_battles_count_period': 'defence_battles_count_period'})
def get_strongholds_statistics(self, callback, clan_id, fields=None):
get_params = urlencode({'performer_id': self._account})
try:
clan_id = int(clan_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/api/strongholds/statistics/%s/' % clan_id
if self._account:
url = '?'.join([url, get_params])
@preprocess_callback(callback, 'strongholds')
def inner_callback(data):
return data[0]
return self._request_data(inner_callback, 'strongholds', url)
def get_wgsh_unit_info(self, callback, periphery_id, unit_server_id, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/'.format(periphery_id=periphery_id, unit_server_id=unit_server_id)
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url)
def set_vehicle(self, callback, periphery_id, unit_server_id, vehicle_cd, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/members/{account_id}/vehicles'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
patch_data = {'vehicle_cd': vehicle_cd}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='PATCH', postData=patch_data)
def set_readiness(self, callback, periphery_id, unit_server_id, is_ready, reset_vehicle, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/members/{account_id}/readiness'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
patch_data = {'is_ready': is_ready,
'reset_vehicle': reset_vehicle}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='PATCH', postData=patch_data)
def invite_players(self, callback, periphery_id, unit_server_id, accounts_to_invite, comment, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/participants/{account_id}/invite'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
post_data = {'accounts_to_invite': accounts_to_invite,
'comment': comment}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='POST', postData=post_data)
def assign_player(self, callback, periphery_id, unit_server_id, account_to_assign, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/participants/{account_id}/assign'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
post_data = {'account_to_assign': account_to_assign}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='POST', postData=post_data)
def unassign_player(self, callback, periphery_id, unit_server_id, account_to_unassign, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/participants/{account_id}/unassign'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
post_data = {'account_to_unassign': account_to_unassign}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='POST', postData=post_data)
def give_leadership(self, callback, periphery_id, unit_server_id, target_account_id, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/members/{account_id}/give_leadership'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
post_data = {'target_account_id': target_account_id}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='PATCH', postData=post_data)
def set_equipment_commander(self, callback, periphery_id, unit_server_id, target_account_id, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/members/{account_id}/equipment_commander'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
post_data = {'equipment_commander_id': target_account_id}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='PATCH', postData=post_data)
def leave_room(self, callback, periphery_id, unit_server_id, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/participants/{account_id}/leave'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='POST')
def take_away_leadership(self, callback, periphery_id, unit_server_id, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery}/units/{unit}/participants/{account}/take_away_leadership'.format(periphery=periphery_id, unit=unit_server_id, account=self._account)
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='PATCH')
def kick_player(self, callback, periphery_id, unit_server_id, account_to_kick, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/members/{account_id}/kick'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
post_data = {'account_to_kick': account_to_kick}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='POST', postData=post_data)
def set_open(self, callback, periphery_id, unit_server_id, is_open, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/participants/{account_id}/set_open'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
post_data = {'is_open': is_open}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='PATCH', postData=post_data)
def lock_reserve(self, callback, periphery_id, unit_server_id, reserve_id, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/members/{account_id}/lock_reserve'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
post_data = {'reserve_id': reserve_id}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='POST', postData=post_data)
def unlock_reserve(self, callback, periphery_id, unit_server_id, reserve_id, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/members/{account_id}/unlock_reserve'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
post_data = {'reserve_id': reserve_id}
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='POST', postData=post_data)
def clan_statistics(self, callback, clan_id, fields=None):
try:
clan_id = int(clan_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/external_api/v1/clan_card/{clan_id}'.format(clan_id=clan_id)
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='GET')
def account_statistics(self, callback, account_id, fields=None):
try:
account_id = int(account_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/external_api/accounts/{account_id}'.format(account_id=account_id)
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='GET')
def join_room(self, callback, periphery_id, unit_server_id, fields=None):
try:
periphery_id = int(periphery_id)
unit_server_id = int(unit_server_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
url = '/unit_api/periphery/{periphery_id}/units/{unit_server_id}/members/{account_id}/join'.format(periphery_id=periphery_id, unit_server_id=unit_server_id, account_id=self._account)
@preprocess_callback(callback, 'wgsh')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'wgsh', url, method='POST')
def user_season_statistics(self, callback, fields=None):
url = '/user-season-statistics/{account_id}/'.format(account_id=self._account)
@preprocess_callback(callback, 'rblb')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'rblb', url, method='GET')
def user_ranked_position(self, callback, fields=None):
url = '/user-position/{account_id}/'.format(account_id=self._account)
@preprocess_callback(callback, 'rblb')
def inner_callback(data):
return data or {}
return self._request_data(inner_callback, 'rblb', url, method='GET')
@convert_data({'defence_hour': lambda x: dt_time(x, 0) if x >= 0 else None})
@mapped_fields({'clan_id': 'clan_id',
'defence_hour': 'defence_hour'})
def get_strongholds_state(self, callback, clan_id, fields=None):
get_params = {'clan_id': clan_id}
try:
clan_id = int(clan_id)
except (TypeError, ValueError):
error = exceptions.BadRequest()
return callback({'description': error.description}, error.status_code, error.response_code)
if self._account:
get_params['performer_id'] = self._account
url = '/api/strongholds/state/?%s' % urlencode(get_params)
@preprocess_callback(callback, 'strongholds')
def inner_callback(data):
return data and data[0] or {}
return self._request_data(inner_callback, 'strongholds', url)
DICT_TYPE = type(dict())
STR_TYPE = type(str())
UNICODE_TYPE = type(unicode())
FLOAT_TYPE = type(float())
INT_TYPE = type(int())
LONG_TYPE = type(long())
BOOL_TYPE = type(bool())
LIST_TYPES = (LIST_TYPE, TUPLE_TYPE)
STR_TYPES = (STR_TYPE, UNICODE_TYPE)
NOMINAL_TYPES = (INT_TYPE, FLOAT_TYPE, LONG_TYPE)
NONE_VALUES = (None, '---', '', 'None')
# 00:00:00; used to combine with date() to form datetime() at midnight
NULL_TIME = dt_time()
# Convenience constants; these are exipration times in seconds
# console escapes for colored output
WHITE = '\033[97m'
BLUE = '\033[96m'
YELLOW = '\033[93m'
GREEN = '\033[92m'
RED = '\033[91m'
ENDC = '\033[0m'
# date only
RE_DATE = re.compile('(\d\d\d\d)-(\d\d)-(\d\d)')
# date only range syntax
RE_DATE_RANGE = re.compile('(\d\d\d\d-\d\d-\d\d):?(\d\d\d\d-\d\d-\d\d)?')
# datetime only
class ServiceEventSchedulingNotice(models.Model):
ALL = 0
DUE = 10
UPCOMING_AND_DUE = 20
UPCOMING = 30
NOTIFICATION_TYPES = (
(ALL, _l("Notify About All Service Event Schedule Due Dates")),
(DUE,
_l("Notify About Scheduled Service Events Currently Due & Overdue")),
(UPCOMING_AND_DUE,
_l("Notify About Scheduled Service Events Currently Due & Overdue, and Upcoming Due Dates"
)),
(UPCOMING,
_l("Notify About Scheduled Service Events Upcoming Due Dates Only")),
)
TIME_CHOICES = [(dt_time(x // 60, x % 60), "%02d:%02d" % (x // 60, x % 60))
for x in range(0, 24 * 60, 15)]
notification_type = models.IntegerField(
verbose_name=_l("Notification Type"),
choices=NOTIFICATION_TYPES,
)
send_empty = models.BooleanField(
verbose_name=_l("Send Empty Notices"),
help_text=_l(
"Check to send notices even if there's no QC to currently notify about"
),
default=False,
)
recurrences = RecurrenceField(
verbose_name=_l("Recurrences"),
help_text=_l(
"Define the schedule this notification should be sent on."),
default="",
)
time = models.TimeField(
verbose_name=_l("Time of day"),
help_text=_l(
"Set the time of day this notice should be sent (00:00-23:59)."),
choices=TIME_CHOICES,
)
future_days = models.PositiveIntegerField(
verbose_name=_l("Future Days"),
blank=True,
null=True,
help_text=_l(
"How many days in the future should notices about upcoming QC due dates include. "
"A value of zero will only include test lists due today."),
)
recipients = models.ForeignKey(
RecipientGroup,
verbose_name=_l("Recipients"),
help_text=_l(
"Choose the group of recipients who should receive these notifications"
),
on_delete=models.PROTECT,
)
units = models.ForeignKey(
UnitGroup,
verbose_name=_l("Unit Group filter"),
help_text=_l(
"Select which group of Units this notification should be limited to. Leave blank to include all units"
),
null=True,
blank=True,
on_delete=models.PROTECT,
)
last_sent = models.DateTimeField(null=True, editable=False)
class Meta:
verbose_name = _l("Service Event Scheduling Notice")
@property
def is_all(self):
return self.notification_type == self.ALL
@property
def is_due(self):
return self.notification_type == self.DUE
@property
def is_upcoming_and_due(self):
return self.notification_type == self.UPCOMING_AND_DUE
@property
def is_upcoming(self):
return self.notification_type == self.UPCOMING
def schedules(self):
"""Return ServiceEventSchedule relevant to this notice"""
schedules = ServiceEventSchedule.objects.filter(active=True)
if self.units_id:
schedules = schedules.filter(
unit_service_area__unit__in=self.units.units.all())
return schedules.order_by(
"unit_service_area__unit__%s" % settings.ORDER_UNITS_BY,
"unit_service_area__service_area__name",
"due_date",
)
def all(self):
return self.schedules().exclude(due_date=None)
def upcoming(self, include_overdue=False):
"""Return Schedules that will be coming due in the future. Optionally
include test lists that are currently due and overdue"""
start, end = today_start_end()
end = end + timezone.timedelta(days=self.future_days)
schedules = self.schedules().filter(due_date__lte=end)
if not include_overdue:
schedules = schedules.filter(due_date__gte=start)
return schedules
def due_and_overdue(self):
"""Return Schedules that are currently due or overdue"""
start, end = today_start_end()
return self.schedules().filter(due_date__lte=end)
def upcoming_and_due(self):
"""Return Schedules that are either coming due soon or currently due or overdue"""
return self.upcoming(include_overdue=True)
def schedules_to_notify(self):
dispatch = {
self.ALL: self.all,
self.DUE: self.due_and_overdue,
self.UPCOMING_AND_DUE: self.upcoming_and_due,
self.UPCOMING: self.upcoming,
}
return dispatch[self.notification_type]()
def send_required(self):
return self.send_empty or self.schedules_to_notify().count() > 0
def testTimeOnlyOffset(self):
# tzoffset doesn't care
tz_offset = tz.tzoffset('+3', 3600)
self.assertEqual(
dt_time(13, 20, tzinfo=tz_offset).utcoffset(),
timedelta(seconds=3600))
def get_forecast_times(forecast_length,
forecast_date=None,
forecast_time=None):
"""
Generate a list of python datetime objects specifying the desired forecast
times. This list will be created from input specifications if provided.
Otherwise defaults are to start today at the most recent 6-hourly interval
(00, 06, 12, 18) and to run out to T+144 hours.
Args:
forecast_length (int):
An integer giving the desired length of the forecast output in
hours (e.g. 48 for a two day forecast period).
forecast_date (string (YYYYMMDD)):
A string of format YYYYMMDD defining the start date for which
forecasts are required. If unset it defaults to today in UTC.
forecast_time (int):
An integer giving the hour on the forecast_date at which to start
the forecast output; 24hr clock such that 17 = 17Z for example. If
unset it defaults to the latest 6 hour cycle as a start time.
Returns:
forecast_times (list of datetime.datetime objects):
A list of python datetime.datetime objects that represent the
times at which diagnostic data should be extracted.
Raises:
ValueError : raised if the input date is not in the expected format.
"""
date_format = re.compile('[0-9]{8}')
if forecast_date is None:
start_date = datetime.utcnow().date()
else:
if date_format.match(forecast_date) and len(forecast_date) == 8:
start_date = datetime.strptime(forecast_date, "%Y%m%d").date()
else:
raise ValueError('Date {} is in unexpected format; should be '
'YYYYMMDD.'.format(forecast_date))
if forecast_time is None:
# If no start hour provided, go back to the nearest multiple of 6
# hours (e.g. utcnow = 11Z --> 06Z).
forecast_start_time = datetime.combine(
start_date, dt_time(divmod(datetime.utcnow().hour, 6)[0] * 6))
else:
forecast_start_time = datetime.combine(start_date,
dt_time(forecast_time))
# Generate forecast times. Hourly to T+48, 3 hourly to T+forecast_length.
forecast_times = [
forecast_start_time + timedelta(hours=x)
for x in range(min(forecast_length, 49))
]
forecast_times = (forecast_times + [
forecast_start_time + timedelta(hours=x)
for x in range(51, forecast_length + 1, 3)
])
return forecast_times
class TestBusinessHour(Base):
_offset = BusinessHour
def setup_method(self, method):
self.d = datetime(2014, 7, 1, 10, 00)
self.offset1 = BusinessHour()
self.offset2 = BusinessHour(n=3)
self.offset3 = BusinessHour(n=-1)
self.offset4 = BusinessHour(n=-4)
from datetime import time as dt_time
self.offset5 = BusinessHour(start=dt_time(11, 0), end=dt_time(14, 30))
self.offset6 = BusinessHour(start="20:00", end="05:00")
self.offset7 = BusinessHour(n=-2,
start=dt_time(21, 30),
end=dt_time(6, 30))
self.offset8 = BusinessHour(start=["09:00", "13:00"],
end=["12:00", "17:00"])
self.offset9 = BusinessHour(n=3,
start=["09:00", "22:00"],
end=["13:00", "03:00"])
self.offset10 = BusinessHour(n=-1,
start=["23:00", "13:00"],
end=["02:00", "17:00"])
@pytest.mark.parametrize(
"start,end,match",
[
(
dt_time(11, 0, 5),
"17:00",
"time data must be specified only with hour and minute",
),
("AAA", "17:00", "time data must match '%H:%M' format"),
("14:00:05", "17:00", "time data must match '%H:%M' format"),
([], "17:00", "Must include at least 1 start time"),
("09:00", [], "Must include at least 1 end time"),
(
["09:00", "11:00"],
"17:00",
"number of starting time and ending time must be the same",
),
(
["09:00", "11:00"],
["10:00"],
"number of starting time and ending time must be the same",
),
(
["09:00", "11:00"],
["12:00", "20:00"],
r"invalid starting and ending time\(s\): opening hours should not "
"touch or overlap with one another",
),
(
["12:00", "20:00"],
["09:00", "11:00"],
r"invalid starting and ending time\(s\): opening hours should not "
"touch or overlap with one another",
),
],
)
def test_constructor_errors(self, start, end, match):
with pytest.raises(ValueError, match=match):
BusinessHour(start=start, end=end)
def test_different_normalize_equals(self):
# GH#21404 changed __eq__ to return False when `normalize` does not match
offset = self._offset()
offset2 = self._offset(normalize=True)
assert offset != offset2
def test_repr(self):
assert repr(self.offset1) == "<BusinessHour: BH=09:00-17:00>"
assert repr(self.offset2) == "<3 * BusinessHours: BH=09:00-17:00>"
assert repr(self.offset3) == "<-1 * BusinessHour: BH=09:00-17:00>"
assert repr(self.offset4) == "<-4 * BusinessHours: BH=09:00-17:00>"
assert repr(self.offset5) == "<BusinessHour: BH=11:00-14:30>"
assert repr(self.offset6) == "<BusinessHour: BH=20:00-05:00>"
assert repr(self.offset7) == "<-2 * BusinessHours: BH=21:30-06:30>"
assert repr(
self.offset8) == "<BusinessHour: BH=09:00-12:00,13:00-17:00>"
assert repr(
self.offset9) == "<3 * BusinessHours: BH=09:00-13:00,22:00-03:00>"
assert repr(
self.offset10) == "<-1 * BusinessHour: BH=13:00-17:00,23:00-02:00>"
def test_with_offset(self):
expected = Timestamp("2014-07-01 13:00")
assert self.d + BusinessHour() * 3 == expected
assert self.d + BusinessHour(n=3) == expected
@pytest.mark.parametrize(
"offset_name",
[
"offset1", "offset2", "offset3", "offset4", "offset8", "offset9",
"offset10"
],
)
def test_eq_attribute(self, offset_name):
offset = getattr(self, offset_name)
assert offset == offset
@pytest.mark.parametrize(
"offset1,offset2",
[
(BusinessHour(start="09:00"), BusinessHour()),
(
BusinessHour(start=["23:00", "13:00"], end=["12:00", "17:00"]),
BusinessHour(start=["13:00", "23:00"], end=["17:00", "12:00"]),
),
],
)
def test_eq(self, offset1, offset2):
assert offset1 == offset2
@pytest.mark.parametrize(
"offset1,offset2",
[
(BusinessHour(), BusinessHour(-1)),
(BusinessHour(start="09:00"), BusinessHour(start="09:01")),
(
BusinessHour(start="09:00", end="17:00"),
BusinessHour(start="17:00", end="09:01"),
),
(
BusinessHour(start=["13:00", "23:00"], end=["18:00", "07:00"]),
BusinessHour(start=["13:00", "23:00"], end=["17:00", "12:00"]),
),
],
)
def test_neq(self, offset1, offset2):
assert offset1 != offset2
@pytest.mark.parametrize(
"offset_name",
[
"offset1", "offset2", "offset3", "offset4", "offset8", "offset9",
"offset10"
],
)
def test_hash(self, offset_name):
offset = getattr(self, offset_name)
assert offset == offset
def test_call(self):
with tm.assert_produces_warning(FutureWarning):
# GH#34171 DateOffset.__call__ is deprecated
assert self.offset1(self.d) == datetime(2014, 7, 1, 11)
assert self.offset2(self.d) == datetime(2014, 7, 1, 13)
assert self.offset3(self.d) == datetime(2014, 6, 30, 17)
assert self.offset4(self.d) == datetime(2014, 6, 30, 14)
assert self.offset8(self.d) == datetime(2014, 7, 1, 11)
assert self.offset9(self.d) == datetime(2014, 7, 1, 22)
assert self.offset10(self.d) == datetime(2014, 7, 1, 1)
def test_sub(self):
# we have to override test_sub here because self.offset2 is not
# defined as self._offset(2)
off = self.offset2
msg = "Cannot subtract datetime from offset"
with pytest.raises(TypeError, match=msg):
off - self.d
assert 2 * off - off == off
assert self.d - self.offset2 == self.d + self._offset(-3)
def testRollback1(self):
assert self.offset1.rollback(self.d) == self.d
assert self.offset2.rollback(self.d) == self.d
assert self.offset3.rollback(self.d) == self.d
assert self.offset4.rollback(self.d) == self.d
assert self.offset5.rollback(self.d) == datetime(2014, 6, 30, 14, 30)
assert self.offset6.rollback(self.d) == datetime(2014, 7, 1, 5, 0)
assert self.offset7.rollback(self.d) == datetime(2014, 7, 1, 6, 30)
assert self.offset8.rollback(self.d) == self.d
assert self.offset9.rollback(self.d) == self.d
assert self.offset10.rollback(self.d) == datetime(2014, 7, 1, 2)
d = datetime(2014, 7, 1, 0)
assert self.offset1.rollback(d) == datetime(2014, 6, 30, 17)
assert self.offset2.rollback(d) == datetime(2014, 6, 30, 17)
assert self.offset3.rollback(d) == datetime(2014, 6, 30, 17)
assert self.offset4.rollback(d) == datetime(2014, 6, 30, 17)
assert self.offset5.rollback(d) == datetime(2014, 6, 30, 14, 30)
assert self.offset6.rollback(d) == d
assert self.offset7.rollback(d) == d
assert self.offset8.rollback(d) == datetime(2014, 6, 30, 17)
assert self.offset9.rollback(d) == d
assert self.offset10.rollback(d) == d
assert self._offset(5).rollback(self.d) == self.d
def testRollback2(self):
assert self._offset(-3).rollback(datetime(2014, 7, 5, 15,
0)) == datetime(
2014, 7, 4, 17, 0)
def testRollforward1(self):
assert self.offset1.rollforward(self.d) == self.d
assert self.offset2.rollforward(self.d) == self.d
assert self.offset3.rollforward(self.d) == self.d
assert self.offset4.rollforward(self.d) == self.d
assert self.offset5.rollforward(self.d) == datetime(2014, 7, 1, 11, 0)
assert self.offset6.rollforward(self.d) == datetime(2014, 7, 1, 20, 0)
assert self.offset7.rollforward(self.d) == datetime(2014, 7, 1, 21, 30)
assert self.offset8.rollforward(self.d) == self.d
assert self.offset9.rollforward(self.d) == self.d
assert self.offset10.rollforward(self.d) == datetime(2014, 7, 1, 13)
d = datetime(2014, 7, 1, 0)
assert self.offset1.rollforward(d) == datetime(2014, 7, 1, 9)
assert self.offset2.rollforward(d) == datetime(2014, 7, 1, 9)
assert self.offset3.rollforward(d) == datetime(2014, 7, 1, 9)
assert self.offset4.rollforward(d) == datetime(2014, 7, 1, 9)
assert self.offset5.rollforward(d) == datetime(2014, 7, 1, 11)
assert self.offset6.rollforward(d) == d
assert self.offset7.rollforward(d) == d
assert self.offset8.rollforward(d) == datetime(2014, 7, 1, 9)
assert self.offset9.rollforward(d) == d
assert self.offset10.rollforward(d) == d
assert self._offset(5).rollforward(self.d) == self.d
def testRollforward2(self):
assert self._offset(-3).rollforward(datetime(2014, 7, 5, 16,
0)) == datetime(
2014, 7, 7, 9)
def test_roll_date_object(self):
offset = BusinessHour()
dt = datetime(2014, 7, 6, 15, 0)
result = offset.rollback(dt)
assert result == datetime(2014, 7, 4, 17)
result = offset.rollforward(dt)
assert result == datetime(2014, 7, 7, 9)
normalize_cases = []
normalize_cases.append((
BusinessHour(normalize=True),
{
datetime(2014, 7, 1, 8): datetime(2014, 7, 1),
datetime(2014, 7, 1, 17): datetime(2014, 7, 2),
datetime(2014, 7, 1, 16): datetime(2014, 7, 2),
datetime(2014, 7, 1, 23): datetime(2014, 7, 2),
datetime(2014, 7, 1, 0): datetime(2014, 7, 1),
datetime(2014, 7, 4, 15): datetime(2014, 7, 4),
datetime(2014, 7, 4, 15, 59): datetime(2014, 7, 4),
datetime(2014, 7, 4, 16, 30): datetime(2014, 7, 7),
datetime(2014, 7, 5, 23): datetime(2014, 7, 7),
datetime(2014, 7, 6, 10): datetime(2014, 7, 7),
},
))
normalize_cases.append((
BusinessHour(-1, normalize=True),
{
datetime(2014, 7, 1, 8): datetime(2014, 6, 30),
datetime(2014, 7, 1, 17): datetime(2014, 7, 1),
datetime(2014, 7, 1, 16): datetime(2014, 7, 1),
datetime(2014, 7, 1, 10): datetime(2014, 6, 30),
datetime(2014, 7, 1, 0): datetime(2014, 6, 30),
datetime(2014, 7, 7, 10): datetime(2014, 7, 4),
datetime(2014, 7, 7, 10, 1): datetime(2014, 7, 7),
datetime(2014, 7, 5, 23): datetime(2014, 7, 4),
datetime(2014, 7, 6, 10): datetime(2014, 7, 4),
},
))
normalize_cases.append((
BusinessHour(1, normalize=True, start="17:00", end="04:00"),
{
datetime(2014, 7, 1, 8): datetime(2014, 7, 1),
datetime(2014, 7, 1, 17): datetime(2014, 7, 1),
datetime(2014, 7, 1, 23): datetime(2014, 7, 2),
datetime(2014, 7, 2, 2): datetime(2014, 7, 2),
datetime(2014, 7, 2, 3): datetime(2014, 7, 2),
datetime(2014, 7, 4, 23): datetime(2014, 7, 5),
datetime(2014, 7, 5, 2): datetime(2014, 7, 5),
datetime(2014, 7, 7, 2): datetime(2014, 7, 7),
datetime(2014, 7, 7, 17): datetime(2014, 7, 7),
},
))
@pytest.mark.parametrize("case", normalize_cases)
def test_normalize(self, case):
offset, cases = case
for dt, expected in cases.items():
assert offset.apply(dt) == expected
on_offset_cases = []
on_offset_cases.append((
BusinessHour(),
{
datetime(2014, 7, 1, 9): True,
datetime(2014, 7, 1, 8, 59): False,
datetime(2014, 7, 1, 8): False,
datetime(2014, 7, 1, 17): True,
datetime(2014, 7, 1, 17, 1): False,
datetime(2014, 7, 1, 18): False,
datetime(2014, 7, 5, 9): False,
datetime(2014, 7, 6, 12): False,
},
))
on_offset_cases.append((
BusinessHour(start="10:00", end="15:00"),
{
datetime(2014, 7, 1, 9): False,
datetime(2014, 7, 1, 10): True,
datetime(2014, 7, 1, 15): True,
datetime(2014, 7, 1, 15, 1): False,
datetime(2014, 7, 5, 12): False,
datetime(2014, 7, 6, 12): False,
},
))
on_offset_cases.append((
BusinessHour(start="19:00", end="05:00"),
{
datetime(2014, 7, 1, 9, 0): False,
datetime(2014, 7, 1, 10, 0): False,
datetime(2014, 7, 1, 15): False,
datetime(2014, 7, 1, 15, 1): False,
datetime(2014, 7, 5, 12, 0): False,
datetime(2014, 7, 6, 12, 0): False,
datetime(2014, 7, 1, 19, 0): True,
datetime(2014, 7, 2, 0, 0): True,
datetime(2014, 7, 4, 23): True,
datetime(2014, 7, 5, 1): True,
datetime(2014, 7, 5, 5, 0): True,
datetime(2014, 7, 6, 23, 0): False,
datetime(2014, 7, 7, 3, 0): False,
},
))
on_offset_cases.append((
BusinessHour(start=["09:00", "13:00"], end=["12:00", "17:00"]),
{
datetime(2014, 7, 1, 9): True,
datetime(2014, 7, 1, 8, 59): False,
datetime(2014, 7, 1, 8): False,
datetime(2014, 7, 1, 17): True,
datetime(2014, 7, 1, 17, 1): False,
datetime(2014, 7, 1, 18): False,
datetime(2014, 7, 5, 9): False,
datetime(2014, 7, 6, 12): False,
datetime(2014, 7, 1, 12, 30): False,
},
))
on_offset_cases.append((
BusinessHour(start=["19:00", "23:00"], end=["21:00", "05:00"]),
{
datetime(2014, 7, 1, 9, 0): False,
datetime(2014, 7, 1, 10, 0): False,
datetime(2014, 7, 1, 15): False,
datetime(2014, 7, 1, 15, 1): False,
datetime(2014, 7, 5, 12, 0): False,
datetime(2014, 7, 6, 12, 0): False,
datetime(2014, 7, 1, 19, 0): True,
datetime(2014, 7, 2, 0, 0): True,
datetime(2014, 7, 4, 23): True,
datetime(2014, 7, 5, 1): True,
datetime(2014, 7, 5, 5, 0): True,
datetime(2014, 7, 6, 23, 0): False,
datetime(2014, 7, 7, 3, 0): False,
datetime(2014, 7, 4, 22): False,
},
))
@pytest.mark.parametrize("case", on_offset_cases)
def test_is_on_offset(self, case):
offset, cases = case
for dt, expected in cases.items():
assert offset.is_on_offset(dt) == expected
apply_cases = [
(
BusinessHour(),
{
datetime(2014, 7, 1, 11):
datetime(2014, 7, 1, 12),
datetime(2014, 7, 1, 13):
datetime(2014, 7, 1, 14),
datetime(2014, 7, 1, 15):
datetime(2014, 7, 1, 16),
datetime(2014, 7, 1, 19):
datetime(2014, 7, 2, 10),
datetime(2014, 7, 1, 16):
datetime(2014, 7, 2, 9),
datetime(2014, 7, 1, 16, 30, 15):
datetime(2014, 7, 2, 9, 30, 15),
datetime(2014, 7, 1, 17):
datetime(2014, 7, 2, 10),
datetime(2014, 7, 2, 11):
datetime(2014, 7, 2, 12),
# out of business hours
datetime(2014, 7, 2, 8):
datetime(2014, 7, 2, 10),
datetime(2014, 7, 2, 19):
datetime(2014, 7, 3, 10),
datetime(2014, 7, 2, 23):
datetime(2014, 7, 3, 10),
datetime(2014, 7, 3, 0):
datetime(2014, 7, 3, 10),
# saturday
datetime(2014, 7, 5, 15):
datetime(2014, 7, 7, 10),
datetime(2014, 7, 4, 17):
datetime(2014, 7, 7, 10),
datetime(2014, 7, 4, 16, 30):
datetime(2014, 7, 7, 9, 30),
datetime(2014, 7, 4, 16, 30, 30):
datetime(2014, 7, 7, 9, 30, 30),
},
),
(
BusinessHour(4),
{
datetime(2014, 7, 1, 11): datetime(2014, 7, 1, 15),
datetime(2014, 7, 1, 13): datetime(2014, 7, 2, 9),
datetime(2014, 7, 1, 15): datetime(2014, 7, 2, 11),
datetime(2014, 7, 1, 16): datetime(2014, 7, 2, 12),
datetime(2014, 7, 1, 17): datetime(2014, 7, 2, 13),
datetime(2014, 7, 2, 11): datetime(2014, 7, 2, 15),
datetime(2014, 7, 2, 8): datetime(2014, 7, 2, 13),
datetime(2014, 7, 2, 19): datetime(2014, 7, 3, 13),
datetime(2014, 7, 2, 23): datetime(2014, 7, 3, 13),
datetime(2014, 7, 3, 0): datetime(2014, 7, 3, 13),
datetime(2014, 7, 5, 15): datetime(2014, 7, 7, 13),
datetime(2014, 7, 4, 17): datetime(2014, 7, 7, 13),
datetime(2014, 7, 4, 16, 30): datetime(2014, 7, 7, 12, 30),
datetime(2014, 7, 4, 16, 30, 30):
datetime(2014, 7, 7, 12, 30, 30),
},
),
(
BusinessHour(-1),
{
datetime(2014, 7, 1, 11):
datetime(2014, 7, 1, 10),
datetime(2014, 7, 1, 13):
datetime(2014, 7, 1, 12),
datetime(2014, 7, 1, 15):
datetime(2014, 7, 1, 14),
datetime(2014, 7, 1, 16):
datetime(2014, 7, 1, 15),
datetime(2014, 7, 1, 10):
datetime(2014, 6, 30, 17),
datetime(2014, 7, 1, 16, 30, 15):
datetime(2014, 7, 1, 15, 30, 15),
datetime(2014, 7, 1, 9, 30, 15):
datetime(2014, 6, 30, 16, 30, 15),
datetime(2014, 7, 1, 17):
datetime(2014, 7, 1, 16),
datetime(2014, 7, 1, 5):
datetime(2014, 6, 30, 16),
datetime(2014, 7, 2, 11):
datetime(2014, 7, 2, 10),
# out of business hours
datetime(2014, 7, 2, 8):
datetime(2014, 7, 1, 16),
datetime(2014, 7, 2, 19):
datetime(2014, 7, 2, 16),
datetime(2014, 7, 2, 23):
datetime(2014, 7, 2, 16),
datetime(2014, 7, 3, 0):
datetime(2014, 7, 2, 16),
# saturday
datetime(2014, 7, 5, 15):
datetime(2014, 7, 4, 16),
datetime(2014, 7, 7, 9):
datetime(2014, 7, 4, 16),
datetime(2014, 7, 7, 9, 30):
datetime(2014, 7, 4, 16, 30),
datetime(2014, 7, 7, 9, 30, 30):
datetime(2014, 7, 4, 16, 30, 30),
},
),
(
BusinessHour(-4),
{
datetime(2014, 7, 1, 11): datetime(2014, 6, 30, 15),
datetime(2014, 7, 1, 13): datetime(2014, 6, 30, 17),
datetime(2014, 7, 1, 15): datetime(2014, 7, 1, 11),
datetime(2014, 7, 1, 16): datetime(2014, 7, 1, 12),
datetime(2014, 7, 1, 17): datetime(2014, 7, 1, 13),
datetime(2014, 7, 2, 11): datetime(2014, 7, 1, 15),
datetime(2014, 7, 2, 8): datetime(2014, 7, 1, 13),
datetime(2014, 7, 2, 19): datetime(2014, 7, 2, 13),
datetime(2014, 7, 2, 23): datetime(2014, 7, 2, 13),
datetime(2014, 7, 3, 0): datetime(2014, 7, 2, 13),
datetime(2014, 7, 5, 15): datetime(2014, 7, 4, 13),
datetime(2014, 7, 4, 18): datetime(2014, 7, 4, 13),
datetime(2014, 7, 7, 9, 30): datetime(2014, 7, 4, 13, 30),
datetime(2014, 7, 7, 9, 30, 30):
datetime(2014, 7, 4, 13, 30, 30),
},
),
(
BusinessHour(start="13:00", end="16:00"),
{
datetime(2014, 7, 1, 11): datetime(2014, 7, 1, 14),
datetime(2014, 7, 1, 13): datetime(2014, 7, 1, 14),
datetime(2014, 7, 1, 15): datetime(2014, 7, 2, 13),
datetime(2014, 7, 1, 19): datetime(2014, 7, 2, 14),
datetime(2014, 7, 1, 16): datetime(2014, 7, 2, 14),
datetime(2014, 7, 1, 15, 30, 15):
datetime(2014, 7, 2, 13, 30, 15),
datetime(2014, 7, 5, 15): datetime(2014, 7, 7, 14),
datetime(2014, 7, 4, 17): datetime(2014, 7, 7, 14),
},
),
(
BusinessHour(n=2, start="13:00", end="16:00"),
{
datetime(2014, 7, 1, 17): datetime(2014, 7, 2, 15),
datetime(2014, 7, 2, 14): datetime(2014, 7, 3, 13),
datetime(2014, 7, 2, 8): datetime(2014, 7, 2, 15),
datetime(2014, 7, 2, 19): datetime(2014, 7, 3, 15),
datetime(2014, 7, 2, 14, 30): datetime(2014, 7, 3, 13, 30),
datetime(2014, 7, 3, 0): datetime(2014, 7, 3, 15),
datetime(2014, 7, 5, 15): datetime(2014, 7, 7, 15),
datetime(2014, 7, 4, 17): datetime(2014, 7, 7, 15),
datetime(2014, 7, 4, 14, 30): datetime(2014, 7, 7, 13, 30),
datetime(2014, 7, 4, 14, 30, 30):
datetime(2014, 7, 7, 13, 30, 30),
},
),
(
BusinessHour(n=-1, start="13:00", end="16:00"),
{
datetime(2014, 7, 2, 11): datetime(2014, 7, 1, 15),
datetime(2014, 7, 2, 13): datetime(2014, 7, 1, 15),
datetime(2014, 7, 2, 14): datetime(2014, 7, 1, 16),
datetime(2014, 7, 2, 15): datetime(2014, 7, 2, 14),
datetime(2014, 7, 2, 19): datetime(2014, 7, 2, 15),
datetime(2014, 7, 2, 16): datetime(2014, 7, 2, 15),
datetime(2014, 7, 2, 13, 30, 15):
datetime(2014, 7, 1, 15, 30, 15),
datetime(2014, 7, 5, 15): datetime(2014, 7, 4, 15),
datetime(2014, 7, 7, 11): datetime(2014, 7, 4, 15),
},
),
(
BusinessHour(n=-3, start="10:00", end="16:00"),
{
datetime(2014, 7, 1, 17): datetime(2014, 7, 1, 13),
datetime(2014, 7, 2, 14): datetime(2014, 7, 2, 11),
datetime(2014, 7, 2, 8): datetime(2014, 7, 1, 13),
datetime(2014, 7, 2, 13): datetime(2014, 7, 1, 16),
datetime(2014, 7, 2, 19): datetime(2014, 7, 2, 13),
datetime(2014, 7, 2, 11, 30): datetime(2014, 7, 1, 14, 30),
datetime(2014, 7, 3, 0): datetime(2014, 7, 2, 13),
datetime(2014, 7, 4, 10): datetime(2014, 7, 3, 13),
datetime(2014, 7, 5, 15): datetime(2014, 7, 4, 13),
datetime(2014, 7, 4, 16): datetime(2014, 7, 4, 13),
datetime(2014, 7, 4, 12, 30): datetime(2014, 7, 3, 15, 30),
datetime(2014, 7, 4, 12, 30, 30):
datetime(2014, 7, 3, 15, 30, 30),
},
),
(
BusinessHour(start="19:00", end="05:00"),
{
datetime(2014, 7, 1, 17): datetime(2014, 7, 1, 20),
datetime(2014, 7, 2, 14): datetime(2014, 7, 2, 20),
datetime(2014, 7, 2, 8): datetime(2014, 7, 2, 20),
datetime(2014, 7, 2, 13): datetime(2014, 7, 2, 20),
datetime(2014, 7, 2, 19): datetime(2014, 7, 2, 20),
datetime(2014, 7, 2, 4, 30): datetime(2014, 7, 2, 19, 30),
datetime(2014, 7, 3, 0): datetime(2014, 7, 3, 1),
datetime(2014, 7, 4, 10): datetime(2014, 7, 4, 20),
datetime(2014, 7, 4, 23): datetime(2014, 7, 5, 0),
datetime(2014, 7, 5, 0): datetime(2014, 7, 5, 1),
datetime(2014, 7, 5, 4): datetime(2014, 7, 7, 19),
datetime(2014, 7, 5, 4, 30): datetime(2014, 7, 7, 19, 30),
datetime(2014, 7, 5, 4, 30, 30):
datetime(2014, 7, 7, 19, 30, 30),
},
),
(
BusinessHour(n=-1, start="19:00", end="05:00"),
{
datetime(2014, 7, 1, 17): datetime(2014, 7, 1, 4),
datetime(2014, 7, 2, 14): datetime(2014, 7, 2, 4),
datetime(2014, 7, 2, 8): datetime(2014, 7, 2, 4),
datetime(2014, 7, 2, 13): datetime(2014, 7, 2, 4),
datetime(2014, 7, 2, 20): datetime(2014, 7, 2, 5),
datetime(2014, 7, 2, 19): datetime(2014, 7, 2, 4),
datetime(2014, 7, 2, 19, 30): datetime(2014, 7, 2, 4, 30),
datetime(2014, 7, 3, 0): datetime(2014, 7, 2, 23),
datetime(2014, 7, 3, 6): datetime(2014, 7, 3, 4),
datetime(2014, 7, 4, 23): datetime(2014, 7, 4, 22),
datetime(2014, 7, 5, 0): datetime(2014, 7, 4, 23),
datetime(2014, 7, 5, 4): datetime(2014, 7, 5, 3),
datetime(2014, 7, 7, 19, 30): datetime(2014, 7, 5, 4, 30),
datetime(2014, 7, 7, 19, 30, 30):
datetime(2014, 7, 5, 4, 30, 30),
},
),
(
BusinessHour(n=4, start="00:00", end="23:00"),
{
datetime(2014, 7, 3, 22): datetime(2014, 7, 4, 3),
datetime(2014, 7, 4, 22): datetime(2014, 7, 7, 3),
datetime(2014, 7, 3, 22, 30): datetime(2014, 7, 4, 3, 30),
datetime(2014, 7, 3, 22, 20): datetime(2014, 7, 4, 3, 20),
datetime(2014, 7, 4, 22, 30, 30):
datetime(2014, 7, 7, 3, 30, 30),
datetime(2014, 7, 4, 22, 30, 20):
datetime(2014, 7, 7, 3, 30, 20),
},
),
(
BusinessHour(n=-4, start="00:00", end="23:00"),
{
datetime(2014, 7, 4, 3): datetime(2014, 7, 3, 22),
datetime(2014, 7, 7, 3): datetime(2014, 7, 4, 22),
datetime(2014, 7, 4, 3, 30): datetime(2014, 7, 3, 22, 30),
datetime(2014, 7, 4, 3, 20): datetime(2014, 7, 3, 22, 20),
datetime(2014, 7, 7, 3, 30, 30):
datetime(2014, 7, 4, 22, 30, 30),
datetime(2014, 7, 7, 3, 30, 20):
datetime(2014, 7, 4, 22, 30, 20),
},
),
(
BusinessHour(start=["09:00", "14:00"], end=["12:00", "18:00"]),
{
datetime(2014, 7, 1, 11):
datetime(2014, 7, 1, 14),
datetime(2014, 7, 1, 15):
datetime(2014, 7, 1, 16),
datetime(2014, 7, 1, 19):
datetime(2014, 7, 2, 10),
datetime(2014, 7, 1, 16):
datetime(2014, 7, 1, 17),
datetime(2014, 7, 1, 16, 30, 15):
datetime(2014, 7, 1, 17, 30, 15),
datetime(2014, 7, 1, 17):
datetime(2014, 7, 2, 9),
datetime(2014, 7, 2, 11):
datetime(2014, 7, 2, 14),
# out of business hours
datetime(2014, 7, 1, 13):
datetime(2014, 7, 1, 15),
datetime(2014, 7, 2, 8):
datetime(2014, 7, 2, 10),
datetime(2014, 7, 2, 19):
datetime(2014, 7, 3, 10),
datetime(2014, 7, 2, 23):
datetime(2014, 7, 3, 10),
datetime(2014, 7, 3, 0):
datetime(2014, 7, 3, 10),
# saturday
datetime(2014, 7, 5, 15):
datetime(2014, 7, 7, 10),
datetime(2014, 7, 4, 17):
datetime(2014, 7, 7, 9),
datetime(2014, 7, 4, 17, 30):
datetime(2014, 7, 7, 9, 30),
datetime(2014, 7, 4, 17, 30, 30):
datetime(2014, 7, 7, 9, 30, 30),
},
),
(
BusinessHour(n=4, start=["09:00", "14:00"], end=["12:00",
"18:00"]),
{
datetime(2014, 7, 1, 11): datetime(2014, 7, 1, 17),
datetime(2014, 7, 1, 13): datetime(2014, 7, 2, 9),
datetime(2014, 7, 1, 15): datetime(2014, 7, 2, 10),
datetime(2014, 7, 1, 16): datetime(2014, 7, 2, 11),
datetime(2014, 7, 1, 17): datetime(2014, 7, 2, 14),
datetime(2014, 7, 2, 11): datetime(2014, 7, 2, 17),
datetime(2014, 7, 2, 8): datetime(2014, 7, 2, 15),
datetime(2014, 7, 2, 19): datetime(2014, 7, 3, 15),
datetime(2014, 7, 2, 23): datetime(2014, 7, 3, 15),
datetime(2014, 7, 3, 0): datetime(2014, 7, 3, 15),
datetime(2014, 7, 5, 15): datetime(2014, 7, 7, 15),
datetime(2014, 7, 4, 17): datetime(2014, 7, 7, 14),
datetime(2014, 7, 4, 16, 30): datetime(2014, 7, 7, 11, 30),
datetime(2014, 7, 4, 16, 30, 30):
datetime(2014, 7, 7, 11, 30, 30),
},
),
(
BusinessHour(n=-4,
start=["09:00", "14:00"],
end=["12:00", "18:00"]),
{
datetime(2014, 7, 1, 11): datetime(2014, 6, 30, 16),
datetime(2014, 7, 1, 13): datetime(2014, 6, 30, 17),
datetime(2014, 7, 1, 15): datetime(2014, 6, 30, 18),
datetime(2014, 7, 1, 16): datetime(2014, 7, 1, 10),
datetime(2014, 7, 1, 17): datetime(2014, 7, 1, 11),
datetime(2014, 7, 2, 11): datetime(2014, 7, 1, 16),
datetime(2014, 7, 2, 8): datetime(2014, 7, 1, 12),
datetime(2014, 7, 2, 19): datetime(2014, 7, 2, 12),
datetime(2014, 7, 2, 23): datetime(2014, 7, 2, 12),
datetime(2014, 7, 3, 0): datetime(2014, 7, 2, 12),
datetime(2014, 7, 5, 15): datetime(2014, 7, 4, 12),
datetime(2014, 7, 4, 18): datetime(2014, 7, 4, 12),
datetime(2014, 7, 7, 9, 30): datetime(2014, 7, 4, 14, 30),
datetime(2014, 7, 7, 9, 30, 30):
datetime(2014, 7, 4, 14, 30, 30),
},
),
(
BusinessHour(n=-1,
start=["19:00", "03:00"],
end=["01:00", "05:00"]),
{
datetime(2014, 7, 1, 17): datetime(2014, 7, 1, 4),
datetime(2014, 7, 2, 14): datetime(2014, 7, 2, 4),
datetime(2014, 7, 2, 8): datetime(2014, 7, 2, 4),
datetime(2014, 7, 2, 13): datetime(2014, 7, 2, 4),
datetime(2014, 7, 2, 20): datetime(2014, 7, 2, 5),
datetime(2014, 7, 2, 19): datetime(2014, 7, 2, 4),
datetime(2014, 7, 2, 4): datetime(2014, 7, 2, 1),
datetime(2014, 7, 2, 19, 30): datetime(2014, 7, 2, 4, 30),
datetime(2014, 7, 3, 0): datetime(2014, 7, 2, 23),
datetime(2014, 7, 3, 6): datetime(2014, 7, 3, 4),
datetime(2014, 7, 4, 23): datetime(2014, 7, 4, 22),
datetime(2014, 7, 5, 0): datetime(2014, 7, 4, 23),
datetime(2014, 7, 5, 4): datetime(2014, 7, 5, 0),
datetime(2014, 7, 7, 3, 30): datetime(2014, 7, 5, 0, 30),
datetime(2014, 7, 7, 19, 30): datetime(2014, 7, 7, 4, 30),
datetime(2014, 7, 7, 19, 30, 30):
datetime(2014, 7, 7, 4, 30, 30),
},
),
]
# long business hours (see gh-26381)
# multiple business hours
@pytest.mark.parametrize("case", apply_cases)
def test_apply(self, case):
offset, cases = case
for base, expected in cases.items():
assert_offset_equal(offset, base, expected)
apply_large_n_cases = [
(
# A week later
BusinessHour(40),
{
datetime(2014, 7, 1, 11): datetime(2014, 7, 8, 11),
datetime(2014, 7, 1, 13): datetime(2014, 7, 8, 13),
datetime(2014, 7, 1, 15): datetime(2014, 7, 8, 15),
datetime(2014, 7, 1, 16): datetime(2014, 7, 8, 16),
datetime(2014, 7, 1, 17): datetime(2014, 7, 9, 9),
datetime(2014, 7, 2, 11): datetime(2014, 7, 9, 11),
datetime(2014, 7, 2, 8): datetime(2014, 7, 9, 9),
datetime(2014, 7, 2, 19): datetime(2014, 7, 10, 9),
datetime(2014, 7, 2, 23): datetime(2014, 7, 10, 9),
datetime(2014, 7, 3, 0): datetime(2014, 7, 10, 9),
datetime(2014, 7, 5, 15): datetime(2014, 7, 14, 9),
datetime(2014, 7, 4, 18): datetime(2014, 7, 14, 9),
datetime(2014, 7, 7, 9, 30): datetime(2014, 7, 14, 9, 30),
datetime(2014, 7, 7, 9, 30, 30):
datetime(2014, 7, 14, 9, 30, 30),
},
),
(
# 3 days and 1 hour before
BusinessHour(-25),
{
datetime(2014, 7, 1, 11): datetime(2014, 6, 26, 10),
datetime(2014, 7, 1, 13): datetime(2014, 6, 26, 12),
datetime(2014, 7, 1, 9): datetime(2014, 6, 25, 16),
datetime(2014, 7, 1, 10): datetime(2014, 6, 25, 17),
datetime(2014, 7, 3, 11): datetime(2014, 6, 30, 10),
datetime(2014, 7, 3, 8): datetime(2014, 6, 27, 16),
datetime(2014, 7, 3, 19): datetime(2014, 6, 30, 16),
datetime(2014, 7, 3, 23): datetime(2014, 6, 30, 16),
datetime(2014, 7, 4, 9): datetime(2014, 6, 30, 16),
datetime(2014, 7, 5, 15): datetime(2014, 7, 1, 16),
datetime(2014, 7, 6, 18): datetime(2014, 7, 1, 16),
datetime(2014, 7, 7, 9, 30): datetime(2014, 7, 1, 16, 30),
datetime(2014, 7, 7, 10, 30, 30):
datetime(2014, 7, 2, 9, 30, 30),
},
),
(
# 5 days and 3 hours later
BusinessHour(28, start="21:00", end="02:00"),
{
datetime(2014, 7, 1, 11): datetime(2014, 7, 9, 0),
datetime(2014, 7, 1, 22): datetime(2014, 7, 9, 1),
datetime(2014, 7, 1, 23): datetime(2014, 7, 9, 21),
datetime(2014, 7, 2, 2): datetime(2014, 7, 10, 0),
datetime(2014, 7, 3, 21): datetime(2014, 7, 11, 0),
datetime(2014, 7, 4, 1): datetime(2014, 7, 11, 23),
datetime(2014, 7, 4, 2): datetime(2014, 7, 12, 0),
datetime(2014, 7, 4, 3): datetime(2014, 7, 12, 0),
datetime(2014, 7, 5, 1): datetime(2014, 7, 14, 23),
datetime(2014, 7, 5, 15): datetime(2014, 7, 15, 0),
datetime(2014, 7, 6, 18): datetime(2014, 7, 15, 0),
datetime(2014, 7, 7, 1): datetime(2014, 7, 15, 0),
datetime(2014, 7, 7, 23, 30): datetime(2014, 7, 15, 21, 30),
},
),
(
# large n for multiple opening hours (3 days and 1 hour before)
BusinessHour(n=-25,
start=["09:00", "14:00"],
end=["12:00", "19:00"]),
{
datetime(2014, 7, 1, 11): datetime(2014, 6, 26, 10),
datetime(2014, 7, 1, 13): datetime(2014, 6, 26, 11),
datetime(2014, 7, 1, 9): datetime(2014, 6, 25, 18),
datetime(2014, 7, 1, 10): datetime(2014, 6, 25, 19),
datetime(2014, 7, 3, 11): datetime(2014, 6, 30, 10),
datetime(2014, 7, 3, 8): datetime(2014, 6, 27, 18),
datetime(2014, 7, 3, 19): datetime(2014, 6, 30, 18),
datetime(2014, 7, 3, 23): datetime(2014, 6, 30, 18),
datetime(2014, 7, 4, 9): datetime(2014, 6, 30, 18),
datetime(2014, 7, 5, 15): datetime(2014, 7, 1, 18),
datetime(2014, 7, 6, 18): datetime(2014, 7, 1, 18),
datetime(2014, 7, 7, 9, 30): datetime(2014, 7, 1, 18, 30),
datetime(2014, 7, 7, 10, 30, 30):
datetime(2014, 7, 2, 9, 30, 30),
},
),
(
# 5 days and 3 hours later
BusinessHour(28, start=["21:00", "03:00"], end=["01:00", "04:00"]),
{
datetime(2014, 7, 1, 11): datetime(2014, 7, 9, 0),
datetime(2014, 7, 1, 22): datetime(2014, 7, 9, 3),
datetime(2014, 7, 1, 23): datetime(2014, 7, 9, 21),
datetime(2014, 7, 2, 2): datetime(2014, 7, 9, 23),
datetime(2014, 7, 3, 21): datetime(2014, 7, 11, 0),
datetime(2014, 7, 4, 1): datetime(2014, 7, 11, 23),
datetime(2014, 7, 4, 2): datetime(2014, 7, 11, 23),
datetime(2014, 7, 4, 3): datetime(2014, 7, 11, 23),
datetime(2014, 7, 4, 21): datetime(2014, 7, 12, 0),
datetime(2014, 7, 5, 0): datetime(2014, 7, 14, 22),
datetime(2014, 7, 5, 1): datetime(2014, 7, 14, 23),
datetime(2014, 7, 5, 15): datetime(2014, 7, 14, 23),
datetime(2014, 7, 6, 18): datetime(2014, 7, 14, 23),
datetime(2014, 7, 7, 1): datetime(2014, 7, 14, 23),
datetime(2014, 7, 7, 23, 30): datetime(2014, 7, 15, 21, 30),
},
),
]
@pytest.mark.parametrize("case", apply_large_n_cases)
def test_apply_large_n(self, case):
offset, cases = case
for base, expected in cases.items():
assert_offset_equal(offset, base, expected)
def test_apply_nanoseconds(self):
tests = [
(
BusinessHour(),
{
Timestamp("2014-07-04 15:00") + Nano(5):
Timestamp("2014-07-04 16:00") + Nano(5),
Timestamp("2014-07-04 16:00") + Nano(5):
Timestamp("2014-07-07 09:00") + Nano(5),
Timestamp("2014-07-04 16:00") - Nano(5):
Timestamp("2014-07-04 17:00") - Nano(5),
},
),
(
BusinessHour(-1),
{
Timestamp("2014-07-04 15:00") + Nano(5):
Timestamp("2014-07-04 14:00") + Nano(5),
Timestamp("2014-07-04 10:00") + Nano(5):
Timestamp("2014-07-04 09:00") + Nano(5),
Timestamp("2014-07-04 10:00") - Nano(5):
Timestamp("2014-07-03 17:00") - Nano(5),
},
),
]
for offset, cases in tests:
for base, expected in cases.items():
assert_offset_equal(offset, base, expected)
def test_datetimeindex(self):
idx1 = date_range(start="2014-07-04 15:00",
end="2014-07-08 10:00",
freq="BH")
idx2 = date_range(start="2014-07-04 15:00", periods=12, freq="BH")
idx3 = date_range(end="2014-07-08 10:00", periods=12, freq="BH")
expected = DatetimeIndex(
[
"2014-07-04 15:00",
"2014-07-04 16:00",
"2014-07-07 09:00",
"2014-07-07 10:00",
"2014-07-07 11:00",
"2014-07-07 12:00",
"2014-07-07 13:00",
"2014-07-07 14:00",
"2014-07-07 15:00",
"2014-07-07 16:00",
"2014-07-08 09:00",
"2014-07-08 10:00",
],
freq="BH",
)
for idx in [idx1, idx2, idx3]:
tm.assert_index_equal(idx, expected)
idx1 = date_range(start="2014-07-04 15:45",
end="2014-07-08 10:45",
freq="BH")
idx2 = date_range(start="2014-07-04 15:45", periods=12, freq="BH")
idx3 = date_range(end="2014-07-08 10:45", periods=12, freq="BH")
expected = idx1
for idx in [idx1, idx2, idx3]:
tm.assert_index_equal(idx, expected)
def test_bday_ignores_timedeltas(self):
idx = date_range("2010/02/01", "2010/02/10", freq="12H")
t1 = idx + BDay(offset=Timedelta(3, unit="H"))
expected = DatetimeIndex(
[
"2010-02-02 03:00:00",
"2010-02-02 15:00:00",
"2010-02-03 03:00:00",
"2010-02-03 15:00:00",
"2010-02-04 03:00:00",
"2010-02-04 15:00:00",
"2010-02-05 03:00:00",
"2010-02-05 15:00:00",
"2010-02-08 03:00:00",
"2010-02-08 15:00:00",
"2010-02-08 03:00:00",
"2010-02-08 15:00:00",
"2010-02-08 03:00:00",
"2010-02-08 15:00:00",
"2010-02-09 03:00:00",
"2010-02-09 15:00:00",
"2010-02-10 03:00:00",
"2010-02-10 15:00:00",
"2010-02-11 03:00:00",
],
freq=None,
)
tm.assert_index_equal(t1, expected)
def testTimeOnlyUTC(self):
# https://github.com/dateutil/dateutil/issues/132
# tzutc doesn't care
tz_utc = tz.tzutc()
self.assertEqual(
dt_time(13, 20, tzinfo=tz_utc).utcoffset(), timedelta(0))
def get_previous_week(date):
iso_date = date.isocalendar()
end = dt.combine(dt_date.fromisocalendar(iso_date[0], iso_date[1], 1),
dt_time(tzinfo=tz.utc))
start = end - dt_delta(weeks=1)
return start, end
def testTimeOnlyLocal(self):
# tzlocal returns None
tz_local = tz.tzlocal()
self.assertIs(dt_time(13, 20, tzinfo=tz_local).utcoffset(), None)
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
process_start_time = time.time() # measure process execution time ...
response.update_status('execution started at : %s ' % dt.now(), 5)
start_time = time.time() # measure init ...
resource = archiveextract(
resource=rename_complexinputs(request.inputs['resource']))
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
regrset = request.inputs['regrset'][0].data
# fix 31 December issue
# refSt = dt.combine(refSt,dt_time(12,0))
# refEn = dt.combine(refEn,dt_time(12,0))
# dateSt = dt.combine(dateSt,dt_time(12,0))
# dateEn = dt.combine(dateEn,dt_time(12,0))
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
# bbox = [-80, 20, 50, 70]
# TODO: Add checking for wrong cordinates and apply default if nesessary
bbox = []
bboxStr = request.inputs['BBox'][0].data
bboxStr = bboxStr.split(',')
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
direction = request.inputs['direction'][0].data
normalize = request.inputs['normalize'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
model_var = request.inputs['reanalyses'][0].data
model, var = model_var.split('_')
try:
if direction == 're2mo':
anaSt = dt.combine(dateSt, dt_time(
0, 0)) #dt.strptime(dateSt[0], '%Y-%m-%d')
anaEn = dt.combine(dateEn, dt_time(
0, 0)) #dt.strptime(dateEn[0], '%Y-%m-%d')
refSt = dt.combine(refSt, dt_time(
12, 0)) #dt.strptime(refSt[0], '%Y-%m-%d')
refEn = dt.combine(refEn, dt_time(
12, 0)) #dt.strptime(refEn[0], '%Y-%m-%d')
r_time_range = [anaSt, anaEn]
m_time_range = [refSt, refEn]
elif direction == 'mo2re':
anaSt = dt.combine(dateSt, dt_time(
12, 0)) #dt.strptime(refSt[0], '%Y-%m-%d')
anaEn = dt.combine(dateEn, dt_time(
12, 0)) #dt.strptime(refEn[0], '%Y-%m-%d')
refSt = dt.combine(refSt, dt_time(
0, 0)) #dt.strptime(dateSt[0], '%Y-%m-%d')
refEn = dt.combine(refEn, dt_time(
0, 0)) #dt.strptime(dateEn[0], '%Y-%m-%d')
r_time_range = [refSt, refEn]
m_time_range = [anaSt, anaEn]
else:
LOGGER.exception(
'failed to find time periods for comparison direction')
except:
msg = 'failed to put simulation and reference time in order'
LOGGER.exception(msg)
raise Exception(msg)
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.exception('output format not valid')
try:
if model == 'NCEP':
getlevel = True
if 'z' in var:
level = var.strip('z')
variable = 'hgt'
# conform_units_to='hPa'
else:
variable = 'slp'
level = None
# conform_units_to='hPa'
elif '20CRV2' in model:
getlevel = False
if 'z' in var:
variable = 'hgt'
level = var.strip('z')
# conform_units_to=None
else:
variable = 'prmsl'
level = None
# conform_units_to='hPa'
else:
LOGGER.exception('Reanalyses model not known')
LOGGER.info('environment set')
except:
msg = 'failed to set environment'
LOGGER.exception(msg)
raise Exception(msg)
# LOGGER.exception("init took %s seconds.", time.time() - start_time)
response.update_status('Read in the arguments', 6)
#################
# get input data
#################
# TODO: do not forget to select years
start_time = time.time() # measure get_input_data ...
response.update_status('fetching input data', 7)
try:
if direction == 're2mo':
nc_reanalyses = reanalyses(start=anaSt.year,
end=anaEn.year,
variable=var,
dataset=model,
getlevel=getlevel)
else:
nc_reanalyses = reanalyses(start=refSt.year,
end=refEn.year,
variable=var,
dataset=model,
getlevel=getlevel)
if type(nc_reanalyses) == list:
nc_reanalyses = sorted(
nc_reanalyses,
key=lambda i: path.splitext(path.basename(i))[0])
else:
nc_reanalyses = [nc_reanalyses]
# For 20CRV2 geopotential height, daily dataset for 100 years is about 50 Gb
# So it makes sense, to operate it step-by-step
# TODO: need to create dictionary for such datasets (for models as well)
# TODO: benchmark the method bellow for NCEP z500 for 60 years, may be use the same (!)
# TODO Now everything regrid to the reanalysis
if ('20CRV2' in model) and ('z' in var):
tmp_total = []
origvar = get_variable(nc_reanalyses)
for z in nc_reanalyses:
tmp_n = 'tmp_%s' % (uuid.uuid1())
b0 = call(resource=z,
variable=origvar,
level_range=[int(level), int(level)],
geom=bbox,
spatial_wrapping='wrap',
prefix='levdom_' + path.basename(z)[0:-3])
tmp_total.append(b0)
tmp_total = sorted(
tmp_total,
key=lambda i: path.splitext(path.basename(i))[0])
inter_subset_tmp = call(resource=tmp_total,
variable=origvar,
time_range=r_time_range)
# Clean
for i in tmp_total:
tbr = 'rm -f %s' % (i)
#system(tbr)
# Create new variable
ds = Dataset(inter_subset_tmp, mode='a')
z_var = ds.variables.pop(origvar)
dims = z_var.dimensions
new_var = ds.createVariable('z%s' % level,
z_var.dtype,
dimensions=(dims[0], dims[2],
dims[3]))
new_var[:, :, :] = squeeze(z_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
nc_subset = call(inter_subset_tmp, variable='z%s' % level)
else:
nc_subset = call(
resource=nc_reanalyses,
variable=var,
geom=bbox,
spatial_wrapping='wrap',
time_range=r_time_range,
# conform_units_to=conform_units_to
)
# nc_subset = call(resource=nc_reanalyses, variable=var, geom=bbox, spatial_wrapping='wrap') # XXXXXX wrap
# LOGGER.exception("get_input_subset_model took %s seconds.", time.time() - start_time)
response.update_status('**** Input reanalyses data fetched', 10)
except:
msg = 'failed to fetch or subset input files'
LOGGER.exception(msg)
raise Exception(msg)
########################
# input data preperation
########################
response.update_status('Start preparing input data', 12)
# Filter resource:
if type(resource) == list:
resource = sorted(resource,
key=lambda i: path.splitext(path.basename(i))[0])
else:
resource = [resource]
tmp_resource = []
m_start = m_time_range[0]
m_end = m_time_range[1]
for re in resource:
s, e = get_timerange(re)
tmpSt = dt.strptime(s, '%Y%m%d')
tmpEn = dt.strptime(e, '%Y%m%d')
if ((tmpSt <= m_end) and (tmpEn >= m_start)):
tmp_resource.append(re)
LOGGER.debug('Selected file: %s ' % (re))
resource = tmp_resource
start_time = time.time() # mesure data preperation ...
# TODO: Check the callendars ! for model vs reanalyses.
# TODO: Check the units! model vs reanalyses.
try:
m_total = []
modvar = get_variable(resource)
# resource properties
ds = Dataset(resource[0])
m_var = ds.variables[modvar]
dims = list(m_var.dimensions)
dimlen = len(dims)
try:
model_id = ds.getncattr('model_id')
except AttributeError:
model_id = 'Unknown model'
LOGGER.debug('MODEL: %s ' % (model_id))
lev_units = 'hPa'
if (dimlen > 3):
lev = ds.variables[dims[1]]
# actually index [1] need to be detected... assuming zg(time, plev, lat, lon)
lev_units = lev.units
if (lev_units == 'Pa'):
m_level = str(int(level) * 100)
else:
m_level = level
else:
m_level = None
if level == None:
level_range = None
else:
level_range = [int(m_level), int(m_level)]
for z in resource:
tmp_n = 'tmp_%s' % (uuid.uuid1())
# select level and regrid
b0 = call(
resource=z,
variable=modvar,
level_range=level_range,
spatial_wrapping='wrap', #cdover='system',
regrid_destination=nc_reanalyses[0],
regrid_options='bil',
prefix=tmp_n)
# select domain
b01 = call(resource=b0,
geom=bbox,
spatial_wrapping='wrap',
prefix='levregr_' + path.basename(z)[0:-3])
tbr = 'rm -f %s' % (b0)
#system(tbr)
tbr = 'rm -f %s' % (tmp_n)
#system(tbr)
# get full resource
m_total.append(b01)
ds.close()
model_subset = call(m_total, time_range=m_time_range)
for i in m_total:
tbr = 'rm -f %s' % (i)
#system(tbr)
if m_level is not None:
# Create new variable in model set
ds = Dataset(model_subset, mode='a')
mod_var = ds.variables.pop(modvar)
dims = mod_var.dimensions
new_modvar = ds.createVariable('z%s' % level,
mod_var.dtype,
dimensions=(dims[0], dims[2],
dims[3]))
new_modvar[:, :, :] = squeeze(mod_var[:, 0, :, :])
# new_var.setncatts({k: z_var.getncattr(k) for k in z_var.ncattrs()})
ds.close()
mod_subset = call(model_subset, variable='z%s' % level)
else:
mod_subset = model_subset
# if direction == 're2mo':
# try:
# response.update_status('Preparing simulation data', 15)
# reanalyses_subset = call(resource=nc_subset, time_range=[anaSt, anaEn])
# except:
# msg = 'failed to prepare simulation period'
# LOGGER.exception(msg)
# try:
# response.update_status('Preparing target data', 17)
# var_target = get_variable(resource)
# # var_simulation = get_variable(simulation)
# model_subset_tmp = call(resource=resource, variable=var_target,
# time_range=[refSt, refEn],
# t_calendar='standard',
# spatial_wrapping='wrap',
# regrid_destination=nc_reanalyses[0],
# regrid_options='bil')
# # model_subset = call(resource=resource, variable=var_target,
# # time_range=[refSt, refEn],
# # geom=bbox,
# # t_calendar='standard',
# # # conform_units_to=conform_units_to,
# # spatial_wrapping='wrap',
# # regrid_destination=reanalyses_subset,
# # regrid_options='bil') # XXXXXXXXXXXX ADD WRAP rem calendar
# model_subset = call(resource=model_subset_tmp,variable=var_target, geom=bbox, spatial_wrapping='wrap', t_calendar='standard')
# # ISSUE: the regrided model has white border with null! Check it.
# # check t_calendar!
# except:
# msg = 'failed subset archive model'
# LOGGER.exception(msg)
# raise Exception(msg)
# else:
# try:
# response.update_status('Preparing target data', 15)
# var_target = get_variable(resource)
# # var_simulation = get_variable(simulation)
# model_subset = call(resource=resource, variable=var_target,
# time_range=[refSt, refEn],
# geom=bbox,
# t_calendar='standard',
# # conform_units_to=conform_units_to,
# # spatial_wrapping='wrap',
# )
# except:
# msg = 'failed subset archive model'
# LOGGER.exception(msg)
# raise Exception(msg)
# try:
# response.update_status('Preparing simulation data', 17)
# reanalyses_subset = call(resource=nc_subset,
# time_range=[anaSt, anaEn],
# regrid_destination=model_subset,
# regrid_options='bil')
# except:
# msg = 'failed to prepare simulation period'
# LOGGER.exception(msg)
except:
msg = 'failed to subset simulation or reference data'
LOGGER.exception(msg)
raise Exception(msg)
# --------------------------------------------
try:
if direction == 'mo2re':
simulation = mod_subset
archive = nc_subset
base_id = model
sim_id = model_id
elif direction == 're2mo':
simulation = nc_subset
archive = mod_subset
base_id = model_id
sim_id = model
else:
LOGGER.exception('direction not valid: %s ' % direction)
except:
msg = 'failed to find comparison direction'
LOGGER.exception(msg)
raise Exception(msg)
try:
if level is not None:
out_var = 'z%s' % level
else:
var_archive = get_variable(archive)
var_simulation = get_variable(simulation)
if var_archive != var_simulation:
rename_variable(archive,
oldname=var_archive,
newname=var_simulation)
out_var = var_simulation
LOGGER.info('varname %s in netCDF renamed to %s' %
(var_archive, var_simulation))
except:
msg = 'failed to rename variable in target files'
LOGGER.exception(msg)
raise Exception(msg)
try:
if seacyc is True:
seasoncyc_base, seasoncyc_sim = analogs.seacyc(
archive, simulation, method=normalize)
else:
seasoncyc_base = None
seasoncyc_sim = None
except:
msg = 'failed to prepare seasonal cycle reference files'
LOGGER.exception(msg)
raise Exception(msg)
ip, output = mkstemp(dir='.', suffix='.txt')
output_file = path.abspath(output)
files = [path.abspath(archive), path.abspath(simulation), output_file]
# LOGGER.exception("data preperation took %s seconds.", time.time() - start_time)
############################
# generating the config file
############################
response.update_status('writing config file', 18)
start_time = time.time() # measure write config ...
try:
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=base_id,
sim_id=sim_id,
timewin=timewin,
varname=var,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
period=[
dt.strftime(refSt, '%Y-%m-%d'),
dt.strftime(refEn, '%Y-%m-%d')
],
bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3]))
except:
msg = 'failed to generate config file'
LOGGER.exception(msg)
raise Exception(msg)
# LOGGER.exception("write_config took %s seconds.", time.time() - start_time)
#######################
# CASTf90 call
#######################
import subprocess
import shlex
start_time = time.time() # measure call castf90
response.update_status('Start CASTf90 call', 20)
try:
# response.update_status('execution of CASTf90', 50)
cmd = 'analogue.out %s' % path.relpath(config_file)
# system(cmd)
args = shlex.split(cmd)
output, error = subprocess.Popen(
args, stdout=subprocess.PIPE,
stderr=subprocess.PIPE).communicate()
LOGGER.info('analogue.out info:\n %s ' % output)
LOGGER.exception('analogue.out errors:\n %s ' % error)
response.update_status('**** CASTf90 suceeded', 90)
except:
msg = 'CASTf90 failed'
LOGGER.exception(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
response.update_status('preparting output', 91)
# Stopper to keep twitcher results, for debug
# dummy=dummy
response.outputs[
'config'].file = config_file #config_output_url # config_file )
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
response.outputs['target_netcdf'].file = archive
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 95)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 99)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
return response
def testTimeOnlyGettz(self):
# gettz returns None
tz_get = tz.gettz('Europe/Minsk')
self.assertIs(dt_time(13, 20, tzinfo=tz_get).utcoffset(), None)
import re
import locale
from datetime import time as dt_time
_DIRECT_PATTERN = re.compile(r'%[A-z]')
_C_KEY = locale.nl_langinfo(locale.D_T_FMT)
_XD_KEY = locale.nl_langinfo(locale.D_FMT)
_XT_KEY = locale.nl_langinfo(locale.T_FMT)
_AMPM_KEY = '({am}|{pm})'.format(am=dt_time().strftime('%p'),
pm=dt_time(12).strftime('%p'))
_DIRECTIVES = {
r'\w+': ('%a', '%A', '%b', '%B'),
r'\d{2}': ('%d', '%H', '%I', '%j', '%m', '%M', '%S', '%U', '%W', '%y'),
r'\d{4,}': ('%Y', ),
r'\d': ('%w', ),
_AMPM_KEY: ('%p', ),
_C_KEY: ('%c', ),
_XD_KEY: ('%x', ),
_XT_KEY: ('%X', ),
r'(UTC|EST|CST)?': ('%Z', ),
r'(\+\d{4})?': ('%z', ),
}
def convert(dt_format):
_special = ('%c', '%x', '%X', )
def get_day_end(day):
tzinfo = None
if isinstance(day, datetime):
tzinfo = day.tzinfo
day = day.date()
return datetime.combine(day, dt_time(23, 59, tzinfo=tzinfo))
def extract_datetime(s):
'''Converts a datetime string into a datetime object'''
now = datetime.now()
args = s.upper().split()
if len(args) == 1:
# This is a good way to check if a single parameter is time,
# because standalone time always has one of these substrings.
# Careful though! If time is preceded by date, it's possible
# to omit the colon for 24-hour time strings (e.g. "pdc 3/2 15").
if ':' in args[0] or 'PM' in args[0] or 'AM' in args[0]:
time = args[0]
date = None
else:
time = None
date = args[0]
elif len(args) == 2:
date, time = args
else:
print('Too many arguments. See "pdc --help" for reference.')
return None
day, month, year = (None, ) * 3
hour, minute, second = (None, ) * 3
pm_am = None
if date is not None:
# Extract date
match = re.match(r'^(?P<day>\d+)\.(?P<month>\d+)(\.(?P<year>\d+))?$',
date)
if match is None:
match = re.match(r'^(?P<month>\d+)/(?P<day>\d+)(/(?P<year>\d+))?$',
date)
if match is None:
match = re.match(
r'^(?P<year>\d+)-(?P<month>\d+)(-(?P<day>\d+))?$', date)
if match is None:
print(
'Unrecognized date format. Available: "dd.mm.yyyy", "mm/dd/yyyy", "yyyy-mm-dd"'
)
return None
# Notice that in the first two formats year can be omitted,
# while in the third format the day can be omitted.
# If a day is omitted, we simply assume it to be 1.
# If a year is omitted however, we have to pick the lowest possible
# year for the rest of the datetime to make sense,
# i.e. if it is currently August and the user request a date
# in November, the year is assumed to be the current one,
# but if the requested a date in March, the year would have to be
# the subsequent one. The check will be performed later.
# Store matches
day = match.group('day')
month = match.group('month')
year = match.group('year')
# Convert known values
day = int(day if day is not None else 1)
month = int(month)
year = int(year) if year is not None else None
# If date is omitted completely and there's only time,
# then depending on the current time the date will either be
# today or tomorrow. This check is also performed later.
if time is not None:
# Extract time
# Some match strings contain unmatchable groups, e.g. '...$(?P<unmatchable>_)?'
# This is a hacky way to enforce that these named groups always exist in the match object,
# to avoid ugly checks later on. Basically, everything that doesn't exist always has None value.
match = re.match(
r'^(?P<hour>\d+):(?P<minute>\d+)(:(?P<second>\d+))?$(?P<pm_am>_)?',
time)
if match is None:
match = re.match(
r'^(?P<hour>\d+)(:(?P<minute>\d+)(:(?P<second>\d+))?)?(?P<pm_am>PM|AM)',
time)
if match is None and date is not None:
match = re.match(
r'^(?P<hour>\d+)$(?P<minute>_)?(?P<second>_)?(?P<pm_am>_)?',
time)
if match is None:
print(
'Unrecognized time format. See "pdc --help" for valid examples.'
)
return None
# Store matches
hour = match.group('hour')
minute = match.group('minute')
second = match.group('second')
pm_am = match.group('pm_am')
# Convert to int and if any time values were omitted, assume 0
hour = int(hour if hour is not None else 0)
minute = int(minute if minute is not None else 0)
second = int(second if second is not None else 0)
# Verify time validity
try:
dt = dt_time(hour, minute, second)
except Exception as e:
print(f'{type(e).__name__}: {e}')
return None
# Convert 12-hour time to 24-hour time
if pm_am is not None:
if hour == 12 and pm_am == 'AM':
hour = 0
elif hour != 12 and pm_am == 'PM':
hour += 12
# If date was omitted, find the closest suitable one
if date is None:
d = datetime(now.year, now.month, now.day, hour, minute, second)
if d < now:
d += timedelta(1)
day = d.day
month = d.month
year = d.year
# If year was omitted, find the closest suitable one
elif year is None:
# Verify month and day validity
# Use 2016 as year, because it was a leap year (Feb 29 is a valid date)
try:
dt = dt_date(2016, month, day)
except Exception as e:
print(f'{type(e).__name__}: {e}')
return None
# Increment year till the date is valid and in the future
year = now.year
dt = datetime(year, month, day, hour, minute, second)
while dt < now:
year += 1
try:
dt = datetime(year, month, day, hour, minute, second)
except ValueError:
continue
# Verify full timedate validity
try:
dt = datetime(year, month, day, hour, minute, second)
except Exception as e:
print(f'{type(e).__name__}: {e}')
return None
return dt
transition_functions = [SimpleTransition, dissolve_changes_only]
else:
raise ValueError("Unsupported platform - must be MacOS or Linux")
# set up list of transit messages - since this is static, it is done outside the loop
lstTransitMessages = []
# lstTransitMessages.append(TransitMessageURL(
# "http://www.norta.com/Mobile/whers-my-busdetail.aspx?stopcode=235&routecode=10123&direction=0", "Street Car"))
# lstTransitMessages.append(TransitMessageURL(
# "http://www.norta.com/Mobile/whers-my-busdetail.aspx?stopcode=145&routecode=10122&direction=0", "Magazine Bus"))
# lstTransitMessages.append(TransitMessageURL(
# "http://www.norta.com/Mobile/whers-my-busdetail.aspx?stopcode=58&routecode=10121&direction=0", "Tchoup Bus"))
q = datetime(1990, 1, 1, 1, 1)
start_time = dt_time(6, 45)
end_time = dt_time(23, 00)
while True:
q = datetime(1990, 1, 1, 1, 1)
now_time_fix = q.now().time()
if start_time < now_time_fix < end_time:
# Reset list of calendars and messages to display
lstCalendars = []
lstMessagestoDisplay = []
lstTemporaryMessages = []
try:
# attempt to get new temporary messages and calendars from the google spreadsheet
# the "check" list is used so that the temporary messages list is only replaced if the internet is up
check = []
GetGoogleSheetData(google_sheet_id, get_credentials(),
def testTimeOnlyUTC(self):
# https://github.com/dateutil/dateutil/issues/132
# tzutc doesn't care
tz_utc = tz.tzutc()
self.assertEqual(dt_time(13, 20, tzinfo=tz_utc).utcoffset(), timedelta(0))
re.sub(ur'<\!--.+?-->', ur'',
unicode(infoboks.parameters[commonargs['week']])).strip())
if infoboks.has_param(commonargs['week2']):
endweek = re.sub(
ur'<\!--.+?-->', ur'',
unicode(infoboks.parameters[commonargs['week2']])).strip()
if endweek == '':
endweek = startweek
else:
endweek = startweek
endweek = int(endweek)
startweek = Week(year, startweek)
endweek = Week(year, endweek)
start = wiki_tz.localize(
datetime.combine(startweek.monday(), dt_time(0, 0, 0)))
end = wiki_tz.localize(
datetime.combine(endweek.sunday(), dt_time(23, 59, 59)))
elif infoboks.has_param(ibcfg['start']) and infoboks.has_param(ibcfg['end']):
startdt = infoboks.parameters[ibcfg['start']].value
enddt = infoboks.parameters[ibcfg['end']].value
start = wiki_tz.localize(
datetime.strptime(startdt + ' 00 00 00', '%Y-%m-%d %H %M %S'))
end = wiki_tz.localize(
datetime.strptime(enddt + ' 23 59 59', '%Y-%m-%d %H %M %S'))
else:
log('!! fant ikke datoer')
sys.exit(0)
year = start.isocalendar()[0]
startweek = start.isocalendar()[1]
def testTimeOnlyOffset(self):
# tzoffset doesn't care
tz_offset = tz.tzoffset("+3", 3600)
self.assertEqual(dt_time(13, 20, tzinfo=tz_offset).utcoffset(), timedelta(seconds=3600))
def parse(datetime_string, localize=True):
_utc_to_local = utc_to_local if localize else lambda x: x
def _to_int(value):
if value is None:
return 0
return int(value)
# match time only '00:45:10'
time_only_match = __RE_MATCH_TIME_ONLY__.match(datetime_string)
if time_only_match:
return _utc_to_local(
datetime.combine(
date.today(),
dt_time(hour=_to_int(time_only_match.group('hour')),
minute=_to_int(time_only_match.group('minute')),
second=_to_int(
time_only_match.group('second'))))).time()
# match date only '2014-11-08'
date_only_match = __RE_MATCH_DATE_ONLY__.match(datetime_string)
if date_only_match:
return _utc_to_local(
date(_to_int(date_only_match.group('year')),
_to_int(date_only_match.group('month')),
_to_int(date_only_match.group('day'))))
# full date time
date_time_match = __RE_MATCH_DATETIME__.match(datetime_string)
if date_time_match:
return _utc_to_local(
datetime(_to_int(date_time_match.group('year')),
_to_int(date_time_match.group('month')),
_to_int(date_time_match.group('day')),
_to_int(date_time_match.group('hour')),
_to_int(date_time_match.group('minute')),
_to_int(date_time_match.group('second'))))
# period - at the moment we support only hours, minutes and seconds (e.g. videos and audio)
period_match = __RE_MATCH_PERIOD__.match(datetime_string)
if period_match:
return timedelta(hours=_to_int(period_match.group('hours')),
minutes=_to_int(period_match.group('minutes')),
seconds=_to_int(period_match.group('seconds')))
# abbreviated match
abbreviated_match = __RE_MATCH_ABBREVIATED__.match(datetime_string)
if abbreviated_match:
month = {
'Jan': 1,
'Feb': 2,
'Mar': 3,
'Apr': 4,
'May': 5,
'June': 6,
'Jun': 6,
'July': 7,
'Jul': 7,
'Aug': 8,
'Sept': 9,
'Sep': 9,
'Oct': 10,
'Nov': 11,
'Dec': 12
}
return _utc_to_local(
datetime(year=_to_int(abbreviated_match.group('year')),
month=month[abbreviated_match.group('month')],
day=_to_int(abbreviated_match.group('day')),
hour=_to_int(abbreviated_match.group('hour')),
minute=_to_int(abbreviated_match.group('minute')),
second=_to_int(abbreviated_match.group('second'))))
raise KodionException("Could not parse iso 8601 timestamp '%s'" %
datetime_string)
def testTimeOnlyLocal(self):
# tzlocal returns None
tz_local = tz.tzlocal()
self.assertIs(dt_time(13, 20, tzinfo=tz_local).utcoffset(), None)
def check_time_in_range(time_range, check_time=None):
"""
Check if the given time is contained in the time_range string.
The time_range can be something like
<DOW>-<DOW>: <hh:mm>-<hh:mm>, <DOW>-<DOW>: <hh:mm>-<hh:mm>
<DOW>-<DOW>: <h:mm>-<hh:mm>, <DOW>: <h:mm>-<hh:mm>
<DOW>: <h>-<hh>
DOW beeing the day of the week: Mon, Tue, Wed, Thu, Fri, Sat, Sun
hh: 00-23
mm: 00-59
If time is omitted the current time is used: time.localtime()
:param time_range: The timerange
:type time_range: basestring
:param time: The time to check
:type time: datetime
:return: True, if time is within time_range.
"""
time_match = False
dow_index = {"mon": 1,
"tue": 2,
"wed": 3,
"thu": 4,
"fri": 5,
"sat": 6,
"sun": 7}
check_time = check_time or datetime.now()
check_day = check_time.isoweekday()
check_hour =dt_time(check_time.hour, check_time.minute)
# remove whitespaces
time_range = ''.join(time_range.split())
# split into list of time ranges
time_ranges = time_range.split(",")
try:
for tr in time_ranges:
# tr is something like: Mon-Tue:09:30-17:30
dow, t = [x.lower() for x in tr.split(":", 1)]
if "-" in dow:
dow_start, dow_end = dow.split("-")
else:
dow_start = dow_end = dow
t_start, t_end = t.split("-")
# determine if we have times like 9:00-15:00 or 9-15
ts = [int(x) for x in t_start.split(":")]
te = [int(x) for x in t_end.split(":")]
if len(ts) == 2:
time_start =dt_time(ts[0], ts[1])
else:
time_start =dt_time(ts[0])
if len(te) == 2:
time_end =dt_time(te[0], te[1])
else:
time_end =dt_time(te[0])
# check the day and the time
if (dow_index.get(dow_start) <= check_day <= dow_index.get(dow_end)
and
time_start <= check_hour <= time_end):
time_match = True
except ValueError:
log.error("Wrong time range format: <dow>-<dow>:<hh:mm>-<hh:mm>")
log.debug("{0!s}".format(traceback.format_exc()))
return time_match
def testTimeOnlyRange(self):
# tzrange returns None
tz_range = tz.tzrange("dflt")
self.assertIs(dt_time(13, 20, tzinfo=tz_range).utcoffset(), None)
def parseEvent(self, event, fg, bg):
"""Method to turn google event into a format that we can use more
easily.
"""
# Set up a reference to UTC (all events need a timezone so we can
# sort them)
utc = pytz.UTC
# Have we got a "date" or "dateTime" event?
# Parse the start end end times as appropriate
if event["start"].get("date", False):
start = dateutil.parser.parse(event["start"]["date"])
end = dateutil.parser.parse(event["end"]["date"])
else:
start = dateutil.parser.parse(event["start"]["dateTime"])
end = dateutil.parser.parse(event["end"]["dateTime"])
# Change the end time to one second earlier (useful to check number
# of days of event)
false_end = end - timedelta(0, 1)
duration = false_end - start
# Empty list for our events
ev_list = []
# Split long events into daily events
for i in range(duration.days + 1):
# Create a new start time if the daily event start time isn't the
# same as the overall start time
new_date = start + timedelta(i)
if new_date.date() != start.date():
st = datetime.combine(new_date.date(),
dt_time(0, 0, tzinfo=start.tzinfo))
else:
st = start
# Create a new end time if the daily event end time isn't the same
# as the overall end time
if new_date.date() != false_end.date():
add_day = new_date.date() + timedelta(1)
en = datetime.combine(add_day,
dt_time(0, 0, tzinfo=start.tzinfo))
else:
en = end
# If there's no timezone set, then let's set one
if st.tzinfo is None:
st = utc.localize(st)
if en.tzinfo is None:
en = utc.localize(en)
# Create a dict of the info we need
ev = {"fg": fg,
"bg": bg,
"summary": event.get("summary", ""),
"location": event.get("location", ""),
"start": st,
"end": en,
"stdate": st.date()}
# Add to our list
ev_list.append(ev)
return ev_list
def testTimeOnlyGettz(self):
# gettz returns None
tz_get = tz.gettz("Europe/Minsk")
self.assertIs(dt_time(13, 20, tzinfo=tz_get).utcoffset(), None)
DICT_TYPE = type(dict())
STR_TYPE = type(str())
UNICODE_TYPE = type(unicode())
FLOAT_TYPE = type(float())
INT_TYPE = type(int())
LONG_TYPE = type(long())
BOOL_TYPE = type(bool())
LIST_TYPES = (LIST_TYPE, TUPLE_TYPE)
STR_TYPES = (STR_TYPE, UNICODE_TYPE)
NOMINAL_TYPES = (INT_TYPE, FLOAT_TYPE, LONG_TYPE)
NONE_VALUES = (None, '---', '', 'None')
# 00:00:00; used to combine with date() to form datetime() at midnight
NULL_TIME = dt_time()
# Convenience constants; these are exipration times in seconds
# console escapes for colored output
WHITE = '\033[97m'
BLUE = '\033[96m'
YELLOW = '\033[93m'
GREEN = '\033[92m'
RED = '\033[91m'
ENDC = '\033[0m'
# date only
RE_DATE = re.compile('(\d\d\d\d)-(\d\d)-(\d\d)')
# date only range syntax
RE_DATE_RANGE = re.compile('(\d\d\d\d-\d\d-\d\d):?(\d\d\d\d-\d\d-\d\d)?')
# datetime only
class FakeDataAccessor(base.BaseDataAccessor):
"""
obtain fake data
`FakeDataAccessor` should be used for test purposes when one want emulate
expected backend response
:Example:
>>> fake_accessor = FakeDataAccessor()
>>> requester = Requester(fake_accessor)
>>> requester.login(str, 12312, 'sdfee23e2')
>>> def printer (*args, **kwargs):
pprint(args)
...
>>> requester.clans.get_account_applications_count_since(printer, 123)
(
{'total': 17},
200,
0
)
Use `requests_before_logout` to emulate session expiration.
Session will be considered as expired when `requests_before_logout` is made
use -1 for endless session (default behavior)
:Example:
>>> fake_accessor = FakeDataAccessor()
>>> fake_accessor.requests_before_logout = 2
>>> requester = Requester(fake_accessor)
>>> requester.login(str, 12312, 'sdfee23e2')
>>> def printer (*args, **kwargs):
print (args)
...
>>> requester.clans.get_account_applications_count_since(printer, 123)
({'total': 17}, 200, 0)
>>> requester.clans.get_account_applications_count_since(printer, 123)
({'total': 17}, 200, 0)
>>> requester.clans.get_account_applications_count_since(printer, 123)
('User is not authentificated', 403, 2)
To set expected result for method use `set_data` method
:Example:
>>> fake_accessor = FakeDataAccessor()
>>> requester = Requester(fake_accessor)
>>> requester.login(str, 12312, 'sdfee23e2')
>>> def printer (*args, **kwargs):
print (args)
>>> fake_accessor.set_data('account_applications_count_since', 14, {'total': 11})
>>> requester.clans.get_account_applications_count_since(printer, 14)
({'total': 11}, 200, 0)
>>> requester.clans.get_account_applications_count_since(printer, 123)
({'total': 17}, 200, 0)
To emulate error in response set data to error instance
:Example:
>>> fake_accessor = FakeDataAccessor()
>>> requester = Requester(fake_accessor)
>>> requester.login(str, 12312, 'sdfee23e2')
>>> def printer (*args, **kwargs):
print (args)
>>> fake_accessor.set_data('account_applications_count_since', 14, exceptions.PermissionDenied())
>>> requester.clans.get_account_applications_count_since(printer, 14)
('Forbidden', 403, 3)
>>> requester.clans.get_account_applications_count_since(printer, 123)
({'total': 17}, 200, 0)
"""
requests_before_logout = -1
def __init__(self,
url_fetcher=None,
config=None,
client_lang=None,
user_agent=None):
super(FakeDataAccessor, self).__init__()
self.client_lang = client_lang
self._account = None
self._storage = {}
self.account = None
self.user_agent = user_agent
return
def login(self, callback, account_id, spa_token):
self.account = account_id
self._account = self.requests_before_logout
result, status_code = ('ok', 200)
response_code = exceptions.ResponseCodes.NO_ERRORS
_doResponse(callback, result, status_code, response_code)
def get_alive_status(self, callback):
result, status_code = {'status': 'I am alive!'}, 200
response_code = exceptions.ResponseCodes.NO_ERRORS
_doResponse(callback, result, status_code, response_code)
def logout(self, callback):
self.account = None
self._account = None
result, status_code = ('ok', 200)
response_code = exceptions.ResponseCodes.NO_ERRORS
_doResponse(callback, result, status_code, response_code)
return
def _filter_data(self, data, fields):
if isinstance(data, list):
return [self._filter_data(i, fields) for i in data]
return {k: v for k, v in data.iteritems() if k in fields}
def _request_data(self, section, entity_id, fields=None):
if not self._account:
raise exceptions.AuthentificationError()
self._account -= 1
try:
result = self._storage[section][entity_id]
except KeyError:
result = EXAMPLES[section]
if callable(result):
result = result(entity_id)
self._storage.setdefault(section, {})[entity_id] = result
if isinstance(result, exceptions.BaseRequestError):
raise result
if fields:
result = self._filter_data(result, fields)
return result
def _compare_keys(self, example, data):
if isinstance(example, list):
for i in data:
self._compare_keys(example[0], i)
if isinstance(example, dict):
if set(example) ^ set(data):
missed = set(example) - set(data)
extra = set(data) - set(example)
message = []
if missed:
message.append('(%s) keys are missed' % ', '.join(missed))
if extra:
message.append('(%s) keys are not needed' %
', '.join(extra))
raise ValueError(' and '.join(message))
def set_data(self, section, entity_id, data):
"""
set fake data for different sections, compare keys while setting
possible sections are following:
- account_applications_count_since
- account_invites
- accounts_clans
- accounts_info
- accounts_names
- clan_applications
- clan_globalmap_stats
- clan_invites_count_since
- clan_invites
- clan_members
- clan_provinces
- clans_info
- clans_ratings
- fronts_info
- search_clans
- stronghold_info
- strongholds_state
- strongholds_statistics
"""
if not section in EXAMPLES:
raise AssertionError
example = EXAMPLES[section]
isinstance(data,
exceptions.BaseRequestError) or self._compare_keys(
example, data)
self._storage.setdefault(section, {})[entity_id] = data
@fake_method(
example=lambda clan_id: {
'clan_id': clan_id,
'xp_avg': random.randrange(1, 1000) / 10.0,
'efficiency': random.randrange(1, 10000),
'battles_count_avg': random.randrange(1, 10000),
'wins_ratio_avg': random.randrange(1, 100),
'gm_elo_rating_6': random.randrange(1, 1000),
'gm_elo_rating_8': random.randrange(1, 1000),
'gm_elo_rating_10': random.randrange(1, 1000),
'gm_elo_rating_6_rank': random.randrange(1, 1000),
'gm_elo_rating_8_rank': random.randrange(1, 1000),
'gm_elo_rating_10_rank': random.randrange(1, 1000),
'fb_elo_rating_8': random.randrange(1, 1000),
'fb_elo_rating_10': random.randrange(1, 1000),
'fb_battles_count_10_28d': random.randrange(1, 100),
'fs_battles_count_10_28d': random.randrange(1, 100),
'gm_battles_count_28d': random.randrange(1, 100),
'fs_battles_count_28d': random.randrange(1, 100),
'fb_battles_count_28d': random.randrange(1, 100)
})
def get_clans_ratings(self, clan_ids, fields=None):
"""
return fake data from `clans_ratings` section
"""
return [
self._request_data('clans_ratings', i, fields=fields)
for i in clan_ids
]
@fake_method(
example=lambda clan_id: {
'name': 'xxx',
'tag': 'ff',
'motto': 'yyyy',
'leader_id': 666,
'members_count': 13,
'clan_id': clan_id,
'created_at': datetime.now(),
'accepts_join_requests': True,
'treasury': 2423
})
def get_clans_info(self, clan_ids, fields=None):
"""
return fake data from `clans_info` section
"""
return [
self._request_data('clans_info', clan_id, fields=fields)
for clan_id in clan_ids
]
@fake_method(example=lambda acc_id: {'id': acc_id, 'name': 'name'})
def get_accounts_names(self, account_ids, fields=None):
"""
return fake data from `accounts_names` section
"""
return [
self._request_data('accounts_names', account_id, fields=fields)
for account_id in account_ids
]
@fake_method(example=lambda clan_id: [{
'account_id': 2324 + i,
'role_name': 'officer',
'role_bw_flag': 1 << i,
'clan_id': clan_id,
'joined_at': datetime.now()
} for i in range(11)])
def get_clan_members(self, clan_id, fields=None):
"""
return fake data from `clan_members` section
"""
return self._request_data('clan_members', clan_id, fields=fields)
@fake_method(
example={
'clan_id': 2790,
'favorite_arena_6': 1,
'favorite_arena_8': 3,
'favorite_arena_10': 65549,
'favorite_primetime': dt_time(19, 0)
})
def get_clan_favorite_attributes(self, clan_id, fields=None):
"""
return fake data from `clan_favorite_attributes` section
"""
return self._request_data('clan_favorite_attributes',
clan_id,
fields=fields)
@fake_method(example={'total': 17})
def get_account_applications_count_since(self, account_id, since=None):
"""
return fake data from `account_applications_count_since` section
"""
return self._request_data('account_applications_count_since',
account_id)
@fake_method(example={'total': 14})
def get_clan_invites_count_since(self, clan_id, since=None):
"""
return fake data from `clan_invites_count_since` section
"""
return self._request_data('clan_invites_count_since', clan_id)
@fake_method(
example={
'account_id': 234,
'joined_at': datetime.now(),
'clan_id': 343,
'role_bw_flag': 13,
'role_name': 'commander',
'in_clan_cooldown_till': datetime.now(),
'clan_tag': 'fake',
'clan_color': 123
})
def get_accounts_clans(self, account_ids, fields):
"""
return fake data from `accounts_clans` section
"""
return [
self._request_data('accounts_clans', i, fields=fields)
for i in account_ids
]
@fake_method(example=lambda (account_id, statuses): [{
'status':
random.choice(statuses or ('active', 'declined', 'cancelled',
'accepted', 'expired', 'error', 'deleted')),
'created_at':
datetime.now(),
'updated_at':
datetime.now(),
'sender_id':
random.randrange(1, 10000),
'id':
random.randrange(1, 1000000),
'account_id':
account_id,
'clan_id':
random.randrange(1, 10000),
'status_changer_id':
random.randrange(1, 10000),
'comment':
'Welcome {}!'.format(random.randrange(1, 10000))
if random.choice((1, 0)) else ''
} for i in range(random.randrange(0, 1000))])
@paginated_method
def get_account_applications(self, fields=None, statuses=None):
"""
return fake data from `account_applications` section
"""
return self._request_data('account_applications',
(self.account, tuple(statuses or [])),
fields=fields)
@fake_method(example=lambda (clan_id, statuses): [{
'status':
random.choice(statuses or ('active', 'declined', 'cancelled',
'accepted', 'expired', 'error', 'deleted')),
'created_at':
datetime.now(),
'updated_at':
datetime.now(),
'sender_id':
random.randrange(1, 10000),
'id':
random.randrange(1, 1000000),
'account_id':
random.randrange(1, 10000),
'clan_id':
clan_id,
'status_changer_id':
random.randrange(1, 10000),
'comment':
'Welcome {}!'.format(random.randrange(1, 10000))
if random.choice((1, 0)) else ''
} for i in range(random.randrange(0, 1000))])
@paginated_method
def get_clan_applications(self, clan_id, fields=None, statuses=None):
"""
return fake data from `clan_applications` section
"""
return self._request_data('clan_applications',
(clan_id, tuple(statuses or [])),
fields=fields)
@fake_method(example=lambda search: ([] if len(
search) % 2 else [{
'name': 'Clan Name %d' % random.randrange(1, 1000),
'tag': 'TCLAN',
'motto': 'Clan Motto',
'leader_id': random.randrange(1, 10000),
'clan_id': random.randrange(1, 100),
'members_count': random.randrange(1, 50),
'created_at': datetime.now(),
'accepts_join_requests': random.choice((True, False))
} for i in range(random.randrange(1, 36))]))
@paginated_method
def search_clans(self, search, fields=None):
"""
return fake data from `clans_info` section
"""
return self._request_data('search_clans', search)
@fake_method(example=lambda account:
[{
'name': 'Clan Name %d' % random.randrange(1, 1000),
'tag': 'TCLAN',
'motto': 'Clan Motto',
'leader_id': random.randrange(1, 10000),
'clan_id': random.randrange(1, 100),
'members_count': random.randrange(1, 50),
'created_at': datetime.now(),
'accepts_join_requests': random.choice((True, False))
} for i in range(random.randrange(1, 36))])
@paginated_method
def get_recommended_clans(self, fields=None):
"""
return fake data from `clans_info` section
"""
return self._request_data('recommended_clans', self.account)
@fake_method(example=lambda (clan_id, statuses): [{
'status':
random.choice(statuses or ('active', 'declined', 'cancelled',
'accepted', 'expired', 'error', 'deleted')),
'created_at':
datetime.now(),
'updated_at':
datetime.now(),
'sender_id':
random.randrange(1, 10000),
'id':
random.randrange(1, 1000000),
'account_id':
random.randrange(1, 10000),
'clan_id':
clan_id,
'comment':
'Welcome {}!'.format(random.randrange(1, 10000))
if random.choice((1, 0)) else '',
'status_changer_id':
2132
} for i in range(random.randrange(0, 1000))])
@paginated_method
def get_clan_invites(self, clan_id, fields=None, statuses=None):
"""
return fake data from `clan_invites` section
"""
return self._request_data('clan_invites',
(clan_id, tuple(statuses or [])),
fields=fields)
@fake_method(example=lambda (account_id, statuses): [{
'status':
random.choice(statuses or ('active', 'declined', 'cancelled',
'accepted', 'expired', 'error', 'deleted')),
'created_at':
datetime.now(),
'updated_at':
datetime.now(),
'sender_id':
random.randrange(1, 10000),
'id':
random.randrange(1, 1000000),
'account_id':
account_id,
'clan_id':
random.randrange(1, 10000),
'status_changer_id':
2132,
'comment':
'Welcome {}!'.format(random.randrange(1, 10000))
if random.choice((1, 0)) else ''
} for i in range(random.randrange(0, 1000))])
@paginated_method
def get_account_invites(self, fields=None, statuses=None):
"""
return fake data from `account_invites` section
"""
return self._request_data('account_invites',
(self.account, tuple(statuses or [])),
fields=fields)
@fake_method(
example=lambda account_id: {
'global_rating': random.randrange(100, 10000),
'battle_avg_xp': random.randrange(100, 10000),
'battles_count': random.randrange(1, 1000),
'battle_avg_performance': random.uniform(0, 1),
'xp_amount': random.randrange(100, 1000),
'account_id': account_id
})
def get_accounts_info(self, account_ids, fields=None):
"""
return fake data from `accounts_info` section
"""
return [
self._request_data('accounts_info', acc_id, fields=fields)
for acc_id in account_ids
]
@fake_method(example=[{
'front_name':
'some_front',
'province_id':
'some_province',
'front_name_localized':
'some_front_localized',
'province_id_localized':
'some_province_localized',
'revenue':
324,
'hq_connected':
True,
'prime_time':
dt_time(18, 0, 0),
'periphery':
333,
'game_map':
'some_map',
'pillage_cooldown':
1,
'pillage_end_datetime':
datetime.now() + timedelta(hours=3),
'turns_owned':
12
}, {
'front_name': 'some_front2',
'province_id': 'some_province2',
'front_name_localized': 'some_front_localized2',
'province_id_localized': 'some_province_localized2',
'revenue': 333,
'hq_connected': True,
'prime_time': dt_time(19, 0, 0),
'periphery': 444,
'game_map': 'some_map2',
'pillage_cooldown': None,
'pillage_end_datetime': None,
'turns_owned': 12,
'arena_id': 5
}])
def get_clan_provinces(self, clan_id, fields=None):
"""
return fake data from `clan_provinces` section
"""
return self._request_data('clan_provinces', clan_id, fields=fields)
@fake_method(
example={
'battles_lost': 12,
'influence_points': 121,
'provinces_captured': 23,
'provinces_count': 234,
'battles_played': 332,
'battles_won': 232,
'battles_played_on_6_level': 21,
'battles_won_on_6_level': 12,
'battles_played_on_8_level': 32,
'battles_won_on_8_level': 21,
'battles_played_on_10_level': 43,
'battles_won_on_10_level': 23
})
def get_clan_globalmap_stats(self, clan_id, fields=None):
"""
return fake data from `clan_globalmap_stats` section
"""
return self._request_data('clan_globalmap_stats',
clan_id,
fields=fields)
@fake_method(example=[{
'front_name': 'front_name',
'front_name_localized': 'front_name_localized',
'min_vehicle_level': 2,
'max_vehicle_level': 4
}])
def get_fronts_info(self, front_names=None, fields=None):
"""
return fake data from `fronts_info` section
"""
return self._request_data('fronts_info', front_names, fields=fields)
@fake_method(
example={
'defence_mode_is_activated':
True,
'defence_hour':
dt_time(10, 0),
'sortie_battles_count':
23,
'sortie_wins':
12,
'sortie_losses':
19,
'sortie_fort_resource_in_absolute':
100,
'sortie_fort_resource_in_champion':
71,
'sortie_fort_resource_in_middle':
60,
'defence_battles_count':
234,
'defence_combat_wins':
21,
'sortie_middle_battles_count':
12,
'sortie_champion_battles_count':
32,
'sortie_absolute_battles_count':
23,
'defence_enemy_base_capture_count':
43,
'defence_capture_enemy_building_total_count':
55,
'defence_loss_own_building_total_count':
65,
'defence_attack_efficiency':
23.2,
'defence_success_attack_count':
122,
'defence_attack_count':
13,
'defence_defence_efficiency':
32.2,
'defence_defence_count':
24,
'defence_success_defence_count':
5,
'total_resource_amount':
321,
'defence_resource_loss_count':
112,
'defence_resource_capture_count':
322,
'fb_battles_count_8':
23,
'fb_battles_count_10':
12,
'level':
2,
'buildings': [{
'type': 1,
'direction': 0,
'level': 2,
'position': 2
}, {
'type': 2,
'direction': 1,
'level': 3,
'position': 2
}]
})
def get_stronghold_info(self, clan_id, fields=None):
"""
return fake data from `stronghold_info` section
"""
return self._request_data('stronghold_info', clan_id, fields=fields)
@fake_method(
example={
'buildings_count':
4,
'directions_count':
3,
'buildings': [{
'type': 1,
'hp': 32,
'storage': 123,
'level': 4,
'position': 7,
'direction': 1
}],
'directions': [1, 2],
'off_day':
3,
'vacation_start':
datetime.utcnow() + timedelta(days=1),
'vacation_finish':
datetime.utcnow() + timedelta(days=4),
'periphery_id':
333,
'clan_tag':
'tag',
'clan_name':
'some_name',
'clan_id':
21,
'level':
2,
'sortie_wins_period':
7,
'sortie_battles_wins_percentage_period':
20.0,
'sortie_battles_count_period':
122,
'defence_battles_count_period':
21
})
def get_strongholds_statistics(self, clan_id, fields=None):
"""
return fake data from `strongholds_statistics` section
"""
return self._request_data('strongholds_statistics',
clan_id,
fields=fields)
@fake_method(example={'clan_id': 234, 'defence_hour': dt_time(10, 0)})
def get_strongholds_state(self, clan_id, fields=None):
"""
return fake data from `strongholds_state` section
"""
return self._request_data('strongholds_state', clan_id, fields=fields)
@fake_method(example=[{'clan_id': 234, 'account_id': 3, 'id': 23}])
def create_invites(self, clan_id, account_ids, comment, fields=None):
"""
return fake data from `create_invites` section
"""
return self._request_data('create_invites', (clan_id, account_ids),
fields=fields)
@fake_method(example=[{'clan_id': 224, 'account_id': 3, 'id': 123}])
def create_applications(self, clan_ids, comment, fields=None):
"""
return fake data from `create_applications` section
"""
return self._request_data('create_applications',
clan_ids,
fields=fields)
@fake_method(example=lambda obj_id: {
'transaction_id': 213,
'id': obj_id,
'account_id': 343,
'clan_id': 17
})
def accept_application(self, application_id, fields=None):
"""
return fake data from `accept_application` section
"""
return self._request_data('accept_application',
application_id,
fields=fields)
@fake_method(example=lambda obj_id: {
'id': obj_id,
'account_id': 343,
'clan_id': 17
})
def decline_application(self, application_id, fields=None):
"""
return fake data from `decline_application` section
"""
return self._request_data('decline_application',
application_id,
fields=fields)
@fake_method(example=lambda obj_id: {
'transaction_id': 213,
'id': obj_id,
'account_id': 343,
'clan_id': 17
})
def accept_invite(self, invite_id, fields=None):
"""
return fake data from `accept_invite` section
"""
return self._request_data('accept_invite', invite_id, fields=fields)
@fake_method(example=lambda obj_id: {
'id': obj_id,
'account_id': 343,
'clan_id': 17
})
def decline_invite(self, invite_id, fields=None):
"""
return fake data from `decline_invite` section
"""
return self._request_data('decline_invite', invite_id, fields=fields)
@fake_method(example=[{
'id': 991,
'account_id': 1001,
'clan_id': 19
}, {
'id': 992,
'account_id': 1001,
'clan_id': 19
}, {
'id': 993,
'account_id': 1001,
'clan_id': 19
}])
def bulk_decline_invites(self, invite_ids):
"""
return fake data from `bulk_decline_invites` section
"""
return self._request_data('bulk_decline_invites', invite_ids)
@fake_method(
example={
'permissions': {
'manage_reserves': [
'commander', 'combat_officer', 'executive_officer',
'personnel_officer'
]
},
'time_to_ready':
900,
'max_level':
10,
'battle_series_duration':
3600,
'enemy_clan':
None,
'industrial_resource_multiplier':
1,
'max_players_count':
15,
'type':
'FORT_BATTLE',
'max_legionaries_count':
0,
'available_reserves': {
'ARTILLERY_STRIKE': [],
'HIGH_CAPACITY_TRANSPORT': [],
'REQUISITION': [],
'AIRSTRIKE': []
},
'direction':
'A',
'min_players_count':
1,
'matchmaker_next_tick':
1475578800,
'battle_series_status': [{
'battle_reward': 0,
'gameplay_id': 0,
'geometry_id': 6,
'first_resp_clan_id': None,
'second_resp_clan_id': None,
'attacker': None,
'clan_owner_id': 14000012972L,
'current_battle': False,
'map_id': 6
}, {
'battle_reward': 0,
'gameplay_id': 0,
'geometry_id': 14,
'first_resp_clan_id': None,
'second_resp_clan_id': None,
'attacker': None,
'clan_owner_id': 14000012972L,
'current_battle': False,
'map_id': 14
}, {
'battle_reward': 0,
'gameplay_id': 0,
'geometry_id': 20,
'first_resp_clan_id': None,
'second_resp_clan_id': None,
'attacker': None,
'clan_owner_id': 14000012972L,
'current_battle': False,
'map_id': 20
}],
'battle_duration':
600,
'requisition_bonus_percent':
None,
'public':
False,
'selected_reserves': [None, None, None],
'min_level':
1
})
)
elif after_entity.attributes.get(
ATTR_DEVICE_CLASS
) == SensorDeviceClass.TIMESTAMP and after_entity.state not in (
STATE_UNAVAILABLE,
STATE_UNKNOWN,
):
after_datetime = dt_util.parse_datetime(after_entity.state)
if after_datetime is None:
return False
after = dt_util.as_local(after_datetime).time()
else:
return False
if before is None:
before = dt_time(23, 59, 59, 999999)
elif isinstance(before, str):
if not (before_entity := hass.states.get(before)):
raise ConditionErrorMessage("time",
f"unknown 'before' entity {before}")
if before_entity.domain == "input_datetime":
before = dt_time(
before_entity.attributes.get("hour", 23),
before_entity.attributes.get("minute", 59),
before_entity.attributes.get("second", 59),
)
elif before_entity.attributes.get(
ATTR_DEVICE_CLASS
) == SensorDeviceClass.TIMESTAMP and before_entity.state not in (
STATE_UNAVAILABLE,
STATE_UNKNOWN,
def _handler(self, request, response):
init_process_logger('log.txt')
response.outputs['output_log'].file = 'log.txt'
LOGGER.info('Start process')
response.update_status('execution started at : {}'.format(dt.now()), 5)
process_start_time = time.time() # measure process execution time ...
start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
# response.update_status('execution started at : %s ' % dt.now(), 5)
# start_time = time.time() # measure init ...
################################
# reading in the input arguments
################################
try:
response.update_status('read input parameter : %s ' % dt.now(), 10)
resource = archiveextract(resource=rename_complexinputs(request.inputs['resource']))
refSt = request.inputs['refSt'][0].data
refEn = request.inputs['refEn'][0].data
dateSt = request.inputs['dateSt'][0].data
dateEn = request.inputs['dateEn'][0].data
seasonwin = request.inputs['seasonwin'][0].data
nanalog = request.inputs['nanalog'][0].data
bboxDef = '-20,40,30,70' # in general format
# level = 500
level = request.inputs['level'][0].data
if (level == 500):
dummylevel = 1000 # dummy workaround for cdo sellevel
else:
dummylevel = 500
LOGGER.debug('LEVEL selected: %s hPa' % (level))
bbox = []
bboxStr = request.inputs['BBox'][0].data
LOGGER.debug('BBOX selected by user: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# Checking for wrong cordinates and apply default if nesessary
if (abs(float(bboxStr[0])) > 180 or
abs(float(bboxStr[1]) > 180) or
abs(float(bboxStr[2]) > 90) or
abs(float(bboxStr[3])) > 90):
bboxStr = bboxDef # request.inputs['BBox'].default # .default doesn't work anymore!!!
LOGGER.debug('BBOX is out of the range, using default instead: %s ' % (bboxStr))
bboxStr = bboxStr.split(',')
# for i in bboxStr: bbox.append(int(i))
bbox.append(float(bboxStr[0]))
bbox.append(float(bboxStr[2]))
bbox.append(float(bboxStr[1]))
bbox.append(float(bboxStr[3]))
LOGGER.debug('BBOX for ocgis: %s ' % (bbox))
LOGGER.debug('BBOX original: %s ' % (bboxStr))
normalize = request.inputs['normalize'][0].data
plot = request.inputs['plot'][0].data
distance = request.inputs['dist'][0].data
outformat = request.inputs['outformat'][0].data
timewin = request.inputs['timewin'][0].data
detrend = request.inputs['detrend'][0].data
LOGGER.info('input parameters set')
response.update_status('Read in and convert the arguments', 20)
except Exception as e:
msg = 'failed to read input prameter %s ' % e
LOGGER.error(msg)
raise Exception(msg)
######################################
# convert types and set environment
######################################
try:
# not nesessary if fix ocgis_module.py
refSt = dt.combine(refSt, dt_time(12, 0))
refEn = dt.combine(refEn, dt_time(12, 0))
dateSt = dt.combine(dateSt, dt_time(12, 0))
dateEn = dt.combine(dateEn, dt_time(12, 0))
# Check if 360_day calendar:
try:
if type(resource) is not list:
resource = [resource]
modcal, calunits = get_calendar(resource[0])
if '360_day' in modcal:
if refSt.day == 31:
refSt = refSt.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (refSt))
if refEn.day == 31:
refEn = refEn.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (refEn))
if dateSt.day == 31:
dateSt = dateSt.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (dateSt))
if dateEn.day == 31:
dateEn = dateEn.replace(day=30)
LOGGER.debug('Date has been changed for: %s' % (dateEn))
except:
LOGGER.debug('Could not detect calendar')
if normalize == 'None':
seacyc = False
else:
seacyc = True
if outformat == 'ascii':
outformat = '.txt'
elif outformat == 'netCDF':
outformat = '.nc'
else:
LOGGER.error('output format not valid')
start = min(refSt, dateSt)
end = max(refEn, dateEn)
LOGGER.info('environment set')
except Exception as e:
msg = 'failed to set environment %s ' % e
LOGGER.error(msg)
raise Exception(msg)
LOGGER.debug("init took %s seconds.", time.time() - start_time)
response.update_status('Read in and convert the arguments', 30)
########################
# input data preperation
########################
# TODO: Check if files containing more than one dataset
response.update_status('Start preparing input data', 40)
start_time = time.time() # mesure data preperation ...
try:
# TODO: Add selection of the level. maybe bellow in call(..., level_range=[...,...])
if type(resource) == list:
# resource.sort()
resource = sorted(resource, key=lambda i: path.splitext(path.basename(i))[0])
else:
resource = [resource]
# ===============================================================
# REMOVE resources which are out of interest from the list
# (years > and < than requested for calculation)
tmp_resource = []
for re in resource:
s,e = get_timerange(re)
tmpSt = dt.strptime(s, '%Y%m%d')
tmpEn = dt.strptime(e, '%Y%m%d')
if ((tmpSt <= end) and (tmpEn >= start)):
tmp_resource.append(re)
LOGGER.debug('Selected file: %s ' % (re))
resource = tmp_resource
# Try to fix memory issue... (ocgis call for files like 20-30 gb... )
# IF 4D - select pressure level before domain cut
#
# resource properties
ds = Dataset(resource[0])
variable = get_variable(resource[0])
var = ds.variables[variable]
dims = list(var.dimensions)
dimlen = len(dims)
try:
model_id = ds.getncattr('model_id')
except AttributeError:
model_id = 'Unknown model'
LOGGER.debug('MODEL: %s ' % (model_id))
lev_units = 'hPa'
if (dimlen > 3):
lev = ds.variables[dims[1]]
# actually index [1] need to be detected... assuming zg(time, plev, lat, lon)
lev_units = lev.units
if (lev_units == 'Pa'):
level = level * 100
dummylevel = dummylevel * 100
# TODO: OR check the NAME and units of vertical level and find 200 , 300, or 500 mbar in it
# Not just level = level * 100.
# Get Levels
from cdo import Cdo
cdo = Cdo(env=environ)
lev_res = []
if(dimlen > 3):
for res_fn in resource:
tmp_f = 'lev_' + path.basename(res_fn)
try:
tmp_f = call(resource=res_fn, variable=variable, spatial_wrapping='wrap',
level_range=[int(level), int(level)], prefix=tmp_f[0:-3])
except:
comcdo = '%s,%s' % (level, dummylevel)
cdo.sellevel(comcdo, input=res_fn, output=tmp_f)
lev_res.append(tmp_f)
else:
lev_res = resource
# ===============================================================
# TODO: Before domain, Regrid to selected grid! (???) if no rean.
# ================================================================
# Get domain
regr_res = []
for res_fn in lev_res:
tmp_f = 'dom_' + path.basename(res_fn)
comcdo = '%s,%s,%s,%s' % (bbox[0], bbox[2], bbox[1], bbox[3])
try:
tmp_f = call(resource=res_fn, geom=bbox, spatial_wrapping='wrap', prefix=tmp_f[0:-3])
except:
cdo.sellonlatbox(comcdo, input=res_fn, output=tmp_f)
regr_res.append(tmp_f)
# ============================
# Block to Detrend data
# TODO 1 Keep trend as separate file
# TODO 2 Think how to add options to plot abomalies AND original data...
# May be do archive and simulation = call.. over NOT detrended data and keep it as well
if (dimlen > 3):
res_tmp = get_level(regr_res, level=level)
variable = 'z%s' % level
else:
res_tmp = call(resource=regr_res, spatial_wrapping='wrap')
if detrend == 'None':
orig_model_subset = res_tmp
else:
orig_model_subset = remove_mean_trend(res_tmp, varname=variable)
# ============================
# archive_tmp = call(resource=regr_res, time_range=[refSt, refEn], spatial_wrapping='wrap')
# simulation_tmp = call(resource=regr_res, time_range=[dateSt, dateEn], spatial_wrapping='wrap')
################################
# Prepare names for config.txt #
################################
# refDatesString = dt.strftime(refSt, '%Y-%m-%d') + "_" + dt.strftime(refEn, '%Y-%m-%d')
# simDatesString = dt.strftime(dateSt, '%Y-%m-%d') + "_" + dt.strftime(dateEn, '%Y-%m-%d')
# Fix < 1900 issue...
refDatesString = refSt.isoformat().strip().split("T")[0] + "_" + refEn.isoformat().strip().split("T")[0]
simDatesString = dateSt.isoformat().strip().split("T")[0] + "_" + dateEn.isoformat().strip().split("T")[0]
archiveNameString = "base_" + variable + "_" + refDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
simNameString = "sim_" + variable + "_" + simDatesString + '_%.1f_%.1f_%.1f_%.1f' \
% (bbox[0], bbox[2], bbox[1], bbox[3])
archive = call(resource=res_tmp, time_range=[refSt, refEn], spatial_wrapping='wrap', prefix=archiveNameString)
simulation = call(resource=res_tmp, time_range=[dateSt, dateEn], spatial_wrapping='wrap', prefix=simNameString)
#######################################################################################
if seacyc is True:
seasoncyc_base, seasoncyc_sim = analogs.seacyc(archive, simulation, method=normalize)
else:
seasoncyc_base = None
seasoncyc_sim = None
except Exception as e:
msg = 'failed to prepare archive and simulation files %s ' % e
LOGGER.debug(msg)
raise Exception(msg)
ip, output = mkstemp(dir='.', suffix='.txt')
output_file = path.abspath(output)
files = [path.abspath(archive), path.abspath(simulation), output_file]
LOGGER.debug("data preperation took %s seconds.", time.time() - start_time)
############################
# generating the config file
############################
# TODO: add MODEL name as argument
response.update_status('writing config file', 50)
start_time = time.time() # measure write config ...
try:
config_file = analogs.get_configfile(
files=files,
seasoncyc_base=seasoncyc_base,
seasoncyc_sim=seasoncyc_sim,
base_id=model_id,
sim_id=model_id,
timewin=timewin,
varname=variable,
seacyc=seacyc,
cycsmooth=91,
nanalog=nanalog,
seasonwin=seasonwin,
distfun=distance,
outformat=outformat,
calccor=True,
silent=False,
# period=[dt.strftime(refSt, '%Y-%m-%d'), dt.strftime(refEn, '%Y-%m-%d')],
period=[refSt.isoformat().strip().split("T")[0], refEn.isoformat().strip().split("T")[0]],
bbox="%s,%s,%s,%s" % (bbox[0], bbox[2], bbox[1], bbox[3]))
except Exception as e:
msg = 'failed to generate config file %s ' % e
LOGGER.debug(msg)
raise Exception(msg)
LOGGER.debug("write_config took %s seconds.", time.time() - start_time)
##############
# CASTf90 call
##############
import subprocess
import shlex
start_time = time.time() # measure call castf90
response.update_status('Start CASTf90 call', 60)
# -----------------------
try:
import ctypes
# TODO: This lib is for linux
mkl_rt = ctypes.CDLL('libmkl_rt.so')
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('Current number of threads: %s' % (nth))
mkl_rt.mkl_set_num_threads(ctypes.byref(ctypes.c_int(64)))
nth = mkl_rt.mkl_get_max_threads()
LOGGER.debug('NEW number of threads: %s' % (nth))
# TODO: Does it \/\/\/ work with default shell=False in subprocess... (?)
environ['MKL_NUM_THREADS'] = str(nth)
environ['OMP_NUM_THREADS'] = str(nth)
except Exception as e:
msg = 'Failed to set THREADS %s ' % e
LOGGER.debug(msg)
# -----------------------
# ##### TEMPORAL WORKAROUND! With instaled hdf5-1.8.18 in anaconda ###############
# ##### MUST be removed after castf90 recompiled with the latest hdf version
# ##### NOT safe
environ['HDF5_DISABLE_VERSION_CHECK'] = '1'
# hdflib = os.path.expanduser("~") + '/anaconda/lib'
# hdflib = os.getenv("HOME") + '/anaconda/lib'
import pwd
hdflib = pwd.getpwuid(getuid()).pw_dir + '/anaconda/lib'
environ['LD_LIBRARY_PATH'] = hdflib
# ################################################################################
try:
# response.update_status('execution of CASTf90', 50)
cmd = 'analogue.out %s' % path.relpath(config_file)
# system(cmd)
args = shlex.split(cmd)
output, error = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
LOGGER.info('analogue.out info:\n %s ' % output)
LOGGER.debug('analogue.out errors:\n %s ' % error)
response.update_status('**** CASTf90 suceeded', 70)
except Exception as e:
msg = 'CASTf90 failed %s ' % e
LOGGER.error(msg)
raise Exception(msg)
LOGGER.debug("castf90 took %s seconds.", time.time() - start_time)
# TODO: Add try - except for pdfs
if plot == 'Yes':
analogs_pdf = analogs.plot_analogs(configfile=config_file)
else:
analogs_pdf = 'dummy_plot.pdf'
with open(analogs_pdf, 'a'): utime(analogs_pdf, None)
response.update_status('preparing output', 80)
response.outputs['analog_pdf'].file = analogs_pdf
response.outputs['config'].file = config_file
response.outputs['analogs'].file = output_file
response.outputs['output_netcdf'].file = simulation
response.outputs['target_netcdf'].file = archive
if seacyc is True:
response.outputs['base_netcdf'].file = seasoncyc_base
response.outputs['sim_netcdf'].file = seasoncyc_sim
else:
# TODO: Still unclear how to overpass unknown number of outputs
dummy_base = 'dummy_base.nc'
dummy_sim = 'dummy_sim.nc'
with open(dummy_base, 'a'): utime(dummy_base, None)
with open(dummy_sim, 'a'): utime(dummy_sim, None)
response.outputs['base_netcdf'].file = dummy_base
response.outputs['sim_netcdf'].file = dummy_sim
########################
# generate analog viewer
########################
formated_analogs_file = analogs.reformat_analogs(output_file)
# response.outputs['formated_analogs'].storage = FileStorage()
response.outputs['formated_analogs'].file = formated_analogs_file
LOGGER.info('analogs reformated')
response.update_status('reformatted analog file', 90)
viewer_html = analogs.render_viewer(
# configfile=response.outputs['config'].get_url(),
configfile=config_file,
# datafile=response.outputs['formated_analogs'].get_url())
datafile=formated_analogs_file)
response.outputs['output'].file = viewer_html
response.update_status('Successfully generated analogs viewer', 95)
LOGGER.info('rendered pages: %s ', viewer_html)
response.update_status('execution ended', 100)
LOGGER.debug("total execution took %s seconds.",
time.time() - process_start_time)
return response
