def toDateOnly(dates, dtype='datetime'): """Converts an array of datetime objects and removes the time component keeping the datetime object intact. Similar to doing just <datetime_object>.date() but makes it comparable to other dates""" if dtype == 'datetime': if isinstance(dates, datetime): #Not an array of dates return datetime.combine(dates, dttime(0, 0, 0)) else: return np.array([ datetime.combine(dates[i], dttime(0, 0, 0)) for i in xrange(len(dates)) ], dtype=object) elif dtype == 'date': if isinstance(dates, datetime): #Not an array of dates return datetime.date() else: return np.array([DATETIME.date() for DATETIME in dates], dtype=object) else: raise ValueError( "[toDateOnly] dtype input needs to be either 'datetime' or 'date'. We got %s" % dtype)
def post(self, lot_id, type) : min_date = self.request.get('mindate') max_date = self.request.get('maxdate') min_hour = self.request.get('minhour') min_minute = self.request.get('minminute') max_hour = self.request.get('maxhour') max_minute = self.request.get('maxminute') if min_date != '' : min_datetime = datetime.strptime(min_date, '%m/%d/%Y') else : min_datetime = datetime.min if max_date != '' : max_datetime = datetime.strptime(max_date, '%m/%d/%Y') else : max_datetime = datetime.max min_time = dttime(int(min_hour), int(min_minute)) max_time = dttime(int(max_hour), int(max_minute)) if int(max_hour) == 0 and int(max_minute) == 0 : # special case, we want end-of-day max_time = dttime.max min_datetime = datetime.combine(min_datetime.date(), min_time) max_datetime = datetime.combine(max_datetime.date(), max_time) # controller query buckets = controller.viewRange(lot_id, min_datetime, max_datetime) max_count = max(e['average'] for e in buckets) values = {'min_date':min_date, 'max_date':max_date, 'buckets':buckets, 'max_count':max_count, 'lot_id':lot_id} path = os.path.join(base_path, 'templates/chart.html') self.response.out.write(template.render(path, values))
def getNextEvent(data: List[str], day, tomorrow=False) -> Tuple[str, datetime]: """Get the next high / low tide event""" valid = [] for event in TideTime: try: tmp = datetime.strptime(data[event.value].strip(), "%H:%M") logging.debug(f"Successfully parsed '{data[event.value].strip()}' as time") valid.append((event.name, tmp.time())) except ValueError: logging.debug(f"Failed parsing '{data[event.value].strip()}' as time") valid.sort(key=lambda x: x[1]) # If the next event is tomorrow set the current time to midnight current = dttime(0, 0, 0) if tomorrow else datetime.now().time() upcoming = [elt for elt in valid if elt[1] >= current] return (upcoming[0][0], datetime.combine(day, upcoming[0][1]))
def __init__(self, input_json, condition_transl, date=None, daily=False, celsius=True): # get current weather self.celsius = celsius if date is None: self.temp = self._round_temp(input_json['main']['temp']) self.condition = condition_transl[input_json['weather'][0]['id']] return # get prediction if daily: # set time to 13:00 for daily date = datetime.combine(date.date(), dttime(13, 00)) date = datetime.utcfromtimestamp(int(time.mktime(date.timetuple()))) ts = int(date.strftime("%s")) # convert time to Unix timestamp for fc1, fc2 in zip(input_json['list'][:-1], input_json['list'][1:]): # find the appropriate time frame if ts >= fc1['dt'] and ts <= fc2['dt']: self.condition = condition_transl[fc1['weather'][0]['id']] # hourly forecast -- interpolate temperature if not daily: slope = (fc2['main']['temp'] - fc1['main']['temp']) / (fc2['dt'] - fc1['dt']) self.temp = self._round_temp(fc1['main']['temp'] + slope * (ts - fc1['dt'])) # daily forecast: use daily high & low else: self.temp = self._round_temp(fc1['temp']['day']) self.min_temp = self._round_temp(fc1['temp']['min']) self.max_temp = self._round_temp(fc1['temp']['max']) break if not hasattr(self, 'temp'): self.temp = self._round_temp(input_json['list'][0]['main']['temp']) self.condition = condition_transl[input_json['list'][0]['weather'][0]['id']]
def getCaptureDateTime(self, data): try: timestr = "%08d" % (struct.unpack('L', data[1])[0]) #in format hhmmsscc time = dttime(int(timestr[:2]), int(timestr[2:4]), int(timestr[4:6]), int(timestr[6:])) except: print "Issue with time format:", timestr, data time = None if self.date == None: self.date = date.utcnow().date() if time == None or time == dttime.min: time = (datetime.utcnow()).time() #TODO address timezones by going to UTC everywhere return datetime.combine(self.date, time)
def getCaptureDateTime(self, data): try: timestr = "%08d" % (struct.unpack('L', data[1])[0]) #in format hhmmsscc time = dttime(int(timestr[:2]), int(timestr[2:4]), int(timestr[4:6]), int(timestr[6:])) except: print "Issue with time format:", timestr, data time = None if self.date == None: self.date = date.today() if time == None or time == dttime.min: time = (datetime.now()).time() #TODO address timezones by going to UTC everywhere return datetime.combine(self.date, time)
def post(self, lot_id, type): min_date = self.request.get('mindate') max_date = self.request.get('maxdate') min_hour = self.request.get('minhour') min_minute = self.request.get('minminute') max_hour = self.request.get('maxhour') max_minute = self.request.get('maxminute') if min_date != '': min_datetime = datetime.strptime(min_date, '%m/%d/%Y') else: min_datetime = datetime.min if max_date != '': max_datetime = datetime.strptime(max_date, '%m/%d/%Y') else: max_datetime = datetime.max min_time = dttime(int(min_hour), int(min_minute)) max_time = dttime(int(max_hour), int(max_minute)) if int(max_hour) == 0 and int(max_minute) == 0: # special case, we want end-of-day max_time = dttime.max min_datetime = datetime.combine(min_datetime.date(), min_time) max_datetime = datetime.combine(max_datetime.date(), max_time) # controller query buckets = controller.viewRange(lot_id, min_datetime, max_datetime) max_count = max(e['average'] for e in buckets) values = { 'min_date': min_date, 'max_date': max_date, 'buckets': buckets, 'max_count': max_count, 'lot_id': lot_id } path = os.path.join(base_path, 'templates/chart.html') self.response.out.write(template.render(path, values))
def build_smartexo(self, start_date, **smartexo_kwargs): self.clear_smartexo() logging.info( "Starting EXO calculation process from: {0}".format(start_date)) if self.smartexo_class.ASSET_LIST is None: warnings.warn( "You must define ASSET_LIST inside SmartEXO class. Aborting..." ) return for ticker in self.smartexo_class.ASSET_LIST: logging.info("Processing: " + ticker) currdate = start_date enddate = datetime.combine(datetime.now().date(), dttime(0, 0, 0)) while currdate <= enddate: start_time = time.time() date = currdate asset_info = self.assetindex.get_instrument_info(ticker) exec_time_end, decision_time_end = AssetIndexMongo.get_exec_time( date, asset_info) logging.info("\t\tRun on {0}".format(decision_time_end)) with self.smartexo_class(ticker, 0, decision_time_end, self.datasource, **smartexo_kwargs) as exo_engine: try: asset_list = exo_engine.ASSET_LIST # Checking if current symbol is present in EXO class ASSET_LIST if asset_list is not None: if ticker not in asset_list: # Skipping assets which are not in the list continue except AttributeError: warnings.warn( "EXO class {0} doesn't contain ASSET_LIST attribute filter, calculating all assets" .format(self.smartexo_class)) # Load EXO information from mongo exo_engine.load() exo_engine.calculate() end_time = time.time() currdate += timedelta(days=1) logging.debug("Elapsed: {0}".format(end_time - start_time)) logging.info('Done')
def toHourFraction(date): """Converts time into fractional hours between 0-24 Parameters ---------- date : numpy array 1 dimensional time series array to convert Returns ------- hour_frac : numpy array, float The hour fraction for each element of array 'date' """ sec = (date-datetime.combine(date.date(), dttime(0,0,0))).total_seconds() hour_frac = sec/3600 return hour_frac
def this_week_last_year(today=None): if not today: today = datetime.combine(date.today(), dttime(0)) #today = datetime.datetime.now() #could use date to same effect, #but that would require exceptions/checks in timestamp for type #today.hour = 0 #today.minute = 0 #today.second = 0 #these are read-only attributes year = timedelta(365) last_year = today - year start = last_year - timedelta(4) end = last_year + timedelta(4) #stamp = start.strftime("%Y%m%d") + '-' + end.strftime("%Y%m%d") return Timerange(start=start, end=end)
def __init__(self, input_json, time=None, daily=False): # get current weather if time is None: self.temp = self._round_temp(input_json['main']['temp']) self.condition = self.CONDITION_TRANSL[input_json['weather'][0]['id']] return # get prediction if daily: # set time to 13:00 for daily time = datetime.combine(time.date(), dttime(13, 00)) ts = int(time.strftime("%s")) # convert time to Unix timestamp for fc1, fc2 in zip(input_json['list'][:-1], input_json['list'][1:]): # find the appropriate time frame if ts >= fc1['dt'] and ts <= fc2['dt']: self.condition = self.CONDITION_TRANSL[fc1['weather'][0]['id']] # hourly forecast -- interpolate temperature if not daily: slope = (fc2['main']['temp'] - fc1['main']['temp']) / (fc2['dt'] - fc1['dt']) self.temp = self._round_temp(fc1['main']['temp'] + slope * (ts - fc1['dt'])) # daily forecast: use daily high & low else: self.temp = self._round_temp(fc1['temp']['day']) self.min_temp = self._round_temp(fc1['temp']['min']) self.max_temp = self._round_temp(fc1['temp']['max'])
def parseTime(self, date, ttime): ttime = re.sub('(\d+).(\d+)', r'\1:\2', ttime) ttime = re.sub('(\d+) (\d+)', r'\1:\2', ttime) ttime = re.sub('am', '', ttime) if re.search('pm', ttime): ttime = re.sub('pm', '', ttime) pm = True else: pm = False if len(ttime) == 1: delta = datetime.now().replace(hour=int(ttime)) if delta < datetime.now(): delta = delta + timedelta(hours=12) ttime = str(delta.hour) + ':' + str(delta.minute) elif ttime[0] > 0 and ttime[1] == ':': ttime = '0' + ttime thistime = time.strptime(ttime, '%H:%M') ttime = dttime(thistime.tm_hour, thistime.tm_min, 0) thisdate = datetime(date.year, date.month, date.day, ttime.hour, ttime.minute) if pm: delta = timedelta(hours=12) thisdate = thisdate + delta return thisdate
base_date = datetime(2014, 1, 13, 12, 45, 0) futures_limit = 3 options_limit = 10 DEBUG = '.' datasource = DataSourceMongo(mongo_connstr, mongo_db_name, assetindex, futures_limit, options_limit, exostorage) server = 'h9ggwlagd1.database.windows.net' user = '******' password = '******' datasource = DataSourceSQL(server, user, password, assetindex, futures_limit, options_limit, exostorage) enddate = datetime.combine(datetime.now().date(), dttime(12, 45, 0)) currdate = base_date instruments = ['CL', 'ES'] directions = [1, -1] # for i in range(100): while currdate <= enddate: start_time = time.time() # date = base_date + timedelta(days=i) date = currdate for ticker in instruments: for dir in directions: with EXOVerticalSpread(ticker, dir,
def __get_start_datetime(start_date, start_time_hours, start_time_minutes): if start_date is None or start_time_hours is None: return None start_datetime = datetime.combine(start_date, dttime()) start_datetime = start_datetime.replace(hour=start_time_hours, minute=start_time_minutes, second=0, microsecond=0) return start_datetime
def main(argv): """ METHODS: read_kpnow_data(source) read_swpam_data(source, limit=5) _execute_sql swtableinit() """ version = "1.0.0" kpsource = '/srv/archive/external/gfz/kp/' swsource = '/srv/archive/external/esa-nasa/ace/raw/' kpname = 'gfzkp*' swename = '*_swepam_1m.txt' magname = '*_mag_1m.txt' configpath = '' # is only necessary for monitoring sqllist = [] debug = False init = False # create table if TRUE statusmsg = {} usage = 'sw_extractor.py -c <config> -k <kpsource> -s <swesource> ' try: opts, args = getopt.getopt(argv,"hc:k:s:o:p:b:e:ID",["config=","kpsource=","swesource=","init=","debug=",]) except getopt.GetoptError: print(usage) sys.exit(2) for opt, arg in opts: if opt == '-h': print('-------------------------------------') print('Description:') print('sw-extractor.py extracts CME predictions from Scoreboard ') print('-------------------------------------') print('Usage:') print(usage) print('-------------------------------------') print('Options:') print('-c : configuration data ') print('-k : Kp source - default is ') print('-s : SWE source - default is ') print('-------------------------------------') print('Examples:') print('---------') print('python3 sw_extractor.py -c /home/cobs/CONF/wic.cfg -k /srv/archive/external/gfz/kp/gfzkp* -s /srv/archive/external/esa-nasa/ace/raw/ -D') print('---------') sys.exit() elif opt in ("-c", "--config"): configpath = os.path.abspath(arg) elif opt in ("-k", "--kpsource"): kpsource = os.path.abspath(arg) elif opt in ("-s", "--swesource"): swsource = os.path.abspath(arg) elif opt in ("-n", "--kpname"): kpname = arg elif opt in ("-w", "--swename"): swname = arg elif opt in ("-m", "--magname"): magname = arg elif opt in ("-I", "--init"): init = True elif opt in ("-D", "--debug"): debug = True if debug: print ("Running sw-extractor version:", version) # 1. conf and logger: # ########################### if debug: print ("Read and check validity of configuration data") print (" and activate logging scheme as selected in config") config = GetConf(configpath) config = DefineLogger(config=config, category = "Info", job=os.path.basename(__file__), newname='mm-info-sw.log', debug=debug) if debug: print (" -> Done") # 2. database: # ########################### try: config = ConnectDatabases(config=config, debug=debug) db = config.get('primaryDB') connectdict = config.get('conncetedDB') except: statusmsg[name1] = 'database failed' # 3. Read Kp data: # ########################### try: sqllist = read_kpnow_data(os.path.join(kpsource,kpname),debug=debug) statusmsg['Kp access'] = 'success' except: statusmsg['Kp access'] = 'failed' # 4. Read ACE swepam data: # ########################### try: newsql = read_swepam_data(os.path.join(swsource,swename),debug=debug) sqllist.extend(newsql) statusmsg['ACE swepam access'] = 'success' except: statusmsg['ACE swepam access'] = 'failed' # 5. Read ACE mag data: # ########################### try: newsql = read_mag_data(os.path.join(swsource,magname),debug=debug) sqllist.extend(newsql) statusmsg['ACE mag access'] = 'success' except: statusmsg['ACE mag access'] = 'failed' # 6. Read GIC data: # ########################### try: newsql = read_gicnow_data(db,source='GICAUT',maxsensor=9, minutes=5, debug=debug) sqllist.extend(newsql) statusmsg['GIC data access'] = 'success' except: statusmsg['GIC data access'] = 'failed' # 7. Read GOES data: # ########################### try: if debug: print ("Running GOES") goespath = '/srv/archive/external/esa-nasa/goes' newsql = read_xrs_data(os.path.join(goespath,'XRS_GOES16*'), debug=debug) sqllist.extend(newsql) statusmsg['XRS data access'] = 'success' except: statusmsg['XRS data access'] = 'failed' # 8. Read PREDSTORM data: # ########################### try: if debug: print ("Running PREDSTORM") predpath = '/srv/archive/external/helio4cast/predstorm' psql = read_predstorm_data(os.path.join(predpath,'PREDSTORM*'), debug=debug) sqllist.extend(psql) statusmsg['PREDSTORM data access'] = 'success' except: # no predstorm data between 23:00 and 2:00 MET # just put success message if hitting except in this time range statusmsg['PREDSTORM data access'] = 'failed' if time_between(datetime.utcnow().time(),dttime(21,0),dttime(0,0)): statusmsg['PREDSTORM data access'] = 'success' sqllist = [el for el in sqllist if el] if debug: print ("Debug selected - sql call looks like:") print (sqllist) else: for dbel in connectdict: db = connectdict[dbel] print (" -- Writing data to DB {}".format(dbel)) if init: swtableinit(db) if len(sqllist) > 0: _execute_sql(db,sqllist, debug=debug) # Logging section # ########################### if not debug and config: martaslog = ml(logfile=config.get('logfile'),receiver=config.get('notification')) martaslog.telegram['config'] = config.get('notificationconfig') martaslog.msg(statusmsg) else: print ("Debug selected - statusmsg looks like:") print (statusmsg)
def strToDateTime( instr ): TimeIn = [0]*5 # 年月日时分 today = datetime.now() TimeType = 0 instrlen = len(instr) strindex = 0 issettime = False unitindata = False #print(today) NextTimeperiod = 0 lastdataindex = 2 if instr[0:2] in TimeSegment.keys(): strindex = 2 k,t = TimeSegment[instr[0:2]] #today += timedelta(hours = t) if t > 23: today += timedelta(hours = t) else: today = datetime.combine(today.date(),dttime(t,0)) TimeType = k else: if instr[0] == u"下": NextTimeperiod = 1 strindex += 1 if instr[strindex] == u"下": NextTimeperiod = 2 strindex += 1 if instr[strindex] == u"周" or instr[strindex:strindex+1] == u"星期": if instr[strindex] == u"周": strindex += 1 else: strindex += 2 number = u"" if instrlen > strindex: word = instr[strindex] if word in ChenseToNumber.keys(): number += ChenseToNumber[word] elif word in u"123456": number += word elif word == u"日": number = u"7" strindex += 1 else: return (-1,"12") try: inum = int(number) -1 except ValueError: return (-1,"10") #issettime = True day = today.weekday() if day <= inum: today = today + timedelta(days = inum-day) else: today = today + timedelta(days = inum-day+7) NextTimeperiod -= 1 if NextTimeperiod > 0: today = today + timedelta(days = NextTimeperiod*7) # else: # return (-1,u"") if u"后" in instr: afday = 0 afhour = 0 afminute = 0 afweek = 0 number = u'' while strindex < instrlen: word = instr[strindex] if word in ChenseToNumber.keys(): number += ChenseToNumber[word] elif word in u"0123456789": number += word elif word == u"天": try: afday = int(number) except ValueError: return (-1,"10") number = u"" # issettime = True elif word == u"小": strindex += 1 word = instr[strindex] if word == u"时": try: afhour = int(number) except ValueError: return (-1,"10") number = u"" lastdataindex = 3 issettime = True else: return (-1,u"1") elif word == u"分": if instr[strindex+1] == u"钟": strindex += 1 try: afminute = int(number) except ValueError: return (-1,"10") lastdataindex = 4 number = u'' issettime = True elif word == u"半": strindex += 1 if instrlen - strindex < 3: return (-1,u"2") if instr[strindex] == u"个": strindex += 1 if instr[strindex:strindex+2] == u"小时": afminute = 30 strindex += 1 lastdataindex = 4 number = u'' else: return (-1,u"3") elif word == u"个": pass elif word == u"后": strindex += 1 number = u'' today = today + timedelta(days = afday,hours = afhour,minutes = afminute,weeks = afweek) break; elif word == u"周": try: afweek = int(number) except ValueError: return (-1,"10") # issettime = True strindex += 1 #else: TimeIn[0] = today.year TimeIn[1] = today.month TimeIn[2] = today.day TimeIn[3] = today.hour TimeIn[4] = today.minute #print(strindex) number = u'' while strindex < instrlen: word = instr[strindex] if word in ChenseToNumber.keys(): tnum = ChenseToNumber[word] if len(number) == 0: number = tnum elif len(number) == 2: number = number[0] + tnum else: return (-1,u"4") #number += ChenseToNumber[word] elif word == u"十": if number == u'': number = u'10' else: number += u"0" elif word == u"刻": try: number = str(int(number)*15) except ValueError: return (-1,"10") if int(number) > 59: return (-1,u"5") try: TimeIn[4] = int(number) except ValueError: return (-1,"10") number = u'' issettime = True break elif word in ChenseData.keys(): lastdataindex = ChenseData[word] unitindata = True #print(word) try: TimeIn[lastdataindex] = int(number) except ValueError: return (-1,"10") if lastdataindex == 3: TimeIn[4] = 0 issettime = True number = u'' if lastdataindex == 4: issettime = True break elif word in u"0123456789-": number += word elif word == u' ': if lastdataindex == 2: derror = False try: if u'-' in number: if number.count(u'-') == 1: index = number.find(u'-') TimeIn[1] = int(number[:index]) TimeIn[2] = int(number[index+1:]) elif number.count(u'-') == 2: index = number.find(u'-') TimeIn[0] = int(number[:index]) number = number[index+1:] index = number.find(u'-') TimeIn[1] = int(number[:index]) TimeIn[2] = int(number[index+1:]) else: derror = True number = u"" else: if len(number) == 2: TimeIn[2] = int(number) elif len(number) == 4: TimeIn[1] = int(number[:2]) TimeIn[2] = int(number[2:]) elif len(number) == 6: TimeIn[0] = int(number[:2])+2000 TimeIn[1] = int(number[2:4]) TimeIn[2] = int(number[4:]) elif len(number) == 8: TimeIn[0] = int(number[:4]) TimeIn[1] = int(number[4:6]) TimeIn[2] = int(number[6:]) else: derror = True number = u"" except ValueError: return (-1,u"6") if derror: #return (-1,u"7") if lastdataindex > 2: break else: lastdataindex = 3 else: break elif instr[strindex:strindex+2] in TimeSegment.keys(): k,t = TimeSegment[instr[strindex:strindex+2]] # today += timedelta(hours = t) TimeType = k strindex += 1 elif word == u'半': if lastdataindex == 3: TimeIn[4] = 30 issettime = True else: return (-1,u"8") elif word == u"&": break; else: break; strindex += 1 else: result = (-1,u"11") # strindex -= 1 # if strindex == instrle: strindex += 1 if number != u'' and unitindata: try: TimeIn[lastdataindex+1] = int(number) except ValueError: return (-1,"10") issettime = True #晚上 if TimeType == 2: if TimeIn[3] < 13: TimeIn[3] += 12 if TimeIn[3] == 24: TimeIn[2] += 1 TimeIn[3] = 0 #中午 elif TimeType == 3: if TimeIn[3] < 3: TimeIn[3] += 12 if issettime == False: return (-1,u"13") #print (TimeIn) try: result = (strindex,datetime(TimeIn[0],TimeIn[1],TimeIn[2],TimeIn[3],TimeIn[4])) except: result = (-1,u"10") return result
import pytz, re from datetime import datetime, time as dttime trading_exchange_timezone = pytz.timezone('US/Eastern') exchange_opening_time = dttime(hour=9, minute = 30) exchange_normal_close_time = dttime(hour=16) exchange_early_close_time = dttime(hour=13) trading_holidays = ( #US Exchanges #We record back to 2010 because we can request up to 5yrs of historical data ('1/1/2010', 'full day'), ('1/18/2010', 'full day'), ('2/15/2010', 'full day'), ('4/4/2010', 'full day'), ('5/31/2010', 'full day'), ('7/5/2010', 'full day'), ('9/6/2010', 'full day'), ('11/25/2010', 'full day'), ('11/26/2010', 'early close'), ('12/24/2010', 'full day'), ('1/17/2011', 'full day'), ('2/21/2011', 'full day'), ('4/22/2011', 'full day'), ('5/30/2011', 'full day'), ('7/4/2011', 'full day'), ('9/5/2011', 'full day'), ('11/24/2011', 'full day'), ('11/25/2011', 'early close'), ('12/26/2011','full day'), ('1/2/2012', 'full day'),
#!/usr/bin/env python3 """Call the script's arguments periodically (see TRIGGER_TIMES).""" import subprocess import sys import time from datetime import datetime, timedelta from datetime import time as dttime # Run the arguments every time we hit one of these times. # Keep this in sorted order... TRIGGER_TIMES = [ dttime(hour=0, minute=13), dttime(hour=6, minute=10), dttime(hour=11, minute=51), dttime(hour=17, minute=49) ] ONEDAY = timedelta(days=1) def find_sleep_info(): now = datetime.now() dts = [datetime.combine(now.date(), t) for t in TRIGGER_TIMES] dts.append(datetime.combine((now + ONEDAY).date(), TRIGGER_TIMES[0])) sleep_until = None sleep_for = None for dt in dts:
def clean(self, value, *args, **kwargs): time_struct = time.strptime(value, "%H:%M") return dttime(hour=time_struct.tm_hour, minute=time_struct.tm_min)