def fetchServer2Data_Live(): #called every minute data = urllib2.urlopen("https://api.bitcoinaverage.com/history/USD/per_minute_24h_sliding_window.csv") PriceWindow = [] firstLine = True for line in data: if firstLine: firstLine = False continue time = line.split(",")[0] time1 = time.split(" ")[0] time2 = time.split(" ")[1] time = time1+"-"+time2 # get rid of second time1 = time.split(":")[0] time2 = time.split(":")[1] time = time1+":"+time2 time = UTCtoLinux(time) price = float(line.split(",")[1]) Price[time] = price PriceWindow.append(price) # updating the internal data structure # now time points to the latest time print "Adding Server2 data at:" + linuxToUTC(time) + " Price " + str(price) calculateTradingStrategies(PriceWindow,time)
def handleRecentsMessage(self, recents): if not recents and syncthing_repositories: return recents = [recent for recent in recents if recent['type'] == 'LocalIndexUpdated'] recents = recents[-10:] folders = {repo['ID']: repo['Directory'] for repo in syncthing_repositories} if recents: # remove all actions in recents menu self.recents.clear() max_length = max([len(recent['data']['name']) for recent in recents]) for recent in recents: filename = recent['data']['name'] directory = recent['data']['repo'] time = recent['time'] time = '+'.join([time.split('+')[0][:-1], time.split('+')[1]]) # hack to let arrow parse it time = arrow.get(time).humanize() action = QAction('%-*s (%s)' % (-max_length, filename, time), self.recents) if os.path.exists(os.path.join(folders[directory], filename)): action.setIcon(QIcon('icons/newfile.png')) action.triggered.connect(lambda: self.open_dir(folders[directory])) else: action.setIcon(QIcon('icons/delfile.png')) folder = os.path.join(folders[directory], '.stversions') if os.path.exists(os.path.join(folder, filename)): action.triggered.connect(lambda: self.open_dir(folder)) self.recents.addAction(action)
def get_bookmarks_time(url, opener=opener): # go to the bookmarks page of the work and find the timestamps for the bookmarks # returns a dict of {month:# of bookmarks in the month} req = urllib2.Request(url) page = bs(opener.open(req)) page_list = [i for i in re.findall('<a href="(.*?)>', str(page)) if "bookmarks?" in i] page_list = sorted(list(set([i.split()[0].replace('"', "") for i in page_list]))) dt = re.findall('<p class="datetime">(.*?)</p>', str(page)) times = [] month_dict = { "Jan": "01", "Feb": "02", "Mar": "03", "Apr": "04", "May": "05", "Jun": "06", "Jul": "07", "Aug": "08", "Sep": "09", "Oct": "10", "Nov": "11", "Dec": "12", } for time in dt: times.append(time.split()[2] + "-" + month_dict.get(time.split()[1])) times = times[1:] if page_list != []: for page in page_list: times += get_bookmarks_time_subpages("http://archiveofourown.org" + page, opener=opener) c = Counter(times) return {time: c[time] for time in times}
def time_to_int(time): if time.count(":") == 2: (hour, min, sec) = time.split(":") return (int(hour) * 3600) + (int(min) * 60) + int(sec) else: (min, sec) = time.split(":") return (int(min) * 60) + int(sec)
def get_song_length(duration, songnumber): duration = str(duration) songnumber = int(songnumber) if songnumber <= 1: time = duration.split(" ")[0] minutes_ev = time.split(":")[0] seconds_ev = time.split(":")[1] minutes = "".join([letter for letter in minutes_ev if letter.isdigit()]) seconds = "".join([letter for letter in seconds_ev if letter.isdigit()]) songtime = (int(minutes) * 60) + int(seconds) return songtime else: songfetch = int(songnumber) - 1 if len(duration.split(" ")) >= songfetch: time = duration.split(" ")[songfetch] minutes_ev = time.split(":")[0] seconds_ev = time.split(":")[1] minutes = "".join([letter for letter in minutes_ev if letter.isdigit()]) seconds = "".join([letter for letter in seconds_ev if letter.isdigit()]) songtime = (int(minutes) * 60) + int(seconds) return songtime else: global __settings__ int(getsl(__settings__.getSetting("sl_custom"))) return int(getsl(__settings__.getSetting("sl_custom")))
def toDateTime(sVal, iDefault=None): """ Suponer formato Iso OrderingDate """ if sVal is None: return iDefault try: if sVal.count("T") > 0: # IsoFormat DateTime (date, time) = sVal.split("T") (an, mois, jour) = date.split('-') (h, m, s) = time.split(':') return datetime.datetime(int(an), int(mois), int(jour), int(h), int(m), int(s)) elif sVal.count("-") == 2: # IsoFormat Date (an, mois, jour) = sVal.split('-') return datetime.date(int(an), int(mois), int(jour)) elif sVal.count("/") == 2: if sVal.count(' ') > 0: (date, time) = sVal.split(" ") (jour, mois, an) = date.split('/') (h, m, s) = time.split(':') return datetime.datetime(int(an), int(mois), int(jour), int(h), int(m), int(s)) else: (jour, mois, an) = date.split('/') return datetime.date(int(an), int(mois), int(jour)) except: return iDefault
def _srtTc2ms(self, time): if ',' in time: split_time = time.split(',') else: split_time = time.split('.') minor = split_time[1] major = split_time[0].split(':') return (int(major[0])*3600 + int(major[1])*60 + int(major[2])) * 1000 + int(minor)
def run(self): """ Gets tracking information from the APRS receiver """ aprsSer = self.APRS.getDevice() while(not self.aprsInterrupt): ### Read the APRS serial port, and parse the string appropriately ### # Format: # "Callsign">CQ,WIDE1-1,WIDE2-2:!"Lat"N/"Lon"EO000/000/A="Alt"RadBug,23C,982mb,001 # ### try: line = str(aprsSer.readline()) print(line) idx = line.find(self.callsign) if(idx != -1): line = line[idx:] line = line[line.find("!") + 1:line.find("RadBug")] line = line.split("/") # Get the individual values from the newly created list ### time = datetime.utcfromtimestamp( time.time()).strftime('%H:%M:%S') lat = line[0][0:-1] latDeg = float(lat[0:2]) latMin = float(lat[2:]) lon = line[1][0:line[1].find("W")] lonDeg = float(lon[0:3]) lonMin = float(lon[3:]) lat = latDeg + (latMin / 60) lon = -lonDeg - (lonMin / 60) alt = float(line[3][2:]) aprsSeconds = float(time.split( ':')[0]) * 3600 + float(time.split(':')[1]) * 60 + float(time.split(':')[2]) ### Create a new location object ### try: newLocation = BalloonUpdate( time, aprsSeconds, lat, lon, alt, "APRS", self.mainWindow.groundLat, self.mainWindow.groundLon, self.mainWindow.groundAlt) except: print( "Error creating a new balloon location object from APRS Data") try: # Notify the main GUI of the new location self.aprsNewLocation.emit(newLocation) except Exception, e: print(str(e)) except: print("Error retrieving APRS Data") ### Clean Up ### try: aprsSer.close() # Close the APRS Serial Port except: print("Error closing APRS serial port") self.aprsInterrupt = False
def get_bookmarks_time_subpages(url, opener=opener): #A work's bookmarks can take up multiple pages. In this case, all timestamp information is add to the first page. req = urllib2.Request(url) page = bs(opener.open(req)) dt = re.findall('<p class="datetime">(.*?)</p>', str(page)) times = [] month_dict = {'Jan':'01', 'Feb':'02','Mar':'03', 'Apr':'04', 'May':'05', 'Jun':'06', 'Jul':'07', 'Aug':'08', 'Sep':'09', 'Oct':'10', 'Nov':'11', 'Dec':'12'} for time in dt: times.append(time.split()[2] + '-' + month_dict.get(time.split()[1])) return times[1:]
def get_bookmarks_time_subpages(url, opener=opener): req = urllib2.Request(url) page = bs(opener.open(req)) dt = re.findall('<p class="datetime">(.*?)</p>', str(page)) times = [] month_dict = {'Jan':'01', 'Feb':'02','Mar':'03', 'Apr':'04', 'May':'05', 'Jun':'06', 'Jul':'07', 'Aug':'08', 'Sep':'09', 'Oct':'10', 'Nov':'11', 'Dec':'12'} for time in dt: times.append(time.split()[2] + '-' + month_dict.get(time.split()[1])) times = times[1:] return times
def time_convert(self, time): print "[YWeather] Time convert" tmp_time = '' if time.endswith('pm'): tmp_time = '%s:%s' % (int(time.split()[0].split(':')[0]) + 12, time.split()[0].split(':')[-1]) else: tmp_time = time.replace('am', '').strip() if len(tmp_time) is 4: return '0%s' % tmp_time else: return tmp_time
def get_timedelta(time): minutes = 0 if len(time.split(':')) > 1: minutes, seconds = [float(t) for t in time.split(':')] else: try: seconds = float(time) except ValueError: seconds = 1000 * 1000 return datetime.timedelta(minutes=minutes, seconds=seconds)
def convertTime(time): timeInSeconds = 0 if time.find(":")>0: min,sec = time.split(":") elif time.find("m")>0: min,sec = time.split("m") sec = sec.replace("s","") else: min = 0 sec = 0 min = int(min) sec = int(sec) return (min*60)+sec
def timeshifter(time): #06/02/2014 02:30 -> 2014-06-02T02:30 try: origdate, origtime = time.split(' ')[0], time.split(' ')[1] date = origdate.split('/')[2] + '-' + origdate.split('/')[0] + '-' + origdate.split('/')[1] time = origtime + ':00' return date + 'T' + time except Exception,e : return None
def generatebeginTimecode(begin_time_list): beginTimecode_list=list() for i in range(0, len(begin_time_list)): datetime = begin_time_list[i].split(' ')[0] time = begin_time_list[i].split(' ')[1] year = int(datetime.split('-')[0]) month = int(datetime.split('-')[1]) day = int(datetime.split('-')[2]) hour = int(time.split(':')[0]) minute = int(time.split(':')[1]) second ='00' beginTimecode = str(year)+str('%02d'%month)+str('%02d'%day)+str('%02d'%hour)+str('%02d'%minute)+second beginTimecode_list.append(beginTimecode) return beginTimecode_list
def save_channels_rambler(date_invert, ADDRESS): source = 1 #rambler source_obj = TvForecastSources.objects.get(id = 1) g.go(ADDRESS) res = g.response.body soup = BeautifulSoup(str(res)) for channel in soup.findAll("table", {"class":"grid-element"}): must_del = True #для каждого нового канала channel_my_id = rambler_to_mine_channel[channel.find('td', {"class" : "plain-event-logo"}).text] channel_obj = TvChannels.objects.get(id = channel_my_id) for param in channel.findAll("div", {"class":"plain-event"}): #if channel_obj.id ==7 or channel_obj.id ==8 or channel_obj.id ==9: #временно только для первого канала #print parmam.text try: name = param.find('a').text #название передачи time = param.find('span').text #время показа time_splitted = time.split(":") except AttributeError: name= param.contents[1].nextSibling # если не передача а сообщение все равно название передачи time= param.find('span').text #время показа если не передача а сообщение time_splitted = time.split(":") #здесь можем сохранить объект в базу tv_forecast_obj = TvForecast(source=source_obj, channel = channel_obj, #date = '2015-01-12', #из андреса date = date_invert, #из андреса time_hour = time_splitted[0], time_minute= time_splitted[1], name = name, date_get = datetime.datetime.now().strftime("%Y-%m-%d") , ) #удалить записи по данному каналу на данную дату if tv_forecast_obj: if must_del ==True: tv_forecast_obj_must_del = TvForecast.objects.filter(source = source_obj, date=date_invert, channel = channel_obj) #print channel_obj.id if tv_forecast_obj_must_del.count() != 0: for tmd in tv_forecast_obj_must_del: #print tmd.id tmd.delete() must_del = False #tv_forecast_obj_must_del.delete() tv_forecast_obj.save()
def ConvertTimeToMinutes(time): """Converts time in HH:MM:SS format to minutes.""" t = time.split('.') if len(t) == 2: time, fraction = time.split('.') t = [int(x) for x in time.split(':')] if len(t) == 2: h = 0 m, s = t else: h, m, s = t mins = h * 60 + m + s / 60.0 return mins
def total(self): logger.debug(u'Fetching MPD total time') time = self.status.get('time') if time: return self._parse_time(time.split(':')[-1]) else: return None
def get_quote_info(ctx, pline, userdata): args = pline.trailing.split() args.pop(0) if args: try: quote_id = int(args.pop(0)) except ValueError: ctx.command('/say Invalid argument.') return else: ctx.command('/say You must provide a quote id.') return init_quotes_db() with closing(sqlite3.connect(quote_db_filename)) as quote_db: with closing(quote_db.cursor()) as cursor: cursor.execute('''SELECT "id", "server", "channel", "date" FROM "quotes" WHERE "id"=?''', (quote_id,)) row = cursor.fetchone() if row is None: ctx.command('/say That quote does not exist.') return quote_id, quote_server, quote_channel, quote_date = row date, time = quote_date.split() year, month, day = map(int, date.split('-')) hour, minute, second = map(int, time.split(':')) dt_utc_naive = datetime(year, month, day, hour, minute, second) dt = pytz.timezone('Europe/Berlin').fromutc(dt_utc_naive) ctx.command(u'/say \x02#{} infos:\x02 {} \x02@\x02 {} \x02at\x02 {}'.format( quote_id, quote_channel, quote_server, dt.strftime(u'%Y-%m-%d %H:%M') ).encode('utf-8'))
def is_valid_time(time): try: hour, minute = map(int, time.split(":")) except: return False else: return (hour < 24 and minute < 60)
def time_to_seconds(time): """ Takes a string in the form of "%M:%S" and returns the total seconds. Can also handle cases where only ":%S" is provided. """ m,s = [int(x) if x != '' else 0 for x in time.split(':')] return 60*m + s
def elapsed(self): logger.debug(u'Fetching MPD elapsed time') time = self.status.get('time') if time: return self._parse_time(time.split(':')[0]) else: return None
def parseString(message): s = message.split("Date: ") s = s[1].split(" Time: ") date = s[0] s = s[1].split(" L") time = s[0] s = s[1].split(" ") light = s[0] hum = s[2].split("%")[0] temp = (s[1].split("C"))[0] moist = (s[3].split("% M"))[0].split("M")[1].split("\r")[0] p = date.split(".") date = p[2]+"-" if len(p[1]) < 2: date+="0" date+= p[1]+"-" if len(p[0]) < 2: date+="0" date+= p[0]+" " p = time.split(":") if len(p[0]) < 2: date+="0" date+= p[0]+":" if len(p[1]) < 2: date+="0" date+= p[1]+":" if len(p[2]) < 2: date+="0" date+= p[2] return [date, temp, hum, moist, light]
def __init__(self, time, when): self.when = when (hours, minutes) = time.split(":") self.minute = int(hours) * 60 + int(minutes)
def parse_datetime(date): """Parse ISO format date/time string into Datetime object. >>> from datetime import datetime >>> print parse_datetime('2007-10-05') None >>> parse_datetime('2007-10-05 08:12:43') == datetime(2007, 10, 5, 8, 12, 43) True >>> parse_datetime('2007-10-06') == datetime(2007, 10, 5) False >>> print parse_datetime('invalid') None """ try: date, time = date.split(' ') year, month, day = date.split('-')[:3] year = int(year) month = int(month) day = int(day) hours, minutes, seconds = time.split(':')[:3] hours = int(hours) minutes = int(minutes) seconds = int(seconds) return datetime(year, month, day, hours, minutes, seconds) except: return None
def validate(self): if not self.iso8601_re.match(self.value): self.log(self.message({"parent":self.parent.name, "element":self.name, "value":self.value})) return work=self.value.split('T') date=work[0].split('-') year=int(date[0]) if len(date)>1: month=int(date[1]) try: if len(date)>2: datetime.date(year,month,int(date[2])) except ValueError as e: return self.log(self.message({"parent":self.parent.name, "element":self.name, "value":str(e)})) if len(work) > 1: time=work[1].split('Z')[0].split('+')[0].split('-')[0] time=time.split(':') if int(time[0])>23: self.log(self.message({"parent":self.parent.name, "element":self.name, "value":self.value})) return if len(time)>1 and int(time[1])>60: self.log(self.message({"parent":self.parent.name, "element":self.name, "value":self.value})) return if len(time)>2 and float(time[2])>60.0: self.log(self.message({"parent":self.parent.name, "element":self.name, "value":self.value})) return self.log(ValidW3CDTFDate({"parent":self.parent.name, "element":self.name, "value":self.value})) return 1
def get_expires(self, mensaje): """ Extraigo el tiempo de expiracion del mensaje """ time = mensaje.split("Expires: ")[1] time = time.split("\r")[0] return(int(time))
def store_index_h5(self, time, index,flag = 1) : """Store information about: * Time-stamp * Total intensity * Beam center * Estimated Particle Size * Estimated particle nr """ self.tot_t[index] = time.split('_')[1] self.tot_int[index] = float(self.img.sum()) self.tot_peak1_int[index] = self.peak1 self.tot_peak2_int[index] = self.peak2 self.tot_streak_m[index] = self.streak_m self.tot_streak_s[index] = self.streak_s self.tot_cx[index] = self.cent[0] self.tot_cy[index] = self.cent[1] self.tot_size[index] = self.radius self.tot_score[index] = self.score if flag : self.ave += self.img
def convert_time(self, time): """ Helper function to convert the displaytime strings to an actual integer minute represenation. Examples: 'Nu' => 0, '8 min.'' => 8, '12:22' (at the time of 12:18) => 4 '9' => 9 """ if 'min' in time: time = time.replace('min', '').replace('.', '').strip() elif 'Nu' in time: time = 0 elif ':' in time: now = self.get_now() # floor below minute now = datetime.datetime(year=now.year, month=now.month, day=now.day, hour=now.hour, minute=now.minute, second=0, microsecond=0) hour, minute = time.split(':') dtime = datetime.datetime(year=now.year, month=now.month, day=now.day, hour=int(hour), minute=int(minute), second=0, microsecond=0) # 00.00 wraparound? if dtime < now: dtime = dtime + datetime.timedelta(days=1) time = round((dtime - now).total_seconds() / 60.0) return int(time)
def as_dict(self): self.photo = None if self.picture: self.photo = "/img?img_id=%s" % self.key() self.created = self.created + datetime.timedelta(hours = 8) self.lastModified = self.lastModified + datetime.timedelta(hours = 8) date, time = self.lastSeen.split(" ") year, month, day = map(int, date.split("-")) hour, minute, second = map(int, time.split(":")) self.lastSeenTime = datetime.datetime(year, month, day, hour, minute, second) d = {'name': self.name, 'age': self.age, 'last_seen': self.lastSeenAt, 'last_seen_date': self.lastSeenTime.strftime("%d %b %Y"), 'last_seen_time': self.lastSeenTime.strftime("%I:%M %p"), 'contact_details': self.contactDetails, 'user': self.user, 'found': self.found, 'posted_date': self.created.strftime("%b %d, %Y"), 'posted_time': self.created.strftime("%I:%M %p") } if self.photo: d['picture'] = self.photo if self.additionalDetails: d['additional_details'] = self.additionalDetails if not (self.created.strftime("%I:%M %p") == self.lastModified.strftime("%I:%M %p") and (self.created.strftime("%b %d, %Y") == self.lastModified.strftime("%b %d, %Y"))): d['last_modified_date'] = self.lastModified.strftime("%b %d, %Y") d['last_modified_time'] = self.lastModified.strftime("%I:%M %p") return d
def time_in_seconds(time): t1 = time.split() t2 = t1[3].split(':') return int(t2[0])* 3600 + int(t2[1])*60 + int(t2[2])
def _gettime(self, date, time): '''get time in datetime from strings date and time''' mm, dd, yy = date.split('/') hh, mins, ss = time.split(':') return mm, dd, yy, hh, mins, ss
def _get_jwbinfosys_course(self, sess, semester_num=4): base = 2018 teacher2num = {} semester2num = {} teacher2course = {} res = None res = self._get( sess=sess, url='http://jwbinfosys.zju.edu.cn/xskbcx.aspx?xh={}'.format( self._stuid)) self._get_jwbinfosys_util(res, teacher2num, semester2num, teacher2course, '2018-2019 春夏') viewstate = re.search('name="__VIEWSTATE" value=".*?"', res.text).group(0)[26:-1] data = { '__VIEWSTATE': viewstate, 'xnd': '2018-2019', 'xqd': '1|秋、冬'.encode('GBK') } res = self._post( sess=sess, url='http://jwbinfosys.zju.edu.cn/xskbcx.aspx?xh={}'.format( self._stuid), data=data) self._get_jwbinfosys_util(res, teacher2num, semester2num, teacher2course, '2018-2019 秋冬') for i in range(1, self._semester_num): viewstate = re.search('name="__VIEWSTATE" value=".*?"', res.text).group(0)[26:-1] data = { '__VIEWSTATE': viewstate, 'xnd': '{}-{}'.format(base - i, base - i + 1), } res = self._post( sess=sess, url='http://jwbinfosys.zju.edu.cn/xskbcx.aspx?xh={}'.format( self._stuid), data=data) self._get_jwbinfosys_util( res, teacher2num, semester2num, teacher2course, '{}-{} 春夏'.format(base - i, base - i + 1)) viewstate = re.search('name="__VIEWSTATE" value=".*?"', res.text).group(0)[26:-1] data = { '__VIEWSTATE': viewstate, 'xnd': '{}-{}'.format(base - i, base - i + 1), 'xqd': '1|秋、冬'.encode('GBK') } res = self._post( sess=sess, url='http://jwbinfosys.zju.edu.cn/xskbcx.aspx?xh={}'.format( self._stuid), data=data) self._get_jwbinfosys_util( res, teacher2num, semester2num, teacher2course, '{}-{} 秋冬'.format(base - i, base - i + 1)) soup = bs4.BeautifulSoup(res.text, 'html.parser') year = soup.find(id='xnd').find(attrs={'selected': "selected"}).text table = soup.find(id='xsgrid') trs = table.findAll('tr')[1:] first_semester_course = [] for tr in trs: tds = tr.findAll('td') course_name, teacher, semester, time, place = tds[1].text, tds[ 2].text, tds[3].text, tds[4].text, tds[5].text if semester.find('秋') != -1: places = place.split('<br/>') times = time.split('<br/>') time, place = None, None if len(times) >= 2: time = sorted(times)[0] place = places[times.index(time)] else: place = places[0] time = times[0] first_semester_course.append( (course_name, teacher, place, time)) first_semester_course = sorted(first_semester_course, key=lambda d: d[-1]) first_course = None if first_semester_course[0][0] == '军训': first_course = { 'name': first_semester_course[1][0], 'teacher': first_semester_course[1][1], 'place': first_semester_course[1][2] } else: first_course = { 'name': first_semester_course[0][0], 'teacher': first_semester_course[0][1], 'place': first_semester_course[0][2] } return teacher2num, teacher2course, semester2num, first_course
def call_api(base_url, cookies): # ---- make REST API Call section ------- # Generate URL used for REST API call to get 5minute temperature data # curly braces {} in string indicates a "replacement field" sensor_url = base_url + 'mo/{}/sys/ch/supslot-1/sup/sensor-3/CDeqptTemp5min.json' # Get all leaf information leaf_url = base_url + '/class/fabricNode.json?query-target-filter=and(eq(fabricNode.role,"leaf"))' leafs = requests.get(leaf_url, cookies=cookies, verify=False).json()['imdata'] leaf_dns = [] # for each of the objects returned by the API we will extract the dn for leaf in leafs: # this will return something like - # "topology/pod-1/node-101" - replaces curly braces {} in sensor_url dn = leaf['fabricNode']['attributes']['dn'] leaf_dns.append(dn) # now we will run a query for the sensor of each node by it's dn for dn in leaf_dns: resp = requests.get(sensor_url.format(dn), cookies=cookies, verify=False).json()['imdata'][0] temp = resp['eqptTemp5min']['attributes']['currentMax'] ts = resp['eqptTemp5min']['attributes']['repIntvEnd'] # clean up dn - only save node name dn = dn.replace('topology/pod-1/', '') # temperature returned as string - # make it an int and assign condition int_temp = int(temp) if int_temp >= 60: # temp is greater than 60 degrees C condition = "high" elif int_temp >= 48: # temp between 48 and 60 degrees C condition = "elevated" else: # temp less than 48 degrees C condition = "normal" # take day/time returned and place into two separate variables date = ts.split('T')[0] # date occurs before 'T' in ts string time = ts.split('T')[1] # time occurs after 'T' in ts string time = time.split('.')[ 0] # remove microseconds from time occurring after period obj = { "dn": dn, "attributes": { "temp": temp, "timestamp": time, "date": date, "condition": condition, "type": 'leaf' } } # send object to RESTAPI function upload = send2_RESTAPI(obj) if upload: print "device successfully uploaded to api (L)" else: print "error uploading device " # Get all spine information spine_url = base_url + '/class/fabricNode.json?query-target-filter=and(eq(fabricNode.role,"spine"))' spines = requests.get(spine_url, cookies=cookies, verify=False).json()['imdata'] print spines spine_dns = [] print spine_dns # for each of the objects returned by the API we will extract the dn for spine in spines: # this will return something like - # "topology/pod-1/node-101" - replaces curly braces {} in sensor_url dn = spine['fabricNode']['attributes']['dn'] spine_dns.append(dn) # now we will run a query for the sensor of each node by it's dn for dn in spine_dns: resp = requests.get(sensor_url.format(dn), cookies=cookies, verify=False).json()['imdata'][0] temp = resp['eqptTemp5min']['attributes']['currentMax'] ts = resp['eqptTemp5min']['attributes']['repIntvEnd'] # clean up dn - only save node name dn = dn.replace('topology/pod-1/', '') # temperature returned as string - # make it an int and assign condition int_temp = int(temp) if int_temp >= 60: # temp is greater than 60 degrees C condition = "high" elif int_temp >= 48: # temp between 48 and 60 degrees C condition = "elevated" else: # temp less than 48 degrees C condition = "normal" # take day/time returned and place into two separate variables date = ts.split('T')[0] # date occurs before 'T' in ts string time = ts.split('T')[1] # time occurs after 'T' in ts string time = time.split('.')[ 0] # remove microseconds from time occuring after period obj = { "dn": dn, "attributes": { "temp": temp, "timestamp": time, "date": date, "condition": condition, "type": 'spine' } } # send object to RESTAPI function upload = send2_RESTAPI(obj) if upload: print "device successfully uploaded to api (S)" else: print "error uploading device "
def run(): config = ConfigParser.ConfigParser() config.read("config.cfg") password = config.get("credentials", "password") username = config.get("credentials", "username") database = config.get("credentials", "database") logger.info("Constructing InfluxDB queries.") if args.average == "1h": retention = "10m" delta = "2h" time_units = 6 elif args.average == "1d": retention = "1h" delta = "2d" time_units = 24 else: return 0 client = InfluxDBClient( "dbod-eschanet.cern.ch", 8080, username, password, "monit_jobs", True, False ) rs_distinct_sets = client.query( """select * from "{}"."jobs" where "prod_source" != '' group by panda_queue, prod_source, resource, job_status limit 1""".format( retention ) ) rs_result = client.query( """select * from "{}"."jobs" where time > now() - {} and "prod_source" != '' group by panda_queue, prod_source, resource, job_status """.format( retention, delta ) ) raw_dict = rs_result.raw series = raw_dict["series"] logger.info("Got data from InfluxDB.") logger.info("Averaging now.") # uploader = InfluxDBClient('dbod-eschanet.cern.ch', 8080, username, password, "test", True, False) points_list = [] for rs in rs_distinct_sets.keys(): rs = rs[1] # rs is a tuple logger.debug(rs) filtered_points = [ p for p in series if p["tags"]["panda_queue"] == rs["panda_queue"] and p["tags"]["resource"] == rs["resource"] and p["tags"]["prod_source"] == rs["prod_source"] and p["tags"]["job_status"] == rs["job_status"] ] if len(filtered_points) == 0: logger.debug("Got no points for this set of keys.") continue filtered_points = filtered_points[0] values = filtered_points["values"] tags = filtered_points["tags"] columns = filtered_points["columns"] # reverse in place, want to have latest points first values.reverse() # get me the last (most recent) point, because this is the one I want to overwrite. latest_value = values[0] # get averaged values if tags["job_status"] in ["failed", "finished", "cancelled", "closed"]: averaged_jobs = get_sum(time_units, values, columns.index("jobs")) else: averaged_jobs = get_average(time_units, values, columns.index("jobs")) # averaged_jobs = get_average(time_units, values, columns.index('jobs')) averaged_cpu = get_average(time_units, values, columns.index("resource_factor")) averaged_corepower = get_average(time_units, values, columns.index("corepower")) averaged_HS06_benchmark = get_average( time_units, values, columns.index("HS06_benchmark") ) averaged_HS06_pledge = get_average( time_units, values, columns.index("federation_HS06_pledge") ) # construct rest of the data dict data = dict(zip(columns, latest_value)) time = data["time"].replace("T", " ").replace("Z", "") if args.average == "1h": hash = time.split(".")[-1].ljust(9, "0") else: # got no hashes in 1h aggregate data yet m = hashlib.md5() m.update( str(tags["panda_queue"]) + str(tags["prod_source"]) + str(tags["resource"]) + str(tags["job_status"]) ) hash = str(int(m.hexdigest(), 16))[0:9] time = unix + int(hash) data.update(tags) data.pop("time", None) data.pop("jobs", None) data.pop("resource_factor", None) data.pop("corepower", None) data.pop("HS06_benchmark", None) data.pop("federation_HS06_pledge", None) json_body = { "measurement": "jobs", "tags": data, "time": time, "fields": { "jobs": averaged_jobs, "resource_factor": averaged_cpu, "corepower": averaged_corepower, "HS06_benchmark": averaged_HS06_benchmark, "federation_HS06_pledge": averaged_HS06_pledge, }, } # sometimes I f**k up and then I want to kill the last measurement... if args.kill_last: for key, value in json_body["fields"].iteritems(): json_body["fields"][key] = 0.0 logger.debug(json_body) points_list.append(json_body) client.write_points( points=points_list, time_precision="n", retention_policy=args.average )
WebDriverWait(driver, 5) name = 'Vitm' count = 1 input('Enter anything after scanning QR code') user = driver.find_element_by_xpath('//span[@title = "{}"]'.format(name)) with open(r'PATH\Train.txt', 'r') as h: sub = h.readlines() re_pattern = r'[0-9]{2}:[0-9]{2}:[0-9]{2},[0-9]{3} -->' regex = re.compile(re_pattern) # Get start times start_times = list(filter(regex.search, sub)) start_times = [time.split(' ')[0] for time in start_times] # Get lines lines = [[]] for sentence in sub: if re.match(re_pattern, sentence): lines[-1].pop() lines.append([]) else: lines[-1].append(sentence) lines = lines[1:] # print(lines) # df = pd.DataFrame(lines) df = pd.DataFrame() for i in lines: print(i)
def get_time(self, match): time = match.select_one(".table__time-text") time = time.text return int(time.split(':')[0]) # mm:ss -> mm
def onMessage(self, Connection, Data): Status = int(Data["Status"]) if (Status == 200): strData = Data["Data"].decode("utf-8", "ignore") response = json.loads(strData) Domoticz.Debug("JSON REPLY: " + str(response)) if (self.melcloud_state == "LOGIN"): if (response["ErrorId"] == None): Domoticz.Log("MELCloud login successfull") self.melcloud_key = response["LoginData"]["ContextKey"] self.melcloud_units_init() elif (response["ErrorId"] == 1): Domoticz.Log( "MELCloud login fail: check login and password") self.melcloud_state = "LOGIN_FAILED" else: Domoticz.Log("MELCloud failed with unknown error " + str(response["ErrorId"])) self.melcloud_state = "LOGIN_FAILED" elif (self.melcloud_state == "UNITS_INIT"): idoffset = 0 Domoticz.Log("Find " + str(len(response)) + " buildings") for building in response: Domoticz.Log("Find " + str(len(building["Structure"]["Areas"])) + " areas in building " + building["Name"]) Domoticz.Log("Find " + str(len(building["Structure"]["Floors"])) + " floors in building " + building["Name"]) Domoticz.Log("Find " + str(len(building["Structure"]["Devices"])) + " devices in building " + building["Name"]) #Search in devices for device in building["Structure"]["Devices"]: self.melcloud_add_unit(device, idoffset) idoffset += len(self.list_switchs) #Search in areas for area in building["Structure"]["Areas"]: for device in area["Devices"]: self.melcloud_add_unit(device, idoffset) idoffset += len(self.list_switchs) #Search in floors for floor in building["Structure"]["Floors"]: for device in floor["Devices"]: self.melcloud_add_unit(device, idoffset) idoffset += len(self.list_switchs) for area in floor["Areas"]: for device in area["Devices"]: self.melcloud_add_unit(device, idoffset) idoffset += len(self.list_switchs) self.melcloud_create_units() elif (self.melcloud_state == "UNIT_INFO"): for unit in self.list_units: if (unit['id'] == response['DeviceID']): Domoticz.Log("Update unit {0} information.".format( unit['name'])) unit['power'] = response['Power'] unit['op_mode'] = response['OperationMode'] unit['room_temp'] = response['RoomTemperature'] unit['set_temp'] = response['SetTemperature'] unit['set_fan'] = response['SetFanSpeed'] unit['vaneH'] = response['VaneHorizontal'] unit['vaneV'] = response['VaneVertical'] unit['next_comm'] = False Domoticz.Debug("Heartbeat unit info: " + str(unit)) self.domoticz_sync_switchs(unit) elif (self.melcloud_state == "SET"): for unit in self.list_units: if (unit['id'] == response['DeviceID']): date, time = response['NextCommunication'].split("T") hours, minutes, sec = time.split(":") sign = Parameters["Mode1"][0] value = Parameters["Mode1"][1:] Domoticz.Debug("TIME OFFSSET :" + sign + value) if (sign == "-"): hours = int(hours) - int(value) if (hours < 0): hours = hours + 24 else: hours = int(hours) + int(value) if (hours > 24): hours = hours - 24 next_comm = date + " " + str( hours) + ":" + minutes + ":" + sec unit[ 'next_comm'] = "Update for last command at " + next_comm Domoticz.Log("Next update for command: " + next_comm) self.domoticz_sync_switchs(unit) else: Domoticz.Log("State not implemented:" + self.melcloud_state) else: Domoticz.Log("MELCloud receive unknonw message with error code " + Data["Status"])
def Hourly_stats_trends(dataArray, inputField): """ Create hourly stats and trends for device data across days """ #get length of retrieved data dataLength = len(dataArray) #initalize hourlyData hourlyData = [] #loop through data for the device for x in range(0, dataLength): #get deviceID, timeStamp and field name timeStamp = str(dataArray[x]["timeStamp"]) deviceId = str(dataArray[x]["deviceID"]) field = dataArray[x][inputField] #split time to get hour timeStampSplit = timeStamp.split("T") day = timeStampSplit[0] time = timeStampSplit[1] timeSplit = time.split(":") hour = timeSplit[0] #get length of hourlyData hourlyDataLength = len(hourlyData) hourlyDataExists = False #loop through hourlyData and check if entry exists for j in range(0, hourlyDataLength): if (deviceId == hourlyData[j]["deviceID"]) and (day == hourlyData[j]["date"]) and (hour == hourlyData[j]["hour"]): #if exists, then update hourly fields accordingly hourlyData[j]["sumField"] += field hourlyData[j]["countEntries"] += 1 if field > hourlyData[j]["maxField"]: hourlyData[j]["maxField"] = field if field < hourlyData[j]["minField"]: hourlyData[j]["minField"] = field hourlyData[j]["avgField"] = hourlyData[j]["sumField"] / hourlyData[j]["countEntries"] hourlyDataExists = True #if entry for hourly data does not exist, then create new one if hourlyDataExists == False: plotTimeStamp = day + "T" + hour + ":30:00.000Z" jsonData = {"deviceID": deviceId, "date": day, "hour": hour, "plotTimeStamp": plotTimeStamp, "maxField": field, "minField": field, "avgField": field, "sumField": field, "countEntries": 1, "maxSlopeLastHour": None, "minSlopeLastHour": None, "avgSlopeLastHour": None, "field": field} hourlyData.append(jsonData) #sort hourlyData per timeStamp sortedHourlyData = sorted(hourlyData, key=lambda k: k['plotTimeStamp']) sortedHourlyDataLength = len(sortedHourlyData) for k in range(1, sortedHourlyDataLength): sortedHourlyData[k]["maxSlopeLastHour"] = sortedHourlyData[k]["maxField"] - sortedHourlyData[k-1]["maxField"] sortedHourlyData[k]["minSlopeLastHour"] = sortedHourlyData[k]["minField"] - sortedHourlyData[k-1]["minField"] sortedHourlyData[k]["avgSlopeLastHour"] = sortedHourlyData[k]["avgField"] - sortedHourlyData[k-1]["avgField"] #return sortedHourlyData return sortedHourlyData
def process_csv_file(self, fh): ''' Iterate through lines of iterator to csv file and pull out data for loading into STOQS ''' ds = {} DA = BaseType() DA.attributes = {'units': 'ng ml-1 ' , 'long_name': 'Domoic Acid', 'standard_name': 'domoic_acid', 'type': 'float', 'description': 'Domoic acid' , 'origin': 'www.sccoos.org' } PD = BaseType() PD.attributes = {'units': 'cells l-1', 'long_name': 'Pseudo-nitzschia delicatissima group', 'standard_name': 'pseudo_nitzschia_delicatissima', 'name': 'pseudo_nitzschia_delicatissima' , 'type': 'float' , 'description': 'Pseudo-nitzschia delicatissima group (cells/L)' , 'origin': 'www.sccoos.org' } PA = BaseType() PA.attributes = {'units': 'cells l-1', 'long_name': 'Pseudo-nitzschia seriata group', 'standard_name': 'pseudo_nitzschia_seriata', 'name': 'pseudo_nitzschia_seriata' , 'type': 'float' , 'description': 'Pseudo-nitzschia seriata group (cells/L)' , 'origin': 'www.sccoos.org' } alexandrium = BaseType() alexandrium.attributes = {'units': 'cells l-1', 'long_name': 'Alexandrium', 'standard_name': 'alexandrium', 'name': 'alexandrium' , 'type': 'float' , 'description': 'Alexandrium spp. (cells/L)' , 'origin': 'www.sccoos.org' } phosphate = BaseType() phosphate.attributes = {'units': 'm-3 mol l-1', 'long_name': 'Phosphate', 'standard_name': 'phosphate_dissolved_in_seawater', 'name': 'Phosphate' , 'type': 'float' , 'description': 'Phosphate (uM)' , 'origin': 'www.sccoos.org' } ammonia = BaseType() ammonia.attributes = {'units': 'm-3 mol l-1', 'long_name': 'Ammonia', 'standard_name': 'ammonia_dissolved_in_seawater', 'name': 'ammonia_dissolved_in_sewater' , 'type': 'float' , 'description': 'Ammonia (uM)' , 'origin': 'www.sccoos.org' } silicate = BaseType() silicate.attributes = {'units': 'm-3 mol l-1', 'long_name': 'Silicate', 'standard_name': 'silicate_dissolved_in_seawater', 'name': 'silicate_dissolved_in_seawater' , 'type': 'float' , 'description': 'Silicate (uM)' , 'origin': 'www.sccoos.org' } chlorophyll = BaseType() chlorophyll.attributes = {'units': 'kg m-3', 'long_name': 'Chlorophyll', 'standard_name': 'mass_concentration_of_chlorophyll_in_sea_water', 'name': 'mass_concentration_of_chlorophyll_in_sea_water' , 'type': 'float' , 'description': 'Chlorophyll (kg/m3)' , 'origin': 'www.sccoos.org' } prorocentrum = BaseType() prorocentrum.attributes = {'units': 'cells l-1', 'long_name': 'Prorocentrum', 'standard_name': 'mass_concentration_of_prorocentrum_in_sea_water', 'name': 'mass_concentration_of_prorocentrum_in_sea_water' , 'type': 'float' , 'description': 'Prorocentrum spp. (cells/L)' , 'origin': 'www.sccoos.org' } self.ds = { 'Domoic Acid (ng/mL)': DA, 'Pseudo-nitzschia seriata group (cells/L)': PA, 'Pseudo-nitzschia delicatissima group (cells/L)': PD, 'Phosphate (uM)': phosphate, 'Silicate (uM)': silicate, 'Ammonia (uM)': ammonia, 'Chlorophyll (mg/m3)': chlorophyll, 'Chlorophyll 1 (mg/m3)': chlorophyll, 'Chlorophyll 2 (mg/m3)': chlorophyll , 'Alexandrium spp. (cells/L)': alexandrium } self.include_names = ['Pseudo-nitzschia seriata group (cells/L)', 'Pseudo-nitzschia delicatissima group (cells/L)', 'Domoic Acid (ng/mL)', 'Chlorophyll (mg/m3)', 'Chlorophyll 1 (mg/m3)', 'Chlorophyll 2 (mg/m3)', 'Prorocentrum spp. (cells/L)', 'Silicate (uM)', 'Ammonia (uM)', 'Nitrate (uM)', 'Phosphate (uM)', 'Alexandrium spp. (cells/L)'] self.initDB() for pn in self.include_names: self.parmCount[pn] = 0 reader = csv.reader(fh) for line in fh: # Skip all lines that don't begin with '"' nor ' ' then open that with csv.DictReader if not line.startswith('"') and not line.startswith(' '): titles = reader.next() reader = csv.DictReader(fh, titles) for r in reader: year = int(r['year']) month = int(r['month']) day = int(r['day']) time = r['time'] lat = float(r['latitude']) lon = float(r['longitude']) depth = float(r['depth (m)']) location = r['location'] hours = int(time.split(':')[0]) mins = int(time.split(':')[1]) secs = int(time.split(':')[2]) parmNameValues = [] for name in self.ds.keys(): if name.startswith('Chlorophyll'): parmNameValues.append((name, 1e-5*float(r[name]))) else: parmNameValues.append((name, float(r[name]))) # Check to make sure all data from this file are from the same location. # The program could be modified to read data in one file from multiple locations by reading data into a hash keyed by location name # and then stepping through each key of the hash saving the data for each location into it's own activity. For now just require # each data file to have data from just one location. try: if lat != lastlat or lon != lastlon: logger.error("lat and lon are not the same for location = %s and lastlocation = %s. The input data should have just one location." % (location, lastlocation)) sys.exit(-1) except NameError, e: # Expected first time through when lastlon & lastlat don't yet exist pass # Load data dt = datetime(year, month, day, hours, mins, secs) self.load_measurement(lon, lat, depth, dt, parmNameValues) # Load sample bName = dt.isoformat() self.load_sample(lon, lat, depth, dt, bName) lastlat = lat lastlon = lon lastlocation = location
def srt_time_to_seconds(time): split_time = time.split(',') major, minor = (split_time[0].split(':'), split_time[1]) return int(major[0]) * 3600 + int(major[1]) * 60 + int(major[2]) + float(minor) / 1000
def convTimezone(time): hour = int(time.split('T')[1].split(':')[0]) new_hr = (hour + 5)%24 return (f'{new_hr:02}')
def open_table(url): xml = "" z1 = m1 table = url.split("/")[-3] key = url.split("/")[-2] tag = url.split("/")[-1] at = Airtable(key, table, api_key='keyikW1exArRfNAWj') match = at.search('category', tag ,view='Grid view') for field in match: try: res = field['fields'] name = res['name'] name = remove_non_ascii(name) thumbnail = res['thumbnail'] fanart = res['fanart'] link1 = res['link1'] link2 = res['link2'] link3 = res['link3'] link4 = res['link4'] link5 = res['link5'] link6 = res['link6'] time = res['Time'] if time == "-": time = "" dsp = name else: if "Final Score" in time: time2 = time dec = "" else: time2 = time.split("@")[-1] dec = time.split("@")[0] (display_time) = convDateUtil(time2, 'default', 'US/Eastern') dsp = ("[B][COLORdodgerblue]%s %s[/COLOR][/B]" % (dec,display_time)) + " " + name if link2 == "-": xml += "<item>"\ "<title>%s</title>"\ "<thumbnail>%s</thumbnail>"\ "<fanart>%s</fanart>"\ "<link>"\ "<sublink>%s</sublink>"\ "</link>"\ "</item>" % (dsp,thumbnail,fanart,link1) elif link3 == "-": xml += "<item>"\ "<title>%s</title>"\ "<thumbnail>%s</thumbnail>"\ "<fanart>%s</fanart>"\ "<link>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "</link>"\ "</item>" % (dsp,thumbnail,fanart,link1,link2) elif link4 == "-": xml += "<item>"\ "<title>%s</title>"\ "<thumbnail>%s</thumbnail>"\ "<fanart>%s</fanart>"\ "<link>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "</link>"\ "</item>" % (dsp,thumbnail,fanart,link1,link2,link3) elif link5 == "-": xml += "<item>"\ "<title>%s</title>"\ "<thumbnail>%s</thumbnail>"\ "<fanart>%s</fanart>"\ "<link>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "</link>"\ "</item>" % (dsp,thumbnail,fanart,link1,link2,link3,link4) elif link6 == "-": xml += "<item>"\ "<title>%s</title>"\ "<thumbnail>%s</thumbnail>"\ "<fanart>%s</fanart>"\ "<link>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "</link>"\ "</item>" % (dsp,thumbnail,fanart,link1,link2,link3,link4,link5) else: xml += "<item>"\ "<title>%s</title>"\ "<thumbnail>%s</thumbnail>"\ "<fanart>%s</fanart>"\ "<link>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "<sublink>%s</sublink>"\ "</link>"\ "</item>" % (dsp,thumbnail,fanart,link1,link2,link3,link4,link5,link6) except: pass jenlist = JenList(xml) display_list(jenlist.get_list(), jenlist.get_content_type())
req = urllib.request.Request(url) req.add_header("Authorization", "Basic %s" % base64string) req.add_header("Referer", "{}://{}/userRpm/SystemLogRpm.htm".format(protocol,ip)) f = urllib.request.urlopen(req) content = f.read() decoded_content = content.decode('utf-8', errors="ignore") time_diff = 0 now_time = datetime.now() current_year = str(now_time.year) print('"Timestamp","Event Created","Module","Severity","Message"') for line in decoded_content.splitlines(): line = line.strip() if len(line) == 0: continue if line.startswith("#"): if "Time = " in line: time = line.split("=") time = time[1].strip() time = time.split(" ") dt = datetime.strptime(time[0] + " " + time[2], "%Y-%m-%d %H:%M:%S") current_year = time[0].split("-")[0] time_diff = (dt - now_time) // timedelta(seconds=1) continue columns = list(map(lambda x: x.strip(), line.split("\t"))) dt = datetime.strptime(columns[0] + " " + current_year, "%b %d %H:%M:%S %Y") dt = dt + timedelta(seconds=time_diff) integer_timestamp = (dt - epoch) // timedelta(seconds=1) print('"' + str(integer_timestamp) + '","' + columns[0] + '","' + columns[1] + '","' + columns[2] + '","' + columns[3] + '"')
def generate_list_dismax_part_and_filter_time_from_info(info): # info is a dict with keys of obj, loc, period, time, timeofday # Time will be in filter --> No score field_clip = 'description_clip' field_des = 'description' delta_hour = 0.5 # For obj and loc --> query as normal obj_sentence = ', '.join(info['obj']) loc_sentence = ', '.join(info['loc']) queries_part = [] basic_obj_part = create_json_query_string_part(query=obj_sentence, field=field_clip, boost=3) loc_as_obj_part = create_json_query_string_part(query=loc_sentence, field=field_clip, boost=3) loc_as_obj_part_des = create_json_query_string_part(query=loc_sentence, field=field_des, boost=5) queries_part = queries_part + [ basic_obj_part ] if basic_obj_part is not None else queries_part queries_part = queries_part + [ loc_as_obj_part ] if loc_as_obj_part is not None else queries_part queries_part = queries_part + [ loc_as_obj_part_des ] if loc_as_obj_part_des is not None else queries_part having_location, location_query = create_location_query(info['loc'], field="address", boost=10) queries_part = queries_part + [location_query ] if having_location else queries_part # Expand object, location here # For time and timeofday --> filter is better filters_part = [] if len(info['time']) > 0: for time in info['time']: if time in [ 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday' ]: time_json = [{"match": {"weekday": time}}] else: time_convert = convert_text_to_date(time) if len(time_convert ) == 10: # full day, and month, year (be added) day = int(time_convert[0:2]) month = int(time_convert[3:5]) else: # only month day = -1 month = int(time_convert) time_json = [{"term": {"month": month}}] if day >= 0: time_json += [{"term": {"day": day}}] filters_part += time_json if len(info['timeofday']) > 0: time_json = [] for time in info['timeofday']: if ';' in time: temp = time.split('; ') oclock = parse(temp[1]) oclock = oclock.hour if temp[0] in ['after']: time_json += [{ "range": { "hour": { "gte": oclock - delta_hour } } }] elif temp[0] in ['to', 'til', 'before']: time_json += [{ "range": { "hour": { "lte": oclock + delta_hour } } }] elif temp[0] in ['at', 'around']: time_json += [{ "range": { "hour": { "gte": oclock - delta_hour, "lte": oclock + delta_hour } } }] else: oclock = parse(time) oclock = oclock.hour time_json += [{"range": {"hour": {"gte": oclock - 1}}}] filters_part += time_json return queries_part, filters_part
def __init__(self, type, time, team): self.type = type self.time = time self.time_in_sec = int(time.split(":")[0]) * 60 + int( time.split(":")[1]) self.team = team
def main(): import matplotlib.pyplot as plt global nodes global cores # Parse command line arguments args = parse_args() global nrun nrun = args.nrun global attribute attribute = 'Time' TUNER_NAME = args.optimization (machine, processor, nodes, cores) = GetMachineConfiguration() print("machine: " + machine + " processor: " + processor + " num_nodes: " + str(nodes) + " num_cores: " + str(cores)) os.environ['MACHINE_NAME'] = machine os.environ['TUNER_NAME'] = TUNER_NAME t = Categoricalnorm(["Time"], transform="onehot", name="t") #t = Real(0., 10., transform="normalize", name="t") k = Integer(5, 105, transform="normalize", name="k") l = Real(0., 1., transform="normalize", name="l") o = Real(float('-Inf'), float('Inf'), name="o") IS = Space([t]) PS = Space([k, l]) OS = Space([o]) constraints = {} problem = TuningProblem(IS, PS, OS, objectives, constraints, None) computer = Computer(nodes=nodes, cores=cores, hosts=None) options = Options() options['model_restarts'] = 1 options['distributed_memory_parallelism'] = False options['shared_memory_parallelism'] = False options['objective_evaluation_parallelism'] = False options['objective_multisample_threads'] = 1 options['objective_multisample_processes'] = 1 options['objective_nprocmax'] = 1 options['model_processes'] = 1 options['model_class'] = 'Model_LCM' #options['model_class'] = 'Model_GPy_LCM' options['verbose'] = False options.validate(computer=computer) giventask = [[attribute]] global X_train global Y_train global X_test global Y_test X_train = [] Y_train = [] X_test = [] Y_test = [] if not os.path.exists("gptune-search-lcm.db"): os.system("mkdir -p gptune-search-lcm.db") with open( "gptune-search-lcm.db/household." + str(nrun) + "." + attribute + ".log", "w") as f_out: f_out.write( "NKnots,Lambda,RegressionTime,InTestTime,InMSE,InR2,InAR2,OutTestTime,OutMSE,OutR2,OutAR2\n" ) with open("household/household_power_consumption.txt", "r") as f_in: f_in.readline() datalines = f_in.readlines() traindata_arr = datalines[0:1000000] testdata_arr = datalines[1000000:1100000] wrong_data_cnt = 0 import time from datetime import datetime start_dt = datetime(2006, 12, 16, 17, 24, 00) start_time_val = int(round(start_dt.timestamp())) for traindata in traindata_arr: data = traindata.split(";") date = data[0] time = data[1] date_split = date.split("/") time_split = time.split(":") try: dt = datetime(int(date_split[2]), int(date_split[1]), int(date_split[0]), int(time_split[0]), int(time_split[1]), int(time_split[2])) time_val = (int(round(dt.timestamp())) - start_time_val) / 60 sub_metering_3 = float(data[-1]) X_train.append(time_val) Y_train.append(sub_metering_3 + 0.00001) except: wrong_data_cnt += 1 print("wrong_data_cnt (train): ", wrong_data_cnt) wrong_data_cnt = 0 for testdata in testdata_arr: data = traindata.split(";") date = data[0] time = data[1] date_split = date.split("/") time_split = time.split(":") try: dt = datetime(int(date_split[2]), int(date_split[1]), int(date_split[0]), int(time_split[0]), int(time_split[1]), int(time_split[2])) time_val = (int(round(dt.timestamp())) - start_time_val) / 60 sub_metering_3 = float(data[-1]) X_test.append(time_val) Y_test.append(sub_metering_3 + 0.00001) except: wrong_data_cnt += 1 print("wrong_data_cnt (test): ", wrong_data_cnt) NI = len(giventask) NS = nrun TUNER_NAME = os.environ['TUNER_NAME'] if (TUNER_NAME == 'GPTune'): data = Data(problem) gt = GPTune(problem, computer=computer, data=data, options=options, driverabspath=os.path.abspath(__file__)) (data, modeler, stats) = gt.MLA(NS=NS, Igiven=giventask, NI=NI, NS1=int(NS / 2)) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(" t:%s " % (data.I[tid][0])) print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) if (TUNER_NAME == 'opentuner'): (data, stats) = OpenTuner(T=giventask, NS=NS, tp=problem, computer=computer, run_id="OpenTuner", niter=1, technique=None) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(" t:%s " % (data.I[tid][0])) print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid])) if (TUNER_NAME == 'hpbandster'): (data, stats) = HpBandSter(T=giventask, NS=NS, tp=problem, computer=computer, run_id="HpBandSter", niter=1) print("stats: ", stats) """ Print all input and parameter samples """ for tid in range(NI): print("tid: %d" % (tid)) print(" t:%s " % (data.I[tid][0])) print(" Ps ", data.P[tid]) print(" Os ", data.O[tid].tolist()) print(' Popt ', data.P[tid][np.argmin(data.O[tid])], 'Oopt ', min(data.O[tid])[0], 'nth ', np.argmin(data.O[tid]))
history = lstm.fit(x=train_gen, epochs=1000, validation_data=val_gen, verbose=1, shuffle=False, callbacks=[es]) plt.plot(history.history['auc'], label='train') plt.plot(history.history['val_auc'], label='val') plt.legend() plt.show() time_end = time.time() time = time_end - time_start time = str(datetime.timedelta(seconds=time)) time = time.split('.')[0] print('Time: ', time) # Testing subjects = range(1, 13) t = pd.DataFrame(index=subjects, columns=['AUROC']) t.index.rename('Subject', inplace=True) # testing for subject in subjects: print('Testing subject %d' % subject) # load testing data x_test = load_subject_data(subject, 'data', 7, 8) y_test = load_subject_data(subject, 'events', 7, 8)
def hour_minute(time): hour, minute = time.split(':') return int(hour.strip()), int(minute.strip())
def getduration(time): (hms, ms) = time.split('.') (h, m, s) = hms.split(':') totalms = int(ms) + (int(s) * 100) + (int(m) * 100 * 60) + (int(h) * 100 * 60 * 60) return totalms
def get_comment_week(self, _str): time = _str[0:10] times = time.split('-'); return (times[0] + '-' + str(datetime.date(int(times[0]), int(times[1]), int(times[2])).isocalendar()[1]).zfill(2))
def set_volume(self, time, present): second = time.split(':')[-1] if not self.data or (self.data[-1][0] != second and self.data[-1][1] == present): self.data.append([second, present])
def sigmaTime(time): hour, minute = time.split(":") return (int(hour)*60 + int(minute))
def _run(self): if self.dry_run: print("Dry run, so not running.") return if "slurm" in self.kind: # Have to jump through a hoop to get the proper node-local storage on cedar/graham. self.local_scratch_prefix = self.get_slurm_var("SLURM_TMPDIR") self.local_scratch = os.path.join( self.local_scratch_prefix, os.path.basename(self.job_path)) # Compute new time limits based on the actual time remaining (protect against e.g. job starting late) print("Time limits before adjustment:") self.print_time_limits() job_id = os.getenv("SLURM_JOBID") command = 'squeue -h -j {} -o "%L"'.format(job_id) returncode, stdout, stderr = self.execute_command(command, frmt=False, robust=False) days = 0 if "-" in stdout: days, time = stdout.split("-") days = int(days) else: time = stdout time = time.split(":") hours = int(time[-3]) if len(time) > 2 else 0 minutes = int(time[-2]) if len(time) > 1 else 0 seconds = int(time[-1]) wall_time_delta = datetime.timedelta(days=days, hours=hours, minutes=minutes, seconds=seconds) wall_time_seconds = int(wall_time_delta.total_seconds()) print("Actual remaining walltime: {}".format(wall_time_delta)) print("Time limits after adjustment:") (self.wall_time_seconds, self.total_seconds_per_step, self.parallel_seconds_per_step, self.python_seconds_per_step) = \ self.compute_time_limits( wall_time_seconds, self.cleanup_time, self.slack_time, self.step_time_limit, self.n_steps) self.print_time_limits() with cd(self.job_path): print("\n" + ("=" * 80)) job_start = datetime.datetime.now() print("Starting job at {}".format(job_start)) job = ReadOnlyJob(self.input_zip) subjobs_remaining = sorted([op.idx for op in job.ready_incomplete_ops(sort=False)]) n_failures = defaultdict(int) dead_jobs = set() i = 0 while subjobs_remaining: step_start = datetime.datetime.now() print("\nStarting step {} at: ".format(i) + "=" * 90) print("{} ({} since start of job)".format(step_start, step_start - job_start)) if not self.host_pool: if self.kind == "pbs": with open(os.path.expandvars("$PBS_NODEFILE"), 'r') as f: self.host_pool = list(set([s.strip() for s in iter(f.readline, '')])) print(self.host_pool) elif "slurm" in self.kind: p = subprocess.run( 'scontrol show hostnames $SLURM_JOB_NODELIST', stdout=subprocess.PIPE, shell=True) self.host_pool = list(set([host.strip() for host in p.stdout.decode().split('\n') if host])) else: raise Exception("NotImplemented") self.hosts, self.n_procs = self.recruit_hosts( self.hpc, self.min_hosts, self.max_hosts, self.host_pool, self.ppn, max_procs=len(subjobs_remaining)) indices_for_step = subjobs_remaining[:self.n_procs] self._step(i, indices_for_step) self._checkpoint(i) job = ReadOnlyJob(self.archive_root) subjobs_remaining = set([op.idx for op in job.ready_incomplete_ops(sort=False)]) for j in indices_for_step: if j in subjobs_remaining: n_failures[j] += 1 if n_failures[j] > self.n_retries: print("All {} attempts at completing job with index {} have failed, " "permanently removing it from set of eligible jobs.".format(n_failures[j], j)) dead_jobs.add(j) subjobs_remaining = [idx for idx in subjobs_remaining if idx not in dead_jobs] subjobs_remaining = sorted(subjobs_remaining) i += 1 print("Step duration: {}.".format(datetime.datetime.now() - step_start)) self.execute_command("rm -rf {archive_root}", robust=True) print("Cleaning up dirty hosts...") command = "rm -rf {local_scratch}" for host in self.dirty_hosts: print("Cleaning host {}...".format(host)) self.ssh_execute(command, host, robust=True)
def process_video(video, lineStart, lineEnd, v_or_h, contourLimit, merchantid, db, date, time): ret, frame1 = video.read() ret, frame2 = video.read() peopleID = 0 custID = 0 people = [] st = time.split(":") hour = int(st[0]) minutes = int(st[1]) s = int(st[2]) entry = 0 exited = 0 while video.isOpened(): check, _ = video.read() if check: milli = video.get(cv2.CAP_PROP_POS_MSEC) sec = milli / 1000 # print(round(sec)) seconds = s + sec if seconds == 60: seconds = 0 minutes += 1 if minutes == 60: minutes = 0 hour += 1 if hour == 24: hour = 0 currentVideoTime = str(hour) + ":" + str(minutes) + ":" + str( round(seconds)) diff = cv2.absdiff(frame1, frame2) gray = cv2.cvtColor(diff, cv2.COLOR_BGR2GRAY) blur = cv2.GaussianBlur(gray, (5, 5), 0) _, thresh = cv2.threshold(blur, 20, 255, cv2.THRESH_BINARY) dilated = cv2.dilate(thresh, None, iterations=3) contours, _ = cv2.findContours(dilated, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE) cv2.line(frame1, (lineStart[0], lineStart[1]), (lineEnd[0], lineEnd[1]), (255, 0, 255), 2) for contour in contours: (x, y, w, h) = cv2.boundingRect(contour) if cv2.contourArea(contour) < contourLimit: continue new = True for i in people: if abs(x - i.getX()) <= w and abs(y - i.getY()) <= h: new = False i.updateCoords(x, y) if v_or_h == 'v': if i.enteringV(lineEnd[1]) == True: entry += 1 custID += 1 insertPeopleData('enter', custID, merchantid, 0.0, date, currentVideoTime, db) if i.exitingV(lineStart[1]) == True: exited += 1 custID += 1 insertPeopleData('exit', custID, merchantid, 0.0, date, currentVideoTime, db) if v_or_h == 'h': if i.enteringH(lineEnd[0]) == True: entry += 1 custID insertPeopleData('enter', custID, merchantid, 0.0, date, currentVideoTime, db) if i.exitingH(lineStart[0]) == True: exited += 1 custID += 1 insertPeopleData('exit', custID, merchantid, 0.0, date, currentVideoTime, db) break if new == True: p = Customer(peopleID, x, y) people.append(p) peopleID += 1 ################################################## # used for finding camera specific parameters# ############################################## # cv2.rectangle(frame1, (x,y), (x + w, y + h), (0, 255, 0), 2) # # cv2.putText(frame1, "Entered: {}".format(entry), (10, 40), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) # cv2.putText(frame1, "Exited: {}".format(exited), (10, 60), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 0, 255), 1) #cv2.imshow('vid', frame1) ################################################## frame1 = frame2 ret, frame2 = video.read() else: break if cv2.waitKey(5) == ord('x'): break cv2.destroyAllWindows() video.release()
def get_date(self): time = self.bsobj.find_all('span', class_='time')[0].text time = time.split('日')[0] time = time.replace('年', '-') time = time.replace('月', '-') return time
def to_secs(date, time): toks = date.split('/') toks.extend(time.split(':')) toks[-1] = round(float(toks[-1])) return calendar.timegm([int(x) for x in toks])
def get_millisecond(time): item = time.split(":") sum = (int(item[0]) * 3600 + int(item[1]) * 60 + int(item[2])) * 1000 + int(item[3]) return sum
def real_time_get_timeline(user_id): allTweets = {} single_col = [] timeline = api.user_timeline(screen_name=user_id, count=100) realTweets = [] for status in timeline: json_form = status._json source_type = "not known" source = json_form["source"].encode('utf8') tweet = json_form["text"].encode('utf8') tweet_raw = json_form["text"].encode('utf8') tweet_arr = tweet.split(" ") interacted = "NULL" used_tag = "NULL" # user_id = json_form["id"].encode('utf8') time = json_form["created_at"].encode('utf8') time_arr = time.split(" ") month = "" if time_arr[1] == "Jan": month = "01" elif time_arr[1] == "Feb": month = "02" elif time_arr[1] == "Mar": month = "03" elif time_arr[1] == "Apr": month = "04" elif time_arr[1] == "May": month = "05" elif time_arr[1] == "Jun": month = "06" elif time_arr[1] == "Jul": month = "07" elif time_arr[1] == "Aug": month = "08" elif time_arr[1] == "Sep": month = "09" elif time_arr[1] == "Oct": month = "10" elif time_arr[1] == "Nov": month = "11" elif time_arr[1] == "Dec": month = "12" time = "%s-%s-%s %s" % (time_arr[5], month, time_arr[2], time_arr[3]) interacted = [] used_tag = [] pure_name = "" for text in tweet_arr: if "://" in text: tweet = tweet.replace(text, '') if "@" in text: pure_name = re.sub("[^A-Za-z]", '', text.strip()) if pure_name.lower() != user_id.lower(): interacted.append(pure_name) if "#" in text: used_tag.append(re.sub("[^A-Za-z]", '', text.strip())) tweet = re.sub("[^A-Za-z]", " ", tweet.strip()) tweet = ' '.join(tweet.split()) retweeted = "false" if tweet[:3] == "RT ": retweeted = "true" if "iphone" in source: source_type = "Iphone" elif "web" in source: source_type = "Web" elif "ipad" in source: source_type = "Ipad" elif "media" in source: soruce_type = "media studio" else: source_type = "Possibly android" if len(tweet) < 3: continue if len(interacted) == 0: interacted = "NULL" if len(used_tag) == 0: used_tag = "NULL" strCon = { "time": time, "text_raw": tweet_raw, "text": tweet, "fav_count": str(status.favorite_count), "source_type": source_type, "is_retweeted": retweeted, "interacted": interacted, "hashtag": used_tag } realTweets.append(strCon) return realTweets
def grasp_weather(year, month, day, city): url = WEATHER_CITY[city].format(year, month, day) html_content = requests.get(url).content b = BeautifulSoup(html_content, "html.parser") table = b.find("table", { "id": "obsTable" }).find_all("tr", {"class": "no-metars"}) item_count = 0 for i in table: a = [] for j in i.find_all("td"): a.append(j.text.replace("\r", "").replace("\n", "")) try: time = a[0] if "AM" in time: hour = int(time.split()[0].split(":")[0]) minute = int(time.split()[0].split(":")[1]) if hour == 12: time = "%02d:%02d:00" % (0, minute) else: time = "%02d:%02d:00" % (hour, minute) elif "PM" in time: hour = int(time.split()[0].split(":")[0]) minute = int(time.split()[0].split(":")[1]) if hour == 12: time = "%02d:%02d:00" % (hour, minute) else: time = "%02d:%02d:00" % (hour + 12, minute) except: time = "-" try: temp = float(a[1].split()[0]) except: temp = float(-999) try: dew_point = float(a[2].split()[0]) except: dew_point = float(-999) try: humidity = float(a[3][:-1]) / 100.0 except: humidity = float(-999) try: pressure = int(a[4].split()[0]) except: pressure = float(-999) try: visibility = float(a[5].split()[0]) except: visibility = float(-999) try: wind_dir = a[6] except: wind_dir = "-" try: wind_speed = float(a[7].split()[0]) except: wind_speed = float(-999) try: conditions = a[-1] except: conditions = "-" weather = Weather(city=city, date=str("%s-%s-%s" % (year, month, day)), time=time, temp=temp, dew_point=dew_point, humidity=humidity, pressure=pressure, visibility=visibility, wind_dir=wind_dir, wind_speed=wind_speed, conditions=conditions) weather.save() item_count += 1 return item_count