def _check_cookie(self, cookie):
""" Check our cookie, possibly returning the session id
"""
expiretime = self._find_chunk(cookie, 'expires=', ' GMT')
dict = {}
print 'sabnzbd-xbmc timecheck'
try:
# Day, dd-mmm-yyyy hh:mm:ss
t = time.strptime(expiretime, '%a, %d-%b-%Y %H:%M:%S')
except ValueError:
# Day, dd mmm yyyy hh:mm:ss
print 'sabnzbd-xbmc timecheck failed!'
t = time.strptime(expiretime, '%a, %d %b %Y %H:%M:%S')
print 'sabnzbd-xbmc checking if expired!'
now = time.gmtime()
# Woops, expired
if now > t:
print 'sabnzbd-xbmc expired!'
return {}
else:
print 'sabnzbd-xbmc extract session from cookie!'
dict['NzbSessionID'] = self._find_chunk(cookie, 'NzbSessionID=', ';')
dict['NzbSmoke'] = self._find_chunk(cookie, 'NzbSmoke=', ';')
print 'sabnzbd-xbmc return session info!'
return dict
def validate_data(info):
hack_license,pick_datetime,drop_datetime,n_passengers,trip_dist,pick_long,\
pick_lat,drop_long,drop_lat,payment_type,fare_amount,\
surcharge,tip_amount,mta_tax,tolls_amount,total_amount=info
time_in_seconds = time.mktime(time.strptime(drop_datetime,'%Y-%m-%d %H:%M:%S'))-\
time.mktime(time.strptime(pick_datetime,'%Y-%m-%d %H:%M:%S'))
try:
pick_long = float(pick_long.strip())
pick_lat = float(pick_lat.strip())
drop_long = float(drop_long.strip())
drop_lat = float(drop_lat.strip())
trip_dist = float(trip_dist.strip())
total_amount = float(total_amount.strip())
n_passengers = int(n_passengers.strip())
except ValueError:
sys.stderr.write('CASTING TO FLOATS FAILED')
return False
# Is the straight distance shorter than the reported distance?
euclidean = validate_euclidean(trip_dist,pick_long,pick_lat,drop_long,drop_lat)
gps_pickup = validate_gps(pick_long,pick_lat) # Are the GPS coordinates present in Manhattan
gps_dropoff = validate_gps(drop_long,drop_lat)
distance = validate_distance(trip_dist,pick_long,pick_lat,drop_long,drop_lat) # Are distances too big
val_time = validate_time(time_in_seconds) # Are times too long or 0? Are they positive?
velocity = validate_velocity(time_in_seconds,trip_dist) # Is velocity too out of reach
amount = validate_amount(total_amount)
pass_validate = validate_passengers(n_passengers)
return(euclidean and gps_pickup and gps_dropoff and distance and val_time and velocity and amount and pass_validate)
def time_comparison(project_pid):
end_time_format = ""
end_time = _time_call(project_pid,"etime")
end_time_dict = end_time.split(":")
if end_time_dict:
if len(end_time_dict) == 2:
ent_time_format = "%M:%S"
elif len(end_time_dict) == 3:
if len(end_time_dict[0].split("-")) == 2:
ent_time_format = "%U-%H:%M:%S"
else:
ent_time_format = "%H:%M:%S"
time_define = time.strptime("00:05:00","%H:%M:%S")
if end_time_format is not None:
if time_define > time.strptime(end_time,ent_time_format):
print_highlight(package_name),
print "running time %s" %(end_time),
print_success()
else:
print_highlight(package_name),
print "running time %s" %(end_time),
print_failure()
def codyFileUpdates(self, trigger):
if '!cody.updates' in trigger\
and self.POST_CODY_FILE_UPDATES:
self.POST_CODY_FILE_UPDATES = False
self.IRC.send('PRIVMSG '+self.MSG_CHANNEL+' :File updates are now turned off.\r\n')
elif '!cody.updates' in trigger\
and not self.POST_CODY_FILE_UPDATES:
self.POST_CODY_FILE_UPDATES = True
self.IRC.send('PRIVMSG '+self.MSG_CHANNEL+' :File updates are now turned on.\r\n')
elif 'codyFileUpdates' in trigger:
if (datetime.datetime.now().second % 3) == 0 :
#fetch current file update time - syntax is 'Sun Apr 21 20:24:36 2013'
lastUpdatedRealTime = time.ctime(os.path.getmtime(self.FILE_PATH))
lastUpdatedCody = lastUpdatedRealTime.split()
#fetch last recorded update time
lastRecorded = open("resources/lastupdated.db", "r")
lastRecordedCody = lastRecorded.read().split()
lastRecorded.close()
#parse recorded time into datetime ints
if len(lastRecordedCody) == 5:
lastRecordedYear = int(lastRecordedCody[4])
lastRecordedMonth = int(time.strptime(lastRecordedCody[1],'%b').tm_mon)
lastRecordedDay = int(lastRecordedCody[2])
lastRecordedTime = lastRecordedCody[3].split(':')
lastRecordedHour = int(lastRecordedTime[0])
lastRecordedMinute = int(lastRecordedTime[1])
lastRecordedSecond = int(lastRecordedTime[2])
#parse current time into datetime ints
lastUpdatedYear = int(lastUpdatedCody[4])
lastUpdatedMonth = int(time.strptime(lastUpdatedCody[1],'%b').tm_mon)
lastUpdatedDay = int(lastUpdatedCody[2])
lastUpdatedTime = lastUpdatedCody[3].split(':')
lastUpdatedHour = int(lastUpdatedTime[0])
lastUpdatedMinute = int(lastUpdatedTime[1])
lastUpdatedSecond = int(lastUpdatedTime[2])
#turn them both into datetime objects
if len(lastRecordedCody) == 5:
lastRecordedCody = datetime.datetime(lastRecordedYear, lastRecordedMonth, lastRecordedDay, lastRecordedHour, lastRecordedMinute, lastRecordedSecond)
else:
lastRecordedCody = datetime.datetime(2000,1,1)
lastUpdatedCody = datetime.datetime(lastUpdatedYear, lastUpdatedMonth, lastUpdatedDay, lastUpdatedHour, lastUpdatedMinute, lastUpdatedSecond)
#check if it has changed
if lastRecordedCody < lastUpdatedCody :
if self.POST_CODY_FILE_UPDATES:
self.IRC.send('PRIVMSG '+self.HOME_CHANNEL+' :'+self.FILE_NAME+' has been updated.\r\n')
dbWrite = open("resources/lastupdated.db", "w")
dbWrite.write(lastUpdatedRealTime)
dbWrite.close()
def getSeconds(time_str):
"""
Convert a string of the form '%d-%H:%M:%S', '%H:%M:%S' or '%M:%S'
to seconds.
"""
# sometimes the timestamp includs a fractional second part
time_str = time_str.split('.')[0]
if '-' in time_str:
days, time_str = time_str.split('-')
st = time.strptime(time_str, '%H:%M:%S')
sec = int(days)*86400+st.tm_hour*3600+st.tm_min*60+st.tm_sec
else:
try:
st = time.strptime(time_str, '%H:%M:%S')
sec = st.tm_hour*3600+st.tm_min*60+st.tm_sec
except ValueError:
try:
st = time.strptime(time_str, '%M:%S')
sec = st.tm_min*60+st.tm_sec
except ValueError:
logging.error('String: %s does not match time format.' % time_str)
return -1
return sec
def _createSearchRequest(self, search=None, tags=None,
notebooks=None, date=None,
exact_entry=None, content_search=None):
request = ""
if notebooks:
for notebook in tools.strip(notebooks.split(',')):
if notebook.startswith('-'):
request += '-notebook:"%s" ' % tools.strip(notebook[1:])
else:
request += 'notebook:"%s" ' % tools.strip(notebook)
if tags:
for tag in tools.strip(tags.split(',')):
if tag.startswith('-'):
request += '-tag:"%s" ' % tag[1:]
else:
request += 'tag:"%s" ' % tag
if date:
date = tools.strip(date.split('-'))
try:
dateStruct = time.strptime(date[0] + " 00:00:00", "%d.%m.%Y %H:%M:%S")
request += 'created:%s ' % time.strftime("%Y%m%d", time.localtime(time.mktime(dateStruct)))
if len(date) == 2:
dateStruct = time.strptime(date[1] + " 00:00:00", "%d.%m.%Y %H:%M:%S")
request += '-created:%s ' % time.strftime("%Y%m%d", time.localtime(time.mktime(dateStruct) + 60 * 60 * 24))
except ValueError, e:
out.failureMessage('Incorrect date format in --date attribute. '
'Format: %s' % time.strftime("%d.%m.%Y", time.strptime('19991231', "%Y%m%d")))
return tools.exitErr()
def add_separator(self, timestamp):
'''Add whitespace and timestamp between chat sessions.'''
time_with_current_year = \
(time.localtime(time.time())[0], ) + \
time.strptime(timestamp, '%b %d %H:%M:%S')[1:]
timestamp_seconds = time.mktime(time_with_current_year)
if timestamp_seconds > time.time():
time_with_previous_year = \
(time.localtime(time.time())[0] - 1, ) + \
time.strptime(timestamp, '%b %d %H:%M:%S')[1:]
timestamp_seconds = time.mktime(time_with_previous_year)
message = TextBox(self,
style.COLOR_BUTTON_GREY, style.COLOR_BUTTON_GREY,
style.COLOR_WHITE, style.COLOR_BUTTON_GREY, False,
None, timestamp_to_elapsed_string(timestamp_seconds))
self._message_list.append(message)
box = Gtk.HBox()
align = Gtk.Alignment.new(
xalign=0.5, yalign=0.0, xscale=0.0, yscale=0.0)
box.pack_start(align, True, True, 0)
align.show()
align.add(message)
message.show()
self._conversation.attach(box, 0, self._row_counter, 1, 1)
box.show()
self._row_counter += 1
self.add_log_timestamp(timestamp)
self._last_msg_sender = None
def do_offset(tuples_list, filename, format ='%b %d, %Y %H:%M:%S', offset_val=0):
new_tuples_list = []
firstval = time.strptime(tuples_list[0][0], format)
if filename != "slide_timestamps.txt":
def_time = 'Apr 01, 2000 00:00:00'
else :
def_time = 'Apr 01 2000 00:00:00'
conversion_timer = time.mktime(time.strptime(def_time, format))
for item in tuples_list:
t= item[0]
timer = time.strptime(t, format) ##3,4,5
timer = time.mktime(timer) - time.mktime(firstval) + conversion_timer + offset_val
timer = time.strftime("%H:%M:%S",time.localtime(timer))
if filename == "spectrum.txt":
line_list = [timer]
for i in json.loads(item[1]):
line_list.append(i)
#print line_list
new_tuples_list.append(tuple(line_list))
else:
line_list = [timer]
for i in item[1:]:
line_list.append(i)
#print line_list
new_tuples_list.append(tuple(line_list))
return new_tuples_list
def get_expire_sec(self, morsel):
expires = None
if morsel.get('max-age'):
return int(morsel['max-age'])
expires = morsel.get('expires')
if not expires:
return None
expires = expires.replace(' UTC', ' GMT')
try:
expires = time.strptime(expires, '%a, %d-%b-%Y %H:%M:%S GMT')
except:
pass
try:
expires = time.strptime(expires, '%a, %d %b %Y %H:%M:%S GMT')
except:
pass
expires = time.mktime(expires)
expires = expires - time.timezone - time.time()
return expires
def time_parse(s):
try:
epoch = int(s)
return epoch
except ValueError:
pass
try:
epoch = int(calendar.timegm(time.strptime(s, '%Y-%m-%d')))
return epoch
except ValueError:
pass
try:
epoch = int(calendar.timegm(time.strptime(s, '%Y-%m-%d %H:%M:%S')))
return epoch
except ValueError:
pass
m = re.match(r'^(?=\d)(?:(\d+)w)?(?:(\d+)d)?(?:(\d+)h)?(?:(\d+)m)?(?:(\d+)s?)?$', s, re.I)
if m:
return -1*(int(m.group(1) or 0)*604800 + \
int(m.group(2) or 0)*86400+ \
int(m.group(3) or 0)*3600+ \
int(m.group(4) or 0)*60+ \
int(m.group(5) or 0))
raise ValueError('Invalid time: "%s"' % s)
def set_filter_date(self):
dialog = xbmcgui.Dialog()
if self.start_date == '':
self.start_date = str(datetime.datetime.now())[:10]
if self.end_date == '':
self.end_date = str(datetime.datetime.now())[:10]
try:
d = dialog.numeric(1, common.getstring(30117) ,strftime("%d/%m/%Y",strptime(self.start_date,"%Y-%m-%d")) )
if d != '':
self.start_date = strftime("%Y-%m-%d",strptime(d.replace(" ","0"),"%d/%m/%Y"))
else:
self.start_date =''
common.log('', str(self.start_date))
d = dialog.numeric(1, common.getstring(30118) ,strftime("%d/%m/%Y",strptime(self.end_date,"%Y-%m-%d")) )
if d != '':
self.end_date = strftime("%Y-%m-%d",strptime(d.replace(" ","0"),"%d/%m/%Y"))
else:
self.end_date =''
common.log('', str(self.end_date))
except:
pass
if self.start_date != '' or self.end_date != '':
self.getControl( BUTTON_DATE ).setLabel( self.start_date + ' ... ' + self.end_date )
else:
self.getControl( BUTTON_DATE ).setLabel( common.getstring(30164) )
self.getControl( BUTTON_DATE ).setVisible(False)
self.getControl( BUTTON_DATE ).setVisible(True)
def to_python(self, value):
if value is None:
return value
if isinstance(value, datetime.datetime):
return value
if isinstance(value, datetime.date):
return datetime.datetime(value.year, value.month, value.day)
# Attempt to parse a datetime:
value = smart_str(value)
# split usecs, because they are not recognized by strptime.
if '.' in value:
try:
value, usecs = value.split('.')
usecs = int(usecs)
except ValueError:
raise exceptions.ValidationError(self.error_messages['invalid'])
else:
usecs = 0
kwargs = {'microsecond': usecs}
try: # Seconds are optional, so try converting seconds first.
return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M:%S')[:6],
**kwargs)
except ValueError:
try: # Try without seconds.
return datetime.datetime(*time.strptime(value, '%Y-%m-%d %H:%M')[:5],
**kwargs)
except ValueError: # Try without hour/minutes/seconds.
try:
return datetime.datetime(*time.strptime(value, '%Y-%m-%d')[:3],
**kwargs)
except ValueError:
raise exceptions.ValidationError(self.error_messages['invalid'])
def getWeatherDataCallback(self, result, errortext):
self['statustext'].text = ''
if result == MSNWeather.ERROR:
self.error(errortext)
else:
self['caption'].text = self.weatherData.city
self.webSite = self.weatherData.url
for weatherData in self.weatherData.weatherItems.items():
item = weatherData[1]
if weatherData[0] == '-1':
self['currentTemp'].text = '%s\xc2\xb0%s' % (item.temperature, self.weatherData.degreetype)
self['condition'].text = item.skytext
self['humidity'].text = _('Humidity: %s %%') % item.humidity
self['wind_condition'].text = item.winddisplay
c = time.strptime(item.observationtime, '%H:%M:%S')
self['observationtime'].text = _('Observation time: %s') % time.strftime('%H:%M', c)
self['observationpoint'].text = _('Observation point: %s') % item.observationpoint
self['feelsliketemp'].text = _('Feels like %s') % item.feelslike + '\xc2\xb0' + self.weatherData.degreetype
else:
index = weatherData[0]
c = time.strptime(item.date, '%Y-%m-%d')
self['weekday%s' % index].text = '%s\n%s' % (item.day, time.strftime('%d. %b', c))
lowTemp = item.low
highTemp = item.high
self['weekday%s_temp' % index].text = '%s\xc2\xb0%s|%s\xc2\xb0%s\n%s' % (highTemp,
self.weatherData.degreetype,
lowTemp,
self.weatherData.degreetype,
item.skytextday)
if self.weatherPluginEntryIndex == 1 and WeatherMSNComp is not None:
WeatherMSNComp.updateWeather(self.weatherData, result, errortext)
return
def date_list(h5file):
dateList = []
tbase = []
ifgramList = h5file['interferograms'].keys()
for ifgram in ifgramList:
dates = h5file['interferograms'][ifgram].attrs['DATE12'].split('-')
dates1= h5file['interferograms'][ifgram].attrs['DATE12'].split('-')
if dates[0][0] == '9':
dates[0] = '19'+dates[0]
else:
dates[0] = '20'+dates[0]
if dates[1][0] == '9':
dates[1] = '19'+dates[1]
else:
dates[1] = '20'+dates[1]
if not dates[0] in dateList: dateList.append(dates[0])
if not dates[1] in dateList: dateList.append(dates[1])
dateList.sort()
dateList1=[]
for ni in range(len(dateList)):
dateList1.append(dateList[ni][2:])
d1 = datetime.datetime(*time.strptime(dateList[0],"%Y%m%d")[0:5])
for ni in range(len(dateList)):
d2 = datetime.datetime(*time.strptime(dateList[ni],"%Y%m%d")[0:5])
diff = d2-d1
tbase.append(diff.days)
dateDict = {}
for i in range(len(dateList)): dateDict[dateList[i]] = tbase[i]
return tbase,dateList,dateDict,dateList1
def to_python(self, value):
if value is None:
return None
if isinstance(value, datetime.time):
return value
if isinstance(value, datetime.datetime):
# Not usually a good idea to pass in a datetime here (it loses
# information), but this can be a side-effect of interacting with a
# database backend (e.g. Oracle), so we'll be accommodating.
return value.time()
# Attempt to parse a datetime:
value = smart_str(value)
# split usecs, because they are not recognized by strptime.
if '.' in value:
try:
value, usecs = value.split('.')
usecs = int(usecs)
except ValueError:
raise exceptions.ValidationError(self.error_messages['invalid'])
else:
usecs = 0
kwargs = {'microsecond': usecs}
try: # Seconds are optional, so try converting seconds first.
return datetime.time(*time.strptime(value, '%H:%M:%S')[3:6],
**kwargs)
except ValueError:
try: # Try without seconds.
return datetime.time(*time.strptime(value, '%H:%M')[3:5],
**kwargs)
except ValueError:
raise exceptions.ValidationError(self.error_messages['invalid'])
def _get_utilization(self, cr, uid, ids, name, arg, context=None):
res = {}
for meter in self.browse(cr, uid, ids, context=context):
Dn = 1.0*calendar.timegm(time.strptime(time.strftime('%Y-%m-%d',time.gmtime()),"%Y-%m-%d"))
Da = Dn - 3600*24*meter.av_time
meter_line_obj = self.pool.get('mro.pm.meter.line')
meter_line_ids = meter_line_obj.search(cr, uid, [('meter_id', '=', meter.id),('date', '<=', time.strftime('%Y-%m-%d',time.gmtime(Da)))], limit=1, order='date desc')
if not len(meter_line_ids):
meter_line_ids = meter_line_obj.search(cr, uid, [('meter_id', '=', meter.id),('date', '>', time.strftime('%Y-%m-%d',time.gmtime(Da)))], limit=1, order='date')
if not len(meter_line_ids):
res[meter.id] = meter.min_utilization
continue
meter_line = meter_line_obj.browse(cr, uid, meter_line_ids[0])
Dci = 1.0*calendar.timegm(time.strptime(meter_line.date, "%Y-%m-%d"))
Ci = meter_line.total_value
number = 0
Us = 0
meter_line_ids = meter_line_obj.search(cr, uid, [('meter_id', '=', meter.id),('date', '>',meter_line.date)], order='date')
for meter_line in meter_line_obj.browse(cr, uid, meter_line_ids):
Dci1 = 1.0*calendar.timegm(time.strptime(meter_line.date, "%Y-%m-%d"))
Ci1 = meter_line.total_value
if Dci1 != Dci:
Us = Us + (3600*24*(Ci1 - Ci))/(Dci1 - Dci)
Dci = Dci1
Ci = Ci1
number += 1
if number:
U = Us/number
if U<meter.min_utilization:
U = meter.min_utilization
else: U = meter.min_utilization
res[meter.id] = U
return res
def findSchedule(self):
for event in self.new_events:
is_scheduled = False
curr_time = self.end_date - event.duration
while not is_scheduled and not curr_time < self.start_date:
event.start = curr_time
event.end = curr_time + event.duration
is_valid = True
# check conflicts with current schedule
for component in self.ical.walk():
if component.name == 'VEVENT':
#try:
dc = component.decoded
dtstart = time.mktime(time.strptime(str(dc('dtstart')), '%Y-%m-%d %H:%M:%S+00:00'))/60
dtend = time.mktime(time.strptime(str(dc('dtend')), '%Y-%m-%d %H:%M:%S+00:00'))/60
if curr_time > dtstart and curr_time < dtend or curr_time + event.duration > dtstart and curr_time + event.duration < dtend or curr_time < dtstart and curr_time + event.duration > dtend or curr_time > dtstart and curr_time + event.duration < dtend or curr_time == dtstart or curr_time + event.duration == dtend:
is_valid = False
break
if is_valid:
for constraint in event.constraints:
if not constraint.isValid(event, self.ical):
is_valid = False
break
if is_valid:
self.addToCalendar(event)
is_scheduled = True
else:
curr_time -= 30
def index(self, report=None, subfolder=None, filter=None, reportid=None, fromdate=None, todate=None):
filter = bool(filter)
if str(fromdate).lower() in ('today', 't') :
fromdate = datetime.datetime.now().date()
elif fromdate is None:
fromdate =datetime.datetime.now().date()
else:
fromdate = datetime.datetime(*time.strptime(fromdate, "%d/%m/%Y")[:6]).date()
if str(todate).lower() in ('today', 't') or todate is None:
todate = datetime.datetime.now().date()
else:
todate = datetime.datetime(*time.strptime(todate, "%d/%m/%Y")[:6]).date()
args = self.parent.getargs()
if filter:
args["pagetitle"] = "Historical reports"
if 0 and not any_of(member_of_ad('Whiteclif - P&L'), member_of('admin'))():
log.debug("Auth Failed")
tabledata = Table(title='Reports',
columns=[' '],
data=[['You do not have sufficient rights to view reports']],
)
reportargs = {
"folder" : None,
"filter" : filter,
"fromdate" : fromdate,
"todate" : todate,
"authorised" : False,
"report" : report,
"subfolder" : subfolder,
"reportmenu" : reportconfig.reportmenu,
"tabledata" : tabledata,
}
else:
if reportid and filter:
reportargs = self.getargsSearch(reportparam=reportid, fromdate=fromdate, todate=todate)
elif report:
groupinfo = reportconfig.groupindex.get(report, None)
if groupinfo and groupinfo.get("groupby", None) == 'groupdir':
reportargs = self.getargsGroupDir(report, subfolder, filter=False, fromdate=fromdate, todate=todate)
else:
reportargs = self.getargsDir(report, subfolder, filter=filter, fromdate=fromdate, todate=todate)
else:
reportargs = self.getargs(report, subfolder, filter=filter, fromdate=fromdate, todate=todate)
args.update(reportargs)
args["include"] = 'reports.html'
args["refresh"] = False
log.debug("Rendering Template")
html = template.render(**args)
log.debug("Rendering Template Complete")
return html
def __init__(self,file):
while True:
line=file.readline()
if line[0]=='-':
#end of header
break
lineSplitPos=line.find(":")
if lineSplitPos>=0:
lineType=line[0:lineSplitPos]
lineValue=line[lineSplitPos+1:]
if lineType=='UUID':
self.patientID=lineValue
elif lineType=='Sampling Rate':
self.samplingRate=float(lineValue)
elif lineType=='Start Time':
offsetPos=lineValue.find("Offset:")
if offsetPos>0:
self.startTime=calendar.timegm(time.strptime(lineValue[0:offsetPos],' %Y-%m-%d %H:%M:%S '))
self.hoursOffset=int(lineValue[offsetPos+7:])
self.startTime-=self.hoursOffset*60*60
else:
self.startTime=calendar.timegm(time.strptime(lineValue,' %Y-%m-%d %H:%M:%S '))
self.hoursOffset=0
self.numSignals=6
fileLen=os.fstat(file.fileno()).st_size
curPos=file.tell()
self.recordCount=(fileLen-curPos)/(14)
def fetch():
posts["upcoming"]=[]
posts["ongoing"]=[]
hackerrank_contests["urls"] = []
thread_list = []
thread_list.append( threading.Thread(target=fetch_codeforces) )
thread_list.append( threading.Thread(target=fetch_topcoder) )
thread_list.append( threading.Thread(target=fetch_hackerearth) )
thread_list.append( threading.Thread(target=fetch_codechef) )
thread_list.append( threading.Thread(target=fetch_hackerrank_general) )
thread_list.append( threading.Thread(target=fetch_hackerrank_college) )
thread_list.append( threading.Thread(target=fetch_google) )
for thread in thread_list:
thread.start()
for thread in thread_list:
thread.join()
posts["upcoming"] = sorted(posts["upcoming"], key=lambda k: strptime(k['StartTime'], "%a, %d %b %Y %H:%M"))
posts["ongoing"] = sorted(posts["ongoing"], key=lambda k: strptime(k['EndTime'], "%a, %d %b %Y %H:%M"))
posts["timestamp"] = strftime("%a, %d %b %Y %H:%M:%S", localtime())
def fetch_codechef():
page = urlopen("http://www.codechef.com/contests")
soup = BeautifulSoup(page,"html.parser")
statusdiv = soup.findAll("div",attrs = {"class":"table-questions"})
upcoming_contests = statusdiv[1].findAll("tr")
if(len(upcoming_contests) <100):
for upcoming_contest in upcoming_contests[1:]:
details = upcoming_contest.findAll("td")
start_time = strptime(details[2].string, "%Y-%m-%d %H:%M:%S")
end_time = strptime(details[3].string, "%Y-%m-%d %H:%M:%S")
duration = get_duration(int(( mktime(end_time)-mktime(start_time) )/60 ))
posts["upcoming"].append({"Name" : details[1].string , "url" : "http://www.codechef.com"+details[1].a["href"] , "StartTime" : strftime("%a, %d %b %Y %H:%M", start_time),"EndTime" : strftime("%a, %d %b %Y %H:%M", end_time),"Duration":duration ,"Platform":"CODECHEF" })
ongoing_contests = statusdiv[0].findAll("tr")
for ongoing_contest in ongoing_contests[1:]:
details = ongoing_contest.findAll("td")
end_time = strptime(details[3].string, "%Y-%m-%d %H:%M:%S")
posts["ongoing"].append({ "Name" : details[1].string , "url" : "http://www.codechef.com"+details[1].a["href"] , "EndTime" : strftime("%a, %d %b %Y %H:%M", end_time) ,"Platform":"CODECHEF"})
else:
upcoming_contests = statusdiv[0].findAll("tr")
for upcoming_contest in upcoming_contests[1:]:
details = upcoming_contest.findAll("td")
start_time = strptime(details[2].string, "%Y-%m-%d %H:%M:%S")
end_time = strptime(details[3].string, "%Y-%m-%d %H:%M:%S")
duration = get_duration(int(( mktime(end_time)-mktime(start_time) )/60 ))
posts["upcoming"].append({"Name" : details[1].string , "url" : "http://www.codechef.com"+details[1].a["href"] , "StartTime" : strftime("%a, %d %b %Y %H:%M", start_time),"EndTime" : strftime("%a, %d %b %Y %H:%M", end_time),"Duration":duration ,"Platform":"CODECHEF" })
def get_dates():
"""
Return a dictionary containing count of submissions
on each date
"""
if len(request.args) < 1:
if session.handle:
handle = str(session.handle)
else:
redirect(URL("default", "submissions", args=[1]))
else:
handle = str(request.args[0])
stable = db.submission
row = db.executesql("SELECT status, time_stamp, COUNT(*) FROM submission WHERE submission.stopstalk_handle='" + handle + "' GROUP BY DATE(submission.time_stamp), submission.status;")
total_submissions = {}
streak = 0
max_streak = 0
prev = curr = start = None
for i in row:
if streak == 0:
streak = 1
prev = time.strptime(str(i[1]), "%Y-%m-%d %H:%M:%S")
prev = date(prev.tm_year, prev.tm_mon, prev.tm_mday)
start = prev
else:
curr = time.strptime(str(i[1]), "%Y-%m-%d %H:%M:%S")
curr = date(curr.tm_year, curr.tm_mon, curr.tm_mday)
delta = (curr - prev).days
if delta == 1:
streak += 1
elif delta != 0:
streak = 0
prev = curr
if streak > max_streak:
max_streak = streak
sub_date = str(i[1]).split()[0]
if total_submissions.has_key(sub_date):
total_submissions[sub_date][i[0]] = i[2]
total_submissions[sub_date]["count"] += i[2]
else:
total_submissions[sub_date] = {}
total_submissions[sub_date][i[0]] = i[2]
total_submissions[sub_date]["count"] = i[2]
today = datetime.today().date()
# If last streak does not match the current day
if (today - start).days + 1 != streak:
streak = 0
return dict(total=total_submissions,
max_streak=max_streak,
curr_streak=streak)
def GenerateShippingCalander(dt):
month = [['', '', '','','', '', ''],
['', '', '','','', '', ''],
['', '', '','','', '', ''],
['', '', '','','', '', ''],
['', '', '','','', '', ''],
['', '', '','','', '', ''],
]
today = time.localtime().tm_mday
start_time = time.strptime(dt, "%m/%d/%Y")
day = start_time.tm_mday
wday = start_time.tm_wday
last_day = calendar.monthrange(start_time.tm_year, start_time.tm_mon)[1]
row_no = 0
while day <= last_day:
d1 = datetime.datetime(start_time.tm_year, start_time.tm_mon, day)
d2 = datetime.datetime.now()
cur_time = time.strptime(time.strftime("%Y/%m/" + str(day), start_time), "%Y/%m/%d")
day = cur_time.tm_mday
wday = cur_time.tm_wday
script = ''
method = ''
bgcolor = "#FFFFFF"
days_diff = (d1 - d2).days + 1
if days_diff < 0:
bgcolor = "card5_grey"
elif days_diff == 0:
bgcolor = "card5_color2"
elif days_diff == 1:
bgcolor = "card5_color3"
method = "NextDay"
script = time.strftime("%m/" + str(day).zfill(2) + "/%Y", start_time)
elif days_diff == 2:
method = "SecondDay"
bgcolor = "card5_color4"
script = time.strftime("%m/" + str(day).zfill(2) + "/%Y", start_time)
elif days_diff > 2:
method = "GroundShipping"
bgcolor = "card5_color7"
script = time.strftime("%m/" + str(day).zfill(2) + "/%Y", start_time)
if days_diff >= 0:
if wday == 6:
bgcolor = "card5_color6"
script = ''
elif wday == 5:
method = "Saturday"
script = time.strftime("%m/" + str(day).zfill(2) + "/%Y", start_time)
bgcolor = "card5_color5"
day_hash = {'wday': wday, 'day': day, 'bgcolor':bgcolor, 'script':script, 'method': method}
month[row_no][wday] = day_hash
if wday == 6:
row_no += 1
day += 1
return month
def calcDays(startDate,endDate):
#对比两个日期的时间差
startDate=time.strptime(startDate,"%Y-%m-%d %H:%M:%S")
endDate=time.strptime(endDate,"%Y-%m-%d %H:%M:%S")
startDate=datetime(startDate[0],startDate[1],startDate[2],startDate[3],startDate[4],startDate[5])
endDate=datetime(endDate[0],endDate[1],endDate[2],endDate[3],endDate[4],endDate[5])
return (endDate-startDate).days
def __init__(self, pidfile, cfgfile):
Daemon.__init__(self, pidfile)
self.jobs = {}
self.immediately = False
self.scheduler = Scheduler(daemonic=False)
self.logger = logging.getLogger(self.__class__.__name__)
if os.path.exists(cfgfile):
with open(cfgfile, 'rt') as f:
config = yaml.load(f.read())
for k1 in config.keys():
if k1 == 'version':
pass
if k1 == 'immediately':
self.immediately = config[k1]
elif k1 == 'taobao':
self.jobs[k1] = config[k1]
self.jobs[k1]['id'] = None
if 'chktime' in self.jobs[k1].keys():
self.jobs[k1]['btime'] = time.strptime(self.jobs[k1]['chktime'].split('-')[0], '%H:%M')
self.jobs[k1]['etime'] = time.strptime(self.jobs[k1]['chktime'].split('-')[1], '%H:%M')
if self.jobs[k1]['btime'] >= self.jobs[k1]['etime']:
raise ValueError('"chktime" is illegal')
else:
raise ValueError('There is no "chktime" be found in configure.')
else:
pass
else:
self.logger.error('{0} not found'.format(cfgfile))
def set_json_aux_matches(self):
match = None
count = 1
if not self.html_aux_matches:
return None
for td in self.html_aux_matches.find_all('td'):
match = td.find("div", {'data-type': 'matches'})
if count == 1:
self.last_match_json = {
'match_id': match.find("div", {'data-type': 'matches'}).attrs.get('data-id'),
'date': datetime.fromtimestamp(time.mktime(time.strptime(match.find("div", {'class': 'm-date'}).attrs.get('data-matchdate'), '%Y%m%d'))),
'date_string': match.find("div", {'class': 'm-date'}).attrs.get('data-matchdate'),
'home': match.find("div", {'class': 'home'}).find('span', {'class': 't-nText'}).get_text(),
'home_logo': match.find('div', {'class': 'home'}).find('img', {'class': 't-i-3-logo'}).attrs.get('src'),
'away': match.find('div', {'class': 'away'}).find('span', {'class': 't-nText'}).get_text(),
'away_logo': match.find('div', {'class': 'away'}).find('img', {'class': 't-i-3-logo'}).attrs.get('src'),
'res': match.find('span', {'class': 's-resText'}).get_text(),
'is_res': True
}
else:
self.next_match_json = {
'match_id': match.find("div", {'data-type': 'matches'}).attrs.get('data-id'),
'date': datetime.fromtimestamp(time.mktime(time.strptime(match.find("div", {'class': 'm-date'}).attrs.get('data-matchdate'), '%Y%m%d'))),
'date_string': match.find("div", {'class': 'm-date'}).attrs.get('data-matchdate'),
'home': match.find("div", {'class': 'home'}).find('span', {'class': 't-nText'}).get_text(),
'home_logo': match.find('div', {'class': 'home'}).find('img', {'class': 't-i-3-logo'}).attrs.get('src'),
'away': match.find('div', {'class': 'away'}).find('span', {'class': 't-nText'}).get_text(),
'away_logo': match.find('div', {'class': 'away'}).find('img', {'class': 't-i-3-logo'}).attrs.get('src'),
'is_res': True
}
count += 1
def attributes(self, node):
attr = VMap()
vfile = node.open()
img = Image.open(vfile)
info = img._getexif()
vfile.close()
for tag, values in info.items():
if tag in self.dateTimeTags:
try:
decoded = str(TAGS.get(tag, tag))
try:
dt = strptime(values, "%Y:%m:%d %H:%M:%S")
except ValueError:
try:
dt = strptime(values[:-6], "%Y-%m-%dT%H:%M:%S")
except ValueError:
dt = strptime(values.rstrip(' '), "%a %b %d %H:%M:%S")
vt = vtime(dt.tm_year, dt.tm_mon, dt.tm_mday, dt.tm_hour, dt.tm_min, dt.tm_sec, 0)
vt.thisown = False
attr[decoded] = Variant(vt)
except Exception as e:
attr[decoded] = Variant(str(values))
else:
decoded = str(TAGS.get(tag, tag))
if isinstance(values, tuple):
vl = VList()
for value in values:
vl.push_back(Variant(value))
attr[decoded] = vl
else:
attr[decoded] = Variant(values)
return attr
def fix_unicode(self):
for conf in self.FEC_CONFIG:
try:
infile = open(os.path.join(self._working_dir(conf), conf.filename), "r")
except:
infile = open(os.path.join(self._working_dir(conf), conf.filename)[:-3] + "dta", "r")
outfile = open(os.path.join(self._working_dir(conf), conf.filename[:-3] + "txt.utf8"), "w")
for line in infile:
try:
fixed_line = line.decode("utf8", "replace").encode("utf8", "replace")
except:
fixed_line = line
self.log.info("utf problem" + line)
continue # don't include this line in the database (!)
if not conf.schema:
parts = fixed_line.split("|")
if len(parts[13]) == 7:
parts[13] = "0" + parts[13][:2] + parts[13][3:]
date = parts[13][-4:] + "-" + parts[13][:2] + "-" + parts[13][2:4]
try:
time.strptime(date, "%Y-%m-%d")
except:
date = ""
parts[13] = date
fixed_line = "|".join(parts)
try:
outfile.write(conf.cycle + "|" + fixed_line)
except:
self.log.info("couldnt write " + fixed_line)
def get_milli_timestamp(time_str):
if len(time_str) == 8:
struct_time = time.strptime(time_str, "%Y%m%d")
elif len(time_str) == 10:
struct_time = time.strptime(time_str, "%Y%m%d%H")
return int(time.mktime(struct_time) * 1000)
def fetch_topcoder():
try:
page = urlopen("https://clients6.google.com/calendar/v3/calendars/[email protected]/events?calendarId=appirio.com_bhga3musitat85mhdrng9035jg%40group.calendar.google.com&singleEvents=true&timeZone=Asia%2FCalcutta&maxAttendees=1&maxResults=250&sanitizeHtml=true&timeMin=2015-04-26T00%3A00%3A00-04%3A00&timeMax=2016-06-07T00%3A00%3A00-04%3A00&key=AIzaSyBNlYH01_9Hc5S1J9vuFmu2nUqBZJNAXxs",timeout=15)
data = json.load(page)["items"]
cur_time = localtime()
for item in data:
if(item["start"].has_key("date")):continue
start_time = strptime(item["start"]["dateTime"][:19], "%Y-%m-%dT%H:%M:%S")
start_time_indian = strftime("%a, %d %b %Y %H:%M",start_time)
end_time = strptime(item["end"]["dateTime"][:19], "%Y-%m-%dT%H:%M:%S")
end_time_indian = strftime("%a, %d %b %Y %H:%M",end_time)
duration = get_duration(int(( mktime(end_time)-mktime(start_time) )/60 ))
name = item["summary"]
if "SRM" in name and "description" in item: url = "http://community.topcoder.com/tc?module=MatchDetails&rd="+ item["description"][110:115]
else : url = "http://tco15.topcoder.com/algorithm/rules/"
if cur_time<start_time:
posts["upcoming"].append({ "Name" : name , "url" : url ,"EndTime" : end_time_indian,"Duration":duration, "StartTime" : start_time_indian,"Platform":"TOPCODER" })
elif cur_time>start_time and cur_time<end_time:
posts["ongoing"].append({ "Name" : name , "url" : url ,"EndTime" : end_time_indian,"Platform":"TOPCODER" })
except Exception, e:
pass
# =========================================
tags = flip_flags(flags)
files = [f for f in listdir(inpath) if isfile(join(inpath, f))]
for file in files:
transactions = openfile( file )
for transaction in transactions:
transaction.append(file)
trans_date = time.strptime(transaction[0], "%d/%m/%Y")
transaction[0] = time.strftime("%Y-%m-%d", trans_date)
if transaction[1][0] == '+':
transaction[1] = transaction[1][1:]
if not add_flags(transaction):
unmatched += 1
unmatched_amount -= float(transaction[1])
unmatched_tags.append(transaction[2][0:30].strip())
messages.append(transaction[1] + ' ' + transaction[2][0:100].strip().ljust(100) + ' ' + transaction[4])
# print transaction[1] + ' ' + transaction[2]
# loop through our tags dict and see if any of the tags match our message
# conflict = transaction[5] == transaction[7]
# transaction.append(str(conflict))
localtime2 = time.asctime(time.localtime(time.time()))
print "当前时间戳为:", ticks
print "本地时间为 :", localtime1
print "本地格式化时间为 :", localtime2
print'\n\n'
# 格式化成2016-03-20 11:45:39形式
print '格式化成2016-03-20 11:45:39形式:', time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
# 格式化成Sat Mar 28 22:24:24 2016形式
print '格式化成Sat Mar 28 22:24:24 2016形式:', time.strftime("%a %b %d %H:%M:%S %Y", time.localtime())
# 将格式字符串转换为时间戳
a = "Sat Mar 28 22:24:24 2016"
print '将格式字符串转换为时间戳:', time.mktime(time.strptime(a, "%a %b %d %H:%M:%S %Y"))
print'\n\n'
import calendar
cal = calendar.month(2018, 3)
print "以下输出2016年1月份的日历:"
print cal;
# 函数 -------------------
# 函数代码块以 def 关键词开头，后接函数标识符名称和圆括号()。
# 任何传入参数和自变量必须放在圆括号中间。圆括号之间可以用于定义参数。
# 函数的第一行语句可以选择性地使用文档字符串—用于存放函数说明。
# 函数内容以冒号起始，并且缩进。
# return [表达式] 结束函数，选择性地返回一个值给调用方。不带表达式的return相当于返回 None。
import sys
import time
for data in sys.stdin:
data = data.strip()
fields = data.split('\t')
if len(fields) != 11:
continue
[_, Time, userid, rc, AC, query, CI, iC, Post, _, dialogue] = fields
if userid[-1]!='f':
continue
if rc == 'godText':
try:
t1 = time.strptime(Time[:20], '%d/%b/%Y:%H:%M:%S')
Time = time.strftime('%Y-%m-%d %H:%M:%S',t1)
#idx = Post.find('[seq]')
#sc = Post[:idx]
#sc = Post.split('[seq]')[0]
sc = 'sc_null'
S = [userid,Time,AC,query,sc]
S = '\t'.join(S)
sys.stdout.write("%s\n" % S)
except:
continue
def HMS2ts(date):
return int(time.mktime(time.strptime(date, '%Y-%m-%d')))
,time.localtime().tm_mday)
#表示方式的相互转换
#元组转化为时间戳
print('元组转换为时间戳：',time.mktime(t))
#当前系统时间转换成时间戳
print(time.mktime(time.localtime()))
#时间戳转换为时间元组
print(time.localtime(1502591382))
print('当前时间戳转换为时间元组:',time.localtime(time.time()))
#元组转换为时间格式字符串,注意区分大小写
print('当前时间转换成时间格式的字符串:',time.strftime('%Y-%m-%d %H:%M:%S',time.localtime()))
#时间格式字符串转换成时间元组,后面的格式描述不能少
print('时间格式字符串转换成时间元组:',time.strptime('2021-02-21 21:00:09','%Y-%m-%d %H:%M:%S'))
#datatime模块，表示日期时间
import datetime
print(datetime.datetime(2021,2,21,21,37,50)) #以时间元组创建日期时间，年月日时必写，时分秒不填默认00:00:00
print(datetime.datetime.now()) #获取当前时间，精确到微秒
print(datetime.datetime.today()) #获取当前时间
#时间表示形式相互转换
print('当前日期时间转换成时间戳',datetime.datetime.now().timestamp()) #
print('把时间戳转换成日期时间',datetime.datetime.fromtimestamp(1613914941.140178))
print('日期时间转换成时间格式字符串',datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
print('时间字符串转换成日期时间',datetime.datetime.strptime('2021-02-21 21:47:44','%Y-%m-%d %H:%M:%S'))
#日历模块
import calendar
def node_start_time(cls, node):
time_str = node.extra['launch_time'].split('.', 2)[0] + 'UTC'
return time.mktime(time.strptime(
time_str,'%Y-%m-%dT%H:%M:%S%Z')) - time.timezone
def parse(self, response):
self.isCOOKIE = True
self.isOK = True
# print('----------------------------')
# print(response.text)
#把当天的航班信息和价格信息分类
time_numbers = response.xpath('//td[contains(@class,"fl_date")]')
prices = response.xpath(
'//td[contains(@class,"outward-total-fare-td")]')
# print('-'*50)
# print(response.text)
# print('+' * 50)
if not time_numbers:
# print('!'*30)
# print '%s' % (response.text.decode('utf-8').encode('gbk', 'ignore'))
# self.log("no data",40)
try:
page = response.xpath('//h2/text()')[0].extract()
# print(page)
# print(response.status)
# # print response.text
# # proxy_invalid = response.xpath('//td[4]/text()')[0].extract()
# # print proxy_invalid
# if response.status == 404:
# self.isOK = False
# yield scrapy.Request(self.start_urls[0],
# method='POST',
# headers=self.custom_settings.get('headers'),
# body=response.meta.get('meta_data').get('form'),
# callback=self.parse,
# meta={'meta_data': response.meta.get('meta_data')},
# errback=self.errback)
if page == 'Are you human?':
self.isCOOKIE = False
yield scrapy.Request(
self.start_urls[0],
method='POST',
headers=self.custom_settings.get('headers'),
body=response.meta.get('meta_data').get('form'),
callback=self.parse,
meta={'meta_data': response.meta.get('meta_data')},
errback=self.errback)
self.task.append(response.meta.get('meta_data').get('invalid'))
except:
self.log("no data", 10)
self.task.append(response.meta.get('meta_data').get('invalid'))
return
#循环取出每个航班信息,year指的是航班的年份
year = response.meta.get('meta_data').get('year')
# print(len(time_numbers))
for i in range(len(time_numbers)):
#取出当次的航班信息,出发时间，到达时间，航班号
time_number = time_numbers[i].xpath('./span/text()').extract()
# 出发时间
deptime = time.strptime(year + time_number[0], '%Y%m/%d\xa0%H:%M')
depTime = time.mktime(deptime)
# 到达时间
arrtime = time.strptime(year + time_number[1], '%Y%m/%d\xa0%H:%M')
arrTime = time.mktime(arrtime)
# 航班号
flightNumber = time_number[2]
carrier = re.search('\D{2}', time_number[2]).group()
#根据n定位当前航班价格，总价除以maxSeats为单人价格
maxSeats = response.meta.get('meta_data').get('maxSeats')
try:
price = prices[i].xpath(
'./div[@id="outward_hp_' + str(i + 1) +
'_total_fare"]//span/text()').extract()
except:
#这种情况是ip有问题，得到数据是错误的
self.log('Dangerous error data....', 40)
self.isOK = False
if price[3] == '0':
price = prices[i].xpath(
'//div[@id="outward_hpp_' + str(i + 1) +
'_total_fare"]//span/text()').extract()
if price[3] == '0':
price = prices[i].xpath(
'//div[@id="outward_prime_' + str(i + 1) +
'_total_fare"]//span/text()').extract()
if price[3] == '0':
self.task.append(response.meta.get('meta_data').get('invalid'))
continue
# print(price)
# 取价格
netFare = int(
re.search(r"\d.*", price[0]).group().replace(',',
'')) / maxSeats
adultTax = int(
re.search(r"\d.*", price[1]).group().replace(',',
'')) / maxSeats
#增加价格打折的判断
promo = response.xpath('//td[@id="outward_hp_' + str(i + 1) +
'_list"]/@class').extract()[0].split(' ')
if promo[-1] == 'promo':
adultPrice = int(
re.search(r"\d.*", price[3]).group().replace(
',', '')) / maxSeats / 0.7
cabin = 'S'
else:
cabin = 'X'
adultPrice = int(
re.search(r"\d.*", price[3]).group().replace(
',', '')) / maxSeats
# 判断网页信息是否虚假
if not price[2]:
return
currency = self.custom_settings.get('CURRENCY_CACHE').get(price[2])
depAirport = response.meta.get('meta_data').get('invalid').get(
'depAirport')
arrAirport = response.meta.get('meta_data').get('invalid').get(
'arrAirport')
isChange = 1
segments = dict(
flightNumber=flightNumber,
aircraftType='',
number=1,
departureTime=time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(depTime)),
destinationTime=time.strftime('%Y-%m-%d %H:%M:%S',
time.localtime(arrTime)),
airline=carrier,
dep=depAirport,
dest=arrAirport,
seats=int(maxSeats),
duration=dataUtil.gen_duration(depTime, arrTime),
depTerminal='')
getTime = time.time()
item = WowSpiderItem()
item['flightNumber'] = flightNumber
item['depTime'] = depTime
item['arrTime'] = arrTime
item['fromCity'] = self.portCitys.get(depAirport, depAirport)
item['toCity'] = self.portCitys.get(arrAirport, arrAirport)
item['depAirport'] = depAirport
item['arrAirport'] = arrAirport
item['currency'] = currency
item['adultPrice'] = adultPrice
item['adultTax'] = adultTax
item['netFare'] = netFare
item['maxSeats'] = maxSeats
item['cabin'] = cabin
item['carrier'] = carrier
item['isChange'] = isChange
item['segments'] = '[]'
item['getTime'] = getTime
yield item
# coding: utf-8
import time
time.stftime('%Y-%m-%d %H:%M:%S', '2018-09-12 12:12:12')
time.strptime('%Y-%m-%d %H:%M:%S', '2018-09-12 12:12:12')
time.strstime('%Y-%m-%d %H:%M:%S', '2018-09-12 12:12:12')
time.strptime('2018-09-12 12:12:12', "%Y-%m-%d %H:%M:%S")
time.strptime('2018-09-12 12:12:12', "%Y-%m-%d")
time.strptime('2018-09-12 12:12:12', "%Y-%m-%d %H:%M:%S")
t = time.strptime('2018-09-12 12:12:12', "%Y-%m-%d %H:%M:%S")
t
dir(t)
time.mktime(t)
ss = time.mktime(t)
datetime.datetime.fromtimestamp(ss)
import datetime
datetime.datetime.fromtimestamp(ss)
datetime.datetime.fromtimestamp(ss).date
datetime.datetime.fromtimestamp(ss).date()
d =datetime.datetime.fromtimestamp(ss).date()
str(d)
datetime.datetime('2018-09-12 12:12:12')
dir(datetime)
dir(datetime.time)
from datetime import datetime
datetime.strptime('2018-08-12 12:12:21', "%Y-%m-%d %H:%M:%S")
datetime.strptime('2018-08-12 12:12:21', "%Y-%m-%d %H:%M:%S").date()
import pandas as pd
d =[array([10, 1, 7, 3]),
array([ 0, 14, 12, 13]),
array([ 3, 10, 7, 8]),
array([7, 5]),
header = fh.readline().strip().split('\t')
header.insert(1, 'Kingdom')
print('\t'.join(header))
for line in fh:
line = line.strip()
word = line.split('\t')
size = word[6]
gc = word[7]
genes = word[14]
proteins = word[15]
release = word[17]
if word[0] not in check:
check[word[0]] = [size, gc, genes, proteins, release]
elif word[0] in check:
try:
if time.strptime(check[word[0]][-1],
"%Y/%m/%d") < time.strptime(
release, "%Y/%m/%d"):
check[word[0]] = [size, gc, genes, proteins, release]
except:
if check[word[0]][-1] == '-':
check[word[0]] = [size, gc, genes, proteins, release]
else:
pass
test1 = {}
with open('bacteria.txt') as fh:
for line in fh:
line = line.strip().split('\t')
if line[0] in check:
test1[line[0]] = line[:4] + check[line[0]]
def timestamp2unix(time_string, pattern='%Y-%m-%d %H:%M:%S'):
time_array = time.strptime(time_string, pattern)
return int(time.mktime(time_array))
def grab_tv_guide(channels):
programs = {}
guide_json = urlquick.get(URL_PROGRAMS, max_age=-1)
guide_json = json.loads(guide_json.text)
guide_items = guide_json['data']['items']
for guide_item in guide_items:
channel = guide_item['channel']
channel_crt_id = channel['id']
if channel_crt_id in ID_CHANNELS and \
ID_CHANNELS[channel_crt_id] in channels:
channel_id = ID_CHANNELS[channel_crt_id]
program_dict = {}
program_dict['duration'] = guide_item['duration'] # sec
start_s = guide_item['startedAt']
start_s = start_s.split('+')[0]
try:
start = datetime.datetime.strptime(start_s, GUIDE_TIME_FORMAT)
except TypeError:
start = datetime.datetime(
*(time.strptime(start_s, GUIDE_TIME_FORMAT)[0:6]))
try:
local_tz = get_localzone()
except Exception:
# Hotfix issue #102
local_tz = pytz.timezone('Europe/Brussels')
start = GUIDE_TIMEZONE.localize(start)
start = start.astimezone(local_tz)
start_time = start.strftime("%Hh%M")
program_dict['start_time'] = start_time
program_dict['genre'] = guide_item['program']['formatGenre'][
'genre']['name']
program_dict['program_id'] = guide_item['id']
program_dict['title'] = guide_item['title']
# https://tel.img.pmdstatic.net/fit/http.3A.2F.2Fimages.2Eone.2Eprismamedia.2Ecom.2Fchannel.2F2.2F3.2F9.2Fe.2F1.2F3.2F1.2Fb.2Ff.2Fd.2Fa.2F0.2F2.2Fa.2F3.2Fd.2Epng/76x76/quality/100/image.png
# https://tel.img.pmdstatic.net/fit/http.3A.2F.2Fimages.2Eone.2Eprismamedia.2Ecom.2Fprogram.2Fe.2F0.2F0.2F2.2F5.2Fc.2F3.2F3.2F2.2F1.2F1.2F1.2F1.2Fa.2F4.2F9.2Ejpg/520x336/quality/75/image.jpg
# https://tel.img.pmdstatic.net/{transformation}/http.3A.2F.2Fimages.2Eone.2Eprismamedia.2Ecom.2Fprogram.2F8.2F0.2F4.2F4.2F4.2F7.2F9.2F3.2F5.2F4.2F9.2F7.2F4.2Fb.2Ff.2Fd.2Ejpg/{width}x{height}/{parameters}/{title}.jpg
if 'image' in guide_item['program']:
image = guide_item['program']['image']['urlTemplate']
image = image.replace('{transformation}', 'fit')
image = image.replace('{width}', '520')
image = image.replace('{height}', '336')
image = image.replace('{parameters}', 'quality/75')
image = image.replace('{title}', 'image')
program_dict['thumb'] = image
programs[channel_id] = program_dict
return programs
def tojstime(d):
return time.mktime(time.strptime(str(d), '%Y-%m-%d')) * 1000
def test_03_stats(self):
"""Test STATS stats method works"""
today = unicode(datetime.date.today())
hour = int(datetime.datetime.utcnow().strftime('%H'))
date_ms = time.mktime(time.strptime(today, "%Y-%m-%d")) * 1000
anon = 0
auth = 0
TaskRunFactory.create(task=self.project.tasks[0])
TaskRunFactory.create(task=self.project.tasks[1])
dates_stats, hours_stats, user_stats = stats.get_stats(self.project.id)
for item in dates_stats:
if item['label'] == 'Anon + Auth':
assert item['values'][-1][0] == date_ms, item['values'][0][0]
assert item['values'][-1][
1] == 10, "There should be 10 answers"
if item['label'] == 'Anonymous':
assert item['values'][-1][0] == date_ms, item['values'][0][0]
anon = item['values'][-1][1]
if item['label'] == 'Authenticated':
assert item['values'][-1][0] == date_ms, item['values'][0][0]
auth = item['values'][-1][1]
if item['label'] == 'Total Tasks':
assert item['values'][-1][0] == date_ms, item['values'][0][0]
assert item['values'][-1][1] == 4, "There should be 4 tasks"
if item['label'] == 'Expected Answers':
assert item['values'][0][0] == date_ms, item['values'][0][0]
for i in item['values']:
assert i[1] == 100, "Each date should have 100 answers"
assert item['values'][0][
1] == 100, "There should be 10 answers"
assert auth + anon == 10, "date stats sum of auth and anon should be 10"
max_hours = 0
for item in hours_stats:
if item['label'] == 'Anon + Auth':
max_hours = item['max']
print item
assert item['max'] == 10, item['max']
assert item['max'] == 10, "Max hours value should be 10"
for i in item['values']:
if i[0] == hour:
assert i[1] == 10, "There should be 10 answers"
assert i[2] == 5, "The size of the bubble should be 5"
else:
assert i[1] == 0, "There should be 0 answers"
assert i[2] == 0, "The size of the buggle should be 0"
if item['label'] == 'Anonymous':
anon = item['max']
for i in item['values']:
if i[0] == hour:
assert i[1] == anon, "There should be anon answers"
assert i[2] == (
anon * 5
) / max_hours, "The size of the bubble should be 5"
else:
assert i[1] == 0, "There should be 0 answers"
assert i[2] == 0, "The size of the buggle should be 0"
if item['label'] == 'Authenticated':
auth = item['max']
for i in item['values']:
if i[0] == hour:
assert i[1] == auth, "There should be anon answers"
assert i[2] == (
auth * 5
) / max_hours, "The size of the bubble should be 5"
else:
assert i[1] == 0, "There should be 0 answers"
assert i[2] == 0, "The size of the buggle should be 0"
assert auth + anon == 10, "date stats sum of auth and anon should be 8"
err_msg = "user stats sum of auth and anon should be 7"
assert user_stats['n_anon'] + user_stats['n_auth'] == 7, err_msg
def create_project(project_name,expired=None):
#创建新项目，project_name是项目名，expired是过期时间，字串输入 2019-01-01 格式
create_project_list = """CREATE TABLE IF NOT EXISTS `project_list` (
`project_name` varchar(255) DEFAULT NULL COMMENT '项目名称',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
`expired_at` int(11) DEFAULT NULL COMMENT '过期时间',
`event_count` bigint(20) DEFAULT NULL COMMENT '事件量',
`device_count` bigint(20) DEFAULT NULL COMMENT '设备数',
`user_count` bigint(20) DEFAULT NULL COMMENT '用户数',
`enable_scheduler` int(4) DEFAULT 1 COMMENT '是否启动定时器支持'
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"""
create_shortcut = """CREATE TABLE if not EXISTS `shortcut` (
`project` varchar(255) DEFAULT NULL COMMENT '项目名',
`short_url` varchar(255) DEFAULT NULL COMMENT '短链地址',
`long_url` varchar(768) DEFAULT NULL COMMENT '长链地址',
`expired_at` int(11) DEFAULT NULL COMMENT '过期时间',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
`src` varchar(10) DEFAULT NULL COMMENT '使用的第三方创建源',
`src_short_url` varchar(255) DEFAULT NULL COMMENT '创建源返回的短地址',
`submitter` varchar(255) DEFAULT NULL COMMENT '由谁提交',
`utm_source` varchar(2048) DEFAULT NULL COMMENT 'utm_source',
`utm_medium` varchar(2048) DEFAULT NULL COMMENT 'utm_medium',
`utm_campaign` varchar(2048) DEFAULT NULL COMMENT 'utm_campaign',
`utm_content` varchar(2048) DEFAULT NULL COMMENT 'utm_content',
`utm_term` varchar(2048) DEFAULT NULL COMMENT 'utm_term',
KEY `short_url` (`short_url`),
KEY `long_url` (`long_url`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"""
create_shortcut_history = """CREATE TABLE IF NOT EXISTS `shortcut_history` (
`short_url` varchar(255) DEFAULT NULL COMMENT '解析短链',
`result` varchar(255) DEFAULT NULL COMMENT '解析的结果',
`cost_time` int(11) DEFAULT NULL COMMENT '耗费时间',
`ip` varchar(255) DEFAULT NULL,
`created_at` int(11) DEFAULT NULL COMMENT '解析时间',
`user_agent` text DEFAULT NULL,
`accept_language` text DEFAULT NULL,
`ua_platform` varchar(255) DEFAULT NULL,
`ua_browser` varchar(255) DEFAULT NULL,
`ua_version` varchar(255) DEFAULT NULL,
`ua_language` varchar(255) DEFAULT NULL,
KEY `created_at` (`created_at`),
KEY `short_url` (`short_url`),
KEY `short_url_result` (`short_url`,`result`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"""
create_mobile_ad_src ="""CREATE TABLE if not EXISTS `mobile_ad_src` (
`src` varchar(255) NOT NULL COMMENT '创建源名称',
`src_name` varchar(255) DEFAULT NULL COMMENT '创建源的中文名字',
`src_args` varchar(1024) DEFAULT NULL COMMENT '创建源自带参数',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
`updated_at` int(11) DEFAULT NULL COMMENT '维护时间',
`utm_source` varchar(255) DEFAULT NULL COMMENT '预制的utm_source',
`utm_medium` varchar(255) DEFAULT NULL COMMENT '预制的utm_medium',
`utm_campaign` varchar(255) DEFAULT NULL COMMENT '预制的utm_campaign',
`utm_content` varchar(255) DEFAULT NULL COMMENT '预制的utm_content',
`utm_term` varchar(255) DEFAULT NULL COMMENT '预制的utm_term',
PRIMARY KEY (`src`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"""
create_mobile_ad_list ="""CREATE TABLE if not EXISTS `mobile_ad_list` (
`project` varchar(255) DEFAULT NULL COMMENT '项目名',
`url` varchar(768) NOT NULL COMMENT '监测地址',
`expired_at` int(11) DEFAULT NULL COMMENT '过期时间',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
`src` varchar(255) DEFAULT NULL COMMENT '使用的检测原id',
`src_url` varchar(1024) DEFAULT NULL COMMENT '使用的检测模板',
`submitter` varchar(255) DEFAULT NULL COMMENT '由谁提交',
`utm_source` varchar(2048) DEFAULT NULL COMMENT 'utm_source',
`utm_medium` varchar(2048) DEFAULT NULL COMMENT 'utm_medium',
`utm_campaign` varchar(2048) DEFAULT NULL COMMENT 'utm_campaign',
`utm_content` varchar(2048) DEFAULT NULL COMMENT 'utm_content',
`utm_term` varchar(2048) DEFAULT NULL COMMENT 'utm_term',
PRIMARY KEY (`url`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"""
created_shortcut_read="""CREATE TABLE if not EXISTS `shortcut_read` (
`short_url` varchar(255) NOT NULL COMMENT '短链地址',
`ip` varchar(20) DEFAULT NULL COMMENT 'ip',
`created_at` int(11) DEFAULT NULL COMMENT '时间',
`user_agent` text DEFAULT NULL COMMENT 'ua',
`accept_language` text DEFAULT NULL COMMENT '语言',
`ua_platform` varchar(255) DEFAULT NULL COMMENT '平台',
`ua_browser` varchar(255) DEFAULT NULL COMMENT '浏览器',
`ua_version` varchar(255) DEFAULT NULL COMMENT '版本号',
`ua_language` varchar(255) DEFAULT NULL COMMENT '语言',
`referrer` text DEFAULT NULL COMMENT '页面',
KEY `short_url` (`short_url`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"""
blacklist_sql_1="""CREATE TABLE `recall_blacklist` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`project` varchar(255) NOT NULL COMMENT '项目名',
`distinct_id` varchar(255) DEFAULT NULL,
`key` varchar(255) NOT NULL COMMENT '渠道key',
`type_id` int(11) NOT NULL COMMENT '渠道类型',
`reason_id` int(11) DEFAULT NULL COMMENT '原因id',
`owner` varchar(255) DEFAULT NULL COMMENT '第一次操作所属人',
`latest_owner` varchar(255) DEFAULT NULL COMMENT '最后一次操作所属人',
`status` int(11) DEFAULT NULL COMMENT '状态',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
`updated_at` int(11) DEFAULT NULL COMMENT '更新时间',
PRIMARY KEY (`id`),
UNIQUE KEY `anti_copy` (`key`,`type_id`,`project`),
KEY `check_blacklist` (`status`,`key`,`type_id`,`project`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1;"""
blacklist_sql_2="""CREATE TABLE `recall_blacklist_history` (
`rbid` int(11) NOT NULL COMMENT 'recall_blacklist的id',
`checker` varchar(255) DEFAULT NULL COMMENT '查询者的名字',
`result_status_id` int(11) DEFAULT NULL COMMENT '返回的status_code里pid是39的状态',
`result_reason_id` int(11) DEFAULT NULL COMMENT '返回的status_code里pid是30的理由',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
KEY `rbid` (`rbid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"""
blacklist_sql_3="""CREATE TABLE `recall_blacklist_reason` (
`rbid` int(11) NOT NULL COMMENT 'recall_blacklist的id',
`reason_id` int(11) DEFAULT NULL COMMENT 'status_code里pid是30的状态',
`reason_owner` varchar(255) DEFAULT NULL COMMENT '修改人',
`reason_comment` varchar(255) DEFAULT NULL COMMENT '修改的备注',
`final_status_id` int(11) DEFAULT NULL COMMENT '最后写入recall_blacklist的status_code里pid是39的状态',
`created_at` varchar(255) DEFAULT NULL COMMENT '创建的时间',
KEY `rbid` (`rbid`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;"""
do_tidb_exe(create_project_list)
do_tidb_exe(create_shortcut)
do_tidb_exe(create_shortcut_history)
do_tidb_exe(create_mobile_ad_src)
do_tidb_exe(create_mobile_ad_list)
do_tidb_exe(created_shortcut_read)
do_tidb_exe(blacklist_sql_1)
do_tidb_exe(blacklist_sql_2)
do_tidb_exe(blacklist_sql_3)
# print('project_list已生成')
check_sql = "show tables"
check_result,check_count = do_tidb_select(check_sql)
tables_name = []
for line in check_result:
tables_name.append(line[0])
# print(tables_name)
check_project_list_sql = """SELECT count(*) FROM `project_list` where project_name='{project_name}'""".format(project_name=project_name)
check_project_list_result,check_project_list_count = do_tidb_select(check_project_list_sql)
# print(check_project_list_result[0][0])
if project_name in tables_name or check_project_list_result[0][0]>0:
print(project_name+'项目表单已存在')
else:
table_sql="""CREATE TABLE `{project_name}` (
`track_id` bigint(17) DEFAULT NULL,
`distinct_id` varchar(64) DEFAULT NULL,
`lib` varchar(255) DEFAULT NULL,
`event` varchar(255) DEFAULT NULL,
`type` varchar(255) DEFAULT NULL,
`all_json` json DEFAULT NULL,
`host` varchar(255) DEFAULT NULL,
`user_agent` varchar(2048) DEFAULT NULL,
`ua_platform` varchar(1024) DEFAULT NULL,
`ua_browser` varchar(1024) DEFAULT NULL,
`ua_version` varchar(1024) DEFAULT NULL,
`ua_language` varchar(1024) DEFAULT NULL,
`connection` varchar(255) DEFAULT NULL,
`pragma` varchar(255) DEFAULT NULL,
`cache_control` varchar(255) DEFAULT NULL,
`accept` varchar(255) DEFAULT NULL,
`accept_encoding` varchar(255) DEFAULT NULL,
`accept_language` varchar(255) DEFAULT NULL,
`ip` varchar(512) DEFAULT NULL,
`ip_city` json DEFAULT NULL,
`ip_asn` json DEFAULT NULL,
`url` text DEFAULT NULL,
`referrer` varchar(2048) DEFAULT NULL,
`remark` varchar(255) DEFAULT NULL,
`created_at` int(11) DEFAULT NULL,
`date` date DEFAULT NULL,
`hour` int(2) DEFAULT NULL,
KEY `date` (`date`),
KEY `distinct_id` (`distinct_id`),
KEY `event` (`event`),
KEY `date_hour` (`date`,`hour`),
KEY `event_date` (`event`,`date`),
KEY `event_remark_date` (`event`,`remark`,`date`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;""".format(project_name=project_name)
table_device_sql ="""CREATE TABLE `{project_name}_device` (
`distinct_id` varchar(255) NOT NULL,
`lib` varchar(255) DEFAULT NULL,
`device_id` varchar(255) DEFAULT NULL,
`manufacturer` varchar(255) DEFAULT NULL,
`model` varchar(255) DEFAULT NULL,
`os` varchar(255) DEFAULT NULL,
`os_version` varchar(255) DEFAULT NULL,
`ua_platform` varchar(1024) DEFAULT NULL,
`ua_browser` varchar(1024) DEFAULT NULL,
`ua_version` varchar(1024) DEFAULT NULL,
`ua_language` varchar(1024) DEFAULT NULL,
`screen_width` int(11) DEFAULT NULL,
`screen_height` int(11) DEFAULT NULL,
`network_type` varchar(255) DEFAULT NULL,
`user_agent` varchar(2048) DEFAULT NULL,
`accept_language` varchar(255) DEFAULT NULL,
`ip` varchar(255) DEFAULT NULL,
`ip_city` json DEFAULT NULL,
`ip_asn` json DEFAULT NULL,
`wifi` varchar(20) DEFAULT NULL,
`app_version` varchar(255) DEFAULT NULL,
`carrier` varchar(255) DEFAULT NULL,
`referrer` text DEFAULT NULL,
`referrer_host` varchar(2048) DEFAULT NULL,
`bot_name` varchar(255) DEFAULT NULL,
`browser` varchar(255) DEFAULT NULL,
`browser_version` varchar(255) DEFAULT NULL,
`is_login_id` varchar(255) DEFAULT NULL,
`screen_orientation` varchar(255) DEFAULT NULL,
`gps_latitude` decimal(11,7) DEFAULT NULL,
`gps_longitude` decimal(11,7) DEFAULT NULL,
`first_visit_time` datetime DEFAULT NULL,
`first_referrer` text DEFAULT NULL,
`first_referrer_host` varchar(768) DEFAULT NULL,
`first_browser_language` varchar(768) DEFAULT NULL,
`first_browser_charset` varchar(768) DEFAULT NULL,
`first_search_keyword` varchar(768) DEFAULT NULL,
`first_traffic_source_type` varchar(768) DEFAULT NULL,
`utm_content` varchar(768) DEFAULT NULL,
`utm_campaign` varchar(768) DEFAULT NULL,
`utm_medium` varchar(768) DEFAULT NULL,
`utm_term` varchar(768) DEFAULT NULL,
`utm_source` varchar(768) DEFAULT NULL,
`latest_utm_content` varchar(768) DEFAULT NULL,
`latest_utm_campaign` varchar(768) DEFAULT NULL,
`latest_utm_medium` varchar(768) DEFAULT NULL,
`latest_utm_term` varchar(768) DEFAULT NULL,
`latest_utm_source` varchar(768) DEFAULT NULL,
`latest_referrer` varchar(2048) DEFAULT NULL,
`latest_referrer_host` varchar(2048) DEFAULT NULL,
`latest_search_keyword` varchar(768) DEFAULT NULL,
`latest_traffic_source_type` varchar(255) DEFAULT NULL,
`created_at` int(11) DEFAULT NULL,
`updated_at` int(11) DEFAULT NULL,
PRIMARY KEY (`distinct_id`),
KEY `utm_campaign` (`utm_campaign`),
KEY `utm_source` (`utm_source`),
KEY `utm_medium` (`utm_medium`),
KEY `utm_term` (`utm_term`),
KEY `utm_content` (`utm_content`),
KEY `created_at` (`created_at`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;""".format(project_name=project_name)
table_user_sql = """CREATE TABLE `{project_name}_user` (
`distinct_id` varchar(200) NOT NULL,
`lib` varchar(127) NOT NULL,
`map_id` varchar(200) NOT NULL,
`original_id` varchar(200) NOT NULL,
`user_id` varchar(255) DEFAULT NULL,
`all_user_profile` json DEFAULT NULL,
`created_at` int(11) DEFAULT NULL,
`updated_at` int(11) DEFAULT NULL,
PRIMARY KEY (`distinct_id`,`lib`,`map_id`,`original_id`),
KEY `distinct_id` (`distinct_id`),
KEY `map_id` (`map_id`),
KEY `original_id` (`original_id`),
KEY `distinct_id_lib` (`distinct_id`,`lib`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;""".format(project_name=project_name)
table_properties_sql = """CREATE TABLE `{project_name}_properties` (
`lib` varchar(255) NOT NULL,
`remark` varchar(255) NOT NULL,
`event` varchar(255) NOT NULL,
`properties` json DEFAULT NULL,
`properties_len` int(10) DEFAULT NULL,
`created_at` int(10) DEFAULT NULL,
`updated_at` int(10) DEFAULT NULL,
`lastinsert_at` int(10) DEFAULT NULL,
`total_count` bigint(20) DEFAULT NULL,
PRIMARY KEY (`lib`,`remark`,`event`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;;""".format(project_name=project_name)
do_tidb_exe(table_sql)
print(project_name+'table表单已插入')
do_tidb_exe(table_device_sql)
print(project_name+'device表单已插入')
do_tidb_exe(table_user_sql)
print(project_name+'user表单已插入')
do_tidb_exe(table_properties_sql)
print(project_name+'properties表单已插入')
sql_insert_status_code = """CREATE TABLE IF NOT EXISTS `status_code` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT 'id',
`desc` varchar(255) DEFAULT NULL COMMENT '含义',
`p_id` int(11) DEFAULT NULL COMMENT '父id',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1;"""
do_tidb_exe(sql_insert_status_code)
print('状态码表创建完')
status_codes = ["INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (1, '分群列表状态', 0);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (2, '创建列表开始', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (3, '分群信息写入中', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (4, '分群写入完成并包含错误', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (5, '分群写入完成', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (6, '分群写入失败', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (7, '生效策略', 0);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (8, '自动', 7);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (9, '手动', 7);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (10, '禁用', 7);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (11, '进入分群队列', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (12, '优先级', 0);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (13, '普通', 12);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (14, '高', 12);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (15, '最高', 12);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (16, '已添加任务队列', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (17, '任务已被选取', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (18, '任务方法加载完', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (19, '任务执行成功', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (20, '分群ETL失败', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (21, '任务执行失败', 1);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (22, '通知方式', 0);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (23, 'email', 22);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (24, '自动分群但不自动应用模板', 7);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (25, '推送状态', 0);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (26, '推送成功', 25);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (27, '推送失败', 25);","INSERT IGNORE INTO `events`.`status_code`(`id`, `desc`, `p_id`) VALUES (28, '自动分群自动应用模板但不自动发送', 7);"]
for code in status_codes:
do_tidb_exe(code)
print('状态码添加完毕')
sql_scheduler_jobs = """CREATE TABLE IF NOT EXISTS `scheduler_jobs` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT '任务id',
`project` varchar(255) DEFAULT NULL COMMENT '项目id',
`group_id` int(11) DEFAULT NULL COMMENT 'group_plan的id',
`list_index` int(11) DEFAULT NULL COMMENT 'group_index任务完成后，补充',
`datetime` int(11) DEFAULT NULL COMMENT '执行的日期，即要执行的那个任务的时间（不是任务执行时间，是要执行的时间。如周三时执行周一的任务。也用来防止任务重复添加）',
`data` json DEFAULT NULL COMMENT '其他附带的参数',
`priority` int(4) DEFAULT NULL COMMENT '优先级',
`status` int(4) DEFAULT NULL COMMENT '状态',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
`updated_at` int(11) DEFAULT NULL COMMENT '更新时间',
PRIMARY KEY (`id`),
UNIQUE KEY `ind_task` (`project`,`group_id`,`datetime`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1;"""
do_tidb_exe(sql_scheduler_jobs)
print('任务计划表添加完毕')
insert_data = """CREATE TABLE IF NOT EXISTS `{project_name}_usergroup_data` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`group_list_id` int(11) DEFAULT NULL COMMENT '分群列表id',
`data_index` int(11) DEFAULT NULL COMMENT '最新一组数据的index_id',
`data_key` varchar(255) DEFAULT NULL COMMENT '数据的唯一识别id',
`data_json` json DEFAULT NULL COMMENT '数据包',
`enable` int(11) DEFAULT NULL COMMENT '生效策略。参考status_code，p_id=7',
`created_at` int(11) DEFAULT NULL,
`updated_at` int(11) DEFAULT NULL,
PRIMARY KEY (`id`),
KEY `group_list_id` (`group_list_id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1;""".format(project_name=project_name)
do_tidb_exe(insert_data)
insert_list = """CREATE TABLE IF NOT EXISTS `{project_name}_usergroup_list` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT '分群列表id',
`group_id` int(11) DEFAULT NULL COMMENT '分群id',
`group_list_index` int(11) DEFAULT NULL COMMENT '分群列表顺位',
`list_init_date` int(11) DEFAULT NULL COMMENT '触发时间',
`list_desc` varchar(255) DEFAULT NULL COMMENT '清单所描述的',
`jobs_id` int(4) DEFAULT NULL COMMENT 'scheduler_jbos的id',
`item_count` int(11) DEFAULT NULL COMMENT '分组条目数',
`status` int(4) DEFAULT NULL COMMENT '分群状态。参考status_code,p_id=1',
`complete_at` int(11) DEFAULT NULL COMMENT '分群完成时间',
`apply_temple_times` int(2) DEFAULT 0 COMMENT '被套用模板的次数',
`created_at` int(11) DEFAULT NULL COMMENT '条目创建时间',
`updated_at` int(11) DEFAULT NULL COMMENT '条目更新时间',
PRIMARY KEY (`id`),
UNIQUE KEY `unique_key` (`group_id`,`group_list_index`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1;""".format(project_name=project_name)
do_tidb_exe(insert_list)
insert_plan = """CREATE TABLE IF NOT EXISTS `{project_name}_usergroup_plan` (
`id` int(11) NOT NULL AUTO_INCREMENT COMMENT '分群id',
`group_title` varchar(255) DEFAULT NULL COMMENT '分群标题',
`group_desc` varchar(255) DEFAULT NULL COMMENT '分群描述',
`func` json DEFAULT NULL COMMENT '分群执行方法参考/scheduler_jobs/scheduler_job_creator.py',
`latest_data_list_index` int(11) DEFAULT NULL COMMENT '最新一组数据的id',
`repeatable` varchar(20) DEFAULT NULL COMMENT '定时器，分，时，日，月，周。不填的用*代替。跟crontab一个逻辑，不支持1-10的方式表达，多日的需要1,2,3,4,5,6,7,8这样的形式填',
`priority` int(4) DEFAULT NULL COMMENT '任务执行优先级',
`latest_data_time` int(11) DEFAULT NULL COMMENT '最新一组数据的完成时间',
`repeat_times` int(11) DEFAULT 0 COMMENT '分群完成次数',
`enable_policy` int(11) DEFAULT NULL COMMENT '生效策略。参考status_code，p_id=7',
`latest_apply_temple_id` int(11) DEFAULT NULL COMMENT '最后一次执行的模板类型',
`latest_apply_temple_time` int(11) DEFAULT NULL COMMENT '最后一次执行的模型时间',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
`updated_at` int(11) DEFAULT NULL COMMENT '更新时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1;""".format(project_name=project_name)
do_tidb_exe(insert_plan)
print(project_name+'的分群附加表表已添加完')
do_tidb_exe(insert_list)
insert_noti = """CREATE TABLE IF NOT EXISTS `{project_name}_noti` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`plan_id` int(11) DEFAULT NULL COMMENT '计划id',
`list_id` int(11) DEFAULT NULL COMMENT '列表id',
`data_id` int(11) DEFAULT NULL COMMENT '数据id',
`temple_id` int(4) DEFAULT NULL COMMENT '模板id',
`noti_group_id` int(11) DEFAULT NULL COMMENT '消息群组id',
`distinct_id` varchar(512) DEFAULT NULL COMMENT '用户识别id',
`priority` int(4) DEFAULT NULL COMMENT '优先级',
`status` int(4) DEFAULT NULL COMMENT '状态',
`owner` varchar(255) DEFAULT NULL COMMENT '添加人',
`level` int(4) DEFAULT NULL COMMENT '消息级别',
`type` int(4) DEFAULT NULL COMMENT '消息类型',
`key` varchar(255) DEFAULT NULL COMMENT '消息接受方式key',
`content` json DEFAULT NULL COMMENT '消息内容',
`send_at` int(11) DEFAULT NULL COMMENT '计划发送时间',
`recall_result` text DEFAULT NULL COMMENT '发送结果',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
`updated_at` int(11) DEFAULT NULL COMMENT '更新时间',
PRIMARY KEY (`id`),
KEY `distinct_id` (`distinct_id`),
KEY `send_plan` (`status`,`priority`,`send_at`),
KEY `key` (`key`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1;""".format(project_name=project_name)
do_tidb_exe(insert_noti)
insert_noti_group = """CREATE TABLE IF NOT EXISTS `{project_name}_noti_group` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`plan_id` int(11) DEFAULT NULL COMMENT '分群计划id',
`list_id` int(11) DEFAULT NULL COMMENT '分群列表id',
`data_id` int(11) DEFAULT NULL COMMENT '分群数据id',
`temple_id` int(11) DEFAULT NULL COMMENT '应用模板id',
`priority` int(4) DEFAULT NULL COMMENT '优先级id',
`status` int(4) DEFAULT NULL COMMENT '状态id',
`owner` varchar(255) DEFAULT NULL COMMENT '添加人',
`send_at` int(11) DEFAULT NULL COMMENT '计划发送时间',
`sent` int(11) DEFAULT NULL COMMENT '已发送数目',
`total` int(11) DEFAULT NULL COMMENT '该计划总数目',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
`updated_at` int(11) DEFAULT NULL COMMENT '更新时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1001;""".format(project_name=project_name)
do_tidb_exe(insert_noti_group)
insert_noti_temple = """CREATE TABLE IF NOT EXISTS `{project_name}_noti_temple` (
`id` int(11) NOT NULL AUTO_INCREMENT,
`name` varchar(255) DEFAULT NULL COMMENT '模板名称',
`temple_desc` varchar(255) DEFAULT NULL COMMENT '模板描述',
`args` json DEFAULT NULL COMMENT '模板参数',
`content` json DEFAULT NULL COMMENT '模板内容',
`apply_times` int(11) DEFAULT 0 COMMENT '应用次数',
`lastest_apply_time` int(11) DEFAULT NULL COMMENT '最后一次应用时间',
`lastest_apply_list` int(11) DEFAULT NULL COMMENT '最后一次应用列表',
`created_at` int(11) DEFAULT NULL COMMENT '创建时间',
`updated_at` int(11) DEFAULT NULL COMMENT '更新时间',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin AUTO_INCREMENT=1001;""".format(project_name=project_name)
do_tidb_exe(insert_noti_temple)
print(project_name+'的消息附加表表已添加完')
if expired == None:
expired_at = 2147483647
else:
expired_at = int(time.mktime(time.strptime(expired, "%Y-%m-%d")))
timenow = int(time.time())
insert_project_list = """insert project_list (`project_name`,`created_at`,`expired_at`) values ('{project_name}',{created_at},{expired_at})""".format(project_name=project_name,created_at=timenow,expired_at=expired_at)
do_tidb_exe(insert_project_list)
print(project_name+'project列表已插入')
def ctime_to_datetime(self): self.datetime = datetime.datetime.utcfromtimestamp(time.mktime(time.strptime(self.ctime, self.struct))) return self.datetime
def main():
if (min_max_len('min') is None) and len(sys.argv) != 3:
print("\n Данных нет. Введите начальное " +
"и конечное значение для диапазона даты")
sys.exit()
elif len(sys.argv) == 1 and min_max_len('min') is not None:
print("\n Введите начальное и конечное значение даты")
sys.exit()
elif sys.argv[1] == "-mml":
print("\n Начальная дата (общ.): %s" %
datetime.utcfromtimestamp(min_max_len("min")))
print(" Конечная дата (общ.): %s" %
datetime.utcfromtimestamp(min_max_len("max")))
print(" Количесвто вопросов: %s" % min_max_len("len"))
sys.exit()
elif sys.argv[1] == "-help":
print_help()
# начало считывания времени выполнения работы цикла
time.clock()
start_date = 0
end_date = 0
if (len(sys.argv) == 2
and (sys.argv[1] != "-mml" or sys.argv[1] != "-help")):
start_date = datetime.utcfromtimestamp(min_max_len("max")).timetuple()
end_date = datetime.utcfromtimestamp(
calendar.timegm(time.strptime(sys.argv[1],
'%d.%m.%Y'))).timetuple()
# print(end_date)
mass = []
mass2 = []
for i in start_date:
mass.append(i)
start_date2 = mass[:6]
for i in end_date:
mass2.append(i)
end_date2 = mass2[:6]
# цикл с вызовом вопросов по дате выполняющийся
# по месяцам, с конечной даты
# по указаную в аргументах
while True:
# если месяц 12-ый, то увеличиваем
# год и начинаем с 1-го месяца
if start_date2[1] >= 11:
start_date2[0] += 1
start_date2[1] = 1
questions = recieve_questions(
min_max_len("max") + 1,
datetime(start_date2[0], start_date2[1], start_date2[2],
start_date2[3], start_date2[4], start_date2[5]))
# указываем если конечная приблизилась
# к указаной и равна или меньше на месяц,
# то мы выполняем извлечение по указаную дату
elif start_date2[0] == end_date2[0] and (
end_date2[1] - start_date2[1] == 1
or end_date2[1] == start_date2[1]):
questions = recieve_questions(
min_max_len("max") + 1,
datetime(end_date2[0], end_date2[1], end_date2[2],
end_date2[3], end_date2[4], end_date2[5]))
save_to_db(questions)
break
# в любом другом случае увеличиваем
# на месяц конечную дату
else:
start_date2[1] += 1
questions = recieve_questions(
min_max_len("max") + 1,
datetime(start_date2[0], start_date2[1], start_date2[2],
start_date2[3], start_date2[4], start_date2[5]))
save_to_db(questions)
# если мы указали в аргументах две даты
# начала и конца, то выполнить извлечение по этим датам
elif len(sys.argv) == 3:
dt = [int(i) for i in sys.argv[1].split('.')]
dt2 = [int(i) for i in sys.argv[2].split('.')]
questions = recieve_questions(datetime(dt[2], dt[1], dt[0]),
datetime(dt2[2], dt2[1], dt2[0]))
save_to_db(questions)
# логирование результатов
logger.info("-" * 80)
logger.info(" Начальная дата (общ.): %s" %
datetime.utcfromtimestamp(min_max_len("min")))
logger.info(" Конечная дата (общ.): %s" %
datetime.utcfromtimestamp(min_max_len("max")))
logger.info(" Всего количество вопросов: %s" % min_max_len("len"))
logger.info(" Время выпонения скрипта " + str(round(time.clock(), 2)) +
" сек. или " + str(round(time.clock() / 60, 2)) + " мин.")
logger.debug("\n")
curs.close()
conn.close()
def ctime_to_structtime(self): self.structtime = time.strptime(self.ctime, self.struct) return self.structtime
from brewer.models import Brewer
from deck.models import Deck, DeckCard
from card.models import Card
WIZARDS = "http://magic.wizards.com"
MTGO_LISTS = "/en/content/deck-lists-magic-online-products-game-info"
page = requests.get(WIZARDS + MTGO_LISTS)
tree = html.fromstring(page.content)
events = tree.xpath("//div[contains(@class,'article-item')]")
for event in events:
url = WIZARDS + event[0].attrib['href']
dates = event.xpath("//span[@class='date']")[0]
month = dates[0].text.strip()
date = str(dates[2].text).strip() + "-" + str(time.strptime(month.strip(), "%B").tm_mon) + "-" + str(dates[1].text).strip()
event_page = requests.get(url)
event_tree = html.fromstring(event_page.content)
decks = event_tree.xpath("//div[@class='deck-group']")
event_name = event_tree.xpath("//div[@id='main-content']/h1/text()")
if (Event.objects.filter(name=event_name, date=date).exists()):
print("It exists!")
break
for deck in decks:
deck_list = []
brewer = deck.xpath("span[@class='deck-meta']/h4/text()")[0]
deck_element = deck.xpath("div/div[@class='deck-list-text']/div[contains(@class, 'sorted-by-overview-container')]")[0]
sideboard_element = deck.xpath("div/div[@class='deck-list-text']/div[contains(@class, 'sorted-by-sideboard-container')]")
deck_subs = deck_element.xpath("div[contains(@class, 'element')]/span[@class='row']")
for card in deck_subs:
numberOf = card.xpath("span[@class='card-count']/text()")[0]
def timestamp_from_string(datestring, template='%Y-%m-%d'):
if datestring:
return calendar.timegm(time.strptime(datestring, '%Y-%m-%d'))
def ctime_to_timestamp(self): self.timestamp = time.mktime(time.strptime(self.ctime, self.struct)) return self.timestamp
def __init__(self):
self.my_tags = dict()
self.my_type = dict()
self.my_mver = dict()
self.my_models = list()
calculation_info = schema.get('calculation_info')
for item in os.listdir(BASEDIR):
if os.path.isdir(BASEDIR + item):
internaldir = os.listdir(BASEDIR + item)
if not 'version0001' in internaldir:
continue
mlabel = None
try:
f = open(BASEDIR + item + '/service-label.txt')
mlabel = f.readline()[:-1]
mtype = f.readline()[:-1]
f.close()
except:
continue
# model properties common to all model versions for this endpoint
rtype = schema.get("result_endpoint").schema
if mtype == 'qualitative':
rtype['properties']['value'] = {
"enum": ["positive", "negative"]
}
# only for exposed models (eTOXlab version > 0.91)
nmodels = 0
try:
f = open(BASEDIR + item + '/service-version.txt')
except:
continue
for line in f:
if line[-1] == '\n': line = line[:-1]
line_versions = line.split('\t')
## old syntax ( eTOXlab < 0.95 ) for backwards compatibility only
if len(line_versions) == 1:
try:
mver = int(line_versions[0]
) # internal (eTOXlab) model version
ever = 1 # external (eTOXsys) model version
except:
break
if not os.path.isdir(BASEDIR + item + '/version%0.4d' %
(mver)):
break
mid = 'eTOXvault ID ' + mlabel + ' ' + PARTNER_ID
## new API with version support
try:
new_model = calculation_info.create_object(
id=mlabel,
category="ENDPOINT",
version=str(ever),
external_id=mid)
except:
new_model = calculation_info.create_object(
id=mlabel,
category="ENDPOINT",
external_id=mid)
new_model['return_type_spec'] = rtype
self.my_models.append(new_model)
self.my_mver[mlabel, str(ever)] = mver
nmodels += 1
break
try:
mver = int(line_versions[0]
) # internal (dir tree ) model version
ever = int(line_versions[1]
) # external (user defined) model version
except:
continue
if ever == 0: # this model has not been exposed
continue
if not os.path.isdir(BASEDIR + item +
'/version%0.4d' % mver
): # make sure this version exists
continue
mid = 'eTOXvault ID ' + mlabel + ' ' + PARTNER_ID
## new API with version support
try:
new_model = calculation_info.create_object(
id=mlabel,
category="ENDPOINT",
version=str(ever),
external_id=mid)
except:
new_model = calculation_info.create_object(
id=mlabel, category="ENDPOINT", external_id=mid)
new_model['return_type_spec'] = rtype
# read licensing information from a file at the endpoint root
if os.path.isfile(BASEDIR + item +
'/licensing-status.txt'):
flic = open(BASEDIR + item + '/licensing-status.txt',
'r')
licenses = []
for licline in flic:
licline = licline.rstrip()
if len(licline) > 10:
liclist = licline.split('\t')
if len(liclist) == 2:
lic_info = {
'license_end':
time.mktime(
time.strptime(
liclist[1], "%d-%b-%Y")),
'license_info':
liclist[0]
}
licenses.append(lic_info)
flic.close()
if len(licenses) > 0:
new_model['license_infos'] = []
for li in licenses:
new_model['license_infos'].append(li)
# append the new model to the list of models
self.my_models.append(new_model)
self.my_mver[mlabel, str(ever)] = mver
nmodels += 1
f.close()
if nmodels == 0: # do not add item and mtype unless there is any published model
continue
# my_program lists the endpoint tags (e.g. CACO2, ABCB1)
self.my_tags[mlabel] = item
self.my_type[mlabel] = mtype
def getFlashDateString():
try:
return time.strftime(_("%Y-%m-%d %H:%M"), time.strptime(open("/etc/version").read().strip(), '%Y%m%d%H%M'))
except:
return _("unknown")
def storeNewCups():
coffees = []
# import pre-booking configurations
if (os.path.isfile(recipeFile) == True):
sweets = "INFO: task-importing operation started"
sweetsOut(sweets)
coffeeBeans = "INFO: task-importing operation started"
beansOut(coffeeBeans)
recipeFromFile = open(recipeFile)
# json file is easily wrong in format, so placed the exception handler
try:
toImportTasks = json.load(recipeFromFile)
except ValueError:
sweets = "ERROR: task-importing operation failed for invalid recipe file, operation canceled"
sweetsOut(sweets)
coffeeBeans = "ERROR: task-importing operation failed for invalid recipe file, operation canceled"
beansOut(coffeeBeans)
else:
for toImportTaskIndex, toImportTask in enumerate(toImportTasks):
toImportTaskTriggerAt = toImportTask['triggerAt']
toImportTaskTriggerAtTimeStructure = time.strptime(
toImportTaskTriggerAt, "%Y-%m-%d %H:%M:%S")
toImportTaskId = time.strftime(
"%Y%m%d%H%M%S", toImportTaskTriggerAtTimeStructure)
toImportTask['taskId'] = toImportTaskId
toImportTask['attemptedTimes'] = 0
coffeeBeans = "INFO: imported task with id [" + toImportTask[
'taskId'] + "], details: " + json.dumps(toImportTask,
ensure_ascii=False)
beansOut(coffeeBeans)
coffees.append(toImportTask)
if (os.path.exists(shelfDirectory) == False):
os.system("mkdir " + shelfDirectory)
for coffee in coffees:
taskFileName = coffee['taskId']
os.system("touch " + shelfDirectory + taskFileName)
coffee = json.dumps(coffee, ensure_ascii=False)
coffeeFile = open(shelfDirectory + taskFileName, 'a')
contentLineToWrite = str(coffee)
coffeeFile.write(coffee)
coffeeFile.close()
os.system('rm ' + recipeFile)
sweets = "INFO: task imported successfully"
sweetsOut(sweets)
coffeeBeans = "INFO: task imported successfully"
beansOut(coffeeBeans)
return True
def convert_time_to_sec(time_string='0:0:0'):
x = time.strptime(time_string,'%H:%M:%S')
return datetime.timedelta(hours=x.tm_hour,minutes=x.tm_min,seconds=x.tm_sec).total_seconds()
def srmeta(indir, mfile, errorlog, folder):
path, dirs, files = next(os.walk(folder))
file_count = len(files)
i = 1
with open(mfile, 'wb') as csvfile:
writer = csv.DictWriter(
csvfile,
fieldnames=[
"id_no", "system:time_start", "product_type", "orbit",
"provider", "instrument", "satellite_id", "number_of_bands",
"epsg_code", "resampling_kernel", "number_of_rows",
"number_of_columns", "gsd", "cloud_cover", "incidence_angle",
"sun_azimuth", "sun_elevation", "azimuth_angle",
"spacecraft_angle", "atmospheric_model", "aerosol_Model",
"aot_method", "aot_std", "aot_used", "aot_Status",
"aot_mean_quality", "luts_version", "aot_coverage",
"aot_source", "atmospheric_correction_algorithm"
],
delimiter=',')
writer.writeheader()
for filename in os.listdir(indir):
if filename.endswith(".tif"):
infilename = os.path.join(
folder, filename.replace("SR.tif", "metadata.xml"))
try:
xmldoc = minidom.parse(infilename)
ps4band = xmldoc.getElementsByTagName(
'ps:EarthObservationMetaData')[0]
eopfilename = xmldoc.getElementsByTagName(
'eop:identifier')[0].firstChild.data
productType = xmldoc.getElementsByTagName(
'eop:productType')[0].firstChild.data
orbit = xmldoc.getElementsByTagName(
'eop:orbitType')[0].firstChild.data
acquisition = xmldoc.getElementsByTagName(
'eop:acquisitionDate')[0].firstChild.data
provider = xmldoc.getElementsByTagName(
"eop:shortName")[0].firstChild.data
instrument = xmldoc.getElementsByTagName(
"eop:shortName")[1].firstChild.data
satellite_id = xmldoc.getElementsByTagName(
"eop:serialIdentifier")[0].firstChild.data
bands = xmldoc.getElementsByTagName(
"ps:numBands")[0].firstChild.data
epsg_code = xmldoc.getElementsByTagName(
"ps:epsgCode")[0].firstChild.data
resampling_kernel = xmldoc.getElementsByTagName(
"ps:resamplingKernel")[0].firstChild.data
number_rows = xmldoc.getElementsByTagName(
"ps:numRows")[0].firstChild.data
number_columns = xmldoc.getElementsByTagName(
"ps:numColumns")[0].firstChild.data
gsd = xmldoc.getElementsByTagName(
"ps:rowGsd")[0].firstChild.data
cloud = xmldoc.getElementsByTagName(
"opt:cloudCoverPercentage")[0].firstChild.data
psb = xmldoc.getElementsByTagName(
"ps:bandNumber")[0].firstChild.data
psb1 = xmldoc.getElementsByTagName(
"ps:bandNumber")[1].firstChild.data
psb3 = xmldoc.getElementsByTagName(
"ps:bandNumber")[2].firstChild.data
psb4 = xmldoc.getElementsByTagName(
"ps:bandNumber")[3].firstChild.data
psia = xmldoc.getElementsByTagName(
"eop:incidenceAngle")[0].firstChild.data
psilaz = xmldoc.getElementsByTagName(
"opt:illuminationAzimuthAngle")[0].firstChild.data
psilelv = xmldoc.getElementsByTagName(
"opt:illuminationElevationAngle")[0].firstChild.data
psaz = xmldoc.getElementsByTagName(
"ps:azimuthAngle")[0].firstChild.data
pssca = xmldoc.getElementsByTagName(
"ps:spaceCraftViewAngle")[0].firstChild.data
date_time = acquisition.split('T')[0]
pattern = '%Y-%m-%d'
epoch = int(time.mktime(time.strptime(date_time,
pattern))) * 1000
print("Processing " + str(i) + " of " + str(file_count))
i = i + 1
gtif = gdal.Open(os.path.join(indir, filename))
date_time = gtif.GetMetadata()['TIFFTAG_DATETIME'].split(
" ")[0]
pattern = '%Y:%m:%d'
epoch = int(time.mktime(time.strptime(date_time,
pattern))) * 1000
conv = json.loads(
gtif.GetMetadata()['TIFFTAG_IMAGEDESCRIPTION'])
sid = str(filename).split("_")[2]
atmmodel = (
conv['atmospheric_correction']['atmospheric_model'])
aotmethod = (conv['atmospheric_correction']['aot_method'])
aotused = (conv['atmospheric_correction']['aot_used'])
aotstat = (conv['atmospheric_correction']['aot_status'])
aotstd = (conv['atmospheric_correction']['aot_std'])
aotmq = (conv['atmospheric_correction']['aot_mean_quality'])
luts = (conv['atmospheric_correction']['luts_version'])
aotcov = (conv['atmospheric_correction']['aot_coverage'])
arsm = (conv['atmospheric_correction']['aerosol_model'])
aotsor = (conv['atmospheric_correction']['aot_source'])
atcoralgo = (conv['atmospheric_correction']
['atmospheric_correction_algorithm'])
with open(mfile, 'a') as csvfile:
writer = csv.writer(csvfile,
delimiter=',',
lineterminator='\n')
writer.writerow([
filename.split('.')[0], epoch, productType, orbit,
provider, instrument, satellite_id, bands, epsg_code,
resampling_kernel, number_rows, number_columns,
format(float(gsd), '.2f'),
format(float(cloud), '.2f'),
format(float(psia), '.4f'),
format(float(psilaz), '.2f'),
format(float(psilelv), '.2f'),
format(float(psaz), '.2f'),
format(float(pssca), '.4f'),
str(atmmodel),
str(arsm),
str(aotmethod),
format(float(aotstd), '.4f'),
format(float(aotused), '.4f'),
str(aotstat), aotmq, luts,
format(float(aotcov), '.4f'),
str(aotsor),
str(atcoralgo)
])
csvfile.close()
except Exception as e:
print(e)
print("Issues with : " + str(os.path.splitext(filename)[0]))
with open(errorlog, 'a') as csvfile:
writer = csv.writer(csvfile,
delimiter=',',
lineterminator='\n')
writer.writerow([filename])
csvfile.close()
def _jinja2_filter_datetime(date, fmt=None):
pyDate = time.strptime(date,'%a %b %d %H:%M:%S +0000 %Y') # convert twitter date string into python date/time
return time.strftime('%Y-%m-%d %H:%M:%S', pyDate) # return the formatted date.
def trans_timestamp(date_str):
time_array = time.strptime(date_str, "%Y_%m_%d")
timestamp = time.mktime(time_array)
return int(timestamp)
currentDate = time.strftime('%Y-%m-%d', time.localtime(time.time()))
currentTime = time.strftime('%H:%M:%S', time.localtime(time.time()))
currentTimestamp = int(time.time())
# first, see whether there are tasks to do
sweets = "INFO: checking task queue for operation"
sweetsOut(sweets)
coffees = fetchStoredCoffeesList()
if (fetchStoredCoffeesList() != []):
for index, coffeeId in enumerate(coffees):
taskTriggerAt = coffeeId
taskTriggerAtTimeStructure = time.strptime(
taskTriggerAt, "%Y%m%d%H%M%S")
taskTriggerAtTimeStamp = time.mktime(
taskTriggerAtTimeStructure)
if (int(currentTimestamp) - int(taskTriggerAtTimeStamp) < 0):
sweets = "INFO: task with id [" + str(
coffeeId) + "] is currently waiting to be triggered"
sweetsOut(sweets)
else:
coffee = fetchStoredCoffee(coffeeId)
attemptedTimes = coffee['attemptedTimes']
expectedTimeStructure = time.strptime(
coffee['triggerAt'], "%Y-%m-%d %H:%M:%S")
expectedTimeStamp = int(time.mktime(expectedTimeStructure))
reload(sys)
sys.setdefaultencoding('utf-8')
import json
import datetime
import time
from rec_driver import *
from pymysql import PyMysql
time_now = int(time.time())
time_value = time.localtime(time_now)
time_year_str = time.strftime('%Y', time_value)
time_day_str = time.strftime('%Y-%m-%d', time_value)
time_day_str_all = time_day_str + " 00:00:00"
time_s = time.mktime(time.strptime(time_day_str_all, '%Y-%m-%d %H:%M:%S'))
time_day = int(time_s)
time_yesterday = time_day - 86400
time_value_yesterday = time.localtime(time_yesterday)
time_yesterday_str = time.strftime('%Y-%m-%d', time_value_yesterday)
time_yesterday_str_all = time_yesterday_str + " 00:00:00"
time_days_before = time_day - (15 * 86400)
time_value_days_before = time.localtime(time_days_before)
time_days_before_str = time.strftime('%Y-%m-%d', time_value_days_before)
time_days_before_str_all = time_days_before_str + " 00:00:00"
# print time_now
print 'start'
print time_day_str_all
def setFirstWeekDate(firstWeekDate):
global DONE_firstWeekDate
DONE_firstWeekDate = time.strptime(firstWeekDate,'%Y%m%d')
print("Now running: setFirstWeekDate():", DONE_firstWeekDate)
