def generatebeginTimecode(begin_time_list):
beginTimecode_list=list()
for i in range(0, len(begin_time_list)):
datetime = begin_time_list[i].split(' ')[0]
time = begin_time_list[i].split(' ')[1]
year = int(datetime.split('-')[0])
month = int(datetime.split('-')[1])
day = int(datetime.split('-')[2])
hour = int(time.split(':')[0])
minute = int(time.split(':')[1])
second ='00'
beginTimecode = str(year)+str('%02d'%month)+str('%02d'%day)+str('%02d'%hour)+str('%02d'%minute)+second
beginTimecode_list.append(beginTimecode)
return beginTimecode_list
def formatDateTime(datetime):
'''
datetime must be inputted exactly like the following example:
12 March 2012 15:30
and return it as R3339 format:
2012-9-6T10:25:00-07:00
Return None if the time is not valid
'''
ar = datetime.split()
# validate month
try:
month = str(convertMonth(ar[1]))
except MonthException:
print '%s is not a valid month' % ar[1]
return None
d = str(ar[2]) + '-' + month + '-' + str(ar[0])
if len(ar) == 3:
return d + 'T00:00:00' + TIMEZONE
else:
t = str(ar[3]) + ':' + '00' + TIMEZONE
dt = d + 'T' + t
return dt
def convert_tags_to_track(tags):
"""Convert our normalized tags to a track.
:param tags: dictionary of tag keys with a list of values
:type tags: :class:`dict`
:rtype: :class:`mopidy.models.Track`
"""
album_kwargs = {}
track_kwargs = {}
track_kwargs['composers'] = _artists(tags, Gst.TAG_COMPOSER)
track_kwargs['performers'] = _artists(tags, Gst.TAG_PERFORMER)
track_kwargs['artists'] = _artists(tags, Gst.TAG_ARTIST,
'musicbrainz-artistid',
'musicbrainz-sortname')
album_kwargs['artists'] = _artists(
tags, Gst.TAG_ALBUM_ARTIST, 'musicbrainz-albumartistid')
track_kwargs['genre'] = '; '.join(tags.get(Gst.TAG_GENRE, []))
track_kwargs['name'] = '; '.join(tags.get(Gst.TAG_TITLE, []))
if not track_kwargs['name']:
track_kwargs['name'] = '; '.join(tags.get(Gst.TAG_ORGANIZATION, []))
track_kwargs['comment'] = '; '.join(tags.get('comment', []))
if not track_kwargs['comment']:
track_kwargs['comment'] = '; '.join(tags.get(Gst.TAG_LOCATION, []))
if not track_kwargs['comment']:
track_kwargs['comment'] = '; '.join(tags.get(Gst.TAG_COPYRIGHT, []))
track_kwargs['track_no'] = tags.get(Gst.TAG_TRACK_NUMBER, [None])[0]
track_kwargs['disc_no'] = tags.get(Gst.TAG_ALBUM_VOLUME_NUMBER, [None])[0]
track_kwargs['bitrate'] = tags.get(Gst.TAG_BITRATE, [None])[0]
track_kwargs['musicbrainz_id'] = tags.get('musicbrainz-trackid', [None])[0]
album_kwargs['name'] = tags.get(Gst.TAG_ALBUM, [None])[0]
album_kwargs['num_tracks'] = tags.get(Gst.TAG_TRACK_COUNT, [None])[0]
album_kwargs['num_discs'] = tags.get(Gst.TAG_ALBUM_VOLUME_COUNT, [None])[0]
album_kwargs['musicbrainz_id'] = tags.get('musicbrainz-albumid', [None])[0]
album_kwargs['date'] = tags.get(Gst.TAG_DATE, [None])[0]
if not album_kwargs['date']:
datetime = tags.get(Gst.TAG_DATE_TIME, [None])[0]
if datetime is not None:
album_kwargs['date'] = datetime.split('T')[0]
# Clear out any empty values we found
track_kwargs = {k: v for k, v in track_kwargs.items() if v}
album_kwargs = {k: v for k, v in album_kwargs.items() if v}
# Only bother with album if we have a name to show.
if album_kwargs.get('name'):
track_kwargs['album'] = Album(**album_kwargs)
return Track(**track_kwargs)
def datetimeformat(datetime):
datetime = str(datetime)
splitDT = datetime.split(" ")
if len(splitDT) > 1:
date = splitDT[0]
splitDT = splitDT[1].split(".")
splitDT = splitDT[0].split(":")
time = splitDT[0] + ":" + splitDT[1]
datetimedict = {'date' : date, 'time' : time}
return datetimedict
else:
return {'date' : '', 'time' : ''}
def _set_alarm_clock(self, request):
entities = request['entities']
context = request['context']
if not self._confident(entities):
self.ai_message = self.nlg.clueless()
return context
datetime = self._first_entity_value(entities, 'datetime')
context = {}
if not datetime:
context['alarm_time_missing'] = self.nlg.alarm_info(None, None)
print "Alarm: %s" % (context)
return context
date = datetime.split("T")[0]
time = ":".join(datetime.split("T")[1].split(".")[0].split(":")[:-1])
self._active_alarms.append({"date": date, "time": time})
print "date: %s, time: %s" % (date, time)
context['alarm_confirmation'] = self.nlg.alarm_info(date, time)
print "Alarm: %s" % (context)
return context
def extract_game_info(self, driver, timestamp):
# Messy splitting of team names.
teams = self.wait_pres(
driver, '//*[@id="betting-odds"]/div/section/div/header/h1').text
home_team = teams.split(' v ')[0].replace(" ", "")
time.sleep(0.5 * uniform(0, 1))
container = teams.split(' v ')[1]
away_team = container.split(' Winner')[0].replace(" ", "")
time.sleep(0.5 * uniform(0, 1))
# Messy splitting of date & time
datetime = self.wait_pres(
driver,
'//*[@id="betting-odds"]/div/section/div/div/div/div/span').text
day = datetime.split(' / ')[0]
clock = datetime.split(' / ')[1]
# Build identifier variable
day_ = day.split(' ')[1][:2]
month = day.split(' ')[2][:3]
h_team = home_team[:5]
a_team = away_team[:5]
identifier = h_team + '_' + a_team + '_' + day_ + '_' + month
# save the info in list
game_info = [home_team, away_team, day, clock, identifier, timestamp]
return game_info
def parse_datetime_to_parts(datetime):
'''
parses date time from the twitter api into parts
'''
#
datetime_parts = []
#
[date, time] = datetime.split(" ")
#
datetime_parts.extend(date.split("-"))
datetime_parts.extend(time.split(":")[:2])
datetime_parts.extend(time.split(":")[2].split("."))
#
return datetime_parts
def minimalPatientResponseDict(p):
responsedict = {}
responsedict["pid"] = p.pid
responsedict["first_name"] = p.first_name
responsedict["last_name"] = p.last_name
#Patient labels
responsedict["labels"] = [
lbl.label_name for lbl in p.labels_for_patient.all()
]
#conditions needed for patient
if len(p.conditions_for_patient.all()) > 0:
responsedict["condition_name"] = p.conditions_for_patient.all(
)[0].condition_name
responsedict["condition_code"] = p.conditions_for_patient.all(
)[0].condition_code
responsedict["condition_desc"] = p.conditions_for_patient.all(
)[0].condition_desc
#observations needed for patient
all_obs = p.obs_for_patients.all().order_by('-obs_date')
if len(all_obs) > 0:
responsedict["latest_observation"] = all_obs[0].obs_desc
datetime = str(all_obs[0].obs_date)
datetime = datetime.split(' ')
responsedict["latest_observation_date"] = datetime[0]
responsedict["latest_observation_time"] = datetime[1]
responsedict["latest_observation_name"] = all_obs[0].obs_name
all_meds = p.medications_for_patient.filter(med_status='active')
all_info_of_med = []
for meds in all_meds:
tempdict = {}
tempdict["medication_name"] = meds.med_name
tempdict["medication_code"] = meds.med_code
tempdict["medication_value"] = meds.med_dosage_value
tempdict["medication_units"] = meds.med_dosage_units
tempdict["medication_text"] = meds.med_dosage_text
tempdict["medication_date"] = meds.med_date_written
all_info_of_med.append(tempdict)
responsedict["active_medication"] = all_info_of_med
return responsedict
def _get_events(self, request):
entities = request['entities']
context = {}
if not self._confident(entities):
self.ai_message = self.nlg.clueless()
return context
datetime = self._first_entity_value(entities, 'datetime')
if not datetime:
context['events_missing'] = "What date do you want me to check?"
return context
date = datetime.split("T")[0]
events = self.calendar.get_events(date)
context['events'] = self.nlg.events(events)
return context
def format_ncsacombinedlog(log):
global _disableColoring
global _begin_time, _end_time
global _filename_searching_only
try:
host, id, username, datetime, tz, method, uri, version, statuscode, bytes, combined = log.split(
' ', 10)
empty, logdatetime = datetime.split('[', 1)
logtime, error = convert_datetime(logdatetime, True, False)
if error: pass
else:
if _begin_time:
if _begin_time <= logtime: pass
else: return False, '', False
if _end_time:
if logtime < _end_time: pass
else: return False, '', False
except Exception as e:
return False, log, True
if _filename_searching_only:
return True, '', False
if _disableColoring:
return True, ' '.join([
host, id, username, datetime, tz, method, uri, version, statuscode,
bytes, combined
]), False
else:
datetimetz = datetime + " " + tz
datetimetz = Colors['date'] + datetimetz + Colors['endc']
request = method + " " + uri + " " + version
request = Colors['green'] + request + Colors['endc']
statuscode = Colors['blue'] + statuscode + Colors['endc']
return True, '%s %s %s %s %s %s %s %s' % (
host, id, username, datetimetz, request, statuscode, bytes,
combined), False
def date_conversion(datetime):
datetimeList = datetime.split(' ')
if datetimeList.__len__()==0:
return None
date = datetimeList[0]
time = ""
if datetimeList.__len__() == 2:
time = datetimeList[1]
'''将中文日期转换成自己想要的格式如：2018年09月29日转换成2018-09-29'''
ls=re.findall(r'(.*)年(.*)月(.*)日', date)
if ls.__len__()==0:
return None
c = list(ls[0])
new_date = ''
# print(c)
if len(c[1]) < 2:
c[1] = '0' + c[1]
# print('-'.join(c))
new_date = '-'.join(c)
else:
# print('-'.join(c))
new_date = '-'.join(c)
return new_date + ' ' + time
# print(date_conversion('2018年09月29日'))
# str=date_conversion('2018年09月29日')
# d=datetime.datetime.strptime('2018-10-15 20:00:00', "%Y-%m-%d %H:%M:%S")
# dd=datetime.datetime.now()+datetime.timedelta(days=1)
# print(dd)
# print(d)
# print('2018-10-15 10:00:00'>'2018-10-15 10:00:00')
# str='2018-10-15'
# d=datetime.datetime.strptime(str,"%Y-%m-%d")+datetime.timedelta(days=1)
# print(d)
# dd=datetime.datetime.now()
# print(dd)
# print(dd>d)
def getDateObject(self, datetime):
print datetime
datetimetokens = datetime.split(',')
monthdatetoken = str(datetimetokens[1]).strip()
monthdatetokenSpaceCut = monthdatetoken.replace(' ', '')
monthdatetokenSpaceCutDigitCount = self.hasNumbers(
monthdatetokenSpaceCut)
yeartoken = str(datetimetokens[2]).strip()
#teo digit day is considered from 10th to 31st
if (monthdatetokenSpaceCutDigitCount == 2):
month = monthdatetokenSpaceCut[:-2]
day = monthdatetokenSpaceCut[-2:]
elif (monthdatetokenSpaceCutDigitCount == 1):
month = monthdatetokenSpaceCut[:-1]
day = monthdatetokenSpaceCut[-1:]
datestring = month + " " + day + " " + yeartoken
datestring = datestring.encode("utf-8")
#print datestring
from datetime import datetime
date_object = datetime.strptime(datestring, '%B %d %Y')
return date_object
def isColdWarm(self, row):
dt = row['date']
# get month, day
fields = dt.split("/")
if (len(fields) < 3):
return row
month = int(fields[0])
day = int(fields[1])
if (month >= 4) and (month <= 10):
if ((month == 4) and (day < 15)):
row['isCold'] = True
return row
if ((month == 10) and (day > 14)):
row['isCold'] = True
return row
row['isWarm'] = True
return row
else:
row['isCold'] = True
return row
def minimalPatientResponseDict(p):
responsedict = {}
responsedict["pid"]=p.pid
responsedict["first_name"]=p.first_name
responsedict["last_name"]=p.last_name
#Patient labels
responsedict["labels"] = [lbl.label_name for lbl in p.labels_for_patient.all()]
#conditions needed for patient
if len(p.conditions_for_patient.all()) > 0:
responsedict["condition_name"] = p.conditions_for_patient.all()[0].condition_name
responsedict["condition_code"] = p.conditions_for_patient.all()[0].condition_code
responsedict["condition_desc"] = p.conditions_for_patient.all()[0].condition_desc
#observations needed for patient
all_obs = p.obs_for_patients.all().order_by('-obs_date')
if len(all_obs) > 0:
responsedict["latest_observation"] = all_obs[0].obs_desc
datetime = str(all_obs[0].obs_date)
datetime = datetime.split(' ')
responsedict["latest_observation_date"] = datetime[0]
responsedict["latest_observation_time"] = datetime[1]
responsedict["latest_observation_name"] = all_obs[0].obs_name
all_meds = p.medications_for_patient.filter(med_status='active')
all_info_of_med = []
for meds in all_meds:
tempdict = {}
tempdict["medication_name"] = meds.med_name
tempdict["medication_code"] = meds.med_code
tempdict["medication_value"] = meds.med_dosage_value
tempdict["medication_units"] = meds.med_dosage_units
tempdict["medication_text"] = meds.med_dosage_text
tempdict["medication_date"] = meds.med_date_written
all_info_of_med.append(tempdict)
responsedict["active_medication"] = all_info_of_med
return responsedict
def mentioned_trend(baseurl,
mysqlhostIP,
mysqlUserName='******',
mysqlPassword='',
dbname='btv_v2'):
list_key_words = list()
box_from = list()
inter = 1
now = int(time.time()) - 86400 * inter
timeArray = time.localtime(now)
otherStyleTime = time.strftime("%Y-%m-%d", timeArray)
print otherStyleTime
# 存储评论数据
# 连接数据库
print(
base64.b64decode(
b'Q29weXJpZ2h0IChjKSAyMDEyIERvdWN1YmUgSW5jLiBBbGwgcmlnaHRzIHJlc2VydmVkLg=='
).decode())
sqlConn = MySQLdb.connect(host=mysqlhostIP,
user=mysqlUserName,
passwd=mysqlPassword,
db=dbname,
charset='utf8')
sqlcursor = sqlConn.cursor()
sqlcursor.execute(
'''CREATE TABLE IF NOT EXISTS key_kk(pk bigint NOT NULL PRIMARY KEY AUTO_INCREMENT, keywords varchar(50)) DEFAULT CHARSET=utf8;'''
)
print '新建库成功'
os.popen('kinit -k -t /home/ctvit/ctvit.keytab ctvit')
kerberos_auth = HTTPKerberosAuth(mutual_authentication=OPTIONAL)
tablename = "DATA:NEWS_Content"
r = requests.get(baseurl + "/" + tablename + "/*",
auth=kerberos_auth,
headers={"Accept": "application/json"})
if issuccessful(r) == False:
print "Could not get messages from HBase. Text was:\n" + r.text
# quit()
bleats = json.loads(r.text)
box = list()
ind = 1
box = list()
for row in bleats['Row']:
# print 000
# count+=1
message = ''
lineNumber = ''
username = ''
type_0 = ''
title = ''
flag = False
flag_t = False
flag_type = False
for cell in row['Cell']:
columnname = base64.b64decode(cell['column'])
value = cell['$']
if value == None:
print 'none'
continue
# if columnname == "base_info:news_from":
# from_f = base64.b64decode(value)
# from_f = re.sub('[^a-zA-Z]','',from_f)
# # print 'hh',from_f,len(from_f)
# if len(from_f) > 1:
# if ('Jane' not in from_f) and ():
# if from_f not in box_from:
# box_from.append(from_f)
# for i in box_from:
# print i
# if 'Jane' not in from_f:
# flag = False
# break
# if columnname == "base_info:type":
# type_0 = base64.b64decode(value)
# if '军事' in type_0:
# flag_t = True
# else:
# break
#
# # if 'anes' not in from_f:
# # flag = False
# # break
if columnname == "base_info:title":
title = base64.b64decode(value)
# # print 'title',title
# if '5月大税改' not in title:
# flag = False
# break
if columnname == "base_info:datetime":
datetime = base64.b64decode(value)
transform_date_time = datetime.split(' ')[0]
# print 'hhh',transform_date_time,otherStyleTime, otherStyleTime
# if transform_date_time >= otherStyleTime:
#
if columnname == "base_info:website":
url = base64.b64decode(value)
if columnname == "base_info:type":
type_0 = base64.b64decode(value)
# if len(type_0) < 1:
# print 'type is blank'
# 不能直接'军事'in type,
# if (type_0 == '军事') or (type_0 == '新浪军事') or (type_0 == '搜狐军事') or (type_0 == '滚动军事') or (type_0 == '凤凰军事') or (type_0 == '军事要闻'):
# # if (type_0 == '军事'):
# flag_type = True
# else:
# break
#
#
#
# if flag and flag_t:
# rowKey = base64.b64decode(row['key'])
# print '00', type_0,rowKey
#
# print title
# ind += 1
# if columnname == "base_info:type":
# type = base64.b64decode(value)
# print 'type',type
# if key_word not in list_key_words:
# list_key_words.append(key_word)
# if flag and flag_type:
if flag:
print 1111
# print transform_date_time,datetime,otherStyleTime, len(datetime),title,url,type_0
# print 'datetime',datetime, type(datetime),len(str(datetime)),str(datetime)[11:19]
# tempData = []
# for i in list_key_words:
# print 'key',i
# tempData.append(str(i))
# sqlcursor.execute('''insert into key_kk(keywords) values (%s)''',tempData)
# sqlConn.commit()
# tempData = []
sqlConn.close()
def datetime_to_time_in_minutes(datetime):
time = datetime.split(" ")[1]
times = time.split(":")
return float(times[0] * 60 + times[1])
def str_to_date(dt):
year, month, day = (int(x) for x in dt.split('-'))
return datetime.date(year, month, day)
def getHour(datetime):
#Input Format: '2015-xx-xx xx:xx:xx'
#Returns the same format but rounded down to the hour
return datetime.split(":")[0] + ":00:00"
def hi_scrap(type_name, table_name, news_cat, news_cat_id, flag=0):
print("scraping started for : ", news_cat)
story_link = []
img_link = []
story_title = []
story = []
news_date = []
sections = []
news_category = []
news_time = []
cat_id = []
links = []
for i in range(1, 5):
html = requests.get('https://www.bhaskar.com/' + type_name + '/' +
str(i),
headers=headers)
# print('https://www.bhaskar.com/'+type_name+'/'+str(i))
page_soup = BeautifulSoup(html.content, 'html.parser')
sections.extend(
page_soup.find_all('section', {'class': 'lpage_bottom_data'}))
for section in sections:
if flag == 1:
STORY_LINK = section.div.a['href']
# story_link.append( )
else:
STORY_LINK = 'https://www.bhaskar.com' + section.div.a['href']
# story_link.append( )
links.append(STORY_LINK)
# print(len(links))
for link in links:
STORY_LINK = link
IMAGE_LINK = None
STORY_TITLE = None
NEWSDATE = None
STORY = None
NEWSTIME = None
try:
# print(link)
html = requests.get(link, headers=headers)
page_soup = BeautifulSoup(html.content, 'html.parser')
# print(link)
try:
image = page_soup.find('div', {
'class': 'db_storyimg openPopup'
}).img
if image == None:
image = page_soup.find('figure', {
'class': 'norfigure'
}).img
IMAGE_LINK = image['src']
# img_link.append()
except:
pass
div = page_soup.find('div', {'class': 'db_storycontent'})
STORY = div.p.text
# story.append()
title = page_soup.find('section', {'class': 'db_storybox'}).h1
STORY_TITLE = title.text
# story_title.append()
datetime = str(page_soup.find('div', {'class': 'db_storytime'}))
# print(datetime)
try:
datetime = datetime.split('</h4>')[1]
except:
datetime = datetime.split('db_storytime">')[1]
datetime = datetime.split(',')
time = datetime[2].split('IST')
NEWSTIME = time[0].strip()
datetime = [i.lstrip().rstrip() for i in datetime]
temp = datetime[0].split(' ')
# print("temp[0]:",temp[0])
temp_dic = {
'Jan': '01',
'Feb': '02',
'Mar': '03',
'Apr': '04',
'May': '05',
'June': '06',
'Jun': '06',
'Jul': '07',
'Aug': '08',
'Sep': '09',
'Oct': '10',
'Nov': '11',
'Dec': '12'
}
if temp[0] in temp_dic:
temp[0] = temp_dic.get(temp[0])
else:
print("Error in date ,something wrong line:219 ")
# print(temp)
tdate = datetime[1] + "-" + str(temp[0]) + "-" + str(
temp[1]) #required for current date matching
# print(date,time)
# news_date.append(date)
# news_time.append(time)
except Exception as e:
print(logging.exception(e))
try:
# print("______________________________________\n")
# print("STORY:",STORY)
# print("STORY_LINK:",STORY_LINK)
# print("STORY_TITLE:",STORY_TITLE)
# print("IMAGE_LINK:",IMAGE_LINK)
# print("NEWSDATE:",NEWSDATE)
# print("NEWSTIME:",NEWSTIME)
# print("news_category:",news_cat)
# print("news_cat_id:",news_cat_id)
if tdate == str(date.today()):
temp = str(temp[1]) + "-" + str(temp[0])
NEWSDATE = str(temp) + "-" + str(datetime[1])
if STORY is not None and STORY_TITLE is not None and STORY_LINK is not None and IMAGE_LINK is not None and NEWSDATE is not None and NEWSTIME is not None and news_cat is not None and news_cat_id is not None:
story.append(STORY)
story_link.append(STORY_LINK)
story_title.append(STORY_TITLE)
img_link.append(IMAGE_LINK)
news_date.append(NEWSDATE)
news_time.append(NEWSTIME)
news_category.append(news_cat)
cat_id.append(news_cat_id)
# print(STORY,STORY_LINK,STORY_TITLE,IMAGE_LINK,NEWSDATE,NEWSTIME)
except Exception as e:
print(logging.exception(e))
# print(len(img_link),len(story_title),len(story_link),len(story),len(news_date),len(news_category))
zip_list = zip(img_link, story_title, story_link, story, news_date,
news_category, cat_id, news_time)
updateDB(zip_list, table_name)
def import_datetime(self,datetime):
date,time = datetime.split()
dt = time_functions.return_datetime(date,time,minute=True,setting='vs')
return dt
qcList = []
for x, y, z in os.walk(d):
for item in z:
fullPath = x + "\\" + item
if u"$av" in item:
if "trial"in item.lower() or "test"in item.lower() or "control"in item.lower() :
qcList.append(item)
fileList.append(item)
datetime = time.ctime(os.path.getmtime(fullPath))
# print item
time_ = datetime.split(" ").pop(3)
mins = sum(int(x) * 60 ** i for i,x in enumerate(reversed(time_.split(":"))))/60
if mins != "None":
# print mins
timeList.append(mins)
a = zip (fileList, timeList)
timeList.sort()
deltaList = []
for t in range(len(timeList)):
u = t+1
try:
def updateEventslist(lon=[-125, -115],
lat=[32, 45],
minMag=4.92,
dates=[dtm.date(2001, 0o1, 0o1),
dtm.date(2010, 12, 31)],
Nmax=999999,
foutname='scorecardevents.dat',
expTimeDelta=dtm.timedelta(days=1)):
import datetime as dtm
# check currency of the events list. if it is expired, create a new data file from ANSS. return 0,1,2 (error, current, updated) as appropriate (?dunno?)
#
# is there an earthquakes datafile?
isAfile = True
getNewANSSdata = True # we start with true default in case the file has no mod_date.
#
fouts = glob.glob(foutname)
if len(fouts) < 0:
isAfile = False
getNewANSSdata = True
#
modDate = dtm.datetime(1900, 1, 1, 0, 0,
0) # just in case there is no mod_date...
# is the file current?
# we could use something like fileInfo=os.stat(fname); modDate=fileInfo.st_mtime
# which returns the mod-date in seconds maybe? my experience is that this is a relatively unreliable way to determine whether
# or not a file is current (funny things can happen to change mod dates and all that good stuff). it is much more reliable
# to evaluate information in the file (or a database query is even better). also, different platforms might use different
# time formats (milliseconts, days, etc.), so .st_mtime might not compare to datetime.datetime.now() properly.
if isAfile:
# is it current:
modDate = dtm.datetime(1900, 1, 1, 0, 0,
0) # just in case there is no mod_date...
fList = []
f = open(foutname, 'r')
for rw in f:
# look for info:
if rw[0:11] == '#!mod_date:':
# this is the modification date...
rws = rw.split('\t')
modDtm = rws[1][:-1].split(' ') #date on in [0], time in [1]
delim = '-'
if '/' in modDtm[0]: delim = ['/']
lstDate = modDtm[0].split(delim)
lstTime = [0, 0, 0]
if len(modDtm) > 1:
lstTime = modDtm[1].split(':')
while len(lstTime) < 3:
lstTime += [0]
for i in range(len(lstDate)):
lstDate[i] = int(lstDate[i])
for i in range(len(lstTime)):
lstTime[i] = int(float(lstTime[i]))
modDate = dtm.datetime(
lstDate[0], lstDate[1], lstDate[2], lstTime[0], lstTime[1],
int(lstTime[2])) # note, we skip fractional seconds.
freshDate = dtm.datetime.today() - expTimeDelta
#print modDate, freshDate, modDate<freshDate
if modDate < freshDate:
getNewANSSdata = True
else:
getNewANSSdata = False
break
#
#
f.close()
#
# at this point, we know if we have an events file and if it is current. if the file does not exist
# or is not current, fetch new data from ANSS. if not, simply return
#
if getNewANSSdata == False: return 1
#
#print "getting fresh data"
# otherwise, retrieve data from ANSS and write a new data file:
# for speed, just code this here; write direcly to the output file:
fin = getANSShtml(lon, lat, minMag, dates, Nmax)
# tempFname='scripts/temp/tempEvents.dat'
tempFname = fullModpyDir('/var/www/quakemaps/scripts/temp/tempEvents.dat')
tempFile = open(tempFname, 'w')
tempFile.write("#!mod_date:\t%s\n" %
str(dtm.datetime.today())) # date the file
tempFile.write("#!CA earthquakes\n")
for rw in fin:
if rw[0] in ["#", "<"] or rw[0:4] in ["Date", "date", "DATE", "----"]:
continue
#anssList+=[rw[:-1]]
# data are fixed width delimited
# return date-time, lat, lon, depth, mag, magType, nst, gap, clo, rms, src, catEventID (because those are all the available bits)
#anssList+=[[rw[0:22].strip(), float(rw[23:31].strip()), float(rw[32:41].strip()), float(rw[42:48].strip()), float(rw[49:54].strip()), rw[55:59].strip(), float(rw[60:64].strip()), rw[65:68].strip(), rw[69:73].strip(), float(rw[74:78].strip()), rw[79:83].strip(), rw[84:96].strip()]]
# we want tab delimited file with lon, lat, mag, date
dtm = rw[0:22].strip()
dt = dtm.split(' ')[0].replace('-',
'/') # i think we want / delimiters...
tempFile.write("%f\t%f\t%s\t%s\n" % (float(rw[32:41]), float(
rw[23:31]), float(rw[49:54]), dtm.split(' ')[0]))
#
tempFile.close()
#
# now, we've written a temp file. swap the old one out for the new...
#
# write a backup (overwrites daily) and copy the temp file into production place.
bkpFname = ''
fnames = foutname.split('.')
fnames[-2] += '-bkp'
for bit in fnames:
bkpFname += bit + '.'
bkpFname = bkpFname[:-1]
os.system('cp %s %s' % (foutname, bkpFname))
#
os.system('cp %s %s' % (tempFname, foutname))
#Splitting month,day and year by'-' if the input is 12-12-2019 then it will be stored as
#array like this ['12','12','2019' and storing it in the variable so that we can access it]
x = date.split("-")
#printing array to check no need to print btw
print(x)
#now getting month,day and year from an array x. we know that indexing of an array starts from 0
#so we have 12 in 0 index and 12 in 1 index and 2019 in 2 index and converting into int because we
#cannot compare string and int
month = int(x[0])
day = int(x[1])
year = int(x[2])
#Now getting date from datetime.date.today() function and datetime will store an integer
#So it is converted to string to use .split() function
datetime = str(datetime.date.today())
y = datetime.split("-")
#Getting presentyear,month and day
presentyear = int(y[0])
presentmonth = int(y[1])
presentday = int(y[2])
#Checking if the month , day and year is error or not
if (int(month) > 12):
print("There are only 12months in a year.Please check your input")
elif (int(day) > 32):
print("!input error!Days aren't supported. please check")
elif (year > presentyear):
print("sorry incorrect date!Please Try Again")
else:
print("Month:", month, "day:", day, "Year:", year)
#we get negative integer if we subtract greater number from smaller number
def store_date(d):
from datetime import date
d = d.split("-")
date1 = date(int(d[0]), int(d[1]), int(d[2]))
return date1
def getformatdate(datetime):
date = datetime.split()[0]
# datelist=date.split('-')
# formattime=datelist[0]+'-'+datelist[1]
return date
def time_set(self, datetime):
currentTime, currentDate = datetime.split('|')
self.time.setText(currentTime)
self.date.setText(currentDate)
def convert_timestamp_for_disqus(datetime):
date, time = datetime.split()
year, mon, day = date.split('-')
hh, mm, ss = time.split(':')
return "%s-%s-%sT%s:%s" %(year, mon, day, hh, mm)
def updateEventslist(lon=[-125, -115], lat=[32, 45], minMag=4.92, dates=[dtm.date(2001,0o1,0o1), dtm.date(2010, 12, 31)], Nmax=999999, foutname='scorecardevents.dat', expTimeDelta=dtm.timedelta(days=1)):
import datetime as dtm
# check currency of the events list. if it is expired, create a new data file from ANSS. return 0,1,2 (error, current, updated) as appropriate (?dunno?)
#
# is there an earthquakes datafile?
isAfile=True
getNewANSSdata=True # we start with true default in case the file has no mod_date.
#
fouts=glob.glob(foutname)
if len(fouts)<0:
isAfile=False
getNewANSSdata=True
#
modDate=dtm.datetime(1900, 1,1, 0,0,0) # just in case there is no mod_date...
# is the file current?
# we could use something like fileInfo=os.stat(fname); modDate=fileInfo.st_mtime
# which returns the mod-date in seconds maybe? my experience is that this is a relatively unreliable way to determine whether
# or not a file is current (funny things can happen to change mod dates and all that good stuff). it is much more reliable
# to evaluate information in the file (or a database query is even better). also, different platforms might use different
# time formats (milliseconts, days, etc.), so .st_mtime might not compare to datetime.datetime.now() properly.
if isAfile:
# is it current:
modDate=dtm.datetime(1900, 1,1, 0,0,0) # just in case there is no mod_date...
fList=[]
f=open(foutname, 'r')
for rw in f:
# look for info:
if rw[0:11] == '#!mod_date:':
# this is the modification date...
rws=rw.split('\t')
modDtm=rws[1][:-1].split(' ') #date on in [0], time in [1]
delim='-'
if '/' in modDtm[0]: delim=['/']
lstDate=modDtm[0].split(delim)
lstTime=[0,0,0]
if len(modDtm)>1:
lstTime=modDtm[1].split(':')
while len(lstTime)<3:
lstTime+=[0]
for i in range(len(lstDate)): lstDate[i]=int(lstDate[i])
for i in range(len(lstTime)): lstTime[i]=int(float(lstTime[i]))
modDate=dtm.datetime(lstDate[0], lstDate[1], lstDate[2], lstTime[0], lstTime[1], int(lstTime[2])) # note, we skip fractional seconds.
freshDate=dtm.datetime.today()-expTimeDelta
#print modDate, freshDate, modDate<freshDate
if modDate<freshDate:
getNewANSSdata=True
else:
getNewANSSdata=False
break
#
#
f.close()
#
# at this point, we know if we have an events file and if it is current. if the file does not exist
# or is not current, fetch new data from ANSS. if not, simply return
#
if getNewANSSdata==False: return 1
#
#print "getting fresh data"
# otherwise, retrieve data from ANSS and write a new data file:
# for speed, just code this here; write direcly to the output file:
fin=getANSShtml(lon, lat, minMag, dates, Nmax)
# tempFname='scripts/temp/tempEvents.dat'
tempFname=fullModpyDir('/var/www/quakemaps/scripts/temp/tempEvents.dat')
tempFile=open(tempFname, 'w')
tempFile.write("#!mod_date:\t%s\n" % str(dtm.datetime.today())) # date the file
tempFile.write("#!CA earthquakes\n")
for rw in fin:
if rw[0] in ["#", "<"] or rw[0:4] in ["Date", "date", "DATE", "----"]: continue
#anssList+=[rw[:-1]]
# data are fixed width delimited
# return date-time, lat, lon, depth, mag, magType, nst, gap, clo, rms, src, catEventID (because those are all the available bits)
#anssList+=[[rw[0:22].strip(), float(rw[23:31].strip()), float(rw[32:41].strip()), float(rw[42:48].strip()), float(rw[49:54].strip()), rw[55:59].strip(), float(rw[60:64].strip()), rw[65:68].strip(), rw[69:73].strip(), float(rw[74:78].strip()), rw[79:83].strip(), rw[84:96].strip()]]
# we want tab delimited file with lon, lat, mag, date
dtm=rw[0:22].strip()
dt=dtm.split(' ')[0].replace('-', '/') # i think we want / delimiters...
tempFile.write("%f\t%f\t%s\t%s\n" % (float(rw[32:41]), float(rw[23:31]), float(rw[49:54]), dtm.split(' ')[0] ))
#
tempFile.close()
#
# now, we've written a temp file. swap the old one out for the new...
#
# write a backup (overwrites daily) and copy the temp file into production place.
bkpFname=''
fnames=foutname.split('.')
fnames[-2]+='-bkp'
for bit in fnames:
bkpFname+=bit+'.'
bkpFname=bkpFname[:-1]
os.system('cp %s %s' % (foutname, bkpFname))
#
os.system('cp %s %s' % (tempFname, foutname))
def import_datetime(datetime):
date,time = datetime.split()
dt = return_datetime(date,time)
return dt
def import_datetime(datetime):
date, time = datetime.split()
dt = return_datetime(date, time)
return dt
def getMonth(datetime):
#Format: 2015-xx-xx xx:xx:xx
#Returns the number month of the time given
return int(datetime.split("-")[1])
# tempFname='scripts/temp/tempEvents.dat'
tempFname = fullModpyDir('/var/www/quakemaps/scripts/temp/tempEvents.dat')
tempFile = open(tempFname, 'w')
tempFile.write("#!mod_date:\t%s\n" %
str(dtm.datetime.today())) # date the file
tempFile.write("#!CA earthquakes\n")
for rw in fin:
if rw[0] in ["#", "<"] or rw[0:4] in ["Date", "date", "DATE", "----"]:
continue
#anssList+=[rw[:-1]]
# data are fixed width delimited
# return date-time, lat, lon, depth, mag, magType, nst, gap, clo, rms, src, catEventID (because those are all the available bits)
#anssList+=[[rw[0:22].strip(), float(rw[23:31].strip()), float(rw[32:41].strip()), float(rw[42:48].strip()), float(rw[49:54].strip()), rw[55:59].strip(), float(rw[60:64].strip()), rw[65:68].strip(), rw[69:73].strip(), float(rw[74:78].strip()), rw[79:83].strip(), rw[84:96].strip()]]
# we want tab delimited file with lon, lat, mag, date
dtm = rw[0:22].strip()
dt = dtm.split(' ')[0].replace('-',
'/') # i think we want / delimiters...
tempFile.write("%f\t%f\t%s\t%s\n" % (float(rw[32:41]), float(
rw[23:31]), float(rw[49:54]), dtm.split(' ')[0]))
#
tempFile.close()
#
# now, we've written a temp file. swap the old one out for the new...
#
# write a backup (overwrites daily) and copy the temp file into production place.
bkpFname = ''
fnames = foutname.split('.')
fnames[-2] += '-bkp'
for bit in fnames:
bkpFname += bit + '.'
bkpFname = bkpFname[:-1]
os.system('cp %s %s' % (foutname, bkpFname))
def get_year(datetime):
datetime = datetime.split("-")
return datetime[0]
# for speed, just code this here; write direcly to the output file:
fin=getANSShtml(lon, lat, minMag, dates, Nmax)
# tempFname='scripts/temp/tempEvents.dat'
tempFname=fullModpyDir('/var/www/quakemaps/scripts/temp/tempEvents.dat')
tempFile=open(tempFname, 'w')
tempFile.write("#!mod_date:\t%s\n" % str(dtm.datetime.today())) # date the file
tempFile.write("#!CA earthquakes\n")
for rw in fin:
if rw[0] in ["#", "<"] or rw[0:4] in ["Date", "date", "DATE", "----"]: continue
#anssList+=[rw[:-1]]
# data are fixed width delimited
# return date-time, lat, lon, depth, mag, magType, nst, gap, clo, rms, src, catEventID (because those are all the available bits)
#anssList+=[[rw[0:22].strip(), float(rw[23:31].strip()), float(rw[32:41].strip()), float(rw[42:48].strip()), float(rw[49:54].strip()), rw[55:59].strip(), float(rw[60:64].strip()), rw[65:68].strip(), rw[69:73].strip(), float(rw[74:78].strip()), rw[79:83].strip(), rw[84:96].strip()]]
# we want tab delimited file with lon, lat, mag, date
dtm=rw[0:22].strip()
dt=dtm.split(' ')[0].replace('-', '/') # i think we want / delimiters...
tempFile.write("%f\t%f\t%s\t%s\n" % (float(rw[32:41]), float(rw[23:31]), float(rw[49:54]), dtm.split(' ')[0] ))
#
tempFile.close()
#
# now, we've written a temp file. swap the old one out for the new...
#
# write a backup (overwrites daily) and copy the temp file into production place.
bkpFname=''
fnames=foutname.split('.')
fnames[-2]+='-bkp'
for bit in fnames:
bkpFname+=bit+'.'
bkpFname=bkpFname[:-1]
os.system('cp %s %s' % (foutname, bkpFname))
#
