def file_list(request, date=None):
template_name = 'index.html'
context = {
'files': [],
'date': date
}
# Реализуйте алгоритм подготавливающий контекстные данные для шаблона по примеру:
files_list = os.listdir(FILES_PATH)
for file in files_list:
file_path = os.path.join(FILES_PATH, file)
file_metadata = os.stat(file_path)
# print(f'ТУТ!!!!!! : {file_metadata}')
time_create_file = datetime.fromtimestamp(file_metadata.st_ctime)
time_change_file = datetime.fromtimestamp(file_metadata.st_mtime)
file_info = {
'name': file,
'ctime': time_create_file,
'mtime': time_change_file
}
if date:
if file_info['ctime'].date() == datetime.strptime(date, '%Y-%m-%d').date():
context['files'].append(file_info)
else:
context['files'].append(file_info)
return render(request, template_name, context)
def filter_data(data, min_item_support, min_session_length):
#y?
session_lengths = data.groupby('SessionId').size()
data = data[np.in1d(data.SessionId,
session_lengths[session_lengths > 1].index)]
#filter item support
item_supports = data.groupby('ItemId').size()
data = data[np.in1d(
data.ItemId, item_supports[item_supports >= min_item_support].index)]
#filter session length
session_lengths = data.groupby('SessionId').size()
data = data[np.in1d(
data.SessionId,
session_lengths[session_lengths >= min_session_length].index)]
#output
data_start = datetime.fromtimestamp(data.Time.min(), timezone.utc)
data_end = datetime.fromtimestamp(data.Time.max(), timezone.utc)
print(
'Filtered data set\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {}\n\n'
.format(len(data), data.SessionId.nunique(), data.ItemId.nunique(),
data_start.date().isoformat(),
data_end.date().isoformat()))
return data
def compare(ds1_path, ds2_path):
"""
Function that compares content of two pickled
:param ds1_path: path string
:param ds2_path: path string
:return: void
"""
ds1_path = str(ds1_path).lower()
ds2_path = str(ds2_path).lower()
# File where the comparision result will be stored
logfile_name = 'diff' + datetime.now().strftime('%Y%m%d%H%M%S') + '.csv'
log = open(logfile_name, 'w')
log.write("TYPE; SNAPSHOT1; SNAPSHOT2; CONTEXT\n")
ds1 = pickle.load(open(ds1_path, 'rb'))
ds2 = pickle.load(open(ds2_path, 'rb'))
# Loop over directory state stored in the first path
for subdir, content in ds1.items():
if subdir in ds2:
for f, f_det in content['file_details'].items():
if f in ds2[subdir]['file_details']:
ds1_time = datetime.fromtimestamp(
ds1[subdir]['file_details'][f][0]).isoformat()
ds2_time = datetime.fromtimestamp(
ds2[subdir]['file_details'][f][0]).isoformat()
# Check if modification date changed
if ds1_time != ds2_time:
log.write("DATE MODIFIED;" + str(ds1_time) + ";" +
str(ds2_time) + ";" +
str(os.path.join(subdir, f)) + "\n")
else:
log.write("MISSING FILE;exists;missing;" +
os.path.join(subdir, f) + "\n")
else:
log.write("MISSING FOLDER;exists;missing;" + subdir + "\n")
# Loop over directory state stored in the second path and check for missing/existing files
for subdir, content in ds2.items():
if subdir in ds1:
for f, f_det in content['file_details'].items():
if f not in ds1[subdir]['file_details']:
log.write("MISSING FILE;missing;exists;" +
os.path.join(subdir, f) + "\n")
else:
log.write("MISSING FOLDER;missing;exists;" + subdir + "\n")
print("Differences stored in: " + logfile_name)
def split_data_retrain_train(data, output_file, days_train, days_test,
retrain_num):
data_start = datetime.fromtimestamp(data.Time.min(), timezone.utc)
train_from = data_start
new_days = retrain_num * days_test
train_to = data_start + timedelta(days=days_train) + timedelta(
days=new_days)
# todo: test_from
# test_to = train_to + timedelta(days=days_test)
session_min_times = data.groupby('SessionId').Time.min()
session_max_times = data.groupby('SessionId').Time.max()
session_train = session_max_times[
(session_min_times >= train_from.timestamp())
& (session_max_times <= train_to.timestamp())].index
train = data[np.in1d(data.SessionId, session_train)]
trlength = train.groupby('SessionId').size()
train = train[np.in1d(train.SessionId, trlength[trlength >= 2].index)]
# print('Full train set\n\tEvents: {}\n\tSessions: {}\n\tItems: {}'.format(len(train), train.SessionId.nunique(),
# train.ItemId.nunique()))
print(
'Full train set\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {}\n\n'
.format(len(train), train.SessionId.nunique(), train.ItemId.nunique(),
train_from.date().isoformat(),
train_to.date().isoformat()))
train.to_csv(output_file + '_train_full.' + str(retrain_num) + '.txt',
sep='\t',
index=False)
data_end = datetime.fromtimestamp(train.Time.max(), timezone.utc)
valid_from = data_end - timedelta(days=days_test)
session_max_times = train.groupby('SessionId').Time.max()
session_train = session_max_times[
session_max_times < valid_from.timestamp()].index
session_valid = session_max_times[
session_max_times >= valid_from.timestamp()].index
train_tr = train[np.in1d(train.SessionId, session_train)]
valid = train[np.in1d(train.SessionId, session_valid)]
valid = valid[np.in1d(valid.ItemId, train_tr.ItemId)]
tslength = valid.groupby('SessionId').size()
valid = valid[np.in1d(valid.SessionId, tslength[tslength >= 2].index)]
print('Train set\n\tEvents: {}\n\tSessions: {}\n\tItems: {}'.format(
len(train_tr), train_tr.SessionId.nunique(),
train_tr.ItemId.nunique()))
train_tr.to_csv(output_file + '_train_tr.' + str(retrain_num) + '.txt',
sep='\t',
index=False)
print('Validation set\n\tEvents: {}\n\tSessions: {}\n\tItems: {}'.format(
len(valid), valid.SessionId.nunique(), valid.ItemId.nunique()))
valid.to_csv(output_file + '_train_valid.' + str(retrain_num) + '.txt',
sep='\t',
index=False)
return train
def photo_shooting_date(self, file):
photo = Image.open(file)
info = photo._getexif()
try:
if 36867 in info:
date = info[36867]
date = datetime.strptime(date, '%Y:%m:%d %H:%M:%S')
else:
date = datetime.fromtimestamp(os.path.getmtime(file))
except:
date = datetime.fromtimestamp(os.path.getmtime(file))
return date
def store_data(my_data):
try:
file_name = 'DATA/DataGatherer_' + str(
datetime.fromtimestamp(time()).strftime('%Y%m%d_%H') + '.csv')
file = open(file_name, 'a')
file.write(my_data)
file.write('\n')
file.close()
except BaseException as e:
print('store_data: ' + str(e) + ' TIME: ' +
str(datetime.fromtimestamp(time()).strftime('%Y%m%d_%H%M')))
sleep(5)
pass
def load_data_prepared(file, version):
data = pd.read_csv(file + '.csv', sep='\t')
#output
data_start = datetime.fromtimestamp(data.Time.min(), timezone.utc)
data_end = datetime.fromtimestamp(data.Time.max(), timezone.utc)
print(
'Loaded data set\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {}\n\n'
.format(len(data), data.SessionId.nunique(), data.ItemId.nunique(),
data_start.date().isoformat(),
data_end.date().isoformat()))
return data
def __init__(self, parent = None):
'''
Constructor
'''
super(VoivoiShow, self).__init__(parent)
#self.setStyleSheet("background: #000;")
self.splashscreen = QImage("res/splashscreen.jpg")
#print(self.splashscreen.isNull())
if not self.splashscreen.isNull():
self.imgList.append(self.splashscreen)
self.imageLabel = QLabel()
self.imageLabel.setSizePolicy(QSizePolicy.Ignored, QSizePolicy.Ignored)
self.imageLabel.setParent(self)
self.httpPool = urllib3.PoolManager()
self.slideIterator = 0
self.timer.timeout.connect(self.nextImage)
self.updateTimer = QTimer()
self.updateTimer.timeout.connect(self.updateImages)
self.mostRecent = datetime.fromtimestamp(0)
self.layout = QHBoxLayout()
self.layout.setMargin(0)
self.layout.addWidget(self.imageLabel)
self.layout.setAlignment(self.imageLabel, Qt.AlignHCenter)
self.setLayout(self.layout)
self.voivoifont = QFont("Helvetica", 48)
self.voivoifont.setBold(True)
def on_data(self, data):
try:
all_data = json.loads(data)
t_date = all_data['created_at']
t_date = datetime.strptime(
t_date, "%a %b %d %H:%M:%S %z %Y") # change to datetime format
t_id = all_data['id']
t_followers = all_data['user']['followers_count']
if 'extended_tweet' in all_data:
t_text = all_data['extended_tweet']['full_text']
else:
t_text = all_data['text']
t_text = t_text.replace(
"'", "") # Gives an error when writing to the db
t_text = t_text.replace(";", "") # Is used as delimiter
t_text = t_text.encode(
"utf-8") # Should get rid of unknown characters
t_user = all_data["user"]['id']
# hashtags = all_data['entities']['hashtags']
my_data = str(t_date) + ";" + str(t_id) + ";" + str(
t_followers) + ";" + str(t_text) + ";" + str(t_user)
store_data(my_data)
return True
except BaseException as e:
print('failed ondata: ' + str(e) + ' TIME: ' +
str(datetime.fromtimestamp(time()).strftime('%Y%m%d_%H%M')))
sleep(5)
pass
def patch(self, user_id):
"""
TODO: cascading update to soccerInfo for user_id change
Updates fields of the User
:param user_id: user id
:return: message dict
"""
data = request.get_json()
try:
user = User.objects(user_id__exact=user_id)
if 'user_id' in data:
user.update(set__user_id=data['user_id'])
if 'first_name' in data:
user.update(set__first_name=data['first_name'])
if 'last_name' in data:
user.update(set__last_name=data['last_name'])
if 'email' in data:
user.update(set__email=data['email'])
if 'password' in data:
user.update(set__password=data['password'])
if 'last_login' in data:
date = datetime.fromtimestamp(data['last_login']['$date'] / 1e3)
user.update(set__last_login=date)
return {"message": "successfully patched!"}, 201
except Exception as err:
return {"message": "Failed to patch: {0}".format(err)}, 405
def mediawiki_time(epoch):
date = datetime.fromtimestamp(epoch)
year = str(date.year)
if date.month < 10:
month = '0' + str(date.month)
else:
month = str(date.month)
if date.day < 10:
day = '0' + str(date.day)
else:
day = str(date.day)
if date.hour < 10:
hour = '0' + str(date.hour)
else:
hour = str(date.hour)
if date.minute < 10:
minute = '0' + str(date.minute)
else:
minute = str(date.minute)
if date.second < 10:
second = '0' + str(date.second)
else:
second = str(date.second)
return year + month + day + hour + minute + second
def print_message(self, message):
text = message['text']
nick = message['username']
timestamp = float(message['time'])
ct = datetime.fromtimestamp(timestamp).strftime('%H:%M:%S')
self.show_text(f'{ct} {nick}\n{text}\n\n')
def dir_l(path):
for f in os.listdir(path):
abspath=os.path.join(path,f) #定位文件 绝对路径
fsize = os.path.getsize(abspath)
mtime = datetime.fromtimestamp(os.path.getmtime(abspath)).strftime('%Y-%m-%d %H:%M:%S') #时间格式化
flag = '/' if os.path.isdir(abspath) else ''
print("%10d %s %s%s" % (fsize,mtime,f,flag)) #格式化输出
def __initializeFileBrowser(self, decryption, encryptedDir, parent=''):
for entry in os.listdir(encryptedDir):
# Check if file is segmented
filename, segment = FilenameUtils.splitSegmentedFileName(entry)
if segment is not None and segment is not 0:
continue
encryptedFullPathForEntry = os.path.join(encryptedDir, entry)
decryptedName = decryption.decryptFileName(filename)
identifier = None
printableModificationDate = datetime.fromtimestamp(os.path.getmtime(encryptedFullPathForEntry)).strftime('%Y-%m-%d %H:%M:%S')
fileSize = None
if os.path.isfile(encryptedFullPathForEntry):
encryptedVirtualFile = None
if segment is not None:
encryptedVirtualFile = SegmentedVirtualFile(os.path.join(encryptedDir, filename))
else:
encryptedVirtualFile = VirtualFile(encryptedFullPathForEntry)
fileSize = decryption.decryptedFileSize(encryptedVirtualFile)
encryptedVirtualFile.closeFileHandle()
identifier = self.fileBrowser.insert(parent, 'end', text=decryptedName)
else:
identifier = self.fileBrowser.insert(parent, 'end', text=decryptedName)
self.__initializeFileBrowser(decryption, encryptedFullPathForEntry, identifier)
fileSize = os.path.getsize(encryptedFullPathForEntry)
self.fileBrowser.set(identifier, 'size', hurry.filesize.size(fileSize))
self.fileBrowser.set(identifier, 'modified', printableModificationDate)
self.fileBrowser.set(identifier, 'encrypted name', entry)
self.fileBrowserShadowInformation[identifier] = encryptedFullPathForEntry
def main():
key = os.environ.get('WEATHER_KEY')
city, country_code = getInput()
query = {
'q': city + "," + country_code,
'units': 'imperial',
'appid': '9db10307657b0ff8224b0da642ac57f7'
}
url = 'http://api.openweathermap.org/data/2.5/forecast'
try:
data = requests.get(url, params=query).json()
forecast_items = data['list']
for forecast in forecast_items:
timestamp = forecast['dt']
date = datetime.fromtimestamp(timestamp)
# I choose to convert the timestamp in to a datetime
# to get a local Minneapolis time, my program is currently built
# to be used in this area for educational purposed and not deployed to others around the world yet
temp = forecast['main']['temp']
description = forecast['weather'][0]['description']
wind_speed = forecast['wind']['speed']
city = city.capitalize()
print(
f'At {date} in {city} the temp is {temp:.2f}F. \n and {description} with a wind speed of {wind_speed:.1f}mph'
)
except:
print("Please enter a valid city/country code")
main()
def get_address(bk=770000):
url = f'https://cpchain.io/explorer/impeachs_by_block/{bk}/0'
r = requests.get(url)
bks = json.loads(r.text).get('impeach_bks')
li = []
addr = {}
for i in range(len(bks)):
time = datetime.fromtimestamp(
bks[i]["timestamp"]).strftime("%Y-%m-%d %H:%M:%S")
impeach_item = {
"address": bks[i]['impeachProposer'],
'number': bks[i]['number'],
'time': time
}
if impeach_item['address'] not in addr:
addr[impeach_item['address']] = 1
else:
addr[impeach_item['address']] += 1
li.append(impeach_item)
li.reverse()
pprint(li)
print("*" * 100)
pprint(sorted(addr.items(), key=lambda x: x[1], reverse=True))
print('total impeach number: ', len(li))
def load_data(path, file_name):
'''
Desc: Loads a tuple of training and test set with the given parameters.
--------
Input:
path : Path of preprocessed data (str)
file : Name of dataset
--------
Output : tuple of DataFrame (train, test)
'''
print('START load data')
st = time.time()
sc = time.perf_counter()
train_appendix = '_train'
test_appendix = '_test'
train = pd.read_csv(path + file_name + train_appendix + '.txt',
sep='\t',
dtype={'ItemId': np.int64})
test = pd.read_csv(path + file_name + test_appendix + '.txt',
sep='\t',
dtype={'ItemId': np.int64})
data_start = datetime.fromtimestamp(train.Time.min(), timezone.utc)
data_end = datetime.fromtimestamp(train.Time.max(), timezone.utc)
print(
'train set\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {}\n'
.format(len(train), train.SessionId.nunique(), train.ItemId.nunique(),
data_start.date().isoformat(),
data_end.date().isoformat()))
data_start = datetime.fromtimestamp(test.Time.min(), timezone.utc)
data_end = datetime.fromtimestamp(test.Time.max(), timezone.utc)
print(
'test set\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {}\n'
.format(len(test), test.SessionId.nunique(), test.ItemId.nunique(),
data_start.date().isoformat(),
data_end.date().isoformat()))
print('END load data ', (time.perf_counter() - sc), 'c / ',
(time.time() - st), 's')
return (train, test)
def download(start=0, end=100000, max_results=10, wait=10, done=None):
no_results = 0
base_url = 'http://export.arxiv.org/api/query?'
categories = ['cat:cs.CV', 'cat:cs.AI', 'cat:cs.LG', 'cat:cs.CL', 'cat:cs.NE', 'cat:stat.ML']
params = {'search_query': '+OR+'.join(categories),
'sortBy' : 'lastUpdatedDate',
'max_results' : max_results}
articles = pd.DataFrame()
for s in range(start, end, max_results):
print(s, end=' ', flush=True)
params.update({'start': s})
param_str = "&".join(f'{k}={v}' for k, v in params.items())
response = requests.get(url=base_url, params=param_str)
parsed = feedparser.parse(response.text)
if len(parsed.entries) == 0:
print('\nReceived no results from arxiv.')
no_results += 1
if no_results < 5:
sleep(60)
else:
save_articles(articles)
break
for entry in parsed.entries:
entry = encode_feedparser_dict(entry)
id = entry['id'].split('/')[-1]
if id in done:
continue
r = dict(
summary=' '.join([t.strip() for t in entry['summary'].replace('\n', ' ').split()]),
tags=','.join([t['term'] for t in entry['tags']]),
category=entry['arxiv_primary_category']['term'],
affiliation=entry.get('arxiv_affiliation'),
pdf_url=entry['link'],
author=entry['author'],
updated=datetime.fromtimestamp(mktime(entry['updated_parsed'])),
published=datetime.fromtimestamp(mktime(entry['published_parsed'])),
title=entry['title'].replace('\n', ' ')
)
articles[id] = pd.Series(r)
sleep(wait + random.uniform(0, 3))
print(articles.T.info(null_counts=True))
save_articles(articles)
def convert_from_timestamp(data):
print("===================================")
print(data)
for pin in data['pin_data']:
print(pin)
t = str(pin['time'])[0:10]
print(t)
pin['time'] = datetime.fromtimestamp(int(t)) #remove milliseconds
def do_dir():
pwd = os.path.abspath(".")
for f in os.listdir(pwd):
fsize = os.path.getsize(f)
mtime = datetime.fromtimestamp(
os.path.getmtime(f)).strftime("%Y.%m.%d %H:%M")
flag = "/" if os.path.isdir(f) else " "
print("%30s %10s %20s %10s" % (f, fsize, mtime, flag))
def get_photo(user_id, access_token, invert=False):
log(user_id)
params_prof_photo = {
'user_id': user_id,
'album_id': 'profile',
'rev': '1',
'access_token': access_token,
'v': '5.120',
'Pragma-directive': 'no-cache',
'ache-directive': 'no-cache',
'Cache-control': 'no-cache',
'Pragma': 'no-cache',
'Expires': 0
}
prof_photo_response = requests.get(
'https://api.vk.com/method/photos.get', params=params_prof_photo)
photo_info = prof_photo_response.json()
log(photo_info)
photo = photo_info['response']['items'][0]
urls = photo['sizes'][-1]
log('url:' + urls['url'])
date = str(datetime.fromtimestamp(photo['date']))
log('date:' + date)
picture = requests.get(urls['url'], stream=True).raw
image = np.asarray(bytearray(picture.read()), dtype="uint8")
image = cv2.imdecode(image, cv2.IMREAD_COLOR)
x0 = 100
x2 = x0 + len(date) * 20
x1 = x2 // 2
y = 50
color1 = image[x0, y - 30]
color2 = image[x1, y - 15]
color3 = image[x2, y]
background = list()
background.append(int(color1[0]) + int(color1[1]) + int(color1[2]))
background.append(int(color2[0]) + int(color2[1]) + int(color2[2]))
background.append(int(color3[0]) + int(color3[1]) + int(color3[2]))
background.sort()
if background[1] < 155:
color_text = (255, 255, 255)
else:
color_text = (0, 0, 0)
text = str("Creation date: " + date)
cv2.putText(image,
text, (x0, y),
cv2.FONT_HERSHEY_COMPLEX,
1,
color=color_text,
thickness=2)
# put the text about creation date
if invert:
image = cv2.bitwise_not(image)
return cv2.imencode('.JPEG', image)[1].tobytes()
else:
cv2.imwrite(f'{user_id}.jpg',
image) # write photo with text in file
return color_text # return color for text
def filter_min_date( data, min_date='2014-04-01' ) :
min_datetime = datetime.strptime(min_date + ' 00:00:00', '%Y-%m-%d %H:%M:%S')
#filter
session_max_times = data.groupby('SessionId').Time.max()
session_keep = session_max_times[ session_max_times > min_datetime.timestamp() ].index
data = data[ np.in1d(data.SessionId, session_keep) ]
#output
data_start = datetime.fromtimestamp( data.Time.min(), timezone.utc )
data_end = datetime.fromtimestamp( data.Time.max(), timezone.utc )
print('Filtered data set\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {}\n\n'.
format( len(data), data.SessionId.nunique(), data.ItemId.nunique(), data_start.date().isoformat(), data_end.date().isoformat() ) )
return data;
def load_data(file):
#load csv
data = pd.read_csv(file + '.csv', sep=';', header=None)
data.columns = ['SessionId', 'ItemId', 'Time']
data.Time = data.Time / 1000
#output
data_start = datetime.fromtimestamp(data.Time.min(), timezone.utc)
data_end = datetime.fromtimestamp(data.Time.max(), timezone.utc)
print(
'Loaded data set\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {}\n\n'
.format(len(data), data.SessionId.nunique(), data.ItemId.nunique(),
data_start.date().isoformat(),
data_end.date().isoformat()))
return data
def split_data_slice( data, output_file, slice_id, days_offset, days_train, days_test ) :
data_start = datetime.fromtimestamp( data.Time.min(), timezone.utc )
data_end = datetime.fromtimestamp( data.Time.max(), timezone.utc )
print('Full data set {}\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {}'.
format( slice_id, len(data), data.SessionId.nunique(), data.ItemId.nunique(), data_start.isoformat(), data_end.isoformat() ) )
start = datetime.fromtimestamp( data.Time.min(), timezone.utc ) + timedelta( days_offset )
middle = start + timedelta( days_train )
end = middle + timedelta( days_test )
#prefilter the timespan
session_max_times = data.groupby('SessionId').Time.max()
greater_start = session_max_times[session_max_times >= start.timestamp()].index
lower_end = session_max_times[session_max_times <= end.timestamp()].index
data_filtered = data[np.in1d(data.SessionId, greater_start.intersection( lower_end ))]
print('Slice data set {}\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {} / {}'.
format( slice_id, len(data_filtered), data_filtered.SessionId.nunique(), data_filtered.ItemId.nunique(), start.date().isoformat(), middle.date().isoformat(), end.date().isoformat() ) )
#split to train and test
session_max_times = data_filtered.groupby('SessionId').Time.max()
sessions_train = session_max_times[session_max_times < middle.timestamp()].index
sessions_test = session_max_times[session_max_times >= middle.timestamp()].index
train = data[np.in1d(data.SessionId, sessions_train)]
print('Train set {}\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {}'.
format( slice_id, len(train), train.SessionId.nunique(), train.ItemId.nunique(), start.date().isoformat(), middle.date().isoformat() ) )
train.to_csv(output_file + '_train_full.'+str(slice_id)+'.txt', sep='\t', index=False)
test = data[np.in1d(data.SessionId, sessions_test)]
test = test[np.in1d(test.ItemId, train.ItemId)]
tslength = test.groupby('SessionId').size()
test = test[np.in1d(test.SessionId, tslength[tslength>=2].index)]
print('Test set {}\n\tEvents: {}\n\tSessions: {}\n\tItems: {}\n\tSpan: {} / {} \n\n'.
format( slice_id, len(test), test.SessionId.nunique(), test.ItemId.nunique(), middle.date().isoformat(), end.date().isoformat() ) )
test.to_csv(output_file + '_test.'+str(slice_id)+'.txt', sep='\t', index=False)
def error_log_append(errorString, sessionID, statusID):
localTimestamp = int(time())
localDateTime = datetime.fromtimestamp(localTimestamp)
with open(paths.errorLog, 'a') as f:
f.writelines(
"Status crashed and the following exception or error was raised:")
f.writelines('\n\t' + errorString)
f.writelines('\n\tSession ID: ' + sessionID)
f.writelines('\n\tCurrent or previous Status ID: ' + statusID)
f.writelines('\n\t' + str(localTimestamp))
f.writelines('\n\t' + str(localDateTime) + '\n')
def save_record(request, encode_params):
param_str = base64.standard_b64decode(encode_params.encode())
param_list = param_str.decode().split(',')
record = models.LocationRecords()
record.imei = param_list[0]
record.longitude = param_list[1]
record.latitude = param_list[2]
record.update_time = datetime.fromtimestamp(int(param_list[3])/1000)
record.save()
print(param_list)
return HttpResponse("success")
def _format_data(self, graph_df):
if self._time_format == "numerate":
return
if self._time_format == "MIL":
graph_df[self._time_col] = graph_df[self._time_col] / 1000 # mili to seconds
if self._time_format == "MIL" or self._time_format == "SEC":
graph_df[self._time_col] = \
graph_df[self._time_col].apply(lambda x: datetime.fromtimestamp(x)) # to datetime format
else:
# to datetime
graph_df[self._time_col] = graph_df[self._time_col].apply(lambda x: datetime.strptime(str(x), self._time_format))
def datetime_filter(t):
delta = int(time.time() - t)
if delta < 60:
return u'1分钟前'
if delta < 3600:
return u'%s分钟前' % (delta // 60)
if delta < 86400:
return u'%s小时前' % (delta // 3600)
if delta < 604800:
return u'%s天前' % (delta // 86400)
dt = datetime.fromtimestamp(t)
return u'%s年%s月%s日' % (dt.year, dt.month, dt.day)
def print_block_chain(self):
current_block = self.tail
index = 0
while current_block:
print("Index: ", index)
print("Timestamp: ",
datetime.fromtimestamp(current_block.timestamp))
print("data: ", current_block.data)
print("SHA256 Hash: ", current_block.hash)
print("Prev Hash: ", current_block.previous_hash)
current_block = current_block.prev
index += 1
def file_info():
path = request.headers['path']
filename = request.headers['file_name']
file = os.path.join(root_dir, path, filename)
if not os.path.exists(file):
return 'file ' + file + ' does not exist'
# collect file info
st = os.stat(file)
s = filename + ' info: \n\n' + \
'mode: ' + str(st.st_mode) + '\n' + \
'I-node: ' + str(st.st_ino) + '\n' + \
'device: ' + str(st.st_dev) + '\n' + \
'number of hard links: ' + str(st.st_nlink) + '\n' + \
'uid: ' + str(st.st_uid) + '\n' + \
'gid: ' + str(st.st_gid) + '\n' + \
'size: ' + str(st.st_size) + ' bytes\n' + \
'last access (atime): ' + str(datetime.fromtimestamp(int(st.st_atime))) + '\n' + \
'last modify (mtime): ' + str(datetime.fromtimestamp(int(st.st_mtime)))
return s
def update_subscriptions(cookie):
query['restful_graph_data'] = {}
results = StationReadings.get_newest_by_cookie("",
cookie["id"])["pin_data"]
if results:
for pin in StationReadings.get_newest_by_cookie(
"", cookie["id"])["pin_data"]:
pin['time'] = datetime.fromtimestamp(int(
pin['time'][0:10])) # remove milliseconds
query['map_data']["pin_data"].append(pin)
query['restful_graph_data'] = StationReadings.get_all_by_cookie(
"", cookie["id"])
print( ' '.join([str(i),'out of',str(len(tickerSet)),'-',tickerSet[i],'options found, set ,',str(k)]))
splitData = re.split('"expirationDates"|"strikes"|"quote"|"options"|"_options"|"epochs"',tickerData[0])
if len(splitData) < 3:
continue
underlyingData = re.split(',',splitData[3][2:-2])
underlyingData = pd.DataFrame([x.replace('"','').split(':') for x in underlyingData])
underlying = pd.DataFrame(underlyingData[1].to_dict(),index=[0])
underlying.columns = underlyingData[0]
underlying = underlying.T.drop_duplicates().T
underlyingOutput.append(underlying)
contracts = re.split('<option ',str(scrape.findAll("option")))
contracts = [re.findall(r'"([^"]*)"', x)[1] for x in contracts[1:]]
contractDates = [np.datetime64(datetime.fromtimestamp(np.float(x)).isoformat()) for x in contracts]
for ii in range(len(contracts)):
print(ii)
if ii != 0:
testPop = []
while len(testPop) == 0:
pullData = requests.Request('GET', url + tickerSet[i] + '+' + contracts[ii])
pullData = pullData.prepare()
dataText, track = tryRequest(funct = pullData, typ = 'sendRaise', track = track)
scrape = BeautifulSoup(dataText,'lxml')
testPop = scrape.findAll("table", {"class": "details-table quote-table Fz-m"})
optionData = scrape.findAll("table", {"class": "details-table quote-table Fz-m"})