def install():
print "Nothing yet"
dl_dir = "C:\\ReviewBoard\\Downloads\\"
p = subprocess.Popen(["svn", "--version", "--quiet"], \
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
n = p.communicate()[0]
m = re.search('[\d]+\W[\d]+\W[\d]+', n)
svn_version = m.group(0)
print svn_version
if sys.version_info[0] == 2:
p_version_minor = sys.version_info[1]
if p_version_minor == 6 and svn_version == "1.5.6":
url = "http://pysvn.tigris.org/files/documents/1233/47202/"\
+ "py26-pysvn-svn156-1.7.2-1280.exe"
pysvn = "py26-pysvn-svn156-1.7.2-1280.exe"
elif p_version_minor == 6 and svn_version == "1.6.12":
url = "http://pysvn.tigris.org/files/documents/1233/48016/"\
+ "py26-pysvn-svn1612-1.7.4-1321.exe"
pysvn = "py26-pysvn-svn1612-1.7.4-1321.exe"
elif p_version_minor == 6 and svn_version >= "1.6.15":
url = "http://pysvn.tigris.org/files/documents/1233/48844/"\
+ "py26-pysvn-svn1615-1.7.5-1360.exe"
pysvn = "py26-pysvn-svn1615-1.7.5-1360.exe"
elif p_version_minor == 7 and svn_version == "1.6.12":
url = "http://pysvn.tigris.org/files/documents/1233/48019/"\
"py27-pysvn-svn1612-1.7.4-1321.exe"
pysvn = "py27-pysvn-svn1612-1.7.4-1321.exe"
elif p_version_minor == 7 and svn_version >= "1.6.15":
url = "http://pysvn.tigris.org/files/documents/1233/48847/"\
+ "py27-pysvn-svn1615-1.7.5-1360.exe"
pysvn = "py27-pysvn-svn1615-1.7.5-1360.exe"
download(url, dl_dir, pysvn)
subprocess.Popen([dl_dir + pysvn, "/silent"]).wait()
def data_deal(list1):#list1是从spider.py接受到的原始数据
list2 = []
if list1 != []:
for person in list1:
student = data2(person.student_ID, person.name, person.department, person.major, person.grade,
person.graduate_time, person.student_status, person.failed_number, person.center_credits,
person.courses_must_to_take, person.a_group, person.b_group, person.c_group, person.d_group,
person.professional_elective_courses, person.enterprise_education_courses,
person.general_courses, person.others, '无', '无')
student.change()
#处理one_direction, another_direction两项
a = student.a_group.replace("\xc2\xa0", " ").split(',')
b = student.b_group.replace("\xc2\xa0", " ").split(',')
c = student.c_group.replace("\xc2\xa0", " ").split(',')
d = student.d_group.replace("\xc2\xa0", " ").split(',')
tmp = [a,b,c,d]
for group in tmp:
if group[2] == ' ':
group[2] = 0
if int(group[2]) + int(group[3]) >= int(group[1]):
student.one_direction = group[0]
elif int(group[2]) + int(group[3]) >= 6:
student.another_direction = group[0]
list2.append(student)
get_and_post.add(list2)#储存数据
download(list2)
return list2
def getLogos(label, url):
path = os.path.join(EXTRAS, 'logos')
zipfile = os.path.join(path, 'logos.zip')
if utils.DialogYesNo('Would you like to install ' + label, 'and make it your active logo-pack?', 'It will be downloaded and installed into your system.'):
download(path, zipfile)
utils.DialogOK(label + ' logo-pack has been installed successfully.', 'It is now set as your active logo-pack.', 'Please restart On-Tapp.TV. Thank you.')
OTT_ADDON.setSetting('dixie.logo.folder', label)
def load_mnist_labels(filename):
if not os.path.exists(filename):
download(filename)
# Read the labels in Yann LeCun's binary format.
with gzip.open(filename, 'rb') as f:
data = np.frombuffer(f.read(), np.uint8, offset=8)
# The labels are vectors of integers now, that's exactly what we want.
return data
def get(what, name):
found = False
if what == "album":
songs = getAlbum(name)
found = True
elif what == "playlist":
songs = getPlaylist(name)
found = True
else:
print("Incorrect Type")
if found and len(songs) > 0:
dirpath = str(Path.home()) + "/Spotify"
try:
# Create target Directory
os.mkdir(dirpath)
print("Directory Spotify created ")
except FileExistsError:
pass
try:
# Create target Directory
os.mkdir(dirpath + "/" + name)
print("Directory ", name, " created ")
except FileExistsError:
pass
path = dirpath + "/" + name
counter = 0
for song in songs:
try:
download(
youtube_query(song.title + " " + song.artist + " lyrics"),
path, song.title)
except youtube_dl.utils.ExtractorError:
pass
counter += 1
file_path = path + "/" + song.title + ".mp3"
audiofile = eyed3.load(file_path)
audiofile.tag.artist = song.artist
audiofile.tag.title = song.title
audiofile.tag.album = song.album
audiofile.tag.track_num = counter
try:
response = urlopen(song.image_url)
imagedata = response.read()
audiofile.tag.images.set(3, imagedata, "image/jpeg", u"")
except urllib.error.HTTPError as e:
print(e)
audiofile.tag.save()
else:
print(name + " not found")
def installSF(sfZip):
sfData = os.path.join('special://profile', 'addon_data',
'plugin.program.super.favourites')
sfDir = xbmc.translatePath(sfData)
path = os.path.join(sfDir, 'Super Favourites')
zipfile = os.path.join(path, 'sfZip.zip')
if not os.path.isdir(path):
sfile.makedirs(path)
download(sfZip, path, zipfile)
def install():
if platform.architecture()[0] == "32bit":
path = "C:\\Program Files\\Perforce\\"
else:
path = "C:\\Program Files (x86)\\Perforce\\"
file_name = "perforce.exe"
url = "http://www.perforce.com/downloads/perforce/r10.2/bin.ntx86/perforce.exe"
dl_dir = "C:\\ReviewBoard\\Downloads"
download(url, dl_dir, file_name)
call(["C:\ReviewBoard\Downloads\perforce.exe", "/S", "/v", "/qn"])
setPath(path)
def run(self):
while self.running:
if urls:
url, name, category, domain = urls.pop()
print(category, name)
notif("{} {}:at {}".format(category, name, len(urls)),
title=domain,
subtitle=url)
download(url, category, name)
if not urls:
notif("empty!!", title="pydown")
def install():
dl_dir = "C:\\ReviewBoard\\Downloads\\"
if platform.architecture()[0] == '32bit':
bit = "win32"
else:
bit = "win-amd64"
file_name = "pywin32-216." + bit + "-py" + str(sys.version_info[0]) \
+ "." + str(sys.version_info[1]) + ".exe"
url = "http://sourceforge.net/projects/pywin32/files/pywin32/Build216/" \
+ file_name + "/download"
download(url, dl_dir, file_name)
subprocess.Popen([dl_dir + file_name, "/silent"]).wait()
def getLogos(label, url):
path = os.path.join(EXTRAS, 'logos')
zipfile = os.path.join(path, 'logos.zip')
if utils.DialogYesNo(
'Would you like to install ' + label,
'and make it your active logo-pack?',
'It will be downloaded and installed into your system.'):
download(path, zipfile)
utils.DialogOK(label + ' logo-pack has been installed successfully.',
'It is now set as your active logo-pack.',
'Please restart On-Tapp.TV. Thank you.')
OTT_ADDON.setSetting('dixie.logo.folder', label)
def load_mnist_images(filename):
if not os.path.exists(filename):
download(filename)
# Read the inputs in Yann LeCun's binary format.
with gzip.open(filename, 'rb') as f:
data = np.frombuffer(f.read(), np.uint8, offset=16)
# The inputs are vectors now, we reshape them to monochrome 2D images,
# following the shape convention: (examples, channels, rows, columns)
data = data.reshape(-1, 1, 28, 28)
# The inputs come as bytes, we convert them to float32 in range [0,1].
# (Actually to range [0, 255/256], for compatibility to the version
# provided at http://deeplearning.net/data/mnist/mnist.pkl.gz.)
return data / np.float32(256)
def getLogos(label, url):
path = os.path.join(EXTRAS, "logos")
zipfile = os.path.join(path, "logos.zip")
if utils.DialogYesNo(
"Would you like to install " + label,
"and make it your active logo-pack?",
"It will be downloaded and installed into your system.",
):
download(path, zipfile)
utils.DialogOK(
label + " logo-pack has been installed successfully.",
"It is now set as your active logo-pack.",
"Please restart On-Tapp.TV. Thank you.",
)
OTT_ADDON.setSetting("dixie.logo.folder", label)
def si(self,s): lan = self.lan sa = 0 sw = 0 if int(s) == 0: sa = 1 a = 'a' return a elif int(s) == 100: sw = 1 a = 'w' return a elif int(s) < 0: o = int(s) * (-1) a = self.si(o) return a elif int(s) < 13: from download import download thisisthedownloadinstance = download(int(s),lan.dictu(),self.tor,self.torin) a = thisisthedownloadinstance.raz() return a else: jezodict = lan.dictu() print "%s: %s" % jezodict[badstacparam],str(s) quit()
def install():
if platform.architecture()[0] == '32bit':
path = "C:\\Program Files\\GnuWin32\\"
else:
path = "C:\\Program Files (x86)\\GnuWin32\\"
file_name = "patch-2.5.9-7-setup.exe"
url = \
"http://downloads.sourceforge.net/project/gnuwin32/patch/2.5.9-7/" \
+ "patch-2.5.9-7-setup.exe?r=http%3A%2F%2Fsourceforge.net%2Fproject" \
+ "%2Fdownloading.php%3Fgroupname%3Dgnuwin32%26file_name%3Dpatch-2." \
+ "5.9-7-setup.exe%26use_mirror%3Dsurfnet&ts=1308215304&use_mirror=" \
+ "cdnetworks-us-1"
dl_dir = "C:\\ReviewBoard\\Downloads"
download(url, dl_dir, file_name)
call([dl_dir + "\\" + file_name, "/silent"])
setPath(path)
def install():
if sys.version_info[0] == 2:
url = "http://mercurial.berkwood.com/binaries/"
p_version_minor = sys.version_info[1]
if p_version_minor < 4:
mercurial_version = "Mercurial-1.0.exe"
elif p_version_minor > 4 and p_version_minor < 5:
mercurial_version = "Mercurial-1.4.exe"
elif p_version_minor >= 5:
mercurial_version = "Mercurial-1.4.3.exe"
url += mercurial_version
folder = "C:\\ReviewBoard\\Downloads\\"
download(url, folder, mercurial_version)
call([folder + mercurial_version, "/qn"])
def download_data():
if request.method == 'POST':
if validate_json(request.data):
return download(request.data)
else:
error = 'Invalid input data'
return error
else:
error = 'Only accepts POST method'
return error
def botman(output):
global knocktime
global username
try:
d = output.split("!")
i = d[1].split(" ")
o = string.replace(output, i[0] + ' ', '')
command = i[0].replace("!", "")
print prefix + command + " received."
if command == 'knocktime':
knocktime = int(i[1])
if command == 'update':
download().update(i[1], i[2])
if command == 'download':
download().download(i[1], i[2])
if command == 'downloadexec':
download().downloadexec(i[1], i[2])
if command == 'terminal':
os.popen(o)
if command == 'get':
dosman('get', i[1], i[2], i[3], i[4])
if command == 'slowget':
dosman('getslow', i[1], i[2], i[3], i[4])
if command == 'udp':
dosman('udp', i[1], i[2], i[3], i[4])
if command == 'udplag':
dosman('udplag', i[1], i[2], i[3], i[4])
if command == 'click':
thread.start_new_thread(settings.clickad, (i[1], useragent))
return True
except:
return False
def process_files(dataset, shared_list, finished_list, config_info):
process_name = multiprocessing.current_process().name
url = dataset['url']
data_type = dataset['type']
data_sub_type = dataset['subtype']
filename = dataset['filename']
save_path = '/usr/src/app/tmp'
if dataset['status'] == 'active':
if url not in shared_list and url not in finished_list:
shared_list.append(url)
download(process_name, url, filename, save_path)
shared_list.remove(url)
if 'filename_uncompressed' in dataset:
decompress(process_name, filename, save_path)
finished_list.append(url)
elif url in finished_list:
logger.info(
'{}: URL already downloaded via another process: {}'.format(
process_name, url))
elif url in shared_list:
logger.info(
'{}: URL already downloading via another process: {}'.format(
process_name, url))
logger.info(
'{}: Waiting for other process\'s download to finish.'.format(
process_name))
while url not in finished_list:
time.sleep(10)
if 'filename_uncompressed' in dataset:
filename = dataset['filename_uncompressed']
logger.info(
'{}: Found uncompressed filename entry, uploading {}.'.format(
process_name, dataset['filename_uncompressed']))
upload_process(process_name, filename, save_path, data_type,
data_sub_type, config_info)
def checkData(satName, lcycle, ogdr_files_page_URL, html, lfile):
print()
print("INSIDE CHECK DATA FUNCTION")
print("satName={} ".format(satName))
print("lcycle={} lfile={}".format(lcycle, lfile))
print("ogdr_files_page_URL={} ".format(ogdr_files_page_URL, lfile))
print()
print("reading files")
for line in html:
if satName in line:
start = line.index(satName)
end = line.index('.nc') + 3
name = line[start:end]
if name > lfile:
print("new file found :", name)
print("*****donwloading new netCDF file ....*****")
download(satName, ogdr_files_page_URL + '/' + name, name,
lcycle)
print("*****download completed.*****")
print()
# print("*****processing the new file*****")
# process.process(satName, lfile)
# print("*****processing complete.*****")
# print()
lfile = name
# writing updates to JAx_last.txt
print("writing last downloads info to", satName + '_last.txt')
print('lcycle={} lfile={}'.format(lcycle, lfile))
with open(
op.join(app_dir, 'data', 'last',
satName + '_last.txt'), "w") as f:
f.write(lcycle + '\n')
f.write(lfile + '\n')
print('write complete')
print("******")
return lfile
def init_page_queue(number_downloader_threads: int = 1) -> List[str]:
ret = []
number_pages = get_number_pages(download(get_listing_url()))
page_numbers = [i for i in range(1, number_pages + 1)]
shuffle(page_numbers)
pages_html = download_multiple({get_listing_url(GAME_VERSION, i) for i in page_numbers})
with concurrent.futures.ProcessPoolExecutor(max_workers=number_downloader_threads) as executor:
project_ids = {
executor.submit(get_project_links, p): p
for p in pages_html if p is not None
}
for future in concurrent.futures.as_completed(project_ids):
ret.extend(future.result())
return ret
def run(self):
dl_name = download(self)
self.dl_name = dl_name
if dl_name and self.stopped:
try:
os.remove(self.dl_directory + '/' + dl_name)
except:
print(f'Failed to remove: {self.dl_directory}/{dl_name}')
if self.paused:
self.signals.update_signal.emit(self.data, [None, None, 'Paused', '0 B/s'])
else:
if not dl_name:
self.complete = True
def iteration(url, index_begin=1, index_end=-1, max_errors=5, getPage=False):
""" 按下标进行排布的链接可以通过这个方法批量获取
"""
num_errors = 0
links = [] if not getPage else {}
page = None
for index in itertools.count(index_begin):
index_url = '%s-%d' % (url, index)
try: page = download(url)
except: pass
if page is not None:
num_errors = 0
if not getPage:
links.append(index_url)
else:
links[index_url]= page
else:
num_errors += 1
if num_errors >= max_errors:
break
if index == index_end:
break
return links
def install():
dl_dir = "C:\\ReviewBoard\\Downloads"
download(url, dl_dir, file_name)
call([pilpath, "/qn"])
def get_data_from_file(username, code_list, name_list): #获取数据
list2 = []
list1 = []
filename = 'static\\' + username + '.txt'
f = open(filename, 'r')
file_list = f.readlines()
f.close()
for one in file_list:
one = one.decode('utf-8')
one_list = one.split(',,')
data_class = data1(one_list[0], one_list[1], one_list[2], one_list[3],
one_list[4], one_list[5], one_list[6], one_list[7],
one_list[8], one_list[9], one_list[10],
one_list[11], one_list[12], one_list[13],
one_list[14], one_list[15], one_list[16],
one_list[17])
list1 += [data_class]
if list1 != []:
for person in list1:
student = data2(person.student_ID, person.name, person.department,
person.major, person.grade, person.graduate_time,
person.student_status, person.failed_number,
person.center_credits, person.courses_must_to_take,
person.a_group, person.b_group, person.c_group,
person.d_group,
person.professional_elective_courses,
person.enterprise_education_courses,
person.general_courses, person.others, '无', '无')
change(student, code_list)
#处理one_direction, another_direction两项
a = student.a_group.replace("\xc2\xa0", " ").split(',')
b = student.b_group.replace("\xc2\xa0", " ").split(',')
c = student.c_group.replace("\xc2\xa0", " ").split(',')
d = student.d_group.replace("\xc2\xa0", " ").split(',')
tmp = [a, b, c, d]
for group in tmp:
if len(group) > 2:
if group[2] == ' ':
group[2] = 0
if int(a[2]) + int(a[3]) >= 15:
student.one_direction = a[0]
student.another_direction = int(b[2]) + int(b[3]) + int(
c[2]) + int(c[3])
elif int(b[2]) + int(b[3]) >= 15:
student.one_direction = b[0]
student.another_direction = int(a[2]) + int(a[3]) + int(
c[2]) + int(c[3])
elif int(c[2]) + int(
c[3]) >= 12 and student.others['SE315'] == '通过':
student.one_direction = c[0]
student.another_direction = int(a[2]) + int(a[3]) + int(
b[2]) + int(b[3]) - 3
else:
student.another_direction = int(a[2]) + int(a[3]) + int(
c[2]) + int(c[3]) + int(b[2]) + int(b[3])
list2.append(student)
download(username, list2, code_list, name_list)
return list2
def download_files():
subprocess.call("rm downloads/* 2>/dev/null", shell=True)
download(ois_reports)
def main(_):
# Get the raw_data.
train_data_filename = download('train-images-idx3-ubyte.gz')
train_labels_filename = download('train-labels-idx1-ubyte.gz')
test_data_filename = download('t10k-images-idx3-ubyte.gz')
test_labels_filename = download('t10k-labels-idx1-ubyte.gz')
# Extract it into numpy arrays.
train_data = extract_data(train_data_filename, 60000)
train_labels = extract_labels(train_labels_filename, 60000)
test_data = extract_data(test_data_filename, 10000)
test_labels = extract_labels(test_labels_filename, 10000)
# Generate a validation set.
validation_data = train_data[:VALIDATION_SIZE, ...]
validation_labels = train_labels[:VALIDATION_SIZE]
train_data = train_data[VALIDATION_SIZE:, ...]
train_labels = train_labels[VALIDATION_SIZE:]
num_epochs = NUM_EPOCHS
train_size = train_labels.shape[0]
x = tf.placeholder(
data_type(),
shape=[
BATCH_SIZE,
IMAGE_SIZE,
IMAGE_SIZE,
NUM_CHANNELS])
y = tf.placeholder(tf.int64, shape=[BATCH_SIZE, ])
eval_data = tf.placeholder(data_type(),
shape=(EVAL_BATCH_SIZE,
IMAGE_SIZE,
IMAGE_SIZE,
NUM_CHANNELS))
# The variables below hold all the trainable weights.
conv1_weights = tf.Variable(tf.truncated_normal([11, 11, NUM_CHANNELS, 64],
stddev=0.1,
seed=SEED,
dtype=data_type()))
conv1_biases = tf.Variable(
tf.zeros([64], dtype=data_type()))
conv2_weights = tf.Variable(tf.truncated_normal([5, 5, 64, 192],
stddev=0.1,
seed=SEED,
dtype=data_type()))
conv2_biases = tf.Variable(
tf.constant(
0.1,
shape=[192],
dtype=data_type()))
conv3_weights = tf.Variable(tf.truncated_normal([3, 3, 192, 384],
stddev=0.1,
seed=SEED,
dtype=data_type()))
conv3_biases = tf.Variable(
tf.constant(
0.1,
shape=[384],
dtype=data_type()))
conv4_weights = tf.Variable(tf.truncated_normal([3, 3, 384, 256],
stddev=0.1,
seed=SEED,
dtype=data_type()))
conv4_biases = tf.Variable(
tf.constant(
0.1,
shape=[256],
dtype=data_type()))
conv5_weights = tf.Variable(tf.truncated_normal([3, 3, 256, 256],
stddev=0.1,
seed=SEED,
dtype=data_type()))
conv5_biases = tf.Variable(
tf.constant(
0.1,
shape=[256],
dtype=data_type()))
# fully connected, depth 1024
fc1_weights = tf.Variable(tf.truncated_normal([1 * 1 * 256, 4096],
stddev=0.1,
seed=SEED,
dtype=data_type()))
fc1_biases = tf.Variable(
tf.constant(
0.1,
shape=[4096],
dtype=data_type()))
fc2_weights = tf.Variable(tf.truncated_normal([4096, 4096],
stddev=0.1,
seed=SEED,
dtype=data_type()))
fc2_biases = tf.Variable(
tf.constant(
0.1,
shape=[4096],
dtype=data_type()))
fc3_weights = tf.Variable(tf.truncated_normal([4096, NUM_LABELS],
stddev=0.1,
seed=SEED,
dtype=data_type()))
fc3_biases = tf.Variable(
tf.constant(
0.1,
shape=[10],
dtype=data_type()))
def model(data):
"""The logs definition"""
# Conv 1
with tf.name_scope('conv1'):
conv1 = tf.nn.conv2d(data,
conv1_weights,
strides=[1, 4, 4, 1],
padding='SAME')
# Bias and rectified linear non_linearity.
relu = tf.nn.relu(tf.nn.bias_add(conv1, conv1_biases))
norm = tf.nn.local_response_normalization(relu,
depth_radius=2,
bias=2.0,
alpha=1e-4,
beta=0.75)
print_activations(conv1)
# Max pooling.The kernel size spec {ksize} also follows the layout.
pool1 = tf.nn.max_pool(norm,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME')
print_activations(pool1)
# Conv 2
with tf.name_scope('conv2'):
conv2 = tf.nn.conv2d(pool1,
conv2_weights,
strides=[1, 1, 1, 1],
padding='SAME')
# Bias and rectified linear non_linearity.
relu = tf.nn.relu(tf.nn.bias_add(conv2, conv2_biases))
norm = tf.nn.local_response_normalization(relu,
alpha=1e-4,
beta=0.75,
depth_radius=2,
bias=2.0)
print_activations(conv2)
# Max pooling.The kernel size spec {ksize} also follows the layout.
pool2 = tf.nn.max_pool(norm,
ksize=[1, 2, 2, 1],
strides=[1, 2, 2, 1],
padding='SAME')
print_activations(pool2)
# Conv 3
with tf.name_scope('conv3'):
conv3 = tf.nn.conv2d(pool2,
conv3_weights,
strides=[1, 1, 1, 1],
padding='SAME')
# Bias and rectified linear non_linearity.
relu = tf.nn.relu(tf.nn.bias_add(conv3, conv3_biases))
print_activations(conv3)
# Conv 4
with tf.name_scope('conv4'):
conv4 = tf.nn.conv2d(relu,
conv4_weights,
strides=[1, 1, 1, 1],
padding='SAME')
# Bias and rectified linear non_linearity.
relu = tf.nn.relu(tf.nn.bias_add(conv4, conv4_biases))
norm = tf.nn.lrn(relu,
4,
bias=1.0,
alpha=0.001 / 9.0,
beta=0.75)
print_activations(conv4)
# Conv 5
with tf.name_scope('conv5'):
conv5 = tf.nn.conv2d(norm,
conv5_weights,
strides=[1, 1, 1, 1],
padding='SAME')
# Bias and rectified linear non_linearity.
relu = tf.nn.relu(tf.nn.bias_add(conv5, conv5_biases))
print_activations(conv5)
# Max pooling.The kernel size spec {ksize} also follows the layout.
pool5 = tf.nn.max_pool(relu,
ksize=[1, 3, 3, 1],
strides=[1, 2, 2, 1],
padding='SAME')
print_activations(pool5)
# Fully 1
fc1 = tf.reshape(pool5, [-1, fc1_weights.get_shape().as_list()[0]])
fc1 = tf.nn.relu(tf.matmul(fc1, fc1_weights) + fc1_biases)
# dropout
fc1 = tf.nn.dropout(fc1, 0.5)
# Fully 2
fc2 = tf.reshape(fc1, [-1, fc2_weights.get_shape().as_list()[0]])
fc2 = tf.nn.relu(tf.matmul(fc2, fc2_weights) + fc2_biases)
# dropout
fc2 = tf.nn.dropout(fc2, 0.5)
# Fully 3
fc3 = tf.reshape(fc2, [-1, fc3_weights.get_shape().as_list()[0]])
out = tf.nn.relu(tf.matmul(fc3, fc3_weights) + fc3_biases)
return out
# Training computation: logits + cross_entropy loss.
logits = model(x)
loss = tf.reduce_mean(
tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=logits, labels=y))
# L2 regularization for the fully connected parameters.
regularizers = (tf.nn.l2_loss(fc1_weights) +
tf.nn.l2_loss(fc1_biases) +
tf.nn.l2_loss(fc2_weights) +
tf.nn.l2_loss(fc2_biases) +
tf.nn.l2_loss(fc3_weights) +
tf.nn.l2_loss(fc3_biases))
loss += 5e-4 * regularizers
# Optimizer: set up a variable that's incremented once per batch
# controls the learning rate decay.
batch = tf.Variable(0, dtype=data_type())
# Decay once per epoch, using an exponential schedule starting at 0.01.
learning_rate = tf.train.exponential_decay(
0.01,
batch * BATCH_SIZE,
train_size,
0.95,
staircase=True)
# Use Adam for the optimization.
optimizer = tf.train.AdamOptimizer(
learning_rate=learning_rate,
beta1=0.9).minimize(loss=loss, global_step=batch)
# Predictions for the current training minibatch.
train_prediction = tf.nn.softmax(logits)
# Predictions for the test and validation.
eval_prediction = tf.nn.softmax(model(eval_data))
def eval_in_batch(session, data):
"""Get all predictions for a dataset by running."""
size = data.shape[0]
if size < EVAL_BATCH_SIZE:
raise ValueError("Batch size for evals larges than dataset:")
pred = np.ndarray(shape=(size, NUM_LABELS))
for begin in range(0, size, EVAL_BATCH_SIZE):
end = begin + EVAL_BATCH_SIZE
if end <= size:
pred[begin:end, :] = session.run(eval_prediction, feed_dict={
eval_data: data[begin:end, ...]})
else:
batch_predictions = session.run(eval_prediction,
feed_dict={eval_data: data[-EVAL_BATCH_SIZE:, ...]})
pred[begin:, :] = batch_predictions[begin - size:, :]
return pred
start_time = time.time()
with tf.Session() as sess:
# Run all the initializers
tf.global_variables_initializer().run()
print("Init all variables complete!")
for step in range(int(num_epochs * train_size) // BATCH_SIZE):
offset = (step * BATCH_SIZE) % (train_size - BATCH_SIZE)
batch_data = train_data[offset:(offset + BATCH_SIZE), ...]
batch_labels = train_labels[offset:(offset + BATCH_SIZE)]
feed_dict = {x: batch_data,
y: batch_labels}
sess.run(optimizer, feed_dict=feed_dict)
if step % EVAL_FREQUENCY == 0:
# fetch some extra node's raw_data
l, lr, predictions = sess.run([loss, learning_rate, train_prediction],
feed_dict=feed_dict)
elapsed_time = time.time() - start_time
start_time = time.time()
print(f"Step {step} "
f"(epoch {(float(step) * BATCH_SIZE / train_size):.2f}) "
f"{(1000 * elapsed_time / EVAL_FREQUENCY):.1f} ms")
print(f"Minibatch loss: {l:.3f}, learning rate: {lr:.6f}")
print(
f"Minibatch error: {error_rate(predictions,batch_labels):.1f}%")
print(
f"Validation error: {error_rate(eval_in_batch(sess, validation_data), validation_labels):.1f}%")
sys.stdout.flush()
# Finally print the result!
test_error = error_rate(eval_in_batch(sess, test_data), test_labels)
print(f"Test error: {test_error:.1f}%.")
if FLAGS.self_test:
print(f"test_error {test_error}")
assert test_error == 0.0, f"expected 0.0 test_error, got {test_error:.2f}"
def data_deal(list1, username): #list1是从spider.py接受到的原始数据
list2 = []
#默认配置
code_list = [
'SE112', 'SE418', 'SE419', 'SE420', 'SE422', 'SE417', 'SE315', 'EI901'
]
name_list = [
"软件工程职业素养， SE112", "软件产品设计与用户体验，SE418", "企业软件质量保证，SE419",
"软件知识产权保护，SE420", "企业软件过程与管理，SE422", "软件工程经济学，SE417", "操作系统，SE315",
"工程实践与科技创新，EI901"
]
if list1 != []:
for person in list1:
student = data2(person.student_ID, person.name, person.department,
person.major, person.grade, person.graduate_time,
person.student_status, person.failed_number,
person.center_credits, person.courses_must_to_take,
person.a_group, person.b_group, person.c_group,
person.d_group,
person.professional_elective_courses,
person.enterprise_education_courses,
person.general_courses, person.others, '无', '无')
change(student, code_list)
#处理one_direction, another_direction两项
a = student.a_group.replace("\xc2\xa0", " ").split(',')
b = student.b_group.replace("\xc2\xa0", " ").split(',')
c = student.c_group.replace("\xc2\xa0", " ").split(',')
d = student.d_group.replace("\xc2\xa0", " ").split(',')
tmp = [a, b, c, d]
for group in tmp:
if len(group) > 2:
if group[2] == ' ':
group[2] = 0
if int(a[2]) + int(a[3]) >= 15:
student.one_direction = a[0]
student.another_direction = int(b[2]) + int(b[3]) + int(
c[2]) + int(c[3])
elif int(b[2]) + int(b[3]) >= 15:
student.one_direction = b[0]
student.another_direction = int(a[2]) + int(a[3]) + int(
c[2]) + int(c[3])
elif int(c[2]) + int(
c[3]) >= 12 and student.others['SE315'] == '通过':
student.one_direction = c[0]
student.another_direction = int(a[2]) + int(a[3]) + int(
b[2]) + int(b[3]) - 3
else:
student.another_direction = int(a[2]) + int(a[3]) + int(
c[2]) + int(c[3]) + int(b[2]) + int(b[3])
list2.append(student)
#储存数据
filename = 'static\\' + username + '.txt'
f = open(filename, 'w')
for person in list1:
one_person = u''
one_person += str(person.student_ID) + ',,' + person.name + ',,' + person.department + ',,' + person.major + ',,' + \
str(person.grade) + ',,' + person.graduate_time + ',,' + person.student_status + ',,' + str(person.failed_number) \
+ ',,' + str(person.center_credits) + ',,' + person.courses_must_to_take + ',,' + person.a_group + ',,' + person.b_group + ',,' + \
person.c_group + ',,' + person.d_group + ',,' + person.professional_elective_courses + ',,' + person.enterprise_education_courses \
+ ',,' + person.general_courses + ',,' + person.others
f.write(one_person + '\n')
f.close()
download(username, list2, code_list, name_list)
return list2
def install():
dl_dir = "C:\\ReviewBoard\\Downloads"
download(url, dl_dir, file_name)
call(["msiexec", "/i", \
"C:\\ReviewBoard\\Downloads\\Silk-Subversion-1.6.17-win32.msi", "/qn"])
print "end of installation"
def on_selection(self, event): w = event.widget selection = w.get(w.curselection()[0]) info = metadata.dictionary(selection) self.general_page.update(info) self.update_icon(download(metadata.icon(selection)))
def process_params(params):
params = get_params(params)
mode = None
try: mode = int(params['mode'])
except: pass
try: url = params['url']
except: url = ''
if mode == _RADIOROOT:
try: return radioRoot(url)
except: pass
if mode == _ONAIR:
try: return onAir(url)
except: pass
if mode == _LISTENAGAIN:
try: return listenAgain(url)
except: pass
if mode == _CHANNEL:
try: return channel(url, params['source'])
except: pass
if mode == _EPISODE:
try: return playEpisode(url, params['name'], params['thumb'], params['fanart'])
except: pass
if mode == _SHOW_DOWNLOAD:
try: return showDownload()
except: pass
if mode == _PLAY_DOWNLOAD:
try: return playDownload(url, params['name'])
except: pass
if mode == _DELETE:
try:
deleteFile(url, params['name'])
return refresh()
except:
pass
if mode == _RESET:
try:
ADDON.setSetting(url, '')
return refresh()
except:
pass
if mode == _DOWNLOAD:
try: return download(url, params['name'])
except: pass
main()
from crawler import *
from download import *
parser = argparse.ArgumentParser(prog="download", conflict_handler="resolve")
parser.add_argument("--key",
"-k",
type=str,
required=True,
help="key word of the music to search")
args = parser.parse_args()
list_link = search(args.key)
print(list_link)
for link in list_link:
download(link)
'NOV', 'DEC'
]
monthNumbers = range(0, 2, 1)
listOfDays = range(0, 31, 1)
pathToSave = "/Users/vinay/PycharmProjects/QuantTradingWithML/Download/src/downloadedFiles/nse/fo/"
secType = "FO"
for year in listOfYears:
for monthInd in monthNumbers:
for dayOfMonth in listOfDays:
day = dayOfMonth + 1
month = listOfMonths[monthInd]
dateStr = str(year) + "-" + month + "-" + str(day)
print "Starting Download for " + dateStr
nseURL = constructNSEurl(secType, day, month, year)
print nseURL
saveAs = "fo" + str(day) + month + str(year) + "bhav.csv.zip"
# weekday = weekDay(year, monthInd, day)
isWeekend = False
# weekday == (6, 'Saturday') or weekday == (0, 'Sunday')
if not isWeekend and download(pathToSave + saveAs, nseURL):
unzip(pathToSave + saveAs, pathToSave)
time.sleep(10)
else:
print "Download wasn't successful for " + dateStr
time.sleep(10)
def download_media(user,
folder,
type='photo',
limit=None,
include_rts=True,
time_range=None):
if time_range:
start_time, end_time = time_range
else:
start_time = time.gmtime(0)
end_time = time.gmtime()
App = twitter_api(user)
page_size = 20
count = page_size
if App.pinned_tweet_id != "":
count = page_size + 1
App.fetch(1)
if (start_time < App.pinned_tweet.created_date < end_time):
media_info = App.pinned_tweet.media_info(type)
download([item for item in media_info], folder)
remain = 20 if not limit else limit
thread = []
i = 0
while True:
# thread=[]
# i=0
if remain <= 20:
count = remain if App.pinned_tweet_id == "" else remain + 1
App.fetch(count)
media_list = []
for tweet_id, tweet in App.tweet_list.items():
if (not include_rts) and (tweet.retweet_id != ""):
continue
if (tweet.created_date < start_time):
return
if not (tweet.created_date < end_time):
continue
if tweet.retweet_id != "":
media_info = App.retweet_list[tweet.retweet_id].media_info(
type)
else:
media_info = tweet.media_info(type)
media_list.extend(media_info)
# thread_download(media_list,folder,4)
# thread download
if i < 4:
t = threading.Thread(target=thread_download,
args=(media_list, folder, 4))
thread.append(t)
t.start()
i += 1
if i == 4:
for t in thread:
t.join()
i = 0
thread = []
remain = remain - page_size if remain > 20 else 0
if not limit:
remain = 20
if remain <= 0 or len(App.timeline) == 0:
break
currently_downloaded = float(numblocks) * blocksize / (1024 * 1024)
kbps_speed = numblocks * blocksize / (time.time() - start_time)
if kbps_speed > 0:
eta = (filesize - numblocks * blocksize) / kbps_speed
else:
eta = 0
kbps_speed = kbps_speed / 1024
total = float(filesize) / (1024 * 1024)
mbs = '%.02f MB of %.02f MB' % (currently_downloaded, total)
e = 'Speed: %.02f Kb/s ' % kbps_speed
e += 'ETA: %02d:%02d' % divmod(eta, 60)
dp.update(percent, mbs, e, ' ')
except:
percent = 100
dp.update(percent)
if dp.iscanceled():
dp.close()
def noconnection():
dialog = xbmcgui.Dialog()
dialog.ok("[COLOR=red][B] ## CONNECTION ERROR ##[/COLOR][/B]",
"Unable to download needed data....", "Will Try Again.",
"Press OK or Back to Continue")
xbmc.sleep(1000)
#dp.close()
download(LOCATION, file2)
xbmc.sleep(1000)
xbmc.executebuiltin('RunAddon(plugin.video.link__tester)')
def __init__(self, pid, password):
self.opener = self.login(pid, password)
self.dl = download(self.opener)
pass
# We are almost done with setting up our database.
# AS a final step we need to also insert the historical prices for various
# indices that trade on the NSE
# NIFTY, BANKNIFTY etc.
# Just like the cm and fo files, there is a daily file published by the NSE with
# index open, low, high ,close for all these indices.
# Let's first define a function to construct the url for this file
# https://www1.nseindia.com/content/indices/ind_close_all_03052016.csv
from download import *
list_of_years = [
2006, 2007, 2008, 2009, 2010, 2011, 2012, 2013, 2014, 2015, 2016
]
localDir = '/Users/swethakolalapudi/pytest/'
for year in list_of_years:
for month in range(12):
for day in range(31):
url = constructIndexURL(day + 1, month + 1, year)
fileName = url.split("/")[-1]
download(localDir + fileName, url)
def install_jar(name, url, filepath):
dst = os.path.join(filepath, name)
if os.path.isfile(dst):
return dst
download(url, dst)
return dst
def install():
file_name = "Git-1.7.6-preview20110708.exe"
url = "http://msysgit.googlecode.com/files/" + file_name
dl_dir = "C:\\ReviewBoard\\Downloads"
download(url, dl_dir, file_name)
call([dl_dir + "\\" + file_name, "/silent"])
def link_crawler(seed_url, link_regex=None, delay=2.0, max_depth=2, max_urls=-1, useProxy=True, local_file=None, save=False):
""" 将网页中符合line_regex正则表达式的链接都筛选出来:
seed_url 根链接
link_regex 匹配的正则规则
delay 延迟
max_length 最大深度
max_urls 最多储存的链接数
local_file 本地保存路径
save 是否保存
"""
begin_time = datetime.now()
pre_time = datetime.now()
crawl_queue = Queue.deque([seed_url]) # 还需要爬的链接的队列
seen = {seed_url: {'depth': 0}} # 初始深度为0 seen用来保存链接和遍历深度
urls_num = 0 # 链接的个数
pages_num = 0 # 页面的个数
index_proxise = -1 # 设置代理的index，同时控制是否进行时延
# rp = get_robots(seed_url) # robot禁止规则
if useProxy: # 从代理池获取IP
proxies = getProxies(url=seed_url, delay=delay, protocol=1)
get_proxies_time = (datetime.now() - begin_time).seconds
num_proxise = len(proxies)
throttle = Throttle(delay) # 速度阀门
okProxy = useProxy and (num_proxise!=0)
while crawl_queue: # 遍历链接
url = crawl_queue.pop()
if okProxy: #循环获取代理进行遍历
if index_proxise == num_proxise:
index_proxise = -1
try:
# if rp.can_fetch('Mozilla/5.0 (Windows NT 6.1; Win64; x64)', url):
if not contain_zh(url): # 防止中文乱码的链接
if index_proxise == -1: throttle.wait(url) # 进行限速
depth = seen[url]['depth']
print u'\n第[', pages_num+1, u']页 已爬取链接数目：', urls_num, u' 深度：', depth
if okProxy:
if index_proxise != -1:
proxy = proxies[index_proxise] #这次使用的代理IP
page = download(url=url, proxy=proxies[index_proxise])
else:
page = download(url) # 使用本机IP访问
index_proxise += 1
else:
page = download(url)
links = []
if depth != max_depth:
links_ = getbyre(page)
if links_ is not None: # 防止空页面
if link_regex: # 如果有给正则表达式则进行匹配
links.extend(link for link in links_ if re.match(link_regex, link))
else:
links.extend(link for link in links_)
for link in links:
link = normalize(seed_url, link) # 将链接进行规范化，转为绝对链接
if link not in seen: # 如果链接没有重复
pre_time = datetime.now()
a_link = {}
a_link['depth'] = depth + 1
seen[link] = a_link # 保存链接
urls_num += 1
if same_domain(seed_url, link): # 如果来自同一个域名, 将链接加入遍历队伍中
crawl_queue.append(link)
pages_num += 1
except Exception, e:
print u'访问超时或其他错误'
# 如果爬取的链接数目超出上限了或者超过600秒链接数都不再增加，视为已经爬完，避免爬虫陷阱 or (datetime.now() - pre_time).seconds > 600
if urls_num >= max_urls and max_urls != -1:
break
listOfYears= [2006,2007,2008,2009,2010,2011,2012,2013,2014,2015,2016]
for year in listOfYears:
for month in listOfMonths:
for dayOfMonth in range(31):
day=dayOfMonth+1
# range(31) will create a sequence starting from 0 but dates start from 1
# so let's add 1
nseURL=constructNSEurl("CM",day,month,year)
fileName="cm" + str(day) + month +str(year)+"bhav.csv.zip"
localFilePath = "/Users/swethakolalapudi/pytest/"
download(localFilePath+fileName,nseURL)
unzip(localFilePath+fileName,localFilePath)
# This will first download and then unzip our file in the given location
time.sleep(10)
# We give it some time between each download request so we don't inadvertently
# overwhelm the NSE website
tracklist = []
try:
if uri[1] == "user":
print("\n" + "||||||||||||||||||||||||||||||||||||||")
print("Playlist Details: ")
print("User: "******"Playlist ID: " + uri[4])
tracklist = get_playlist(spotify, uri[2], uri[4])
elif uri[1] == "album":
print("\n" + "||||||||||||||||||||||||||||||||||||||")
print("Album Details: ")
print("Album ID: " + uri[2])
tracklist = get_album(spotify, uri[2])
except:
print("Invalid URI!")
sys.exit(0)
print("\n" + "||||||||||||||||||||||||||||||||||||||")
print("T R A C K L I S T")
for song in tracklist:
print(song)
print("\n" + "||||||||||||||||||||||||||||||||||||||")
ch = input("Download All Songs (y/n): ")
if ch == 'n' or ch == 'N':
sys.exit(0)
for song in tracklist:
download(song)
for p in sorted(records.keys()):
n = records.get(p, 0)
if n >= out_num:
nn = n - out_num
out_num = 0
else:
out_num = out_num - n
nn = 0
records.update({p: nn})
if out_num == 0:
break
precious_num = records.get(current_price, 0)
current_num = precious_num + volume[i]
records.update({current_price: current_num})
# print(current_price)
# print(dates[i], current_price, floatCapitalOfAShares[i], floatSharesOfAShares[i])
price = [k for k in records.keys()]
num = [records[k] for k in records.keys()]
print(code_name[c], min([k for k in records.keys() if records[k]]))
# l1=plt.bar(price,num)
# plt.title(c+" - "+et)
# plt.show()
from download import download
et = '2021-01-28'
download(et=et)
for c in code_list:
cal(c, et)
