def scrape_historic_daily_time_series(symbol):
log(name, f"Scrapping {symbol}")
url = f"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol={symbol}&apikey={config['api_key']}&outputsize=full"
response = requests.get(url)
data = json.dumps(response.json()["Time Series (Daily)"])
save(f"{symbol}_Daily.txt", data, 'w')
log(name, f"{symbol} Dump file saved.")
def position_detail(self, url):
text = self.send_request(url)
obj = BeautifulSoup(text, 'lxml')
data = {}
data['url'] = url
data['source'] = u'拉勾网'
data['create_date'] = datetime.now()
data['_id'] = int(re.findall(r'\d{5,10}', url)[0])
data['position'] = obj.find('span', class_='name').get_text().strip()
job_request = [
span.string.replace('/', '').strip()
for span in obj.find('dd', class_='job_request').find_all('span')
]
data['salary'], data['address'], data['exp'], data[
'edu'], span = job_request
data['release_date'] = obj.find(
'p', class_='publish_time').get_text().split()[0].strip()
data['company'] = obj.find('h2', class_='fl').contents[0].strip()
data['desc'] = obj.find('dd', class_='job_bt').div.get_text().strip()
data['label'] = [
ul.string.strip() for ul in obj.find('ul', class_='position-label')
if ul.string.strip()
]
data['position_type'] = obj.find(
'ul', class_='c_feature').li.contents[2].strip()
store.save(data)
def main(argv):
if len(argv) < 2:
print __doc__
sys.exit(-1)
pathname = argv[1]
store = getStore(False)
store.load(pathname)
wlib = getWeblib()
print "Loaded %s\ncategory_description:\n%s\n#tags %s\n#webpages %s" % (
argv[1],
wlib.category.getDescription().encode("raw_unicode_escape")[:300],
len(wlib.tags),
len(wlib.webpages),
)
newTag = weblib.Tag(name="hello tag")
print >> sys.stderr, "id", newTag.id
newTag = store.writeTag(newTag)
print >> sys.stderr, "id", newTag.id
store.removeItem(newTag)
# save
if len(argv) > 2:
store.save(wlib, argv[2])
def removes(list):
user_input = input("\nPlease enter the number to remove: ")
print("")
n = int(user_input) - 1
list.pop(n)
store.save(list)
def complete(list):
user_input = input("\nPlease enter the number to complete: ")
print("")
n = int(user_input) - 1
list[n]["status"] = "DONE"
store.save(list)
def fetch_sentence_images():
content = load()
for index, sentence in enumerate(content["sentences"]):
query = content["search_term"] + ' ' + sentence['keywords'][0]
content["sentences"][index]["google_search_query"] = query
content["sentences"][index]["images"] = fetch_images_urls(query)
save(content)
def break_into_sentences(limit=5):
content = load()
content["sentences"] = []
for sentence in sent_tokenize(content["source_sanitized_content"])[:limit]:
content["sentences"].append({'text':sentence, 'keywords':watson_keywords(sentence), 'images':[]})
save(content)
def remove_completed(list):
for n in list:
if n["status"] == "DONE":
list.remove(n)
for n in list:
if n["status"] == "DONE":
list.remove(n)
print("\nAll DONE items deleted!\n")
store.save(list)
def redis_expiration_test():
save('test_redis/expiration', 'Yeah')
save('test_redis/expiration1', 'Yeah')
expire('test_redis/expiration1', 1)
expire('test_redis/expiration', 10)
assert (expiration('test_redis/expiration') == 10)
assert (expiration('test_redis/expiration1') == 1)
assert (recall('test_redis/expiration1') == 'Yeah')
sleep(1)
assert (expiration('test_redis/expiration') == 9)
assert (not recall('test_redis/expiration1'))
def redis_expiration_test():
save ('test_redis/expiration','Yeah')
save ('test_redis/expiration1','Yeah')
expire('test_redis/expiration1',1)
expire('test_redis/expiration',10)
assert(expiration('test_redis/expiration')==10)
assert(expiration('test_redis/expiration1')==1)
assert(recall('test_redis/expiration1')=='Yeah')
sleep(1)
assert(expiration('test_redis/expiration')==9)
assert(not recall('test_redis/expiration1'))
def fetch_wikipedia_article():
content = load()
algorithmia_key = ""
with open("credentials.json", "r") as file:
algorithmia_key = json.loads(file.read())["algorithmia_key"]
client = Algorithmia.client(algorithmia_key)
algo = client.algo('web/WikipediaParser/0.1.2')
content["source_original_content"] = algo.pipe(content['search_term']).result["content"]
save(content)
def sanitize_content():
content = load()
article = content['source_original_content']
# Removes blank lines and markdown
article = "".join([i for i in article.splitlines(True) if not (len(i) < 2 or i.startswith('='))])
# Removes dates in parentheses
article = re.sub("/\((?:\([^()]*\)|[^()])*\)/gm", '', article)
content["source_sanitized_content"] = article
save(content)
def scrape(url, file, parser, attempts):
if (attempts > 3):
log(name, 'Failure, too many attempts')
response = balanz.session.get(url)
if (response.status_code == 401 or response.status_code == 403):
log(name, 'Auth Error!')
balanz.refreshSession()
scrape(url, file, parser, attempts + 1)
elif (response.status_code == 200):
data = json.loads(response.content)['cotizaciones']
data = [*map(parser, data)]
dump = '\n'.join(data) + '\n'
save(file, dump, 'a+') #change to dump
def diff(testname):
t1 = output(testname)
t2 = correct(testname)
if not t2:
save(tname(testname)+'.correct',t1)
t2 = t1
if t1!=t2:
print '\n_____________________________________________________\n'
print 'FAIL: %-35s'%testname, len(t1), len(t2)
print '_____________________________________________________'
print diff_string(t1,t2)
return True
else:
return False
def diff(testname):
t1 = output(testname)
t2 = correct(testname)
if not t2:
save(tname(testname) + '.correct', t1)
t2 = t1
if t1 != t2:
print '\n_____________________________________________________\n'
print 'FAIL: %-35s' % testname, len(t1), len(t2)
print '_____________________________________________________'
print diff_string(t1, t2)
return True
else:
return False
def page_diff(url):
t1 = page_text(url)
t2 = page_correct(url)
if not t2:
save(url + '.correct', t1)
t2 = t1
if t1 != t2:
print '\n_____________________________________________________\n'
print 'FAIL: %-35s' % url, len(t1), len(t2)
print '_____________________________________________________'
print diff_string(t1, t2)
return True
else:
return False
def page_diff(url):
t1 = page_text(url)
t2 = page_correct(url)
if not t2:
save(url+'.correct',t1)
t2 = t1
if t1!=t2:
print '\n_____________________________________________________\n'
print 'FAIL: %-35s'%url, len(t1), len(t2)
print '_____________________________________________________'
print diff_string(t1,t2)
return True
else:
return False
def execute_test(testname):
if not is_cached(testname) or diff(testname):
start = datetime.now()
#print 'Execute shell command', tname(testname)+'.tst'
if not exists(tname(testname)+'.tst'):
print 'error: Missing command',tname(testname)+'.tst'
text = do_command('bash '+tname(testname)+'.tst')
end = datetime.now()
t = end-start
print "%d.%1d seconds"%(t.seconds, t.microseconds/100000)
#print 'OUTPUT: \n ',tname(testname)+'.out', '\n ',recall(tname(testname)+'.out'), '\n',
save(tname(testname)+'.out', text)
save(tname(testname)+'.cache', text)
expire(tname(testname)+'.cache', 10+t.seconds*100)
def execute_test(testname):
if not is_cached(testname) or diff(testname):
start = datetime.now()
#print 'Execute shell command', tname(testname)+'.tst'
if not exists(tname(testname) + '.tst'):
print 'error: Missing command', tname(testname) + '.tst'
text = do_command('bash ' + tname(testname) + '.tst')
end = datetime.now()
t = end - start
print "%d.%1d seconds" % (t.seconds, t.microseconds / 100000)
#print 'OUTPUT: \n ',tname(testname)+'.out', '\n ',recall(tname(testname)+'.out'), '\n',
save(tname(testname) + '.out', text)
save(tname(testname) + '.cache', text)
expire(tname(testname) + '.cache', 10 + t.seconds * 100)
def main():
store.load()
while True:
# Run the program in the loop.
run()
print()
# Ask the user if he wants more entries.
should_continue = input('More Entries (Y/N)? ')
# If the answer is not 'Y', exit.
if should_continue.upper() != 'Y':
break
# This will save the user data to a file.
store.save()
def post(self):
for field_name, fs in self.request.files.items():
for f in fs:
filename, body, content_type = f["filename"], f['body'], f["content_type"]
path = store.save(body, filename)
break
self.json(R.ok().add("url", path))
def saveCurPos (fileName, line, col):
assert fileName != None
if (line == 0) and (col == 0):
pass
else:
db = store.load(storeName, {})
key = os.path.realpath(fileName)
oldVal = db.get(key)
if oldVal == None:
toDel = None
for k, v in db.iteritems():
l, c, last = v
if last == CUR_POS_DB_LEN - 1:
assert toDel == None
toDel = k
else:
db[k] = (l, c, last + 1)
if toDel != None:
del db[toDel]
if Trace: print toDel, 'removed from cursor position db'
else:
oldLast = oldVal[2]
for k, v in db.iteritems():
l, c, last = v
if last < oldLast:
db[k] = (l, c, last + 1)
db[key] = (line, col, 0)
ok = store.save(storeName, db)
if ok:
if Trace: print 'saved cursor position:', line, col
def add_network_saved(network):
if not 'ssid' in network:
raise ValueError()
if not 'pass' in network:
raise ValueError()
network_saved_list = store.load('networks', [])
# Remove a previous entry if it was there...
network_saved_list = [
x for x in network_saved_list if x['ssid'] != network['ssid']]
network_saved_list.append(network)
store.save('networks', network_saved_list)
def set_state(self, state):
# filter out empty patterns
for node in state['nodes']:
for pattern in state['nodes'][node]['properties']['patterns'][:]:
if not pattern:
state['nodes'][node]['properties']['patterns'].remove(
pattern)
return save('state', state)
def render_components(self):
images = requests.get(url=self.baseURI + '/v1/images/' +
self.projectKey,
params={
'format':
'svg',
'ids':
','.join(list(self.get_components().keys()))
},
headers=self.headers).json()['images']
return save('images', images)
def redis_test():
#assert(recall('test_redis/name')=='Eric')
assert(save ('test_redis/name','Mark')==None)
assert(recall('test_redis/name')=='Mark')
assert(save ('test_redis/name','Eric')==None)
assert(recall('test_redis/name')=='Eric')
def accept(collection):
t = recall(collection + '/current')
save(collection + '/past', t)
def next_counts(collection, text):
save(collection + '/current', text)
def save_page_text(browser,page):
text = get_page_text(browser,page)
save(page+'.txt', text)
page_diff(page)
def correct_load(testname, host):
correct_text = read_text(join(environ['pt'],host,testname))[:-1]
save(tname(testname)+'.correct', correct_text)
def previous_counts(collection, text):
save(collection+'/past',text)
def next_counts(collection, text):
save(collection+'/current',text)
def save_code(testname, code):
save(tname(testname)+'.tst', code)
def save_code(testname, code):
save(tname(testname) + '.tst', code)
def update_user_poster(data, pd, id):
code = data.get('code', C.code())
name = data['name']
buf, mimetype = poster.drawio(pd, 0.4)
path = store.save(buf.getvalue(), f"a.{pd['type']}", 'preview')
return db_update_poster(id, code, name, path, data['json'])
def like(testname):
text = recall(tname(testname) + '.out')
save(tname(testname) + '.correct', text)
def save_user_poster(data, pd):
code = C.code()
name = data['name']
buf, mimetype = poster.drawio(pd, 0.4)
path = store.save(buf.getvalue(), f"a.{pd['type']}", 'preview')
return db_save_poster(code, name, path, data['json'])
def get_updates():
check.check_and_update()
store.print_update(updater.bot)
threading.Timer(60 * 5, get_updates).start()
store.save()
prompt = "This place does not exist, please type again" + "\n"
else:
prompt = "You may have already told this, please try again" + "\n"
isPlaying += 1
elif reply_ending.upper() != current_city[:1].upper():
prompt = "It is not starting with " + reply_ending.upper(
) + ", try again" + "\n"
isPlaying += 1
else:
score += 10
content = content.replace(current_city, '')
ending = current_city[-1:]
reply = city.get_city(ending, content)
tts2.speak(reply)
content = content.replace(reply, '')
print("Computer replies with " + reply + "...\n")
reply_ending = reply[-1:]
prompt = 'Tell a city or a country starting with ' + reply_ending.upper(
) + "\n"
store.save(score, name)
leaderboard = store.leader()
if (leaderboard['1st'][1] > score):
tts2.speak("Don't worry, Not everyone is a computer")
else:
tts2.speak("Go get a life, Nerd!")
print("\n 1st : " + leaderboard['1st'][0] + " - " +
str(leaderboard['1st'][1]) + "\n 2nd : " + leaderboard['2nd'][0] +
" - " + str(leaderboard['2nd'][1]) + "\n 3rd : " +
leaderboard['3rd'][0] + " - " + str(leaderboard['3rd'][1]) + "\n")
print("Game Over!! Your score is " + str(score))
def redis_test():
#assert(recall('test_redis/name')=='Eric')
assert (save('test_redis/name', 'Mark') == None)
assert (recall('test_redis/name') == 'Mark')
assert (save('test_redis/name', 'Eric') == None)
assert (recall('test_redis/name') == 'Eric')
def like(testname):
text = recall(tname(testname)+'.out')
save(tname(testname)+'.correct', text)
def add(list):
user_input = input("\033[47m" + "\nPlease enter a new items to your to-do list: ")
user_input_date = input("Enter the due date in the format(2015.12.03): ")
list.append({"description": user_input, "status": "TO DO", "due date": user_input_date})
store.save(list)
def correct_load(testname, host):
correct_text = read_text(join(environ['pt'], host, testname))[:-1]
save(tname(testname) + '.correct', correct_text)
def accept(collection):
t = recall(collection+'/current')
save(collection+'/past',t)
def previous_counts(collection, text):
save(collection + '/past', text)
choice = int(input("Choose a task: "))
print('----' * 10)
except ValueError:
print('You have to enter the number of your choice')
else:
if choice == 1:
commands.print_todos(items)
print('--' *16)
if choice == 2:
commands.new_element(items)
if choice == 3:
commands.print_in_progress(items)
if choice == 4:
commands.remaining_time(items)
if choice == 5:
commands.expiring(items)
if choice == 6:
commands.complete_task(items)
if choice == 7:
commands.print_completed(items)
if choice == 8:
commands.remove_task(items)
if choice == 9:
for item in items:
print(item)
if choice == 10:
commands.make_task_inprogress(items)
if choice == 11:
store.save(items, 'todos.txt')
exit()
answer = {'message': 'csdgegbmerngmekjtnerinevrigunrg w'}
elif message_list[1] == 'help':
answer = {
'message':
'--alive It alives the bot <br>--poweroff It do nothing<br>--cmatrix :)'
}
socketio.emit('bot response', answer)
#emits bot self conversation
def bot_speaking():
time.sleep(5)
dialog = [
'hi', 'are you alone?', 'im very lonely', 'say something',
'i dont hear you', 'it was my fault', 'forgive me', 'say smth',
'please', 'i beg you', 'ok, good bye'
]
for number in range(len(dialog) - 1):
socketio.emit('bot response', {"message": dialog[number]})
time.sleep(1)
if __name__ == "__main__":
try:
socketio.run(app, host='localhost', port=5000, debug=True)
except Exception as e:
print(e)
print('why we here?')
store.save()
def saveMeshData(self, event=None):
save(self.canvasFrame.mesh.save(), self.inputimage)
def saveTrimsOnEEPROM(self):
trims = [0, 0, 0, 0, 0, 0]
for i in range(0, self._servo_totals):
trims[i] = self._servo[i].getTrim()
store.save('Trims', trims)
if old != None:
r = dialog.set_font_name(old)
if Trace: print 'set font name:', r
resp = dialog.run()
if resp == gtk.RESPONSE_OK:
new = dialog.get_font_name()
if Trace: print 'selected font:', new
else:
new = old
dialog.destroy()
return new
defaultSettings = { 'font': None }
_settingsStoreName = 'settings'
loadSettings = lambda: store.load(_settingsStoreName, defaultSettings)
saveSettings = lambda settings: store.save(_settingsStoreName, settings)
def translateBuilder (builder):
for obj in builder.get_objects():
# делаем так, потому что при попытке получения свойств по крейней мере у SeparatorMenuItem он исчезает (где-то ошибка в gtk или в pygtk)
if type(obj) in (gtk.Window, gtk.MenuItem, gtk.Label, gtk.Button, gtk.CheckButton):
if type(obj) is gtk.Window:
label = obj.get_title().decode('utf-8')
else:
label = obj.get_property('label').decode('utf-8')
# print label
if label.startswith('#'):
label = tr(label.encode('ascii'))
if type(obj) is gtk.Window:
obj.set_title(label.encode('utf-8'))
else:
def like(url):
text = recall(url+'.txt')
save(url+'.correct', text)
