def scrape_historic_daily_time_series(symbol): log(name, f"Scrapping {symbol}") url = f"https://www.alphavantage.co/query?function=TIME_SERIES_DAILY&symbol={symbol}&apikey={config['api_key']}&outputsize=full" response = requests.get(url) data = json.dumps(response.json()["Time Series (Daily)"]) save(f"{symbol}_Daily.txt", data, 'w') log(name, f"{symbol} Dump file saved.")
def position_detail(self, url): text = self.send_request(url) obj = BeautifulSoup(text, 'lxml') data = {} data['url'] = url data['source'] = u'拉勾网' data['create_date'] = datetime.now() data['_id'] = int(re.findall(r'\d{5,10}', url)[0]) data['position'] = obj.find('span', class_='name').get_text().strip() job_request = [ span.string.replace('/', '').strip() for span in obj.find('dd', class_='job_request').find_all('span') ] data['salary'], data['address'], data['exp'], data[ 'edu'], span = job_request data['release_date'] = obj.find( 'p', class_='publish_time').get_text().split()[0].strip() data['company'] = obj.find('h2', class_='fl').contents[0].strip() data['desc'] = obj.find('dd', class_='job_bt').div.get_text().strip() data['label'] = [ ul.string.strip() for ul in obj.find('ul', class_='position-label') if ul.string.strip() ] data['position_type'] = obj.find( 'ul', class_='c_feature').li.contents[2].strip() store.save(data)
def main(argv): if len(argv) < 2: print __doc__ sys.exit(-1) pathname = argv[1] store = getStore(False) store.load(pathname) wlib = getWeblib() print "Loaded %s\ncategory_description:\n%s\n#tags %s\n#webpages %s" % ( argv[1], wlib.category.getDescription().encode("raw_unicode_escape")[:300], len(wlib.tags), len(wlib.webpages), ) newTag = weblib.Tag(name="hello tag") print >> sys.stderr, "id", newTag.id newTag = store.writeTag(newTag) print >> sys.stderr, "id", newTag.id store.removeItem(newTag) # save if len(argv) > 2: store.save(wlib, argv[2])
def removes(list): user_input = input("\nPlease enter the number to remove: ") print("") n = int(user_input) - 1 list.pop(n) store.save(list)
def complete(list): user_input = input("\nPlease enter the number to complete: ") print("") n = int(user_input) - 1 list[n]["status"] = "DONE" store.save(list)
def fetch_sentence_images(): content = load() for index, sentence in enumerate(content["sentences"]): query = content["search_term"] + ' ' + sentence['keywords'][0] content["sentences"][index]["google_search_query"] = query content["sentences"][index]["images"] = fetch_images_urls(query) save(content)
def break_into_sentences(limit=5): content = load() content["sentences"] = [] for sentence in sent_tokenize(content["source_sanitized_content"])[:limit]: content["sentences"].append({'text':sentence, 'keywords':watson_keywords(sentence), 'images':[]}) save(content)
def remove_completed(list): for n in list: if n["status"] == "DONE": list.remove(n) for n in list: if n["status"] == "DONE": list.remove(n) print("\nAll DONE items deleted!\n") store.save(list)
def redis_expiration_test(): save('test_redis/expiration', 'Yeah') save('test_redis/expiration1', 'Yeah') expire('test_redis/expiration1', 1) expire('test_redis/expiration', 10) assert (expiration('test_redis/expiration') == 10) assert (expiration('test_redis/expiration1') == 1) assert (recall('test_redis/expiration1') == 'Yeah') sleep(1) assert (expiration('test_redis/expiration') == 9) assert (not recall('test_redis/expiration1'))
def redis_expiration_test(): save ('test_redis/expiration','Yeah') save ('test_redis/expiration1','Yeah') expire('test_redis/expiration1',1) expire('test_redis/expiration',10) assert(expiration('test_redis/expiration')==10) assert(expiration('test_redis/expiration1')==1) assert(recall('test_redis/expiration1')=='Yeah') sleep(1) assert(expiration('test_redis/expiration')==9) assert(not recall('test_redis/expiration1'))
def fetch_wikipedia_article(): content = load() algorithmia_key = "" with open("credentials.json", "r") as file: algorithmia_key = json.loads(file.read())["algorithmia_key"] client = Algorithmia.client(algorithmia_key) algo = client.algo('web/WikipediaParser/0.1.2') content["source_original_content"] = algo.pipe(content['search_term']).result["content"] save(content)
def sanitize_content(): content = load() article = content['source_original_content'] # Removes blank lines and markdown article = "".join([i for i in article.splitlines(True) if not (len(i) < 2 or i.startswith('='))]) # Removes dates in parentheses article = re.sub("/\((?:\([^()]*\)|[^()])*\)/gm", '', article) content["source_sanitized_content"] = article save(content)
def scrape(url, file, parser, attempts): if (attempts > 3): log(name, 'Failure, too many attempts') response = balanz.session.get(url) if (response.status_code == 401 or response.status_code == 403): log(name, 'Auth Error!') balanz.refreshSession() scrape(url, file, parser, attempts + 1) elif (response.status_code == 200): data = json.loads(response.content)['cotizaciones'] data = [*map(parser, data)] dump = '\n'.join(data) + '\n' save(file, dump, 'a+') #change to dump
def diff(testname): t1 = output(testname) t2 = correct(testname) if not t2: save(tname(testname)+'.correct',t1) t2 = t1 if t1!=t2: print '\n_____________________________________________________\n' print 'FAIL: %-35s'%testname, len(t1), len(t2) print '_____________________________________________________' print diff_string(t1,t2) return True else: return False
def diff(testname): t1 = output(testname) t2 = correct(testname) if not t2: save(tname(testname) + '.correct', t1) t2 = t1 if t1 != t2: print '\n_____________________________________________________\n' print 'FAIL: %-35s' % testname, len(t1), len(t2) print '_____________________________________________________' print diff_string(t1, t2) return True else: return False
def page_diff(url): t1 = page_text(url) t2 = page_correct(url) if not t2: save(url + '.correct', t1) t2 = t1 if t1 != t2: print '\n_____________________________________________________\n' print 'FAIL: %-35s' % url, len(t1), len(t2) print '_____________________________________________________' print diff_string(t1, t2) return True else: return False
def page_diff(url): t1 = page_text(url) t2 = page_correct(url) if not t2: save(url+'.correct',t1) t2 = t1 if t1!=t2: print '\n_____________________________________________________\n' print 'FAIL: %-35s'%url, len(t1), len(t2) print '_____________________________________________________' print diff_string(t1,t2) return True else: return False
def execute_test(testname): if not is_cached(testname) or diff(testname): start = datetime.now() #print 'Execute shell command', tname(testname)+'.tst' if not exists(tname(testname)+'.tst'): print 'error: Missing command',tname(testname)+'.tst' text = do_command('bash '+tname(testname)+'.tst') end = datetime.now() t = end-start print "%d.%1d seconds"%(t.seconds, t.microseconds/100000) #print 'OUTPUT: \n ',tname(testname)+'.out', '\n ',recall(tname(testname)+'.out'), '\n', save(tname(testname)+'.out', text) save(tname(testname)+'.cache', text) expire(tname(testname)+'.cache', 10+t.seconds*100)
def execute_test(testname): if not is_cached(testname) or diff(testname): start = datetime.now() #print 'Execute shell command', tname(testname)+'.tst' if not exists(tname(testname) + '.tst'): print 'error: Missing command', tname(testname) + '.tst' text = do_command('bash ' + tname(testname) + '.tst') end = datetime.now() t = end - start print "%d.%1d seconds" % (t.seconds, t.microseconds / 100000) #print 'OUTPUT: \n ',tname(testname)+'.out', '\n ',recall(tname(testname)+'.out'), '\n', save(tname(testname) + '.out', text) save(tname(testname) + '.cache', text) expire(tname(testname) + '.cache', 10 + t.seconds * 100)
def main(): store.load() while True: # Run the program in the loop. run() print() # Ask the user if he wants more entries. should_continue = input('More Entries (Y/N)? ') # If the answer is not 'Y', exit. if should_continue.upper() != 'Y': break # This will save the user data to a file. store.save()
def post(self): for field_name, fs in self.request.files.items(): for f in fs: filename, body, content_type = f["filename"], f['body'], f["content_type"] path = store.save(body, filename) break self.json(R.ok().add("url", path))
def saveCurPos (fileName, line, col): assert fileName != None if (line == 0) and (col == 0): pass else: db = store.load(storeName, {}) key = os.path.realpath(fileName) oldVal = db.get(key) if oldVal == None: toDel = None for k, v in db.iteritems(): l, c, last = v if last == CUR_POS_DB_LEN - 1: assert toDel == None toDel = k else: db[k] = (l, c, last + 1) if toDel != None: del db[toDel] if Trace: print toDel, 'removed from cursor position db' else: oldLast = oldVal[2] for k, v in db.iteritems(): l, c, last = v if last < oldLast: db[k] = (l, c, last + 1) db[key] = (line, col, 0) ok = store.save(storeName, db) if ok: if Trace: print 'saved cursor position:', line, col
def add_network_saved(network): if not 'ssid' in network: raise ValueError() if not 'pass' in network: raise ValueError() network_saved_list = store.load('networks', []) # Remove a previous entry if it was there... network_saved_list = [ x for x in network_saved_list if x['ssid'] != network['ssid']] network_saved_list.append(network) store.save('networks', network_saved_list)
def set_state(self, state): # filter out empty patterns for node in state['nodes']: for pattern in state['nodes'][node]['properties']['patterns'][:]: if not pattern: state['nodes'][node]['properties']['patterns'].remove( pattern) return save('state', state)
def render_components(self): images = requests.get(url=self.baseURI + '/v1/images/' + self.projectKey, params={ 'format': 'svg', 'ids': ','.join(list(self.get_components().keys())) }, headers=self.headers).json()['images'] return save('images', images)
def redis_test(): #assert(recall('test_redis/name')=='Eric') assert(save ('test_redis/name','Mark')==None) assert(recall('test_redis/name')=='Mark') assert(save ('test_redis/name','Eric')==None) assert(recall('test_redis/name')=='Eric')
def accept(collection): t = recall(collection + '/current') save(collection + '/past', t)
def next_counts(collection, text): save(collection + '/current', text)
def save_page_text(browser,page): text = get_page_text(browser,page) save(page+'.txt', text) page_diff(page)
def correct_load(testname, host): correct_text = read_text(join(environ['pt'],host,testname))[:-1] save(tname(testname)+'.correct', correct_text)
def previous_counts(collection, text): save(collection+'/past',text)
def next_counts(collection, text): save(collection+'/current',text)
def save_code(testname, code): save(tname(testname)+'.tst', code)
def save_code(testname, code): save(tname(testname) + '.tst', code)
def update_user_poster(data, pd, id): code = data.get('code', C.code()) name = data['name'] buf, mimetype = poster.drawio(pd, 0.4) path = store.save(buf.getvalue(), f"a.{pd['type']}", 'preview') return db_update_poster(id, code, name, path, data['json'])
def like(testname): text = recall(tname(testname) + '.out') save(tname(testname) + '.correct', text)
def save_user_poster(data, pd): code = C.code() name = data['name'] buf, mimetype = poster.drawio(pd, 0.4) path = store.save(buf.getvalue(), f"a.{pd['type']}", 'preview') return db_save_poster(code, name, path, data['json'])
def get_updates(): check.check_and_update() store.print_update(updater.bot) threading.Timer(60 * 5, get_updates).start() store.save()
prompt = "This place does not exist, please type again" + "\n" else: prompt = "You may have already told this, please try again" + "\n" isPlaying += 1 elif reply_ending.upper() != current_city[:1].upper(): prompt = "It is not starting with " + reply_ending.upper( ) + ", try again" + "\n" isPlaying += 1 else: score += 10 content = content.replace(current_city, '') ending = current_city[-1:] reply = city.get_city(ending, content) tts2.speak(reply) content = content.replace(reply, '') print("Computer replies with " + reply + "...\n") reply_ending = reply[-1:] prompt = 'Tell a city or a country starting with ' + reply_ending.upper( ) + "\n" store.save(score, name) leaderboard = store.leader() if (leaderboard['1st'][1] > score): tts2.speak("Don't worry, Not everyone is a computer") else: tts2.speak("Go get a life, Nerd!") print("\n 1st : " + leaderboard['1st'][0] + " - " + str(leaderboard['1st'][1]) + "\n 2nd : " + leaderboard['2nd'][0] + " - " + str(leaderboard['2nd'][1]) + "\n 3rd : " + leaderboard['3rd'][0] + " - " + str(leaderboard['3rd'][1]) + "\n") print("Game Over!! Your score is " + str(score))
def redis_test(): #assert(recall('test_redis/name')=='Eric') assert (save('test_redis/name', 'Mark') == None) assert (recall('test_redis/name') == 'Mark') assert (save('test_redis/name', 'Eric') == None) assert (recall('test_redis/name') == 'Eric')
def like(testname): text = recall(tname(testname)+'.out') save(tname(testname)+'.correct', text)
def add(list): user_input = input("\033[47m" + "\nPlease enter a new items to your to-do list: ") user_input_date = input("Enter the due date in the format(2015.12.03): ") list.append({"description": user_input, "status": "TO DO", "due date": user_input_date}) store.save(list)
def correct_load(testname, host): correct_text = read_text(join(environ['pt'], host, testname))[:-1] save(tname(testname) + '.correct', correct_text)
def accept(collection): t = recall(collection+'/current') save(collection+'/past',t)
def previous_counts(collection, text): save(collection + '/past', text)
choice = int(input("Choose a task: ")) print('----' * 10) except ValueError: print('You have to enter the number of your choice') else: if choice == 1: commands.print_todos(items) print('--' *16) if choice == 2: commands.new_element(items) if choice == 3: commands.print_in_progress(items) if choice == 4: commands.remaining_time(items) if choice == 5: commands.expiring(items) if choice == 6: commands.complete_task(items) if choice == 7: commands.print_completed(items) if choice == 8: commands.remove_task(items) if choice == 9: for item in items: print(item) if choice == 10: commands.make_task_inprogress(items) if choice == 11: store.save(items, 'todos.txt') exit()
answer = {'message': 'csdgegbmerngmekjtnerinevrigunrg w'} elif message_list[1] == 'help': answer = { 'message': '--alive It alives the bot <br>--poweroff It do nothing<br>--cmatrix :)' } socketio.emit('bot response', answer) #emits bot self conversation def bot_speaking(): time.sleep(5) dialog = [ 'hi', 'are you alone?', 'im very lonely', 'say something', 'i dont hear you', 'it was my fault', 'forgive me', 'say smth', 'please', 'i beg you', 'ok, good bye' ] for number in range(len(dialog) - 1): socketio.emit('bot response', {"message": dialog[number]}) time.sleep(1) if __name__ == "__main__": try: socketio.run(app, host='localhost', port=5000, debug=True) except Exception as e: print(e) print('why we here?') store.save()
def saveMeshData(self, event=None): save(self.canvasFrame.mesh.save(), self.inputimage)
def saveTrimsOnEEPROM(self): trims = [0, 0, 0, 0, 0, 0] for i in range(0, self._servo_totals): trims[i] = self._servo[i].getTrim() store.save('Trims', trims)
if old != None: r = dialog.set_font_name(old) if Trace: print 'set font name:', r resp = dialog.run() if resp == gtk.RESPONSE_OK: new = dialog.get_font_name() if Trace: print 'selected font:', new else: new = old dialog.destroy() return new defaultSettings = { 'font': None } _settingsStoreName = 'settings' loadSettings = lambda: store.load(_settingsStoreName, defaultSettings) saveSettings = lambda settings: store.save(_settingsStoreName, settings) def translateBuilder (builder): for obj in builder.get_objects(): # делаем так, потому что при попытке получения свойств по крейней мере у SeparatorMenuItem он исчезает (где-то ошибка в gtk или в pygtk) if type(obj) in (gtk.Window, gtk.MenuItem, gtk.Label, gtk.Button, gtk.CheckButton): if type(obj) is gtk.Window: label = obj.get_title().decode('utf-8') else: label = obj.get_property('label').decode('utf-8') # print label if label.startswith('#'): label = tr(label.encode('ascii')) if type(obj) is gtk.Window: obj.set_title(label.encode('utf-8')) else:
def like(url): text = recall(url+'.txt') save(url+'.correct', text)