def get_live_from_google(self):
latest = Scrap()
print "Getting Live Data from Google (Infinte Loop) : ", self.stock.name
while (True):
data = latest.get_from_google(self.stock, 300, 1, True)
for d in data:
self.db.proper_insert(d)
time.sleep(30)
def get_max_from_google(self):
max = Scrap()
print "Getting N days Records from Google for :", self.max_days, self.stock.name
data = max.get_from_google(self.stock, 300, self.max_days, False)
if self.max_days > 10:
self.db.remove_all_records(self.stock)
if len(data) > 0:
self.db.proper_insert_all(data)
def get_new(self):
latest = Scrap()
print "In Live Data of : ", self.stock.name
while (True):
data = latest.get_latest(self.stock)
for d in data:
self.db.proper_insert(d)
time.sleep(50)
def fill_from_cloud(self):
print "Inserting Complete Data of ", self.stock.symbol
cloud = Scrap()
records = cloud.get_complete_data(self.stock)
# self.db.remove_all_records(self.stock)
self.db.proper_insert_all(records)
# for record in records:
# self.db.proper_insert(record)
print "All Data Inserted for ", self.stock.symbol
print "-----------------------"
gen_touch = pygame.sprite.spritecollideany(lifeform, grass_gens)
if gen_touch is not None and lifeform.health < 100:
lifeform.health = lifeform.health + 1
for enemy in enemies:
if pygame.sprite.collide_rect(
enemy, lifeform
) and enemy is not lifeform and lifeform.enemy == False:
enemy.health = enemy.health - 1
lifeform.health = lifeform.health - 1
screen.blit(lifeform.surf, lifeform.rect)
if lifeform.health <= 0:
scrap = Scrap(lifeform.rect.x, lifeform.rect.y)
scraps.add(scrap)
if lifeform in enemies:
enemies.remove(lifeform)
lifeforms.remove(lifeform)
# Fill the screen with white
# Draw the player on the screen
pygame.display.flip()
# Ensure program maintains a rate of 30 frames per second
clock.tick(30)
def __init__(self, cache_path='./.cache'):
self.cache_path = cache_path
self.scrap = Scrap()
self.create_dir()
class CachedScrap:
def __init__(self, cache_path='./.cache'):
self.cache_path = cache_path
self.scrap = Scrap()
self.create_dir()
def get_data(self, method, word):
data = self.seek_cache(method, word)
if data:
return data['data']
else:
if method == 'syn':
data = self.scrap.syn(word)
elif method == 'dictionary_informal':
data = self.scrap.dictionary_informal(word)
elif method == 'dictionary':
data = self.scrap.dictionary(word)
elif method == 'autocomplete':
data = self.scrap.autocomplete(word)
self.save_cache(method, word, data)
return data
def syn(self, word):
return self.get_data('syn', word)
def dictionary_informal(self, word):
return self.get_data('dictionary_informal', word)
def dictionary(self, word):
return self.get_data('dictionary', word)
def autocomplete(self, word):
return self.get_data('autocomplete', word)
def file_name(self, method, word):
return f'{method}.{word}.cache'
def seek_cache(self, method, word):
filename = self.file_name(method, word)
if self.has_file(filename):
file = open(self.cache_path + '/' + filename, 'rb')
data = pickle.load(file)
file.close()
return data
def save_cache(self, method, word, data):
filename = self.file_name(method, word)
file = open(self.cache_path + '/' + filename, 'wb')
pickle.dump({
'data': data,
'cacheDate': datetime.now().timestamp()
}, file)
file.close()
def create_dir(self):
import os
if not os.path.exists(self.cache_path):
os.makedirs(self.cache_path)
def has_file(self, name):
return path.exists(self.cache_path + '/' +
name) and path.isfile(self.cache_path + '/' + name)
response = http.request('GET', url)
soup = BeautifulSoup(response.data, "html.parser")
#print(soup)
urlbox = []
special_divs = soup.find_all('div', {'class': 'nooverflow'})
for text in special_divs:
download = text.find_all('a', href=re.compile('\.html$'))
for text in download:
hrefText = (text['href'])
urlbox.append(hrefText)
finalurlbox = []
for num in urlbox:
if num not in finalurlbox:
finalurlbox.append(num)
print(finalurlbox)
scrapfinalforwrite = [[]]
for i in range(len(finalurlbox)):
scrapfinal = Scrap(page, finalurlbox[i])
scrapfinalforwrite.append((scrapfinal))
print(scrapfinalforwrite)
list2 = [x for x in scrapfinalforwrite if x != []]
print(list2)
Filewriter(list2)