def write_file(bucket: str,
file: str,
data: bytes,
credentials=None) -> None:
client = storage.Client(credentials=(
credentials.get_credentials() if credentials else None))
# logging.info(f'Writing {file} to GCS: {len(data)}')
try:
client.get_bucket(bucket).blob(file).upload_from_string(data)
except Exception as ex:
content = None
logging.error(f'Error writing file {file}\n{ex}')
def upload_to_cloud(bucket_id, filename):
client = storage.Client()
bucket = client.get_bucket(bucket_id)
blob = bucket.blob(filename)
blob.upload_from_filename(filename)
print filename, 'upload successful'
return 'gs://' + bucket_id + '/' + filename
def download_html_company_information(company_symbols, bucket_name):
"""Downloads HTML files with company information and stores localy in the VM and upload in Cloud Storage"""
client = storage.Client()
bucket = client.get_bucket(bucket_name)
# Set the Display for the VM to download the HTML COde
display = Display(visible=0, size=(800, 600))
display.start()
print "Downloading the Company Information HTML files for: " + str(
len(company_symbols)) + " symbols"
company_symbols = ["AXON"]
i = 1
for symbol in company_symbols:
print "Downloading symbol " + str(i) + ": " + str(
symbol) + " from the list"
# Download locally in a vm specified folder
url = "http://financials.morningstar.com/company-profile/c.action?t=" + symbol + "®ion=GRC&culture=en_US"
#htmlFile = urllib2.urlopen(url)
htmlFilePath = "/home/project_investing/datasources/html_" + symbol + "_" + datetime.datetime.today(
).strftime('%Y%m%d') + ".html"
driver = webdriver.Chrome()
#options = webdriver.ChromeOptions()
#options.binary_location = '/opt/google/chrome/google-chrome'
#service_log_path = "{}/chromedriver.log".format("/home/project_investing")
#service_args = ['--verbose']
#driver = webdriver.Chrome('/usr/bin/chromedriver', chrome_options=options, service_args=service_args, service_log_path=service_log_path)
driver.get(url)
#waiting for the page to load - TODO: change
#wait = WebDriverWait(driver, 10)
#wait.until(EC.visibility_of_element_located((By.ID, "content")))
data = driver.page_source
with open(htmlFilePath, 'wb') as output:
output.write(data)
driver.close()
# Upload to Cloud Storage
blob_fileName = "datasources/html_files/html_" + symbol + "_" + datetime.datetime.today(
).strftime('%Y%m%d') + ".html"
blob = Blob(blob_fileName, bucket)
with open(htmlFilePath, 'rb') as input:
blob.upload_from_file(input)
i += 1
print "All HTML files have been succesfully stored in Cloud Storage in the following path: gs://athens_stock_exchange/datasources/html_files/"
def fetch_file(bucket: str, file: str, credentials=None) -> str:
"""fetch a file from GCS
Arguments:
bucket {str} -- bucket name
file {str} -- file name
Returns:
{str} -- file content
"""
client = storage.Client(credentials=(
credentials.get_credentials() if credentials else None))
# logging.info('Fetching {f} from GCS'.format(f=file))
try:
content = client.get_bucket(bucket).blob(file).download_as_string()
except Exception as ex:
content = None
logging.error('Error fetching file {f}\n{e}'.format(f=file, e=ex))
return content
def register():
form = RegistrationForm()
if form.validate_on_submit() == True:
flash('Account created!', 'success')
hashed_password = bcrypt.generate_password_hash(
form.password.data).decode('utf-8')
if (form.dp.data):
client = storage.Client()
bucket = client.get_bucket('a2iot-carshare.appspot.com')
blob = bucket.blob('dp_' + form.username.data + '.jpg')
with open('/home/pi/Desktop/' + form.dp.data, "rb") as my_file:
blob.upload_from_file(my_file)
dp = "https://storage.cloud.google.com/a2iot-carshare.appspot.com/dp_" + form.username.data + ".jpg"
else:
dp = None
user = User(username=form.username.data,
email=form.email.data,
password=hashed_password,
dp=dp)
db.session.add(user)
db.session.commit()
return redirect(url_for('home'))
return render_template('register.html', title='Register', form=form)
def __init__(self, bucket_name=None, project=None, public=False):
client = gc_storage.Client()
self.bucket_name = (
bucket_name
if bucket_name
else settings.DJANGO_GCS_BUCKET
)
self.project = project if project else settings.DJANGO_GCS_PROJECT
self.public = public
try:
self.gc_bucket = client.get_bucket(self.bucket_name)
except exceptions.NotFound:
# if the bucket hasn't been created, create one
# TODO: creating buckets here is not functional,
# buckets won't be created.
self.gc_bucket = self.gc_connection.new_bucket(self.bucket_name)
except oauth2client.client.AccessTokenRefreshError:
# Temporarily ignore this exception
# It causes 502 Bad Gateway on GCE,
# but it seems OK to ignore this exception.
pprint(sys.exc_info())
pass