"""Uploads a file to the bucket."""
storage_client = storage.Client.from_service_account_json(
'service_account.json')
bucket = storage_client.bucket(bucket_name)
blob = bucket.blob(destination_blob_name)
blob.upload_from_filename(source_file_name)
print(f"File {source_file_name} uploaded to {destination_blob_name}.")
if __name__ == "__main__":
try:
with open("data.json", "r") as f:
list_data = json.load(f)
print("JSON file parsed")
except:
exception_handler("Read JSON failed")
if (len(list_data) == 0):
print("No image to upload, exiting...")
exit(0)
try:
for data in list_data:
upload_image(data["enhanced_file"], data["output_file"])
print("Image upload complete")
except:
exception_handler("Cannot establish connection to GCS")
print("Initializing")
try:
display = Display(visible=0, size=(800, 600))
display.start()
firefox_profile = FirefoxProfile()
firefox_profile.set_preference(
'dom.ipc.plugins.enabled.libflashplayer.so', 'false')
driver = webdriver.Firefox(firefox_profile)
driver.set_page_load_timeout(90)
driver.implicitly_wait(30)
except:
driver.quit()
display.stop()
exception_handler("Initialization failed")
print("Loading Pixiv daily rankings")
try:
load_and_retry(driver, URL_RANKING, MAX_RETRIES)
driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
page_title = driver.title
# Slash not allowed in folder names
timestamp = page_title.split(" ")[-1].replace('/', '-')
print("Rankings loaded {}".format(timestamp))
# ----- Prepare Output Path -----
output_dir = Path('.') / OUTPUT_BASE_DIR / timestamp
output_dir = str(output_dir.resolve())
# ----- Detect Duplicate Download -----
from error_handlers import exception_handler
from google.api_core import retry
from google.cloud import pubsub_v1
if __name__ == "__main__":
try:
project_id = os.environ['GCP_PROJ']
subscription_id = os.environ['SUB_ID']
subscriber = pubsub_v1.SubscriberClient.from_service_account_json(
'service_account.json')
subscription_path = subscriber.subscription_path(
project_id, subscription_id)
print("Connected to Pub/Sub")
except:
exception_handler("Connection to Pub/Sub failed")
try:
list_messages = []
with subscriber:
while True:
response = subscriber.pull(
request={"subscription": subscription_path,
"max_messages": 1},
retry=retry.Retry(deadline=300),
)
if len(response.received_messages) == 1:
received_message = response.received_messages[0]
list_messages.append(
dict((key, received_message.message.attributes[key]) for key in received_message.message.attributes))
print(f"Received: {received_message.message.attributes}.")
'service_account.json')
bucket = storage_client.bucket(bucket_name)
blob = bucket.blob(source_blob_name)
blob.download_to_filename(destination_file_name)
print("Blob {} downloaded to {}.".format(source_blob_name,
destination_file_name))
if __name__ == "__main__":
try:
with open("data.json", "r") as f:
list_data = json.load(f)
print("JSON file parsed")
except:
exception_handler("Read JSON failed")
if (len(list_data) == 0):
print("No image to process, exiting...")
exit(0)
try:
Path("./imgs").mkdir(parents=True, exist_ok=True)
list_new_data = []
for data in list_data:
save_path = download_image(data["input_file"])
list_new_data.append({**data, "saved_file": save_path})
print("Images downloaded")
except:
exception_handler("Image download failed")
if not isinstance(image, Image.Image):
image = tf.clip_by_value(image, 0, 255)
image = Image.fromarray(tf.cast(image, tf.uint8).numpy())
image.save(filename)
print("Saved as %s" % filename)
if __name__ == "__main__":
start_time = time.time()
print("Starting image super-resolution")
try:
model = hub.load("https://tfhub.dev/captain-pool/esrgan-tf2/1")
print("Model downloaded")
except:
exception_handler("Model download failed")
try:
with open("data.json", "r") as f:
list_data = json.load(f)
print("JSON file parsed")
except:
exception_handler("Read JSON failed")
if (len(list_data) == 0):
print("No image to process, exiting...")
exit(0)
try:
list_new_data = []
img_batch = []