def upload_to_s3(cls, *, directory: DirectoryPath, bucket: str = 'e2e-distributed-stress-tests'): s3_key = os.getenv('TFID') if 'TFID' in os.environ else str( uuid.uuid4()) S3(bucket=bucket).add(path=directory, key=s3_key)
def upload(): photo = request.args.get("photo.jpg") #photo.jpg par exemple user_id = request.args.get("user_id") # Upload la photo sur S3. photo_url = S3.upload_photo(photo) # Écrire l'URL en base de données. database.query("INSERT INTO photos (user_id, url) VALUES {user_id}, {photo_url}") # SQL database.write({ "photo_url": photo_url, "user_id": user_id }) # NoSQL # Récupérer les metadonnées. metadata = Analyze.analyze_photo(photo) # Écrire les métadonnées en base de données. database.query("UPDATE photos SET metadata = {metadata} WHERE photo_url = {photo_url}") # SQL database.update({ "photo_url": photo_url, "user_id": user_id, "metadata": metadata }) # NoSQL return 'OK'
def __init__(self, dataset_location = 'caltech-dataset/dataset'): self.dataset_location = dataset_location self.annotations = None self.anchors = Anchors([30, 60, 100, 200, 350], [0.41]) CaltechDataset.LOSS_LAMBDA = 2 * float(CaltechDataset.OUTPUT_SIZE[0] * CaltechDataset.OUTPUT_SIZE[1] * self.anchors.num) / float(CaltechDataset.MINIBATCH_SIZE) self.epoch = 0 self.training_minibatch = 0 self.validation_minibatch = 0 self.testing_minibatch = 0 self.s3 = S3(boto3.client("s3")) self.s3.bucket = "caltech-dataset" # self.set_training([(0, 1, 975), (3, 8, 240), (3, 8, 262), (3, 8, 279), (3, 8, 280), (3, 8, 293), (3, 8, 294), (3, 8, 295), (3, 8, 299), (3, 8, 300), (3, 8, 306), (3, 8, 308), (3, 8, 309), (3, 8, 313), (3, 8, 314), (3, 8, 315), (3, 8, 316), (3, 8, 317), (3, 8, 318), (3, 8, 321), (3, 8, 322), (3, 8, 326), (3, 8, 327), (3, 8, 334), (3, 8, 335), (3, 8, 336), (3, 8, 342), (3, 8, 345), (3, 8, 347), (3, 8, 348), (3, 8, 349), (3, 8, 350), (3, 8, 351), (3, 8, 358), (3, 8, 359), (3, 8, 360), (3, 8, 361), (3, 8, 362), (3, 8, 363), (3, 8, 364), (3, 8, 365), (3, 8, 368), (3, 8, 369), (3, 8, 370), (3, 8, 371), (3, 8, 372), (3, 8, 373), (3, 8, 374), (3, 8, 380), (3, 8, 381), (3, 8, 382), (3, 8, 391), (3, 8, 392), (3, 8, 393), (3, 8, 396), (3, 8, 397), (3, 8, 398), (3, 8, 401), (3, 8, 404), (3, 8, 410), (3, 8, 420), (3, 8, 427), (3, 8, 429), (3, 8, 430), (3, 8, 431), (3, 8, 434), (3, 8, 437), (3, 8, 443), (3, 8, 444), (3, 8, 451), (3, 8, 455), (3, 8, 456), (3, 8, 457), (3, 8, 458), (3, 8, 459), (3, 8, 460), (3, 8, 462), (3, 8, 466), (3, 8, 467), (3, 8, 478), (3, 8, 479), (3, 8, 480), (3, 8, 492), (3, 8, 493), (3, 8, 514), (3, 8, 515)]) self.discover_training() self.discover_testing()
def download_file(s3_key, base_dir=None): """Downloads a file from S3 Args: s3_key (str): S3 location of a file base_dir (str, optional): Directory to download files to. Defaults to current working dir. Returns: str: Path to downloaded file """ assert s3_key.startswith("s3://") base_dir = base_dir if base_dir else os.getcwd() bucket, key = re.findall(r"s3://(.+?)/(.+)", s3_key)[0] S3().download_file(os.path.join(base_dir, key), bucket, key, overwrite=True) return os.path.join(base_dir, key)
def digital_recon(filename): if type(filename) == list: sdq_list = [] item_list = [] for file in filename: a, b = read_recon_file(file) sdq_list.extend(a) item_list.extend(b) else: sdq_list, item_list = read_recon_file(filename) a, b, c, d = reconDynamo(sdq_list, item_list) s3 = S3('dmf-prod') missing_sdqs = set() if len(sdq_list): sdq_bucket = s3['pine-dmf-sdqresponse'] all_sdq_list = [ str(i.key)[0:5] for i in sdq_bucket.bucket.objects.all() ] missing_sdqs = set(sdq_list) - set(all_sdq_list) print("Number of SDQ responses received: {}".format(len(sdq_list))) print("Number of SDQ responses missing: {}".format(len(missing_sdqs))) print(missing_sdqs) print("Number of SDQ responses loaded into DynamoDB: {}".format(a)) print("Number of SDQs not loaded: {}".format(len(b))) print(b) print() missing_items = set() if len(item_list): item_bucket = s3['pine-dmf-itemresponse'] all_item_list = [ str(i.key)[0:5] for i in item_bucket.bucket.objects.all() ] missing_items = set(item_list) - set(all_item_list) print("Number of item responses received: {}".format(len(item_list))) print("Number of item responses missing: {}".format(len(missing_items))) print(missing_items) print("Number of Item responses loaded into DynamoDB: {}".format(c)) print("Number of Items not loaded: {}".format(len(d))) print(d)
def load_merchants(self, acquirer: str) -> list: df = [] try: region = 'us-east-1' keys ={ "AME": { "region": "sa-east-1" } } if acquirer in keys: region = keys[acquirer]["region"] s3 = S3(p_bucket=f"concil-{environ.get('STAGE', 'dev')}-blackbox-{acquirer.lower()}", region_name=region) merchants = s3.get_object(p_key=f"establishment/CFR_ESTABELECIMENTOS_{acquirer.upper()}.csv") df = pd.read_csv(StringIO(merchants), sep=";") df = df.query("FILIAL_CODIGO != 'FILIAL_GERAL'", engine="python") df['ESTABELECIMENTO'] = df['ESTABELECIMENTO'].map(str) self.merchants = df # self.merchants.to_csv('test_establishments.csv', sep=',') df.rename(columns={"ESTABELECIMENTO": "merchantCode", "PFJ_CODIGO": "clientCode"}, inplace=True) except Exception as error: print(str(error)) finally: return df[["merchantCode", "clientCode"]].to_dict(orient="records"), []
import datetime import os import time from logging import getLogger from pathlib import Path import pandas as pd from bitflyer_api import (cancel_child_order, get_balance, get_child_orders, send_child_order) from manage import CHILD_ORDERS_DIR, REF_LOCAL from utils import df_to_csv, path_exists, read_csv, rm_file if not REF_LOCAL: from aws import S3 s3 = S3() logger = getLogger(__name__) class AI: """自動売買システムのアルゴリズム """ def __init__(self, latest_summary, product_code, min_size, min_volume_short=2000, min_volume_long=10000, max_volume_short=10000,
def lambda_handler(event, context): for _event in event['Records']: data = S3().read_object(_event['s3']['bucket']['name'], _event['s3']['object']['key']) SentimentAnalysis().main(body=data)
def download_from_s3(cls, *, key: str, local_directory: str = '.', bucket: str = 'e2e-distributed-stress-tests'): S3(bucket=bucket).get(key=key, local_path=f'{local_directory}/{key}')
from aws import S3 import datetime import pandas as pd RAWDATA_BUCKET = 'kpu-gradutation-team-rawdata-dev' S3_CLIENT = S3() dt_obj = datetime.datetime.now(datetime.timezone.utc) print(dt_obj) raw_files = [ raw_file["Key"] for raw_file in S3_CLIENT.get_keys( Bucket=RAWDATA_BUCKET, Prefix="/kpu/rawdata/{:04d}/{:02d}/{:02d}/{:02d}".format( dt_obj.year, dt_obj.month, dt_obj.day, dt_obj.hour), is_sorted=False) if raw_file.get('Key', '').endswith('.csv') ] print("raw file count : ", len(raw_files)) print(raw_files) new_report_df = pd.DataFrame( columns=['timestamp_from', 'timestamp_to', 'status']) merged_df = pd.concat([ pd.read_csv( S3_CLIENT.read_file(Bucket=RAWDATA_BUCKET, Key=raw_file)['Body']) for raw_file in raw_files ], join='outer', sort=False)