def test_resource_uses_existing_session(self, setup_session):
ibm_boto3.DEFAULT_SESSION = self.Session()
ibm_boto3.resource('sqs')
assert not setup_session.called
assert ibm_boto3.DEFAULT_SESSION.resource.called
def test_resource_passes_through_arguments(self):
ibm_boto3.DEFAULT_SESSION = self.Session()
ibm_boto3.resource('s3', region_name='us-west-2', verify=False)
ibm_boto3.DEFAULT_SESSION.resource.assert_called_with(
's3', region_name='us-west-2', verify=False)
def test_resource_creates_default_session(self, setup_session):
ibm_boto3.DEFAULT_SESSION = None
ibm_boto3.resource('sqs')
assert setup_session.called
assert ibm_boto3.DEFAULT_SESSION.resource.called
def test_resource_uses_existing_session(self, setup_session):
ibm_boto3.DEFAULT_SESSION = self.Session()
ibm_boto3.resource('s3')
self.assertFalse(setup_session.called,
'setup_default_session should not have been called')
self.assertTrue(ibm_boto3.DEFAULT_SESSION.resource.called,
'Default session resource method not called')
def test_resource_creates_default_session(self, setup_session):
ibm_boto3.DEFAULT_SESSION = None
ibm_boto3.resource('s3')
self.assertTrue(setup_session.called,
'setup_default_session not called')
self.assertTrue(ibm_boto3.DEFAULT_SESSION.resource.called,
'Default session resource method not called')
def init():
try:
message = flask.request.get_json(force=True, silent=True)
if message and not isinstance(message, dict):
flask.abort(404)
cos = ibm_boto3.resource(
's3',
ibm_api_key_id='apikey',
ibm_service_instance_id='resource_instance_id',
ibm_auth_endpoint='https://iam.bluemix.net/oidc/token',
config=Config(signature_version='oauth'),
endpoint_url='https://s3-api.us-geo.objectstorage.softlayer.net')
obj = cos.Object("tensorflow", "retrained_graph_cozmo.pb").get()
graph_def = tf.GraphDef()
graph_def.ParseFromString(obj["Body"].read())
with graph.as_default():
tf.import_graph_def(graph_def)
obj = cos.Object("tensorflow", "retrained_labels_cozmo.txt").get()
for i in obj["Body"].read().decode("utf-8").split():
labels.append(i)
except Exception as e:
print("Error in downloading content")
print(e)
response = flask.jsonify({'error downloading models': e})
response.status_code = 512
return ('OK', 200)
def upload():
try:
cos = ibm_boto3.resource(
's3',
ibm_api_key_id='apikey',
ibm_service_instance_id='resource_instance_id',
ibm_auth_endpoint='https://iam.bluemix.net/oidc/token',
config=Config(signature_version='oauth'),
endpoint_url='https://s3-api.us-geo.objectstorage.softlayer.net')
zipFileName = 'cozmo-photos'
shutil.make_archive(zipFileName, 'zip', '../1-take-pictures/pictures')
print("Done: Zipping Pictures")
container = 'tensorflow'
cos.create_bucket(Bucket=container)
with open('./' + zipFileName + '.zip', 'rb') as local:
cos.Object(container,
zipFileName + '.zip').upload_file(zipFileName + '.zip')
print("Done: Uploading Pictures")
except Exception as e:
print("Error: Uploading Pictures")
print(e)
return
def get_data():
COS_ENDPOINT = "https://s3.eu.cloud-object-storage.appdomain.cloud" # Current list avaiable at https://control.cloud-object-storage.cloud.ibm.com/v2/endpoints
COS_API_KEY_ID = "phWenEzTBmpBfuluOeGpVZ2vJD6J6YOtwvPdKtYWFZ2-" # eg "W00YiRnLW4a3fTjMB-oiB-2ySfTrFBIQQWanc--P3byk"
COS_AUTH_ENDPOINT = "https://iam.cloud.ibm.com/identity/token"
COS_RESOURCE_CRN = "crn:v1:bluemix:public:cloud-object-storage:global:a/22a5864ba3754cf0a274ba36ea94a358:72601e19-4b09-4f8c-8bf5-757c0c2eddfb::" # eg "crn:v1:bluemix:public:cloud-object-storage:global:a/3bf0d9003abfb5d29761c3e97696b71c:d6f04d83-6c4f-4a62-a165-696756d63903::"
cos = ibm_boto3.resource("s3",
ibm_api_key_id=COS_API_KEY_ID,
ibm_service_instance_id=COS_RESOURCE_CRN,
ibm_auth_endpoint=COS_AUTH_ENDPOINT,
config=Config(signature_version="oauth"),
endpoint_url=COS_ENDPOINT)
BUCKET_NAME = "zt-fraud-detection"
FILE_PREFIX = "resource"
train_identity = "ieee-fraud-detection/train_identity.csv"
train_transaction = "ieee-fraud-detection/train_transaction.csv"
test_identity = "ieee-fraud-detection/test_identity.csv"
test_transaction = "ieee-fraud-detection/test_transaction.csv"
ms_transfer_config = AsperaConfig(multi_session="all",
target_rate_mbps=2500,
multi_session_threshold_mb=100)
transfer_manager = AsperaTransferManager(client=cos,
config=ms_transfer_config)
transfer_manager.download(BUCKET_NAME, train_identity,
"{}/{}".format(FILE_PREFIX, train_identity))
transfer_manager.download(BUCKET_NAME, train_transaction,
"{}/{}".format(FILE_PREFIX, train_transaction))
transfer_manager.download(BUCKET_NAME, test_identity,
"{}/{}".format(FILE_PREFIX, test_identity))
transfer_manager.download(BUCKET_NAME, test_transaction,
"{}/{}".format(FILE_PREFIX, test_transaction))
def main():
print("init")
# Start by shrinking all images to 224x224 (default for sota cnn).
resize_all_images(IMAGE_DIRECTORY, SCALED_IMAGE_DIRECTORY)
# Load labels and images.
images = get_image_list(SCALED_IMAGE_DIRECTORY)
labels = get_label_list(LABELS_PATH)
# Create the _annotations.json
mapped_labels = [label_for_depth(label, LABEL_MAP) for label in labels]
json_content = build_annotations(images, mapped_labels,
index_from_filename)
with open(ANNOTATIONS_PATH, 'w') as f:
json.dump(json_content, f, indent=2)
quit()
# Upload to object storage.
credentials = {
'ibm_auth_endpoint': 'https://iam.ng.bluemix.net/oidc/token',
'ibm_service_instance_id': RESOURCE_INSTANCE_ID,
'endpoint_url': ENDPOINT,
'ibm_api_key_id': API_KEY,
'config': Config(signature_version='oauth')
}
bucket = ibm_boto3.resource('s3', **credentials).Bucket(BUCKET_NAME)
print('uploading {}...'.format(ANNOTATIONS_PATH))
bucket.upload_file(ANNOTATIONS_PATH, ANNOTATIONS_PATH)
for filename in images:
print('uploading {}...'.format(filename))
bucket.upload_file(os.path.join(SCALED_IMAGE_DIRECTORY, filename),
filename)
def __init__(self, SECRET_CONFIG):
self.cos = ibm_boto3.resource(
service_name='s3',
ibm_api_key_id=SECRET_CONFIG["cos_credentials"]['apikey'],
ibm_auth_endpoint=SECRET_CONFIG["auth_endpoint"],
config=Config(signature_version='oauth'),
endpoint_url=SECRET_CONFIG["service_endpoint"])
def main():
"""
Script to deploy the larger parts of this model to Cloud Object Storage.
Before running this script, you'll need to perform the following manual steps:
* Create a file `ibm_cloud_credentials.json` in this directory, if such a
file doesn't already exist.
Initialize the file with an empty JSON record, i.e. "{ }".
* Create a Cloud Object Storage instance.
* Go to the COS web UI and create a set of credentials with write
permissions on your COS instance. Paste the JSON version of the credentials
into `ibm_cloud_credentials.json` under the key "COS_credentials".
* Figure out an endpoint that your COS instance can talk to.
Go back to the web UI for COS and click on "Endpoints". Take one of the
endpoint names, prepend it with "https://", and store the resulting URL
under the key "COS_endpoint" in `ibm_cloud_credentials.json`
* Figure out what "location constraint" works for your COS bucket. Today
there is a list of potential values at
https://console.bluemix.net/docs/infrastructure/cloud-object-storage
-infrastructure/buckets.html#create-a-bucket
(though this part of the docs has a habit of moving around).
Enter your location constraint string into `ibm_cloud_credentials.json`
under the key "COS_location_constraint".
"""
# STEP 1: Read IBM Cloud authentication data from the user's local JSON
# file.
with open("./ibm_cloud_credentials.json") as f:
creds_json = json.load(f)
print("creds_json is:\n{}".format(creds_json))
_COS_ENDPOINT = creds_json["COS_endpoint"]
_COS_API_KEY_ID = creds_json["COS_credentials"]["apikey"]
_COS_RESOURCE_CRN = creds_json["COS_credentials"]["resource_instance_id"]
_COS_LOCATION_CONSTRAINT = creds_json["COS_location_constraint"]
# STEP 2: Create a bucket on Cloud Object Storage to hold the SavedModel
cos = ibm_boto3.resource("s3",
ibm_api_key_id=_COS_API_KEY_ID,
ibm_service_instance_id=_COS_RESOURCE_CRN,
ibm_auth_endpoint=_COS_AUTH_ENDPOINT,
config=Config(signature_version="oauth"),
endpoint_url=_COS_ENDPOINT
)
_empty_cos_bucket(cos, _MODEL_BUCKET, _COS_LOCATION_CONSTRAINT)
# STEP 3: Convert the SavedModel directory to a tarball.
if os.path.exists(_SAVED_MODEL_TARBALL):
os.remove(_SAVED_MODEL_TARBALL)
subprocess.run(["tar", "--create", "--gzip", "--verbose",
"--directory={}".format(_SAVED_MODEL_DIR),
"--file={}".format(_SAVED_MODEL_TARBALL),
"saved_model.pb"])
# STEP 4: Upload the SavedModel tarball to the COS bucket.
_cp_to_cos(cos, _SAVED_MODEL_TARBALL, _MODEL_BUCKET,
_SAVED_MODEL_TARBALL_PATH_IN_COS, replace=True)
print("Done.")
def __init__(self):
"""Initialize the instance using global configuration."""
self._conf = get_cephci_config()["cos"]
self._api_key = self._conf["api-key"]
self._resource_id = self._conf["resource-id"]
self.endpoint = self._conf["endpoint"]
self.location_constraint = dict(
{"LocationConstraint": self._conf["location-constraint"]})
self.client = ibm_boto3.client(
"s3",
ibm_api_key_id=self._api_key,
ibm_service_instance_id=self._resource_id,
config=Config(signature_version="oauth"),
endpoint_url=self.endpoint,
)
self.resource = ibm_boto3.resource(
"s3",
ibm_api_key_id=self._api_key,
ibm_service_instance_id=self._resource_id,
config=Config(signature_version="oauth"),
endpoint_url=self.endpoint,
)
def initCos():
return ibm_boto3.resource(
"s3",
ibm_api_key_id=CONFIG['ibm_cos']['api_key'],
ibm_auth_endpoint='https://iam.ng.bluemix.net/oidc/token',
config=Config(signature_version="oauth"),
endpoint_url=CONFIG['ibm_cos']['endpoint'])
def __init__(self, wml_vcap, cos_vcap, auth_endpoint, service_endpoint):
api_key = cos_vcap['apikey']
service_instance_id = cos_vcap['resource_instance_id']
self.instance_id = wml_vcap['instance_id']
self.cos = ibm_boto3.resource('s3',
ibm_api_key_id=api_key,
ibm_service_instance_id=service_instance_id,
ibm_auth_endpoint=auth_endpoint,
config=Config(signature_version='oauth'),
endpoint_url=service_endpoint)
self.bucket_names = [self.instance_id + '-style-data', self.instance_id + '-style-results']
for bucket in self.bucket_names:
try:
print('Creating bucket "{}"...'.format(bucket))
self.cos.create_bucket(Bucket=bucket)
except ClientError as ex:
if ex.response['Error']['Code'] == 'BucketAlreadyExists':
print('Already exists.')
else:
raise ex
self.data_bucket = self.cos.Bucket(self.bucket_names[0])
self.results_bucket = self.cos.Bucket(self.bucket_names[1])
def __init__(self, recipe):
"""
:type recipe: Recipe
"""
# self.log.setLevel(DEBUG)
self.log.info("Connection to COS")
cos_creds = recipe.cos_creds_content()
api_key = cos_creds['apikey']
auth_endpoint = 'https://iam.bluemix.net/oidc/token'
service_instance_id = cos_creds['resource_instance_id']
service_endpoint = recipe["COS"]["endpoint"]
# service_endpoint = "s3.eu-de.objectstorage.service.networklayer.com"
self.bucket = recipe["COS"]["bucket"]
self.log.info("service endpoint '%s'", service_endpoint)
self.log.info("service bucket '%s'", self.bucket)
try:
self.resource = ibm_boto3.resource(
's3',
ibm_api_key_id=api_key,
ibm_service_instance_id=service_instance_id,
ibm_auth_endpoint=auth_endpoint,
config=Config(signature_version='oauth'),
endpoint_url=service_endpoint)
except ClientError as e:
self.log.fatal('Exception: %s', e)
raise SystemExit(-1)
def main(dict):
try:
# setup of information
resource = ibm_boto3.resource(
's3',
ibm_api_key_id=dict['keyAPI1'],
ibm_service_instance_id=dict['bucket1CNR'],
config=Config(signature_version="oauth"),
endpoint_url=dict['endpoint1'],
region_name="us-east")
client = ibm_boto3.client('s3',
ibm_api_key_id=dict['keyAPI1'],
ibm_service_instance_id=dict['bucket1CNR'],
config=Config(signature_version="oauth"),
endpoint_url=dict['endpoint1'],
region_name="us-east")
bucket = 'kbigelow-testing'
bucket2 = 'copy-bucket'
newFile = 'testing'
key = dict['key']
encoded_newFile = newFile.encode('utf-8')
# copy file to backup bucket
response = resource.Object(bucket2, key).copy_from(CopySource=bucket +
"/" + key)
# get time for the log file
now = datetime.now()
dt_string = now.strftime("%d/%m/%Y %H:%M:%S")
# output for built in logs
print('File: ' + str(key))
print('Time: ' + dt_string)
print('Source bucket: ' + str(bucket))
print('Backup bucket: ' + str(bucket2))
# gets current log file
try:
response2 = client.get_object(Bucket=bucket2, Key='log.txt')
currFile = response2['Body'].read().decode('utf-8')
except Exception as e:
currFile = 'Logs of Backup Function\n\n'
# adds new log to log file
newFile = currFile + 'Creating backup of file: ' + \
str(key) + '\nTime: ' + dt_string + '\nSource bucket: ' + \
str(bucket) + '\nBackup bucket: ' + str(bucket2) + '\n\n'
encoded_newFile = newFile.encode('utf-8')
# uploading new log file
resource.Bucket(bucket2).put_object(Key='log.txt',
Body=encoded_newFile)
return {'message': 'Created backup'}
except Exception as e:
print(e)
print('Error creating backup of file {} from bucket {}.'.format(
key, bucket))
raise e
def upload_object(bucket_name, data, key, cos_cli):
s3 = ibm_boto3.resource('s3')
bucket = s3.Bucket(bucket_name)
try:
bucket.upload_fileobj(data, key, extraArgs=cos_cli)
except:
print("Failed Upload")
def get_cos_resource():
cos = ibm_boto3.resource("s3",
ibm_api_key_id=COS_API_KEY_ID,
ibm_service_instance_id=COS_RESOURCE_CRN,
ibm_auth_endpoint=COS_AUTH_ENDPOINT,
config=Config(signature_version="oauth"),
endpoint_url=COS_ENDPOINT)
return cos
def connect(self, credentials):
return ibm_boto3.resource(
's3',
ibm_api_key_id=credentials["apikey"],
ibm_service_instance_id=credentials["resource_instance_id"],
ibm_auth_endpoint=credentials["auth_endpoint"],
config=ibm_botocore.client.Config(signature_version='oauth'),
endpoint_url=credentials["service_endpoint"])
def get_cos_resource(location):
"""return IBM COS resource object"""
cos_endpoint = "https://s3.%s.cloud-object-storage.appdomain.cloud" % location
return ibm_boto3.resource("s3",
ibm_api_key_id=COS_API_KEY,
ibm_service_instance_id=COS_RESOURCE_CRN,
ibm_auth_endpoint=COS_AUTH_ENDPOINT,
config=Config(signature_version="oauth"),
endpoint_url=cos_endpoint)
def __init__(self):
self.cos = ibm_boto3.resource(
"s3",
ibm_api_key_id='Ykpx0He0ReyY8Gi4lTlIDA0Sl67PL067mJkFIZ6EFwNo',
ibm_service_instance_id=
'crn:v1:bluemix:public:cloud-object-storage:global:a/ab3491678a354cef8c67c5ce7734eaeb:3c76ff27-8531-4e1b-9585-4adad2442443:bucket:covidstudypublic-donotdelete-pr-vpm7tz8euu9ufn',
ibm_auth_endpoint="https://iam.cloud.ibm.com/oidc/token",
config=Config(signature_version="oauth"),
endpoint_url='https://s3.eu.cloud-object-storage.appdomain.cloud')
def upload_to_cos(properties, src_path, bucket_name, dest_path):
service_endpoint = properties['url']
aws_access_key_id = properties['access_key']
aws_secret_access_key = properties['secret_key']
cos_client = ibm_boto3.resource('s3',
endpoint_url=service_endpoint,
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key)
cos_client.meta.client.upload_file(src_path, bucket_name, dest_path)
def initConnection():
global resource
resource = ibm_boto3.resource(
's3',
ibm_api_key_id=credentials['apikey'],
ibm_service_instance_id=credentials['resource_instance_id'],
ibm_auth_endpoint=auth_endpoint,
config=Config(signature_version='oauth'),
endpoint_url=service_endpoint)
def create_bucket(bucket_name):
cos = ibm_boto3.resource('s3',
ibm_api_key_id='COS_API_KEY',
ibm_service_instance_id='COS_SERVICE_INSTANCE',
ibm_auth_endpoint='https://iam.cloud.ibm.com/identity/token',
config=Config(signature_version='oauth'),
endpoint_url='https://s3.us-south.cloud-object-storage.appdomain.cloud'
)
cos.create_bucket(Bucket=bucket_name)
def cos_uploadImage(image):
cos = ibm_boto3.resource(
's3',
ibm_api_key_id=coscred['apikey'],
ibm_service_instance_id=coscred['resource_instance_id'],
ibm_auth_endpoint=auth_endpoint,
config=Config(signature_version='oauth'),
endpoint_url=service_endpoint)
for bucket in cos.buckets.all():
cos.Bucket(bucket.name).upload_file("./" + image, image)
def get_cos_client(credentials):
cos = ibm_boto3.resource(
"s3",
ibm_api_key_id=credentials['apikey'],
ibm_service_instance_id=credentials['resource_instance_id'],
ibm_auth_endpoint='https://iam.ng.bluemix.net/oidc/token',
config=Config(signature_version="oauth"),
endpoint_url="https://s3.us-east.cloud-object-storage.appdomain.cloud",
)
return cos
def build_cos_client() -> ServiceResource:
"""Uses environment variables to build a COS client."""
cos_endpoint = os.environ.get('COS_ENDPOINT')
cos_api_key_id = os.environ.get('COS_API_KEY_ID')
cos_instance_crn = os.environ.get('COS_INSTANCE_CRN')
cos_client = ibm_boto3.resource('s3',
ibm_api_key_id=cos_api_key_id,
ibm_service_instance_id=cos_instance_crn,
config=Config(signature_version='oauth'),
endpoint_url=cos_endpoint)
return cos_client
def s3(self):
if not hasattr(self, '_s3'):
kwargs = {
'ibm_auth_endpoint':
"https://iam.cloud.ibm.com/identity/token",
}
kwargs.update(self.kwargs)
config = {'signature_version': "oauth"}
config.update(self.kwargs.get('config') or {})
kwargs['config'] = ibm_botocore.client.Config(**config)
self._s3 = ibm_boto3.resource("s3", **kwargs)
return self._s3
def initServices(app):
# Setup MQTT
app.config['MQTT_BROKER_URL'] = 'test.mosquitto.org'
app.config['MQTT_BROKER_PORT'] = 1883
mqtt = Mqtt(app)
app.config['MQTT_CLIENT'] = mqtt
# Setup IBM Watson
load_dotenv()
authenticator = IAMAuthenticator(os.getenv("STT_API_KEY"))
service = SpeechToTextV1(authenticator=authenticator)
service.set_service_url(os.getenv("STT_URL"))
app.config['SPEECH_TO_TEXT'] = service
authenticator_translate = IAMAuthenticator(os.getenv("TRANSLATE_API_KEY"))
language_translator = LanguageTranslatorV3(
version='2018-05-01', authenticator=authenticator_translate)
language_translator.set_service_url(os.getenv("TRANSLATE_URL"))
app.config['LANGUAGE_TRANSLATOR'] = language_translator
# IBM COS
app.config['COS_ENDPOINT'] = os.getenv("COS_ENDPOINT")
if not app.config['COS_ENDPOINT'].startswith(
'http') or not app.config['COS_ENDPOINT'].startswith('https'):
app.config['COS_ENDPOINT'] = 'https://' + app.config['COS_ENDPOINT']
cos = ibm_boto3.resource(
"s3",
ibm_api_key_id=os.getenv("COS_API_KEY"),
ibm_service_instance_id=os.getenv("COS_IAM_ROLE_CRN"),
ibm_auth_endpoint='https://iam.cloud.ibm.com/identity/token',
config=Config(signature_version="oauth"),
endpoint_url=app.config['COS_ENDPOINT'])
app.config['COS'] = cos
app.config['COS_BUCKET_NAME'] = os.getenv("COS_BUCKET_NAME")
# Setup config
# app.config['BASE'] = os.path.join(os.path.dirname(os.getcwd()),'cfc-covid-19-video-transcriber-starter')
app.config['BASE'] = os.path.join(os.path.dirname(os.getcwd()), '/app')
app.config['BASE'] = os.path.join(app.config['BASE'], 'server')
app.config['UPLOAD_FOLDER'] = os.path.join(app.config['BASE'],
'video_uploads')
app.config['AUDIO_FOLDER'] = os.path.join(app.config['BASE'],
'audio_extractions')
app.config['OUTPUT_FOLDER'] = os.path.join(app.config['BASE'],
'output_transcripts')
os.makedirs(app.config['UPLOAD_FOLDER'], exist_ok=True)
os.makedirs(app.config['AUDIO_FOLDER'], exist_ok=True)
os.makedirs(app.config['OUTPUT_FOLDER'], exist_ok=True)
return
def get_connection(module):
endpoint = module.params['ibm_endpoint']
api_key = module.params['ibm_api_key']
auth_endpoint = module.params['ibm_auth_endpoint']
resource_id = module.params['ibm_resource_id']
cos = ibm_boto3.resource("s3",
ibm_api_key_id=api_key,
ibm_service_instance_id=resource_id,
ibm_auth_endpoint=auth_endpoint,
config=Config(signature_version="oauth"),
endpoint_url=endpoint)
return cos
