def add_to_db(message):
# Initialize the Cosmos client
endpoint = 'https://talkingdb.documents.azure.com:443/'
key = 'nL5f0ZJPKLwxSaKigi2BNlk9eVn5jtZCeoFIg85TeCC73vkBCy2BctEY27YDMCh8B4UXg8Gq1GvrNBeltrNYgg=='
#create cosmos client
client = CosmosClient(endpoint, key)
try:
# Create a database
database_name = 'tcdb'
database = client.create_database_if_not_exists(id=database_name)
except exceptions.CosmosResourceExistsError:
pass
# Create a container
# Using a good partition key improves the performance of database operations.
try:
container_name = 'c1'
container = database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path="/messages"),
offer_throughput=400)
except exceptions.CosmosResourceExistsError:
pass
container.create_item(body=message)
def update_run_id(device_id, run_id):
# first get the appropriate farm
client = CosmosClient(Cosmos.URL, Cosmos.KEY)
farms_container = client.get_database_client(
Cosmos.DATABASE).get_container_client(Cosmos.FARMS_CONTAINER)
query = """SELECT c.id as farm_id, f
FROM farms f
JOIN c IN f.farms_arr
WHERE c.base_station.deviceId = @dev_id"""
items = farms_container.query_items(query=query,
parameters=[{
"name": "@dev_id",
"value": device_id
}],
enable_cross_partition_query=True)
item = next(items)
farm_id = item['farm_id']
document = item['f']
# loop through the farms and find the correct farm to update run id
for farm_obj in document['farms_arr']:
if farm_obj['id'] == farm_id:
farm_obj['last_run'] = run_id
# finally update document
farms_container.upsert_item(document)
return farm_id
def main(req: func.HttpRequest) -> func.HttpResponse:
# Initialize the Cosmos client
endpoint = "https://parking-spots.documents.azure.com:443/"
key = 'c9av0FP6xBrnz7XNUdBNIUFdCuo1IvE3H2SWvvB8Pxo92ALKzfW2gDXxgQlBlyUJtLDiJSZK6Ew8YO8yPNepnA=='
client = CosmosClient(endpoint, key)
# Select database
database_name = 'Parking-Data'
database = client.create_database_if_not_exists(id=database_name)
# Select container
container_name = 'Items'
container = database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path="/slots"),
offer_throughput=400
)
# Query slots using SQL
query = "SELECT * FROM c"
items = list(container.query_items(
query=query,
enable_cross_partition_query=True
))
return func.HttpResponse(json.dumps(items))
def store_url_counts(url_list, unique_url_list):
print ("**** COUNT URLS ****\n")
print(url_list)
url_counts = dict(Counter(url_list))
print(url_counts)
uri = os.environ.get('ACCOUNT_URI')
key = os.environ.get('ACCOUNT_KEY')
database_id = os.environ.get('DATABASE_ID')
container_id = os.environ.get('URL_CONTAINER_ID')
client = CosmosClient(uri, {'masterKey': key})
database = client.get_database_client(database_id)
container = database.get_container_client(container_id)
date_str = datetime.today().strftime('%Y-%m-%d %H:%M:%S')
id_date = int((datetime.utcnow()).timestamp())
id_date_str = str(id_date)
output = []
for k,v in url_counts.items():
output.append({'url':k, 'count':v})
container.upsert_item({'id': id_date_str,
'date_time': id_date_str,
'date': date_str,
'urls_and_counts': output})
def hello(request):
context = {'items': [], 'status_text': 'ok', 'created': {}}
client = CosmosClient(endpoint, key)
database = client.create_database_if_not_exists(id=database_name)
container = database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path="/name"),
offer_throughput=400)
query = "SELECT * FROM c items"
items = list(
container.query_items(query=query, enable_cross_partition_query=True))
context['items'] = items
if request.method == 'GET':
return render(request, template_name='index.html', context=context)
elif request.method == 'POST':
picture = request.FILES.get('picture')
item = push_image(picture, request.POST['name'], request.POST['age'],
request.POST['category'])
container.upsert_item(body=item)
items.append(item)
context['items'] = items
return render(request, template_name='index.html', context=context)
elif request.method == 'PUT':
print(request.PUT)
pass
else:
pass
return render(request, template_name='index.html', context=context)
def get_item(request, pk, name):
context = {'item': [], 'status_text': 'ok', 'created': {}}
client = CosmosClient(endpoint, key)
database = client.create_database_if_not_exists(id=database_name)
container = database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path="/name"),
offer_throughput=400)
print(request)
if request.method == "DELETE":
print('delete')
try:
container.delete_item(item=pk, partition_key=name)
except CosmosResourceNotFoundError:
return HttpResponse("Resource not Found")
return redirect('http://127.0.0.1:8000')
# return render(request, template_name='index.html', context=context)
elif request.method == 'POST':
item = {
'id': request.POST.get('id'),
'name': request.POST.get('name'),
'age': request.POST.get('age'),
'category': request.POST.get('category'),
'picture': request.POST.get('picture')
}
# picture = request.FILES.get('picture')
# item = push_image(picture, request.POST['name'], request.POST['age'], request.POST['category'])
container.replace_item(item=item['id'], body=item)
context['item'] = [item]
return render(request, template_name='index.html', context=context)
else:
item = container.read_item(pk, partition_key=name)
context['item'] = item
return render(request, template_name='index.html', context=context)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
endpoint = os.environ["uri"]
key = os.environ["pkey"]
client = CosmosClient(endpoint, key)
database_name = 'usernames'
database = client.create_database_if_not_exists(id=database_name)
container_name = 'usernameStorage'
container = database.create_container_if_not_exists(id=container_name, partition_key=PartitionKey(path="/id"), offer_throughput=400)
getletters = requests.get('https://agris123.azurewebsites.net/api/service3?code=AfwsKOwVIblqhzmcOVH4IJxaFQ0eMIiNQdsA1RnWaIHtqQKICUbL3w==')
getnumbers = requests.get('https://agris123.azurewebsites.net/api/service2?code=LZAj225WWasaZLmUIOTMrpoeWOIMxQrwqir2TtFLdhiP8ufPqjKgTw==')
letters = getletters.text
numbers= getnumbers.text
answer = []
for i in range(5):
answer.append(letters[i])
answer.append(numbers[i])
ans="".join(answer)
ansItem = {'id': ans}
container.create_item(body=ansItem)
return str(ans)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
uid = uuid.uuid4()
url = os.environ['ACCOUNT_URI']
key = os.environ['ACCOUNT_KEY']
client = CosmosClient(url, credential=key)
database_name = 'ToDoDB'
database = client.get_database_client(database_name)
container_name = 'Actions'
container = database.get_container_client(container_name)
try:
req_body = req.get_json()
title = req_body.get('title')
container.upsert_item({
'id': str(uid),
'title': title,
'location': 'SouthAfrica',
'complete': False
})
return func.HttpResponse(
"Saved",
status_code=201
)
except Exception:
return func.HttpResponse("Error", status_code=500)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
id = str(req.route_params.get('id'))
if id:
endpoint = "https://localhost:8081"
key = os.environ['key']
client = CosmosClient(endpoint, key)
database_name = 'AzureDatabase'
database = client.create_database_if_not_exists(id=database_name)
container_name = 'DishesContainer'
container = database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path="/name"),
offer_throughput=400)
query = "SELECT * FROM ALL"
items = list(
container.query_items(query=query,
enable_cross_partition_query=True))
for item in items:
if (str(item['id']) == id):
container.delete_item(item, item['name'])
return func.HttpResponse(f"Deleted dish having {id}!")
return func.HttpResponse("Element not found", status_code=404)
else:
return func.HttpResponse("Please pass a dish ID", status_code=400)
def database():
client = CosmosClient(ENDPOINT, KEY)
default_database_name = 'FastApiDB'
database = client.create_database_if_not_exists(id=default_database_name)
return database
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
getnumbers = requests.get(
'https://jadeazfunction.azurewebsites.net/api/HttpTrigger2?code=rFIbZDWG04Um4U1tQ1LtZkVuOrUda13UrMc7buN3nHCBu/BEGoniug=='
)
getletters = requests.get(
'https://jadeazfunction.azurewebsites.net/api/HttpTrigger3?code=3vYSLRruH2cdxNqFDmH1OW56C0BNmJpnENKhpuk08uwWR4bGbhU2Ig=='
)
numbers = getnumbers.text
letters = getletters.text
answer = []
for i in range(5):
answer.append(numbers[i])
answer.append(letters[i])
finalans = "".join(answer)
key = "nvzumL7YLMAIS9H6P3Ya2aID31eY0n4yvwzkioiIgPXLw8Tvy9KGt8RX4EPFi81PN4fgN5nQ4LxMTio8S6XmWA=="
endpoint = "https://serverlessmilestone-1.documents.azure.com:443/;AccountKey=nvzumL7YLMAIS9H6P3Ya2aID31eY0n4yvwzkioiIgPXLw8Tvy9KGt8RX4EPFi81PN4fgN5nQ4LxMTio8S6XmWA==;"
client = CosmosClient(endpoint, key)
database_name = "Usernames"
client.create_database_if_not_exist(id=database_name)
container_name = "UsernameContainer"
container = database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path / "id"),
offer_throughput=400)
return str(finalans)
def main(req: func.HttpRequest) -> func.HttpResponse:
# fetch the farm_id from the request
try:
farm_id = req.params['farm_id']
run_id = req.params['run_id']
pred_type = req.params['pred_type']
except (ValueError, KeyError) as e:
return func.HttpResponse(
"Incorrect request body",
status_code=422
)
client = CosmosClient(Cosmos.URL, Cosmos.KEY)
pred_container = client.get_database_client(Cosmos.DATABASE).get_container_client(Cosmos.PRED_CONTAINER)
results, timestamp = get_predictions_from_runId(pred_container, farm_id, run_id, pred_type)
payload = {
'results': results,
'timestamp': timestamp
}
return func.HttpResponse(
json.dumps(payload),
mimetype="application/json",
status_code=200
)
def get_farm_data(device_id):
# query farm data correspoding to device
client = CosmosClient(Cosmos.URL, Cosmos.KEY)
farms_container = client.get_database_client(Cosmos.DATABASE).get_container_client(Cosmos.FARMS_CONTAINER)
query = """SELECT c
FROM farms f
JOIN c IN f.farms_arr
WHERE c.base_station.deviceId = @dev_id"""
items = list(farms_container.query_items(
query=query,
parameters=[{ "name":"@dev_id", "value": device_id }],
enable_cross_partition_query=True
))
props = {
'TYPE': 'RESPONSE',
'EVENT': 'FARM_DATA_LOADED'
}
payload = json.dumps(items[0]['c'])
# write back response to the device
registry_manager = IoTHubRegistryManager(IoTHub.CONNECTION_STRING)
registry_manager.send_c2d_message(device_id, payload, properties=props)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
con = 0
try:
req_body = req.get_json()
name = req_body['name']
dish = req_body['dish']
veg = req_body['vegetarian']
vegen = req_body['vegan']
except:
pass
else:
endpoint = "https://localhost:8081"
key = os.environ['key']
client = CosmosClient(endpoint, key)
database_name = 'AzureDatabase'
database = client.create_database_if_not_exists(id=database_name)
container_name = 'DishesContainer'
container = database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path="/name"),
offer_throughput=400)
dish = new_item(name, dish, veg, vegen)
items = [dish]
for item in items:
container.create_item(body=item)
con = 1
if (con == 1):
return func.HttpResponse(f"Dish feeded")
else:
return func.HttpResponse(
"Please pass a valid query in the request body", status_code=400)
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
url = os.environ['COSMOS_URL']
key = os.environ['COSMOS_KEY']
# Create connection to cosmos
client = CosmosClient(url, credential=key)
db = client.get_database_client('database1')
container = db.get_container_client('container1')
# Query to select all records or to select only a particular patient's records
sql_query = 'SELECT * FROM container1'
patient_id = req.params.get('patient_id')
if patient_id:
sql_query += f' WHERE container1.patient_id = "{patient_id}"'
# enable_cross_partition_query should be set to True as the container is partitioned
items = list(
container.query_items(query=sql_query,
enable_cross_partition_query=True))
return func.HttpResponse(
json.dumps(items),
status_code=200,
mimetype="application/json",
)
def connect(self):
self._client = CosmosClient(self._cosmos_db_endpoint,
self._cosmos_db_key)
self._database = self._client.get_database_client(
self._cosmos_db_database_name)
self._container = self._database.get_container_client(
self._cosmos_db_container_name)
def get_db_container(container_id):
endpoint = "https://kawin.documents.azure.com:443/"
key = 'U6VUfkGeu2vzNVKumpXsTWHkPyjbYi9ffmHmmsz9XUz6mqAwwcFTs7FAAzmb4ZF3SptDLFfs2vbCALrzimiJNQ=='
client = CosmosClient(endpoint, key)
id = "hpsdb"
try:
db = client.get_database_client(id)
print('Database with id \'{0}\' was found, it\'s link is {1}'.format(
id, db.database_link))
except exceptions.CosmosResourceNotFoundError:
print('A database with id \'{0}\' does not exist'.format(id))
id = container_id
try:
container = db.get_container_client(id)
print('Container with id \'{0}\' was found, it\'s link is {1}'.format(
container.id, container.container_link))
except exceptions.CosmosResourceNotFoundError:
print('A container with id \'{0}\' does not exist'.format(id))
return container
def get_records_from_cosmos():
latest_2_query_results = []
print ("**** GET RECORDS FROM COSMOS ****")
uri = os.environ.get('ACCOUNT_URI')
key = os.environ.get('ACCOUNT_KEY')
database_id = os.environ.get('DATABASE_ID')
results_container_id = os.environ.get('RESULTS_CONTAINER_ID')
dummy_container_id = os.environ.get('DUMMY_CONTAINER_ID')
client = CosmosClient(uri, {'masterKey': key})
print (client)
database = client.get_database_client(database_id)
tango_container = database.get_container_client(results_container_id) # Results limited to those submitted by UUID
netcraft_container = database.get_container_client(dummy_container_id) # Results include redirects, all results reported in portal
latest_2_query_results_tango = list(tango_container.query_items(query = 'SELECT TOP 2 * FROM c ORDER BY c._ts DESC', enable_cross_partition_query = True))
latest_2_query_results_netcraft = list(netcraft_container.query_items(query = 'SELECT TOP 2 * FROM c ORDER BY c._ts DESC', enable_cross_partition_query = True))
for result in latest_2_query_results_tango:
print (json.dumps(result, indent=True))
for result in latest_2_query_results_netcraft:
print (json.dumps(result, indent=True))
return latest_2_query_results_tango, latest_2_query_results_netcraft
def get_latest_sensor_data(farm_id):
client = CosmosClient(Cosmos.URL, Cosmos.KEY)
pred_container = client.get_database_client(Cosmos.DATABASE).get_container_client(Cosmos.TELEMETRY)
query = """SELECT c.data
FROM c
WHERE c.farmId=@farm_id AND c.deviceType=@device_type
ORDER BY c._ts DESC
OFFSET 0 LIMIT 1"""
# loop through each device and fetch latest sensor reading for each device
agg_result = {}
for device in DEVICE_TYPES:
items = list(pred_container.query_items(
query=query,
parameters=[{ "name":"@farm_id", "value": farm_id },{ "name":"@device_type", "value": device }],
enable_cross_partition_query=True
))
try:
agg_result[device] = {
'unit': items[0]['data']['unit'],
'value': items[0]['data']['value']
}
except:
agg_result[device] = None
return agg_result
def cosdb(db, ctr, prtn):
# Connect to a CosmosDB database and container
# Setup Logging
logging.basicConfig(level=os.environ.get("LOGLEVEL", "INFO"))
# Initialize the Cosmos client
endpoint = os.environ.get("OURO_DOCUMENTS_ENDPOINT",
"SET OURO_DOCUMENTS_ENDPOINT IN ENVIRONMENT")
key = os.environ.get("OURO_DOCUMENTS_KEY",
"SET OURO_DOCUMENTS_KEY IN ENVIRONMENT")
client = CosmosClient(endpoint, key)
database = client.create_database_if_not_exists(id=db)
# Connect to the daily_indicators container
try:
container = database.create_container_if_not_exists(
id=ctr,
partition_key=PartitionKey(path=prtn),
offer_throughput=400)
logging.info('Azure-Cosmos client initialized; connected to ' + db +
'.' + ctr + ' at ' + endpoint)
return container
except Exception as ex:
logging.critical('Unable to create connection to ' + db + '.' + ctr +
' at ' + endpoint)
logging.critical(ex)
quit(-1)
def main(req: func.HttpRequest) -> func.HttpResponse:
URL = "https://junctionx.documents.azure.com:443/"
KEY = "BCw9nhNiTSl9ndUokCw6uHhzZSDgsSGoMjPUa0Ech6E9yiuGKCrsswFIXRatcguQYcNLnYdhFPWPhuaRtLfoxg=="
DATABASE_NAME = "MitoSoftCorp"
CONTAINER_NAME = "workType"
client = CosmosClient(URL, credential=KEY)
work_type = req.params.get("work_type")
try:
database = client.create_database(DATABASE_NAME)
except exceptions.CosmosResourceExistsError:
database = client.get_database_client(DATABASE_NAME)
try:
container = database.create_container(
id=CONTAINER_NAME, partition_key=PartitionKey(path="/name"))
except exceptions.CosmosResourceExistsError:
container = database.get_container_client(CONTAINER_NAME)
user_list = []
for item in container.query_items(
query='SELECT * FROM {} w WHERE w.work_type = "{}"'.format(
CONTAINER_NAME, work_type),
enable_cross_partition_query=True):
user_list.append(item.get("user_id"))
return_data = dict(id=user_list)
return func.HttpResponse(json.dumps(return_data),
headers={"Content-Type": "application/json"},
status_code=200)
def run_sample():
AUTH_URI = os.environ.get("ACCOUNT_URI")
AUTH_KEY = os.environ.get("ACCOUNT_KEY")
DATABASE_ID = "testdocumentmanagementdb"
CONTAINER_ID = "testdocumentmanagementcollection"
client = CosmosClient(AUTH_URI, AUTH_KEY)
database = client.create_database(id=DATABASE_ID)
partition_key = PartitionKey(path="/purchase_order_number")
try:
container = database.create_container(id=CONTAINER_ID,
partition_key=partition_key)
print(f'Container with id "{CONTAINER_ID}"" created')
except HTTPFailure as e:
if e.status_code == 409:
print(f"Container with id {CONTAINER_ID} already exists")
container = database.get_container(container=CONTAINER_ID)
else:
raise
DocumentManagement.create_documents(container)
DocumentManagement.read_document(container, "SalesOrder1")
DocumentManagement.read_documents(container)
client.delete_database(database=DATABASE_ID)
print("\nrun_sample done")
def __init___(image_name,section,label):
# Initialize the Cosmos client
endpoint = os.getenv('AZURE_DATABASE_URL')
key = os.getenv('AZURE_DATABASE_KEY')
# <create_cosmos_client>
client = CosmosClient(endpoint, key)
# </create_cosmos_client>
# Create a database
# <create_database_if_not_exists>
database_name = 'satlabelingdb'
database = client.create_database_if_not_exists(id=database_name)
# </create_database_if_not_exists>
# Create a container
# Using a good partition key improves the performance of database operations.
# <create_container_if_not_exists>
container_name = 'labelinfo'
container = database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path="/info"),
offer_throughput=400
)
image_info = get_image_info_json(image_name,section,label)
read_item = container.read_item(item=image_name, partition_key=key)
if read_item:
container.replace_item(item=read_item, boby=image_info)
else:
container.create_item(body=image_info)
print("updated datbase")
def main(req: func.HttpRequest) -> func.HttpResponse:
logging.info('Python HTTP trigger function processed a request.')
name = req.params.get('name')
if not name:
try:
req_body = req.get_json()
except ValueError:
pass
else:
name = req_body.get('name')
if name:
return func.HttpResponse(
f"Hello, {name}. This HTTP triggered function executed successfully."
)
else:
endpoint = "https://pythondemo.documents.azure.com:443/"
key = 'Qyxm4LRY5vEh41KzNz6HMFzqCbzLZDorySjzm1sZ26QSVduiEUuLOl96y5YA7jSemG7NLPIimdEQXiRWeSU4qA=='
client = CosmosClient(endpoint, key)
database_name = 'pythondemo'
database = client.create_database_if_not_exists(id=database_name)
container_name = 'container1'
container = database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path="/id"),
offer_throughput=400)
query = 'SELECT * FROM c WHERE c.name = "sneha"'
items = list(
container.query_items(query=query,
enable_cross_partition_query=True))
print("bla bla" + str(items))
return func.HttpResponse("success", status_code=200)
def __init__(self):
self.client = CosmosClient(url=connection_key, credential=master_key)
self.db = self.client.create_database_if_not_exists(
id=CONSTANTS['COSMOS']['DATABASE'])
self.container = self.db.create_container_if_not_exists(
id=CONSTANTS['COSMOS']['CONTAINER'],
partition_key=PartitionKey(path="/_partitionKey"),
offer_throughput=400)
def store_deltas(delta_tango, delta_netcraft):
print ("**** STORE DELTAS IN COSMOS DB ****")
uri = os.environ.get('ACCOUNT_URI')
key = os.environ.get('ACCOUNT_KEY')
database_id = os.environ.get('DATABASE_ID')
container_id = os.environ.get('DELTA_CONTAINER_ID')
client = CosmosClient(uri, {'masterKey': key})
database = client.get_database_client(database_id)
container = database.get_container_client(container_id)
# Get date
date_str = datetime.today().strftime('%Y-%m-%d %H:%M:%S')
id_date = int((datetime.utcnow()).timestamp())
id_date_str = str(id_date)
phishing_delta = list(delta_tango['phishing'])
already_blocked_delta = list(delta_tango['already_blocked'])
suspicious_delta = list(delta_tango['suspicious'])
malware_delta = list(delta_tango['malware'])
tango_unique_list = list(set(phishing_delta) | set(already_blocked_delta) | set(suspicious_delta) | set(malware_delta))
#print (type(tango_unique_list))
#print (type(delta_netcraft))
delta_netcraft_list = list(delta_netcraft)
all_unique_list = tango_unique_list + delta_netcraft_list
all_unique_list_str = ' '.join(map(str, all_unique_list))
all_phishing_delta_str = ' '.join(map(str, phishing_delta))
all_already_blocked_delta_str = ' '.join(map(str, already_blocked_delta))
all_suspicious_delta_str = ' '.join(map(str, suspicious_delta))
all_malware_delta_str = ' '.join(map(str, malware_delta))
all_unique_str = ' '.join(map(str, all_unique_list))
all_netcraft_delta_str = ' '.join(map(str, delta_netcraft_list))
container.upsert_item( { 'id': id_date_str,
'date_time': id_date_str,
'date': date_str,
'n_unique': str(len(all_unique_list)),
'unique' : all_unique_list_str,
'n_phishing': str(len(phishing_delta)),
'phishing_delta': all_phishing_delta_str,
'n_blocked_delta': str(len(already_blocked_delta)),
'already_blocked_delta': all_already_blocked_delta_str,
'n_suspicious_delta': str(len(suspicious_delta)),
'suspicious_delta': all_suspicious_delta_str,
'n_malware_delta': str(len(malware_delta)),
'malware_delta': all_malware_delta_str,
'n_netcraft_delta': str(len(delta_netcraft_list)),
'netcraft_delta': all_netcraft_delta_str })
write_attack_urls_to_output(all_unique_list, tango_unique_list, delta_netcraft_list, date_str)
def __init__(self, endpoint, key, database_name, container_name):
self.key = key
self.database_name = database_name
client = CosmosClient(endpoint, key)
self.database = client.create_database_if_not_exists(id=database_name)
self.container = self.database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path="/id"),
offer_throughput=400)
def __init__(self, config: ConfigParser):
config = config
CosmosURI = config['AzureCosmos']['URI']
CosmosKey = config['AzureCosmos']['Key']
client = CosmosClient(CosmosURI, credential=CosmosKey)
database = client.get_database_client(config['AzureCosmos']['DatabaseName'])
self.users = database.get_container_client(config['AzureCosmos']['UsersContainerName'])
self.events = database.get_container_client(config['AzureCosmos']['EventsContainerName'])
self.counters = database.get_container_client(config['AzureCosmos']['CountersContainerName'])
def main(req: func.HttpRequest) -> func.HttpResponse:
##SQL
conn = pyodbc.connect('Driver={SQL Server};'
'Server=.;'
'Database=StackOverflow;'
'Trusted_Connection=yes;')
cursor = conn.cursor()
cursor.execute("""SELECT (SELECT CAST([id] AS varchar(50)) AS [id]
,[AcceptedAnswerId]
,[AnswerCount]
,[Body]
,[ClosedDate]
,[CommentCount]
,[CommunityOwnedDate]
,[CreationDate]
,[FavoriteCount]
,[LastActivityDate]
,[LastEditDate]
,[LastEditorDisplayName]
,[LastEditorUserId]
,[OwnerUserId]
,[ParentId]
,[PostTypeId]
,[Score]
,[Tags]
,[Title]
,[ViewCount]
FOR JSON PATH, WITHOUT_ARRAY_WRAPPER) AS Result
FROM [StackOverflow].[dbo].[Posts] ORDER BY [id] ASC""")
##Cosmos
client = CosmosClient(os.environ["ACCOUNT_URI"], os.environ["ACCOUNT_KEY"], logging_enable=False)
database_name = 'StackOverflow'
database = client.create_database_if_not_exists(id=database_name)
container_name = 'Posts'
container = database.create_container_if_not_exists(
id=container_name,
partition_key=PartitionKey(path="/id"),
offer_throughput=400,
analytical_storage_ttl=-1
)
i = 0
startTime = datetime.now()
for row in cursor:
obj = json.loads(row.Result)
obj["ExtractionDate"] = datetime.now().isoformat()
container.create_item(obj,False)
i=i+1
if i%100==0:
logging.warning("{} elapse for the last 100 Posts insertions".format(datetime.now()-startTime))
startTime = datetime.now()
return func.HttpResponse(
"This HTTP triggered function executed successfully. Pass a name in the query string or in the request body for a personalized response.",
status_code=200
)
def get_cosmos_client(self, cosmos_url: str, cosmos_key: str):
logger = logging.getLogger(__name__)
logger.info('LOGGER: acquiring Cosmos client connection')
try:
self._cosmos_client = CosmosClient(cosmos_url, cosmos_key)
except Exception as e:
logger.error(f'LOGGER: Exception occurred: {e}')
raise e
return self._cosmos_client
