def approval_notify(request):
# loading secrets as lazy globals
# can't be global as this creates issues with automated deployment
# as cold start on initial deployment can't access the variables
global slack_channel, slack_token, notification_key
if not notification_key:
notification_key = get_secret(gcp_project_id, 'notification-key')
if ('X-Api-Key' not in request.headers
or not api_key_valid(request.headers['X-Api-Key'])):
log.fatal('API key is invalid')
return 'unauthorized', 403
if not api_key_long_enough(request.headers['X-Api-Key']):
log.warning(
'API key is too short please make it at least 10 characters')
try:
request_json = request.get_json(silent=True)
title = request_json["title"]
approval_type = request_json["type"]
data = json.loads(request_json["data"])
msg_fields = request_json["msg_fields"]
slack_channel = request_json["slack_channel"]
except KeyError as err:
log.error(
'payload malformed or mandatory data missing: {}'.format(err))
return 'payload malformed or mandatory data missing', 500
log.debug(json.dumps(request_json))
request_id = uuid.uuid4().hex
if not slack_token:
slack_token = get_secret(gcp_project_id, 'slack-token')
client = slack.WebClient(token=slack_token)
# Message posted to Slack as blocks
msg_blocks = [
construct_title_msg_blocks(title),
construct_field_msg_blocks(msg_fields, data),
construct_actions_msg_block(request_id)
]
write_to_datastore(request_id, approval_type, data)
# Send message to Slack
try:
client.chat_postMessage(channel=slack_channel, blocks=msg_blocks)
except slack.errors.SlackApiError as err:
log.error('could not post to slack: {}'.format(err))
return 'error posting to slack', 500
return 'ok', 200
class Sighthound_Interface:
subscription_key = get_secret('MS_API_KEY')
headers = {
"Content-type": "application/json",
"X-Access-Token": get_secret('SIGHTHOUND_API_KEY')
}
def __init__(self, directory_path, outfile_path, overwrite_file=True):
self.directory_path = directory_path
self.outfile_path = outfile_path
self.columns = [
'neutral', 'sadness', 'disgust', 'anger', 'surprise', 'fear',
'happiness'
]
if overwrite_file:
with open(outfile_path, "w+", newline='') as fp:
wr = csv.writer(fp)
wr.writerow(['Image'] + self.columns)
def save_result_to_file(self, result):
with open(self.outfile_path, "a", newline='') as fp:
wr = csv.writer(fp)
wr.writerow(result)
def find_face_emotions(self):
all_image_files = all_images_in_directory(self.directory_path)
i = 0
for file_name in all_image_files:
print(i, file_name)
i += 1
conn = httplib.HTTPSConnection("dev.sighthoundapi.com",
context=ssl.SSLContext(
ssl.PROTOCOL_TLSv1))
image_data = base64.b64encode(open(file_name,
"rb").read()).decode()
params = json.dumps({"image": image_data})
conn.request("POST",
"/v1/detections?type=face,person&faceOption=emotion",
params, self.headers)
response = conn.getresponse()
result = response.read()
#print("Detection Results = " + str(result))
result = json.loads(result)
row = [file_name.split('/')[-1]]
time.sleep(2)
if "objects" in result:
for o in result['objects']:
if o.get('attributes'):
emotions = o['attributes']['emotionsAll']
for c in self.columns:
row.append(emotions[c])
self.save_result_to_file(row)
def run(data):
global secrets
secrets = get_secret("twitter-case-db-cred")
logger.info(
f"msg='Trying to connect to RDS' db_host='{secrets['host']}' db='{secrets['dbname']}' db_user='******'username']}'"
)
try:
con = pymysql.connect(user=secrets['username'],
password=secrets['password'],
host=secrets['host'],
database=secrets['dbname'])
cursor = con.cursor()
except Exception as e:
panic_out(e)
logger.info(
f"msg='Connected to RDS' db_host='{secrets['host']}' db='{secrets['dbname']}' db_user='******'username']}'"
)
insert(cursor, data[0], "user")
insert(cursor, data[1], "tweet")
try:
cursor.close()
con.commit()
con.close()
logger.info(
f"msg='Commited to RDS' db_host='{secrets['host']}' db='{secrets['dbname']}' db_user='******'username']}'"
)
except Exception as e:
panic_out(e)
return 0
def approval_response(request):
# loading secrets as lazy globals
# can't be global as this creates issues with automated deployment
# as cold start on initial deployment can't access the variables
global slack_signing_secret
if not slack_signing_secret:
slack_signing_secret = get_secret(gcp_project_id,
'slack-signing-secret')
content_type = request.headers['content-type']
if content_type == 'application/x-www-form-urlencoded':
if not verify_slack_signature(request):
raise ValueError("slack response signature invalid")
# Slack Guide:
# Your Action URL will receive a HTTP POST request, including a payload
# body parameter, itself containing an application/x-www-form-urlencoded
# JSON string.
json_data = json.loads(request.form["payload"])
log.debug(json_data)
msg_type = json_data['type']
action_id = json_data['actions'][0]['action_id']
if msg_type == "block_actions":
block_action_process(json_data, action_id)
return "action taken", 200
return "no action taken", 500
else:
raise ValueError("Unknown content type: {}".format(content_type))
class Face_pp_Interface:
headers = {
"api_key": get_secret('FACE++_KEY'),
"api_secret": get_secret('FACE++_SECRET'),
"return_attributes": "emotion",
}
def __init__(self, directory_path, outfile_path, overwrite_file=True):
self.directory_path = directory_path
self.outfile_path = outfile_path
self.columns = [
'sadness', 'neutral', 'disgust', 'anger', 'surprise', 'fear',
'happiness'
]
if overwrite_file:
with open(outfile_path, "w+", newline='') as fp:
wr = csv.writer(fp)
wr.writerow(['Image'] + self.columns)
def save_result_to_file(self, result):
with open(self.outfile_path, "a", newline='') as fp:
wr = csv.writer(fp)
wr.writerow(result)
def find_face_emotions(self):
all_image_files = all_images_in_directory(self.directory_path)
i = 0
for file_name in all_image_files:
row = [file_name.split('/')[-1]]
print(i, file_name)
i += 1
files = {"image_file": open(file_name, 'rb')}
url = "https://api-us.faceplusplus.com/facepp/v3/detect"
r = requests.post(url, data=self.headers, files=files)
json_string = json.loads(r.content)
emotions = json_string['faces'][0]['attributes']['emotion']
for c in self.columns:
row.append(emotions[c])
self.save_result_to_file(row)
def send_message(message):
url = secrets.get_secret("slack-webhook-url")
urlfetch.fetch(url,
method="POST",
headers={"Content-Type": "application/json"},
payload=json.dumps({
"username": "******",
"icon_emoji": ":github:",
"text": message,
}))
def connect_to_github():
git_credentials = json.loads(secrets.get_secret())
username = git_credentials['username']
password = git_credentials['password']
gh = login(username=username, password=password)
repo = gh.repository(username, "chapter7")
branch = repo.branch("master")
return gh,repo,branch
def send_message(message):
url = secrets.get_secret("slack-webhook-url")
urlfetch.fetch(
url,
method="POST",
headers={"Content-Type": "application/json"},
payload=json.dumps({
"username": "******",
"icon_emoji": ":github:",
"text": message,
}))
def okta_workflows_callback(data):
global workflows_callback_url, workflows_callback_token
if not workflows_callback_url:
workflows_callback_url = get_secret(gcp_project_id,
'workflows-callback-url')
if not workflows_callback_token:
workflows_callback_token = get_secret(gcp_project_id,
'workflows-callback-token')
response = requests.post(workflows_callback_url,
json=data,
headers={
'x-api-client-token':
workflows_callback_token,
'Content-Type': 'application/json'
})
if response.status_code != 200:
raise ValueError(
'Request to okta workflows returned an error %s, the response is:\n%s'
% (response.status_code, response.text))
def getitem():
if request.method == 'POST':
input_data = {
'source': request.form['source'],
'destination': request.form['destination'],
}
input_data.update(get_secret())
is_ok, get_item_response = post_get_item(input_data)
if not is_ok:
return render_template('index.html', result=get_item_response)
compatibility = get_compatibility(get_item_response)
result = post_revise_item(input_data, compatibility)
return render_template('index.html', result=result)
return render_template('index.html', result='')
def _fetch_github_api(url, method=urlfetch.GET, payload=None):
full_url = urlparse.urljoin("https://api.github.com", url)
logging.info("Making %r request to %r.", method, full_url)
headers = {
"Accept": "Accept: application/vnd.github.v3+json",
"Authorization": "Basic " + secrets.get_secret("community-lead-bot-auth"),
}
if method == urlfetch.POST and payload is not None:
headers["Content-Type"] = "application/json"
encoded_payload = None if payload is None else json.dumps(payload)
return urlfetch.fetch(
full_url,
headers=headers,
method=method,
payload=encoded_payload)
def send_msg(message, channel=DEFAULT_CHANNEL, icon=ICON):
conn = http.client.HTTPSConnection(POST_HOST)
body = {
"text": message,
"username": USER,
"icon_emoji": icon,
"channel": channel,
"mrkdwn": True,
}
post_path = POST_PATH + get_secret("slack.chezbob_token")
conn.request(
"POST", post_path, json.dumps(body),
{"Content-type": "application/json"})
return conn.getresponse().read()
def _fetch_github_api(url, method=urlfetch.GET, payload=None):
full_url = urlparse.urljoin("https://api.github.com", url)
logging.info("Making %r request to %r.", method, full_url)
headers = {
"Accept": "Accept: application/vnd.github.v3+json",
"Authorization":
"Basic " + secrets.get_secret("community-lead-bot-auth"),
}
if method == urlfetch.POST and payload is not None:
headers["Content-Type"] = "application/json"
encoded_payload = None if payload is None else json.dumps(payload)
return urlfetch.fetch(full_url,
headers=headers,
method=method,
payload=encoded_payload)
def run(query, tag):
sql = sqls[query]
if tag is not None:
sql = sql.format(tag)
global secrets
secrets = get_secret()
logger.info(
f"msg='Trying to connect to RDS' db_host='{secrets['host']}' db='{secrets['dbname']}' db_user='******'username']}'"
)
try:
con = pymysql.connect(user=secrets['username'],
password=secrets['password'],
host=secrets['host'],
database=secrets['dbname'])
cursor = con.cursor()
except Exception as e:
panic_out(e)
logger.info(
f"msg='Connected to RDS' db_host='{secrets['host']}' db='{secrets['dbname']}' db_user='******'username']}'"
)
logger.info(f"msg='Querying data' query='{query}' tag='{tag}'")
start_time = time.time()
try:
cursor.execute(sql)
response = cursor.fetchall()
cursor.close()
con.close()
seconds = time.time() - start_time
logger.info(
f"msg='Query done' query='{query}' tag='{tag}' exec_time='{seconds:.2f}'"
)
logger.info(
f"msg='Disconnected from RDS' db_host='{secrets['host']}' db='{secrets['dbname']}' db_user='******'username']}'"
)
except Exception as e:
panic_out(e)
response = formatter(query, response)
return response
def submit_to_tape(tape_archive):
tarname = tape_archive.id
cmdlist = Path(get_secret("staging_cmddir")) / \
f'{os.path.basename(tarname)}.cmd'
cmdlist.parent.mkdir(parents=True, exist_ok=True)
items = [copy.member_name for copy in archive.contents]
with open(cmdlist, 'w') as f:
for destination in items:
f.write(f'{destination}\n')
script = f"""#!/bin/bash
#SBATCH -q xfer
#SBATCH -N 1
#SBATCH -A {get_secret("nersc_account")}
#SBATCH -t 48:00:00
#SBATCH -L project,SCRATCH
#SBATCH -C haswell
#SBATCH -J {Path(tarname).name}
/usr/common/mss/bin/htar cvf {tarname} -L {cmdlist}
for f in `cat {cmdlist}`; do
rm -v $f
done
"""
shfile = f'{cmdlist.resolve()}.sh'
with open(shfile, 'w') as f:
f.write(script)
subfile = shfile.replace('.sh', '.sub.sh')
with open(subfile, 'w') as f:
f.write(f'''#!/bin/bash
module load esslurm
sbatch {Path(shfile).resolve()}'''
)
subprocess.check_call(f'bash {subfile}'.split())
import json
import sys
from flask import Blueprint, jsonify, request
from flask_cors import cross_origin
from slackclient import SlackClient
from . import slack_commands
from secrets import get_secret
blueprint = Blueprint('slack_events', __name__)
SLACK_VERIFICATION_TOKEN = get_secret('slack.ucsdcse_verification_token')
sc = SlackClient(get_secret('slack.ucsdcse_token'))
# Slack event handlers -- populated via decorator
EVENT_HANDLERS = {}
REQUEST_HANDLERS = {}
def handles_event(*handled_events):
"""Decorator for slack event routing."""
def decorator(func):
for event in handled_events:
EVENT_HANDLERS[event] = func
return func
return decorator
# Load data downloaded from S3
train_data = np.load(os.path.join(args.train, 'train.npz'))['data']
train_labels = np.load(os.path.join(args.train, 'train.npz'))['labels']
test_data = np.load(os.path.join(args.test, 'test.npz'))['data']
test_labels = np.load(os.path.join(args.test, 'test.npz'))['labels']
train = chainer.datasets.TupleDataset(train_data, train_labels)
test = chainer.datasets.TupleDataset(test_data, test_labels)
model_dir = args.model_dir
# Define an Optuna study.
import optuna
secret = get_secret(args.db_secret, args.region_name)
connector = 'mysqlconnector'
db = 'mysql+{}://{}:{}@{}/{}'.format(connector, secret['username'],
secret['password'], args.host,
args.db_name)
study = optuna.study.load_study(study_name=args.study_name, storage=db)
study.optimize(objective, n_trials=args.n_trials)
print('Number of finished trials: ', len(study.trials))
print('Best trial:')
trial = study.best_trial
print(' Value: ', trial.value)
import json
import boto3
import os
import random
import time
import tweepy
from secrets import get_secret
auth = tweepy.OAuthHandler(get_secret('TWITTER_CONSUMER_KEY'),
get_secret('TWITTER_CONSUMER_SECRET'))
auth.set_access_token(get_secret('TWITTER_ACCESS_TOKEN_KEY'),
get_secret('TWITTER_ACCESS_TOKEN_SECRET'))
api = tweepy.API(auth)
sqs = boto3.client('sqs')
dynamodb = boto3.client('dynamodb')
def send_job_tweet(tweet_text):
# Sends a tweet to Twitter
api.update_status(status=tweet_text)
def handler(event, context):
print(event)
response = sqs.receive_message(QueueUrl=sqs.get_queue_url(
QueueName='jobsQueue.fifo',
QueueOwnerAWSAccountId=os.environ['ACCOUNT_ID'])['QueueUrl'],
import re
import json
from azure.mgmt.monitor import MonitorManagementClient
from azure.identity import DefaultAzureCredential
from azure.common.client_factory import get_client_from_cli_profile
from azure.mgmt.resource import SubscriptionClient
from azure.mgmt.servicebus import ServiceBusManagementClient
from azure.servicebus.management import ServiceBusAdministrationClient
from secrets import get_secret
secrets = json.loads(get_secret())
APP_NAME = "dead-letter-watcher"
SHORT_APP_NAME = "dl-wtcr"
ENDPOINT = secrets["PULUMI"]["DEADLETTER_WATCHER_ENDPOINT"]
subscription_client = get_client_from_cli_profile(SubscriptionClient)
for sub in subscription_client.subscriptions.list():
subscription_id = sub.subscription_id
sb_client = ServiceBusManagementClient(credential=DefaultAzureCredential(),
subscription_id=subscription_id)
for sb in sb_client.namespaces.list():
if sb.tags.get('group') == 'FileTrust':
resource_id = sb.id
resource_group_match = re.search('resourceGroups/(.*)/providers',
sb.id)
class MS_Face_Interface:
subscription_key = get_secret('MS_API_KEY')
face_api_url = 'https://westcentralus.api.cognitive.microsoft.com/face/v1.0/detect'
headers = {
'Ocp-Apim-Subscription-Key': subscription_key,
'Content-Type': 'application/octet-stream'
}
params = {
'returnFaceId': 'false',
'returnFaceLandmarks': 'false',
'returnFaceAttributes': 'emotion',
}
def __init__(self, directory_path, outfile_path, overwrite_file=True):
self.directory_path = directory_path
self.outfile_path = outfile_path
if overwrite_file:
columns = [
'image', 'anger', 'contempt', 'disgust', 'fear', 'happiness',
'neutral', 'sadness', 'surprise'
]
with open(outfile_path, "w+", newline='') as fp:
wr = csv.writer(fp)
wr.writerow(columns)
def save_result_to_file(self, result, imagename):
keys = list(result.keys())
keys.sort()
row = [imagename]
for k in keys:
row.append(result[k])
with open(self.outfile_path, "a", newline='') as fp:
wr = csv.writer(fp)
wr.writerow(row)
def find_face_emotions(self):
i = 0
start = time.time()
previous = start - 3
already_done = pd.read_csv(self.outfile_path)
print(already_done)
all_image_files = all_images_in_directory(self.directory_path)
for filename in all_image_files:
print(filename.split('/')[-1])
if already_done.image.str.contains(filename.split('/')[-1]).any():
print('already done')
continue
i += 1
print(i)
with open(filename, 'rb') as image:
im_data = image.read()
while time.time() - previous < 3:
time.sleep(0.1)
previous = time.time()
response = requests.post(self.face_api_url,
data=im_data,
params=self.params,
headers=self.headers)
print('request', time.time() - start)
result = (json.dumps(response.json()))
result = json.loads(result)[0]['faceAttributes']['emotion']
self.save_result_to_file(result, filename.split('/')[-1])
import requests
import spotipy
from spotipy.oauth2 import SpotifyClientCredentials
from tqdm import tqdm
from secrets import get_secret
C_ID = get_secret("C_ID")
C_SCT = get_secret("C_SCT")
# prompts for playlist id as input, displays number of songs
def select_playlist():
"""
Prompts for playlist id.
Returns:
basic_playlist_info - information from get_basic_playlist_info()
"""
playlist = input("Enter a spotify playlist id: ")
# check if url, handle if necessary
if "/" in playlist:
from urllib.parse import urlparse
parsed = urlparse(playlist).path.split("/")
if "playlist" in parsed:
playlist = parsed[parsed.index("playlist") + 1]
else:
return "ERROR"
basic_playlist_info = get_basic_playlist_info(playlist)
# if returns an error code as int
if isinstance(basic_playlist_info, int):
from slackclient import SlackClient
from secrets import get_secret
CHANNEL_IDS = {
"slackbot": "D0DCD2PJ5",
}
SLACK_API_URL = "https://slack.com/api/"
DEFAULT_CHANNEL = "#chezbob"
sc = SlackClient(get_secret("slack.ucsdcse_token"))
def populate_channel_mapping():
result = sc.api_call("channels.list")
if not result['ok']:
return
for channel in result['channels']:
CHANNEL_IDS['#' + channel['name']] = channel['id']
def get_channel_id(channel):
channel_id = CHANNEL_IDS.get(channel, None)
if not channel_id:
populate_channel_mapping()
from datetime import timedelta
from collections import OrderedDict
import isodate
import requests
import pafy
from apiclient.discovery import build
from secrets import get_secret
# ----------INIT----------
# MAYBE SET THESE AS INPUTS LATER?
MAX_RESULTS = 5
TIME_DIFF = 3
# basic initialization stuff
GOOGLE_API = get_secret('GOOGLE_API')
YT_API_SERVICE = 'youtube'
YT_API_VERSION = 'v3'
pafy.set_api_key(GOOGLE_API)
YT = "https://www.youtube.com/watch?v="
CLIENT_SECRET = "client_secret.json"
# simple http requests - doesn't use api
# returns youtube search results as list
# returns a lot of entries, can't control how many
def search_youtube(trackname):
'''
Uses HTTP requests (requests lib) w/o using the YouTube API
This can possibly save some quota information, but could be unreliable.
from librus import get_timetables, format_date, get_monday, LibrusNotAvailible
from secrets import get_secret
from config import get_config
from google.cloud import datastore
import sentry_sdk
from sentry_sdk.integrations.serverless import serverless_function
sentry_url = get_secret("SENTRY_URL")
sentry_sdk.init(sentry_url)
with sentry_sdk.configure_scope() as scope:
scope.set_tag('service', 'update')
username = get_config('USERNAME')
password = get_config('PASSWORD')
kind = 'Timetable'
client = datastore.Client()
@serverless_function
def update_timetable(data, context):
timetables = None
timetables = get_timetables(username, password, 4)
entities = []
for element in timetables:
key = client.key(kind, element.date)
common.dt2ts(
datetime.fromisoformat(disqus_comment['createdAt'])))
votes = {
'comment_id': disqus_comment['id'],
'post_id': disqus_comment['thread'],
'ups': disqus_comment['likes'],
'downs': disqus_comment['dislikes'],
'heat': heat
}
tally.append(votes)
return tally
mongo_host = os.getenv('MONGO_HOST') or 'localhost'
rabbitmq_host = os.getenv('RABBITMQ_HOST') or 'localhost'
disqus_key = get_secret('DISQUS_API_KEY')
logging.basicConfig(format='%(asctime)s - %(message)s', level=logging.INFO)
while True:
mongo = MongoClient(mongo_host)
db = mongo['okdiariocom-bot']
comment_count = 0
urls = []
for comment in db['comments'].find():
if comment['url'] not in urls:
urls.append(comment['url'])
tally = get_tally(urls)
updated_count = 0
processed_count = 0
import telebot
from bs4 import BeautifulSoup
import secrets
import requests
import nltk
nltk.downloader.download('vader_lexicon')
from nltk.sentiment.vader import SentimentIntensityAnalyzer
bot = telebot.TeleBot(secrets.get_secret())
def get_headlines(ticker):
url = f"https://finviz.com/quote.ashx?t={ticker}"
headers = {
"User-Agent":
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/70.0.3538.77 Safari/537.36"
}
response = requests.get(url, headers=headers, allow_redirects=False)
if response.status_code == 200:
soup = BeautifulSoup(response.content, "html.parser")
headlines = soup.find_all("a", class_="tab-link-news")
return headlines
return False
def get_sentiment_score(ticker):
score = 0
num = 0
vader = SentimentIntensityAnalyzer()
headlines = get_headlines(ticker)
import decimal
class DecimalFriendlyJSONEncoder(JSONEncoder):
def default(self, obj):
if isinstance(obj, decimal.Decimal):
return float(obj)
return super().default(obj)
app = Flask(__name__)
app.json_encoder = DecimalFriendlyJSONEncoder
conn = db.get_conn()
bobapi = BobApi(conn)
stripe.api_key = get_secret("stripe.live_key")
# TODO - make things log in a better way. This dumps to apache's error logs.
def log(*args):
sys.stderr.write(" ".join([str(x) for x in args]))
sys.stderr.write("\n")
@app.route('/api/v0.1/validate_user/<string:username>', methods=['GET'])
def validate_user(username):
return jsonify({
"success": bobapi.is_valid_username(username),
"username": username})
"""TODO: Change me!"""
import time
import sys
from flask import Blueprint, jsonify, request
from flask_cors import cross_origin
from jose import jwt, JWTError
from secrets import get_secret
from private_api.bob_api import bobapi
VALID_FOR_TIME_S = 300
JWT_SECRET = get_secret('jwt.secret')
JWT_ALGO = get_secret('jwt.algo')
blueprint = Blueprint('userauth', __name__)
"""
>>> token = jwt.encode({'key': 'value'}, JWT_SECRET, algorithm=JWT_ALGO)
>>> data = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGO])
"""
def log(*args):
sys.stderr.write(" ".join([str(x) for x in args]))
sys.stderr.write("\n")
def _mktoken(userid):
exp = int(time.time()) + VALID_FOR_TIME_S
def main(event, context):
try:
secrets = json.loads(get_secret(AWS_SECRET_REGION, AWS_SECRET_NAME))
except Exception as e:
logger.error("Unable to load secrets: {}".format(str(e)))
exit(1)
if AWS_SECRET_KEY_USER not in secrets or AWS_SECRET_KEY_PASS not in secrets:
logger.error("Unable to find secrets in {} or {}".format(
AWS_SECRET_KEY_USER, AWS_SECRET_KEY_PASS))
exit(1)
logger.info("Opening iRODS session")
with iRODSSession(host=IRODS_HOST,
port=IRODS_PORT,
user=secrets[AWS_SECRET_KEY_USER],
password=secrets[AWS_SECRET_KEY_PASS],
zone=IRODS_ZONE) as session:
event_name = event['Records'][0]['eventName']
bucket_name = event['Records'][0]['s3']['bucket']['name']
object_key = urllib.parse.unquote_plus(
event['Records'][0]['s3']['object']
['key']) # irods/Vault/home/rods/requirements.txt
rods_path = remove_prefix(
object_key, IRODS_VAULT_PREFIX) # /home/rods/requirements.txt
now = int(time.time())
logger.info("Processing {} for s3://{}/{}".format(
event_name, bucket_name, object_key))
if not object_key.startswith(IRODS_VAULT_PREFIX):
# This object isn't exposed to iRODS, skip...
logger.warning(
"Skipping object {}, isn't within Vault context {}. Consider setting Prefix filter to \"{}\" in S3 event config"
.format(object_key, IRODS_VAULT_PREFIX, IRODS_VAULT_PREFIX))
exit(0)
# https://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html#supported-notification-event-types
if "ObjectCreated" in event_name:
irods_logical = "/{}{}".format(IRODS_ZONE, rods_path)
s3_physical = "/{}/{}".format(bucket_name, object_key)
size = event['Records'][0]['s3']['object']['size']
try:
session.data_objects.register(s3_physical,
irods_logical,
rescName=IRODS_S3_RESC,
dataSize=size,
dataCreate=now,
dataModify=now)
except CATALOG_ALREADY_HAS_ITEM_BY_THAT_NAME as e:
session.data_objects.unregister(irods_logical)
session.data_objects.register(s3_physical,
irods_logical,
rescName=IRODS_S3_RESC,
dataSize=size,
dataCreate=now,
dataModify=now)
except S3_FILE_STAT_ERR as e:
logger.error("iRODS not able to find {} in S3: {}".format(
s3_physical, repr(e)))
exit(1)
except Exception as e:
logger.error(repr(e))
exit(1)
elif "ObjectRemoved" in event_name:
try:
session.data_objects.unregister("/{}{}".format(
IRODS_ZONE, rods_path))
except CAT_NO_ROWS_FOUND as e:
logger.warning(
"Tried to unregister {} but the object was already unregistered: {}"
.format(rods_path, repr(e)))
except Exception as e:
logger.error(repr(e))
exit(1)
logger.info("Done!")
return {'message': "Updated iRODS object record {}".format(object_key)}
response = response_message + "".join(str(x) for x in meal_list)
else:
meal_list.insert(-1, ', and')
response_message = "The following {} meals are {}: ".format(
meal_type, meal_restriction)
response = response_message + ", ".join(
str(x) for x in meal_list[:-2]) + " ".join(
str(x) for x in meal_list[-2:])
return response
#fetches token from sodexo authentication server
def fetch_token(password):
response = requests.post(
'http://api-staging.sodexomyway.net/api/authenticate',
json={
"Username": "******",
"Password": password['Sodexo']
})
json_response = response.json()
claims = jwt.get_unverified_claims(json_response['token'])
expiration = claims['exp']
response_object = {
'tokenValue': json_response['token'],
'expiration': expiration
}
return response_object
password = get_secret()
token = fetch_token(password)
import logging
import pymysql
import boto3
from secrets import get_secret
logger = logging.getLogger()
logger.setLevel(logging.INFO)
s3 = boto3.client('s3')
#rds_host = 'YOUR_RDS_ENDPOINT' # RDS endpoint
rds_host = 'YOUR_RDS_PROXY_ENDPOINT' # Proxy endpoint
secret = get_secret(secret_name='YOUR_SECRET', region_name='YOUR_REGION')
try:
logger.info(f'Trying to connect to MySQL instance...')
conn = pymysql.connect(rds_host,
user=secret['username'],
passwd=secret['password'],
db=secret['dbname'],
connect_timeout=10)
logger.info("SUCCESS: Connection to RDS MySQL instance succeeded.")
except pymysql.MySQLError as e:
logger.error('Error: Unexpected error: could not connect to MySQL instance.')
logger.error(e)
exit(99)
import sys
import telnetlib
import re
from secrets import get_secret
# your router creds
IP = get_secret('ndms_host')
LOGIN = get_secret('ndms_user')
PWD = get_secret('ndms_password')
# router interface (Home by default)
MAC = CMD = None
TIMEOUT = 5
STATUS_REGEXP = 'access: (.*)\r'
if len(sys.argv) != 3 or sys.argv[2] not in ['deny', 'permit', 'status']:
print("""Usage:
python3 access.py mydevice deny
python3 access.py mydevice permit
where mydevice is the MAC address of a device in the secrets.yaml"""
)
sys.exit(-1)
else:
CMD = sys.argv[2]
MAC = get_secret(sys.argv[1])
class Telnet(object):
def send_command(self, cmd, prompt):
result = self.tn.expect([prompt.encode('utf-8')], TIMEOUT)
if result[0] == -1:
print('Something went wrong, exited')
from secrets import get_secret
from helpers import get_timetable, display_date, is_night
from flask import Flask, render_template
import sentry_sdk
from sentry_sdk.integrations.flask import FlaskIntegration
sentry_url = get_secret('SENTRY_URL')
sentry_sdk.init(sentry_url, integrations=[FlaskIntegration()])
with sentry_sdk.configure_scope() as scope:
scope.set_tag('service', 'front')
app = Flask(__name__)
@app.route('/')
def root():
response = get_timetable()
date = response['date']
timetable = response['timetable']
classes = []
time = 'Nie ma lekcji 👍'
if timetable and timetable['beginning']:
classes = timetable['classes']
time = f'{timetable["beginning"]} – {timetable["end"]}'
dark = is_night()
return render_template('index.html',
import sys
from flask import Blueprint, jsonify, request
from flask_cors import cross_origin
from slackclient import SlackClient
from . import slack_commands
from secrets import get_secret
blueprint = Blueprint('slack_events', __name__)
SLACK_VERIFICATION_TOKEN = get_secret('slack.ucsdcse_verification_token')
sc = SlackClient(get_secret('slack.ucsdcse_token'))
# Slack event handlers -- populated via decorator
EVENT_HANDLERS = {}
REQUEST_HANDLERS = {}
def handles_event(*handled_events):
"""Decorator for slack event routing."""
def decorator(func):
for event in handled_events:
EVENT_HANDLERS[event] = func
return func
return decorator
import yaml
import boto3
import base64
from botocore.exceptions import ClientError
from secrets import get_secret
connections_file = "/usr/local/airflow/connections.yaml"
print("Setting up connections found in connections configuration file")
with open(connections_file) as f:
config = yaml.safe_load(f)
for name, settings in config['connections'].items():
print("Creating connection: ", name)
#obtain DB credentials from the AWS secrets manager
secret = eval(get_secret(settings['secrets'], "eu-west-1"))
subprocess.run([
'airflow', 'connections', '--add', '--conn_id',
settings['conn_id'], '--conn_type', settings['conn_type'],
'--conn_host', settings['conn_host'], '--conn_port',
str(settings['conn_port']), '--conn_schema',
settings['conn_schema'], '--conn_login', secret["username"],
'--conn_password', secret["password"]
])
"""TODO: Change me!"""
import time
import sys
from flask import Blueprint, jsonify, request
from flask_cors import cross_origin
from jose import jwt, JWTError
from secrets import get_secret
from private_api.bob_api import bobapi
VALID_FOR_TIME_S = 300
JWT_SECRET = get_secret('jwt.secret')
JWT_ALGO = get_secret('jwt.algo')
blueprint = Blueprint('userauth', __name__)
"""
>>> token = jwt.encode({'key': 'value'}, JWT_SECRET, algorithm=JWT_ALGO)
>>> data = jwt.decode(token, JWT_SECRET, algorithms=[JWT_ALGO])
"""
def log(*args):
sys.stderr.write(" ".join([str(x) for x in args]))
sys.stderr.write("\n")
def _mktoken(userid):
exp = int(time.time()) + VALID_FOR_TIME_S
verified = {
