def start_app():
# If running in Google Cloud Run, use cloud logging
if "K_SERVICE" in os.environ:
# Setup Google Cloud logging
# By default this captures all logs at INFO level and higher
log_client = google.cloud.logging.Client()
log_client.get_default_handler()
log_client.setup_logging()
logging.info("Using google cloud logging")
else:
logging.getLogger().setLevel(logging.INFO)
logging.info("Using standard logging")
logging.info("Starting app")
data.Start()
return app
def connectCloudStuff(agent, do_cloud_logging):
""" Connects to logging and firestore DB and returns the db connection"""
# Initialze google stuff
credKey = "GOOGLE_APPLICATION_CREDENTIALS"
if not credKey in os.environ:
print("Creds:", agent.googleCredentialFile)
os.environ[credKey] = agent.googleCredentialFile
cred = credentials.Certificate(os.environ.get(credKey))
firebase_admin.initialize_app(cred, {'projectId': agent.googleProjectId})
db = firestore.client()
# Initialize logging
if do_cloud_logging:
# logger = logging_client.logger("tkbuild-agent-" + agent.name )
logging_client = google.cloud.logging.Client()
logname = "tkbuild-agent-" + agent.name
print("Log name is :", logname)
logging_handler = google.cloud.logging.handlers.CloudLoggingHandler(
logging_client, name=logname)
google.cloud.logging.handlers.setup_logging(logging_handler)
# Also echo to stdout
rootLogger = logging.getLogger()
#rootLogger.setLevel(logging.DEBUG)
stdoutHandler = logging.StreamHandler(sys.stdout)
#stdoutHandler.setLevel(logging.DEBUG)
formatter = logging.Formatter('%(levelname)s: %(message)s')
stdoutHandler.setFormatter(formatter)
rootLogger.addHandler(stdoutHandler)
else:
print("Cloud logging is off")
# Just run with stdout logging for testing
logging.basicConfig(level=logging.INFO)
# logging.debug("log debug")
# logging.info("log info")
# logging.warning("log warn")
# logging.error("log error")
logging.info(f"Agent: {agent.name}: {agent.desc}")
testRepoProj = None
for p in agent.projects.values():
fetchRepoUrl = "(No Fetch Step Defined)"
pfetch = p.getFetchWorkstep()
if pfetch:
fetchRepoUrl = pfetch.repoUrl
logging.info(f"Project: {p.projectId} -- {fetchRepoUrl}")
return db
def logging_handler(client):
# [START create_default_handler]
import logging
handler = client.get_default_handler()
cloud_logger = logging.getLogger('cloudLogger')
cloud_logger.setLevel(logging.INFO)
cloud_logger.addHandler(handler)
cloud_logger.error('bad news')
# [END create_default_handler]
# [START create_cloud_handler]
from google.cloud.logging.handlers import CloudLoggingHandler
handler = CloudLoggingHandler(client)
cloud_logger = logging.getLogger('cloudLogger')
cloud_logger.setLevel(logging.INFO)
cloud_logger.addHandler(handler)
cloud_logger.error('bad news')
# [END create_cloud_handler]
# [START create_named_handler]
handler = CloudLoggingHandler(client, name='mycustomlog')
def logging_handler(client):
# [START create_default_handler]
import logging
handler = client.get_default_handler()
cloud_logger = logging.getLogger('cloudLogger')
cloud_logger.setLevel(logging.INFO)
cloud_logger.addHandler(handler)
cloud_logger.error('bad news')
# [END create_default_handler]
# [START create_cloud_handler]
from google.cloud.logging.handlers import CloudLoggingHandler
handler = CloudLoggingHandler(client)
cloud_logger = logging.getLogger('cloudLogger')
cloud_logger.setLevel(logging.INFO)
cloud_logger.addHandler(handler)
cloud_logger.error('bad news')
# [END create_cloud_handler]
# [START create_named_handler]
handler = CloudLoggingHandler(client, name='mycustomlog')
def logging_handler(client):
# [START create_default_handler]
import logging
handler = client.get_default_handler()
cloud_logger = logging.getLogger("cloudLogger")
cloud_logger.setLevel(logging.INFO)
cloud_logger.addHandler(handler)
cloud_logger.error("bad news")
# [END create_default_handler]
# [START create_cloud_handler]
from google.cloud.logging.handlers import CloudLoggingHandler
from google.cloud.logging_v2.handlers import setup_logging
handler = CloudLoggingHandler(client)
setup_logging(handler)
# [END create_cloud_handler]
# [START create_named_handler]
handler = CloudLoggingHandler(client, name="mycustomlog")
location = ''
# Payload file name that user uploads
payload_name = ''
# Name of PubSub host topic (unique)
topic_name = ''
# Name of PubSub host subscription (unique)
sub_name = ''
# User defined name of a particular experiment
experimentName = ''
# Where the mock ransomware payload is directed to begin execution
rootPairString = '-root=.\\files'
# Maximum number of retries allowed for a test before marked as failed
max_retries = 3
# Logging object
logger = None
# PubSub Host Subscriber object (pulls messages off sub_name)
subscriber = None
# PubSub Host Publisher object (publishes messages to various host VM topics)
publisher = None
# Future corresponding to asynchronous subscription callback (pulling messages off sub_name published by various VMs)
future = None
#Code used to setup GCP logger START
client = google.cloud.logging.Client()
handler = CloudLoggingHandler(client)
cloud_logger = logging.getLogger('cloudLogger')
#NOTE: Can set minimum log level here
cloud_logger.setLevel(logging.INFO) # normally defaults to WARN
cloud_logger.addHandler(handler)
logger = cloud_logger
#Code used to setup GCP logger END
import notifications.alter as alter
from rest.developer import developer_api
from rest.health_check import health_check_api
from rest.language import language_api
from rest.repository import repository_api
from rest.search import search_api
app = Flask(__name__)
app.register_blueprint(health_check_api)
app.register_blueprint(developer_api)
app.register_blueprint(repository_api)
app.register_blueprint(language_api)
app.register_blueprint(search_api)
logging.basicConfig(level=logging.DEBUG, format='%(message)s')
requests_logging = logging.getLogger("urllib3")
requests_logging.setLevel(logging.CRITICAL)
requests_logging.propagate = False
@app.errorhandler(Exception)
def handler_requests_exceptions(error):
alter.to_slack({'text': 'Exception: {}'.format(error)})
return 'Oops, something went wrong', 500
@app.after_request
def apply_headers_and_status_code(response):
response.headers['Content-Type'] = 'application/json'
return response
def main():
# Parse arguments
parser = argparse.ArgumentParser()
parser.add_argument(
'--discord-token',
help=
'Token to use when running the bot. You can either use the raw token string or a path to a text file containing the token.',
dest='discord_bot_token',
default='discord_token.txt')
parser.add_argument('--ffmpeg-path',
help='Path to ffmpeg executable.',
dest='ffmpeg_path',
default='ffmpeg')
parser.add_argument(
'--google-credentials-path',
help='Path to Google application credentials JSON file.',
dest='google_credentials_path',
default='google_credentials.json')
parser.add_argument(
'--dictionary-api',
help=
'A list of dictionary API\'s to use for fetching definitions. These should be in order of priority and separated by comma\'s. Available API\'s are'
'\'google\', \'owlbot\', \'webster\', and \'rapid-words\'. Some API\'s require tokens that must be provided with the appropriate arguments.',
dest='dictionary_api',
default='google')
parser.add_argument(
'--owlbot-api-token',
help=
'The token to use for the Owlbot dictionary API. You can use either the raw token string or a path to a text file containing the token.',
dest='owlbot_api_token',
default='owlbot_api_token.txt')
parser.add_argument(
'--webster-api-token',
help=
'The token to use for the Merriam Webster dictionary API. You can use either the raw token string or a path to a text file containing the token.',
dest='webster_api_token',
default='webster_api_token.txt')
parser.add_argument(
'--rapid-words-api-token',
help=
'The token to use for the RapidAPI WordsAPI dictionary API. You can use either the raw token string or a path to a text file containing the token.',
dest='rapid_words_api_token',
default='rapid_words_api_token.txt')
args = parser.parse_args()
# Set Google API credentials
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = args.google_credentials_path
# Set up GCP logging
gcp_logging_client = google.cloud.logging.Client()
gcp_logging_handler = CloudLoggingHandler(gcp_logging_client,
name='discord-dictionary-bot')
gcp_logging_handler.addFilter(gcp_logging_filter)
logging.getLogger().addHandler(gcp_logging_handler)
# Check which dictionary API we should use
dictionary_apis = []
for name in args.dictionary_api.split(','):
if name == 'google':
dictionary_apis.append(UnofficialGoogleAPI())
elif name == 'owlbot':
if 'owlbot_api_token' not in args:
print(
f'You must specify an API token with --owlbot-api-token to use the owlbot dictionary API!'
)
return
# Read owlbot API token from file
owlbot_api_token = try_read_token(args.owlbot_api_token)
dictionary_apis.append(OwlBotDictionaryAPI(owlbot_api_token))
elif name == 'webster':
if 'webster_api_token' not in args:
print(
f'You must specify an API token with --webster-api-token to use the Merriam Webster dictionary API!'
)
return
# Read API token from file
webster_api_token = try_read_token(args.webster_api_token)
dictionary_apis.append(MerriamWebsterAPI(webster_api_token))
elif name == 'rapid-words':
if 'rapid_words_api_token' not in args:
print(
f'You must specify an API token with --rapid-words-api-token to use the Rapid API WordsAPI dictionary API!'
)
return
# Read API token from file
rapid_words_api_token = try_read_token(args.rapid_words_api_token)
dictionary_apis.append(RapidWordsAPI(rapid_words_api_token))
else:
print(f'Invalid dictionary API: {args.dictionary_api}')
return
# Start client
bot = DiscordBotClient(BackupDictionaryAPI(dictionary_apis),
args.ffmpeg_path)
bot.run(try_read_token(args.discord_bot_token))
def process_chunk(output, group):
name = multiprocessing.current_process().name
logger = logging.getLogger("ltv/{}".format(name))
try:
logger.debug("Processing a chunk to {}".format(output))
import google.cloud.logging
import logging
import os
from battleforcastile_auth.custom_formatter import CustomFormatter
logger = logging.getLogger()
logger.setLevel(logging.INFO)
if os.getenv('PRODUCTION_MODE'):
client = google.cloud.logging.Client()
handler = client.get_default_handler()
handler.setFormatter(CustomFormatter())
logger.addHandler(handler)
def bq_to_yahoo(
event: dict,
content,
) -> bool:
# logging
logging_client = google.cloud.logging.Client()
handler = CloudLoggingHandler(logging_client)
cloud_logger = logging.getLogger('cloudLogger')
cloud_logger.setLevel(logging.INFO)
cloud_logger.addHandler(handler)
# error reporting
error_reporting_client = error_reporting.Client()
# start
func_name = sys._getframe().f_code.co_name
cloud_logger.info('%s start.' % (func_name))
# environmet variable
project_id = os.environ.get('project_id', 'all-project-264506')
bq_dataset_id = os.environ.get('bq_dataset_id', 'mk_demo_project')
start_date = os.environ.get('start_date', '20200701')
end_date = os.environ.get('end_date', '20200731')
bq_query = os.environ.get(
'bq_query', 'SELECT segmentId, clientId, idfa, adid ' +
'FROM `{project_id}.{bq_dataset_id}.yahoo_demo1` ' +
'WHERE created_at BETWEEN {start_date} AND {end_date}').format(
project_id=project_id,
bq_dataset_id=bq_dataset_id,
start_date=start_date,
end_date=end_date,
)
try:
success_count = 0
# create model
ebty = ExportBqDataToY(
project_id,
bq_dataset_id,
)
# get data from BigQuery
bq_data = ebty.get_data_from_bq(bq_query)
bq_data_length = len(list(bq_data))
# send data by API
for row in bq_data:
url_param_idfa = {}
url_param_adid = {}
url_param_gclid = {}
try:
if row.idfa != '':
# IDFA
url_param_idfa['referrer'] = 'idfa_referrer'
url_param_idfa['key'] = 'idfa'
url_param_idfa['value'] = row.idfa
url_param_idfa['flag'] = str(row.segmentId)
ebty.send_api(url_param_idfa)
if row.adid != '':
# AAID
url_param_adid['referrer'] = 'adid_referrer'
url_param_adid['key'] = 'adid'
url_param_adid['value'] = row.adid
url_param_adid['flag'] = str(row.segmentId)
ebty.send_api(url_param_adid)
# GA client ID
url_param_gclid['referrer'] = 'gaid_referrer'
url_param_gclid['key'] = 'ga_client_id'
url_param_gclid['value'] = row.clientId
url_param_gclid['flag'] = str(row.segmentId)
ebty.send_api(url_param_gclid)
except Exception as e:
msg = '%s: %s.' % (__file__, e)
cloud_logger.error(msg)
error_reporting_client.report(msg)
continue
else:
success_count += 1
# wait
sleep(0.01)
except Exception as e:
msg = '%s: %s.' % (__file__, e)
cloud_logger.error(msg)
error_reporting_client.report(msg)
return False
# end
cloud_logger.info('total send count:%d, success send count:%d.' % (
bq_data_length,
success_count,
))
cloud_logger.info('%s end.' % (func_name))
return True
import google.cloud.logging
import logging
from spaceship.config import Config
# enable google cloud logging
# from: https://cloud.google.com/logging/docs/setup/python
if Config.IN_PRODUCTION:
client = google.cloud.logging.Client()
client.setup_logging(log_level=logging.INFO)
# set log level on the root logger
root = logging.getLogger()
root.setLevel(logging.INFO)
# in production, set gunicorn handlers to what google provided
if Config.IN_PRODUCTION:
gunicorn_logger = logging.getLogger('gunicorn.error')
gunicorn_logger.handlers = root.handlers
# if google logging is not available, set up our own (to stdout)
else:
# send output to stdout
handler = logging.StreamHandler()
root.addHandler(handler)
# our log format
formatter = logging.Formatter(
'%(asctime)s %(levelname)s %(name)s : %(message)s')
handler.setFormatter(formatter)
@app.route(API_VERSION + "/del_policy", methods=["GET"])
def del_policy():
"""
Delete a policy
Returns:
"""
name = request.args.get("policy")
with client.context():
res = PolicyModel.query(PolicyModel.Name == name).get()
if not res:
return "not found", 404
res.key.delete()
return "ok", 200
@app.route("/")
def index():
"""
Main Page
:return:
"""
return "ok", 200
if __name__ == "__main__":
logging.getLogger("googleapiclient.discovery_cache").setLevel(logging.info)
app.run(debug=False)
) # Documentation Description
CORS(app)
app.register_blueprint(blueprint)
try:
# This is used when running locally only. When deploying to Google App
# Engine, a webserver process such as Gunicorn will serve the app. This
# can be configured by adding an `entrypoint` to app.yaml.
logging.basicConfig(filemode='w',
format='%(asctime)s - %(message)s',
datefmt='%d-%b-%y %H:%M:%S',
level=logging.INFO)
logging.info('LET THE GAMES BEGIN! API STARTS')
logging.info('==========================================')
logger = logging.getLogger('werkzeug')
handler = logging.FileHandler('./log/Api_log.log')
logger.addHandler(handler)
# Also add the handler to Flask's logger for cases
# where Werkzeug isn't used as the underlying WSGI server.
app.logger.addHandler(handler)
except BaseException:
pass
client = MongoClient(config.MONGO_URI, config.MONGO_PORT)
db = client[config.MONGO_DB]
parser = reqparse.RequestParser()
LOCATION = 'location'
KEY_TERMS = 'key_terms'
}
},
'handlers': {
'console': {
'class': 'logging.StreamHandler',
'level': 'DEBUG',
'stream': 'ext://sys.stdout',
}
},
'root': {
'level': 'DEBUG',
'handlers': ['console'],
}
})
logging.getLogger('root').info("Logging init!")
# If `entrypoint` is not defined in app.yaml, App Engine will look for an app
# called `app` in `main.py`.
app = Flask(__name__)
auth = HTTPBasicAuth()
from auth_credentials_store.all_credentials import all_credentials
app.register_blueprint(all_credentials)
from quick_follow_up.follow_up import follow_up
app.register_blueprint(follow_up, url_prefix="/follow_up")
@auth.verify_password
def verify_password(username, password):
def bq_to_yahoo(
event: dict,
content,
) -> bool:
# logging
logging_client = google.cloud.logging.Client()
handler = CloudLoggingHandler(logging_client)
cloud_logger = logging.getLogger('cloudLogger')
cloud_logger.setLevel(logging.INFO)
cloud_logger.addHandler(handler)
# error reporting
error_reporting_client = error_reporting.Client()
# get parameters
args = docopt(__doc__)
conf_file_path = args['--conf_file_path']
# start
cloud_logger.info('%s start.' % (__file__))
# check lock file
lock_file_path = os.path.join(
os.path.abspath(os.path.dirname(__file__)) + '/../lock/',
os.path.splitext(os.path.basename(conf_file_path))[0] + '.lock',
)
if os.path.isfile(lock_file_path):
msg = '%s: msg:"%s".' % (__file__, 'already running.')
cloud_logger.error(msg)
error_reporting_client.report(msg)
sys.exit(1)
# make lock file
Path(lock_file_path).touch()
try:
# setting
with open(conf_file_path) as f:
conf_data = yaml.full_load(f)
bq_project_id = conf_data['bq']['project_id']
bq_dataset_id = conf_data['bq']['dataset_id']
bq_query = conf_data['bq']['query'].format(
project_id=bq_project_id,
)
success_count = 0
# create model
ebdty = bq_to_yahoo_src.ExportBqDataToY(
bq_project_id,
bq_dataset_id,
)
# get data from BigQuery
bq_data = ebdty.get_data_from_bq(bq_query)
bq_data_length = len(list(bq_data))
# send data by API
for row in bq_data:
url_param_idfa = {}
url_param_aaid = {}
url_param_gclid = {}
try:
if row.idfa != '':
# IDFA
url_param_idfa['referrer'] = 'idfa_referrer'
url_param_idfa['key'] = 'idfa'
url_param_idfa['value'] = row.idfa
url_param_idfa['flag'] = str(row.segmentId)
ebdty.send_api(url_param_idfa)
if row.aaid != '':
# AAID
url_param_aaid['referrer'] = 'adid_referrer'
url_param_aaid['key'] = 'adid'
url_param_aaid['value'] = row.aaid
url_param_aaid['flag'] = str(row.segmentId)
ebdty.send_api(url_param_aaid)
# GA client ID
url_param_gclid['referrer'] = 'gaid_referrer'
url_param_gclid['key'] = 'ga_client_id'
url_param_gclid['value'] = row.clientId
url_param_gclid['flag'] = str(row.segmentId)
ebdty.send_api(url_param_gclid)
except Exception as e:
msg = '%s: %s.' % (__file__, e)
cloud_logger.error(msg)
error_reporting_client.report(msg)
else:
success_count += 1
# wait
sleep(0.01)
# delete lock file
os.remove(lock_file_path)
except Exception as e:
msg = '%s: %s.' % (__file__, e)
cloud_logger.error(msg)
error_reporting_client.report(msg)
# delete lock file
os.remove(lock_file_path)
sys.exit(1)
# end
cloud_logger.info('total send count:%d, success send count:%d.' % (
bq_data_length,
success_count,
))
cloud_logger.info('%s end.' % (__file__))
sys.exit(0)
currentReq = currentRequest.get()
TRACE = "projects/{}/traces/{}".format(client.project,
currentReq._traceID)
currentReq.maxLogLevel = max(currentReq.maxLogLevel,
record.levelno)
except:
TRACE = None
self.transport.send(record,
message,
resource=self.resource,
labels=self.labels,
trace=TRACE)
logger = logging.getLogger()
logger.setLevel(logging.DEBUG)
handler = ViURDefaultLogger(client,
name="ViUR-Messages",
resource=Resource(type="gae_app", labels={}))
logger.addHandler(handler)
sh = logging.StreamHandler()
formatter = logging.Formatter(
"%(levelname)-8s %(asctime)s %(filename)s:%(lineno)s] %(message)s")
sh.setFormatter(formatter)
logger.addHandler(sh)
for logger_name in EXCLUDED_LOGGER_DEFAULTS:
logger = logging.getLogger(logger_name)
# Imports the Google Cloud client library
import google.cloud.logging
# Instantiates a client
client = google.cloud.logging.Client()
# Connects the logger to the root logging handler; by default this captures
# all logs at INFO level and higher
cloud_handler = client.get_default_handler()
# Imports Python standard library logging
import logging
logger = logging.getLogger('Cloud Logger')
logger.setLevel(logging.INFO)
logger.addHandler(cloud_handler)
# Emits the data using the standard logging module
logger.warning('get default handler')
import sys
DRY_RUN = True
SUICIDE_FLAG = False
PROJECT_ID = sys.env["GLOUCD_PROJECT"]
pnconfig = PNConfiguration()
pnconfig.subscribe_key = 'sub-c-52a9ab50-291b-11e5-baaa-0619f8945a4f'
# pnconfig.reconnect_policy = PNReconnectionPolicy.LINEAR
pubnub = PubNub(pnconfig)
import logging
logger = logging.getLogger('bitflyer-collector')
if not DRY_RUN:
logging_client = google.cloud.logging.Client(PROJECT_ID)
logger.addHandler(logging_client.get_default_handler())
logger.setLevel(logging.INFO)
class StreamDataProcessing():
def __init__(self):
pass
def stream_data(self, rows):
pass
:param record:
:return:
"""
return 'discord_dictionary_bot' in record.name or 'discord_dictionary_bot' in record.pathname
def gcp_logging_filter(record):
return 'google.cloud.logging_v2.handlers.transports.background_thread' not in record.name
# Set up logging
logging.basicConfig(
format='%(asctime)s [%(name)s] [%(levelname)s] %(message)s',
level=logging.DEBUG,
datefmt='%m/%d/%Y %H:%M:%S')
logging.getLogger().handlers[0].addFilter(gcp_logging_filter)
def try_read_token(token_or_path: str) -> str:
"""
Try to read from the given file. If the file can be read, return the file contents. Otherwise, return the argument.
:param token_or_path:
:return:
"""
try:
with open(token_or_path) as file:
return file.read()
except IOError:
pass # Ignore and assume the argument is a token string not a file path
return token_or_path
# Instantiates a client
logging_client = google.cloud.logging.Client()
# Connects the logger to the root logging handler; by default this captures
# all logs at INFO level and higher
logging_client.setup_logging()
logger = logging_client.logger(__name__)
logger.log_struct = log_struct_flatten(logger.log_struct)
else:
# must be imported after google.cloud.logging
import logging
import types
from pprint import pformat
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def log_text(self, text, **kw):
level = logging.INFO
if 'severity' in kw:
level = getattr(logging, kw.pop('severity'))
self.log(level, text, **kw)
logger.log_text = log_text.__get__(logger)
def log_struct(self, info, **kw):
logger.log_text(pformat(info), **kw)
logger.log_struct = log_struct.__get__(logger)
logger.log_text("local logging mode")
import google.cloud.logging
from flask import Flask, request, render_template
from chatterbot import ChatBot
from chatterbot.trainers import ListTrainer
from chatterbot.trainers import ChatterBotCorpusTrainer
import re
from logging.config import dictConfig
import random, logging
from google.cloud.logging.handlers import CloudLoggingHandler
from logging.config import dictConfig
client = google.cloud.logging.Client()
handler = CloudLoggingHandler(client)
cloud_logger = logging.getLogger()
cloud_logger.addHandler(handler)
app = Flask(__name__)
# app.config["DEBUG"] = True
# logging.basicConfig(level=logging.INFO, format=, handlers=[
# logging.FileHandler("log.log")
# ])
db_user = os.environ["DB_USER"]
db_pass = os.environ["DB_PASS"]
db_name = os.environ["DB_NAME"]
db_host = os.environ["DB_HOST"]
db = f"mongodb+srv://{db_user}:{db_pass}@{db_host}/{db_name}?retryWrites=true&w=majority"
bot = ChatBot(
