def main():
#Load Logging Default Config
lc.setup_logging()
logger = logging.getLogger(__name__)
logger.info("----- -> START_UP <- "+"-"*50)
#Init Holiday Class
holiday.init("START HOLIDAY CONFIG LOADER")
date_for_test = "2017-06-11"
#1
logger.info(holiday.holidays)
#2
logger.info(holiday.holidays[3])
#3
logger.info('is_sunday("{}")? [{}]'.format(date_for_test, holiday.is_sunday(date_for_test)))
#4
logger.info('is_holiday("{}")? [{}]'.format(date_for_test, holiday.is_holiday(date_for_test)))
#5
logger.info('is_work_day_off("{}")? [{}]'.format(date_for_test, holiday.is_work_day_off(date_for_test)))
print("END OF TESTS")
def tmp_main():
"""."""
setup_logging()
logger = logging.getLogger(__name__)
cik = int('0000000020')
scrape = Scrape()
company, urls = scrape.document_list(cik)
for url in urls:
doc, files = scrape.file_list(url)
logger.info('fiscal_year_end_%s' % doc.filing_date_)
for record in files:
f = scrape.extract_rawdata(record)
logger.info('文件大小：%s，文件实际大小：%s' % (f.size_, f.msize))
hintInterest, hintForex = get_hint(f.source_.decode('utf-8'))
logger.info('hintInterest:%s, hintForex:%s' %
(hintInterest, hintForex))
from multiprocessing import Pool
# from paths_usa import era_path
DOWNLOAD_DIR = '/data/users/kgruber/Data/era5/TX'
COUNTRY = 'TX'
YEARS = range(1950, 1979)
MONTHS = list(range(1, 13))
north = 36
south = 25
west = -107
east = -93
setup_logging(op.join(DOWNLOAD_DIR, 'download.log'))
def download_era5(year):
# API documentation for downloading a subset:
# https://confluence.ecmwf.int/display/CKB/Global+data%3A+Download+data+from+ECMWF+for+a+particular+area+and+resolution
# https://retostauffer.org/code/Download-ERA5/
# Format for downloading ERA5: North/West/South/East
bounding_box = "{}/{}/{}/{}".format(north, west, south, east)
logging.info("Downloading bounding_box=%s for years=%s and months=%s",
bounding_box, YEARS, MONTHS)
import os
import logging
import logging.config
import json
import requests
from flask import escape
from flask import abort
from logging_config import setup_logging
setup_logging()
logger = logging.getLogger(__name__)
from config import CONFIG
from bitbucket2chat import handle_bitbucket_event
def main(request):
bot_name = CONFIG['bot_name'].get()
if request is None:
logger.info('Hi, I\'m %s!', bot_name)
return 'OK'
logger.info('Bot %s is alive!', bot_name)
logger.info('This request came from: %s', request.remote_addr)
if request.headers.get('User-Agent') == 'Bitbucket-Webhooks/2.0':
event = request.get_json(silent=True)
replySpider_start_urls_num, userSpider_start_urls_num)
logger.info(time.asctime() + '\t' + 'begin' + '\t' + s)
if user_urls_num != 0:
if userSpider_start_urls_num == 0:
logger.info(time.asctime() + '\t' + 'run_user_spider')
run_user_spider(r, 50)
if reply_urls_num != 0:
if replySpider_start_urls_num == 0:
logger.info(time.asctime() + '\t' + 'run_reply_spider')
run_reply_spider(r, 1000)
elif post_urls_num != 0:
if postSpider_start_urls_num == 0:
logger.info(time.asctime() + '\t' + 'run_post_spider')
run_post_spider(r, 1000)
else:
if listSpider_start_urls_num == 0:
logger.info(time.asctime() + '\t' + 'run_list_spider')
run_list_spider(r)
# print(time.asctime() + '\t' + 'sleep')
time.sleep(60)
if __name__ == '__main__':
yaml_path = 'logging_config.yaml'
setup_logging(yaml_path)
run()
old_key = set(sqlserver._query(
'select [KEY] from IT_OPS.XQY_TGT_TEST2')['KEY'])
add_key = list(execl_key-old_key)
if len(add_key) > 0:
logger.info(f"new key:{add_key}")
new_df = DataFrame(columns=table_column)
for key in add_key:
new_df = new_df.append(
df.loc[df['KEY'] == key], ignore_index=True)
logger.info(f'insert into new data number {len(new_df)}')
# new_df.to_sql(name='XQY_TGT_TEST2', con=engine,
# if_exists='append', schema='IT_OPS', index=False, chunksize=1000)
sqlserver._insert(new_df, 'XQY_TGT_TEST2', 'IT_OPS')
logger.info(f'Insertion successful number {len(new_df)}')
else:
logger.info('no change data')
end = time.time()
runtime = end-starttime
logger.info(f"Running time:{runtime}")
if __name__ == "__main__":
logger = logging.getLogger(__name__)
try:
setup_logging(default_path='logging.json',
default_level=logging.INFO,
env_key='LOG_CFG')
main()
except Exception as e:
logger.error(e)
def main():
"""保存公司基本信息."""
setup_logging()
logger = logging.getLogger(__name__)
conn, cur = get_conn()
sql = "SELECT url from sec.tb_file"
sqlOk = "SELECT id, is_ok from sec.tb_company where cik = %s"
sqlInsertCompany = """INSERT into sec.tb_company(cik, name, sic, location, state_of_inc,
fiscal_year_end, baddr_state, baddr_city,
baddr_street, baddr_zip, baddr_phone,
maddr_street, maddr_city, maddr_state,
maddr_zip, ctime)
values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,
%s,%s,%s,%s,%s)"""
sqkUpdateCompany = "UPDATE sec.tb_company set is_ok = 1 where id = %s"
sqlDoc = "SELECT id from sec.tb_document where url = %s"
sqlInsetDoc = """INSERT into sec.tb_document(acc_no, url, company_id, type, filing_date,
period_of_report, accepted, documents, company_name,
sic, state_of_inc, fiscal_year_end, baddr_street,
baddr_city, baddr_state, baddr_zip, baddr_phone,
maddr_street, maddr_city, maddr_state, maddr_zip)
values(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,
%s,%s,%s,%s,%s,%s)"""
sqlInsertFile = "INSERT into sec.tb_file(company_id, doc_id, url, seq, describtion, type, \
size) values(%s,%s,%s,%s,%s,%s,%s)"
fileSet = set()
if cur.execute(sql):
fileSet = set(record[0] for record in cur.fetchall())
scrape = Scrape()
wb = load_workbook('./demand20180810/Sample_short.xlsx')
sheet = wb.get_sheet_by_name('Sample_short2')
for v in sheet.values:
ctime = time.strftime('%Y-%m-%d %H:%M:%S')
cik = v[7]
try:
int(cik)
except ValueError as e:
print("cik格式不符", cik)
continue
if len(cik) != 10:
print("cik长度不符", cik)
continue
logger.debug('cik:%s' % cik)
companyid = None
isOk = 0
if cur.execute(sqlOk, cik):
record = cur.fetchone()
companyid = record[0]
isOk = record[1]
if not isOk:
print("cik采集", cik)
try:
company, urls = scrape.document_list(cik)
except Exception as e:
if e.args[0] == 'No matching CIK':
logger.debug('No matching CIK:%s' % cik)
continue
else:
raise e
else:
print("cik已存在", cik)
continue
if not companyid:
print(company)
cur.execute(sqlInsertCompany,
(company.cik_, company.name_, company.sic_,
company.location_, company.state_of_inc_,
company.fiscal_year_end_, company.baddr_state_,
company.baddr_city_, company.baddr_street_,
company.baddr_zip_, company.baddr_phone_,
company.maddr_street_, company.maddr_city_,
company.maddr_state_, company.maddr_zip_, ctime))
companyid = cur.lastrowid
conn.commit()
logger.debug('公司ID:%s,cik:%s' % (companyid, cik))
for url in urls:
doc, files = scrape.file_list(url)
doc.company_id_ = companyid
if cur.execute(sqlDoc, url):
docId = cur.fetchone()[0]
else:
cur.execute(
sqlInsetDoc,
(doc.acc_no_, url, doc.company_id_, doc.type_,
doc.filing_date_, doc.period_of_report_, doc.accepted_,
doc.documents_, doc.company_name_, doc.sic_,
doc.state_of_inc_, doc.fiscal_year_end_,
doc.baddr_street_, doc.baddr_city_, doc.baddr_state_,
doc.baddr_zip_, doc.baddr_phone_, doc.maddr_street_,
doc.maddr_city_, doc.maddr_state_, doc.maddr_zip_))
docId = cur.lastrowid
conn.commit()
for record in files:
if record[2] not in fileSet:
f = File()
f.seq_ = record[0]
f.describtion_ = record[1]
f.url_ = record[2]
f.type_ = record[3]
f.size_ = record[4]
f.company_id_ = companyid
f.doc_id_ = docId
cur.execute(sqlInsertFile,
(f.company_id_, f.doc_id_, f.url_, f.seq_,
f.describtion_, f.type_, f.size_))
conn.commit()
fileSet.add(f.url_)
cur.execute(sqkUpdateCompany, companyid)
conn.commit()
if alert.failed_alerts:
if environ.get('SKIP_EMAIL_FOR_SUCCESSFUL_REMEDIATION',
'false') == 'true':
if not all(
failure.get('remediated', False)
for failure in alert.failed_alerts):
alert.email_results()
else:
alert.email_results()
log.handlers = [logging_config.get_normal_formatter()]
log.info(f'Finished processing alerts for Cluster: {cluster}.')
if __name__ == '__main__':
logger: Logger = logging_config.setup_logging()
cluster_contexts: List[str] = environ.get('CLUSTER_CONTEXTS',
'current').split(',')
# Import all modules below the `alerts/` directory
alert_plugins = (import_module(f'alerts.{alert_module.stem}')
for alert_module in Path('./alerts').glob('*.py'))
# Load all classes from each module that inherits from the BaseAlert abstract class
alert_classes = (
cls for plugin in alert_plugins
for cls in getmembers(plugin,
predicate=lambda curr_class: isclass(curr_class)
and BaseAlert in curr_class.__bases__) if cls)
schedule.every(1).hour.do(process_alerts_and_remediations, logger,
cluster_contexts, alert_classes)
