async def start_scrap():
while True:
next_time = time.time() + 15
await scraping.scraping()
wait_for = next_time - time.time()
if wait_for > 0:
time.sleep(wait_for)
scraping.scraping().close()
def servScrape(cloudprovider, companyname, keyword):
""" Scrape for the cloud services of the company.
Args:
cloudprovider (string): the cloudprovider to check for.
companyname (string): the name of the company we are looking for.
keyword (string): optional a keyword to specify the search.
Outprints:
the found results of the search.
"""
print(f'{Fore.CYAN}Checking for {cloudprovider} services{Style.RESET_ALL}')
request = []
if cloudprovider == 'Azure':
request.append('site:core.windows.net')
elif cloudprovider == 'AWS':
request.append('site:http://s3.amazonaws.com/*/')
request.append('site:http://*.s3.amazonaws.com/')
elif cloudprovider == 'Google Cloud Platform':
request.append('site:*.storage.googleapis.com')
else:
print(f'Cloudprovider {cloudprovider} not supported')
associated_list = []
potentially_associated_list = []
for r in request:
if keyword:
output = filter_for_correct_links(
scraping.scraping(r + ' ' + companyname + ' ' + keyword))
associated_list.extend(output[0])
potentially_associated_list.extend(output[1])
else:
output = filter_for_correct_links(
scraping.scraping(r + ' ' + companyname))
associated_list.extend(output[0])
potentially_associated_list.extend(output[1])
sorted(list(dict.fromkeys(associated_list)))
sorted(list(dict.fromkeys(potentially_associated_list)))
if associated_list:
print(
f'{Fore.RED}We\'ve got them! Their associated services are as follows:{Style.RESET_ALL}'
)
p.pprint(sorted(associated_list))
if potentially_associated_list:
print(
f'{Fore.YELLOW}Potentially associated services are as follows:{Style.RESET_ALL}'
)
p.pprint(sorted(potentially_associated_list))
if not associated_list and not potentially_associated_list:
print('Nothing detected here.')
def main(args):
"""
最初に実行される関数.
@param args コンソールで渡された引数をdict型にしたもの
"""
# 設定ファイルのロード
config = read_config(args)
# プログラムの実行
if args.module_name == "scraping":
scraping.scraping(args, config)
def convertion(date, link, dest_path, latest=False):
total_somministrations = 0
total_available = 0
total_population = 0
total_health = 0
day, month = date.split(sep="_")
data = [str(datetime.date(2021, int(month), int(day)))] * (len(POPOLAZIONE))
data.insert(0, "Data")
regions, administration, available, percentage = scraping(link)
zipped = zip(regions, administration, available, percentage)
sort = sorted(zipped, key = lambda x: x[0])
regions = [regions for (regions, administration, available, percentage) in sort]
administration = [administration for (regions, administration, available, percentage) in sort]
available = [available for (regions, administration, available, percentage) in sort]
percentage = [percentage for (regions, administration, available, percentage) in sort]
copertura = []
copertura.append('Copertura')
copertura_dosi = []
copertura_dosi.append(('Copertura Teorica'))
for x in range(1, len(POPOLAZIONE)):
copertura.append(str(round(int(administration[x].replace('.', '')) / int(POPOLAZIONE[x]) * 100, 3)) + '%')
copertura_dosi.append(str(round(int(available[x].replace('.', '')) / int(POPOLAZIONE[x]) * 100, 3)) + '%')
total_somministrations += int(administration[x].replace('.', ''))
total_available += int(available[x].replace('.', ''))
total_population += int(POPOLAZIONE[x])
total_health += int(PERSONALE_SANITARIO[x])
print(copertura)
print(copertura_dosi)
regions.append("Totale")
administration.append(total_somministrations)
available.append(total_available)
percentage.append(str(round(total_somministrations / total_available * 100, 3)) + '%')
POPOLAZIONE.append(total_population)
PERSONALE_SANITARIO.append(total_health)
copertura.append(str(round(total_somministrations / total_population * 100, 3)) + '%')
copertura_dosi.append(str(round(total_available / total_population * 100, 3)) + '%')
print("\n")
result = [list(zipped) for zipped in
zip(regions, administration, available, percentage, POPOLAZIONE, PERSONALE_SANITARIO, copertura,
copertura_dosi, data)]
print(result)
np.savetxt(dest_path + str(datetime.date(2021, int(month), int(day))) + '.csv', result,
delimiter=',', fmt='%s')
if latest:
np.savetxt(dest_path + 'latest' + '.csv', result,
delimiter=',', fmt='%s')
def handle_message(event):
recived = event.message.text
trends = scraping(recived)
n = len(trends[0])
if n == 0:
line_bot_api.reply_message(
event.reply_token, TextSendMessage(text='そのようなタグの記事は存在しませんでした'))
else:
text = ''
for i in range(n):
text += trends[0][i] + '\n' + trends[1][i] + '\n'
line_bot_api.reply_message(event.reply_token,
TextSendMessage(text=text))
def __init__(self,
working_dir,
executable_path,
download_path,
cache_path,
mode='csv',
directory_polling_interval=2.,
directory_polling_limit=10):
self.working_dir = Path(working_dir)
self.working_dir.mkdir(exist_ok=True)
self.download_dir = self.working_dir / 'publication'
self.download_dir.mkdir(exist_ok=True)
self.database_dir = self.working_dir / 'database'
self.database_dir.mkdir(exist_ok=True)
self.gds_path = self.database_dir / 'gds.csv'
self.publication_path = self.database_dir / 'publication.csv'
self.source_path = self.database_dir / 'source.csv'
if self.gds_path.exists():
self.gds = pd.read_csv(self.gds_path)
self.gds_index = set(self.gds.gds_uid)
else:
self.gds = pd.DataFrame(columns=columns['gds'])
self.gds_index = set()
if self.publication_path.exists():
self.publication = pd.read_csv(self.publication_path)
self.publication_index = set(self.publication.doi)
else:
self.publication = pd.DataFrame(columns=columns['publication'])
self.publication_index = set()
if self.source_path.exists():
self.source = pd.read_csv(self.source_path)
else:
self.source = pd.DataFrame(columns=columns['source'])
self.scraping = scraping(executable_path, cache_path, download_path)
self.crawler = paper_crawler(executable_path, str(self.download_dir),
cache_path)
self.directory_polling_interval = directory_polling_interval
self.directory_polling_limit = directory_polling_limit
def result(request, product):
scraping_object = scraping.scraping(product)
items = scraping_object.get_item_list
return render(request, 'search/result.html', {
'item': items,
})
print('\n\t\t MENU')
print('1- Generate wordlist')
print('2- Validate password')
print('3- Close')
op = input('What is your choice number: ')
return op
if __name__ == "__main__":
view.clear()
view.logo()
url = get_URL()
profile = {}
if (url != ''):
profile = fs.scraping(url)
else:
profile["name"] = get_name()
get_information(profile)
view.clear()
view.show_info(profile)
finish = False
t = Combinations(profile)
info = t.info
op = menu('')
while (not finish):
if (op == '1'):
wg(info, profile)
message = 'Your wordlist is ready, the file is wordlist.txt'
op = menu(message)
async def stop_scrap():
scraping.scraping().close()
return render_template('index.html')
from flask import Flask, jsonify
from gevent.pywsgi import WSGIServer
import logging
from flask import request
from scraping import scraping
LOG_FORMAT = "%(levelname)s : %(filename)s : %(asctime)s : %(message)s"
logging.basicConfig(filename=".logs/ApiLogs",
level=logging.INFO,
format=LOG_FORMAT,
filemode="w")
logger = logging.getLogger()
scrap = scraping()
app = Flask(__name__)
@app.route("/getDict/", methods=["GET"])
def getDictionary():
letter = request.args.get('letter')
logger.info(f"getDictionary of {letter}")
words = scrap.getDictionary(letter)
if (words == None):
logger.info(f"Letra não encontrda")
return jsonify({"None": "Nenhuma palavra encontrada com essa letra"})
# -*_ encoding: utf-8 -*-
import os
from scraping import scraping
from download import download
from dataProcessing import processing
def clear():
return os.system('clear')
url = "http://www.ans.gov.br/prestadores/tiss-troca-de-informacao-de-saude-suplementar"
pdfFileName = download(scraping(url))
csvFileName = processing(pdfFileName)
clear()
print('URL: ' + url)
print('Arquivo ' + pdfFileName + ' baixado.')
print('Tabelas extraidas e salvas em ' + csvFileName + '.')
import matplotlib as m
import numpy as np
import pandas as pd
from scipy import stats
import scraping as sc
import seaborn as sns
import matplotlib.pyplot as plt
import pingouin as pg
# Imports para formatação dos gráficos
sns.set_style('whitegrid')
dfScraping = sc.scraping()
# Convertendo a coluna "Chip Time" em apenas minutos
# Chip time é o tempo total de corrida medido com a leitura do sensor RFID no selo da camisa do participante
time_list = dfScraping[' Chip Time'].tolist()
# Visualizando uma amostra de dados
# print(time_list[1:5])
# Lista para receber o resultado da conversão
time_mins = []
# Interação para conversão em minutos
for i in time_list:
i = i.strip(' ')
if len(i) != 7:
i = '0:' + i
h, m, s = i.split(':')
math = (int(h) * 3600 + int(m) * 60 + int(s)) / 60
sys.setdefaultencoding("utf-8")
tmpl_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'templates')
app = Flask(__name__, template_folder=tmpl_dir)
DEBUG = True
SECRET_KEY = 'development key'
tmpl_dir = os.path.join(os.path.dirname(os.path.abspath(__file__)),
'templates')
app = Flask(__name__, template_folder=tmpl_dir)
app.config.from_object(__name__)
scraping()
username = None
conn = None
cur = None
def encrypt_password(password):
encrypted_pass = hashlib.sha1(password.encode('utf-8')).hexdigest()
return encrypted_pass
conn = lite.connect('orbis.sqlite')
cur = conn.cursor()
import scraping rawtext = 'スライド作成には大きな労力が必要です。プレゼン発表の前はただでさえ内容確認、Q&A対策、などの準備に追われているのに、発表用のスライドも用意しなくてはなりません。しかも、スライド作成はすぐに終わる作業ではなく、大きな手間を必要とします。 \ まず、発表内容から要点だけを抽出しなくてはなりません。原稿そのままの文章をスライドに載せてしまうと非常に見にくく、またどの部分に着目したら良いかもわからないため聞いている人に上手く伝えることができません。 \ また、その内容に合うグラフやイラストも用意する必要があります。視覚的に聞いている人に訴えかけることは大切な手法の一つです。 \ さらに、文章と画像をいい感じにスライドに配置する必要もあります。 \ 最後に、テンプレートを選択する必要があります。' word_list = scraping.scraping(rawtext, 1) print(word_list) path = scraping.irasutoya(word_list, 1) print(path)
if not associated_websites and not potential_associated_websites:
print('Nothing detected here.')
"""
In case of 429, try:
https://findwork.dev/blog/advanced-usage-python-requests-timeouts-retries-hooks/
Category: Retry on failure
"""
companyname = input('Provide the companyname: ')
main_domain = input('Provide the company\'s classic domain: ')
keyword = None
keyword_option = input('Do you want to use an additional keyword? [y/n]: ')
if keyword_option == 'y' or keyword_option == 'yes':
keyword = input('Enter keyword: ')
print(f'{Fore.CYAN}Checking Amazon websites{Style.RESET_ALL}')
if keyword:
get_associated_websites(scraping.scraping("site:s3.amazonaws.com -filetype:pdf " + companyname + " " + keyword), companyname, main_domain, keyword)
else:
get_associated_websites(scraping.scraping("site:s3.amazonaws.com -filetype:pdf " + companyname), companyname, main_domain, keyword)
print(f'{Fore.CYAN}Checking Azure websites{Style.RESET_ALL}')
if keyword:
get_associated_websites(scraping.scraping("site:azurewebsites.net -filetype:pdf " + companyname + " " + keyword), companyname, main_domain, keyword)
else:
get_associated_websites(scraping.scraping("site:azurewebsites.net -filetype:pdf " + companyname), companyname, main_domain, keyword)
def get():
return scraping.scraping()
async def tenki(self, message: discord.message):
sc = scraping()
whether = sc.forecast("名古屋", "横浜")
await self.send_message(message, whether)