def run_command():
# create the argument parser
parser = argparse.ArgumentParser()
# add command option
parser.add_argument("command", help="the command to run")
# get the command
command = parser.parse_args().command
# get the config
conf = common.config()
# get the command
command_obj = None
for comm in conf['commands']:
if comm == command:
command_obj = conf['commands'][comm]
break
# if command not found, show error
if not command_obj:
raise CommandNotFound()
else:
# run the command
filename = command_obj['filename']
mod = imp.load_module(filename=filename, name=comm,
file=open(filename), details=('', 'r', imp.PY_SOURCE))
mod.main()
def __init__(self, config = common.config()):
self.config = config
self.train_lable = None
self.train_data = None
self.test_label = None
self.test_data = None
self.model = None
self.cluster_info_dict = {}
self.predict_result_dict = {}
def __init__(self, host, port, headers={}, use_ssl=True, use_urllib=False):
self.host = host
self.port = port
self.headers = headers
self.use_ssl = use_ssl
self.use_urllib = use_urllib
self.log = logging.getLogger(__name__)
FORMAT = '%(asctime)s [%(levelname)s] %(message)s'
logging.basicConfig(filename=config('DEBUG_LOGFILE'),
level=logging.DEBUG, format=FORMAT)
article = news.ArticlePage(news_site_uid, _build_link(
host, link)) #revisa que los vinculos esten bien construidos
except (HTTPError, MaxRetryError) as e:
#si ocurre un error, invalida el articulo
logger.warning('Error while fechting the article', exc_info=False)
#si el articulo no tiene cuerpo, queda invalidado
if article and not article.body:
logger.warning('Invalid article. There is no body')
return None
return article
#Revisa los links y valida los principales con las expresiones regulares
def _build_link(host, link):
if is_well_formed_link.match(
link): #si es de tipo https://example.com/hello
return link
elif is_root_path.match(link): #si es de tipo /some-text
return '{}{}'.format(host, link)
else:
return '{}/{}'.format(host, link)
if __name__ == '__main__':
#enlista los nombres de los sites
news_site_choices = list(config('config.yaml')['news_sites'].keys())
for choices in news_site_choices:
_news_scraper(choices)
def __init__(self, news_site_uid, url):
self._config = config()['news_sites'][news_site_uid]
self._queries = self._config['queries']
self._html = None
self._url = url
self._visit(url)
#!/usr/bin/env python
import yara
import re
import sys
import json
import os
import argparse
import time
import common
from check_base64 import extract_base64_strings
from checkcryptonote import is_valid_wallet
#load config file
config = common.config()
#load needed params
samples_dir = config['samples_dir']
rules_dir = config['rules_dir']
def MoneroWallet(sha256, *base64list):
#base64list = extract_base64_strings(sha256)
regex_monero = r"(4[0-9AB][0-9a-zA-Z]{93,104})"
sample_path = samples_dir + sha256
Monero_rule = yara.compile(filepath=rules_dir + './monerowallet.yara')
matches = Monero_rule.match(sample_path)
if matches != []:
filtered_matches = []
for match in matches[0].strings:
wallet_addr = re.search(regex_monero, str(match[2]))
if not wallet_addr.group(0).islower() and not wallet_addr.group(
import datetime
import logging
import os
import lxml.html as html
import requests
import yaml
# Utilites
from common import config
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
HOME_URL = str(config()["larepublica"]["url"])
XPATH_LINK_TO_ARTICLE = str(config()["larepublica"]["links"])
XPATH_TITLE = str(config()["larepublica"]["titulo"])
XPATH_SUMMARY = str(config()["larepublica"]["resumen"])
XPATH_BODY = str(config()["larepublica"]["cuerpo"])
def error_handler():
"""
docstring
"""
pass
def parse_notice(link, today):
"""
docstring
"""
def start_analysis(self, options, monitor):
"""Start the analysis by uploading all required files.
@param options: the task options
@param monitor: identifier of the monitor to be used.
"""
log.info("Starting analysis on guest (id=%s, ip=%s)",
self.machine.label, self.machine.addr)
self.options = options
self.timeout = options["timeout"] + config("cuckoo:timeouts:critical")
# Wait for the agent to come alive.
self.wait_available()
# Could be beautified a bit, but basically we have to perform the
# same check here as we did in wait_available().
if db.guest_get_status(self.task_id) != "starting":
return
# Check whether this is the new Agent or the old one (by looking at
# the status code of the index page).
r = self.get("/", do_raise=False)
if r.status_code != 200:
log.critical(
"While trying to determine the Agent version that your VM is "
"running we retrieved an unexpected HTTP status code: %s. If "
"this is a false positive, please report this issue to the "
"Cuckoo Developers. HTTP response headers: %s",
r.status_code,
json.dumps(dict(r.headers)),
)
db.guest_set_status(self.task_id, "failed")
return
try:
status = r.json()
version = status.get("version")
features = status.get("features", [])
except:
log.critical(
"We were unable to detect either the Old or New Agent in the "
"Guest VM, are you sure you have set it up correctly? Please "
"go through the documentation once more and otherwise inform "
"the Cuckoo Developers of your issue.")
db.guest_set_status(self.task_id, "failed")
return
log.info("Guest is running Cuckoo Agent %s (id=%s, ip=%s)", version,
self.machine.label, self.machine.addr)
# Pin the Agent to our IP address so that it is not accessible by
# other Virtual Machines etc.
if "pinning" in features:
self.get("/pinning")
# Obtain the environment variables.
self.query_environ()
# Upload the analyzer.
self.upload_analyzer(monitor)
# Pass along the analysis.conf file.
self.add_config(options)
# Allow Auxiliary modules to prepare the Guest.
self.aux.callback("prepare_guest")
# If the target is a file, upload it to the guest.
if options["category"] == "file" or options["category"] == "archive":
data = {
"filepath":
os.path.join(self.determine_temp_path(), options["file_name"]),
}
files = {
"file": ("sample.bin", open(options["target"], "rb")),
}
self.post("/store", files=files, data=data)
if "execpy" in features:
data = {
"filepath": "%s/analyzer.py" % self.analyzer_path,
"async": "yes",
"cwd": self.analyzer_path,
}
self.post("/execpy", data=data)
else:
# Execute the analyzer that we just uploaded.
data = {
"command": "C:\\Python27\\pythonw.exe %s\\analyzer.py" %
self.analyzer_path,
"async": "yes",
"cwd": self.analyzer_path,
}
self.post("/execute", data=data)
def __init__(self, config = common.config()):
self.config = config
def articles_and_categories_extraction(host, article_url, iterator):
''' Function that extracts the articles for url, and returns it in a dictionary, also, it returns a list for each category. '''
# Variables definition
title_query = config()['news_sites'][iterator]['queries']['title']
subtitle_query = config()['news_sites'][iterator]['queries']['subtitle']
body_query = config()['news_sites'][iterator]['queries']['content']
images_query = config()['news_sites'][iterator]['queries']['images']
category_long_query = config()['news_sites'][iterator]['queries']['category_long']
tags_query = config()['news_sites'][iterator]['queries']['tags']
author_query = config()['news_sites'][iterator]['queries']['author']
publication_date_query = config()['news_sites'][iterator]['queries']['publication_date']
categories_query = config()['news_sites'][iterator]['queries']['categories']
data = {}
try:
logger.info(f'Extracting article and category content from {article_url}')
# Requesting info from the categories list
article_page = requests.get(article_url)
if article_page.status_code == 200:
home = article_page.content.decode('utf-8')
parsed = html.fromstring(home)
# Extracting the content for each article
try:
title = parsed.xpath(title_query)
title = replacer(title)
except ValueError as e:
logger.warning(f'there is no title')
title = None
try:
subtitle = parsed.xpath(subtitle_query)
subtitle = replacer(subtitle)
except ValueError as e:
logger.warning('There is no subtitle')
subtitle = None
try:
body = parsed.xpath(body_query)
body = replacer(body)
except ValueError as e:
logger.warning(f'there is no body')
body = None
try:
category_long = parsed.xpath(category_long_query)
category_long = replacer(category_long)
except ValueError as e:
logger.warning(f'there is no category')
category_long = None
try:
tags = parsed.xpath(tags_query)
tags = replacer(tags)
except ValueError as e:
logger.warning(f'there is no tags')
tags = None
try:
author = parsed.xpath(author_query)
author = replacer(author)
except ValueError as e:
logger.warning(f'there is no author')
author = None
try:
categories = parsed.xpath(categories_query)
categories = replacer(categories)
except ValueError as e:
logger.warning(f'there are no categories')
categories = None
try:
images = parsed.xpath(images_query)
images = replacer(images)
except ValueError as e:
logger.warning(f'there are no images')
images = None
try:
publication_date = parsed.xpath(publication_date_query)
publication_date = replacer(publication_date)
except ValueError as e:
logger.warning(f'there is no publication date')
publication_date = None
data = {
'title': title,
'subtitle': subtitle,
'body': body,
'images': parsed.xpath(images_query),
'category_long': category_long,
'tags': tags,
'author': author,
'publication_date': publication_date,
'news_url': article_url,
'host': host
}
category = "".join(categories)
else:
logger.warning(f'{article_url}: {article_page.status_code}')
raise ValueError(f'Error.')
except (HTTPError, MaxRetryError) as e:
logger.warning('Error while fetching article', exc_info=False)
return data, category.capitalize()
def _news_scraper(news_site_uid):
host = config()['news_sites'][news_site_uid]['url']
logging.info("Beginning scraper for {}".format(host))
homepage = news.HomePage(news_site_uid, host)
for link in homepage.article_list:
print(link)
def v_main():
knn = KnnClassifer(common.config())
knn.knn()
if __name__ == '__main__':
data['categories'] = []
data['articles'] = []
articles = []
articles_recovered = []
categories_recovered = []
if os.path.isfile('urls.txt'):
articles_recovered = recover_text_file('urls.txt')
if os.path.isfile('categories.txt'):
categories_recovered = recover_text_file('categories.txt')
articles_to_scrape = []
categories = []
for i in range(6):
host = config()['news_sites'][i]['url']
logger.info(f'Begining scraper for {host}')
categories_urls = categories_urls_extraction(host, i)
articles_links = articles_urls_extraction(host, categories_urls, i)
for article in articles_links:
if article not in articles_recovered:
articles_to_scrape.append(article)
articles, category = articles_and_categories_extraction(host, article, i)
data['articles'].append(articles)
categories.append(category)
categories = list(set(categories))
for category in categories:
if category not in categories_recovered:
data['categories'].append({'categories':category})
writer = csv.writer(f)
writer.writerow(csv_headers)
for i in range(0,len(articles_title)):
row = [category_id,articles_title[i],articles_price[i].strip(),articles_link[i],articles_image[i]]
writer.writerow(row)
def _find_article_info_in_page(news_site_uid,category_id,page):
page = pages.HomePage(news_site_uid,category_id,page)
return page.articles
if __name__ == '__main__':
parser = argparse.ArgumentParser()
retail_site_choices = list(config()['retail_sites'].keys())
parser.add_argument('retail_site',
help='The retail site that you want to scrape',
type=str,
choices=retail_site_choices)
parser.add_argument('category_id',
help='The category of articles that you want to scrape',
type=str)
parser.add_argument('num_pages',
help='The number of pages in the selected category',
type=int)
args = parser.parse_args()
_prices_scraper(args.retail_site, args.category_id, args.num_pages)
return link
elif is_root_path.match(link):
if news_site_uid == 'elpais':
host = host.rstrip('/').rstrip(news_section_uid).rstrip('/')
return '{}{}'.format(host, link)
else:
return '{}{}'.format(host, link)
else:
if news_site_uid == 'elpais':
link = link.rstrip('/america/')
return '{host}/{uri}'.format(host=host, uri=link)
else:
return '{host}/{uri}'.format(host=host, uri=link)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
news_site_choices = list(config()['news_sites'].keys())
parser.add_argument('news_site',
help='the news site that you want to scrape',
type=str,
choices=news_site_choices)
args = parser.parse_args()
news_section_choices = list(
config()['news_sites'][args.news_site]['queries'].keys())
news_section_uid = input(
'choose a section: {}\n'.format(news_section_choices))
_news_scraper(args.news_site, news_section_uid)
def _news_scraper(news_site_uid):
host = config()['news_sites'][news_site_uid]['url']
logging.info(f'Beginning scrape for {host}')
return None
return article
def _build_link(host, link):
if is_well_formed_link.match(link):
return link
elif is_root_path.match(link):
return '{}{}'.format(host, link)
else:
return '{host}/{url}'.format(host=host, url=link)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
new_site_choices = list(
config()["news_sites"].keys()) #Obtiene lista de opciones para scraper
parser.add_argument('news_sites',
help='Los Sitios para scraper',
type=str,
choices=new_site_choices
) #Agrega argumentos para ejecucion del usuario
parser.add_argument(
'Time',
help='Fecha desde la que se desea scraper. Ej: 2020/05/01',
type=str) #Agrega argumentos para ejecucion del usuario
args = parser.parse_args()
_news_scraper(args.news_sites, args.Time)
def _news_scraper(news_site_uid):
host = config()['news_sites'][news_site_uid]['url']
logging.info('Begging scrarper for {}'.format(host))
import connections as conne import pandas as pd from common import config import os source = conne.database() parameters = config()['destination'] sqlmaxtl = 'SELECT DimD.FullDateAlternateKey,DimO.OrganizationName,Ds.ScenarioName,DiA.AccountDescription, Facf.Amount \ FROM FactFinance Facf INNER JOIN DimDate DimD ON Facf.DateKey = DimD.DateKey INNER JOIN DimOrganization DimO ON Facf.OrganizationKey = DimO.OrganizationKey INNER JOIN DimScenario Ds ON Facf.ScenarioKey = Ds.ScenarioKey INNER JOIN DimAccount DiA ON Facf.AccountKey = DiA.AccountKey WHERE YEAR(DimD.FullDateAlternateKey) = 2010' df = source.__execute__(sqlmaxtl) #print(type(df)) #print(df.columns) #print(df.head()) #print(df.shape) #print(df.ndim) #print(df.tail()) #print(df.dtypes) print(parameters['path']) df.to_csv(parameters['path'])
def save_articles(news_site_uid, articles):
now = datetime.datetime.now()
csv_headers = list(
filter(lambda properties: not property.startwith('_'),
dir(articles[0])))
out_file_name = '{news_site_uid}_{datetime}_articles.csv'.format(
news_site_uid=news_site_uid, datetime=now.strftime('%Y_%m_%d'))
with open(out_file_name, mode='w+') as f:
writer = csv.writer(f)
writer.writerow(csv_headers)
for article in articles:
row = [str(getattr(article, prop)) for prop in csv_headers]
writer.writerow(row)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
news_options = list(config()['sports_news_sites'].keys())
parser.add_argument('sports_news_site',
help='Select the sports news site for scraping',
type=str,
choices=news_options)
args = parser.parse_args()
sports_news_scraper(args.sports_news_site)
def __init__(self, news_site_uid, url):
self._config = config()["news_sites"][news_site_uid]
self.queries = self._config["queries"]
self._url = url
self._html = None
self._visit(url)
def __init__(self, config=common.config()):
self.config = config
self.db_engine = DB_Engine(self.config.db_string)
self.filename = self.config.pickle_records_file
import argparse
import logging
from common import config
import news_page_objects as news
logging.basicConfig(level=logging.INFO)
logger = logging.getLogger(__name__)
def _news_scrapper(_news_site_uid):
logging.info('Start::..')
host = config()['news_site'][_news_site_uid]['url']
logging.info('Beginning sraper for {}'.format(host))
homepage = news.HomePage(_news_site_uid, host)
for link in homepage.article_links:
print(link)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
logging.info('Middle::.. {}'.format(list(config()['news_site'])))
new_sites_choices = list(config()['news_site'].keys())
parser.add_argument('news_site',
help='The news sites that u want to scrape',
type=str,
choices=new_sites_choices)
args = parser.parse_args()
_news_scrapper(args.news_site)
elif opt == '--use_ram':
pref['use_ram'] = arg
elif opt == '--gpgpu':
pref['gpgpu'] = True
elif opt in ('-v', '--verbose'):
pref['verbose'] = True
elif opt == '--tmp':
pref['tmp'] = arg
else:
print('Option: "', opt, '" is not defined.')
if __name__ == "__main__":
pref = {}
init(pref)
cfg = config()
cfg.setDir({
'ws': pref['root'],
'app': os.path.join(pref['root'], 'MyApp'),
'ref': pref['refdir'],
'pref': pref['paramdir'],
'sample': os.path.join(pref['root'], 'Sample'),
'tmp': pref['tmp'],
'test': os.path.join(pref['root'], 'test'),
'out': pref['outdir']
})
cfg.makeDirs()
app = apprun(cfg)
pref['reference'] = os.path.join(pref['refdir'], pref['reference'])
if pref['verbose']:
print(pref)
def _news_scraper(news_site_uid):
host = config()['news_sites'][news_site_uid]['url']
logging.info('Beginning scraper for {}'.format(host))
homepage = news.HomePage(host)
links = homepage.article_links
homepage.save_articles(news_site_uid, links)
import pandas as pd
import pyodbc as conn_d
from common import config
parameters = config()['source']
class database:
""" Connection class to source and target databases """
def __init__(self):
self.driver_source = parameters['driver_source']
self.database_source = parameters['database_source']
self.host_source = parameters['host_source']
self.port_distination = parameters['port_distination']
self.user_source = parameters['user_source']
self.password_source = parameters['password_source']
self.trusted_connection = parameters['trusted_connection']
def __source__connect__(self):
self.conn_source = conn_d.connect(
Driver=self.driver_source,
Server=self.host_source,
Database=self.database_source,
user=self.user_source,
Trusted_Connection=self.trusted_connection,
password=self.password_source)
self.cur_des = self.conn_source.cursor()
def __disconnect_source__(self):
self.conn_source.close()
except (HTTPError, MaxRetryError) as e:
logger.warning('Error while fetching the article', exc_info=False)
if article and not article.body:
logger.warning('Invalid article. There is no body.')
return None
return article
def _build_link(host, link):
if is_well_formed_link.match(link):
return link
elif is_root_path.match(link):
return f'{host}{link}'
else:
return '{host}/{uri}'.format(host=host, uri=link)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
news_site_choices = list(config()['news_sites'].keys()) # Python3 nos regresa un iterador, por eso lo convertimos a lista
parser.add_argument(
'news_site',
help='The news site that you want to scrappe',
type=str,
choices=news_site_choices)
args = parser.parse_args()
_news_scrapper(args.news_site)
def __init__(self,url):
self._config = config()['amazon']
self._queries = self._config['products']
self._html = None
self._visit(url)
self.url = url
def _news_scraper(news_site_uid):
paths = config()['news_sites'][news_site_uid]
host = paths['url']
logging.info(f'Beginning scraper for {host}')
try:
response = requests.get(paths['url'])
if response.status_code == 200:
logger.info(f'Parsing url...')
final_articles = []
home = response.content.decode('utf-8')
parsed = html.fromstring(home)
links_to_news = parsed.xpath(
paths['queries']['XPATH_HOMEPAGE_LINKS_TO_ARTICLES'])
good_links = _fix_links(links_to_news, host)
for i, link in enumerate(good_links):
try:
article_response = requests.get(link, timeout=6)
article = article_response.content.decode('utf-8')
article_parsed = html.fromstring(article)
article_elements = {}
title = article_parsed.xpath(
paths['queries']['XPATH_TITLE'])
if len(title):
article_elements['title'] = title[0]
else:
article_elements['title'] = None
body = article_parsed.xpath(paths['queries']['XPATH_BODY'])
p_elements = []
for text in body:
if str(text)[0] in [',', '.', ' ']:
p_elements.append(str(text))
else:
p_elements.append(' ' + str(text))
body = ''.join(p_elements)
if len(body):
article_elements['body'] = body
else:
article_elements['body'] = None
date = article_parsed.xpath(paths['queries']['XPATH_DATE'])
if len(date):
article_elements['date'] = date[0]
else:
article_elements['date'] = None
author = article_parsed.xpath(
paths['queries']['XPATH_AUTHOR'])
if len(author):
article_elements['author'] = author[0]
else:
article_elements['author'] = None
article_elements['url'] = link
final_articles.append(article_elements)
logger.info(f'Article {i+1}/{len(good_links)} scraped!')
except Exception as e:
print(e)
return final_articles
else:
print(f'Error. Status code {response.status_code}')
except ValueError as ve:
print(ve)
def __init__(self, config=common.config()):
self.config = config
self.db_engine = DB_Engine(self.config.db_string)
self.filename = self.config.pickle_records_file
def __init__(self, config=common.config()):
self.__config = config
def _news_scraper(news_site):
host = config()['news_sites'][news_site]['url']
logging.info('Beginning scraper for {}'.format(host))
logging.info('Finding links in homepage...')
try:
article = news.ArticlePage(news_sites_uid, _build_link(host, link))
except (HTTPError, MaxRetryError) as e:
logger.warning('Error while fetching the article', exc_info = false)
if article and not article.body:
logger.warning('Articulo Invalido')
return None
return article
def _build_link(host, link):
if is_well_formed_url.match(link):
return link
elif is_root_path.match(link)
return '{}{}'.format(host, link)
else:
return'{host}/{url}'.format(host=host, url=link)
if __name__ == '__main__':
parser = argparse.ArgumentParser()
news_sites_choices = list(config()[new_site].keys())
parser.add_argument('news_sites', help = 'el nuevo sitio para hacer scrape',
type = str,
choices = news_sites_choices)
args = parser.parse_args()
_news_scrapper(args.news_sites)
articles.append(article)
_save_articles(news_site_id, articles)
def _save_articles(news_site_id, articles):
#now = datetime.now().strftime("%Y_%m_%d")
#output_filename = f"{news_site_id}_{now}_articles.csv"
output_filename = f"{news_site_id}.csv"
csv_headers = list(filter(lambda property: not property.startswith("_"), dir(articles[0])))
with open(output_filename, mode="w+") as f:
writer = csv.writer(f)
writer.writerow(csv_headers)
for article in articles:
row = [str(getattr(article, prop)) for prop in csv_headers]
writer.writerow(row)
if __name__ == "__main__":
parser = argparse.ArgumentParser()
news_sites_choices = list(config()["news_sites"].keys())
parser.add_argument('news_site',
help="The news site to be scraped",
type=str,
choices=news_sites_choices)
args = parser.parse_args()
_news_scraper(args.news_site)
article = news.ArticlePage(news_site_uid, _build_link(host, link))
except (HTTPError, MaxRetryError) as e:
logger.warning('Error while fetching the article', exc_info=False)
if article and not article.body:
logger.warning('Invalid article. There is no body')
return None
return article
def _build_link(host, link):
if is_well_formed_link.match(link):
return link
elif is_root_path.match(link):
return '{}{}'.format(host, link)
else:
return '{host}/{uri}'.format(host=host, uri=link)
if __name__ == "__main__":
news_sites_choices = list(config()['news_sites'].keys())
parser = argparse.ArgumentParser()
parser.add_argument('news_site',
help='The news site you want to scrape',
type=str,
choices=news_sites_choices)
args = parser.parse_args()
_new_scraper(args.news_site)
def v_main():
knn = KnnClassifer(common.config())
knn.knn()
