def connect(self):
"""
Connects to the Sybl service
Parametes:
None
Returns:
None
Raises:
AttributeError: raised when accesstoken or model id has not been loaded
PermissionError: raised when the accesstoken cannot be authorized
"""
# Check the user has specified a callback here
if self.callback is None:
raise AttributeError("Callback has not been registered")
if not self._access_token or not self._model_id:
log.error("Model has not been loaded")
raise AttributeError(
"Model access token and ID have not been loaded")
self._connect_to_sock()
log.info("Connected")
if not self._is_authenticated():
raise PermissionError(
"Model access token has not been authenticated")
self._state = State.HEARTBEAT
self._begin_state_machine()
def load_access_token(email, model_name) -> Tuple[str, str]:
"""
Loads the access token from XDG_DATA_HOME using email and model name
Parameters:
email (str): The email the model is registered with
model_name (str): The name of the model to be loaded
Returns:
Tuple[str, str]: Tuple of access token and model id
Raise:
ValueError: raised if the model name and email pair is not found
in XDG_DATA_HOME
FileNotFoundError: raised if XDG_DATA_HOME/sybl.json is not found
meaning no access token has been stored
"""
model_key: str = f"{email}.{model_name}"
path = xdg_data_home() / "sybl.json"
with path.open("r") as f: # pylint: disable=invalid-name
file_contents: str = f.read()
models_dict: Dict = json.loads(file_contents)
try:
model_data = models_dict[model_key]
return model_data["access_token"], model_data["model_id"]
except KeyError as e: # pylint: disable=invalid-name
log.error("Model not registered")
raise ValueError(
f"Model {model_name} not registered to {email}") from e
def ask_for_components(services: dict, topics: dict, actions: dict):
services_choices = component_options(services)
topics_choices = component_options(topics)
actions_choices = component_options(actions)
all_choices = services_choices + topics_choices + actions_choices
there_are_valid_choices = any("disabled" not in choice for choice in all_choices)
if not there_are_valid_choices:
logging.error(
"There are no available components to fuzz!\n"
"Check for TODOs in the fuzz.yaml file."
)
exit(-1)
choices = [
Separator("Topics"),
*topics_choices,
Separator("Services"),
*services_choices,
Separator("Actions"),
*actions_choices,
]
questions = [
{
"type": "checkbox",
"message": "What do you want to fuzz?",
"name": "to_fuzz_components",
"choices": choices,
}
]
return prompt(questions)["to_fuzz_components"]
def parse(type_name: str, prepath: str = "") -> ROSType:
# Base case
if type_name in TypeParser.PRIMITIVES:
return ROSType(type_name=type_name, is_primitive=True)
# Recursive case
full_topic_path = prepath + type_name
try:
ros2_process_result = subprocess.check_output(
["ros2", "interface", "show", full_topic_path],
stderr=subprocess.STDOUT,
)
except:
logging.error(
f"Couldn't call `ros2 interface show {full_topic_path}`\n"
"Have you sourced install/setup.bash?"
)
exit(-1)
ros2_process_result = ros2_process_result.decode("utf-8")
# If it is a *.srv file content (request --- response),
# isolate and process only the request part
if "---" in ros2_process_result:
ros2_process_result = ros2_process_result.split("---")[0]
ros2_process_result = ros2_process_result.splitlines()
fields = [TypeParser.line2field(line) for line in ros2_process_result]
fields = [field for field in fields if field is not None]
return ROSType(type_name=type_name, fields=fields)
def _process_job_config(self) -> None:
if self._message_stack:
job_config = self._message_stack.pop()
else:
log.error("Empty Message Stack!\n RETURNING TO HEARTBEAT")
self._state = State.HEARTBEAT
return
assert self.config is not None
if "JobConfig" not in job_config:
log.warning("Invalid Job Config Message")
self._state = State.HEARTBEAT
return
accept_job: bool = self.config.compare(job_config["JobConfig"])
if not accept_job:
self._send_message({"ConfigResponse": {"accept": False}})
log.info("REJECTING JOB")
else:
self._send_message({"ConfigResponse": {"accept": True}})
self.recv_job_config = job_config["JobConfig"]
log.info("ACCEPTING JOB")
self._state = State.HEARTBEAT
def _read_message(self) -> Dict:
size_bytes = self._sock.recv(4)
if size_bytes == b"":
log.error("Empty Message: Closing")
self._sock.close()
sys.exit(1)
size = struct.unpack(">I", size_bytes)[0]
log.debug("Message size: %d", size)
if size > 4096:
remaining_size = size
buf: List[int] = []
while remaining_size > 0:
chunk = self._sock.recv(4096)
buf.extend(chunk)
remaining_size -= 4096
return json.loads(bytes(buf))
message: Dict = json.loads(self._sock.recv(size))
# Error handle
if "Server" in message.keys():
# There has been an error in communication
if "text" in message["Server"].keys():
payload: Dict = json.loads(message["Server"]["text"])
code = message["Server"]["code"]
self._handle_server_error(code, payload)
log.info(message)
return message
async def play_game(websocket, gametype: str):
# Wait until the game begins
global bot
while True:
message = await parse_game_message(websocket)
if message is None:
log.error("Failed to get a valid message from the socket.")
continue
if isinstance(message, BidRequest):
args = pickle.loads(bytes.fromhex(message.arguments))
value = None
if gametype == "value":
value = str(bot.get_bid_game_type_value(**args))
else:
value = str(bot.get_bid_game_type_collection(**args))
await websocket.send(value)
elif isinstance(message, (AuctionEnd, MultiAuctionEnd)):
log.info(f"Auction winners: {message.winners}")
return
elif isinstance(message, ResetBot):
log.info("Resetting bot")
bot = Bot()
def text_to_mp3(text, outfile, lang):
url = "http://translate.google.com/translate_tts?ie=UTF-8&q=%s&tl=%s&prev=input" % (urllib.quote(saythis), lang)
try:
urllib.urlretrieve(url, outfile)
except IOError:
log.error('Got IOError -- are you online?')
raise
def table_to_list(table: Tag, section: str,
id_base: int) -> Tuple[List[Mod], int]:
"""Convert an HTML table into a list of Mods."""
mods: List[Mod] = []
modid = id_base
row: Tag
for row in table.find_all('tr'):
try:
tag_column_1: Tag = row.find('td', 'col1')
tag_column_2: Tag = row.find('td', 'col2')
tag_column_3: Tag = row.find('td', 'col3')
if tag_column_1 and tag_column_2 and tag_column_3:
webpages = get_webpages_from_tag(tag_column_2)
requirements = get_requirements_from_tag(tag_column_3)
mod = Mod(modid, get_name_from_tag(tag_column_2),
get_description_from_tag(tag_column_3),
get_notes_from_tag(tag_column_3), section,
get_media_from_tag(tag_column_1),
Oldrim(True, webpages, requirements),
Sse(False, Webpages(None, None, None, []), []), [],
get_deprecated_status_from_tag(tag_column_3), False)
mods.append(mod)
modid += 1
except BadEntryException as exc:
log.error(str(exc))
return (mods, modid)
def display_access(self, message: Dict):
"""
Parses an access token from a message and displays them to the user.
Args:
message: The message to parse
Raises:
KeyError: If the message doesn't contain the "token" and "id" keys
"""
try:
self.access_token = message["token"]
self.model_id = message["id"]
log.info("Successfully authenticated with the Sybl system")
# log.info(f"\tACCESS TOKEN: {self.access_token}")
# log.info(f"\tMODEL ID: {self.model_id}")
log.info(
"Please go to https://sybl.tech/models to unlock your new model"
)
except KeyError:
log.error(f"Expected 'token' and 'id' keys but got data={message}")
finally:
self.stream.close()
def authenticate_challenge(self, message: Dict[Any, Any]):
"""
Authenticates a challenge message and responds to the requestor.
Args:
challenge: The challenge message itself
Raises:
KeyError: If the user does not have their private key in their
environment
"""
challenge = message["challenge"]
log.info("Authenticating a challenge from the server")
try:
signed_challenge = sign_challenge(base64.b64decode(challenge),
self.private_key)
message = {
"ChallengeResponse": {
"email": self.email,
"model_name": self.model_name,
"response":
base64.b64encode(signed_challenge).decode("utf-8"),
}
}
self._send_message(message)
except KeyError:
log.error("Failed to find the private key in the environment")
def generate_cpp_file(fuzz_target: FuzzTarget, source_file: str,
template_name: str):
__location__ = os.path.realpath(
os.path.join(os.getcwd(), os.path.dirname(__file__)))
plain_source_file_name = Path(source_file).name
without_extension = os.path.splitext(plain_source_file_name)[0]
# Read template
env = Environment(loader=FileSystemLoader(__location__))
template = env.get_template(template_name)
logging.debug("Template read")
# Populate template
template_arguments = fuzz_target.get_mapping()
template_arguments["FILE_NAME"] = plain_source_file_name
fuzzing_path = os.path.join(os.path.dirname(__file__), "fuzzing_api.hpp")
template_arguments["FUZZING_API"] = open(fuzzing_path).read()
template = template.render(template_arguments)
logging.debug("Template populated")
# Write the populated file
full_path = os.path.join(os.path.dirname(source_file),
without_extension + "_generated.cpp")
try:
with open(full_path, "w") as fd:
fd.write(template)
logging.debug(
f"Template written with {fuzz_target.client_name} client")
except Exception:
logging.error("Couldn't write generated file", exc_info=True)
return full_path
def upload_archive(self, content, description):
try:
archive = self.vault.upload_archive(archiveDescription=description,
body=content)
return archive
except ClientError as e:
log.error(e)
def validate(fontpkg):
"""Checks if the font package is valid."""
# check if JSON passes
try:
json.loads(os.path.join(fontpkg, "fontpackage.json"))
except ValueError:
log.error("Invalid JSON in fontpackage.json file.")
raise
def load_priv_key():
try:
priv_key = os.environ["PRIVATE_KEY"]
except KeyError:
log.error("PRIVATE_KEY not found in environment. Exiting...")
sys.exit(1)
return priv_key
def get_db(view):
db = getattr(g, '_database', None)
if db is None:
log.error("creating for the first time")
couch = couchdb.Server(app.config['COUCHDB_SERVER'])
couch.resource.credentials = (app.config['COUCHDB_USER'], app.config['COUCHDB_PASSWORD'])
db = g._database = couch[app.config['COUCHDB_DATABASE']]
else:
log.info('nothing to do here, flies away')
return db
def parse_event_info(event, info):
# event is a dict, info is a BeautifulSoup object
if event['type'] == u"Baixa comissão para discussão":
com_name = info.find('span', id=RE_COMNAME)
if com_name:
event['comission_name'] = com_name.text.strip()
elif event['type'] == u"Publicação":
url_nodes = info.findAll('a')
urls = [{'url': node['href'], 'title': node.text.strip('[]')} for node in url_nodes]
event['references'] = urls
elif event['type'] in (u"Votação na generalidade", u"Votação Deliberação", u"Votação final global"):
vote_info = info.find('span', id=RE_VOTEINFO)
# funky parse loop for understanding how each party voted
results = {'for': [], 'against': [], 'abstain': []}
current_vote = None
for c in vote_info.contents:
if type(c) == bs4.element.Tag:
if c.name == "br":
continue
elif c.name == "i":
results[current_vote].append(c.text)
else:
log.error("Unrecognized vote tag: %s" % c)
elif type(c) == bs4.element.NavigableString:
c = c.strip()
if c == ",":
continue
if c.startswith(u'Contra:'):
current_vote = "against"
if not c == u'Contra:':
# cases with one voter, in one line
# ex. "Abstenção: Isabel Oneto (PS)"
c = c.replace(u'Contra: ', '')
results[current_vote].append(c)
elif c.startswith(u"A Favor:"):
current_vote = "for"
if not c == u'A Favor:':
c = c.replace(u'A Favor: ', '')
results[current_vote].append(c)
elif c.startswith(u"Abstenção:"):
current_vote = "abstain"
if not c == u'Abstenção:':
c = c.replace(u'Abstenção: ', '')
results[current_vote].append(c)
else:
log.error("Unrecognized vote string: %s" % c)
event['vote_info'] = results
pass
else:
if info.text.strip():
event['raw_info'] = info.text.strip()
return event
def test_output():
# All of these just need to output without errors.
from zenlog import log
log.debug("A quirky message only developers care about")
log.info("Curious users might want to know this")
log.warn("Something is wrong and any user should be informed")
log.warning("Something is wrong and any user should be informed")
log.error("Serious stuff, this is red for a reason")
log.critical("OH NO everything is on fire")
log.c("OH NO everything is on fire")
log.crit("OH NO everything is on fire")
def check_warning_lights_on(self):
warning_light = self.topics.get("*/track/0/warning_light/0")
if warning_light.payload == "1":
log.info("Track sequence OK. Warning lights are on.")
return True
else:
log.error(
"Track Sequence ERROR! Payload not matching expected state: Warning lights are not on!"
)
return False
def _connect_to_sock(self):
"""
Connects to the DCL for communications.
"""
try:
self._sock.connect(self._address)
except ConnectionRefusedError:
log.error(f"Could not connect to address: {self._address[0]}")
sys.exit(1)
log.info(f"Successfully connected to {self._address}")
def test_output():
# All of these just need to output without errors.
from zenlog import log
log.debug("A quirky message only developers care about")
log.info("Curious users might want to know this")
log.warn("Something is wrong and any user should be informed")
log.warning("Something is wrong and any user should be informed")
log.error("Serious stuff, this is red for a reason")
log.critical("OH NO everything is on fire")
log.c("OH NO everything is on fire")
log.crit("OH NO everything is on fire")
def parse_type(type_name: str) -> ROSType:
if "/" not in type_name:
logging.error(
f"The type name `{type_name}` does not contain any slash (/).\n"
f'Please write the whole path (i.e. "tutorial_interfaces/srv/AddThreeInts")'
)
exit(-1)
prepath, type_name = type_name.rsplit("/", 1)
prepath += "/"
return TypeParser.parse(type_name=type_name, prepath=prepath)
def check_train_passed(self):
train_light_east = self.topics.get("*/track/0/train_light/0")
train_light_west = self.topics.get("*/track/0/train_light/1")
if train_light_east.payload == "0" and train_light_west.payload == "0":
log.info("Track sequence OK. Train lights are red.")
return True
else:
log.error(
"Track Sequence ERROR! Payload not matching expected state: Train lights are still green!"
)
return False
async def parse(url, session, collected):
try:
data = await fetch_html(url, session)
collected[url] = data
except (aiohttp.ClientError,
aiohttp.http_exceptions.HttpProcessingError) as exc:
status = getattr(exc, 'status', None)
message = getattr(exc, 'message', None)
log.error(f'aiothtp exception for {url} [{status}]: {message}')
except Exception as exc: # pylint: disable=broad-except
exception_attributes = getattr(exc, '__dict__', {})
log.error(f'Non-aiohttp exception occured: {exception_attributes}')
def _update_all(self):
default_args = self.get_default_args()
for manga in self.db.get_all(): # type: Manga
self.show_log() and log.info('Update %s', manga.url)
_args = default_args.copy()
data = json.loads(manga.data)
data_args = data.get('args', {})
del data_args['rewrite_exists_archives']
del data_args['user_agent']
del data_args['url']
if not fs.is_dir(
fs.path_join(data_args['destination'], data_args['name'])):
self.show_log() and log.warn('Destination not exists. Skip')
continue
_args.update({ # re-init args
'url': manga.url,
**data_args,
})
provider = self._get_provider(_args) # type: Provider
if provider:
provider.before_provider(_args)
provider.http.cookies = data.get('cookies')
provider.http.ua = data.get('browser')
provider.run(_args)
provider.after_provider()
provider.update_db()
self.global_info.add_info(info)
else:
self.show_log() and log.error('Provider not exists')
def get_file_stats(self):
file_stats = {}
try:
stats = self.file_path.stat()
file_stats['st_mode'] = stats.st_mode
file_stats['st_ino'] = stats.st_ino
file_stats['st_dev'] = stats.st_dev
file_stats['st_nlink'] = stats.st_nlink
file_stats['st_uid'] = stats.st_uid
file_stats['st_gid'] = stats.st_gid
file_stats['st_size'] = stats.st_size
file_stats['st_atime'] = stats.st_atime
file_stats['st_mtime'] = stats.st_mtime
except Exception as e:
log.error(e)
return file_stats
def play_by_station_uuid(self, _uuid):
print(_uuid)
# Pyradios by default don't let you search by uuid
# a trick is to call click_counter(uuid) directly to get the statioon info
is_ok = "false"
try:
self.target_station = self.API.click_counter(_uuid)
log.debug(self.target_station)
is_ok = self.target_station["ok"]
except Exception as e:
log.error("Could not find a station by the UUID")
sys.exit(0)
self.API.search(name=self.target_station["name"], name_exact=True)
# againg register a valid click
if is_ok == "false":
res = self.API.click_counter(self.target_station["stationuuid"])
log.debug(res)
def check_train_passing(self):
self.check_warning_lights_on()
train_light_east = self.topics.get("*/track/0/train_light/0")
train_light_west = self.topics.get("*/track/0/train_light/1")
if train_light_east.payload == "0" and train_light_west.payload == "0":
log.error(
"Track sequence ERROR! Payload not matching expected state: Both train lights are red!"
)
return
if train_light_east.payload == "1" and train_light_west.payload == "1":
log.error(
"Track sequence ERROR! Payload not matching expected state: Both train lights are green!"
)
return
log.info("Track sequence OK. Train light is green")
def verify(self):
"""
Creates a new model for the user and authenticates it with the
challenge response method.
Raises:
IndexError: If an invalid message is encountered
"""
# Connect to the socket
self._connect()
message = {
"NewModel": {
"email": self.email,
"password": self.password,
"model_name": self.model_name,
}
}
self._send_message(message)
while True:
# Read some data
data = self._read_message()
log.debug(f"Received data={data}")
try:
variant, data = parse_message(data)
if variant == "Challenge":
self.authenticate_challenge(data)
elif variant == "AccessToken":
self.display_access(data)
self.save_access_tokens()
break
else:
log.warn(
f"Encountered an unexpected message variant={variant}")
except IndexError:
log.error(f"Failed to parse a message from data={data}")
def _is_authenticated(self) -> bool:
response = {
"AccessToken": {
"id": self._model_id,
"token": self._access_token,
}
}
self._send_message(response)
message = self._read_message()
try:
if message["message"] == "Authentication successful":
return True
except KeyError:
pass
log.error("Authentication not successful")
return False
def main():
if not os.path.exists(dest):
os.mkdir(dest)
log.info("Directory 'imgs/' created.")
mp_json = json.loads(open(mp_file, 'r').read())
for mp_id in mp_json:
url = pic_url_formatter % mp_id
filename = '%s.jpg' % os.path.join(dest, mp_id)
if os.path.exists(filename):
log.debug("File for id %s already exists, skipping." % mp_id)
continue
log.info('Retrieving picture with id: %s' % mp_id)
try:
urlretrieve(url, filename)
except IOError:
log.error('Socket error! :(')
log.info('Done. Now do find ./imgs/ -size -722c -exec rm {} \;')
log.info('to clean up things.')
def main():
if not os.path.exists(dest):
os.mkdir(dest)
log.info("Directory 'imgs/' created.")
mp_json = json.loads(open(mp_file, 'r').read())
for mp_id in mp_json:
url = pic_url_formatter % mp_id
filename = '%s.jpg' % os.path.join(dest, mp_id)
if os.path.exists(filename):
log.debug("File for id %s already exists, skipping." % mp_id)
continue
log.info('Retrieving picture with id: %s' % mp_id)
try:
urlretrieve(url, filename)
except IOError:
log.error('Socket error! :(')
log.info('Done. Now do find ./imgs/ -size -722c -exec rm {} \;')
log.info('to clean up things.')
def station_validator(self):
if len(self.response) == 0:
log.error("No stations found by the name")
sys.exit(0)
if len(self.response) > 1:
log.info("Multiple stations found by the name")
stations_name = ""
for station in self.response:
# stations_name = stations_name + "," + station["name"]
log.info("name: {} | id: {} | country: {}".format(
station["name"], station["stationuuid"],
station["country"]))
log.info(stations_name)
sys.exit(0)
if len(self.response) == 1:
log.info("Station found: {}".format(self.response[0]["name"]))
log.debug(self.response[0])
self.target_station = self.response[0]
self.API.click_counter(self.target_station["stationuuid"])
def parse_file(f):
bank = {}
html = open(f, 'r').read()
soup = BeautifulSoup(html)
items_headings = soup.findAll("th")
items = soup.findAll("td")
item_pairs = zip(items_headings, items)
# encontrar duplicados
country_needs_fixing = False
urls_need_fixing = False
dups = set([x.text for x in items_headings if items_headings.count(x) > 1])
if dups:
if "Country" in dups:
country_needs_fixing = True
elif "Related decisions" in dups and "Published accounts" in dups:
pass
else:
log.error("Campo duplicado:")
for d in dups:
print ' ' + d
for name, value in item_pairs:
name = name.text.lower().replace(' ', '_').replace('-', '_')
if country_needs_fixing and name == "country":
country_needs_fixing = False
name = "city"
if value.find("a"):
bank[name] = value.find('a')['href']
else:
bank[name] = value.text
bank_filename = f.split('/')[-1]
bank_id = bank_filename.split('.')[0]
bank_url = 'https://www.bportugal.pt/en-US/Supervisao/Pages/CreditInstitution.aspx?IcID=' + bank_id
bank['id'] = bank_id
bank['url'] = bank_url
return bank
rows.append(headerline)
for item in jsondata:
values = []
for header in headers:
if not item.has_key(header):
if header == "institution_name":
# abre o banks.json
name = ""
for f_item in fatherjsondata:
if f_item['url'] == item['url'].replace("https", "http"):
name = f_item['name'].replace('"', "'")
values.append('"%s"' % name)
break
if not name:
log.error("Não encontrei o nome ;(")
continue
elif header == "institution_type":
# abre o banks.json
bank_type = ""
for f_item in fatherjsondata:
if f_item['url'] == item['url'].replace("https", "http"):
bank_type = f_item['type'].replace('"', "'")
values.append('"%s"' % bank_type)
break
if not bank_type:
log.error("Não encontrei o tipo ;(")
continue
else:
value = ""
values.append(value)
def parse_event_info(event, info):
# event is a dict, info is a BeautifulSoup object
if event['type'] == u"Baixa comissão para discussão":
com_name = info.find('span', id=RE_COMNAME)
if com_name:
event['comission_name'] = com_name.text.strip()
elif event['type'] == u"Publicação":
url_nodes = info.findAll('a')
urls = [{'url': node['href'], 'title': node.text.strip('[]')} for node in url_nodes]
event['references'] = urls
elif event['type'] in (u"Votação na generalidade", u"Votação Deliberação", u"Votação final global"):
vote_info = info.find('span', id=RE_VOTEINFO)
# funky parse loop for understanding how each party voted
# I really have to refactor this, please pester me if you need it -- rlafuente
results = {'for': [], 'against': [], 'abstain': []}
current_vote = None
for c in vote_info.contents:
if type(c) == bs4.element.Tag:
if c.name == "br":
continue
elif c.name == "i":
results[current_vote].append(c.text)
else:
log.error("Unrecognized vote tag: %s" % c)
elif type(c) == bs4.element.NavigableString:
c = c.strip()
if c == ",":
continue
if c.startswith(u'Contra:'):
current_vote = "against"
if not c == u'Contra:':
# cases with voters in one line (individual MPs)
# ex. "Abstenção: Isabel Oneto (PS)"
c = c.replace(u'Contra: ', '').split(', ')
for mp in c:
if mp:
results[current_vote].append(mp.strip(','))
elif c.startswith(u"A Favor:"):
current_vote = "for"
if not c == u'A Favor:':
c = c.replace(u'A Favor: ', '').split(', ')
for mp in c:
if mp:
results[current_vote].append(mp.strip(','))
elif c.startswith(u"Abstenção:"):
current_vote = "abstain"
if not c == u'Abstenção:':
c = c.replace(u'Abstenção: ', '').split(', ')
for mp in c:
if mp:
results[current_vote].append(mp.strip(','))
else:
log.warn("Orphan vote string: %s -- saving as voter" % c)
c = c.split(', ')
for mp in c:
if mp:
results[current_vote].append(mp)
event['vote_info'] = results
pass
else:
if info.text.strip():
event['raw_info'] = info.text.strip()
return event
def format_date(value, format='medium'):
if type(value) not in (datetime.date, datetime.datetime):
log.error(type(value))
value = dateparser.parse(value)
return babel_format_date(value, format, locale="pt_PT")
bank_list = []
pagecount = 1
while True:
log.debug("Novo loop, pagecount eh " + str(pagecount))
wait = UI.WebDriverWait(driver, 10)
links = driver.find_elements_by_css_selector(".AreaResultados td a")
log.debug("Encontrei %d links..." % len(links))
if len(links) == 0:
from time import sleep
sleep(3)
links = driver.find_elements_by_css_selector(".AreaResultados td a")
if len(links) == 0:
log.error("Não há links, snif")
else:
log.debug("Iupi, %d links!" % len(links))
rows = driver.find_elements_by_css_selector(".AreaResultados tbody tr")
# skip first row
first = True
for row in rows:
if first:
first = False
continue
bank_name = row.find_element_by_css_selector("a")
bank_type = row.find_element_by_css_selector("span")
bank = { "name": bank_name.text,
"url": bank_name.get_attribute("href"),
"type": bank_type.text
def generate(offline, fetch_only):
'''Main function that takes care of the whole process.'''
# set up the output directory
if not os.path.exists(output_dir):
os.mkdir(output_dir)
# set up the dir for storing repositories
if not os.path.exists(repo_dir):
log.info("Directory %s doesn't exist, creating it." % repo_dir)
os.mkdir(repo_dir)
# create dir for dataset pages
if not os.path.exists(os.path.join(output_dir, datasets_dir)):
os.mkdir(os.path.join(output_dir, datasets_dir))
# create download dir for zip and csv/json/* dataset files
if not os.path.exists(os.path.join(output_dir, files_dir)):
os.mkdir(os.path.join(output_dir, files_dir))
# create static dirs
# TODO: only update changed files -- right now we regenerate the whole static dir
css_dir = os.path.join(output_dir, "css")
js_dir = os.path.join(output_dir, "js")
img_dir = os.path.join(output_dir, "img")
fonts_dir = os.path.join(output_dir, "fonts")
if os.path.exists(css_dir):
shutil.rmtree(css_dir)
shutil.copytree("static/css", css_dir)
if os.path.exists(js_dir):
shutil.rmtree(js_dir)
shutil.copytree("static/js", js_dir)
if os.path.exists(img_dir):
shutil.rmtree(img_dir)
shutil.copytree("static/img", img_dir)
if os.path.exists(fonts_dir):
shutil.rmtree(fonts_dir)
shutil.copytree("static/fonts", fonts_dir)
# read the config file to get the datasets we want to publish
parser = SafeConfigParser()
parser.read(config_file)
packages = []
if not parser.items('repositories'):
log.critical('No repository data in settings.conf (does it even exist?). Cannot proceed :(')
sys.exit()
# go through each specified dataset
for r in parser.items('repositories'):
name, url = r
dir_name = os.path.join(repo_dir, name)
# do we have a local copy?
if os.path.isdir(dir_name):
if not offline:
log.info("Checking for changes in repo '%s'..." % name)
repo = git.Repo(dir_name)
origin = repo.remotes.origin
try:
origin.fetch()
except AssertionError:
# usually this fails on the first run, try again
origin.fetch()
except git.exc.GitCommandError:
log.critical("Fetch error connecting to repository, this dataset will be ignored and not listed in the index!")
continue
# connection errors can also happen if fetch succeeds but pull fails
try:
result = origin.pull()[0]
except git.exc.GitCommandError:
log.critical("Pull error connecting to repository, this dataset will be ignored and not listed in the index!")
continue
# we get specific flags for the results Git gave us
# and we set the "updated" var in order to signal whether to
# copy over the new files to the download dir or not
if result.flags & result.HEAD_UPTODATE:
log.info("No new changes in repo '%s'." % name)
updated = False
elif result.flags & result.ERROR:
log.error("Error pulling from repo '%s'!" % name)
updated = False
else:
# TODO: figure out other git-python flags and return more
# informative log output
log.info("Repo changed, updating. (returned flags: %d)" % result.flags)
updated = True
else:
log.info("Offline mode, using cached version of package %s..." % name)
# we set updated to True in order to re-generate everything
# FIXME: See checksum of CSV files to make sure they're new before
# marking updated as true
updated = True
repo = git.Repo(dir_name)
if fetch_only:
# if the --fetch-only flag was set, skip to the next dataset
continue
else:
if offline:
log.warn("Package %s specified in settings but no local cache, skipping..." % name)
continue
else:
log.info("We don't have repo '%s', cloning..." % name)
repo = git.Repo.clone_from(url, dir_name)
updated = True
# get datapackage metadata
pkg_info = process_datapackage(name)
# set last updated time based on last commit, comes in Unix timestamp format so we convert
import datetime
d = repo.head.commit.committed_date
last_updated = datetime.datetime.fromtimestamp(int("1284101485")).strftime('%Y-%m-%d %H:%M:%S')
log.debug(last_updated)
pkg_info['last_updated'] = last_updated
# add it to the packages list for index page generation after the loop ends
packages.append(pkg_info)
# re-generate the dataset HTML pages
create_dataset_page(pkg_info)
# if repo was updated, copy over CSV/JSON/* and ZIP files to the download dir
# (we always generate them if offline)
if updated or offline:
create_dataset_page(pkg_info)
datafiles = pkg_info['datafiles']
zipf = zipfile.ZipFile(os.path.join(output_dir, files_dir, name + '.zip'), 'w')
for d in datafiles:
# copy CSV file
target = os.path.join(output_dir, files_dir, os.path.basename(d['path']))
shutil.copyfile(os.path.join(dir_name, d['path']), target)
# generate JSON version
csv2json(target, target.replace(".csv", ".json"))
# make zip file
zipf.write(os.path.join(dir_name, d['path']), d['basename'], compress_type=zipfile.ZIP_DEFLATED)
try:
zipf.write(pkg_info['readme_path'], 'README.md')
except OSError:
pass
zipf.close()
# generate the HTML index with the list of available packages
create_index_page(packages)
# generate the static JSON API of the data packages
create_api(packages)
def generate(offline=False,
fetch_only=False,
output_dir=OUTPUT_DIR,
theme_dir=os.path.join(THEMES_DIR, 'centraldedados'),
repo_dir=REPO_DIR,
config_file=CONFIG_FILE):
'''Main function that takes care of the whole process.'''
global env, packages
# Read the config file
parser = SafeConfigParser()
parser.read(config_file)
# Load the theme and set up Jinja
theme_name = parser.get('ui', 'theme')
theme_dir = os.path.join(THEMES_DIR, theme_name)
template_dir = os.path.join(theme_dir, "templates")
env = jinja2.Environment(loader=jinja2.FileSystemLoader([template_dir]))
# Set up the output directory
if not os.path.exists(output_dir):
os.mkdir(output_dir)
# Set up the dir for storing repositories
if not os.path.exists(repo_dir):
log.debug("Directory %s doesn't exist, creating it." % repo_dir)
os.mkdir(repo_dir)
# Copy htaccess file
shutil.copyfile(os.path.join(theme_dir, 'static/htaccess'), os.path.join(output_dir, ".htaccess"))
# Static CSS files
css_dir = os.path.join(output_dir, "css")
if os.path.exists(css_dir):
shutil.rmtree(css_dir)
shutil.copytree(os.path.join(theme_dir, "static/css"), css_dir)
# Static JavaScript files
js_dir = os.path.join(output_dir, "js")
if os.path.exists(js_dir):
shutil.rmtree(js_dir)
shutil.copytree(os.path.join(theme_dir, "static/js"), js_dir)
# Theme images
img_dir = os.path.join(output_dir, "img")
if os.path.exists(img_dir):
shutil.rmtree(img_dir)
shutil.copytree(os.path.join(theme_dir, "static/img"), img_dir)
# Fonts
fonts_dir = os.path.join(output_dir, "fonts")
if os.path.exists(fonts_dir):
shutil.rmtree(fonts_dir)
shutil.copytree(os.path.join(theme_dir, "static/fonts"), fonts_dir)
if not parser.items('repositories'):
log.critical('No repository data in settings.conf (does it even exist?). Cannot proceed :(')
sys.exit()
# go through each specified dataset
for r in parser.items('repositories'):
name, url = r
dir_name = os.path.join(repo_dir, name)
repo = None
# do we have a local copy?
if os.path.isdir(dir_name):
if not os.path.isdir(os.path.join(dir_name, '.git')):
if url.endswith(".json"):
log.info("%s: Data package, refreshing" % name)
updated = fetch_data_package(url, dir_name)
else:
log.info('%s: Unsupported repo, skipping update' % name)
continue
elif not offline:
repo = git.Repo(dir_name)
origin = repo.remotes.origin
try:
origin.fetch()
except AssertionError:
# usually this fails on the first run, try again
origin.fetch()
except git.exc.GitCommandError:
log.critical("%s: Fetch error, this dataset will be left out." % name)
continue
# see if we have updates
if not local_and_remote_are_at_same_commit(repo, origin):
log.debug("%s: Repo has new commits, updating local copy." % name)
updated = True
# connection errors can also happen if fetch succeeds but pull fails
try:
result = origin.pull()[0]
except git.exc.GitCommandError:
log.critical("%s: Pull error, this dataset will be left out." % name)
continue
if result.flags & result.ERROR:
log.error("%s: Pull error, but going ahead." % name)
updated = False
else:
log.info("%s: No changes." % name)
updated = False
else:
log.debug("%s: Offline mode, using cached version." % name)
# we set updated to True in order to re-generate everything
updated = True
repo = git.Repo(dir_name)
if fetch_only:
# if the --fetch-only flag was set, skip to the next dataset
continue
else:
if offline:
log.warn("%s: No local cache, skipping." % name)
continue
else:
if url.endswith(".git"):
# Handle GIT Repository URL
log.info("%s: New repo, cloning." % name)
try:
repo = git.Repo.clone_from(url, dir_name)
# For faster checkouts, one file at a time:
# repo = git.Repo.clone_from(url, dir_name, n=True, depth=1)
# repo.git.checkout("HEAD", "datapackage.json")
except git.exc.GitCommandError as inst:
log.warn("%s: skipping %s" % (inst, name))
continue
updated = True
elif url.endswith(".json"):
# Handle Data Package URL
log.info("%s: New data package, fetching." % name)
updated = fetch_data_package(url, dir_name)
else:
log.warn("Unsupported repository: %s" % url)
# get datapackage metadata
try:
pkg_info = process_datapackage(name, repo_dir, url)
except ParseException as inst:
log.warn("%s: skipping %s" % (inst, name))
continue
# set last updated time based on last commit, comes in Unix timestamp format so we convert
import datetime
if repo is not None:
d = repo.head.commit.committed_date
else:
d = int(time.mktime(time.localtime()))
last_updated = datetime.datetime.fromtimestamp(int(d)).strftime('%Y-%m-%d %H:%M:%S')
pkg_info['last_updated'] = last_updated
# add it to the packages list for index page generation after the loop ends
packages.append(pkg_info)
# re-generate the dataset HTML pages
create_dataset_page(pkg_info, output_dir)
# if repo was updated, copy over CSV/JSON/* and ZIP files to the download dir
# (we always generate them if offline)
if updated or offline:
create_dataset_page(pkg_info, output_dir)
datafiles = pkg_info['datafiles']
zipf = zipfile.ZipFile(os.path.join(output_dir, name + '.zip'), 'w')
for d in datafiles:
log.info("Copying %s" % d['path'])
# copy file
target = os.path.join(output_dir, os.path.basename(d['path']))
shutil.copyfile(os.path.join(dir_name, d['path']), target)
# generate JSON version of CSV
if target.endswith('.csv'):
csv2json(target, target.replace(".csv", ".json"))
# make zip file
zipf.write(os.path.join(dir_name, d['path']), d['basename'], compress_type=zipfile.ZIP_DEFLATED)
if 'readme_path' in pkg_info:
try:
zipf.write(pkg_info['readme_path'], 'README.md')
except OSError:
pass
zipf.close()
# HTML index with the list of available packages
create_index_page(packages, output_dir)
# Static JSON API of the data packages
create_api(packages, output_dir, repo_dir)
# Static pages
create_static_pages(output_dir)
# Contact page
create_contact_page(output_dir, parser.get('credentials', 'contact_email'))
log.info("All static content is ready inside '%s'." % OUTPUT_DIR)
