def get_modified_items(config: ns.Namespace, itemtype: str):
""" Retrieves all items of specified type from the server """
logging.info(f"Retrieving available items of type {itemtype}")
# Assemble URL and create request
svr = config.Server
scheme = 'https' if svr.https else 'http'
generated = '1' if config.Project.generated else '0'
url = f"{scheme}://{svr.host}:{svr.port}/api/atelier/v1/{svr.namespace}/modified/{itemtype}?generated={generated}"
# Get JSON response
try:
rsp = tls.session.post(url, json=[]) # pylint:disable=undefined-variable
except requests.exceptions.RequestException:
logging.error(f"Accessing {url}:")
raise
data = rsp.json()
# Check for configuration issue:
if data['status']['errors']:
e = data['status']['errors'][0]
if e['code'] == 16004:
raise ConfigurationError(
f"Error from server: unknown item type '{itemtype}'.")
return data
def setup_session(config):
global session
svr = config.Server
session = requests.Session()
session.auth = (svr.user, svr.password)
url = f"http://{svr.host}:{svr.port}/api/atelier/"
try:
session.get(url)
except requests.RequestException as e:
msg = f"Error connecting to server for converting UDL to XML:\n{e}."
raise ConfigurationError(msg) from None
def get_repo(config):
"""Returns the configured repository."""
if config.Source.type == 'github':
from github import get_data
elif config.Source.type == 'directory':
from fsrepo import get_data
else:
raise ConfigurationError(
f"Invalid repository type '{config.Source.type}' in configuration."
)
return get_data(config)
def main():
CM = ConfigurationManager()
configs = CM.build_configs()
mode = configs["mode"]
agent_name = configs["agent"]
agent = getattr(agents, agent_name)(configs)
try:
if mode == "play":
agent.run()
elif mode == "learn":
agent.train()
else:
raise AttributeError
except(AttributeError):
raise ConfigurationError("Error trying the mode '{}' on the agent '{}': possibly not implemented.".format(mode, agent_name))
def split_csp(config, name: str):
""" Split CSP item in app and page. """
for i, parser in enumerate(config.CSP.parsers):
match = re.fullmatch(parser.regex, name)
if not match:
if parser.nomatch != 'error':
logging.debug(
f"Item {name} does not match regex in parser {i+1} ('{parser.regex}')."
)
continue
raise ConfigurationError(
f"Error: item {name} does not match regex in parser {i+1} ('{parser.regex}')."
)
app = parser.app
for i, v in enumerate(match.groups()):
if v is not None:
app = app.replace(f'\\{i+1}', v)
page = parser.item
for i, v in enumerate(match.groups()):
if v is not None:
page = page.replace(f'\\{i+1}', v)
# Some basic validity checks:
if app[0] != '/':
raise ValueError('Invalid application: must start with a slash')
if app[-1] == '/':
raise ValueError('Invalid application: must not end with a slash')
if page[0] == '/':
raise ValueError('Invalid page: must not start with a slash')
if page[-1] == '/':
raise ValueError('Invalid page: must not end with a slash')
return app, page
return None
def save_item(config: ns.Namespace, item: Dict[str, Any]):
""" Retrieves an item and saves it to disk """
logging.info(f"Retrieving and saving {item['name']}")
data = ret.retrieve_item(config, item)
fname = determine_filename(config, item)
dir = dirname(fname)
os.makedirs(dir, exist_ok=True)
# Write the data to the output file. If this fails, catch the exception
# to log the name of the file we tried to write to, and reraise; function
# unhandled_exception will log the stack trace.
try:
if not isinstance(data, bytes):
# Text document; store in specified encoding (default UTF-8)
with open(fname, 'wt', encoding=config['encoding']) as ft:
ft.write(data)
else:
# Binary document (e.g. image from CSP application)
with open(fname, 'wb') as fb:
fb.write(data)
except UnicodeEncodeError as e:
faulty = data[e.start - 1:e.end]
msg = f"Error saving {item['name']}: some characters can't be saved" \
f" in the configured encoding ({config['encoding']}). Problem" \
f" starts around character {e.start}; data: {faulty}."
raise ConfigurationError(msg) from None
except Exception:
logging.error(f"\nException detected writing to file {fname}")
raise
# Set modified date/time to that of item in IRIS
set_file_datetime(fname, item['ts'])