def load_tool_definition_by_name(tool_configuration_path,
default_tool_name=None):
tool_definition_by_name = {}
configuration = RawCaseSensitiveConfigParser()
configuration.read(tool_configuration_path)
configuration_folder = dirname(tool_configuration_path)
d = {
u'configuration_path': tool_configuration_path,
u'configuration_folder': configuration_folder,
}
for section_name in configuration.sections():
try:
tool_name = TOOL_NAME_PATTERN.match(section_name).group(1).strip()
except AttributeError:
continue
if not tool_name:
tool_name = default_tool_name
tool_definition = {
unicode_safely(k): unicode_safely(v)
for k, v in configuration.items(section_name)
}
for key in tool_definition:
if key in ('show_standard_output', 'show_standard_error'):
tool_definition[key] = asbool(tool_definition[key])
elif key.endswith('.dependencies'):
tool_definition[key] = aslist(tool_definition[key])
tool_definition[u'tool_name'] = tool_name
tool_definition[u'argument_names'] = parse_tool_argument_names(
tool_definition.get('command_template', u''))
tool_definition_by_name[tool_name] = dict(
make_absolute_paths(tool_definition, configuration_folder), **d)
return tool_definition_by_name
def _transform_field_value(field_value, field_type):
if field_type in (ogr.OFTInteger, ogr.OFTInteger64):
field_value = int(field_value)
elif field_type in (ogr.OFTIntegerList, ogr.OFTInteger64List):
field_value = [int(x) for x in field_value]
elif field_type in (ogr.OFTString, ogr.OFTWideString):
field_value = unicode_safely(field_value)
elif field_type in (ogr.OFTStringList, ogr.OFTWideStringList):
field_value = [unicode_safely(x) for x in field_value]
elif field_type in (ogr.OFTDate, ogr.OFTDateTime):
try:
field_value = datetime(*map(int, field_value))
except ValueError:
field_value = np.nan
return field_value
def load_tool_arguments(notebook):
gv, lv = OrderedDict(), OrderedDict()
code_cells = get_code_cells(notebook)
if not code_cells:
raise CrossComputeError('cannot make a tool without code')
try:
exec(code_cells[0]['source'], gv, lv)
except Exception as e:
raise CrossComputeError(e)
d = OrderedDict()
for k, v in lv.items():
if k.startswith('__'):
continue
d[unicode_safely(k)] = unicode_safely(v)
return d
def from_csv(Class,
source_path,
source_proj4=None,
target_proj4=None,
**kw):
try:
t = load_csv_safely(source_path, **kw)
except pd.errors.EmptyDataError:
raise EmptyGeoTableError('file empty (%s)' % source_path)
try:
geometry_columns = _get_geometry_columns(t)
except GeoTableError as e:
raise GeoTableError(str(e) + ' (%s)' % source_path)
load_geometry_object = _get_load_geometry_object(geometry_columns)
source_proj4 = _get_proj4_from_path(source_path, source_proj4)
geometry_objects = []
if 'geometry_layer' not in t:
t['geometry_layer'] = unicode_safely(get_file_stem(source_path))
if _has_one_proj4(t):
row_proj4 = t.iloc[0].get('geometry_proj4', source_proj4)
f = get_transform_shapely_geometry(row_proj4, target_proj4)
for index, row in t.iterrows():
geometry_objects.append(f(load_geometry_object(row)))
t['geometry_proj4'] = normalize_proj4(target_proj4 or row_proj4)
else:
geometry_proj4s = []
for index, row in t.iterrows():
row_proj4 = row.get('geometry_proj4', source_proj4)
f = get_transform_shapely_geometry(row_proj4, target_proj4)
geometry_objects.append(f(load_geometry_object(row)))
geometry_proj4s.append(
normalize_proj4(target_proj4 or row_proj4))
t['geometry_proj4'] = geometry_proj4s
t['geometry_object'] = geometry_objects
return Class(t.drop(geometry_columns, axis=1))
def find_tool_definition_by_name(folder, default_tool_name=None):
tool_definition_by_name = {}
folder = unicode_safely(folder)
default_tool_name = unicode_safely(default_tool_name)
for root_folder, folder_names, file_names in walk(folder):
if are_same_path(root_folder, folder):
tool_name = default_tool_name or basename(folder)
else:
tool_name = basename(root_folder)
for file_name in file_names:
if not fnmatch(file_name, '*.ini'):
continue
tool_configuration_path = join(root_folder, file_name)
for tool_name, tool_definition in load_tool_definition_by_name(
tool_configuration_path, tool_name).items():
tool_name = _get_unique_tool_name(tool_name,
tool_definition_by_name)
tool_definition_by_name[tool_name] = tool_definition
return tool_definition_by_name
def wrapped_function(*args, **kw):
self = args[0].copy()
_make_geotable(self)
arguments = inspect.signature(f).bind(*args, **kw).arguments
try:
target_path = arguments['target_path']
except KeyError:
pass
else:
target_stem = unicode_safely(get_file_stem(target_path))
self['geometry_layer'].replace('', target_stem, inplace=True)
return f(self, *args[1:], **kw)
def from_gdal(Class, source_path, source_proj4=None, target_proj4=None):
try:
gdal_dataset = gdal.OpenEx(source_path)
except RuntimeError:
raise GeoTableError('file unloadable (%s)' % source_path)
instances = []
for layer_index in range(gdal_dataset.GetLayerCount()):
gdal_layer = gdal_dataset.GetLayer(layer_index)
row_proj4 = _get_proj4_from_gdal_layer(gdal_layer, source_proj4)
f = _get_transform_gdal_geometry(row_proj4, target_proj4)
t = _get_instance_from_gdal_layer(Class, gdal_layer, f)
t['geometry_layer'] = unicode_safely(gdal_layer.GetName())
t['geometry_proj4'] = normalize_proj4(target_proj4 or row_proj4)
instances.append(t)
t = concatenate_tables(instances)
if source_path.endswith('.kmz') or source_path.endswith('.kml'):
t = t.drop(columns=KML_COLUMNS, errors='ignore')
return t
def post(self):
stop_servers()
notebook_path = self.get_argument('notebook_path')
tool_port = S['tool_port']
process_arguments = [
'crosscompute', 'serve',
abspath(notebook_path), '--host', S['tool_host'], '--port',
str(tool_port), '--base_url', S['tool_base_url'],
'--without_browser', '--without_logging', '--with_debugging'
]
for x in 'brand_url', 'website_name', 'website_owner':
y = expect_variable(x, '')
if y:
process_arguments.extend(('--' + x, y))
open(join(
make_folder(DEBUGGING_FOLDER),
'preview-tool.sh',
), 'wt').write(' '.join(
('"%s"' % x if ' ' in x else x) for x in process_arguments))
process = Popen(process_arguments, stderr=PIPE)
d = {}
for x in range(10):
try:
requests.get('http://127.0.0.1:%s' % tool_port)
except ConnectionError:
sleep(1)
else:
status_code = 200
d['tool_url'] = self._get_tool_url()
break
if process.poll():
status_code = 400
d['text'] = unicode_safely(process.stderr.read().strip())
break
else:
status_code = 400
self.set_status(status_code)
self.write(d)