def get_renderer_class(project_title, entity_name, asset):
try:
renderer_class = import_from(
'hydra.renderers.{0}'.format(project_title),
'{0}{1}Renderer'.format(entity_name.title(), asset.title()))
except (ImportError, AttributeError):
renderer_class = import_from(
'hydra.renderers.standard',
'{0}{1}Renderer'.format(entity_name.title(), asset.title()))
return renderer_class
def xls(request):
"""Serve xls output"""
title = request.matchdict['title']
config = get_config(title)
with transaction.manager:
models.populate_session(config.get('paths', 'database'))
project = models.Project.fetch(title)
from xlwt import Workbook, easyxf
workbook = Workbook()
heading_xf = easyxf('font: bold on; align: wrap on, '
'vert centre, horiz center')
sheet = workbook.add_sheet('apartment')
sheet.set_panes_frozen(True)
sheet.set_horz_split_pos(1)
sheet.set_remove_splits(True)
attribs = config.get('project:data_renderer',
'apartment').split(', ')
for attr_num, name in enumerate(attribs):
sheet.col(attr_num).width = 256 * (len(name) + 3)
sheet.write(0, attr_num, name, heading_xf)
entity_class = import_from('art3dutils.models', 'Apartment')
instances = entity_class.fetch_all(project.title)
for num, instance in enumerate(instances):
for attr_num, name in enumerate(attribs):
sheet.write(num+1, attr_num, getattr(instance, name))
response = Response(content_type='application/vnd.ms-excel')
workbook.save(response.body_file)
return response
def render(project_title, entity_name='project', shorthand=None, assets=None,
limit=None, format_=None, available_only=False, **filters):
"""Render materials for an entity"""
config = get_config(project_title)
assets = assets or config.get(entity_name, 'assets')
core_models.populate_session(config.get('paths', 'database'))
entity_class = import_from('art3dutils.models', entity_name.title())
instances = entity_class.fetch_all(project_title, shorthand=shorthand,
limit=limit,
available_only=available_only,
**filters)
for asset in assets.split(', '):
print(cyan('Rendering `{0}`...'.format(asset)))
logger = setup_logging(project_title, entity_name, asset)
renderer_class = get_renderer_class(project_title, entity_name, asset)
container = get_progress(instances)
if len(instances) is 1:
container = instances
for instance in container:
renderer = renderer_class(instance, config)
try:
renderer.render(format_)
except core_models.ApartmentError as e:
print(red(e))
logger.error(e)
def render_xls(self):
"""Render data as Excel file"""
project = self.entity
from xlwt import Workbook, easyxf
workbook = Workbook()
heading_xf = easyxf('font: bold on; align: wrap on, '
'vert centre, horiz center')
sheet = workbook.add_sheet('apartment')
sheet.set_panes_frozen(True)
sheet.set_horz_split_pos(1)
sheet.set_remove_splits(True)
attribs = self.config.get('project:data_renderer',
'apartment').split(', ')
for attr_num, name in enumerate(attribs):
sheet.col(attr_num).width = 256 * (len(name) + 3)
sheet.write(0, attr_num, name, heading_xf)
sheet.write(0, len(attribs), 'pl', heading_xf)
entity_class = import_from('art3dutils.models', 'Apartment')
instances = entity_class.fetch_all(project.title)
for num, instance in enumerate(utils.progressbar(instances)):
for attr_num, name in enumerate(attribs):
sheet.write(num+1, attr_num, getattr(instance, name))
sheet.write(num+1, len(attribs), getattr(instance, 'pl'))
workbook.save(self.output_path)
def mass_set_property(project_title, entity_name='apartment', **attributes):
"""Set property for all lots in project"""
config = get_config(project_title)
session = core_models.populate_session(config.get('paths', 'database'))
entity_class = import_from('art3dutils.models', entity_name.title())
instances = entity_class.fetch_all(project_title)
container = get_progress(instances)
for instance in container:
for attribute_name in attributes:
setattr(instance, attribute_name, attributes[attribute_name])
session.commit()
def multi_render(project_title, entity_name='project', shorthand=None,
assets=None, limit=None, workers_count=2):
"""Multiprocessing version of `render`"""
from multiprocessing import Queue, Process
config = get_config(project_title)
assets = assets or config.get(entity_name, 'assets').split(', ')
def worker(work_queue, count):
for key in get_progress(iter(work_queue.get, 'STOP'),
length=count):
for asset in assets:
try:
renderer_class = import_from(
'hydra.renderers.{0}'.format(project_title),
'{0}{1}Renderer'.format(entity_name.title(),
asset.title()))
except (ImportError, AttributeError):
renderer_class = import_from(
'hydra.renderers.standard',
'{0}{1}Renderer'.format(entity_name.title(),
asset.title()))
instance_to_render = entity_class.fetch(shorthand_key=key)
renderer = renderer_class(instance_to_render, config)
try:
renderer.render()
except core_models.ApartmentError as e:
print(red(e))
core_models.populate_session(config.get('paths', 'database'))
entity_class = import_from('art3dutils.models', entity_name.title())
instances = entity_class.fetch_all(project_title, shorthand=shorthand,
limit=limit)
total = len(instances)
queue = Queue()
print(cyan('Forming queue...'))
for instance in get_progress(instances):
queue.put(instance.shorthand)
workers_count = int(workers_count)
processes = []
print(cyan('Spawning {0} processes...'.format(workers_count)))
for w in xrange(workers_count):
p = Process(target=worker, args=(queue, total))
p.start()
processes.append(p)
queue.put('STOP')
for p in processes:
p.join()
def worker(work_queue, count):
for key in get_progress(iter(work_queue.get, 'STOP'),
length=count):
for asset in assets:
try:
renderer_class = import_from(
'hydra.renderers.{0}'.format(project_title),
'{0}{1}Renderer'.format(entity_name.title(),
asset.title()))
except (ImportError, AttributeError):
renderer_class = import_from(
'hydra.renderers.standard',
'{0}{1}Renderer'.format(entity_name.title(),
asset.title()))
instance_to_render = entity_class.fetch(shorthand_key=key)
renderer = renderer_class(instance_to_render, config)
try:
renderer.render()
except core_models.ApartmentError as e:
print(red(e))
def update(project_title, clean=False):
"""Add apartments to DB, canonize first"""
config = get_config(project_title)
if config.has_section('input_data'):
execute(canonize, project_title)
db = config.get('paths', 'database')
session = core_models.populate_session(db, create_tables=True)
if clean:
#remove project
project = core_models.Project.fetch(project_title)
if project:
session.delete(project)
session.commit()
#write new data
print(cyan('Updating project data...'))
canonic_json = config.get('paths:input', 'canonic_json')
try:
with codecs.open(canonic_json, 'r', 'utf-8') as f:
json_data = json.load(f)
for apt_obj in get_progress(json_data):
if 'number' not in apt_obj:
apt_obj['number'] = 1
if 'note' in apt_obj and type(apt_obj['note']) in (list, dict):
apt_obj['note'] = json.dumps(apt_obj['note'])
core_models.Apartment.add(project_title, **apt_obj)
session.commit()
except IOError:
print(red('No `{0}` file!'.format(canonic_json)))
if config.has_option('database', 'post_update_filters'):
print(cyan('Applying post-update filters...'))
filters = []
filter_names = config.get('database',
'post_update_filters').split(', ')
for filter_name in filter_names:
filter_ = import_from('hydra.'
'filters.{0}'.format(project_title),
filter_name)
filters.append(filter_)
apartments = core_models.Apartment.fetch_all(project_title)
for apartment in get_progress(apartments, 'apartments'):
for filter_ in filters:
filter_(apartment, config)
session.commit()
def __init__(self, entity, config):
self.entity = entity
self.config = config
self.output_path = None
self.input_path = config.get('paths:input', 'project')
filters = []
if config.has_option('entity:basic_renderer', 'filters'):
filter_names = config.get('entity:basic_renderer',
'filters').split(', ')
for filter_name in filter_names:
filter_ = import_from('art3d_hydra.'
'filters.{0}'
.format(config.get('entity:basic_renderer',
'project_title')),
filter_name)
filters.append(filter_)
for filter_ in filters:
filter_(self.entity, config)
def canonize(project_title):
"""Create a canonic JSON data file"""
config = get_config(project_title)
iterator_lib = config.get('input_data', 'iterator').split('.')[0]
iterator_name = config.get('input_data', 'iterator').split('.')[1]
apt_data = list()
iterator_class = import_from('hydra.'
'iterators.{0}'.format(iterator_lib),
iterator_name)
iterator = iterator_class(config)
if len(iterator):
print(cyan('Creating canonic JSON with {0}...'.format(iterator_name)))
for apt_dict in get_progress(iterator):
if apt_dict:
apt_data.append(apt_dict)
canonic_path = config.get('paths:input', 'canonic_json')
core_utils.create_dirs_in_path(canonic_path)
with codecs.open(canonic_path,
'w', 'utf-8') as f:
f.write(json.dumps(apt_data, ensure_ascii=False, indent=2,
separators=(',', ':')))
else:
print(yellow('Iterator empty!'))
def deploy(project_title, entity_name='project', shorthand=None, assets=None,
limit=None, file_format='json', available_only=False, **filters):
current_host = env.hosts[0]
config = get_config(project_title)
core_models.populate_session(config.get('paths', 'database'))
entity_class = import_from('art3dutils.models', entity_name.title())
instances = entity_class.fetch_all(project_title, shorthand=shorthand,
limit=limit,
available_only=available_only,
**filters)
role = 'staging'
if config.has_option('remote:production', 'host'):
production_host = config.get('remote:production', 'host')
if current_host == production_host:
role = 'production'
webroot = config.get('remote:{0}'.format(role), 'webroot')
assets = assets or config.get(entity_name, 'assets')
for asset in assets.split(', '):
print(cyan('*** DEPLOYING {entity}{shorthand} {asset} TO `{role}` ***'.format(
entity=entity_name, shorthand=" {}".format(shorthand) if shorthand else '',
asset=asset, role=role)))
remote_asset_path = config.get(
'paths:deployment',
'{entity}_{asset}'.format(entity=entity_name, asset=asset))
remote_path = '{webroot}/' \
'{asset_path}'.format(webroot=webroot,
asset_path=remote_asset_path)
remote_dir = os.path.dirname(remote_path)
if shorthand:
renderer_class = get_renderer_class(project_title, entity_name, asset)
def local_paths_gen():
for instance in instances:
entity_dir = config.get('paths:input', 'entity').format(
entity=instance.fs_name)
local_path = renderer_class(instance, config).set_output_path()
if os.path.exists(local_path):
yield local_path
else:
print(yellow("No {} for {} {}").format(asset, instance.fs_name,
instance.shorthand))
local_paths = local_paths_gen()
elif entity_name == 'project' and\
asset == 'data' and \
config.has_option('paths:output', '{0}_{1}'.format(entity_name, asset)):
local_path = (config.get('paths:output',
'{0}_{1}'.format(entity_name, asset))) \
.format(format=file_format, entity=entity_name, asset=asset, id='')
local_paths = [local_path]
else:
local_path = config.get('paths:output',
'entity_asset_dir').format(entity=entity_name,
asset=asset)
local_paths = ['{0}/*'.format(local_path)]
remote_dir_exists = True
with cd(remote_dir):
with quiet():
if run('ls').failed:
remote_dir_exists = False
dir_created = None
if not remote_dir_exists:
with quiet():
dir_created = run('mkdir {0}'.format(remote_dir)).succeeded
if dir_created:
print(cyan('Directory `{0}` created...'.format(remote_dir)))
for instance, local_path in izip(get_progress(instances, asset),
local_paths):
put(local_path, remote_path)
def render_json(self):
project = self.entity
import codecs
import art3dutils.models as models
output_file_path = self.output_path
config = self.config
dict_ = OrderedDict()
#add additional fields if any
if config.has_section('project:extras'):
extra_attribs = dict()
for attrib, short_type in config.items('project:extras'):
extra_attribs[attrib] = tuple(short_type.split(', '))
models.ATTR_TYPES.update(extra_attribs)
filters = []
if config.has_option('project:data_renderer', 'filters'):
filter_names = config.get('project:data_renderer',
'filters').split(', ')
config.remove_option('project:data_renderer', 'filters')
for filter_name in filter_names:
filter_ = import_from('art3d_hydra.'
'filters.{0}'.format(project.title),
filter_name)
filters.append(filter_)
for entity, attribs in config.items('project:data_renderer')[2:]:
id_pattern = config.get(entity, 'id_pattern')
dict_['{0}s'.format(entity)] = OrderedDict()
print('Rendering {0}s...'.format(entity))
entity_class = import_from('art3dutils.models', entity.title())
instances = entity_class.fetch_all(project.title)
for instance in utils.progressbar(instances):
instance_dict = OrderedDict()
for attrib in attribs.split(', '):
short, typ = models.ATTR_TYPES[attrib]
value = getattr(instance, attrib)
instance_dict[short] = utils.process_value(value, typ)
# insert fixed room counts
if attrib == 'available_detail' \
and config.has_option('project', 'room_counts'):
room_counts = config.get('project',
'room_counts').split(', ')
room_counts = [int(rc) for rc in room_counts]
room_counts.append('t')
instance_dict[short] = dict()
for rc in room_counts:
if rc in instance.available_detail:
instance_dict[short][rc] = \
instance.available_detail[rc]
else:
instance_dict[short][rc] = 0
# calc total cost
if attrib == 'total_cost' and not value:
instance_dict[short] = instance.calc_total_cost()
# check note for json
if attrib == 'note':
try:
instance_dict[short] = json.loads(value)
except (TypeError, ValueError):
pass
for filter_ in filters:
filter_(instance_dict, instance)
try:
key = id_pattern.format(**instance)
except TypeError:
key = id_pattern.format(
building_number=instance.building_number,
number=instance.number)
dict_['{0}s'.format(entity)][key] = instance_dict
utils.create_dirs_in_path(output_file_path)
with codecs.open(output_file_path, 'w', 'utf-8') as f:
if not self.minified:
data_string = json.dumps(dict_, ensure_ascii=False, indent=2,
separators=(',', ':'))
else:
data_string = json.dumps(dict_, ensure_ascii=False)
f.write(data_string)
