def set_home(self, home, memory=False):
"""Sets the home folder"""
# self._settings['home'] = home
self.home = home
if not memory:
self.storage = JSONStorage(self.get_home(), 'settings')
self.reload()
def set_settings(self, params):
"""Loads the parameters overriding the defaults"""
settings = self._default.copy()
if params is not None:
if not isinstance(params, dict):
raise ValueError("Params must be a dict")
for defined in self.param_definition.keys():
if defined in params:
settings[defined] = params[defined]
logging.debug('Set configuration key ' + defined + ' to ' +
str(params[defined]))
self._settings = settings
self.storage = JSONStorage(self.get_home(), 'settings')
def scrape_locations(country_id):
req = RequestHandler()
with JSONStorage(f'lmigroup_{country_id}') as stor:
st = [0]
while st:
n = st.pop()
print(n)
key = f'loc_{n}'
if stor.has_item(key):
data = stor.read_item(key)
else:
data = get_locations(req, n, country_id)
stor.store_item(key, data)
for i in data:
st.append(i['id'])
def show_construction_types(country_id):
d = []
ii = {0: 'root'}
with JSONStorage(f'lmigroup_{country_id}') as stor:
for i in stor.items():
if i.startswith('constr_'):
data = stor.read_item(i)
for v in data:
if v:
d.append((v['id'], v))
ii[v['id']] = v['name']
for i, v in sorted(d):
print(i, v['name'])
print('parent: ', ii[v['parentId']])
print()
def scrape_construction_types(country_id):
req = RequestHandler()
with JSONStorage(f'lmigroup_{country_id}') as stor:
st = [(0, True), (0, False)]
while st:
n, p = st.pop()
print(n, p)
key = f'constr_{n}'
if p:
key += 'p'
if stor.has_item(key):
data = stor.read_item(key)
else:
data = get_construction_types(req, n, p, country_id)
stor.store_item(key, data)
for i in data:
st.append((i['id'], p))
def configure(self):
"""Configures the storage system"""
# Read params
if self.params is None:
self.params = {}
params = self.params
self.memory = params.get('memory', False)
data = params.get('data', None)
if self.path is not None:
self.data_storage = JSONStorage(
self.path,
self.subcollection,
self.memory)
self.items = self.data_storage.load()
if self.items is None:
self.items = []
if data is not None:
self.items = data
self._calc_autoid()
def create_csv(country_id):
con_n = {}
loc_n = {}
occ_used = set()
loc_used = set()
cost_array = defaultdict(lambda: defaultdict(list))
with JSONStorage(f'lmigroup_{country_id}') as stor:
stor.clear_errors()
for i in stor.items():
if i.startswith('loc_'):
for l in stor.read_item(i):
loc_n[l['id']] = l['name'].strip()
if i.startswith('constr_'):
for c in stor.read_item(i):
con_n[c['id']] = c['name'].strip()
if i.startswith('cost_'):
cost_data = stor.read_item(i)
occ_used.add(cost_data['construction']['id'])
loc_used.add(cost_data['location']['id'])
cost_array[cost_data['construction']['id']][
cost_data['location']['id']] = [
cost_data['lowerCost'], cost_data['upperCost']
]
occ_ids = sorted(occ_used)
loc_ids = sorted(loc_used)
with open(f'country_{country_id}.csv', 'w') as f:
fr = ['']
fr.extend(map(loc_n.get, loc_ids))
f.write(','.join(fr) + '\n')
for oid in occ_ids:
r = [f'"{con_n[oid]}"']
for lid in loc_ids:
r.append('"' + '\n'.join(map(str, cost_array[oid][lid])) + '"')
f.write(','.join(r) + '\n')
def scrape_costs(country_id):
req = RequestHandler()
with JSONStorage(f'lmigroup_{country_id}') as stor:
stor.clear_errors()
locations = []
constructions = []
for i in stor.items():
if i.startswith('loc_'):
locations.extend(stor.read_item(i))
if i.startswith('constr_'):
constructions.extend(stor.read_item(i))
for location, construction in product(locations, constructions):
print(location['id'], construction['id'])
key = f'cost_{location["id"]}_{construction["id"]}'
if not stor.has_item(key):
data = calculate_cost(req, location['id'], construction['id'])
data['location'] = location
data['construction'] = construction
if 'message' in data:
print(data['message'])
stor.mark_notfound(key, data['message'])
else:
stor.store_item(key, data)
class Config(object):
"""Config is the main class for this module; initialize the values with
the default configuration, allows query/edit or save it's state."""
_instance = None
OSX = 1
WINDOWS = 2
OTHER = 3
DIR_MODE = 0700
# Settings description
param_definition = {
'build_user_dir': "Build the user data directory",
'plugins_enabled': "List of plugins enabled",
'lang': """The application language must be locale_COUNTRY
or ':system:'.
Examples:
en_UK for English (United Kingdom)
ca_ES for catalan
es_ES for spanish
""",
'copy': """Copy files to the collection folder policy:
never never copy files
always copy all the files
remote only network files
""",
}
# Default settings
_default = {
'build_user_dir': True,
'plugins_enabled': [],
'lang': ':system:',
'copy': 'http',
}
def __init__(self, platform=None):
""" Constructor for Config, initialize all the attributes"""
if not Config._instance is None:
raise Exception("Called more that once")
Config.isntance = self
self.__file__ = FILEPATH
if platform is None:
platform = Config.get_current_os()
elif platform not in [Config.OSX, Config.WINDOWS, Config.OTHER]:
raise Exception("Platform identifier %s not found" % platform)
self.platform = platform
self.resources = self.get_resources_path()
self.home = ":auto:"
# Parse all the parameters
self._settings = []
self.storage = None
self.set_settings(None)
def save(self):
"""Persistences call, stores all the data at disc"""
if self.storage is not None:
self.storage.save(self._settings)
def set(self, key, value):
"""Overrides a value of the settings or creates a new one"""
self._settings[key] = value
def set_home(self, home, memory=False):
"""Sets the home folder"""
# self._settings['home'] = home
self.home = home
if not memory:
self.storage = JSONStorage(self.get_home(), 'settings')
self.reload()
def set_settings(self, params):
"""Loads the parameters overriding the defaults"""
settings = self._default.copy()
if params is not None:
if not isinstance(params, dict):
raise ValueError("Params must be a dict")
for defined in self.param_definition.keys():
if defined in params:
settings[defined] = params[defined]
logging.debug('Set configuration key ' + defined + ' to ' +
str(params[defined]))
self._settings = settings
self.storage = JSONStorage(self.get_home(), 'settings')
def get_settings(self):
"""Returns the settings"""
return self._settings
def reload(self):
"""Reloads the stored configuration (if exists)"""
if self.storage is not None:
self.set_settings(self.storage.load())
@staticmethod
def get_instance():
"""Returns the config instance"""
if Config._instance is None:
Config._instance = Config()
return Config._instance
def get_resources_path(self):
"""Returns the resources path, in MacOs is the folder Resources
inside the bundle """
# determine if application is a script file or frozen exe
application_path = ''
if getattr(sys, 'frozen', False):
application_path = os.path.dirname(sys.executable)
if self.platform == Config.OSX:
application_path = application_path.replace(
'MacOS',
'Resources'
)
else:
application_path = self.__file__.replace(
os.path.join('collector', 'core'),
'')
return os.path.abspath(application_path)
def get_appdata_path(self):
"""Returns the app data folder, this folder is inside the
application"""
return os.path.join(self.resources, 'data')
def get_home(self):
"""Returns the user data path (home)"""
value = self.home
if value == ':auto:':
value = Config.calculate_data_path(self.platform)
elif value == ':resources:':
value = self.get_appdata_path()
return value
get_data_path = get_home
def get_plugin_path(self):
"""Returns the user plugin path"""
return os.path.join(self.get_home(), 'plugins')
@staticmethod
def get_current_os(platform=None):
"""Try to discover the current OS"""
if platform is None:
platform = sys.platform
platform = platform.lower()
if platform in 'win32' or platform in 'win64':
return Config.WINDOWS
elif platform in 'darwin':
return Config.OSX
else:
return Config.OTHER
@staticmethod
def calculate_data_path(platform=None):
""" Calculates the user data directory,
the exact location depends of the OS"""
config_dir = None
if platform is None:
platform = Config.get_current_os()
if platform == Config.WINDOWS:
# TODO try to use distutils get_special_folder_path
#config_dir = get_special_folder_path("CSIDL_APPDATA")
config_dir = os.path.expanduser('~')
config_dir = os.path.join(config_dir, 'Collector')
elif platform == Config.OSX:
config_dir = os.path.expanduser(
'~/Library/Application Support/Collector')
else:
bdir = os.path.abspath(os.path.expanduser(
os.environ.get('XDG_CONFIG_HOME', '~/.config')))
config_dir = os.path.join(bdir, 'collector')
return config_dir
def build_data_directory(self):
"""Creates the content inside the user data directory"""
subfolders = ['user_plugins', 'config', 'collections']
import shutil
base = self.get_home()
if not os.path.exists(base):
os.makedirs(base, mode=Config.DIR_MODE)
appdata = self.get_appdata_path()
# Do a full copy because no previus data exists
for folder in subfolders:
dst = os.path.join(base, folder)
src = os.path.join(appdata, folder)
shutil.copytree(src, dst)
# TODO check if some data is missing?
def conf(self, key):
"""Returns the setting value for the requested key"""
if key != 'home':
return self._settings[key]
else:
return self.get_home()
get = conf
def archive_data(country_id):
with JSONStorage(f'lmigroup_{country_id}') as stor:
stor.archive()
class PersistenceDict(Persistence):
"""Implementation of persistence using a python dictionary"""
_autoid = 1
def __init__(self, schema, path, params=None):
super(PersistenceDict, self).__init__(schema, path, params)
self.items = []
self.data_storage = None
self.memory = False
self.class_ = type(str(schema.collection + "_" + schema.id),
(FileDict,), {"schema": schema.id})
# configure
self.configure()
def configure(self):
"""Configures the storage system"""
# Read params
if self.params is None:
self.params = {}
params = self.params
self.memory = params.get('memory', False)
data = params.get('data', None)
if self.path is not None:
self.data_storage = JSONStorage(
self.path,
self.subcollection,
self.memory)
self.items = self.data_storage.load()
if self.items is None:
self.items = []
if data is not None:
self.items = data
self._calc_autoid()
def _calc_autoid(self):
"""Searchs the max id used and set's the new autoid value"""
maxid = 0
for i in self.items:
maxid = max(maxid, i['id'])
self._autoid = maxid + 1
def filter(self, fitlers):
# TODO
raise Exception("Not Implemented")
def get_filters(self):
#TODO
return []
def get_last(self, count):
"""Returns the last items created, the number of items are defined
with the count parameter, the items are orderded by last inserted"""
result = self.items[-count:]
# Reverse the count
objects = []
result.reverse()
for item in result:
objects.append(FileDict(item))
return objects
def get(self, _id):
if isinstance(_id, str):
_id = int(_id)
for item in self.items:
if _id == item['id']:
return FileDict(item)
return None
def get_all(self, start_at, limit, order=None):
"""Returns all the items starting at *start_at*, the results could
be limited whit *limit*"""
result = []
objects = []
if limit == 0:
objects = self.items[start_at:]
else:
objects = self.items[start_at:(start_at + limit)]
for item in objects:
result.append(FileDict(item))
return result
def search(self, term):
results = []
term = term.lower()
for item in self.items:
if item['title'].lower().find(term) != -1:
results.append(FileDict(item))
return results
def save(self, values):
if 'id' in values:
for item in self.items:
if values['id'] == item['id']:
item.update(values)
else:
values['id'] = self._autoid
self._autoid += 1
self.items.append(values)
self.commit()
def delete(self, _id):
for item in self.items:
if item['id'] == _id:
self.items.remove(item)
self.commit()
def commit(self):
"""Stores the changes"""
if self.data_storage is not None:
self.data_storage.save(self.items)
def load_references(self, collections, item):
if 'refLoaded' in item:
return
item = item.copy()
item['refLoaded'] = True
return item