def value2key(keys, val):
if len(keys) == 1:
if isinstance(val, Mapping):
return val[keys[0]]
elif isinstance(val, (list, tuple)):
return val[0]
else:
return val
else:
if isinstance(val, Mapping):
return dictwrap({k: val[k] for k in keys})
elif isinstance(val, (list, tuple)):
return dictwrap(dict(zip(keys, val)))
else:
Log.error("do not know what to do here")
def value2key(keys, val):
if len(keys) == 1:
if isinstance(val, Mapping):
return val[keys[0]]
elif isinstance(val, (list, tuple)):
return val[0]
else:
return val
else:
if isinstance(val, Mapping):
return dictwrap({k: val[k] for k in keys})
elif isinstance(val, (list, tuple)):
return dictwrap(dict(zip(keys, val)))
else:
Log.error("do not know what to do here")
def main():
try:
settings = startup.read_settings(defs=[{
"name": ["--id"],
"help": "id(s) to process. Use \"..\" for a range.",
"type": str,
"dest": "id",
"required": False
}])
constants.set(settings.constants)
Log.start(settings.debug)
if settings.args.id:
etl_one(settings)
return
hg = HgMozillaOrg(settings=settings.hg)
resources = Dict(hg=dictwrap(hg))
stopper = Signal()
for i in range(coalesce(settings.param.threads, 1)):
ETL(
name="ETL Loop " + unicode(i),
work_queue=settings.work_queue,
resources=resources,
workers=settings.workers,
settings=settings.param,
please_stop=stopper
)
Thread.wait_for_shutdown_signal(stopper, allow_exit=True)
except Exception, e:
Log.error("Problem with etl", e)
def __getitem__(self, key):
try:
key = value2key(self._keys, key)
d = self._data.get(key)
return dictwrap(d)
except Exception, e:
Log.error("something went wrong", e)
def main():
try:
settings = startup.read_settings(defs=[{
"name": ["--id"],
"help": "id(s) to process. Use \"..\" for a range.",
"type": str,
"dest": "id",
"required": False
}])
constants.set(settings.constants)
Log.start(settings.debug)
if settings.args.id:
etl_one(settings)
return
hg = HgMozillaOrg(settings=settings.hg)
resources = Dict(hg=dictwrap(hg))
stopper = Signal()
for i in range(coalesce(settings.param.threads, 1)):
ETL(name="ETL Loop " + unicode(i),
work_queue=settings.work_queue,
resources=resources,
workers=settings.workers,
settings=settings.param,
please_stop=stopper)
Thread.wait_for_shutdown_signal(stopper, allow_exit=True)
except Exception, e:
Log.error("Problem with etl", e)
def select(self, key):
"""
simple `select`
"""
if not dictwrap:
_late_import()
return DictList(vals=[unwrap(dictwrap(v)[key]) for v in _get(self, "list")])
def _get_managed_instances(ec2_conn, name):
requests = UniqueIndex(["instance_id"], data=_get_managed_spot_requests(ec2_conn, name).filter(lambda r: r.instance_id != None))
reservations = ec2_conn.get_all_instances()
output = []
for res in reservations:
for instance in res.instances:
if instance.tags.get('Name', '').startswith(name) and instance._state.name == "running":
instance.request = requests[instance.id]
output.append(dictwrap(instance))
return wrap(output)
def _get_managed_instances(ec2_conn, name):
requests = UniqueIndex(["instance_id"], data=_get_managed_spot_requests(ec2_conn, name).filter(lambda r: r.instance_id != None))
reservations = ec2_conn.get_all_instances()
output = []
for res in reservations:
for instance in res.instances:
if instance.tags.get('Name', '').startswith(name) and instance._state.name == "running":
instance.request = requests[instance.id]
output.append(dictwrap(instance))
return wrap(output)
def remove(self, val):
key = value2key(self._keys, dictwrap(val))
if key == None:
Log.error("Expecting key to not be None")
d = self._data.get(key)
if d is None:
# ALREADY GONE
return
else:
del self._data[key]
self.count -= 1
def remove(self, val):
key = value2key(self._keys, dictwrap(val))
if key == None:
Log.error("Expecting key to not be None")
d = self._data.get(key)
if d is None:
# ALREADY GONE
return
else:
del self._data[key]
self.count -= 1
def add(self, val):
val = dictwrap(val)
key = value2key(self._keys, val)
if key == None:
Log.error("Expecting key to be not None")
d = self._data.get(key)
if d is None:
self._data[key] = unwrap(val)
self.count += 1
elif d is not val:
if self.fail_on_dup:
Log.error("{{new|json}} with key {{key|json}} already filled with {{old|json}}", key=key, new=val, old=self[val])
elif DEBUG:
Log.warning("key {{key|json}} already filled\nExisting\n{{existing|json|indent}}\nValue\n{{value|json|indent}}",
key=key,
existing=d,
value=val
)
def add(self, val):
val = dictwrap(val)
key = value2key(self._keys, val)
if key == None:
Log.error("Expecting key to not be None")
d = self._data.get(key)
if d is None:
self._data[key] = unwrap(val)
self.count += 1
elif d is not val:
if self.fail_on_dup:
Log.error("key {{key|json}} already filled", key=key)
else:
Log.warning("key {{key|json}} already filled\nExisting\n{{existing|json|indent}}\nValue\n{{value|json|indent}}",
key=key,
existing=d,
value=val
)
def add(self, val):
val = dictwrap(val)
key = value2key(self._keys, val)
if key == None:
Log.error("Expecting key to be not None")
d = self._data.get(key)
if d is None:
self._data[key] = unwrap(val)
self.count += 1
elif d is not val:
if self.fail_on_dup:
Log.error("key {{key|json}} already filled", key=key)
else:
Log.warning(
"key {{key|json}} already filled\nExisting\n{{existing|json|indent}}\nValue\n{{value|json|indent}}",
key=key,
existing=d,
value=val)
def _get_managed_spot_requests(ec2_conn, name):
output = wrap([dictwrap(r) for r in ec2_conn.get_all_spot_instance_requests() if not r.tags.get("Name") or r.tags.get("Name").startswith(name)])
return output
def _get_managed_spot_requests(ec2_conn, name):
output = wrap([
dictwrap(r) for r in ec2_conn.get_all_spot_instance_requests()
if not r.tags.get("Name") or r.tags.get("Name").startswith(name)
])
return output
def __iter__(self):
return (dictwrap(v) for v in self._data.itervalues())
