def validator(val):
val = val.split(' ')
if val == ['']:
return choices
new_val = []
if cast:
for i in val:
try:
i = cast(i)
new_val.append(i)
except ValueError:
print "Invalid Selection: %s" % i
raise ConfigError('Invalid Selection')
val = new_val
invalid_choices = [i for i in val if i not in choices]
if len(invalid_choices):
print "Invalid selections: %s" % ', '.join(
str(i) for i in invalid_choices)
raise ConfigError("Invalid selections")
return val
def __init__(self, unique_id, model, position, vision_radius,
violent_affinity, active_affinity, quiet_affinity,
cop_affinity, media_affinity, flag_affinity,
obstacle_affinity, citizen_type, state, hardship,
perceived_legitimacy, risk_tolerance, threshold):
super().__init__(unique_id, model, position, vision_radius,
violent_affinity, active_affinity, quiet_affinity,
cop_affinity, media_affinity, flag_affinity,
obstacle_affinity)
self.citizen_type = citizen_type
self.state = state
if citizen_type not in Citizen.citizen_types:
raise ConfigError("invalid citizen type.")
if state not in Citizen.states:
raise ConfigError("invalid state.")
self.hardship = hardship
self.perceived_legitimacy = perceived_legitimacy
self.risk_tolerance = risk_tolerance
self.arrest_delay = self.model.arrest_delay # The time between a fight with police starting and arrest.
self.arrested = False
self.arrested_count = 0
self.jail_time = 0
# Expected utility of not acting.
self.threshold = threshold
def configure(self):
if not os.access(DM_CONF_FN, os.R_OK):
self._logger.warning("Cannot access download mgr configuration " +
"(" + DM_CONF_FN + "), ")
else:
try:
self._dm_port, self._download_dir = get_dm_config(
DM_CONF_FN, self._logger)
except Exception as e:
self._logger.error("Error checking DM configuration: " + ` e `)
raise
if None == self._download_dir or '' == self._download_dir:
raise ConfigError("No download directory")
if None == self._dm_port or '' == self._dm_port:
raise ConfigError("No DM port")
check_or_make_dir(self._download_dir, self._logger)
#now add this year's dir
year = str(time.gmtime().tm_year)
yr_dld = os.path.join(self._download_dir, year)
check_or_make_dir(yr_dld, self._logger)
port_ok = self.wait_for_port()
self._dm_url = DM_URL_TEMPLATE % self._dm_port
return port_ok
def submit_dar(self, dar):
if None == self._dm_port:
raise ConfigError("No port for DM")
if None == self._ie_port:
raise ConfigError("No IE port.")
if None == self._dar_resp_url:
self._dar_resp_url = IE_DAR_RESP_URL_TEMPLATE % self._ie_port
# lock the queue since access is potentially from the
# work_flow_manager worker threads
self._lock_queue.acquire()
try:
dar_seq_id = mkIdBase() + ` self._get_next_seq_id() `
dar_q_item = (dar_seq_id, dar)
self._dar_queue.append(dar_q_item)
except Exception as e:
raise
finally:
self._lock_queue.release()
dm_dl_url = os.path.join(self._dm_url, DM_DOWNDLOAD_COMMAND)
dar_url = self._dar_resp_url + "/" + dar_seq_id
post_data = "darUrl=" + dar_url
self._logger.info("Submitting request to DM to retieve DAR:\n" \
+ post_data)
try:
dm_resp = json.loads(read_from_url(dm_dl_url, post_data))
self._logger.debug("dm_response: " + ` dm_resp `)
except HTTPError as e:
self._logger.error("Download Manager Error: " + ` e.code ` + ' ' +
` e `)
self._logger.error(" " + e.read())
raise DMError("HTTPError")
except URLError as e:
self._logger.error("Download Manager Error: " + ` e `)
if e.reason:
self._logger.error(" reason: " + ` e.reason `)
raise DMError("URLError")
dm_dar_id = None
if "success" in dm_resp and dm_resp["success"]:
self._logger.info("DM accepted DAR.")
if 'darUuid' in dm_resp:
dm_dar_id = dm_resp['darUuid']
elif "errorType" in dm_resp and \
dm_resp["errorType"] == "DataAccessRequestAlreadyExistsException":
# TODO: try to auto-recover/find the status/resume the dar
# but this should occur only exceptionally/almost never
return ("DAR_EXISTS", None, None)
else:
msg = None
if not "errorMessage" in dm_resp:
msg = "Unknown error, no 'errorMessage' found in response"
else:
msg = "DM reports error:\n" + dm_resp["errorMessage"]
raise DMError(msg)
return ("OK", dar_url, dm_dar_id)
def parse_ss(value):
# make sure string is int between 1 and 4
if value == 'None':
return None
try:
value = int(value)
except ValueError:
raise ConfigError("Invalid int value: %s" % value)
if not (1 <= value <= 4):
raise ConfigError("Invalid ss value: %s" % value)
return value
def read_configs(self, filename, names, settings):
'''
read config file, validate selections, store in settings
'''
f = None
try:
f = open(filename, 'r')
for line in f:
key, value = line.strip().split('=')
key = key.strip()
value = value.strip()
# check that this is a valid config key
if key not in names:
raise ConfigError("Invalid config - %s : %s" %
(key, value))
# validate value
value = self.validators[key](value)
settings[key] = value
finally:
if f is not None:
f.close()
def main(argv=None):
global api_key
if argv is None:
argv = sys.argv[1:]
try:
if api_key == "XXX":
raise ConfigError(
"You need to register for an Open Calaise API key and configure it in config.py\n"
)
try:
opts, args = getopt.getopt(argv, "h", ["help"])
except getopt.error as msg:
raise UsageError(msg)
for o, a in opts:
if (o in ['-h', '--help']):
# print help and exit
sys.stdout.write(__doc__)
sys.stdout.flush()
return 0
opencalaistags(opts, args)
except UsageError, err:
sys.stderr.writelines([str(err.msg) + '\n', "for help use --help\n"])
sys.stderr.flush()
return 2
def parse_bool(value):
# parse boolean values
if value == 'True':
return True
if value == 'False':
return False
raise ConfigError('Invalid boolean: %s' % value)
def _inner(value):
# compare against choices
value = value.split(',')
if cast:
# try to cast each entry using cast function
try:
value = [cast(v) for v in value]
except exc_class:
# improper config file
raise ConfigError("Invalid value: %s" % value)
# find file selections not in valid choices
invalid_choices = [v for v in value if v not in choices]
if len(invalid_choices):
raise ConfigError("Invalid Selections: %s" %
','.join(invalid_choices))
return value
def validate_mcs(value):
# check against mcs regex
if value == 'ALL':
return value
if value == 'None':
return None
if not re.match(r'^[0-9]+(-[0-9]+)?(,[0-9]+(-[0-9]+)?)*$', value):
raise ConfigError("Invalid mcs value: %s" % value)
return value
raise UsageError(msg)
for o, a in opts:
if (o in ['-h', '--help']):
print __doc__
return 0
try:
pdfpath = args[-1]
except IndexError:
raise UsageError("You must provide the name of a valid PDF to analyse")
pdffn = os.path.split(pdfpath)[-1]
tmpdir = tempfile.mkdtemp(suffix='.d', prefix=pdffn)
tmppath = os.path.join(tmpdir, "{0}.xml".format(pdffn))
if not os.path.exists(pdf2xmlexe):
raise ConfigError(
"pdftoxml exectutable does not exist at specified path: '{0}'\nPlease check config.py"
.format(pdf2xmlexe))
cmdline = "{0} -q -blocks {1} {2}".format(pdf2xmlexe, pdfpath, tmppath)
commands.getoutput(cmdline)
try:
with open(tmppath, 'r') as fh:
tree = etree.parse(fh)
except IOError:
raise UsageError(
"Could not convert to XML. Are you sure you provided the name of a valid PDF?"
)
else:
return tree
def pdf2xml(argv=None):
def __init__(self, initial_num_citizens, initial_num_media,
hardcore_density, hanger_on_density, observer_density,
agent_vision_radius, agent_move_falibility,
default_hardcore_move_vector, default_hanger_on_move_vector,
default_observer_move_vector, default_cop_move_vector,
default_media_move_vector, citizen_jailed_sensitivity,
citizen_pictures_sensitivity, citizen_cops_sensitivity,
max_days, height, width, agent_regions, obstacle_regions,
flag_regions, cop_regions, arrest_delay, jail_time):
super().__init__()
self.steps_per_day = 12
# Population initialisation
self.initial_num_cops = len(co_ords_for_area(cop_regions))
self.initial_num_citizens = initial_num_citizens
self.initial_num_media = initial_num_media
self.hardcore_density = hardcore_density
self.hanger_on_density = hanger_on_density
self.hanger_on_density = observer_density
# Agent init
# Agent movement factors
self.agent_vision_radius = agent_vision_radius
self.agent_move_falibility = agent_move_falibility
# vector order:
# [violent, active, quiet, cop, media, flag, obstacle]
self.default_hardcore_move_vector = default_hardcore_move_vector
self.default_hanger_on_move_vector = default_hanger_on_move_vector
self.default_observer_move_vector = default_observer_move_vector
self.default_cop_move_vector = default_cop_move_vector
self.default_media_move_vector = default_media_move_vector
# Citizen legitimacy update factors
self.citizen_jailed_sensitivity = citizen_jailed_sensitivity
self.citizen_pictures_sensitivity = citizen_pictures_sensitivity
self.citizen_cops_sensitivity = citizen_cops_sensitivity
# Core model code
# Model step represents 2 hours
self.max_iters = max_days * self.steps_per_day
self.iterations = 0
self.schedule = RandomActivation(self)
self.grid = Grid(width, height, torus=False)
self.height = height
self.width = width
self.running = True
self.previous_day_jailed_count = 0
self.previous_day_pictures_count = 0
self.previous_day_cops_count = self.initial_num_cops
self.jailed_count = 0
self.pictures_count = 0
self.cops_count = self.initial_num_cops
# Set such that when cops/agents are 2:1, the perceived arrest chance is 0.9
self.arrest_delay = arrest_delay
self.arrest_constant = 1.15
self.jail = [] # stores jailed agents
self.jail_time = jail_time # Represents "harshness" of current regime.
if not (hardcore_density + hanger_on_density + observer_density == 1):
raise ConfigError("Protestor densities must add up to 1")
if self.initial_num_cops + initial_num_citizens + initial_num_media > (
height * width):
raise ConfigError("Too many humans for the given grid")
self.total_fights = 0
self.total_jailed = 0
self.hours_without_protest = 0
self.hours_without_conflict = 0
self.datacollector = DataCollector(
model_reporters={
"Quiet":
lambda model: model.num_in_state("quiet"),
"Active":
lambda model: model.num_in_state("active"),
"Violent":
lambda model: model.num_in_state("violent"),
"Fighting":
lambda model: model.num_in_state("fighting"),
"Protesting":
lambda model: model.num_protesting(),
"Jailed":
lambda model: model.num_jailed(),
"Frustrated":
lambda model: model.num_frustrated(),
"Average legitimacy":
lambda model: model.average_legitimacy(),
"Average grievance":
lambda model: model.average_grievance(),
"Average ripeness":
lambda model: model.average_grievance() * model.num_in_state(
"quiet") / float(model.average_risk_aversion()),
"Cop count":
lambda model: model.num_cops(),
"Num pictures":
lambda model: model.num_pictures(),
"Total fights":
lambda model: model.total_fights,
"Total jailed":
lambda model: model.total_jailed,
"Protest waiting time":
lambda model: model.hours_without_protest,
"Conflict waiting time":
lambda model: model.hours_without_conflict,
},
agent_reporters={
"perceived_gain":
lambda agent: agent.perceived_gain()
if isinstance(agent, Citizen) else 0,
"net_risk_active":
lambda agent: agent.net_risk("active")
if isinstance(agent, Citizen) else 0,
"net_risk_violent":
lambda agent: agent.perceived_gain() - agent.net_risk(
"violent") if isinstance(agent, Citizen) else 0,
"act_utils":
lambda agent: agent.net_risk("violent")
if isinstance(agent, Citizen) else 0,
"threshold":
lambda agent: agent.threshold
if isinstance(agent, Citizen) else 0,
})
self.agent_regions = agent_regions
self.cop_regions = cop_regions
self.flag_regions = flag_regions
self.obstacle_regions = obstacle_regions
# Place objects
for position in co_ords_for_area(obstacle_regions):
self.grid[position[0]][position[1]] = Object("obstacle", position)
for position in co_ords_for_area(flag_regions):
self.grid[position[0]][position[1]] = Object("flag", position)
unique_id = 1
for cop_region in cop_regions:
frozen = cop_region["frozen"]
for position in co_ords_for_area([cop_region]):
self.add_cop(unique_id, frozen, position[0], position[1])
unique_id += 1
placed_media = 0
placed_citizens = 0
population = initial_num_media + initial_num_citizens
while (placed_media + placed_citizens) < population:
(x, y) = random.choice(co_ords_for_area(agent_regions))
if self.grid.is_cell_empty((x, y)):
seed = random.random()
# Optimised for adding citizens
if seed > (float(initial_num_media) / population):
if placed_citizens < initial_num_citizens:
self.add_citizen(unique_id, x, y)
placed_citizens += 1
else:
if placed_media < initial_num_media:
placed_media += 1
self.add_media(unique_id, x, y)
unique_id += 1