def mpld3ify(fig, sanitize=True, jsonify=True):
''' Do all the processing steps that might be necessary to render a figure displayable '''
import mpld3 # Do this here since currently a bug that sets the wrong backend
# Nothing to do if already a string
if sc.isstring(fig):
return fig
# If it's empty or null, make a small figure
if not fig:
fig = pl.figure(figsize=(1, 1)) # Make a dummy plot since no legend
fig.add_subplot(111) # So axis commands work
fig.get_axes()[0].set_visible(False) # Turn off axes
# Convert to mpld3 -- this is the big step
if isinstance(fig, pl.Figure):
fig = mpld3.fig_to_dict(fig)
# Optionally do additional sanitization
if sanitize: fig = sc.sanitizejson(fig)
if jsonify and not sc.isstring(fig):
fig = json.dumps(fig) # Could cause problems to jsonify a string
return fig
def day(obj, *args, start_day=None):
'''
Convert a string, date/datetime object, or int to a day (int), the number of
days since the start day. See also date() and daydiff(). Used primarily via
sim.day() rather than directly.
Args:
obj (str, date, int, or list): convert any of these objects to a day relative to the start day
args (list): additional days
start_day (str or date): the start day; if none is supplied, return days since 2020-01-01.
Returns:
days (int or str): the day(s) in simulation time
**Example**::
sim.day('2020-04-05') # Returns 35
'''
# Do not process a day if it's not supplied
if obj is None:
return None
if start_day is None:
start_day = '2020-01-01'
# Convert to list
if sc.isstring(obj) or sc.isnumber(obj) or isinstance(
obj, (dt.date, dt.datetime)):
obj = sc.promotetolist(obj) # Ensure it's iterable
elif isinstance(obj, np.ndarray):
obj = obj.tolist() # Convert to list if it's an array
obj.extend(args)
days = []
for d in obj:
if d is None:
days.append(d)
elif sc.isnumber(d):
days.append(int(d)) # Just convert to an integer
else:
try:
if sc.isstring(d):
d = sc.readdate(d).date()
elif isinstance(d, dt.datetime):
d = d.date()
d_day = (d - date(start_day)
).days # Heavy lifting -- actually compute the day
days.append(d_day)
except Exception as E:
errormsg = f'Could not interpret "{d}" as a date: {str(E)}'
raise ValueError(errormsg)
# Return an integer rather than a list if only one provided
if len(days) == 1:
days = days[0]
return days
def initialize(self, sim):
''' Fix days and store beta '''
if sc.isstring(self.days) or not sc.isiterable(self.days):
self.days = sc.promotetolist(self.days)
if isinstance(self.days, list):
for d, day in enumerate(self.days):
self.days[d] = sim.day(
day
) # Ensure it's an integer and not a string or something
self.days = sc.promotetoarray(self.days)
self.changes = sc.promotetoarray(self.changes)
if len(self.days) != len(self.changes):
errormsg = f'Number of days supplied ({len(self.days)}) does not match number of changes in beta ({len(self.changes)})'
raise ValueError(errormsg)
self.orig_betas = {}
self.layers = sc.promotetolist(self.layers, keepnone=True)
for lkey in self.layers:
if lkey is None:
self.orig_betas['overall'] = sim['beta']
else:
self.orig_betas[lkey] = sim['beta_layer'][lkey]
self.initialized = True
return
def validate_pars(self):
''' Some parameters can take multiple types; this makes them consistent '''
# Handle start day
start_day = self['start_day'] # Shorten
if start_day in [None, 0]: # Use default start day
start_day = dt.date(2020, 1, 1)
elif sc.isstring(start_day):
start_day = sc.readdate(start_day)
if isinstance(start_day,dt.datetime):
start_day = start_day.date()
self['start_day'] = start_day
# Handle contacts
contacts = self['contacts']
if sc.isnumber(contacts): # It's a scalar instead of a dict, assume it's all contacts
self['contacts'] = {'a':contacts}
self['beta_layers'] = {'a':1.0}
# Handle population data
popdata_choices = ['random', 'microstructure', 'synthpops']
if sc.isnumber(self['pop_type']) or isinstance(self['pop_type'], bool): # Convert e.g. pop_type=1 to 'microstructure'
self['pop_type'] = popdata_choices[int(self['pop_type'])] # Choose one of these
if self['pop_type'] not in popdata_choices:
choice = self['pop_type']
choicestr = ', '.join(popdata_choices)
errormsg = f'Population type "{choice}" not available; choices are: {choicestr}'
raise ValueError(errormsg)
# Handle interventions
self['interventions'] = sc.promotetolist(self['interventions'], keepnone=False)
return
def get_quar_inds(quar_policy, sim):
'''
Helper function to return the appropriate indices for people in quarantine
based on the current quarantine testing "policy". Used by test_num and test_prob.
Not for use by the user.
If quar_policy is a number or a list of numbers, then it is interpreted as
the number of days after the start of quarantine when a test is performed.
It can also be a function that returns the list of indices.
Args:
quar_policy (str, int, list, func): 'start', people entering quarantine; 'end', people leaving; 'both', entering and leaving; 'daily', every day in quarantine
sim (Sim): the simulation object
'''
t = sim.t
if quar_policy is None: quar_test_inds = np.array([])
elif quar_policy == 'start': quar_test_inds = cvu.true(sim.people.date_quarantined==t-1) # Actually do the day after since testing usually happens before contact tracing
elif quar_policy == 'end': quar_test_inds = cvu.true(sim.people.date_end_quarantine==t+1) # +1 since they are released on date_end_quarantine, so do the day before
elif quar_policy == 'both': quar_test_inds = np.concatenate([cvu.true(sim.people.date_quarantined==t-1), cvu.true(sim.people.date_end_quarantine==t+1)])
elif quar_policy == 'daily': quar_test_inds = cvu.true(sim.people.quarantined)
elif sc.isnumber(quar_policy) or (sc.isiterable(quar_policy) and not sc.isstring(quar_policy)):
quar_policy = sc.promotetoarray(quar_policy)
quar_test_inds = np.unique(np.concatenate([cvu.true(sim.people.date_quarantined==t-1-q) for q in quar_policy]))
elif callable(quar_policy):
quar_test_inds = quar_policy(sim)
else:
errormsg = f'Quarantine policy "{quar_policy}" not recognized: must be a string (start, end, both, daily), int, list, array, set, tuple, or function'
raise ValueError(errormsg)
return quar_test_inds
def date(obj, *args, **kwargs):
'''
Convert a string or a datetime object to a date object. To convert to an integer
from the start day, use sim.date() instead.
Args:
obj (str, date, datetime): the object to convert
args (str, date, datetime): additional objects to convert
Returns:
dates (date or list): either a single date object, or a list of them
**Examples**::
cv.date('2020-04-05') # Returns datetime.date(2020, 4, 5)
'''
# Convert to list
if sc.isstring(obj) or sc.isnumber(obj) or isinstance(
obj, (dt.date, dt.datetime)):
obj = sc.promotetolist(obj) # Ensure it's iterable
obj.extend(args)
dates = []
for d in obj:
try:
if type(
d
) == dt.date: # Do not use isinstance, since must be the exact type
pass
elif sc.isstring(d):
d = sc.readdate(d).date()
elif isinstance(d, dt.datetime):
d = d.date()
else:
errormsg = f'Could not interpret "{d}" of type {type(d)} as a date'
raise TypeError(errormsg)
dates.append(d)
except Exception as E:
errormsg = f'Conversion of "{d}" to a date failed: {str(E)}'
raise ValueError(errormsg)
# Return an integer rather than a list if only one provided
if len(dates) == 1:
dates = dates[0]
return dates
def day(self, day, *args):
'''
Convert a string, date/datetime object, or int to a day (int).
Args:
day (str, date, int, or list): convert any of these objects to a day relative to the simulation's start day
Returns:
days (int or str): the day(s) in simulation time
**Example**::
sim.day('2020-04-05') # Returns 35
'''
# Do not process a day if it's not supplied
if day is None:
return None
# Convert to list
if sc.isstring(day) or sc.isnumber(day) or isinstance(
day, (dt.date, dt.datetime)):
day = sc.promotetolist(day) # Ensure it's iterable
day.extend(args)
days = []
for d in day:
if sc.isnumber(d):
days.append(int(d)) # Just convert to an integer
else:
try:
if sc.isstring(d):
d = sc.readdate(d).date()
elif isinstance(d, dt.datetime):
d = d.date()
d_day = (d - self['start_day']).days
days.append(d_day)
except Exception as E:
errormsg = f'Could not interpret "{d}" as a date: {str(E)}'
raise ValueError(errormsg)
# Return an integer rather than a list if only one provided
if len(days) == 1:
days = days[0]
return days
def process_days(sim, days):
'''
Ensure lists of days are in consistent format. Used by change_beta, clip_edges,
and some analyzers. If day is 'end' or -1, use the final day of the simulation.
'''
if sc.isstring(days) or not sc.isiterable(days):
days = sc.promotetolist(days)
for d, day in enumerate(days):
if day in ['end', -1]:
day = sim['end_day']
days[d] = sim.day(
day) # Ensure it's an integer and not a string or something
days = sc.promotetoarray(days)
return days
def validate_pars(self):
''' Some parameters can take multiple types; this makes them consistent '''
# Handle types
for key in ['pop_size', 'pop_infected', 'pop_size', 'n_days']:
self[key] = int(self[key])
# Handle start day
start_day = self['start_day'] # Shorten
if start_day in [None, 0]: # Use default start day
start_day = dt.date(2020, 1, 1)
elif sc.isstring(start_day):
start_day = sc.readdate(start_day)
if isinstance(start_day, dt.datetime):
start_day = start_day.date()
self['start_day'] = start_day
# Handle contacts
contacts = self['contacts']
if sc.isnumber(
contacts
): # It's a scalar instead of a dict, assume it's all contacts
self['contacts'] = {'a': contacts}
# Handle key mismaches
beta_layer_keys = set(self.pars['beta_layer'].keys())
contacts_keys = set(self.pars['contacts'].keys())
quar_eff_keys = set(self.pars['quar_eff'].keys())
if not (beta_layer_keys == contacts_keys == quar_eff_keys):
errormsg = f'Layer parameters beta={beta_layer_keys}, contacts={contacts_keys}, quar_eff={quar_eff_keys} have inconsistent keys'
raise cvm.KeyNotFoundError(errormsg)
if self.people is not None:
pop_keys = set(self.people.contacts.keys())
if pop_keys != beta_layer_keys:
errormsg = f'Please update your parameter keys {beta_layer_keys} to match population keys {pop_keys}. You may find sim.reset_layer_pars() helpful.'
raise cvm.KeyNotFoundError(errormsg)
# Handle population data
popdata_choices = ['random', 'hybrid', 'clustered', 'synthpops']
choice = self['pop_type']
if choice not in popdata_choices:
choicestr = ', '.join(popdata_choices)
errormsg = f'Population type "{choice}" not available; choices are: {choicestr}'
raise cvm.KeyNotFoundError(errormsg)
# Handle interventions
self['interventions'] = sc.promotetolist(self['interventions'],
keepnone=False)
return
def process_days(sim, days, return_dates=False):
'''
Ensure lists of days are in consistent format. Used by change_beta, clip_edges,
and some analyzers. If day is 'end' or -1, use the final day of the simulation.
Optionally return dates as well as days.
'''
if sc.isstring(days) or not sc.isiterable(days):
days = sc.promotetolist(days)
for d,day in enumerate(days):
if day in ['end', -1]:
day = sim['end_day']
days[d] = sim.day(day) # Ensure it's an integer and not a string or something
days = np.sort(sc.promotetoarray(days)) # Ensure they're an array and in order
if return_dates:
dates = [sim.date(day) for day in days] # Store as date strings
return days, dates
else:
return days
def process_days_changes(sim, days, changes):
'''
Ensure lists of days and lists of changes are in consistent format. Used by
change_beta and clip_edges.
'''
if sc.isstring(days) or not sc.isiterable(days):
days = sc.promotetolist(days)
if isinstance(days, list):
for d, day in enumerate(days):
days[d] = sim.day(
day) # Ensure it's an integer and not a string or something
days = sc.promotetoarray(days)
changes = sc.promotetoarray(changes)
if len(days) != len(changes):
errormsg = f'Number of days supplied ({len(days)}) does not match number of changes in beta ({len(changes)})'
raise ValueError(errormsg)
return days, changes
def validate_pars(self):
''' Some parameters can take multiple types; this makes them consistent '''
# Handle start day
start_day = self['start_day'] # Shorten
if start_day in [None, 0]: # Use default start day
start_day = dt.date(2020, 1, 1)
elif sc.isstring(start_day):
start_day = sc.readdate(start_day)
if isinstance(start_day, dt.datetime):
start_day = start_day.date()
self['start_day'] = start_day
# Handle contacts
contacts = self['contacts']
if sc.isnumber(
contacts
): # It's a scalar instead of a dict, assume it's all contacts
self['contacts'] = {'a': contacts}
# Handle key mismaches
beta_layer_keys = set(self.pars['beta_layer'].keys())
contacts_keys = set(self.pars['contacts'].keys())
quar_eff_keys = set(self.pars['quar_eff'].keys())
if not (beta_layer_keys == contacts_keys == quar_eff_keys):
errormsg = f'Layer parameters beta={beta_layer_keys}, contacts={contacts_keys}, quar_eff={quar_eff_keys} are not consistent'
raise ValueError(errormsg)
# Handle population data
popdata_choices = ['random', 'hybrid', 'clustered', 'synthpops']
choice = self['pop_type']
if choice not in popdata_choices:
choicestr = ', '.join(popdata_choices)
errormsg = f'Population type "{choice}" not available; choices are: {choicestr}'
raise ValueError(errormsg)
# Handle interventions
self['interventions'] = sc.promotetolist(self['interventions'],
keepnone=False)
return
def initialize(self, sim):
# Handle days
self.start_day = cvm.date(sim['start_day'], as_date=False) # Get the start day, as a string
self.end_day = cvm.date(sim['end_day'], as_date=False) # Get the start day, as a string
if self.days is None:
self.days = self.end_day # If no day is supplied, use the last day
self.days = cvi.process_days(sim, self.days) # Ensure days are in the right format
self.days = np.sort(self.days) # Ensure they're in order
self.dates = [sim.date(day) for day in self.days] # Store as date strings
max_hist_day = self.days[-1]
max_sim_day = sim.day(self.end_day)
if max_hist_day > max_sim_day:
errormsg = f'Cannot create histogram for {self.dates[-1]} (day {max_hist_day}) because the simulation ends on {self.end_day} (day {max_sim_day})'
raise ValueError(errormsg)
# Handle edges and age bins
if self.edges is None: # Default age bins
self.edges = np.linspace(0,100,11)
self.bins = self.edges[:-1] # Don't include the last edge in the bins
# Handle states
if self.states is None:
self.states = ['exposed', 'dead', 'tested', 'diagnosed']
self.states = sc.promotetolist(self.states)
for s,state in enumerate(self.states):
self.states[s] = state.replace('date_', '') # Allow keys starting with date_ as input, but strip it off here
# Handle the data file
if self.datafile is not None:
if sc.isstring(self.datafile):
self.data = cvm.load_data(self.datafile, check_date=False)
else:
self.data = self.datafile # Use it directly
self.datafile = None
self.initialized = True
return
def sanitize(vals, forcefloat=False, verbose=True, allowstrings=True):
''' Make sure values are numeric, and either return nans or skip vals that aren't -- WARNING, duplicates lots of other things!'''
if verbose: print('Sanitizing vals of %s: %s' % (type(vals), vals))
if isinstance(vals, list):
as_array = False if forcefloat else True
else:
vals = [vals]
as_array = False
output = []
for val in vals:
if val in [None, '']:
sanival = ''
else:
try:
sanival = val
factor = 1.0
if sc.isstring(sanival):
sanival = sanival.replace(',',
'') # Remove commas, if present
sanival = sanival.replace('$',
'') # Remove dollars, if present
# if val.endswith('%'): factor = 0.01 # Scale if percentage has been used -- CK: not used since already converted from percentage
sanival = float(sanival) * factor
except Exception as E:
if allowstrings:
print('Allowing string "%s" to pass (%s)' % (val, repr(E)))
sanival = str(val)
else:
print('Could not sanitize value "%s": %s; returning nan' %
(val, repr(E)))
sanival = np.nan
output.append(sanival)
if as_array:
return output
else:
return output[0]
def _get_ia(self, which, label=None, partial=False, as_list=False, as_inds=False, die=True, first=False):
''' Helper method for get_interventions() and get_analyzers(); see get_interventions() docstring '''
# Handle inputs
if which not in ['interventions', 'analyzers']:
errormsg = f'This method is only defined for interventions and analyzers, not "{which}"'
raise ValueError(errormsg)
ia_list = self.pars[which] # List of interventions or analyzers
n_ia = len(ia_list) # Number of interventions/analyzers
if label == 'summary': # Print a summary of the interventions
df = pd.DataFrame(columns=['ind', 'label', 'type'])
for ind,ia_obj in enumerate(ia_list):
df = df.append(dict(ind=ind, label=str(ia_obj.label), type=type(ia_obj)), ignore_index=True)
print(f'Summary of {which}:')
print(df)
return
else: # Standard usage case
position = 0 if first else -1 # Choose either the first or last element
if label is None:
label = position # Get the last element
labels = sc.promotetolist(label)
# Calculate the matches
matches = []
match_inds = []
for label in labels:
if sc.isnumber(label):
matches.append(ia_list[label]) # This will raise an exception if an invalid index is given
label = n_ia + label if label<0 else label # Convert to a positive number
match_inds.append(label)
elif sc.isstring(label) or isinstance(label, type):
for ind,ia_obj in enumerate(ia_list):
if sc.isstring(label) and ia_obj.label == label or (partial and (label in str(ia_obj.label))):
matches.append(ia_obj)
match_inds.append(ind)
elif isinstance(label, type) and isinstance(ia_obj, label):
matches.append(ia_obj)
match_inds.append(ind)
else:
errormsg = f'Could not interpret label type "{type(label)}": should be str, int, or {which} class'
raise TypeError(errormsg)
# Parse the output options
if as_inds:
output = match_inds
elif as_list:
output = matches
else: # Normal case, return actual interventions
if len(matches) == 0:
if die:
errormsg = f'No {which} matching "{label}" were found'
raise ValueError(errormsg)
else:
output = None
else:
output = matches[position] # Return either the first or last match
return output
def _do_RPC(self, verbose=False):
# Check to see whether the RPC is getting passed in in request.form.
# If so, we are doing an upload, and we want to download the RPC
# request info from the form, not request.data.
if 'funcname' in request.form: # Pull out the function name, args, and kwargs
fn_name = request.form.get('funcname')
try:
args = json.loads(request.form.get('args', "[]"),
object_pairs_hook=OrderedDict)
except:
args = [] # May or may not be present
try:
kwargs = json.loads(request.form.get('kwargs', "{}"),
object_pairs_hook=OrderedDict)
except:
kwargs = {} # May or may not be present
else: # Otherwise, we have a normal or download RPC, which means we pull the RPC request info from request.data.
reqdict = json.loads(request.data, object_pairs_hook=OrderedDict)
fn_name = reqdict['funcname']
args = reqdict.get('args', [])
kwargs = reqdict.get('kwargs', {})
if verbose:
print('RPC(): RPC properties:')
print(' fn_name: %s' % fn_name)
print(' args: %s' % args)
print(' kwargs: %s' % kwargs)
# If the function name is not in the RPC dictionary, return an error.
if not sc.isstring(fn_name):
return robustjsonify(
{'error': 'Invalid RPC - must be a string (%s)' % fn_name})
if not fn_name in self.RPC_dict:
return robustjsonify(
{'error': 'Could not find requested RPC "%s"' % fn_name})
found_RPC = self.RPC_dict[fn_name] # Get the RPC we've found.
## Do any validation checks we need to do and return errors if they don't pass.
# If the RPC is disabled, always return a Status 403 (Forbidden)
if found_RPC.validation == 'disabled':
if verbose: print('RPC(): RPC disabled')
abort(403)
# Only do other validation if DataStore and users are included -- NOTE: Any "unknown" validation values are treated like 'none'.
if self.config['USE_DATASTORE'] and self.config['USE_USERS']:
if found_RPC.validation == 'any' and not (
current_user.is_anonymous
or current_user.is_authenticated):
abort(
401
) # If the RPC should be executable by any user, including an anonymous one, but there is no authorization or anonymous login, return a Status 401 (Unauthorized)
elif found_RPC.validation == 'named' and (
current_user.is_anonymous
or not current_user.is_authenticated):
abort(
401
) # Else if the RPC should be executable by any non-anonymous user, but there is no authorization or there is an anonymous login, return a Status 401 (Unauthorized)
elif found_RPC.validation == 'admin': # Else if the RPC should be executable by any admin user, but there is no admin login or it's an anonymous login...
if current_user.is_anonymous or not current_user.is_authenticated:
abort(
401
) # If the user is anonymous or no authenticated user is logged in, return Status 401 (Unauthorized).
elif not current_user.is_admin:
abort(
403
) # Else, if the user is not an admin user, return Status 403 (Forbidden).
# If we are doing an upload...
if found_RPC.call_type == 'upload':
if verbose: print('Starting upload...')
thisfile = request.files[
'uploadfile'] # Grab the formData file that was uploaded.
filename = secure_filename(
thisfile.filename
) # Extract a sanitized filename from the one we start with.
try:
uploaded_fname = os.path.join(
self.datastore.tempfolder,
filename) # Generate a full upload path/file name.
except Exception as E:
errormsg = 'Could not create filename for uploaded file: %s' % str(
E)
raise Exception(errormsg)
try:
thisfile.save(
uploaded_fname) # Save the file to the uploads directory
except Exception as E:
errormsg = 'Could not save uploaded file: %s' % str(E)
raise Exception(errormsg)
args.insert(
0, uploaded_fname) # Prepend the file name to the args list.
# Show the call of the function.
callcolor = ['cyan', 'bgblue']
successcolor = ['green', 'bgblue']
failcolor = ['gray', 'bgred']
timestr = '[%s]' % sc.now(astype='str')
try:
userstr = ' <%s>' % current_user.username
except:
userstr = ' <no user>'
RPCinfo = sc.objdict({
'time': timestr,
'user': userstr,
'module': found_RPC.call_func.__module__,
'name': found_RPC.funcname
})
if self.config['LOGGING_MODE'] == 'FULL':
string = '%s%s RPC called: "%s.%s"' % (
RPCinfo.time, RPCinfo.user, RPCinfo.module, RPCinfo.name)
sc.colorize(callcolor, string, enable=self.colorize)
# Execute the function to get the results, putting it in a try block in case there are errors in what's being called.
try:
if verbose: print('RPC(): Starting RPC...')
T = sc.tic()
result = found_RPC.call_func(*args, **kwargs)
if isinstance(
result, dict
) and 'error' in result: # If the RPC returns an error, return it
return robustjsonify({'error': result['error']})
elapsed = sc.toc(T, output=True)
if self.config['LOGGING_MODE'] == 'FULL':
string = '%s%s RPC finished in %0.2f s: "%s.%s"' % (
RPCinfo.time, RPCinfo.user, elapsed, RPCinfo.module,
RPCinfo.name)
sc.colorize(successcolor, string, enable=self.colorize)
except Exception as E:
if verbose: print('RPC(): Exception encountered...')
shortmsg = str(E)
exception = traceback.format_exc() # Grab the trackback stack
hostname = '|%s| ' % socket.gethostname()
tracemsg = '%s%s%s Exception during RPC "%s.%s" \nRequest: %s \n%.10000s' % (
hostname, RPCinfo.time, RPCinfo.user, RPCinfo.module,
RPCinfo.name, request, exception)
sc.colorize(
failcolor, tracemsg, enable=self.colorize
) # Post an error to the Flask logger limiting the exception information to 10000 characters maximum (to prevent monstrous sqlalchemy outputs)
if self.config['SLACK']:
self.slacknotification(tracemsg)
if isinstance(
E, HTTPException
): # If we have a werkzeug exception, pass it on up to werkzeug to resolve and reply to.
raise E
code = 500 # Send back a response with status 500 that includes the exception traceback.
fullmsg = shortmsg + '\n\nException details:\n' + tracemsg
reply = {
'exception': fullmsg
} # NB, not sure how to actually access 'traceback' on the FE, but keeping it here for future
return make_response(robustjsonify(reply), code)
# If we are doing a download, prepare the response and send it off.
if found_RPC.call_type == 'download':
# To download a file, use `this.$sciris.download` instead of `this.$sciris.rpc`. Decorate the RPC with
# `@RPC(call_type='download')`. Finally, the RPC needs to specify the file and optionally the filename.
# This is done with tuple unpacking. The following outputs are supported from `rpc_function()`
#
# 1 - filename_on_disk
# 2 - BytesIO
# 3 - filename_on_disk, download_filename
# 4- BytesIO, download_filename
#
# Examples return values from the RPC are as follows
#
# 1 - "E:/test.xlsx" (uses "test.xlsx")
# 2 - <BytesIO> (default filename will be generated in this function)
# 3 - ("E:/test.xlsx","foo.xlsx")
# 4 - (<BytesIO>,"foo.xlsx")
#
# On the RPC end, the most common cases would be it might look like
#
# return "E:/test.xlsx"
#
# OR
#
# return Blobject.to_file(), "foo.xlsx"
if verbose: print('RPC(): Starting download...')
if result is None: # If we got None for a result (the full file name), return an error to the client.
return robustjsonify({
'error':
'Could not find resource to download from RPC "%s": result is None'
% fn_name
})
elif sc.isstring(result):
from_file = True
dir_name, file_name = os.path.split(result)
output_name = file_name
elif isinstance(result, io.BytesIO):
from_file = False
bytesio = result
output_name = 'download.obj'
else:
try:
content = result[0]
output_name = result[1]
if sc.isstring(content):
from_file = True
dir_name, file_name = os.path.split(content)
elif isinstance(content, io.BytesIO):
from_file = False
bytesio = content
else:
return robustjsonify(
{'error': 'Unrecognized RPC output'})
except Exception as E:
return robustjsonify(
{'error': 'Error reading RPC result (%s)' % E})
if from_file:
response = send_from_directory(dir_name,
file_name,
as_attachment=True)
response.status_code = 201 # Status 201 = Created
# Unfortunately, we cannot remove the actual file at this point
# because it is in use during the actual download, so we rely on
# later cleanup to remove download files.
else:
response = send_file(bytesio,
as_attachment=True,
attachment_filename=output_name)
response.headers['filename'] = output_name
print(response)
return response # Return the response message.
# Otherwise (normal and upload RPCs),
else:
if found_RPC.call_type == 'upload': # If we are doing an upload....
try:
os.remove(
uploaded_fname
) # Erase the physical uploaded file, since it is no longer needed.
if verbose:
print('RPC(): Removed uploaded file: %s' %
uploaded_fname)
except Exception as E:
if verbose:
print('RPC(): Could not remove uploaded file: %s' %
str(E)) # Probably since moved by the user
if result is None: # If None was returned by the RPC function, return ''.
if verbose: print('RPC(): RPC finished, returning None')
return ''
else: # Otherwise, convert the result (probably a dict) to JSON and return it.
output = robustjsonify(result)
if verbose: print('RPC(): RPC finished, returning result')
return output
def datathief(filename,
xlim=None,
ylim=None,
xcol=None,
ycol=None,
dcol=None,
debug=False):
'''
Extract data from a figure. To use, add a single pixel at the start and end
of the x-axis (default color: pure blue) and the y-axis (default color: pure
red), and then one pixel for each data point (default color: pure green). This
function will then return the x and y coordinates of each data point.
Args:
filename (str): the filename of the image to load
xlim (list): a list of 2 floats defining the start and end values of the x-axis
ylim (list): likewise, for the y-axis
xcol (str, list): a hex string or list of 3 floats defining the RGB color for the x-axis pixels
ycol (str, list): likewise, for the y-axis pixels
dcol (str, list): likewise, for the data points
debug (bool): whether to print extra information about the data transformations
Returns:
Dict with keys x and y corresponding to the data points
**Examples**::
import datathief as dt
import pylab as pl
data1 = dt.datathief('my-data-fig.png', xlim=[0,20], ylim=[0,30])
data2 = dt.datathief('my-other-fig.png', xlim=[0,1], ylim=[0,1], xcol='#f00', ycol='#f0f', dcol='#face00')
pl.scatter(data2.x, data2.y)
Inspired by the DataThief Java tool.
'''
if xlim is None: xlim = [0, 1]
if ylim is None: ylim = [0, 1]
if xcol is None: xcol = [0, 0, 1]
if ycol is None: ycol = [1, 0, 0]
if dcol is None: dcol = [0, 1, 0]
if sc.isstring(xcol): xcol = sc.hex2rgb(xcol)
if sc.isstring(ycol): xcol = sc.hex2rgb(ycol)
if sc.isstring(dcol): xcol = sc.hex2rgb(dcol)
# Read image data
lim = sc.objdict(x=xlim, y=ylim)
perpix = sc.objdict()
ref = sc.objdict()
d = sc.objdict()
img = mpl.image.imread(filename)
ref.x, tmp_xy = findpixels(img, xcol)
tmp_yx, ref.y = findpixels(img, ycol)
d.x, d.y = findpixels(img, dcol)
ref.y = np.sort(img.shape[0] - ref.y) # Flip y-axis
d.y = img.shape[0] - d.y # Flip y-axis
assert len(
ref.x
) == 2, f'Wrong number of x coordinates found ({len(ref.x)}): please ensure exactly 2 pixels have color {xcol}'
assert len(
ref.y
) == 2, f'Wrong number of y coordinates found ({len(ref.y)}): please ensure exactly 2 pixels have color {ycol}'
if debug:
print(f'Image shape: {img.shape}')
print(f'Reference pixels: {ref}')
print(f'Data pixels: {d}')
# Process data
data = sc.objdict()
order = np.argsort(d.x)
d.x = d.x[order]
d.y = d.y[order]
for k in ['x', 'y']:
perpix = (np.diff(lim[k]) / np.diff(ref[k]))[0]
orig = np.array(d[k], dtype=float)
rmref = orig - ref[k][0]
bypix = rmref * perpix
data[k] = bypix + lim[k][0]
if debug:
print('\n\nFor variable:\n ', k)
print(f'Pixels-per-value: {1.0/perpix}; pixel ratio: {perpix}')
print(f'Original:\n orig={orig}')
print(f'Removing reference:\n rmref={rmref}')
print(f'By pixel:\n bypix={bypix}')
print(f'Adding limit:\n data={data[k]}')
return data
def date(obj, *args, start_date=None, dateformat=None, as_date=True):
'''
Convert a string or a datetime object to a date object. To convert to an integer
from the start day, it is recommended you supply a start date, or use sim.date()
instead; otherwise, it will calculate the date counting days from 2020-01-01.
This means that the output of cv.date() will not necessarily match the output
of sim.date() for an integer input.
Args:
obj (str, date, datetime, list, array): the object to convert
args (str, date, datetime): additional objects to convert
start_date (str, date, datetime): the starting date, if an integer is supplied
dateformat (str): the format to return the date in
as_date (bool): whether to return as a datetime date instead of a string
Returns:
dates (date or list): either a single date object, or a list of them
**Examples**::
cv.date('2020-04-05') # Returns datetime.date(2020, 4, 5)
cv.date('2020-04-14', start_date='2020-04-04', as_date=False) # Returns 10
cv.date([35,36,37], as_date=False) # Returns ['2020-02-05', '2020-02-06', '2020-02-07']
'''
if obj is None:
return None
# Convert to list and handle other inputs
if isinstance(obj, np.ndarray):
obj = obj.tolist() # If it's an array, convert to a list
obj = sc.promotetolist(obj) # Ensure it's iterable
obj.extend(args)
if dateformat is None:
dateformat = '%Y-%m-%d'
if start_date is None:
start_date = '2020-01-01'
dates = []
for d in obj:
if d is None:
dates.append(d)
continue
try:
if type(
d
) == dt.date: # Do not use isinstance, since must be the exact type
pass
elif sc.isstring(d):
d = sc.readdate(d).date()
elif isinstance(d, dt.datetime):
d = d.date()
elif sc.isnumber(d):
if start_date is None:
errormsg = f'To convert the number {d} to a date, you must supply start_date'
raise ValueError(errormsg)
d = date(start_date) + dt.timedelta(days=int(d))
else:
errormsg = f'Cannot interpret {type(d)} as a date, must be date, datetime, or string'
raise TypeError(errormsg)
if as_date:
dates.append(d)
else:
dates.append(d.strftime(dateformat))
except Exception as E:
errormsg = f'Conversion of "{d}" to a date failed: {str(E)}'
raise ValueError(errormsg)
# Return an integer rather than a list if only one provided
if len(dates) == 1:
dates = dates[0]
return dates
