def mpld3ify(fig, sanitize=True, jsonify=True): ''' Do all the processing steps that might be necessary to render a figure displayable ''' import mpld3 # Do this here since currently a bug that sets the wrong backend # Nothing to do if already a string if sc.isstring(fig): return fig # If it's empty or null, make a small figure if not fig: fig = pl.figure(figsize=(1, 1)) # Make a dummy plot since no legend fig.add_subplot(111) # So axis commands work fig.get_axes()[0].set_visible(False) # Turn off axes # Convert to mpld3 -- this is the big step if isinstance(fig, pl.Figure): fig = mpld3.fig_to_dict(fig) # Optionally do additional sanitization if sanitize: fig = sc.sanitizejson(fig) if jsonify and not sc.isstring(fig): fig = json.dumps(fig) # Could cause problems to jsonify a string return fig
def day(obj, *args, start_day=None): ''' Convert a string, date/datetime object, or int to a day (int), the number of days since the start day. See also date() and daydiff(). Used primarily via sim.day() rather than directly. Args: obj (str, date, int, or list): convert any of these objects to a day relative to the start day args (list): additional days start_day (str or date): the start day; if none is supplied, return days since 2020-01-01. Returns: days (int or str): the day(s) in simulation time **Example**:: sim.day('2020-04-05') # Returns 35 ''' # Do not process a day if it's not supplied if obj is None: return None if start_day is None: start_day = '2020-01-01' # Convert to list if sc.isstring(obj) or sc.isnumber(obj) or isinstance( obj, (dt.date, dt.datetime)): obj = sc.promotetolist(obj) # Ensure it's iterable elif isinstance(obj, np.ndarray): obj = obj.tolist() # Convert to list if it's an array obj.extend(args) days = [] for d in obj: if d is None: days.append(d) elif sc.isnumber(d): days.append(int(d)) # Just convert to an integer else: try: if sc.isstring(d): d = sc.readdate(d).date() elif isinstance(d, dt.datetime): d = d.date() d_day = (d - date(start_day) ).days # Heavy lifting -- actually compute the day days.append(d_day) except Exception as E: errormsg = f'Could not interpret "{d}" as a date: {str(E)}' raise ValueError(errormsg) # Return an integer rather than a list if only one provided if len(days) == 1: days = days[0] return days
def initialize(self, sim): ''' Fix days and store beta ''' if sc.isstring(self.days) or not sc.isiterable(self.days): self.days = sc.promotetolist(self.days) if isinstance(self.days, list): for d, day in enumerate(self.days): self.days[d] = sim.day( day ) # Ensure it's an integer and not a string or something self.days = sc.promotetoarray(self.days) self.changes = sc.promotetoarray(self.changes) if len(self.days) != len(self.changes): errormsg = f'Number of days supplied ({len(self.days)}) does not match number of changes in beta ({len(self.changes)})' raise ValueError(errormsg) self.orig_betas = {} self.layers = sc.promotetolist(self.layers, keepnone=True) for lkey in self.layers: if lkey is None: self.orig_betas['overall'] = sim['beta'] else: self.orig_betas[lkey] = sim['beta_layer'][lkey] self.initialized = True return
def validate_pars(self): ''' Some parameters can take multiple types; this makes them consistent ''' # Handle start day start_day = self['start_day'] # Shorten if start_day in [None, 0]: # Use default start day start_day = dt.date(2020, 1, 1) elif sc.isstring(start_day): start_day = sc.readdate(start_day) if isinstance(start_day,dt.datetime): start_day = start_day.date() self['start_day'] = start_day # Handle contacts contacts = self['contacts'] if sc.isnumber(contacts): # It's a scalar instead of a dict, assume it's all contacts self['contacts'] = {'a':contacts} self['beta_layers'] = {'a':1.0} # Handle population data popdata_choices = ['random', 'microstructure', 'synthpops'] if sc.isnumber(self['pop_type']) or isinstance(self['pop_type'], bool): # Convert e.g. pop_type=1 to 'microstructure' self['pop_type'] = popdata_choices[int(self['pop_type'])] # Choose one of these if self['pop_type'] not in popdata_choices: choice = self['pop_type'] choicestr = ', '.join(popdata_choices) errormsg = f'Population type "{choice}" not available; choices are: {choicestr}' raise ValueError(errormsg) # Handle interventions self['interventions'] = sc.promotetolist(self['interventions'], keepnone=False) return
def get_quar_inds(quar_policy, sim): ''' Helper function to return the appropriate indices for people in quarantine based on the current quarantine testing "policy". Used by test_num and test_prob. Not for use by the user. If quar_policy is a number or a list of numbers, then it is interpreted as the number of days after the start of quarantine when a test is performed. It can also be a function that returns the list of indices. Args: quar_policy (str, int, list, func): 'start', people entering quarantine; 'end', people leaving; 'both', entering and leaving; 'daily', every day in quarantine sim (Sim): the simulation object ''' t = sim.t if quar_policy is None: quar_test_inds = np.array([]) elif quar_policy == 'start': quar_test_inds = cvu.true(sim.people.date_quarantined==t-1) # Actually do the day after since testing usually happens before contact tracing elif quar_policy == 'end': quar_test_inds = cvu.true(sim.people.date_end_quarantine==t+1) # +1 since they are released on date_end_quarantine, so do the day before elif quar_policy == 'both': quar_test_inds = np.concatenate([cvu.true(sim.people.date_quarantined==t-1), cvu.true(sim.people.date_end_quarantine==t+1)]) elif quar_policy == 'daily': quar_test_inds = cvu.true(sim.people.quarantined) elif sc.isnumber(quar_policy) or (sc.isiterable(quar_policy) and not sc.isstring(quar_policy)): quar_policy = sc.promotetoarray(quar_policy) quar_test_inds = np.unique(np.concatenate([cvu.true(sim.people.date_quarantined==t-1-q) for q in quar_policy])) elif callable(quar_policy): quar_test_inds = quar_policy(sim) else: errormsg = f'Quarantine policy "{quar_policy}" not recognized: must be a string (start, end, both, daily), int, list, array, set, tuple, or function' raise ValueError(errormsg) return quar_test_inds
def date(obj, *args, **kwargs): ''' Convert a string or a datetime object to a date object. To convert to an integer from the start day, use sim.date() instead. Args: obj (str, date, datetime): the object to convert args (str, date, datetime): additional objects to convert Returns: dates (date or list): either a single date object, or a list of them **Examples**:: cv.date('2020-04-05') # Returns datetime.date(2020, 4, 5) ''' # Convert to list if sc.isstring(obj) or sc.isnumber(obj) or isinstance( obj, (dt.date, dt.datetime)): obj = sc.promotetolist(obj) # Ensure it's iterable obj.extend(args) dates = [] for d in obj: try: if type( d ) == dt.date: # Do not use isinstance, since must be the exact type pass elif sc.isstring(d): d = sc.readdate(d).date() elif isinstance(d, dt.datetime): d = d.date() else: errormsg = f'Could not interpret "{d}" of type {type(d)} as a date' raise TypeError(errormsg) dates.append(d) except Exception as E: errormsg = f'Conversion of "{d}" to a date failed: {str(E)}' raise ValueError(errormsg) # Return an integer rather than a list if only one provided if len(dates) == 1: dates = dates[0] return dates
def day(self, day, *args): ''' Convert a string, date/datetime object, or int to a day (int). Args: day (str, date, int, or list): convert any of these objects to a day relative to the simulation's start day Returns: days (int or str): the day(s) in simulation time **Example**:: sim.day('2020-04-05') # Returns 35 ''' # Do not process a day if it's not supplied if day is None: return None # Convert to list if sc.isstring(day) or sc.isnumber(day) or isinstance( day, (dt.date, dt.datetime)): day = sc.promotetolist(day) # Ensure it's iterable day.extend(args) days = [] for d in day: if sc.isnumber(d): days.append(int(d)) # Just convert to an integer else: try: if sc.isstring(d): d = sc.readdate(d).date() elif isinstance(d, dt.datetime): d = d.date() d_day = (d - self['start_day']).days days.append(d_day) except Exception as E: errormsg = f'Could not interpret "{d}" as a date: {str(E)}' raise ValueError(errormsg) # Return an integer rather than a list if only one provided if len(days) == 1: days = days[0] return days
def process_days(sim, days): ''' Ensure lists of days are in consistent format. Used by change_beta, clip_edges, and some analyzers. If day is 'end' or -1, use the final day of the simulation. ''' if sc.isstring(days) or not sc.isiterable(days): days = sc.promotetolist(days) for d, day in enumerate(days): if day in ['end', -1]: day = sim['end_day'] days[d] = sim.day( day) # Ensure it's an integer and not a string or something days = sc.promotetoarray(days) return days
def validate_pars(self): ''' Some parameters can take multiple types; this makes them consistent ''' # Handle types for key in ['pop_size', 'pop_infected', 'pop_size', 'n_days']: self[key] = int(self[key]) # Handle start day start_day = self['start_day'] # Shorten if start_day in [None, 0]: # Use default start day start_day = dt.date(2020, 1, 1) elif sc.isstring(start_day): start_day = sc.readdate(start_day) if isinstance(start_day, dt.datetime): start_day = start_day.date() self['start_day'] = start_day # Handle contacts contacts = self['contacts'] if sc.isnumber( contacts ): # It's a scalar instead of a dict, assume it's all contacts self['contacts'] = {'a': contacts} # Handle key mismaches beta_layer_keys = set(self.pars['beta_layer'].keys()) contacts_keys = set(self.pars['contacts'].keys()) quar_eff_keys = set(self.pars['quar_eff'].keys()) if not (beta_layer_keys == contacts_keys == quar_eff_keys): errormsg = f'Layer parameters beta={beta_layer_keys}, contacts={contacts_keys}, quar_eff={quar_eff_keys} have inconsistent keys' raise cvm.KeyNotFoundError(errormsg) if self.people is not None: pop_keys = set(self.people.contacts.keys()) if pop_keys != beta_layer_keys: errormsg = f'Please update your parameter keys {beta_layer_keys} to match population keys {pop_keys}. You may find sim.reset_layer_pars() helpful.' raise cvm.KeyNotFoundError(errormsg) # Handle population data popdata_choices = ['random', 'hybrid', 'clustered', 'synthpops'] choice = self['pop_type'] if choice not in popdata_choices: choicestr = ', '.join(popdata_choices) errormsg = f'Population type "{choice}" not available; choices are: {choicestr}' raise cvm.KeyNotFoundError(errormsg) # Handle interventions self['interventions'] = sc.promotetolist(self['interventions'], keepnone=False) return
def process_days(sim, days, return_dates=False): ''' Ensure lists of days are in consistent format. Used by change_beta, clip_edges, and some analyzers. If day is 'end' or -1, use the final day of the simulation. Optionally return dates as well as days. ''' if sc.isstring(days) or not sc.isiterable(days): days = sc.promotetolist(days) for d,day in enumerate(days): if day in ['end', -1]: day = sim['end_day'] days[d] = sim.day(day) # Ensure it's an integer and not a string or something days = np.sort(sc.promotetoarray(days)) # Ensure they're an array and in order if return_dates: dates = [sim.date(day) for day in days] # Store as date strings return days, dates else: return days
def process_days_changes(sim, days, changes): ''' Ensure lists of days and lists of changes are in consistent format. Used by change_beta and clip_edges. ''' if sc.isstring(days) or not sc.isiterable(days): days = sc.promotetolist(days) if isinstance(days, list): for d, day in enumerate(days): days[d] = sim.day( day) # Ensure it's an integer and not a string or something days = sc.promotetoarray(days) changes = sc.promotetoarray(changes) if len(days) != len(changes): errormsg = f'Number of days supplied ({len(days)}) does not match number of changes in beta ({len(changes)})' raise ValueError(errormsg) return days, changes
def validate_pars(self): ''' Some parameters can take multiple types; this makes them consistent ''' # Handle start day start_day = self['start_day'] # Shorten if start_day in [None, 0]: # Use default start day start_day = dt.date(2020, 1, 1) elif sc.isstring(start_day): start_day = sc.readdate(start_day) if isinstance(start_day, dt.datetime): start_day = start_day.date() self['start_day'] = start_day # Handle contacts contacts = self['contacts'] if sc.isnumber( contacts ): # It's a scalar instead of a dict, assume it's all contacts self['contacts'] = {'a': contacts} # Handle key mismaches beta_layer_keys = set(self.pars['beta_layer'].keys()) contacts_keys = set(self.pars['contacts'].keys()) quar_eff_keys = set(self.pars['quar_eff'].keys()) if not (beta_layer_keys == contacts_keys == quar_eff_keys): errormsg = f'Layer parameters beta={beta_layer_keys}, contacts={contacts_keys}, quar_eff={quar_eff_keys} are not consistent' raise ValueError(errormsg) # Handle population data popdata_choices = ['random', 'hybrid', 'clustered', 'synthpops'] choice = self['pop_type'] if choice not in popdata_choices: choicestr = ', '.join(popdata_choices) errormsg = f'Population type "{choice}" not available; choices are: {choicestr}' raise ValueError(errormsg) # Handle interventions self['interventions'] = sc.promotetolist(self['interventions'], keepnone=False) return
def initialize(self, sim): # Handle days self.start_day = cvm.date(sim['start_day'], as_date=False) # Get the start day, as a string self.end_day = cvm.date(sim['end_day'], as_date=False) # Get the start day, as a string if self.days is None: self.days = self.end_day # If no day is supplied, use the last day self.days = cvi.process_days(sim, self.days) # Ensure days are in the right format self.days = np.sort(self.days) # Ensure they're in order self.dates = [sim.date(day) for day in self.days] # Store as date strings max_hist_day = self.days[-1] max_sim_day = sim.day(self.end_day) if max_hist_day > max_sim_day: errormsg = f'Cannot create histogram for {self.dates[-1]} (day {max_hist_day}) because the simulation ends on {self.end_day} (day {max_sim_day})' raise ValueError(errormsg) # Handle edges and age bins if self.edges is None: # Default age bins self.edges = np.linspace(0,100,11) self.bins = self.edges[:-1] # Don't include the last edge in the bins # Handle states if self.states is None: self.states = ['exposed', 'dead', 'tested', 'diagnosed'] self.states = sc.promotetolist(self.states) for s,state in enumerate(self.states): self.states[s] = state.replace('date_', '') # Allow keys starting with date_ as input, but strip it off here # Handle the data file if self.datafile is not None: if sc.isstring(self.datafile): self.data = cvm.load_data(self.datafile, check_date=False) else: self.data = self.datafile # Use it directly self.datafile = None self.initialized = True return
def sanitize(vals, forcefloat=False, verbose=True, allowstrings=True): ''' Make sure values are numeric, and either return nans or skip vals that aren't -- WARNING, duplicates lots of other things!''' if verbose: print('Sanitizing vals of %s: %s' % (type(vals), vals)) if isinstance(vals, list): as_array = False if forcefloat else True else: vals = [vals] as_array = False output = [] for val in vals: if val in [None, '']: sanival = '' else: try: sanival = val factor = 1.0 if sc.isstring(sanival): sanival = sanival.replace(',', '') # Remove commas, if present sanival = sanival.replace('$', '') # Remove dollars, if present # if val.endswith('%'): factor = 0.01 # Scale if percentage has been used -- CK: not used since already converted from percentage sanival = float(sanival) * factor except Exception as E: if allowstrings: print('Allowing string "%s" to pass (%s)' % (val, repr(E))) sanival = str(val) else: print('Could not sanitize value "%s": %s; returning nan' % (val, repr(E))) sanival = np.nan output.append(sanival) if as_array: return output else: return output[0]
def _get_ia(self, which, label=None, partial=False, as_list=False, as_inds=False, die=True, first=False): ''' Helper method for get_interventions() and get_analyzers(); see get_interventions() docstring ''' # Handle inputs if which not in ['interventions', 'analyzers']: errormsg = f'This method is only defined for interventions and analyzers, not "{which}"' raise ValueError(errormsg) ia_list = self.pars[which] # List of interventions or analyzers n_ia = len(ia_list) # Number of interventions/analyzers if label == 'summary': # Print a summary of the interventions df = pd.DataFrame(columns=['ind', 'label', 'type']) for ind,ia_obj in enumerate(ia_list): df = df.append(dict(ind=ind, label=str(ia_obj.label), type=type(ia_obj)), ignore_index=True) print(f'Summary of {which}:') print(df) return else: # Standard usage case position = 0 if first else -1 # Choose either the first or last element if label is None: label = position # Get the last element labels = sc.promotetolist(label) # Calculate the matches matches = [] match_inds = [] for label in labels: if sc.isnumber(label): matches.append(ia_list[label]) # This will raise an exception if an invalid index is given label = n_ia + label if label<0 else label # Convert to a positive number match_inds.append(label) elif sc.isstring(label) or isinstance(label, type): for ind,ia_obj in enumerate(ia_list): if sc.isstring(label) and ia_obj.label == label or (partial and (label in str(ia_obj.label))): matches.append(ia_obj) match_inds.append(ind) elif isinstance(label, type) and isinstance(ia_obj, label): matches.append(ia_obj) match_inds.append(ind) else: errormsg = f'Could not interpret label type "{type(label)}": should be str, int, or {which} class' raise TypeError(errormsg) # Parse the output options if as_inds: output = match_inds elif as_list: output = matches else: # Normal case, return actual interventions if len(matches) == 0: if die: errormsg = f'No {which} matching "{label}" were found' raise ValueError(errormsg) else: output = None else: output = matches[position] # Return either the first or last match return output
def _do_RPC(self, verbose=False): # Check to see whether the RPC is getting passed in in request.form. # If so, we are doing an upload, and we want to download the RPC # request info from the form, not request.data. if 'funcname' in request.form: # Pull out the function name, args, and kwargs fn_name = request.form.get('funcname') try: args = json.loads(request.form.get('args', "[]"), object_pairs_hook=OrderedDict) except: args = [] # May or may not be present try: kwargs = json.loads(request.form.get('kwargs', "{}"), object_pairs_hook=OrderedDict) except: kwargs = {} # May or may not be present else: # Otherwise, we have a normal or download RPC, which means we pull the RPC request info from request.data. reqdict = json.loads(request.data, object_pairs_hook=OrderedDict) fn_name = reqdict['funcname'] args = reqdict.get('args', []) kwargs = reqdict.get('kwargs', {}) if verbose: print('RPC(): RPC properties:') print(' fn_name: %s' % fn_name) print(' args: %s' % args) print(' kwargs: %s' % kwargs) # If the function name is not in the RPC dictionary, return an error. if not sc.isstring(fn_name): return robustjsonify( {'error': 'Invalid RPC - must be a string (%s)' % fn_name}) if not fn_name in self.RPC_dict: return robustjsonify( {'error': 'Could not find requested RPC "%s"' % fn_name}) found_RPC = self.RPC_dict[fn_name] # Get the RPC we've found. ## Do any validation checks we need to do and return errors if they don't pass. # If the RPC is disabled, always return a Status 403 (Forbidden) if found_RPC.validation == 'disabled': if verbose: print('RPC(): RPC disabled') abort(403) # Only do other validation if DataStore and users are included -- NOTE: Any "unknown" validation values are treated like 'none'. if self.config['USE_DATASTORE'] and self.config['USE_USERS']: if found_RPC.validation == 'any' and not ( current_user.is_anonymous or current_user.is_authenticated): abort( 401 ) # If the RPC should be executable by any user, including an anonymous one, but there is no authorization or anonymous login, return a Status 401 (Unauthorized) elif found_RPC.validation == 'named' and ( current_user.is_anonymous or not current_user.is_authenticated): abort( 401 ) # Else if the RPC should be executable by any non-anonymous user, but there is no authorization or there is an anonymous login, return a Status 401 (Unauthorized) elif found_RPC.validation == 'admin': # Else if the RPC should be executable by any admin user, but there is no admin login or it's an anonymous login... if current_user.is_anonymous or not current_user.is_authenticated: abort( 401 ) # If the user is anonymous or no authenticated user is logged in, return Status 401 (Unauthorized). elif not current_user.is_admin: abort( 403 ) # Else, if the user is not an admin user, return Status 403 (Forbidden). # If we are doing an upload... if found_RPC.call_type == 'upload': if verbose: print('Starting upload...') thisfile = request.files[ 'uploadfile'] # Grab the formData file that was uploaded. filename = secure_filename( thisfile.filename ) # Extract a sanitized filename from the one we start with. try: uploaded_fname = os.path.join( self.datastore.tempfolder, filename) # Generate a full upload path/file name. except Exception as E: errormsg = 'Could not create filename for uploaded file: %s' % str( E) raise Exception(errormsg) try: thisfile.save( uploaded_fname) # Save the file to the uploads directory except Exception as E: errormsg = 'Could not save uploaded file: %s' % str(E) raise Exception(errormsg) args.insert( 0, uploaded_fname) # Prepend the file name to the args list. # Show the call of the function. callcolor = ['cyan', 'bgblue'] successcolor = ['green', 'bgblue'] failcolor = ['gray', 'bgred'] timestr = '[%s]' % sc.now(astype='str') try: userstr = ' <%s>' % current_user.username except: userstr = ' <no user>' RPCinfo = sc.objdict({ 'time': timestr, 'user': userstr, 'module': found_RPC.call_func.__module__, 'name': found_RPC.funcname }) if self.config['LOGGING_MODE'] == 'FULL': string = '%s%s RPC called: "%s.%s"' % ( RPCinfo.time, RPCinfo.user, RPCinfo.module, RPCinfo.name) sc.colorize(callcolor, string, enable=self.colorize) # Execute the function to get the results, putting it in a try block in case there are errors in what's being called. try: if verbose: print('RPC(): Starting RPC...') T = sc.tic() result = found_RPC.call_func(*args, **kwargs) if isinstance( result, dict ) and 'error' in result: # If the RPC returns an error, return it return robustjsonify({'error': result['error']}) elapsed = sc.toc(T, output=True) if self.config['LOGGING_MODE'] == 'FULL': string = '%s%s RPC finished in %0.2f s: "%s.%s"' % ( RPCinfo.time, RPCinfo.user, elapsed, RPCinfo.module, RPCinfo.name) sc.colorize(successcolor, string, enable=self.colorize) except Exception as E: if verbose: print('RPC(): Exception encountered...') shortmsg = str(E) exception = traceback.format_exc() # Grab the trackback stack hostname = '|%s| ' % socket.gethostname() tracemsg = '%s%s%s Exception during RPC "%s.%s" \nRequest: %s \n%.10000s' % ( hostname, RPCinfo.time, RPCinfo.user, RPCinfo.module, RPCinfo.name, request, exception) sc.colorize( failcolor, tracemsg, enable=self.colorize ) # Post an error to the Flask logger limiting the exception information to 10000 characters maximum (to prevent monstrous sqlalchemy outputs) if self.config['SLACK']: self.slacknotification(tracemsg) if isinstance( E, HTTPException ): # If we have a werkzeug exception, pass it on up to werkzeug to resolve and reply to. raise E code = 500 # Send back a response with status 500 that includes the exception traceback. fullmsg = shortmsg + '\n\nException details:\n' + tracemsg reply = { 'exception': fullmsg } # NB, not sure how to actually access 'traceback' on the FE, but keeping it here for future return make_response(robustjsonify(reply), code) # If we are doing a download, prepare the response and send it off. if found_RPC.call_type == 'download': # To download a file, use `this.$sciris.download` instead of `this.$sciris.rpc`. Decorate the RPC with # `@RPC(call_type='download')`. Finally, the RPC needs to specify the file and optionally the filename. # This is done with tuple unpacking. The following outputs are supported from `rpc_function()` # # 1 - filename_on_disk # 2 - BytesIO # 3 - filename_on_disk, download_filename # 4- BytesIO, download_filename # # Examples return values from the RPC are as follows # # 1 - "E:/test.xlsx" (uses "test.xlsx") # 2 - <BytesIO> (default filename will be generated in this function) # 3 - ("E:/test.xlsx","foo.xlsx") # 4 - (<BytesIO>,"foo.xlsx") # # On the RPC end, the most common cases would be it might look like # # return "E:/test.xlsx" # # OR # # return Blobject.to_file(), "foo.xlsx" if verbose: print('RPC(): Starting download...') if result is None: # If we got None for a result (the full file name), return an error to the client. return robustjsonify({ 'error': 'Could not find resource to download from RPC "%s": result is None' % fn_name }) elif sc.isstring(result): from_file = True dir_name, file_name = os.path.split(result) output_name = file_name elif isinstance(result, io.BytesIO): from_file = False bytesio = result output_name = 'download.obj' else: try: content = result[0] output_name = result[1] if sc.isstring(content): from_file = True dir_name, file_name = os.path.split(content) elif isinstance(content, io.BytesIO): from_file = False bytesio = content else: return robustjsonify( {'error': 'Unrecognized RPC output'}) except Exception as E: return robustjsonify( {'error': 'Error reading RPC result (%s)' % E}) if from_file: response = send_from_directory(dir_name, file_name, as_attachment=True) response.status_code = 201 # Status 201 = Created # Unfortunately, we cannot remove the actual file at this point # because it is in use during the actual download, so we rely on # later cleanup to remove download files. else: response = send_file(bytesio, as_attachment=True, attachment_filename=output_name) response.headers['filename'] = output_name print(response) return response # Return the response message. # Otherwise (normal and upload RPCs), else: if found_RPC.call_type == 'upload': # If we are doing an upload.... try: os.remove( uploaded_fname ) # Erase the physical uploaded file, since it is no longer needed. if verbose: print('RPC(): Removed uploaded file: %s' % uploaded_fname) except Exception as E: if verbose: print('RPC(): Could not remove uploaded file: %s' % str(E)) # Probably since moved by the user if result is None: # If None was returned by the RPC function, return ''. if verbose: print('RPC(): RPC finished, returning None') return '' else: # Otherwise, convert the result (probably a dict) to JSON and return it. output = robustjsonify(result) if verbose: print('RPC(): RPC finished, returning result') return output
def datathief(filename, xlim=None, ylim=None, xcol=None, ycol=None, dcol=None, debug=False): ''' Extract data from a figure. To use, add a single pixel at the start and end of the x-axis (default color: pure blue) and the y-axis (default color: pure red), and then one pixel for each data point (default color: pure green). This function will then return the x and y coordinates of each data point. Args: filename (str): the filename of the image to load xlim (list): a list of 2 floats defining the start and end values of the x-axis ylim (list): likewise, for the y-axis xcol (str, list): a hex string or list of 3 floats defining the RGB color for the x-axis pixels ycol (str, list): likewise, for the y-axis pixels dcol (str, list): likewise, for the data points debug (bool): whether to print extra information about the data transformations Returns: Dict with keys x and y corresponding to the data points **Examples**:: import datathief as dt import pylab as pl data1 = dt.datathief('my-data-fig.png', xlim=[0,20], ylim=[0,30]) data2 = dt.datathief('my-other-fig.png', xlim=[0,1], ylim=[0,1], xcol='#f00', ycol='#f0f', dcol='#face00') pl.scatter(data2.x, data2.y) Inspired by the DataThief Java tool. ''' if xlim is None: xlim = [0, 1] if ylim is None: ylim = [0, 1] if xcol is None: xcol = [0, 0, 1] if ycol is None: ycol = [1, 0, 0] if dcol is None: dcol = [0, 1, 0] if sc.isstring(xcol): xcol = sc.hex2rgb(xcol) if sc.isstring(ycol): xcol = sc.hex2rgb(ycol) if sc.isstring(dcol): xcol = sc.hex2rgb(dcol) # Read image data lim = sc.objdict(x=xlim, y=ylim) perpix = sc.objdict() ref = sc.objdict() d = sc.objdict() img = mpl.image.imread(filename) ref.x, tmp_xy = findpixels(img, xcol) tmp_yx, ref.y = findpixels(img, ycol) d.x, d.y = findpixels(img, dcol) ref.y = np.sort(img.shape[0] - ref.y) # Flip y-axis d.y = img.shape[0] - d.y # Flip y-axis assert len( ref.x ) == 2, f'Wrong number of x coordinates found ({len(ref.x)}): please ensure exactly 2 pixels have color {xcol}' assert len( ref.y ) == 2, f'Wrong number of y coordinates found ({len(ref.y)}): please ensure exactly 2 pixels have color {ycol}' if debug: print(f'Image shape: {img.shape}') print(f'Reference pixels: {ref}') print(f'Data pixels: {d}') # Process data data = sc.objdict() order = np.argsort(d.x) d.x = d.x[order] d.y = d.y[order] for k in ['x', 'y']: perpix = (np.diff(lim[k]) / np.diff(ref[k]))[0] orig = np.array(d[k], dtype=float) rmref = orig - ref[k][0] bypix = rmref * perpix data[k] = bypix + lim[k][0] if debug: print('\n\nFor variable:\n ', k) print(f'Pixels-per-value: {1.0/perpix}; pixel ratio: {perpix}') print(f'Original:\n orig={orig}') print(f'Removing reference:\n rmref={rmref}') print(f'By pixel:\n bypix={bypix}') print(f'Adding limit:\n data={data[k]}') return data
def date(obj, *args, start_date=None, dateformat=None, as_date=True): ''' Convert a string or a datetime object to a date object. To convert to an integer from the start day, it is recommended you supply a start date, or use sim.date() instead; otherwise, it will calculate the date counting days from 2020-01-01. This means that the output of cv.date() will not necessarily match the output of sim.date() for an integer input. Args: obj (str, date, datetime, list, array): the object to convert args (str, date, datetime): additional objects to convert start_date (str, date, datetime): the starting date, if an integer is supplied dateformat (str): the format to return the date in as_date (bool): whether to return as a datetime date instead of a string Returns: dates (date or list): either a single date object, or a list of them **Examples**:: cv.date('2020-04-05') # Returns datetime.date(2020, 4, 5) cv.date('2020-04-14', start_date='2020-04-04', as_date=False) # Returns 10 cv.date([35,36,37], as_date=False) # Returns ['2020-02-05', '2020-02-06', '2020-02-07'] ''' if obj is None: return None # Convert to list and handle other inputs if isinstance(obj, np.ndarray): obj = obj.tolist() # If it's an array, convert to a list obj = sc.promotetolist(obj) # Ensure it's iterable obj.extend(args) if dateformat is None: dateformat = '%Y-%m-%d' if start_date is None: start_date = '2020-01-01' dates = [] for d in obj: if d is None: dates.append(d) continue try: if type( d ) == dt.date: # Do not use isinstance, since must be the exact type pass elif sc.isstring(d): d = sc.readdate(d).date() elif isinstance(d, dt.datetime): d = d.date() elif sc.isnumber(d): if start_date is None: errormsg = f'To convert the number {d} to a date, you must supply start_date' raise ValueError(errormsg) d = date(start_date) + dt.timedelta(days=int(d)) else: errormsg = f'Cannot interpret {type(d)} as a date, must be date, datetime, or string' raise TypeError(errormsg) if as_date: dates.append(d) else: dates.append(d.strftime(dateformat)) except Exception as E: errormsg = f'Conversion of "{d}" to a date failed: {str(E)}' raise ValueError(errormsg) # Return an integer rather than a list if only one provided if len(dates) == 1: dates = dates[0] return dates