def add_assignments(org_url, token, projectId, assignments): """ Adds the assignments to project :param org_url: (string) The organizational url to use :param token: (string) The token to authenticate with :param projectId: (string) The project Id :param assignments: (list) The list of assignments to add :return: The json response of the addFeatures REST API Call """ logger = logging.getLogger() assignments_url = workforcehelpers.get_assignments_feature_layer_url( org_url, token, projectId) assignment_to_post = [x["data"] for x in assignments] add_url = "{}/addFeatures".format(assignments_url) data = { 'token': token, 'f': 'json', 'features': json.dumps(assignment_to_post) } logger.debug("Adding Assignments...") response = workforcehelpers.post(add_url, data) # Assign the returned object ids to the assignment dictionary object for i in range(len(response["addResults"])): assignments[i]["OBJECTID"] = response["addResults"][i]["objectId"] # Add the attachments if len(assignments) > 0 and "attachmentFile" in assignments[0]: add_attachments(org_url, token, projectId, assignments) return response
def add_attachments(org_url, token, projectId, assignments): """ This adds attachments to the assignments if they have one :param org_url: The org url to use :param token: The token to authenticate with :param projectId: The project Id to use :param assignments: The list of assignment json objects :return: """ assignment_fl_url = workforcehelpers.get_assignments_feature_layer_url( org_url, token, projectId) logging.getLogger().debug("Adding Attachments...") for assignment in assignments: if assignment["attachmentFile"] and assignment["attachmentFile"] != "": # url to hit to add attachments (<feature-service-layer>/<object-id>/addAttachment) add_url = "{}/{}/addAttachment".format(assignment_fl_url, assignment["OBJECTID"]) # Create the file dictionary that requests expects, guess the mimetype based on the file files = { "attachment": (os.path.basename(assignment["attachmentFile"]), open(os.path.abspath(assignment["attachmentFile"]), "rb"), mimetypes.guess_type( os.path.abspath(assignment["attachmentFile"]))) } data = {'f': 'json', 'token': token} response = workforcehelpers.post(add_url, data, files) logging.getLogger().debug(response)
def main(args): """ The main sequence of steps :param args: The argparse args :return: """ # Authenticate logging.getLogger().info("Authenticating...") token = workforcehelpers.get_token(args.org_url, args.username, args.password) # Get the assignments feature layer logging.getLogger().info("Getting assignments feature layer...") assignment_fl_url = workforcehelpers.get_assignments_feature_layer_url(args.org_url, token, args.projectId) # Get the target feature layer logging.getLogger().info("Getting target feature layer...") target_fl_url = args.targetFL # Open the field mappings config file logging.getLogger().info("Reading field mappings...") with open(args.configFile, 'r') as f: field_mappings = json.load(f) logging.getLogger().info("Validating field mappings...") if validate_config(field_mappings, target_fl_url, token): # Copy the assignments copy_assignments(assignment_fl_url, target_fl_url, field_mappings, token) logging.getLogger().info("Completed") else: logging.getLogger().critical("Invalid field mappings detected")
def main(args): """ The main sequence of steps :param args: The argparse args :return: """ # Authenticate logging.getLogger().info("Authenticating...") token = workforcehelpers.get_token(args.org_url, args.username, args.password) # Get the assignments feature layer logging.getLogger().info("Getting assignments feature layer...") assignment_fl_url = workforcehelpers.get_assignments_feature_layer_url(args.org_url, token, args.projectId) # Get the target feature layer logging.getLogger().info("Getting target feature layer...") target_fl_url = args.targetFL # Open the field mappings config file logging.getLogger().info("Reading field mappings...") with open(args.configFile, 'r') as f: field_mappings = json.load(f) logging.getLogger().info("Validating field mappings...") if validate_config(field_mappings, target_fl_url, token): # Copy the assignments copy_assignments(assignment_fl_url, target_fl_url, field_mappings, token, where=args.where) logging.getLogger().info("Completed") else: logging.getLogger().critical("Invalid field mappings detected")
def filter_assignment_types(org_url, token, project_id, assignment_types): """ Filters the assignment type, so that we don't have duplicates :param org_url: (string) The organization url to use :param token: (string) The token to use for authentication :param project_id: (string) The project Id :param assignment_types: (string) The list of assignment types to add :return: List<dict> The list of assignment types to add """ logger = logging.getLogger() assignments_url = workforcehelpers.get_assignments_feature_layer_url( org_url, token, project_id) data = {'token': token, 'f': 'json'} assignment_types_to_add = [] assignment_fl = workforcehelpers.get(assignments_url, params=data) for field in assignment_fl["fields"]: if field["name"] == "assignmentType": for assignment_type in assignment_types: if assignment_types.count(assignment_type) > 1: logger.critical("Duplicate types detected in csv...") return [] duplicate = False for current_type in field["domain"]["codedValues"]: if current_type["name"] == assignment_type: logger.warning( "Duplicate type detected and will not be added: {}" .format(assignment_type)) duplicate = True if not duplicate: assignment_types_to_add.append(assignment_type) break return assignment_types_to_add
def add_attachments(org_url, token, projectId, assignments): """ This adds attachments to the assignments if they have one :param org_url: The org url to use :param token: The token to authenticate with :param projectId: The project Id to use :param assignments: The list of assignment json objects :return: """ assignment_fl_url = workforcehelpers.get_assignments_feature_layer_url(org_url, token, projectId) logging.getLogger().debug("Adding Attachments...") for assignment in assignments: if assignment["attachmentFile"] and assignment["attachmentFile"] != "": # url to hit to add attachments (<feature-service-layer>/<object-id>/addAttachment) add_url = "{}/{}/addAttachment".format(assignment_fl_url, assignment["OBJECTID"]) # Create the file dictionary that requests expects, guess the mimetype based on the file files = {"attachment": (os.path.basename(assignment["attachmentFile"]), open(os.path.abspath(assignment["attachmentFile"]), "rb"), mimetypes.guess_type(os.path.abspath(assignment["attachmentFile"])))} data = { 'f': 'json', 'token': token } response = workforcehelpers.post(add_url, data, files) logging.getLogger().debug(response)
def add_assignments(org_url, token, projectId, assignments): """ Adds the assignments to project :param org_url: (string) The organizational url to use :param token: (string) The token to authenticate with :param projectId: (string) The project Id :param assignments: (list) The list of assignments to add :return: The json response of the addFeatures REST API Call """ logger = logging.getLogger() assignments_url = workforcehelpers.get_assignments_feature_layer_url(org_url, token, projectId) assignment_to_post = [x["data"] for x in assignments] add_url = "{}/addFeatures".format(assignments_url) data = { 'token': token, 'f': 'json', 'features': json.dumps(assignment_to_post) } logger.debug("Adding Assignments...") response = workforcehelpers.post(add_url, data) # Assign the returned object ids to the assignment dictionary object for i in range(len(response["addResults"])): assignments[i]["OBJECTID"] = response["addResults"][i]["objectId"] # Add the attachments if len(assignments) > 0 and "attachmentFile" in assignments[0]: add_attachments(org_url, token, projectId, assignments) return response
def validate_assignments(org_url, token, projectId, assignments): """ Validates the provided values against the codedValues specified in the FS :param org_url: (string) The organization url to use :param token: (string) The token to use for authentication :param projectId: (string) The project Id :param assignments: (string) The list of assignments to check :return: True if valid, False if not """ # Grab the item logger = logging.getLogger() assignment_fl = workforcehelpers.get_assignments_feature_layer_url(org_url, token, projectId) dispatcher_fl = workforcehelpers.get_dispatchers_feature_layer_url(org_url, token, projectId) dispatchers = workforcehelpers.query_feature_layer(dispatcher_fl, token) statuses = [] priorities = [] assignmentTypes = [] dispatcherIds = [] # Get the dispatcherIds for dispatcher in dispatchers["features"]: dispatcherIds.append(dispatcher["attributes"]["OBJECTID"]) # Get the codes of the domains for field in workforcehelpers.get_feature_layer(assignment_fl, token)["fields"]: if field["name"] == "status": statuses = [cv["code"] for cv in field["domain"]["codedValues"]] if field["name"] == "priority": priorities = [cv["code"] for cv in field["domain"]["codedValues"]] if field["name"] == "assignmentType": assignmentTypes = [cv["code"] for cv in field["domain"]["codedValues"]] logger.debug("Validating assignments...") # check the values against the fields that have domains for assignment in assignments: if assignment["data"]["attributes"]["status"] not in statuses: logging.getLogger().critical("Invalid Status for: {}".format(assignment)) return False if "priority" in assignment["data"]["attributes"] and assignment["data"]["attributes"][ "priority"] not in priorities: logging.getLogger().critical("Invalid Priority for: {}".format(assignment)) return False if assignment["data"]["attributes"]["assignmentType"] not in assignmentTypes: logging.getLogger().critical("Invalid Assignment Type for: {}".format(assignment)) return False if assignment["data"]["attributes"]["dispatcherId"] not in dispatcherIds: logging.getLogger().critical("Invalid Dispatcher Id for: {}".format(assignment)) return False if "attachmentFile" in assignment and assignment["attachmentFile"]: if not os.path.isfile(os.path.abspath(assignment["attachmentFile"])): logging.getLogger().critical("Attachment file not found: {}".format(assignment["attachmentFile"])) return False return True
def main(args): # First step is to authenticate and get a valid token logging.getLogger().info("Authenticating...") token = workforcehelpers.get_token(args.org_url, args.username, args.password) # Get the assignment feature layer logging.getLogger().info("Getting assignment feature layer...") assignment_fl_url = workforcehelpers.get_assignments_feature_layer_url(args.org_url, token, args.projectId) # Query the assignment feature layer to get certain assignments logging.getLogger().info("Querying assignments...") assignments = workforcehelpers.query_feature_layer(assignment_fl_url, token, where=args.where, outSR=args.outSR)["features"] # Write the assignments to the csv file logging.getLogger().info("Writing to CSV...") write_assignments_to_csv(args.outCSV, assignments, args.dateFormat) logging.getLogger().info("Completed")
def main(args): # Authenticate with AGOL and get the required token logging.getLogger().info("Authenticating...") token = workforcehelpers.get_token(args.org_url, args.username, args.password) # Get the assignments feature layer url logging.getLogger().info("Getting assignment feature layer...") assignment_fl_url = workforcehelpers.get_assignments_feature_layer_url( args.org_url, token, args.projectId) logging.getLogger().info("Deleting assignments...") response = delete_assignments(assignment_fl_url, token, args.objectIDs, args.where) logging.getLogger().info(response) logging.getLogger().info("Completed")
def main(args): """ The main sequence of steps :param args: The argparse args :return: """ # Authenticate logging.getLogger().info("Authenticating...") token = workforcehelpers.get_token(args.org_url, args.username, args.password) # Get the assignments feature layer logging.getLogger().info("Getting assignments feature layer...") assignment_fl_url = workforcehelpers.get_assignments_feature_layer_url( args.org_url, token, args.projectId) # Get the target feature layer logging.getLogger().info("Getting target feature layer...") target_fl_url = args.targetFL # if a specific workers weren't specified, let's use all workers if not args.workers: workers_fl_url = workforcehelpers.get_workers_feature_layer_url( args.org_url, token, args.projectId) features = workforcehelpers.query_feature_layer( workers_fl_url, token, where="1=1")["features"] workers = [feature["attributes"]["userId"] for feature in features] else: workers = args.workers # Open the field mappings config file logging.getLogger().info("Reading field mappings...") with open(args.configFile, 'r') as f: field_mappings = json.load(f) # Check the mapping to the target feature service is valid logging.getLogger().info("Validating field mappings...") if validate_config(field_mappings, target_fl_url, token): for worker in workers: # Get the query string that represents the invalid assignment completions query_string = get_invalid_completions(args.org_url, token, args.projectId, worker, args.timeTol, args.distTol, args.minAccuracy) # Use that query to copy the assignments to feature service (if they don't already exist) copy_assignments(assignment_fl_url, target_fl_url, field_mappings, token, where=query_string) else: logging.getLogger().critical("Invalid field mappings detected") return
def add_assignment_types(org_url, token, project_id, assignment_types): """ Adds the assignments to project :param org_url: (string) The organizational url to use :param token: (string) The token to authenticate with :param project_id: (string) The project Id :param assignment_types: (list) The list of assignment types to add :return: The json response of the addFeatures REST API Call """ logger = logging.getLogger() # get the assignments feature layer assignments_url = workforcehelpers.get_assignments_feature_layer_url( org_url, token, project_id) data = {'token': token, 'f': 'json'} # get the json assignment_fl = workforcehelpers.get(assignments_url, params=data) for field in assignment_fl["fields"]: if field["name"] == "assignmentType": length = len(field["domain"]["codedValues"]) # Add new types here, auto-increment the coded values for assignment_type in assignment_types: field["domain"]["codedValues"].append({ "name": assignment_type, "code": length + 1 }) length += 1 break # clear out lastEditDate (otherwise ArcGIS online throws an error about it) if "editingInfo" in assignment_fl and "lastEditDate" in assignment_fl[ "editingInfo"]: del assignment_fl["editingInfo"]["lastEditDate"] # clear out hasGeometry properties before submitting (known issue with FS) if "hasGeometryProperties" in assignment_fl: del assignment_fl["hasGeometryProperties"] # need to use the admin url to update a service definition index = assignments_url.index("/services/") update_definition_url = assignments_url[ 0:index] + "/admin" + assignments_url[index:] + "/updateDefinition" data = { 'token': token, 'f': 'json', 'updateDefinition': json.dumps(assignment_fl) } logger.debug("Adding Assignment types...") # send the data response = workforcehelpers.post(update_definition_url, data) return response
def main(args): # First step is to authenticate and get a valid token logging.getLogger().info("Authenticating...") token = workforcehelpers.get_token(args.org_url, args.username, args.password) # Get the assignment feature layer logging.getLogger().info("Getting assignment feature layer...") assignment_fl_url = workforcehelpers.get_assignments_feature_layer_url( args.org_url, token, args.projectId) # Query the assignment feature layer to get certain assignments logging.getLogger().info("Querying assignments...") assignments = workforcehelpers.query_feature_layer( assignment_fl_url, token, where=args.where, outSR=args.outSR)["features"] # Write the assignments to the csv file logging.getLogger().info("Writing to CSV...") write_assignments_to_csv(args.outCSV, assignments, args.dateFormat) logging.getLogger().info("Completed")
def validate_assignments(org_url, token, projectId, assignments): """ Validates the provided values against the codedValues specified in the FS :param org_url: (string) The organization url to use :param token: (string) The token to use for authentication :param projectId: (string) The project Id :param assignments: (string) The list of assignments to check :return: True if valid, False if not """ # Grab the item logger = logging.getLogger() assignment_fl = workforcehelpers.get_assignments_feature_layer_url( org_url, token, projectId) dispatcher_fl = workforcehelpers.get_dispatchers_feature_layer_url( org_url, token, projectId) dispatchers = workforcehelpers.query_feature_layer(dispatcher_fl, token) statuses = [] priorities = [] assignmentTypes = [] dispatcherIds = [] # Get the dispatcherIds for dispatcher in dispatchers["features"]: dispatcherIds.append(dispatcher["attributes"]["OBJECTID"]) # Get the codes of the domains for field in workforcehelpers.get_feature_layer(assignment_fl, token)["fields"]: if field["name"] == "status": statuses = [cv["code"] for cv in field["domain"]["codedValues"]] if field["name"] == "priority": priorities = [cv["code"] for cv in field["domain"]["codedValues"]] if field["name"] == "assignmentType": assignmentTypes = [ cv["code"] for cv in field["domain"]["codedValues"] ] logger.debug("Validating assignments...") # check the values against the fields that have domains for assignment in assignments: if assignment["data"]["attributes"]["status"] not in statuses: logging.getLogger().critical( "Invalid Status for: {}".format(assignment)) return False if "priority" in assignment["data"]["attributes"] and assignment[ "data"]["attributes"]["priority"] not in priorities: logging.getLogger().critical( "Invalid Priority for: {}".format(assignment)) return False if assignment["data"]["attributes"][ "assignmentType"] not in assignmentTypes: logging.getLogger().critical( "Invalid Assignment Type for: {}".format(assignment)) return False if assignment["data"]["attributes"][ "dispatcherId"] not in dispatcherIds: logging.getLogger().critical( "Invalid Dispatcher Id for: {}".format(assignment)) return False if "attachmentFile" in assignment and assignment["attachmentFile"]: if not os.path.isfile(os.path.abspath( assignment["attachmentFile"])): logging.getLogger().critical( "Attachment file not found: {}".format( assignment["attachmentFile"])) return False return True
def get_invalid_completions(org_url, token, projectId, worker, time_tolerance, distance_tolerance, min_accuracy): """ Generates a query string that represents the assignments that were completed either outside of the specified time window or outside of the specified distance :param org_url: (string) The url of the organizational account to use :param token: (string) The token to authenticate with :param projectId: (string) The projectId to use :param worker: (string) The worker to check :param time_tolerance: (int) The number of minutes of tolerance to use :param distance_tolerance: (int or float) The distance tolerance to use :param min_accuracy: (int or float) The minimum accuracy to require when querying points :return: (string) A query that uses the OBJECTID to identify invalid assignment completions """ logging.getLogger().info("Getting assignments feature layer url") assignment_fl_url = workforcehelpers.get_assignments_feature_layer_url( org_url, token, projectId) logging.getLogger().info("Getting location feature layer url") location_fl_url = workforcehelpers.get_location_feature_layer_url( org_url, token, projectId) logging.getLogger().info("Getting workerId for {}".format(worker)) worker_id = get_worker_id(org_url, token, projectId, worker) if not worker_id: logging.critical("Invalid worker detected") return completed_assignments = workforcehelpers.query_feature_layer( assignment_fl_url, token, where="workerId = {} AND completedDate is not NULL".format( worker_id))["features"] invalid_assignment_oids = [] for assignment in completed_assignments: # The coordinates of the assignment start_coords = (assignment["geometry"]["x"], assignment["geometry"]["y"]) # When the assignment was completed completion_date = datetime.datetime.utcfromtimestamp( int(assignment["attributes"]["completedDate"]) / 1000) # Add/Subtract some minutes to give a little leeway start_date = completion_date - datetime.timedelta( minutes=time_tolerance) end_date = completion_date + datetime.timedelta(minutes=time_tolerance) # Make a query string to select location by the worker during the time period loc_query_string = "Editor = '{}' AND CreationDate >= '{}' AND CreationDate <= '{}' AND Accuracy <= {}" \ .format(worker, start_date.strftime('%Y-%m-%d %H:%M:%S'), end_date.strftime('%Y-%m-%d %H:%M:%S'), min_accuracy) # Query the feature layer locations_to_check = workforcehelpers.query_feature_layer( location_fl_url, token, where=loc_query_string)["features"] # Bool to see if this assignment is valid or not is_valid = False for location in locations_to_check: # Make a list of coordinate pairs to get the distance of coords = [] coords.append( (location["geometry"]["x"], location["geometry"]["y"])) # If we include the accuracy, we need to make four variations (+- the accuracy) accuracy = float(location["attributes"]["Accuracy"]) coords.append((location["geometry"]["x"] + accuracy, location["geometry"]["y"] + accuracy)) coords.append((location["geometry"]["x"] + accuracy, location["geometry"]["y"] - accuracy)) coords.append((location["geometry"]["x"] - accuracy, location["geometry"]["y"] + accuracy)) coords.append((location["geometry"]["x"] - accuracy, location["geometry"]["y"] - accuracy)) distances = [ get_simple_distance(start_coords, coordinates) for coordinates in coords ] # if any of the distances is less than the threshold then this assignment is valid if any(distance < distance_tolerance for distance in distances): is_valid = True break # if it's not valid add the OBJECTID to the list of invalid assignment OBJECTIDS if not is_valid: logging.debug("Location Query: {}".format(loc_query_string)) invalid_assignment_oids.append( str(assignment["attributes"]["OBJECTID"])) if invalid_assignment_oids: return "OBJECTID in ({})".format(",".join(invalid_assignment_oids)) else: logging.getLogger().info( "All assignments were completed within the specified time and distance. Nothing to copy." ) return "1=0"