"""Accumulate the data to be used when posting a form."""

PY2 = sys.version_info[0] == 2

PY3 = sys.version_info[0] == 3

files = []

form_fields = []

boundary = None

form_data = ""

#----------------------------------------------------------------------

def __init__(self, param_dict, files):

self.boundary = None

self.files = []

self.form_data = ""

if len(self.form_fields) > 0:

self.form_fields = []

if len(param_dict) == 0:

self.form_fields = []

else:

for key, value in param_dict.items():

self.form_fields.append((key, value))

del key, value

if len(files) == 0:

self.files = []

else:

for key, value in files.items():

self.add_file(fieldname=key,

filename=os.path.basename(value),

filepath=value,

mimetype=None)

self.boundary = email.generator._make_boundary()

#----------------------------------------------------------------------

def get_content_type(self):

"""Gets the content type."""

return 'multipart/form-data; boundary=%s' % self.boundary

#----------------------------------------------------------------------

def add_field(self, name, value):

"""Add a simple field to the form data."""

self.form_fields.append((name, value))

#----------------------------------------------------------------------

def add_file(self, fieldname, filename, filepath, mimetype=None):

"""Add a file to be uploaded.

Inputs:

fieldname - name of the POST value

fieldname - name of the file to pass to the server

filePath - path to the local file on disk

mimetype - MIME stands for Multipurpose Internet Mail Extensions.

It's a way of identifying files on the Internet according to

their nature and format. Default is None.

"""

body = filepath

if mimetype is None:

mimetype = mimetypes.guess_type(filename)[0] or 'application/octet-stream'

self.files.append((fieldname, filename, mimetype, body))

#----------------------------------------------------------------------

@property

def make_result(self):

"""Returns the data for the request."""

if self.PY2:

self._py2()

elif self.PY3:

self._py3()

return self.form_data

#----------------------------------------------------------------------

def _py2(self):

"""python 2.x version of formatting body data"""

boundary = self.boundary

buf = StringIO()

for (key, value) in self.form_fields:

buf.write('--%s\r\n' % boundary)

buf.write('Content-Disposition: form-data; name="%s"' % key)

buf.write('\r\n\r\n%s\r\n' % value)

for (key, filename, mimetype, filepath) in self.files:

if os.path.isfile(filepath):

buf.write('--{boundary}\r\n'

'Content-Disposition: form-data; name="{key}"; '

'filename="{filename}"\r\n'

'Content-Type: {content_type}\r\n\r\n'.format(

boundary=boundary,

key=key,

filename=filename,

content_type=mimetype))

with open(filepath, "rb") as local_file:

shutil.copyfileobj(local_file, buf)

buf.write('\r\n')

buf.write('--' + boundary + '--\r\n\r\n')

buf = buf.getvalue()

self.form_data = buf

#----------------------------------------------------------------------

def _py3(self):

""" python 3 method"""

boundary = self.boundary

buf = io.BytesIO()

textwriter = io.TextIOWrapper(

buf, 'utf8', newline='', write_through=True)

for (key, value) in self.form_fields:

textwriter.write(

'--{boundary}\r\n'

'Content-Disposition: form-data; name="{key}"\r\n\r\n'

'{value}\r\n'.format(

boundary=boundary, key=key, value=value))

for(key, filename, mimetype, filepath) in self.files:

if os.path.isfile(filepath):

textwriter.write(

'--{boundary}\r\n'

'Content-Disposition: form-data; name="{key}"; '

'filename="{filename}"\r\n'

'Content-Type: {content_type}\r\n\r\n'.format(

boundary=boundary, key=key, filename=filename,

content_type=mimetype))

with open(filepath, "rb") as local_file:

shutil.copyfileobj(local_file, buf)

textwriter.write('\r\n')

textwriter.write('--{}--\r\n\r\n'.format(boundary))

def __init__(self):

self._config_options = {}

def _read_config(self, config_file):

"""Read the config and set global variables used in the script.

Keyword arguments:

config_file - Path to the configuration file.

If None it will look for a file called overwrite_hosted_features.cfg in the same directory as the executing script.

"""

config = configparser.ConfigParser()

if config_file is None:

config_file = os.path.join(os.path.dirname(__file__), 'overwrite_hosted_features.cfg')

config.readfp(open(config_file))

log_path = _validate_input(config, 'Log File', 'path', 'path', False)

if log_path is not None:

self._config_options['log_path'] = log_path

is_verbose = _validate_input(config, 'Log File', 'isVerbose', 'bool', False)

if is_verbose is not None:

self._config_options['is_verbose'] = is_verbose

self._start_logging()

self._config_options['feature_service_id'] = _validate_input(config, 'Existing ItemIDs', 'featureServiceItemID', 'id', True)

feature_collection_id = _validate_input(config, 'Existing ItemIDs', 'featureCollectionItemID', 'id', False)

if feature_collection_id is not None:

self._config_options['feature_collection_id'] = feature_collection_id

fgdb = _validate_input(config, 'Data Sources', 'fgdb', 'path', False)

if fgdb is not None:

self._config_options['fgdb'] = fgdb

self._config_options['org_url'] = _validate_input(config, 'Portal Sharing URL', 'baseURL', 'url', True)

self._config_options['username'] = _validate_input(config, 'Portal Credentials', 'username', 'string', True)

self._config_options['pw'] = _validate_input(config, 'Portal Credentials', 'pw', 'string', True)

token_url = _validate_input(config, 'Portal Sharing URL', 'tokenURL', 'url', False)

if token_url is not None:

self._config_options['token_url'] = token_url

max_allowable_offset = _validate_input(config, 'Generalization', 'maxAllowableOffset', 'int', False)

if max_allowable_offset is not None:

self._config_options['max_allowable_offset'] = max_allowable_offset

layer_mapping = _validate_input(config, 'Layers', 'nameMapping', 'mapping', False)

if layer_mapping is not None:

self._config_options['layer_mapping'] = layer_mapping

def _start_logging(self):

"""If a log file is specified in the config,

create it if it doesn't exist and write the start time of the run."""

self._config_options['start_time'] = datetime.datetime.now()

if 'log_path' in self._config_options:

log_path = self._config_options['log_path']

is_file = os.path.isfile(log_path)

logfile_location = os.path.abspath(os.path.dirname(log_path))

if not os.path.exists(logfile_location):

os.makedirs(logfile_location)

if is_file:

path = log_path

else:

path = os.path.join(logfile_location, "OverwriteLog.txt")

log_path = path

log = open(path, "a")

date = self._config_options['start_time'].strftime('%Y-%m-%d %H:%M:%S')

log.write("----------------------------" + "\n")

log.write("Begin Data Overwrite: " + str(date) + "\n")

log.close()

def _log_message(self, my_message, is_error=False):

"""Log a new message and print to the python output.

Keyword arguments:

my_message - the message to log

is_error - indicates if the message is an error, used to log even when verbose logging is disabled

"""

date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

if 'log_path' in self._config_options and (('is_verbose' in self._config_options and self._config_options['is_verbose']) or is_error):

log = open(self._config_options['log_path'], "a")

log.write(" " + str(date) + " - " +my_message + "\n")

log.close()

print(" " + str(date) + " - " +my_message + "\n")

def _end_logging(self):

"""If a log file is specified in the config write the elapsed time."""

if 'log_path' in self._config_options:

log = open(self._config_options['log_path'], "a")

endtime = datetime.datetime.now()

log.write("Elapsed Time: " + str(endtime - self._config_options['start_time']) + "\n")

log.close()

def _log_error(self):

"""Log an error message."""

pymsg = "PYTHON ERRORS:\nTraceback info:\n{1}\nError Info:\n{0}".format(str(sys.exc_info()[1]), "".join(traceback.format_tb(sys.exc_info()[2])))

self._log_message(pymsg, True)

def _url_request(self, url, request_parameters, request_type='GET', files=None, repeat=0, error_text="Error", raise_on_failure=True):

"""Send a new request and format the json response.

Keyword arguments:

url - the url of the request

request_parameters - a dictionay containing the name of the parameter and its correspoinsding value

request_type - the type of request: 'GET', 'POST'

files - the files to be uploaded

repeat - the nuber of times to repeat the request in the case of a failure

error_text - the message to log if an error is returned

raise_on_failure - indicates if an exception should be raised if an error is returned and repeat is 0"""

if files is not None:

mpf = _MultiPartForm(param_dict=request_parameters, files=files)

req = request(url)

body = mpf.make_result

req.add_header('Content-type', mpf.get_content_type())

req.add_header('Content-length', len(body))

req.data = body

elif request_type == 'GET':

req = request('?'.join((url, encode(request_parameters))))

else:

headers = {'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',}

req = request(url, encode(request_parameters).encode('UTF-8'), headers)

req.add_header('Accept-encoding', 'gzip')

response = urlopen(req)

if response.info().get('Content-Encoding') == 'gzip':

buf = io.BytesIO(response.read())

with gzip.GzipFile(fileobj=buf) as gzip_file:

response_bytes = gzip_file.read()

else:

response_bytes = response.read()

response_text = response_bytes.decode('UTF-8')

response_json = json.loads(response_text)

if "error" in response_json:

if repeat == 0:

if raise_on_failure:

raise Exception("{0}: {1}".format(error_text, response_json))

return response_json

repeat -= 1

time.sleep(2)

response_json = self._url_request(

url, request_parameters, request_type, files, repeat, error_text)

return response_json

def _get_token(self):

"""Returns a token for the given user and organization."""

query_dict = {'username': self._config_options['username'],

'password': self._config_options['pw'],

'expiration': '60',

'referer': self._config_options['org_url'],

'f': 'json'}

token_url = "https://www.arcgis.com/sharing/rest/generateToken"

if 'token_url' in self._config_options:

token_url = self._config_options['token_url']

token_response = self._url_request(token_url, query_dict, 'POST', repeat=2, raise_on_failure=False)

if "token" not in token_response:

raise Exception("Unable to connect to specified portal. Please verify you are passing in your correct portal url, token url, username and password.")

else:

return token_response['token']

def _wait_on_job(self, item_id, job_type, job_id, error_text):

"""Waits for a job to complete, if it fails an exception is raised.

Keyword arguments:

item_id - the id of the item to get the status for

job_type - the type of job currently processing

job_id - the id of the pending job

error_text - the error to raise if the job fails"""

url = '{0}sharing/rest/content/users/{1}/items/{2}/status'.format(self._config_options['org_url'], self._config_options['username'], item_id)

parameters = {'token': self._config_options['token'], 'f': 'json', 'jobType' : job_type, 'jobId' : job_id}

status = "processing"

while status != "completed":

response = self._url_request(url, parameters, repeat=2, error_text=error_text)

status = response['status'].lower()

if status == 'failed':

raise Exception("{0}: {1}".format(error_text, str(response['statusMessage'])))

elif status == "completed":

break

time.sleep(2)

def _get_published_items(self):

"""Validates the feature service and feature collection exist and sets global variables."""

url = '{0}sharing/rest/content/items/{1}'.format(self._config_options['org_url'], self._config_options['feature_service_id'])

request_parameters = {'f' : 'json', 'token' : self._config_options['token']}

item = self._url_request(url, request_parameters, error_text='Unable to find feature service with ID: {}'.format(self._config_options['feature_service_id']))

if not item['type'] == 'Feature Service':

raise Exception("Item {} is not a feature service".format(self._config_options['feature_service_id']))

self._config_options['basename'] = item['title']

if 'feature_collection_id' in self._config_options:

url = '{0}sharing/rest/content/items/{1}'.format(self._config_options['org_url'], self._config_options['feature_collection_id'])

item = self._url_request(url, request_parameters, error_text='Unable to find feature collection with ID: {}'.format(self._config_options['feature_collection_id']))

if not item['type'] == 'Feature Collection':

raise Exception("Item {} is not a feature collection".format(self._config_options['feature_collection_id']))

self._config_options['temp_fc_name'] = item['title'] + "_temp"

self._config_options['owner_folder'] = item['ownerFolder']

def _delete_item(self, item_id):

"""Delete an item from the portal with a given id.

Keyword arguments:

item_id - the id of the item to delete"""

url = '{0}sharing/rest/content/users/{1}/items/{2}/delete'.format(self._config_options['org_url'], self._config_options['username'], item_id)

request_parameters = {'f' : 'json', 'token' : self._config_options['token']}

return self._url_request(url, request_parameters, 'POST', repeat=2, raise_on_failure=False)

def _find_and_delete_gdb(self, gdb_name):

"""Search the portal for a geodatabase with a given name owned by the owner specified and if found delete the item.

Keyword arguments:

gdb_name - the name of the geodatabase"""

url = '{0}sharing/rest/search'.format(self._config_options['org_url'])

request_parameters = {'f' : 'json', 'q' : 'OverwriteHostedFeatures owner:{0} type:"File Geodatabase"'.format(self._config_options['username']),

'token' : self._config_options['token']}

response = self._url_request(url, request_parameters, error_text='Failed to upload file geodatabase')

results = response['results']

existing_gdb = next((r['id'] for r in results if r['name'] == gdb_name and "OverwriteHostedFeatures" in r['tags']), None)

if existing_gdb is None:

self._log_message("Failed to find file geodatabase on the portal named {0}: {1}".format(gdb_name, response))

return

self._log_message("File geodatabase {} found on the portal, deleting the item".format(gdb_name))

response = self._delete_item(existing_gdb)

if 'success' in response and response['success']:

self._log_message("File geodatabase deleted")

else:

self._log_message("Failed to delete file geodatabase: {0}".format(response))

def _upload_fgdb(self):

"""Uploads the file geodatabase to the portal."""

fgdb = self._config_options['fgdb']

if not os.path.exists(fgdb):

raise Exception("File geodatabase {} could not be found".format(fgdb))

gdb_name = os.path.basename(fgdb)

self._log_message("Uploading file geodatabase {}".format(fgdb))

try:

request_parameters = {'f' : 'json', 'token' : self._config_options['token'], 'tags' : 'OverwriteHostedFeatures',

'itemType' : 'file', 'async' : False,

'type' : 'File Geodatabase', 'descriptipion' : 'GDB',

'filename' : os.path.basename(fgdb), 'title' : self._config_options['basename']}

url = '{0}sharing/rest/content/users/{1}/addItem'.format(self._config_options['org_url'], self._config_options['username'])

files = {}

files['file'] = fgdb

response = self._url_request(url, request_parameters, files=files, error_text='Failed to upload file geodatabase')

except Exception:

self._log_message("Failed to upload file geodatabase. Searching the portal for a file geodatabase with the same name")

self._find_and_delete_gdb(gdb_name)

response = self._url_request(url, request_parameters, files=files, error_text='Failed to upload file geodatabase')

self._config_options['gdb_item_id'] = response['id']

self._log_message("File geodatabase upload complete")

def _update_feature_service(self):

"""Overwrites the feature service using the uploaded file geodatabase."""

basename = self._config_options['basename']

org_url = self._config_options['org_url']

feature_service_id = self._config_options['feature_service_id']

self._log_message("Updating {} feature service".format(basename))

url = '{0}sharing/rest/content/items/{1}'.format(org_url, feature_service_id)

request_parameters = {'f' : 'json', 'token' : self._config_options['token']}

response = self._url_request(url, request_parameters, error_text='Unable to find feature service with ID: {}'.format(feature_service_id))

fs_url = response['url']

feature_service = self._url_request(fs_url, request_parameters, repeat=2, error_text='Unable to get JSON definition of feature service')

publish_params = feature_service

publish_params['name'] = os.path.basename(os.path.dirname(fs_url))

layers = self._url_request(fs_url + "/layers", request_parameters, repeat=2, error_text='Unable to get JSON definition of feature service layers')

publish_params['layers'] = layers['layers']

publish_params['tables'] = layers['tables']

if 'layer_mapping' in self._config_options: #The name of the layer needs to match the name of the feature class for it to be succesfully overwritten

for mapping in self._config_options['layer_mapping']:

lyr = next((i for i in publish_params['layers'] if i['name'] == mapping[0]), None)

if lyr is not None:

lyr['name'] = mapping[1]

#Need to pass the C encoding for '<' and '>', otherwise publish fails.

publish_params_json = json.dumps(publish_params).replace('<', '\\u003c').replace('>', '\\u003e')

attempt_count = 2

for i in range(attempt_count):

try:

url = '{0}sharing/rest/content/users/{1}/publish'.format(org_url, self._config_options['username'])

request_parameters = {'f' : 'json', 'token' : self._config_options['token'],

'publishParameters' : publish_params_json, 'itemID' : self._config_options['gdb_item_id'],

'overwrite' : True, 'fileType' : 'fileGeodatabase'}

response = self._url_request(url, request_parameters, "POST", error_text='Failed to update {} feature service'.format(basename))

if 'services' not in response:

raise Exception("Failed to update {0} feature service: {1}".format(basename, response))

service = response['services'][0]

if 'jobId' not in service:

raise Exception("Failed to update {0} feature service: {1}".format(basename, response))

self._wait_on_job(feature_service_id, "publish", service['jobId'], "Failed to update {0} feature service, job id: {1}".format(basename, service['jobId']))

break

except Exception as ex:

ex.args = ("Attempt {0}: {1}".format(i+1, str(ex)),)

if i+1 == attempt_count:

raise ex

self._log_message(str(ex))

self._log_message("{} feature service updated".format(basename))

def _update_feature_collection(self):

"""Overwrite the feature collection using the feature service."""

org_url = self._config_options['org_url']

feature_service_id = self._config_options['feature_service_id']

feature_collection_id = self._config_options['feature_collection_id']

self._log_message("Updating feature collection")

exp_params = {}

if 'max_allowable_offset' in self._config_options:

exp_params.update({"maxAllowableOffset":self._config_options['max_allowable_offset']})

attempt_count = 2

for i in range(attempt_count):

try:

url = '{0}sharing/rest/content/users/{1}/export'.format(org_url, self._config_options['username'])

request_parameters = {'f' : 'json', 'token' : self._config_options['token'],

'itemID' : feature_service_id, 'exportFormat' : 'feature collection',

'exportParameters' : json.dumps(exp_params),

'overwrite' : True, 'resultItemId' : feature_collection_id}

response = self._url_request(url, request_parameters, "POST", raise_on_failure=False)

if 'jobId' not in response:

raise Exception("Failed to export temporary feature collection: {0}".format(response))

self._wait_on_job(feature_collection_id, "export", response['jobId'], "Failed to update feature collection, job id: {0}".format(response['jobId']))

break

except Exception as ex:

ex.args = ("Attempt {0}: {1}".format(i+1, str(ex)),)

if i+1 == attempt_count:

raise ex

self._log_message(str(ex))

date = datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')

user_folder = self._config_options['username']

owner_folder = self._config_options['owner_folder']

if owner_folder is not None:

user_folder = user_folder + "/" + str(owner_folder)

url = '{0}sharing/rest/content/users/{1}/items/{2}/update'.format(org_url, user_folder, self._config_options['feature_collection_id'])

request_parameters = {'f' : 'json', 'token' : self._config_options['token'], 'snippet' : str(date)}

self._url_request(url, request_parameters, "POST", repeat=2, error_text='Failed to update feature collection')

self._log_message("Feature collection updated")

def _remove_temp_content(self):

"""Remove the temporary file geodatabase if it exists."""

if 'gdb_item_id' in self._config_options:

response = self._delete_item(self._config_options['gdb_item_id'])

if 'success' in response and response['success']:

self._log_message("File geodatabase deleted")

else:

self._log_message("Failed to delete file geodatabase: {0}".format(response))

def run(self, config_file):

"""Overwrite hosted features."""

try:

self._read_config(config_file)

self._config_options['token'] = self._get_token()

self._get_published_items()

if 'fgdb' in self._config_options:

self._upload_fgdb()

self._update_feature_service()

if 'feature_collection_id' in self._config_options:

self._update_feature_collection()

except Exception:

self._log_error()

finally:

self._remove_temp_content()

"""Validates and returns the correspoinding value defined in the config.

Keyword arguments:

config - the instance of the configparser

group - the name of the group containing the property

name - the name of the property to get that value for

variable_type - the type of property, 'path', 'mapping' 'bool', otherwise return the raw string

required - if the option is required and none is found than raise an exception

"""

try:

value = config.get(group, name)

if value == '':

raise configparser.NoOptionError(name, group)

if variable_type == 'path':

return os.path.normpath(value)

elif variable_type == 'mapping':

return list(v.split(',') for v in value.split(';'))

elif variable_type == 'bool':

return value.lower() == 'true'

else:

return value

except (configparser.NoSectionError, configparser.NoOptionError):

if required:

raise

elif variable_type == 'bool':

return False

else: