def cartodb_append(sqlite_db_path,
out_cartodb_name,
gfw_env,
where_clause=None):
"""
Append a local FC to a cartoDB dataset
:param sqlite_db_path: path to local sqlite db
:param out_cartodb_name: cartoDB table
:param gfw_env: gfw_env
:param where_clause: where_clause to apply to the dataset
:return:
"""
key = token_util.get_token(
settings.get_settings(gfw_env)['cartodb']['token'])
account_name = get_account_name(gfw_env)
# Help: http://www.gdal.org/ogr2ogr.html
# The -dim 2 option ensures that only two dimensional data is created; no Z or M values
cmd = [
r'C:\Program Files\GDAL\ogr2ogr', '--config', 'CARTODB_API_KEY', key,
'-append', '-skipfailures', '-t_srs', 'EPSG:4326', '-f', 'CartoDB',
'-nln', out_cartodb_name, '-dim', '2',
'CartoDB:{0}'.format(account_name)
]
cmd = add_fc_to_ogr2ogr_cmd(sqlite_db_path, cmd)
cmd = add_where_clause_to_ogr2ogr_cmd(where_clause, cmd)
util.run_subprocess(cmd)
def send_email(body_text):
"""
Send an email given a body text
:param body_text: text to include in the email
:return:
"""
username = '******'
fromaddr = "{0}@gmail.com".format(username)
msg = MIMEMultipart()
msg['From'] = fromaddr
msg['Subject'] = "gfw-sync2 results"
msg.attach(MIMEText(body_text, 'html'))
server = smtplib.SMTP('smtp.gmail.com', 587)
server.ehlo()
server.starttls()
server.login(fromaddr, token_util.get_token(username))
for toaddr in ["*****@*****.**", "*****@*****.**", "*****@*****.**"]:
msg['To'] = toaddr
text = msg.as_string()
server.sendmail(fromaddr, toaddr, text)
server.quit()
def get_api_key_and_url(gfw_env):
key = token_util.get_token(
settings.get_settings(gfw_env)['cartodb']['token'])
api_url = settings.get_settings(gfw_env)["cartodb"]["sql_api"]
# sync_api = settings.get_settings(gfw_env)["cartodb"]["synchronization_api"]
return key, api_url #, sync_api
def hit_vizz_webhook(dataset_name):
# POST to kick off GLAD Alerts subscriptions now that we've updated the country-pages data
api_token = get_token('gfw-rw-api-prod')
headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer {0}'.format(api_token)}
url = r'https://production-api.globalforestwatch.org/subscriptions/notify-updates/{0}'.format(dataset_name)
r = requests.post(url, headers=headers)
logging.debug(r.text)
print r.text
def kickoff(proc_name, regions, years, gfw_env):
token_info = token_util.get_token('s3_read_write.config')
aws_access_key = token_info[0][1]
aws_secret_key = token_info[1][1]
region_str = ' '.join(regions.split(';'))
year_str = ' '.join(years.split(';'))
if proc_name == 'umd_landsat_alerts':
final_cmd = 'python /home/ubuntu/glad-data-management/update_glad_data.py ' \
'-r {} -y {}'.format(region_str, year_str)
if gfw_env == 'staging':
final_cmd += ' --staging'
else:
# Generate the mapnik tiles and push to s3
tile_cmd = 'python /home/ubuntu/mapnik-forest-change-tiles/generate-tiles.py'
tile_cmd += ' -l terrai -r {} -y {} --world --threads 60 '.format(region_str, year_str)
# Write the rasters to point and push to s3
point_cmd = 'python /home/ubuntu/raster-vector-to-tsv/processing/utilities/weekly_updates.py'
point_cmd += ' -l terrai -r {} -y {}'.format(region_str, year_str)
# add staging flags if necessary
if gfw_env == 'staging':
tile_cmd += ' --staging'
point_cmd += ' --staging'
# required because fabric will wait if process is not actively connected to this machine
# can't do multiple fabric.api.run calls for some reason
# http://docs.fabfile.org/en/1.6/faq.html#my-cd-workon-export-etc-calls-don-t-seem-to-work
final_cmd = ' && '.join([tile_cmd, point_cmd])
# Required, even though these are set for ubuntu in .bashrc
with fabric.api.shell_env(S3_ACCESS_KEY=aws_access_key, S3_SECRET_KEY=aws_secret_key):
fabric.api.run(final_cmd)
# Important to signal the global_forest_change_layer to kill the subprocess
print '****FAB SUBPROCESS COMPLETE****'
def spider_taobao_address(self):
member_url = f'https://member1.taobao.com/member/fresh/deliver_address.htm'
# search_url = 'http://pv.sohu.com/cityjson%3Fie=utf-8'
print(self.cookies)
res = self.req_session.get(member_url,
headers=headers2,
proxies=proxies,
verify=False,
timeout=self.timeout)
print(res.text)
url = 'https://h5api.m.taobao.com/h5/mtop.taobao.mbis.getdeliveraddrlist/1.0/?jsv=2.4.2&api=mtop.taobao.mbis.getDeliverAddrList&v=1.0&ecode=1&needLogin=true&dataType=jsonp&type=jsonp&callback=mtopjsonp4'
appkey = '27769795'
token = token_util.get_token(self.req_session.cookies)
url = token_util.init_url('{}', url, appkey, token, self.ctx)
response = self.req_session.get(url,
headers=headers,
proxies=proxies,
verify=False,
timeout=self.timeout)
print(response.text)
def validate_osm_source(osm_source):
"""
Used to check that all the job uids for an osm datasource exists for our osm HOT export account
:param osm_source: list of job uids. Example: 2c5d8ae4-940a-445b-b34a-0e922a40598c,
3b88a831-e2a0-4c80-8b25-cd3bc64ffd2f
:return: true/false based on if all job uids are valid
"""
osm_id_list = osm_source.split(',')
auth_key = get_token('thomas.maschler@hot_export')
headers = {"Content-Type": "application/json", "Authorization": "Token " + auth_key}
url = "http://export.hotosm.org/api/runs?job_uid={0}"
is_valid = True
for osm_id in osm_id_list:
request = urllib2.Request(url.format(osm_id))
for key, value in headers.items():
request.add_header(key, value)
try:
# If the input uid is in the correct format, but doesn't exist, will return an empty list
if json.load(urllib2.urlopen(request)):
# Success! We have a response, and can assume that the job uid exists
pass
else:
is_valid = False
# If formatted improperly/etc, will return a 500 HTTP Error
except urllib2.HTTPError:
is_valid = False
if not is_valid:
logging.error("HOT OSM job uid {0} is invalid\n".format(osm_id))
break
return is_valid