def fetch_weather_data():
"""Fetches weather data from openweathermap.org.
Data is cached in /run/shm, updates after 1000 seconds.
Returns a dict/list data structure derived from the JSON data.
"""
c = None
try:
f = open(SAVEDIR+'/weather', 'rd')
c = pickle.load(f)
f.close()
except: pass
if c is None or c['checktime'] < time.time() - 1000.0:
try:
weatherdata = urlopen(WEATHERDATAURL % (WEATHERCITY, WEATHERAPPID))
forecastdata = urlopen(WEATHERFORECASTURL % (WEATHERCITY, WEATHERAPPID))
c = jsonload(weatherdata)
c['forecast'] = jsonload(forecastdata)
c['checktime'] = time.time()
try:
f = open(SAVEDIR+'/weather', 'wb')
pickle.dump(c, f)
f.close()
except Exception as e:
print('ERR: cannot save weather in %s/weather' % SAVEDIR)
except Exception as e:
print("ERR: cannot fetch weather: " + e.message)
return c
def urbandictionary(event, bot):
""" urbandictionary [TERM]. Searches Urban Dictionary for TERM if supplied.
Otherwise a random definition will be displayed."""
if not event.argument:
json_obj = jsonload(urlopen(RANDOM_URL))
else:
json_obj = jsonload(urlopen(API_URL + urlquote(event.argument)))
if not json_obj['list']:
return bot.say("No definition found for '%s'." % bold(event.argument))
return bot.say(format_definition(json_obj['list'][0]))
def readGoogleEngineAsset(asset_id):
asset_url = (
"https://www.googleapis.com/mapsengine/v1/tables/"
+ asset_id
+ "/features?version=published&key="
+ environ.get("GOOGLE_ENGINE_KEY")
)
done = False
readings = []
url = asset_url
pages = []
# Too many requests in too short a time
# can cause a 429 error
# The Internet connection may also cut out
# Right now either of those will crash the program
# TODO: create try-catch statements for urlopen to improve robustness
# TODO: Bad JSON is also not inconceivable
while not done:
f = urlopen(url)
data = jsonload(f)
f.close()
if "nextPageToken" in data:
url = asset_url + "&pageToken=" + str(data["nextPageToken"])
else:
done = True
pages.append(data)
return pages
def main():
f1 = open("conf_path", "r")
conf_path = f1.readline()
f1.close()
# Retrieve the configuration file
f = open(conf_path, "r")
conf = jsonload(f)
f.close()
services = {
"route": route,
"train": train,
"predict": predict,
"source": source,
"register": register,
"deregister": deregister,
"validate": validate,
}
# This script pulls from the MySQL database
# Should be served over https, otherwise passwords are
# available on the wire as plain text
form = cgi.FieldStorage()
if "service" in form:
result = services[form.getfirst("service")](form, conf)
else:
result = ServiceResult(
"text/html",
"<!DOCTYPE html><html><head><title>Failed Request</title></head><body>Request must contain a service argument</body></html>",
)
# For now expect CGI, but in the future use FastCGI
out_pipe = stdout
result.writeToFile(out_pipe)
def load_settings(settings_file, default_settings):
try:
with open(settings_file, 'r') as f:
settings = jsonload(f)
except Exception as e:
sg.popup_quick_message(f'exception {e}', 'No settings file found... will create one for you', keep_on_top=True, background_color='red', text_color='white')
settings = default_settings
save_settings(settings_file, settings, None)
return settings
def cancel(bot, update, user_data):
if 'lang' in user_data:
lang = user_data['lang']
else:
lang = getLang(update.message.from_user)
with open("{0}/{1}.json".format(transDir, lang), 'r') as language:
botText = jsonload(language)
user_data.clear()
bot.send_message(chat_id=update.message.chat_id, text=botText['cancelled'])
return ConversationHandler.END
def read_patterns(self, source=None):
if source is None:
source = self.source
data = None
with open(self.source, 'r') as f:
data = jsonload(f)
try:
return data['patterns']
except KeyError:
return []
def _processCmdLineArgs(self):
try:
self._opts, self._args = getopt.getopt(
self._cmdLineArgs, 'h', ['help', 'env', 'pickle=', 'json='])
except getopt.GetoptError as v:
# print help information and exit:
print(v)
print(self.usage())
sys.exit(2)
for o, a in self._opts:
if o in ('-h', '--help'):
print(self.usage())
sys.exit()
if o == '--env':
self._template.searchList().insert(0, os.environ)
if o == '--pickle':
if a == '-':
if hasattr(sys.stdin, 'buffer'):
stdin = sys.stdin.buffer # Read binary data from stdin
else:
stdin = sys.stdin
unpickled = load(stdin)
self._template.searchList().insert(0, unpickled)
else:
f = open(a, 'rb')
unpickled = load(f)
f.close()
self._template.searchList().insert(0, unpickled)
if o == '--json':
if a == '-':
if hasattr(sys.stdin, 'buffer'):
stdin = sys.stdin.buffer # Read binary data from stdin
else:
stdin = sys.stdin
unjsoned = jsonload(stdin)
self._template.searchList().insert(0, unjsoned)
else:
f = open(a, 'r')
unjsoned = jsonload(f)
f.close()
self._template.searchList().insert(0, unjsoned)
def configuration(key, config_file='config.json'):
if consul_present():
c = Consul()
try:
value = c.kv.get(key)[1]['Value'].decode()
config = jsonloads(value)
return config
except ConsulException:
print("==> Configurator: Couldn't get configuration from Consul, "
"reading from file")
return jsonload(open(config_file, 'r'))
def load_settings(self, json_file_path=None):
if json_file_path:
try:
with open(json_file_path, 'r') as f:
return jsonload(f)
except FileNotFoundError as e:
print(f'{e} \nLoading previous settings')
try:
with open(self.previous_settings_file_name, 'r') as f:
return jsonload(f)
except FileNotFoundError:
print('No previous settings found, loading default')
try:
with open(self.default_settings_file_name, 'r') as f:
return jsonload(f)
except FileNotFoundError:
print('No default settings found, making some new ones :)')
self.make_default_settings()
with open(self.default_settings_file_name, 'r') as f:
return jsonload(f)
def load_settings():
try:
with open(SETTINGS_FILE, 'r') as f:
settings = jsonload(f)
except:
sg.popup_quick_message(
'No settings file found... will create one for you',
keep_on_top=True,
background_color='red',
text_color='white')
settings = change_settings(DEFAULT_SETTINGS)
save_settings(settings)
return settings
def read_check_patterns(self, source=None):
if source is None:
source = self.source
data = None
with open(self.source, 'r') as f:
data = jsonload(f)
try:
x = data['check_patterns']
if x == 'True':
return True
if x == 'False':
return False
except KeyError:
return True
def read_child_vocabs(self, source=None):
if source is None:
source = self.source
data = None
with open(self.source, 'r') as f:
data = jsonload(f)
try:
result = [
ControlledVocabFromJson(join(self.sourceroot, x))
for x in data['child_vocabs']
]
return result
except KeyError:
return []
def setLang(bot, update, user_data):
# Check if input is valid and set language
if update.message.text in langs:
user_data['lang'] = update.message.text
else:
with open(
"{0}/{1}.json".format(transDir,
getLang(update.message.from_user)),
'r') as language:
botText = jsonload(language)
bot.send_message(chat_id=update.message.chat_id,
text=botText['useKeyboard'],
reply_markup=createReplyKeyboard(
botText['underUssageKeyboard']))
return SETLANG
with open("{0}/{1}.json".format(transDir, user_data['lang']),
'r') as language:
botText = jsonload(language)
# Send message to enter email
bot.send_message(chat_id=update.message.chat_id,
text=botText['langSet'] + botText['enterEmail'],
reply_markup=ReplyKeyboardRemove())
return EMAIL
def read_edit_before_build(self, source=None):
if source is None:
source = self.source
data = None
with open(self.source, 'r') as f:
data = jsonload(f)
try:
x = data['edit_before_build']
if x == 'True':
return True
if x == 'False':
return False
except KeyError:
return False
def read_case_sensitive(self, source=None):
if source is None:
source = self.source
data = None
with open(self.source, 'r') as f:
data = jsonload(f)
try:
x = data['case_sensitive']
if x == 'True':
return True
if x == 'False':
return False
except KeyError:
return False
def get(self):
if not current_app.debug:
current_app.config['RESTFUL_JSON'] = {}
url = f"{current_app.config['LOGS_BASE']}api/getposts"
# Get data
user = request.authorization.username
pw = request.authorization.password
req = requests.get(url, auth=HTTPBasicAuth(user, pw))
items = jsonload(req.content)
output = map(self.create_data_map, items.items())
return {'posts': [*output]}, 200
def confirmation(bot, update, user_data):
with open("{0}/{1}.json".format(transDir, user_data['lang']),
'r') as language:
botText = jsonload(language)
if update.message.text == user_data['code']:
oldID = dbFuncs.getTelegramID(mail=user_data['mail'])
if not helpFuncs.sendMail(
igmimail, [user_data['mail']], botText['confirmedMail'].format(
igmimail, user_data['mail'])):
if oldID and oldID[0] != update.message.chat_id:
bot.send_message(chat_id=oldID[0],
text=botText['foreignLogin'])
try:
bot.delete_message(chat_id=oldID[0], message_id=oldID[2])
except:
bot.edit_message_text(chat_id=oldID[0],
message_id=oldID[2],
text="⬇")
bot.send_message(chat_id=update.message.chat_id,
text=botText['confirmed'])
if oldID:
try:
bot.delete_message(chat_id=oldID[0], message_id=oldID[2])
except:
bot.edit_message_text(chat_id=oldID[0],
message_id=oldID[2],
text="⬇")
dbFuncs.insertMember(
user_data['mail'], user_data['lang'],
update.message.from_user['id'],
bot.send_message(chat_id=update.message.chat_id,
text=botText['mainScreen'],
reply_markup=createInlineKeyboard(
botText['mainScreenKeyboard'])).message_id)
return ConversationHandler.END
else:
user_data['tries'] -= 1
if user_data['tries'] > 0:
bot.send_message(chat_id=update.message.chat_id,
text=botText['wrongCode'].format(
user_data['tries']))
return CONFIRMATION
else:
bot.send_message(
update.message.chat_id,
text=botText['wrongCodeExit']) #TODO set a better message
user_data.clear()
return ConversationHandler.END
def load_parameters(parameters_file, default_parameters,
PARAMETER_KEYS_TO_ELEMENT_KEYS):
try:
with open(parameters_file, 'r') as f:
parameters = jsonload(f)
except Exception as e:
sg.popup_quick_message(
f'exception {e}',
'No parameters file found... will create one for you',
keep_on_top=True,
background_color='red',
text_color='white')
parameters = default_parameters
save_parameters(parameters_file, parameters, None,
PARAMETER_KEYS_TO_ELEMENT_KEYS)
return parameters
def __init__(self, path_to_image_folder=TEST_DIRECTORY):
## Read config
with open(self.CONFIG_PATH, 'r') as configfile:
data = jsonload(configfile)
self.controls = data['controls']
self.labels = list(data['labels'].keys())
self.color_map = {
key: value['color']
for key, value in data['labels'].items()
}
## Read image
if not os.path.isdir(path_to_image_folder):
raise ValueError(
f"The path \'{path_to_image_folder}\' is not a valid directory!"
)
self._path_to_image_folder = path_to_image_folder
self._paths = [
os.path.join(path_to_image_folder, filename)
for filename in os.listdir(self.path_to_image_folder)
if os.path.splitext(filename)[1] in
PascalVocAnnotator.SUPPORTED_FILE_EXTENSIONS
]
self._paths.sort(key=os.path.getmtime)
self._current_image = None
self._saved_annotations = Annotation.load_annotations(
self.path_to_image_folder,
self.color_map,
annotation_dir=ANNOTATION_DEFAULT_DIR)
self._window = None
self.tag_index = 0
self._annotations = []
self._annotation_in_progress = None
self._changed = False
self.index = 0
self._key_events = {
self.controls['NEXT']: self._next,
self.controls['PREV']: self._prev,
self.controls['UNDO']: self._undo,
self.controls['CLEAR']: self._clear
}
def check_google_capatcha(request):
if settings.DEBUG:
return True
recaptcha_is_valid = False
try:
cap_values = {
'secret': settings.GOOGLE_RECAPTCHA_SECRET_KEY,
'response': request.POST.get('g-recaptcha-response')
}
cap_rep = post_req('https://www.google.com/recaptcha/api/siteverify',
data=cap_values)
cp_rep_json = jsonload(cap_rep.text)
if cp_rep_json['success']:
recaptcha_is_valid = True
except Exception as e:
LogThat(msg="check_google_capatcha {}".format(e))
return recaptcha_is_valid
def load_settings(settings_file, default_settings):
try:
with open(settings_file, 'r') as f:
settings = jsonload(f)
except FileNotFoundError:
if sg.__name__ == 'PySimpleGUI' or no_PySimpleGUI_module:
settings = default_settings
save_settings(settings_file, settings, None)
else:
settings = None
print(
"PLEASE SET `text_mode` TO `False` IN ORDER TO RUN THIS DEMO "
"FIRST TIME WITH GUI. SET UP THE MOCK-UP CONFIGURATIONS,"
"THEN EXIT THE PROGRAM AND SET `text_mode` TO `True` TO TRY THE GUI-LESS MODE."
)
return settings
def sendMail(bot, user_data, chatId):
with open("{0}/{1}.json".format(transDir, user_data['lang']),
'r') as language:
botText = jsonload(language)
# Get mail text and insert some values inside
mailText = botText['mailText'].format(igmimail, user_data['mail'],
user_data['code'])
# User gets 5 tries to insert the right code (maybe too generous, rework later)
user_data['tries'] = 5
# Try to send the mail. If it worked, tell the user. If not, tell the user too
if helpFuncs.sendMail(igmimail, [user_data['mail']], mailText):
bot.send_message(chat_id=chatId, text=botText['codeSent'])
logger.info(user_data['code'])
return CONFIRMATION
else:
bot.send_message(chat_id=chatId, text=botText['codeNotSent'])
return EMAIL
def mailUnderUsage(bot, update, user_data):
choice = update.message.text
with open("{0}/{1}.json".format(transDir, user_data['lang']),
'r') as language:
botText = jsonload(language)
if choice == botText['underUsageKeyboard'][0][0][0]:
return sendMail(bot, user_data, update.message.from_user['id'])
elif choice == botText['underUsageKeyboard'][0][1][0]:
bot.send_message(chat_id=update.message.from_user['id'],
text=botText['enterEmail'])
return EMAIL
else:
bot.send_message(chat_id=update.message.chat_id,
text=botText['useKeyboard'],
reply_markup=createReplyKeyboard(
botText['underUssageKeyboard']))
return MAILUNDERUSAGE
def get_contributions(self):
run('node {} {} {}'.format(self.scraper, self.user,
self.data)) # call the scraper submod
with open(self.data) as source:
all_contributions = jsonload(source)
if len(all_contributions) < self.window:
self.window = len(
all_contributions
) # Just in case contributions are fewer than 256
self.contributions.extend(
all_contributions[-1:0 - self.window -
1:-1]) # Step backward from most recent
self.contributions.reverse(
) # To return the list to its original order
print('.json file parsed...')
return self.contributions
def load_settings():
global API_KEY
try:
with open(SETTINGS_FILE, 'r') as f:
settings = jsonload(f)
API_KEY = settings['api key']
except:
sg.popup_quick_message(
'No settings file found... will create one for you',
keep_on_top=True,
background_color='red',
text_color='white',
auto_close_duration=3,
non_blocking=False)
settings = change_settings(DEFAULT_SETTINGS)
save_settings(settings)
return settings
def start(bot, update):
# Try to get user's language. Otherwise, use english
lang = getLang(update.message.from_user)
# Get the right translation and store it in variable botText
with open("{0}/{1}.json".format(transDir, lang), 'r') as language:
botText = jsonload(language)
# Check if telegram-user is already registered and send message if yes
if dbFuncs.isTGMember(tid=update.message.from_user['id']):
bot.send_message(chat_id=update.message.chat_id,
text=botText['tgUnderUsage'],
reply_markup=createReplyKeyboard(
botText['underUsageKeyboard']))
return TGUNDERUSAGE
# Send welcome message
bot.send_message(chat_id=update.message.chat_id,
text=botText['welcome'],
reply_markup=createReplyKeyboard([langs]))
return SETLANG
def load_json_parameters(path, value_cast=False):
# loads in json parameters and turns it into a dictionary
try:
with codecsopen(path, 'r', encoding='utf-8') as f:
parameters = []
try:
parameters = jsonload(f)
if value_cast:
parameters = _cast_parameters(parameters)
except ValueError:
raise ValueError("Parameters file is formatted incorrectly!")
f.close()
except IOError:
raise IOError("Incorrect path to parameters given! Please try again.")
return parameters
async def run(r):
url = "http://api.myntra.com/assorted/v1/urlrouter/?path={}"
baseUrl = 'https://www.jabong.com/'
tasks = []
# Fetch all responses within one Client session,
# keep connection alive for all requests.
connector = TCPConnector(verify_ssl=False)
async with ClientSession(connector=connector) as session:
data = list()
dataappend = data.append
# with open('2000URLs', 'r', newline='', encoding='utf-8') as csvfile:
# reader = csvreader(csvfile, lineterminator='\n', delimiter=',', quotechar='"',
# escapechar='\\', doublequote=False, quoting=QUOTE_NONE, strict=True)
reader = pandas.read_csv('./SeoData(4473).csv')
#rows = reader['Old URL']
#fetcher = len(rows) / 1000
#while i <= fetcher:
for row in reader['Old URL']:
dataappend(row)
#queryparams = (quote(i) for i in row)
urlAppendStr = row.replace(baseUrl, '')
updatedUrl = url.format(quote_plus(urlAppendStr.strip('/')))
#print(updatedUrl)
task = asyncio.ensure_future(
fetch(updatedUrl, session))
tasks.append(task)
responses = await asyncio.gather(*tasks)
# you now have all response bodies in this variable
with open('OldUrlToNewUrl.csv', 'w+', newline='', encoding='utf-8', buffering=1) as csvoutfile:
writer = csvwriter(csvoutfile, lineterminator='\n', delimiter=',', quotechar='"',
escapechar='\\', doublequote=False, quoting=QUOTE_NONE, strict=True)
index = 0
for x in data:
try:
jsonStr = jsonload(responses[index])
if jsonStr["data"]:
a = True
writer.writerow([x, getNewUrl(baseUrl, jsonStr["data"])])
except:
writer.writerow([x, "failed"])
index += 1
def load_from_json(dst: Path = SETTING_PATH):
print(f"< load setting {dst}")
try:
with open(dst, "r") as f:
userSettingDict = jsonload(f)
except FileNotFoundError:
userSettingDict = DEFAULT_SETTING
print(f"setting file {dst} not found")
except json.decoder.JSONDecodeError:
dst.unlink()
userSettingDict = DEFAULT_SETTING
print(f"setting file {dst} is broken")
try:
userSetting = UserSetting(**userSettingDict)
except TypeError:
dst.unlink()
userSettingDict = DEFAULT_SETTING
userSetting = UserSetting(**userSettingDict)
print("userSetting broken >")
return userSetting
def load_json_parameters(path: str, value_cast: bool = False) -> dict:
"""Load JSON Parameters.
Given a path to a json of parameters, convert to a dictionary and optionally
cast the type.
Expects the following format:
"fake_data": {
"value": "true",
"section": "bci_config",
"readableName": "Fake Data Sessions",
"helpTip": "If true, fake data server used",
"recommended_values": "",
"type": "bool"
}
PARAMETERS
----------
:param: path: string path to the parameters file.
:param: value_case: True/False cast values to specified type.
"""
# loads in json parameters and turns it into a dictionary
try:
with codecsopen(path, 'r', encoding='utf-8') as f:
parameters = []
try:
parameters = jsonload(f)
if value_cast:
parameters = _cast_parameters(parameters)
except ValueError:
raise ValueError(
"Parameters file is formatted incorrectly!")
f.close()
except IOError:
raise IOError("Incorrect path to parameters given! Please try again.")
return parameters
def __init__(self, layer, stack=None, stackfile=None):
""" Make a new Composite.Provider.
Arguments:
layer:
The current TileStache.Core.Layer
stack:
A list or dictionary with configuration for the image stack, parsed
by build_stack(). Also acceptable is a URL to a JSON file.
stackfile:
*Deprecated* filename for an XML representation of the image stack.
"""
self.layer = layer
if type(stack) in (str, unicode):
stack = jsonload(
urlopen(urljoin(layer.config.dirpath, stack)).read())
if type(stack) in (list, dict):
self.stack = build_stack(stack)
elif stack is None and stackfile:
#
# The stackfile argument is super-deprecated.
#
stackfile = pathjoin(self.layer.config.dirpath, stackfile)
stack = parseXML(stackfile).firstChild
assert stack.tagName == 'stack', \
'Expecting root element "stack" but got "%s"' % stack.tagName
self.stack = makeStack(stack)
else:
raise Exception(
'Note sure what to do with this stack argument: %s' %
repr(stack))
def _get_latest_version_info(self):
# Get index that contains file URLs and latest
# version strings from Swift Nav's website.
try:
f = urlopen(INDEX_URL, timeout=120)
self.index = jsonload(f)
f.close()
except URLError:
self._write("\nError: Failed to download latest file index from Swift Navigation's website (%s). Please visit our website to check that you're running the latest Piksi firmware and Piksi console.\n" % INDEX_URL)
return
# Make sure index contains all keys we are interested in.
try:
self.newest_stm_vers = self.index['piksi_v2.3.1']['stm_fw']['version']
self.newest_nap_vers = self.index['piksi_v2.3.1']['nap_fw']['version']
self.index['piksi_v2.3.1']['stm_fw']['url']
self.index['piksi_v2.3.1']['nap_fw']['url']
self.newest_console_vers = self.index['piksi_v2.3.1']['console']['version']
except KeyError:
self._write("\nError: Index downloaded from Swift Navigation's website (%s) doesn't contain all keys. Please contact Swift Navigation.\n" % INDEX_URL)
return
def setMail(bot, update, user_data):
with open("{0}/{1}.json".format(transDir, user_data['lang']),
'r') as language:
botText = jsonload(language)
# Save whatever is sent as lowercase
email = update.message.text.lower()
# When the mail is not registered, tell the user
if not dbFuncs.isIGMIMember(email):
bot.send_message(chat_id=update.message.chat_id,
text=botText['notRegistered'])
return EMAIL
# Otherwise store mail temporarily and generate a unique code
user_data['mail'] = email
user_data['code'] = helpFuncs.createCode()
# When the mail is already registered from a telegram user, ask for confirmation
if dbFuncs.isTGMember(email=email):
bot.send_message(chat_id=update.message.chat_id,
text=botText['mailUnderUsage'],
reply_markup=createReplyKeyboard(
botText['underUsageKeyboard']))
return MAILUNDERUSAGE
# Otherwise, send the confirmation mail and tell the user if it worked or not
return sendMail(bot, user_data, update.message.chat_id)
def __init__(self, layer, stack=None, stackfile=None):
""" Make a new Composite.Provider.
Arguments:
layer:
The current TileStache.Core.Layer
stack:
A list or dictionary with configuration for the image stack, parsed
by build_stack(). Also acceptable is a URL to a JSON file.
stackfile:
*Deprecated* filename for an XML representation of the image stack.
"""
self.layer = layer
if type(stack) in (str, unicode):
stack = jsonload(urlopen(urljoin(layer.config.dirpath, stack)).read())
if type(stack) in (list, dict):
self.stack = build_stack(stack)
elif stack is None and stackfile:
#
# The stackfile argument is super-deprecated.
#
stackfile = pathjoin(self.layer.config.dirpath, stackfile)
stack = parseXML(stackfile).firstChild
assert stack.tagName == 'stack', \
'Expecting root element "stack" but got "%s"' % stack.tagName
self.stack = makeStack(stack)
else:
raise Exception('Note sure what to do with this stack argument: %s' % repr(stack))
async def run(r):
url = "https://gw.jabong.com/v1/search/?url={}"
tasks = []
# Fetch all responses within one Client session,
# keep connection alive for all requests.
connector = TCPConnector(verify_ssl=False)
async with ClientSession(connector=connector) as session:
data = list()
dataappend = data.append
# with open('2000URLs', 'r', newline='', encoding='utf-8') as csvfile:
# reader = csvreader(csvfile, lineterminator='\n', delimiter=',', quotechar='"',
# escapechar='\\', doublequote=False, quoting=QUOTE_NONE, strict=True)
reader = pandas.read_csv('./OldUrlToNewUrl.csv')
for row in reader['old_urls']:
dataappend(row)
#queryparams = (quote(i) for i in row)
urlAppendStr = row.replace('https://www.jabong.com', '')
task = asyncio.ensure_future(
fetch(url.format(urlAppendStr), session))
tasks.append(task)
responses = await asyncio.gather(*tasks)
# you now have all response bodies in this variable
with open('OldUrlCounts.csv', 'w', newline='', encoding='utf-8', buffering=1) as csvoutfile:
writer = csvwriter(csvoutfile, lineterminator='\n', delimiter=',', quotechar='"',
escapechar='\\', doublequote=False, quoting=QUOTE_NONE, strict=True)
index = 0
for x in data:
try:
jsonStr = jsonload(responses[index])
if jsonStr["data"]:
a = True
writer.writerow([x, jsonStr["data"]["summary"]["productCnt"]])
except:
writer.writerow([x, "failed"])
index += 1
def __init__(self, root_dir=''): f = urlopen(INDEX_URL) self.index = jsonload(f) self.root_dir = '' f.close()
for lon in range(-164, -127, 4):
for lat in range(52, 69, 4):
yield lat, lon
if __name__ == '__main__':
tt_host = 'this-tract.s3.amazonaws.com'
fcc_host, fcc_path = 'data.fcc.gov', '/api/block/2000/find'
for (lat, lon) in generate_locations():
q = {'format': 'json', 'latitude': lat, 'longitude': lon}
conn = HTTPConnection(fcc_host)
conn.request('GET', fcc_path + '?' + urlencode(q))
resp = conn.getresponse()
data = jsonload(resp)
if resp.status == 500:
assert data['Err']['msg'] == 'There are no results for this location', data['Err']
print '--'
continue
assert resp.status == 200
assert data['status'] == 'OK'
block = data['Block']['FIPS']
state, county, tract = block[0:2], block[2:5], block[5:11]
print float(lat), float(lon), state, county, tract
def run(self):
# Create prompt objects.
fw_update_prompt = \
UpdatePrompt(
title='Firmware Update',
actions=[update_button, close_button],
update_callback=self.manage_firmware_updates,
)
console_outdated_prompt = \
UpdatePrompt(
title="Console Outdated",
actions=[close_button],
)
# Get index that contains file URLs and latest
# version strings from Swift Nav's website.
try:
f = urlopen(INDEX_URL)
index = jsonload(f)
f.close()
except URLError:
self.write("\nError: Failed to download latest file index from Swift Navigation's website (%s). Please visit our website to check that you're running the latest Piksi firmware and Piksi console.\n\n" % INDEX_URL)
return
# Make sure index contains all keys we are interested in.
try:
index['piksi_v2.3.1']['stm_fw']['version']
index['piksi_v2.3.1']['stm_fw']['url']
index['piksi_v2.3.1']['nap_fw']['version']
index['piksi_v2.3.1']['nap_fw']['url']
index['piksi_v2.3.1']['console']['version']
except KeyError:
self.write("\nError: Index downloaded from Swift Navigation's website (%s) doesn't contain all keys. Please contact Swift Navigation.\n\n" % INDEX_URL)
return
# Make sure settings contains Piksi firmware version strings.
try:
self.settings['system_info']['firmware_version'].value
self.settings['system_info']['nap_version'].value
except:
self.write("\nError: Settings received from Piksi don't contain firmware version keys. Please contact Swift Navigation.\n\n" % INDEX_URL)
return
# Assign text to UpdatePrompt's
fw_update_prompt.text = \
"Local STM Version :\n\t%s\n" % \
self.settings['system_info']['firmware_version'].value + \
"Newest STM Version :\n\t%s\n\n" % \
index['piksi_v2.3.1']['stm_fw']['version'] + \
"Local SwiftNAP Version :\n\t%s\n" % \
self.settings['system_info']['nap_version'].value + \
"Newest SwiftNAP Version :\n\t%s\n\n" % \
index['piksi_v2.3.1']['nap_fw']['version']
console_outdated_prompt.text = \
"Your Console is out of date and may be incompatible\n" + \
"with current firmware. We highly recommend upgrading\n" + \
"to ensure proper behavior.\n\n" + \
"Please visit http://download.swift-nav.com to\n" + \
"download the newest version.\n\n" + \
"Local Console Version :\n\t" + \
CONSOLE_VERSION + \
"\nNewest Console Version :\n\t" + \
index['piksi_v2.3.1']['console']['version'] + "\n"
# Do local version match latest from website?
self.stm_fw_outdated = index['piksi_v2.3.1']['stm_fw']['version'] \
!= self.settings['system_info']['firmware_version'].value
self.nap_fw_outdated = index['piksi_v2.3.1']['nap_fw']['version'] \
!= self.settings['system_info']['nap_version'].value
self.console_outdated = index['piksi_v2.3.1']['console']['version'] \
!= CONSOLE_VERSION
# Get firmware files from Swift Nav's website.
self.nap_ihx = None
if self.nap_fw_outdated:
try:
f = urlopen(index['piksi_v2.3.1']['nap_fw']['url'])
self.nap_ihx = IntelHex(f)
f.close()
except URLError:
self.write("\nError: Failed to download latest Piksi SwiftNAP firmware from Swift Navigation's website (%s). Please visit our website to check that you're running the latest firmware.\n" % index['piksi_v2.3.1']['nap_fw']['url'])
self.stm_ihx = None
if self.stm_fw_outdated:
try:
f = urlopen(index['piksi_v2.3.1']['stm_fw']['url'])
self.stm_ihx = IntelHex(f)
f.close()
except URLError:
self.write("\nError: Failed to download latest Piksi STM firmware from Swift Navigation's website (%s). Please visit our website to check that you're running the latest firmware.\n" % index['piksi_v2.3.1']['stm_fw']['url'])
# Prompt user to update firmware(s). Only update if firmware was
# successfully downloaded. If both are out of date, only allow
# update if we successfully downloaded both files.
if (self.stm_fw_outdated and self.stm_ihx and not self.nap_fw_outdated) or \
(self.nap_fw_outdated and self.nap_ihx and not self.stm_fw_outdated) or \
(self.stm_fw_outdated and self.stm_ihx and \
self.nap_fw_outdated and self.nap_ihx):
fw_update_prompt.run()
# For timing aesthetics between windows popping up.
sleep(0.5)
# Check if console is out of date and notify user if so.
if self.console_outdated:
console_outdated_prompt.run()
def __init__(self): f = urlopen(INDEX_URL) self.index = jsonload(f) f.close()
def decode_routes_page(uri):
f = urlopen(uri)
page = jsonload(f)
f.close()
return [page]
def get_translation(path_to_translationJSON):
"""Returns dictionary with translations.json data"""
logger.debug(f'Function start arguments: {locals()}')
with path_to_translationJSON.open() as translationjson:
return (jsonload(translationjson))
def read_rec_def(path):
return jsonload(open(path, "r"))
"""