def make_query_url(parameters):
debug_print("Calling make_query_url")
url = 'https://api.twitter.com/1.1/search/tweets.json?lang=en'
for k, v in parameters.items():
url += '&{}={}'.format(k, v)
debug_print("Resulting url: {}".format(url))
return url
def transform_jsonToCsv_socialDistancingData(self, jsonFilename,
csvFilename):
jsonMetaData = []
countyFieldnames = [
'stateFips', 'stateName', 'countyFips', 'countyName'
]
dataFieldnames = [
'date', 'totalGrade', 'visitationGrade', 'encountersGrade',
'travelDistanceGrade'
]
jsonMetaData = self._loadData(jsonFilename)
jsonCountiesData = jsonMetaData['hits']['hits']
singleCountyData = {}
with open(_CSV_Directory_ + csvFilename, 'w') as csvFile:
csvDriver = csv.DictWriter(csvFile,
fieldnames=(countyFieldnames +
dataFieldnames))
csvDriver.writeheader()
for county in jsonCountiesData:
# set county data
singleCountyData = {
field: (county['_source'][field]).encode('utf-8')
for field in countyFieldnames
}
# set Grade of county for each day, specified with 'date'
for data in county['_source']['data']:
singleCountyData.update(
{field: data[field]
for field in dataFieldnames})
csvDriver.writerow(singleCountyData)
debug.debug_print("SUCCESS: transform completed(socialDistancingData)",
2)
def merge_csvFiles_addColumns(self, csvFilename1, csvFilename2, destinationFilename, file1Keys, file2Keys, newColumns):
# Get csvFiles data
csvData1, csvFile1Fieldnames = self._loadData(csvFilename1)
csvData2 = self._loadData(csvFilename2)[0]
# Generate new file fieldnames, add newClumns to fieldnames
mergedDataFieldnames = csvFile1Fieldnames + newColumns
# Merge and save data
with open(_CSV_Directory_ + destinationFilename, 'w') as destinationFile:
csvDriver = csv.DictWriter(destinationFile, fieldnames=mergedDataFieldnames)
csvDriver.writeheader()
for row in csvData1:
# Find correspondingRow on csvData2
found = False
for item in csvData2:
if self._isEqual(self._keyValues(row, file1Keys), self._keyValues(item, file2Keys)):
correspondingRow = {key:item[key] for key in newColumns}
found = True
break
if found == False:
debug.debug_print("key not found:\t" + ', '.join(str(item) for item in self._keyValues(row, file1Keys)), 3)
continue
# Generate and add row to file
row.update(correspondingRow)
csvDriver.writerow(row)
debug.debug_print("SUCCESS: merge completed", 2)
def await_reply(sock, path):
reply = loop_recv(sock)
if reply["status"] == "ok":
debug_print(path + " done loading")
else:
debug_print("Error: " + reply["status"])
return reply
def search(regex, text, flags=0):
"""Search for REGEX in TEXT with optional FLAGS"""
my_re.match = re.search(regex, text, flags)
if my_re.match:
# TODO: while my_re.match.group(i): print my_re.match.group(i)
debug_print("match: %s" % my_re.match.group(), 6)
return (my_re.match)
def transform_jsonToCsv_confirmAndDeathData(self, jsonFilename, csvFilename):
startDay = date.fromisoformat('2020-01-22')
jsonMetaData = []
countyFieldnames = ['stateFIPS', 'stateAbbr', 'countyFIPS', 'county']
dataFieldnames = ['date', 'confirmed', 'deaths']
jsonMetaData = self._loadData(jsonFilename)
singleCountyData = {}
with open(_CSV_Directory_ + csvFilename, 'w') as csvFile:
csvDriver = csv.DictWriter(csvFile, fieldnames=(countyFieldnames+dataFieldnames))
csvDriver.writeheader()
numberOfDays = len(jsonMetaData[0]['deaths'])
for day in range(numberOfDays):
singleCountyData = {'date':(startDay+timedelta(days=day)).isoformat()}
for county in jsonMetaData:
# pass invalid county
if int(county['countyFIPS'], 10) == 0:
continue
# set county data
singleCountyData.update({field:county[field] for field in countyFieldnames})
singleCountyData.update({field:county[field][day] for field in dataFieldnames if field in county})
# remove summation
if day != 0:
singleCountyData.update({field:(county[field][day] - county[field][day - 1]) for field in ['confirmed', 'deaths'] if field in county})
# write in file
csvDriver.writerow(singleCountyData)
debug.debug_print("SUCCESS: transform completed(confirmAndDeathData)", 2)
def test_ColorSensorSpeed():
i = 0
start = timeit.default_timer()
while i < 10000:
i = i + 1
#debug.debug_print(str(ev3.colorSensor_lt.value()))
stop = timeit.default_timer()
debug.debug_print('Time: ', stop - start)
def test_comparison():
i = 0
start = timeit.default_timer()
while i < 10000:
i = i + 1
#debug.debug_print("1")
stop = timeit.default_timer()
debug.debug_print('Time: ', stop - start)
def assume_role(account_id, role_name):
debug_print("Assuming role.....")
role_arn = "arn:aws:iam::{0}:role/{1}".format(account_id, role_name)
client = get_boto_client('sts')
assumed_role = client.assume_role(
RoleArn=role_arn, RoleSessionName="account_vending_machine_lambda")
return assumed_role['Credentials']
def _shutdown_socket(self, file_descriptor):
del self.socket_timing[file_descriptor]
self.poller.unregister(file_descriptor)
if file_descriptor in self.clients:
self.clients[file_descriptor].close()
del self.clients[file_descriptor]
else:
debug_print('tried to close a socket that was not in clients: %i' % file_descriptor)
def get_confirmAndDeathData(self, destinationFilename):
jsonData = requests.get(
'https://usafactsstatic.blob.core.windows.net/public/2020/coronavirus-timeline/allData.json'
).json()
with open(_JSON_Directory_ + destinationFilename, 'w') as jsonFile:
json.dump(jsonData, jsonFile)
debug.debug_print("SUCCESS: data extracted (confirmAndDeathData)", 2)
def _shutdown_socket(self, file_descriptor) -> None:
del self.socket_timing[file_descriptor]
self.poller.unregister(file_descriptor)
if file_descriptor in self.clients:
self.clients[file_descriptor].close()
del self.clients[file_descriptor]
else:
debug_print('tried to close a socket that was not in clients: %i' %
file_descriptor)
def invoke_statemachine(arn, input):
client = get_boto_client("stepfunctions")
account_name = input.get("accountName")
response = client.start_execution(stateMachineArn=arn,
name="{}-creation-{}".format(
account_name, time.time()),
input=json.dumps(input))
debug_print(response)
return (response)
def change_directory(dir):
"""Changes into file system DIR (ignoring errors)"""
debug_print("change_directory(%s)" % dir, 4)
try:
os.chdir(dir)
except:
print_stderr("Unable to change into directory '%s': %s" %
(filename, sys.exc_info()))
return
def make_directory(dir):
"""Creates file system DIR (ignoring errors)"""
debug_print("make_directory(%s)" % dir, 4)
try:
os.mkdir(dir)
except:
print_stderr("Unable to create directory '%s': %s" %
(dir, sys.exc_info()))
return
def do_query_url(url):
debug_print("Sending query to url {}".format(url))
r = requests.get(url, auth=get_oauth())
if r.status_code == 200:
return r.json()
else:
print("Something went wrong with the query")
debug_print(r.text)
return None
def publishVoltage(mqttClient, topic):
while True:
data = {}
data['measured_volts'] = powerSupply.measured_volts
#print("sensor reading published as " + json.dumps(data) )
debug.debug_print(json.dumps(data))
messageJson = json.dumps(data)
mqttClient.publish(topic, messageJson, 1)
time.sleep(1)
def check_files(options):
p = Path(options['output_dir']) / options['search_name']
p.mkdir(parents=True, exist_ok=True)
last_id_path = p / 'last_id.txt'
since_id = 0
if last_id_path.exists():
with last_id_path.open('r') as last_id_file:
since_id = int(last_id_file.read().strip())
debug_print("Previous since id = {}".format(since_id))
return since_id
def write_file(filename, text):
"""Outpus TEXT to FILENAME"""
debug_print("write_file(%s, _)" % filename, 5)
try:
with open(filename, 'w') as f:
f.write(text)
except:
print_stderr("Unable to write file '%s': %s" %
(filename, sys.exc_info()))
return
def get_socialDistancingData(self, stateFIPS, destinationFilename):
jsonData = requests.get(
'https://covid19-scoreboard-api.unacastapis.com/api/search/covidcountyaggregates_v3?q=stateFips:%02d&size=4000'
% (stateFIPS)).json()
with open(_JSON_Directory_ + destinationFilename, 'w') as jsonFile:
json.dump(jsonData, jsonFile)
debug.debug_print(
"SUCCESS: data extracted (socialDistancingData on stateFIPS:%02d)"
% (stateFIPS), 2)
def alt_get_url_source(url, retries=1):
"""Returns HTML source at URL with retries after pausing"""
tries = retries + 1
while (tries > 0):
contents = get_url_source(url)
if not re.search("Error: ERR_ACCESS_DENIED", contents):
break
debug_print("Warning: access denied error; trying again after pause")
tries -= 1
time.sleep(5)
return (contents)
def simplify_csvFile(self, csvFilename, destinationFilename, fieldnames):
# Get csvFile data
csvData = self._loadData(csvFilename)[0]
with open(_CSV_Directory_ + destinationFilename, 'w') as DestinationFile:
csvDriver = csv.DictWriter(DestinationFile, fieldnames)
csvDriver.writeheader()
for row in csvData:
csvDriver.writerow({k:row[k] for k in (fieldnames) if k in row})
debug.debug_print("SUCCESS: simplify completed", 2)
def execute_command(self, command):
if Context().clear_before_command:
self.clear_screen()
self.print_prompt()
print(command)
debug_print(command)
if command in self.commands_dict:
self.commands_dict[command]()
else:
self.run_instruction(command)
def send_param(self, pac, sendStatus=False):
Process.send_param(self, pac, sendStatus)
result = True
## read the data
data, data2 = self.meter.read_data(pac.type, pac.data)
if pac.type != "LONG":
debug_print(" Data: (%s) %s, RAW: %s" % (pac.type, data, data2))
else:
debug_print(" Data: (LONG) %d or (FLOAT) %f" % (data, data2))
## deal with packet contents
if pac.param == 10 and pac.type == "CHAR": # Control device
if data == "I": # Initialise process
print("->INITIALISE NODE %d PROCESS %d" %
(pac.node, pac.process_number))
elif data == "R": # Reset device
print("->RESET MODULE")
if self.temp_pri_node_num >= 0:
self.meter.pri_node_num = self.temp_pri_node_num
if self.temp_sec_node_num >= 0:
self.meter.sec_node_num = self.temp_sec_node_num
if self.temp_next_node_num >= 0:
self.meter.next_node_num = self.temp_next_node_num
if self.temp_last_node_num >= 0:
self.meter.last_node_num = self.temp_last_node_num
elif pac.param == 1 and pac.type == "CHAR": # Set primary node address
print("->SET PRIMARY NODE ADDRESS TO %d" % ord(data))
self.temp_pri_node_num = ord(data)
elif pac.param == 2 and pac.type == "CHAR": # Set secondary node address
print("->SET SECONDARY NODE ADDRESS TO %d" % ord(data))
self.temp_sec_node_num = ord(data)
elif pac.param == 3 and pac.type == "CHAR": # Set next node address
print("->SET NEXT NODE ADDRESS TO %d" % ord(data))
self.temp_next_node_num = ord(data)
elif pac.param == 4 and pac.type == "CHAR": # Set last node address
print("->SET LAST NODE ADDRESS TO %d" % ord(data))
self.temp_last_node_num = ord(data)
elif pac.param == 5 and pac.type == "CHAR": # Set bus management
bm = "EVERYTHING AUTO" if ord(data) == 67 else \
"AUTO ARBITRATION" if ord(data) == 3 else \
"ALWAYS BUSMASTER" if ord(data) == 2 else \
"TEMPORARY" if ord(data) == 1 else "UNKNOWN"
print("->SET BUS MANAGEMENT TO %d (%s)" % (ord(data), bm))
else:
result = False
## send status if we need to
if sendStatus:
self.meter.send_packet(
Status.build(
Container(length=4,
node=pac.node,
status='NO_ERROR',
index=0)))
return result
def run_command(command_line, level=5):
"""Runs COMMAND_LINE and returns the output (as a string), with debug tracing at specified LEVEL"""
# Issue command
debug_print("Running command: %s" % command_line, level=level)
(status, output) = getstatusoutput(command_line)
if (status != 0):
print_stderr("Warning: problem running command (status=%d): %s" %
(status, command_line))
# Return output
debug_print("Command output: %s\n" % output, level=level + 1)
return (output)
def main(event):
data = event[0]
account_id = data.get("accountId")
admin_account_role_name = data.get("adminAccountRole")
admin_account_number = data.get("adminAccount")
message = "New account with ID: {} was created successfully. Role: {} with Trust to Account: {} was setup for Admin Access.".format(
account_id, admin_account_role_name, admin_account_number)
debug_print(message)
postMessageToSns("Account was created", message)
return data
def handler(event, context):
debug_print(json.dumps(event, indent=2))
s3_event = event["Records"][0]["s3"]
s3_bucket = s3_event["bucket"]["name"]
s3_object = s3_event["object"]["key"]
client = get_boto_client("s3")
response = client.get_object(Bucket=s3_bucket, Key=s3_object)
content = json.loads(response['Body'].read().decode('utf-8'))
debug_print(json.dumps(content, indent=2))
step_function_arn = os.environ["ACCOUNT_CREATOR_STEPFUNCTION"]
invoke_statemachine(step_function_arn, content)
def write_tweet_info(tweet_object, options):
debug_print("Writing tweet info")
output_path = Path(options['output_dir']) / options['search_name'] / (options['search_name'] + '.txt')
with output_path.open('a') as output_file:
output_file.write('Tweet info:\n')
output_file.write('\tID:{}\n'.format(tweet_object['id_str']))
output_file.write('\tCreated at:{}\n'.format(tweet_object['created_at']))
output_file.write('\tAuthor:{}\n'.format(tweet_object['user']['screen_name']))
output_file.write('\tText:{}\n'.format(clean_text(tweet_object['text'])))
try:
output_file.write('\tURL:{}\n'.format(tweet_object['entities']['urls'][0]['expanded_url']))
except Exception:
pass
def create_account(name, email, role):
account_request_id = None
client = get_boto_client('organizations')
debug_print("Creating account with {} name and e-mail {}".format(
name, email))
response = client.create_account(Email=email,
AccountName=name,
RoleName=role,
IamUserAccessToBilling="ALLOW")
account_request_id = response['CreateAccountStatus']['Id']
return account_request_id
def generate_allWeatherData(self, startDate, endDate):
stationsData = self.csvHandler._loadData('stations.csv')[0]
numberOfStations = len(stationsData)
progressBarWidget = [
progressbar.Percentage(),
' ',
progressbar.Bar('#', '|', '|'),
' ',
progressbar.Variable('FIPS', width=12, precision=12),
' ',
progressbar.Variable('ID', width=12, precision=12),
]
progressBar = progressbar.ProgressBar(maxval=numberOfStations,
widgets=progressBarWidget,
redirect_stdout=True)
progressBar.start()
step = 0
try:
logFile = open('weather.log', 'r')
step = int(logFile.read(), 10)
logFile.close()
except:
logFile = open('weather.log', 'w')
logFile.write(str(step))
logFile.close()
for i in range(step, numberOfStations):
with open('weather.log', 'w') as logFile:
logFile.write(str(i))
stationID = stationsData[i]['id'].split(':')[1]
countyFips = stationsData[i]['county_fips']
progressBar.update(i, FIPS=countyFips, ID=stationID)
# First step, create weather.csv file
if i == 0:
self.downloadHandler.get_countyWeatherData(
countyFips, stationID, startDate, endDate, 'weather.csv')
# Other steps, merge new data to weather.csv file
else:
self.downloadHandler.get_countyWeatherData(
countyFips, stationID, startDate, endDate, 'temp.csv')
self.csvHandler.merge_csvFiles_addRows('weather.csv',
'temp.csv',
'weather.csv')
progressBar.finish()
debug.debug_print("SUCCESS: data extracted (weather data)", 2)
def publishIoT(mqttClient, topic, QoS):
# Run publisher
start = datetime.now()
numOfPublishedMQTTMessage = 0
while vars.PublishingIoT:
try:
publishEV3Status(mqttClient, topic, QoS)
numOfPublishedMQTTMessage = numOfPublishedMQTTMessage + 1
vars.messagesPerSecond = 0
if (numOfPublishedMQTTMessage >= vars.msgRateEvaluateBatch):
total_seconds = (datetime.now() - start).total_seconds()
vars.messagesPerSecond = vars.msgRateEvaluateBatch / total_seconds
numOfPublishedMQTTMessage = 0
except Exception as e:
debug.debug_print("publishIoT" + e)
def run(self):
while True:
msg = self.pipe.recv()
#debug_print(str(self.state['name']) + str(msg))
if msg == "finalize":
debug_print("Finalizing node " + str(self.state['name']))
self.strategy['finalizer'].finalize(self.state)
self.pipe.send("ok")
elif msg == "inference":
debug_print("Doing inference on node " + str(self.state['name']))
# if self.state['name'] == (0,0):
# import profile
# profile.runctx("self.strategy['inference_maker'].inference(self.state)", globals(), {'self':self,})
#else:
self.strategy['inference_maker'].inference(self.state)
# debug_print("Node " + str(self.state['name']) + " output message: " +\
# str(self.state['output_msg']))
self.pipe.send("ok")
elif msg == "get_output":
send_array(self.pipe, "", self.state['output_msg'])
elif msg == "clone_state":
debug_print("Cloning node " + str(self.state['name']) + " state")
self.pipe.send(self.clone_state())
elif msg == "reset_input":
self.state['input_msg'] = []
# debug_print(str(self.state['name']) + \
# " input has been reset")
elif msg[0] == "set_state":
debug_print("Setting state on node " + str(self.state['name']))
self.set_state(msg[1])
elif msg[0] == "train":
# debug_print("Training node " + str(self.state['name']))
self.strategy['trainer'].train(self.state, msg[1])
debug_print("Node " + str(self.state['name']) + " coincidences: " + \
str(self.state['coincidences'].shape))
# debug_print("Node " + str(self.state['name']) + \
# " new coincidences:" + \
# str(self.state['coincidences']))
self.pipe.send("ok")
elif msg[0] == "set_input":
t = msg[1]['dtype']
s = msg[1]['shape']
input_pattern = recv_array(self.pipe, t, s)
self.state['input_msg'] = input_pattern
#print "input is", self.state['input_msg']
# debug_print(str(self.state['name']) + \
# " new input: " + \
# str(self.state['input_msg']))
self.pipe.send("ok")
elif msg[0] == "append_input":
t = msg[1]['dtype']
s = msg[1]['shape']
input_pattern = recv_array(self.pipe, t, s)
self.state['input_msg'].append(input_pattern)
#print "Node's new input", self.state['input_msg']
# debug_print(str(self.state['name']) + \
# " new input: " + \
# str(self.state['input_msg']))
self.pipe.send("ok")
else:
debug_print(str(self.state['name']) + \
": received unknown message")
def send_packet(self, packet):
ascii_packet = ":%s\r\n" % (''.join('{:02x}'.format(x) for x in packet))
ascii_packet = ascii_packet.upper() # Flowmeter only uses uppercase
debug_print("> Sending %s" % ascii_packet)
self.port.write(bytes(ascii_packet, "ASCII"))
def process(self, line):
if line[0] != ":":
print("< Got bad command: %s" % line.strip())
return
debug_print("< Got %s" % line.strip())
line = line[1:].strip() # trim starting colon and newline
bs = bytes([int(line[i:i+2], 16) for i in range(0,len(line),2)]) # get chars in pairs and read as hex int
length = bs[0]
if len(bs) != length+1:
print(" Got packet with invalid length: %s" % line)
return
node = bs[1]
if node!=128 and node!=self.pri_node_num:
print(" Command not addressed to me (Sent to node %d, I'm node %d)" % (node, self.pri_node_num))
return
command = bs[2]
if command==0x00:
pac = Status.parse(bs)
debug_print(" Got a STATUS packet:")
debug_print(" %s" % pac.status)
elif command==0x01:
pac = SendParamStatus.parse(bs)
debug_print(" Got a SEND PARAMETER packet for node %d:" % (pac.node))
if not self.processes[pac.process_number].send_param(pac, True):
print(" UNKNOWN PACKET: %s" % line)
elif command==0x02:
pac = SendParamNoStatus.parse(bs)
debug_print(" Got a SEND PARAMETER (NO STATUS) packet for node %d:" % (pac.node))
if not self.processes[pac.process_number].send_param(pac):
print(" UNKNOWN PACKET: %s" % line)
elif command==0x04:
pac = RequestParam.parse(bs)
debug_print(" Got a REQUEST PARAMETER packet for node %d:" % (pac.node))
if not self.processes[pac.process_number].req_param(pac):
print(" UNKNOWN PACKET: %s" % line)
else:
print("Got unknown command type %d:" % command)
print(" %s" % line)