def make_query_url(parameters): debug_print("Calling make_query_url") url = 'https://api.twitter.com/1.1/search/tweets.json?lang=en' for k, v in parameters.items(): url += '&{}={}'.format(k, v) debug_print("Resulting url: {}".format(url)) return url
def transform_jsonToCsv_socialDistancingData(self, jsonFilename, csvFilename): jsonMetaData = [] countyFieldnames = [ 'stateFips', 'stateName', 'countyFips', 'countyName' ] dataFieldnames = [ 'date', 'totalGrade', 'visitationGrade', 'encountersGrade', 'travelDistanceGrade' ] jsonMetaData = self._loadData(jsonFilename) jsonCountiesData = jsonMetaData['hits']['hits'] singleCountyData = {} with open(_CSV_Directory_ + csvFilename, 'w') as csvFile: csvDriver = csv.DictWriter(csvFile, fieldnames=(countyFieldnames + dataFieldnames)) csvDriver.writeheader() for county in jsonCountiesData: # set county data singleCountyData = { field: (county['_source'][field]).encode('utf-8') for field in countyFieldnames } # set Grade of county for each day, specified with 'date' for data in county['_source']['data']: singleCountyData.update( {field: data[field] for field in dataFieldnames}) csvDriver.writerow(singleCountyData) debug.debug_print("SUCCESS: transform completed(socialDistancingData)", 2)
def merge_csvFiles_addColumns(self, csvFilename1, csvFilename2, destinationFilename, file1Keys, file2Keys, newColumns): # Get csvFiles data csvData1, csvFile1Fieldnames = self._loadData(csvFilename1) csvData2 = self._loadData(csvFilename2)[0] # Generate new file fieldnames, add newClumns to fieldnames mergedDataFieldnames = csvFile1Fieldnames + newColumns # Merge and save data with open(_CSV_Directory_ + destinationFilename, 'w') as destinationFile: csvDriver = csv.DictWriter(destinationFile, fieldnames=mergedDataFieldnames) csvDriver.writeheader() for row in csvData1: # Find correspondingRow on csvData2 found = False for item in csvData2: if self._isEqual(self._keyValues(row, file1Keys), self._keyValues(item, file2Keys)): correspondingRow = {key:item[key] for key in newColumns} found = True break if found == False: debug.debug_print("key not found:\t" + ', '.join(str(item) for item in self._keyValues(row, file1Keys)), 3) continue # Generate and add row to file row.update(correspondingRow) csvDriver.writerow(row) debug.debug_print("SUCCESS: merge completed", 2)
def await_reply(sock, path): reply = loop_recv(sock) if reply["status"] == "ok": debug_print(path + " done loading") else: debug_print("Error: " + reply["status"]) return reply
def search(regex, text, flags=0): """Search for REGEX in TEXT with optional FLAGS""" my_re.match = re.search(regex, text, flags) if my_re.match: # TODO: while my_re.match.group(i): print my_re.match.group(i) debug_print("match: %s" % my_re.match.group(), 6) return (my_re.match)
def transform_jsonToCsv_confirmAndDeathData(self, jsonFilename, csvFilename): startDay = date.fromisoformat('2020-01-22') jsonMetaData = [] countyFieldnames = ['stateFIPS', 'stateAbbr', 'countyFIPS', 'county'] dataFieldnames = ['date', 'confirmed', 'deaths'] jsonMetaData = self._loadData(jsonFilename) singleCountyData = {} with open(_CSV_Directory_ + csvFilename, 'w') as csvFile: csvDriver = csv.DictWriter(csvFile, fieldnames=(countyFieldnames+dataFieldnames)) csvDriver.writeheader() numberOfDays = len(jsonMetaData[0]['deaths']) for day in range(numberOfDays): singleCountyData = {'date':(startDay+timedelta(days=day)).isoformat()} for county in jsonMetaData: # pass invalid county if int(county['countyFIPS'], 10) == 0: continue # set county data singleCountyData.update({field:county[field] for field in countyFieldnames}) singleCountyData.update({field:county[field][day] for field in dataFieldnames if field in county}) # remove summation if day != 0: singleCountyData.update({field:(county[field][day] - county[field][day - 1]) for field in ['confirmed', 'deaths'] if field in county}) # write in file csvDriver.writerow(singleCountyData) debug.debug_print("SUCCESS: transform completed(confirmAndDeathData)", 2)
def test_ColorSensorSpeed(): i = 0 start = timeit.default_timer() while i < 10000: i = i + 1 #debug.debug_print(str(ev3.colorSensor_lt.value())) stop = timeit.default_timer() debug.debug_print('Time: ', stop - start)
def test_comparison(): i = 0 start = timeit.default_timer() while i < 10000: i = i + 1 #debug.debug_print("1") stop = timeit.default_timer() debug.debug_print('Time: ', stop - start)
def assume_role(account_id, role_name): debug_print("Assuming role.....") role_arn = "arn:aws:iam::{0}:role/{1}".format(account_id, role_name) client = get_boto_client('sts') assumed_role = client.assume_role( RoleArn=role_arn, RoleSessionName="account_vending_machine_lambda") return assumed_role['Credentials']
def _shutdown_socket(self, file_descriptor): del self.socket_timing[file_descriptor] self.poller.unregister(file_descriptor) if file_descriptor in self.clients: self.clients[file_descriptor].close() del self.clients[file_descriptor] else: debug_print('tried to close a socket that was not in clients: %i' % file_descriptor)
def get_confirmAndDeathData(self, destinationFilename): jsonData = requests.get( 'https://usafactsstatic.blob.core.windows.net/public/2020/coronavirus-timeline/allData.json' ).json() with open(_JSON_Directory_ + destinationFilename, 'w') as jsonFile: json.dump(jsonData, jsonFile) debug.debug_print("SUCCESS: data extracted (confirmAndDeathData)", 2)
def _shutdown_socket(self, file_descriptor) -> None: del self.socket_timing[file_descriptor] self.poller.unregister(file_descriptor) if file_descriptor in self.clients: self.clients[file_descriptor].close() del self.clients[file_descriptor] else: debug_print('tried to close a socket that was not in clients: %i' % file_descriptor)
def invoke_statemachine(arn, input): client = get_boto_client("stepfunctions") account_name = input.get("accountName") response = client.start_execution(stateMachineArn=arn, name="{}-creation-{}".format( account_name, time.time()), input=json.dumps(input)) debug_print(response) return (response)
def change_directory(dir): """Changes into file system DIR (ignoring errors)""" debug_print("change_directory(%s)" % dir, 4) try: os.chdir(dir) except: print_stderr("Unable to change into directory '%s': %s" % (filename, sys.exc_info())) return
def make_directory(dir): """Creates file system DIR (ignoring errors)""" debug_print("make_directory(%s)" % dir, 4) try: os.mkdir(dir) except: print_stderr("Unable to create directory '%s': %s" % (dir, sys.exc_info())) return
def do_query_url(url): debug_print("Sending query to url {}".format(url)) r = requests.get(url, auth=get_oauth()) if r.status_code == 200: return r.json() else: print("Something went wrong with the query") debug_print(r.text) return None
def publishVoltage(mqttClient, topic): while True: data = {} data['measured_volts'] = powerSupply.measured_volts #print("sensor reading published as " + json.dumps(data) ) debug.debug_print(json.dumps(data)) messageJson = json.dumps(data) mqttClient.publish(topic, messageJson, 1) time.sleep(1)
def check_files(options): p = Path(options['output_dir']) / options['search_name'] p.mkdir(parents=True, exist_ok=True) last_id_path = p / 'last_id.txt' since_id = 0 if last_id_path.exists(): with last_id_path.open('r') as last_id_file: since_id = int(last_id_file.read().strip()) debug_print("Previous since id = {}".format(since_id)) return since_id
def write_file(filename, text): """Outpus TEXT to FILENAME""" debug_print("write_file(%s, _)" % filename, 5) try: with open(filename, 'w') as f: f.write(text) except: print_stderr("Unable to write file '%s': %s" % (filename, sys.exc_info())) return
def get_socialDistancingData(self, stateFIPS, destinationFilename): jsonData = requests.get( 'https://covid19-scoreboard-api.unacastapis.com/api/search/covidcountyaggregates_v3?q=stateFips:%02d&size=4000' % (stateFIPS)).json() with open(_JSON_Directory_ + destinationFilename, 'w') as jsonFile: json.dump(jsonData, jsonFile) debug.debug_print( "SUCCESS: data extracted (socialDistancingData on stateFIPS:%02d)" % (stateFIPS), 2)
def alt_get_url_source(url, retries=1): """Returns HTML source at URL with retries after pausing""" tries = retries + 1 while (tries > 0): contents = get_url_source(url) if not re.search("Error: ERR_ACCESS_DENIED", contents): break debug_print("Warning: access denied error; trying again after pause") tries -= 1 time.sleep(5) return (contents)
def simplify_csvFile(self, csvFilename, destinationFilename, fieldnames): # Get csvFile data csvData = self._loadData(csvFilename)[0] with open(_CSV_Directory_ + destinationFilename, 'w') as DestinationFile: csvDriver = csv.DictWriter(DestinationFile, fieldnames) csvDriver.writeheader() for row in csvData: csvDriver.writerow({k:row[k] for k in (fieldnames) if k in row}) debug.debug_print("SUCCESS: simplify completed", 2)
def execute_command(self, command): if Context().clear_before_command: self.clear_screen() self.print_prompt() print(command) debug_print(command) if command in self.commands_dict: self.commands_dict[command]() else: self.run_instruction(command)
def send_param(self, pac, sendStatus=False): Process.send_param(self, pac, sendStatus) result = True ## read the data data, data2 = self.meter.read_data(pac.type, pac.data) if pac.type != "LONG": debug_print(" Data: (%s) %s, RAW: %s" % (pac.type, data, data2)) else: debug_print(" Data: (LONG) %d or (FLOAT) %f" % (data, data2)) ## deal with packet contents if pac.param == 10 and pac.type == "CHAR": # Control device if data == "I": # Initialise process print("->INITIALISE NODE %d PROCESS %d" % (pac.node, pac.process_number)) elif data == "R": # Reset device print("->RESET MODULE") if self.temp_pri_node_num >= 0: self.meter.pri_node_num = self.temp_pri_node_num if self.temp_sec_node_num >= 0: self.meter.sec_node_num = self.temp_sec_node_num if self.temp_next_node_num >= 0: self.meter.next_node_num = self.temp_next_node_num if self.temp_last_node_num >= 0: self.meter.last_node_num = self.temp_last_node_num elif pac.param == 1 and pac.type == "CHAR": # Set primary node address print("->SET PRIMARY NODE ADDRESS TO %d" % ord(data)) self.temp_pri_node_num = ord(data) elif pac.param == 2 and pac.type == "CHAR": # Set secondary node address print("->SET SECONDARY NODE ADDRESS TO %d" % ord(data)) self.temp_sec_node_num = ord(data) elif pac.param == 3 and pac.type == "CHAR": # Set next node address print("->SET NEXT NODE ADDRESS TO %d" % ord(data)) self.temp_next_node_num = ord(data) elif pac.param == 4 and pac.type == "CHAR": # Set last node address print("->SET LAST NODE ADDRESS TO %d" % ord(data)) self.temp_last_node_num = ord(data) elif pac.param == 5 and pac.type == "CHAR": # Set bus management bm = "EVERYTHING AUTO" if ord(data) == 67 else \ "AUTO ARBITRATION" if ord(data) == 3 else \ "ALWAYS BUSMASTER" if ord(data) == 2 else \ "TEMPORARY" if ord(data) == 1 else "UNKNOWN" print("->SET BUS MANAGEMENT TO %d (%s)" % (ord(data), bm)) else: result = False ## send status if we need to if sendStatus: self.meter.send_packet( Status.build( Container(length=4, node=pac.node, status='NO_ERROR', index=0))) return result
def run_command(command_line, level=5): """Runs COMMAND_LINE and returns the output (as a string), with debug tracing at specified LEVEL""" # Issue command debug_print("Running command: %s" % command_line, level=level) (status, output) = getstatusoutput(command_line) if (status != 0): print_stderr("Warning: problem running command (status=%d): %s" % (status, command_line)) # Return output debug_print("Command output: %s\n" % output, level=level + 1) return (output)
def main(event): data = event[0] account_id = data.get("accountId") admin_account_role_name = data.get("adminAccountRole") admin_account_number = data.get("adminAccount") message = "New account with ID: {} was created successfully. Role: {} with Trust to Account: {} was setup for Admin Access.".format( account_id, admin_account_role_name, admin_account_number) debug_print(message) postMessageToSns("Account was created", message) return data
def handler(event, context): debug_print(json.dumps(event, indent=2)) s3_event = event["Records"][0]["s3"] s3_bucket = s3_event["bucket"]["name"] s3_object = s3_event["object"]["key"] client = get_boto_client("s3") response = client.get_object(Bucket=s3_bucket, Key=s3_object) content = json.loads(response['Body'].read().decode('utf-8')) debug_print(json.dumps(content, indent=2)) step_function_arn = os.environ["ACCOUNT_CREATOR_STEPFUNCTION"] invoke_statemachine(step_function_arn, content)
def write_tweet_info(tweet_object, options): debug_print("Writing tweet info") output_path = Path(options['output_dir']) / options['search_name'] / (options['search_name'] + '.txt') with output_path.open('a') as output_file: output_file.write('Tweet info:\n') output_file.write('\tID:{}\n'.format(tweet_object['id_str'])) output_file.write('\tCreated at:{}\n'.format(tweet_object['created_at'])) output_file.write('\tAuthor:{}\n'.format(tweet_object['user']['screen_name'])) output_file.write('\tText:{}\n'.format(clean_text(tweet_object['text']))) try: output_file.write('\tURL:{}\n'.format(tweet_object['entities']['urls'][0]['expanded_url'])) except Exception: pass
def create_account(name, email, role): account_request_id = None client = get_boto_client('organizations') debug_print("Creating account with {} name and e-mail {}".format( name, email)) response = client.create_account(Email=email, AccountName=name, RoleName=role, IamUserAccessToBilling="ALLOW") account_request_id = response['CreateAccountStatus']['Id'] return account_request_id
def generate_allWeatherData(self, startDate, endDate): stationsData = self.csvHandler._loadData('stations.csv')[0] numberOfStations = len(stationsData) progressBarWidget = [ progressbar.Percentage(), ' ', progressbar.Bar('#', '|', '|'), ' ', progressbar.Variable('FIPS', width=12, precision=12), ' ', progressbar.Variable('ID', width=12, precision=12), ] progressBar = progressbar.ProgressBar(maxval=numberOfStations, widgets=progressBarWidget, redirect_stdout=True) progressBar.start() step = 0 try: logFile = open('weather.log', 'r') step = int(logFile.read(), 10) logFile.close() except: logFile = open('weather.log', 'w') logFile.write(str(step)) logFile.close() for i in range(step, numberOfStations): with open('weather.log', 'w') as logFile: logFile.write(str(i)) stationID = stationsData[i]['id'].split(':')[1] countyFips = stationsData[i]['county_fips'] progressBar.update(i, FIPS=countyFips, ID=stationID) # First step, create weather.csv file if i == 0: self.downloadHandler.get_countyWeatherData( countyFips, stationID, startDate, endDate, 'weather.csv') # Other steps, merge new data to weather.csv file else: self.downloadHandler.get_countyWeatherData( countyFips, stationID, startDate, endDate, 'temp.csv') self.csvHandler.merge_csvFiles_addRows('weather.csv', 'temp.csv', 'weather.csv') progressBar.finish() debug.debug_print("SUCCESS: data extracted (weather data)", 2)
def publishIoT(mqttClient, topic, QoS): # Run publisher start = datetime.now() numOfPublishedMQTTMessage = 0 while vars.PublishingIoT: try: publishEV3Status(mqttClient, topic, QoS) numOfPublishedMQTTMessage = numOfPublishedMQTTMessage + 1 vars.messagesPerSecond = 0 if (numOfPublishedMQTTMessage >= vars.msgRateEvaluateBatch): total_seconds = (datetime.now() - start).total_seconds() vars.messagesPerSecond = vars.msgRateEvaluateBatch / total_seconds numOfPublishedMQTTMessage = 0 except Exception as e: debug.debug_print("publishIoT" + e)
def run(self): while True: msg = self.pipe.recv() #debug_print(str(self.state['name']) + str(msg)) if msg == "finalize": debug_print("Finalizing node " + str(self.state['name'])) self.strategy['finalizer'].finalize(self.state) self.pipe.send("ok") elif msg == "inference": debug_print("Doing inference on node " + str(self.state['name'])) # if self.state['name'] == (0,0): # import profile # profile.runctx("self.strategy['inference_maker'].inference(self.state)", globals(), {'self':self,}) #else: self.strategy['inference_maker'].inference(self.state) # debug_print("Node " + str(self.state['name']) + " output message: " +\ # str(self.state['output_msg'])) self.pipe.send("ok") elif msg == "get_output": send_array(self.pipe, "", self.state['output_msg']) elif msg == "clone_state": debug_print("Cloning node " + str(self.state['name']) + " state") self.pipe.send(self.clone_state()) elif msg == "reset_input": self.state['input_msg'] = [] # debug_print(str(self.state['name']) + \ # " input has been reset") elif msg[0] == "set_state": debug_print("Setting state on node " + str(self.state['name'])) self.set_state(msg[1]) elif msg[0] == "train": # debug_print("Training node " + str(self.state['name'])) self.strategy['trainer'].train(self.state, msg[1]) debug_print("Node " + str(self.state['name']) + " coincidences: " + \ str(self.state['coincidences'].shape)) # debug_print("Node " + str(self.state['name']) + \ # " new coincidences:" + \ # str(self.state['coincidences'])) self.pipe.send("ok") elif msg[0] == "set_input": t = msg[1]['dtype'] s = msg[1]['shape'] input_pattern = recv_array(self.pipe, t, s) self.state['input_msg'] = input_pattern #print "input is", self.state['input_msg'] # debug_print(str(self.state['name']) + \ # " new input: " + \ # str(self.state['input_msg'])) self.pipe.send("ok") elif msg[0] == "append_input": t = msg[1]['dtype'] s = msg[1]['shape'] input_pattern = recv_array(self.pipe, t, s) self.state['input_msg'].append(input_pattern) #print "Node's new input", self.state['input_msg'] # debug_print(str(self.state['name']) + \ # " new input: " + \ # str(self.state['input_msg'])) self.pipe.send("ok") else: debug_print(str(self.state['name']) + \ ": received unknown message")
def send_packet(self, packet): ascii_packet = ":%s\r\n" % (''.join('{:02x}'.format(x) for x in packet)) ascii_packet = ascii_packet.upper() # Flowmeter only uses uppercase debug_print("> Sending %s" % ascii_packet) self.port.write(bytes(ascii_packet, "ASCII"))
def process(self, line): if line[0] != ":": print("< Got bad command: %s" % line.strip()) return debug_print("< Got %s" % line.strip()) line = line[1:].strip() # trim starting colon and newline bs = bytes([int(line[i:i+2], 16) for i in range(0,len(line),2)]) # get chars in pairs and read as hex int length = bs[0] if len(bs) != length+1: print(" Got packet with invalid length: %s" % line) return node = bs[1] if node!=128 and node!=self.pri_node_num: print(" Command not addressed to me (Sent to node %d, I'm node %d)" % (node, self.pri_node_num)) return command = bs[2] if command==0x00: pac = Status.parse(bs) debug_print(" Got a STATUS packet:") debug_print(" %s" % pac.status) elif command==0x01: pac = SendParamStatus.parse(bs) debug_print(" Got a SEND PARAMETER packet for node %d:" % (pac.node)) if not self.processes[pac.process_number].send_param(pac, True): print(" UNKNOWN PACKET: %s" % line) elif command==0x02: pac = SendParamNoStatus.parse(bs) debug_print(" Got a SEND PARAMETER (NO STATUS) packet for node %d:" % (pac.node)) if not self.processes[pac.process_number].send_param(pac): print(" UNKNOWN PACKET: %s" % line) elif command==0x04: pac = RequestParam.parse(bs) debug_print(" Got a REQUEST PARAMETER packet for node %d:" % (pac.node)) if not self.processes[pac.process_number].req_param(pac): print(" UNKNOWN PACKET: %s" % line) else: print("Got unknown command type %d:" % command) print(" %s" % line)