def authorize(self): """Shows basic usage of the Drive v3 API. Prints the names and ids of the first 10 files the user has access to. """ creds = None # The file token.pickle stores the user's access and refresh tokens, and is # created automatically when the authorization flow completes for the first # time. if os.path.exists('token.pickle'): with open('token.pickle', 'rb') as token: creds = pickle.load(token) else: error("token.pickle doesn't exist") # If there are no (valid) credentials available, let the user log in. if not creds or not creds.valid: if creds and creds.expired and creds.refresh_token: # notify("Login again", crashed=True) error("Creds expired. Refreshing....") creds.refresh(Request()) else: flow = InstalledAppFlow.from_client_secrets_file( 'credentials.json', SCOPES) creds = flow.run_local_server(port=0) # Save the credentials for the next run with open('token.pickle', 'wb') as token: pickle.dump(creds, token) info("Authorized with Drive API") self.service = build('drive', 'v3', credentials=creds)
def update_song_link(song_key, song_name, old=False): link = search_by_name(song_name) if link != -1: query = 'update song_list set link_found = true, link = \'{}\' where id = {}'.format( link, song_key) execute_query(query) info("Link for song {} found".format(song_name)) return link else: if not old: notify("Link for song {} not found".format(song_name), customText=True) info("Link for song {} not found".format(song_name)) return None
def upload(self, key, name, file_path): try: self.authorize() file_metadata = {'name': name, 'key': key} media = MediaFileUpload(file_path, mimetype='audio/mpeg') result_file = self.service.files().create(body=file_metadata, media_body=media, fields='id').execute() info("Song {} uploaded to drive".format(name)) self.delete_duplicate() return True if result_file else False except Exception: import traceback traceback.print_exc() stacktrace = traceback.format_exc() error(stacktrace)
def notify(name, crashed=False, customText=False): from pyfcm import FCMNotification push_service = FCMNotification(api_key="api_key") registration_id = "device_registration_token" message_title = "Song downloaded!" if not crashed else "Program crashed" if customText: message_title = "My Shazam Songs" if not customText: message_body = "Your song \"{}\" is ready in Google Drive ([email protected])".format( name) if not crashed else name else: message_body = name if not crashed: result = push_service.notify_single_device( registration_id=registration_id, message_title=message_title, message_body=message_body) if result['success'] == 1: info("Notification sent for song {}".format(name) if not crashed else "Crash notification sent with msg = {}".format(name))
def delete_duplicate(self): info("Deleting duplicate files in drive") try: self.authorize() pageToken = None songs = [] dups = {} while True: results = self.service.files().list( pageToken=pageToken, fields="nextPageToken, files(id,name,modifiedTime)", orderBy="modifiedTime asc").execute() items = results.get('files', []) pageToken = results.get('nextPageToken', None) songs += items if pageToken is None: break found_dups = False for song in songs: if song['name'] in dups: found_dups = True dups[song["name"]].append({ 'name': song['name'], 'id': song['id'], 'modifiedTime': song['modifiedTime'] }) else: dups[song["name"]] = [{ 'name': song['name'], 'id': song['id'], 'modifiedTime': song['modifiedTime'] }] if found_dups: info("No duplicate files found") for dup in dups: if len(dups[dup]) > 1: others = sorted(dups[dup], key=lambda x: x['modifiedTime'])[1:] for inst in others: info(("Deleting duplicate song: {}".format( inst['name']))) self.service.files().delete( fileId=inst['id']).execute() except Exception: import traceback traceback.print_exc() stacktrace = traceback.format_exc() error(stacktrace)
def download_and_upload(song_key, song_name, link): file_path = download_file(song_name, link) if filesize(file_path) > 5 * 1024: info("Uploading song {} to drive".format(song_name)) GDrive().upload(song_key, song_name, file_path) else: info( "File for song {} was less than 5 KB, therefore not uploading song" .format(song_name)) delete_file(file_path) info("Song {} deleted from temp storage".format(song_name)) notify(song_name)
if os.path.exists('{}.index'.format(weights_path)): debug('weights file exists at {}'.format(weights_path)) try: training_model.load_weights(weights_path) training_model.compile( optimizer=tf.keras.optimizers.Adam(), loss=tf.keras.losses.CategoricalCrossentropy(), metrics=['accuracy']) except Exception as ex: error('error during weights loading: {}'.format(ex)) exit() else: debug( 'weights file does not exist => this network needs to be trained') training_model.compile(optimizer=tf.keras.optimizers.Adam(), loss=tf.keras.losses.CategoricalCrossentropy(), metrics=['accuracy']) info('Start training:') training_model.fit(training_ds, validation_data=validation_ds, verbose=1, batch_size=5, epochs=20, callbacks=[TB_callback, checkpoint_callback]) info('Model summary:') training_model.summary() info('Performance evaluation:') results = training_model.evaluate(test_ds, verbose=1) info('Results: \n\tLoss => {}\n\t Accuracy => {:5.2f}%'.format( results[0], results[1] * 100))
def test_logging(args): msg = args[0] info(msg)
from Logger import warn, debug, info import Logger import multiprocessing Logger.basicConfig(level=Logger.DEBUG, format="%(asctime)s %(levelname)s %(message)s", filename="logger_test.log") def test_logging(args): msg = args[0] info(msg) if __name__ == "__main__": # # call from main process to make sure works # info("starting") # # call from child processes in pool # pool = multiprocessing.Pool(processes=4) # start 4 worker processes function_parameters = list() for a in range(200): function_parameters.append(("message #%3d" % a,)) pool.map(test_logging, function_parameters) print Logger.getCounts()
def delete_file(file_path): import os if os.path.exists(file_path): os.remove(file_path) else: info("The file '{}' does not exist".format(file_path))
def get_song_list_from_shazam(upto_timestamp=None): url = "https://www.shazam.com/discovery/v4/en-US/US/web/-/tag/AC53CA17-11C1-4F7D-91FC-93FCD57B878B" i = 0 song_names = [] song_keys = set() token = "" random_uuid = str(uuid.uuid4()) # found = False # change this to false when blocked on some song while True: if i == 0: # tagId is random uuid, backup=9c26cc1d-a28f-470f-90b0-161a8d04e886 token = '{"accountId": {"s": "accountid"},' \ '"tagId": {"s": "' + random_uuid + '"},' '"timestamp": {"n": "' \ + str(int(str(time.time()).split('.')[0] + '000')) + '"}}' # print(token) token = str(token).encode("utf-8") token = str(base64.b64encode(token))[2:-1] querystring = {"limit": "100", "token": token} headers = { 'authority': "www.shazam.com", 'pragma': "no-cache", 'cache-control': "no-cache,no-cache", 'user-agent': "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_1) AppleWebKit/537.36 (KHTML, like Gecko) " "Chrome/78.0.3904.97 Safari/537.36", 'dnt': "1", 'accept': "*/*", 'sec-fetch-site': "same-origin", 'sec-fetch-mode': "cors", 'referer': "https://www.shazam.com/myshazam", 'accept-encoding': "gzip, deflate, br", 'accept-language': "en-US,en;q=0.9,hi;q=0.8,mr;q=0.7", 'cookie': "fbm_210827375150=base_domain=.shazam.com; geoip_country=US; geoip_lat=32.925; " "geoip_long=-96.892; " "codever" "=a3b244b8d73c4f5e426954d6822ca00e1c1bdcb949071b552b8165dccaa1d86e4e5dd7bbc50ba313bd270b5208be195a167094d3bfc988a8102dc288dda8f2e8550e629473614471cc148a5d307fb5d0cc548e0b0d3369f5fabef9d71c65dd6e0a93d41232981ee6409c03a760ef8fe0a4c57f308300f1db3defa3a8351b38754737ac20caf41f67adeb349ffb7eb66504f4fb083e4033cdfe92", 'Host': "www.shazam.com", 'Connection': "keep-alive" } response = requests.get(url, headers=headers, params=querystring) # print(response) if not response.text: # print("returning") return song_names # print(response.text) j = json.loads(response.text) tags = j["tags"] breakout = False for x in tags: # stuck_song_key = -1 # get id from mac intellij db # if not found and x['track']['key'] == str(stuck_song_key): # found = True # if not found: # continue if x['track']['key'] not in song_keys: key = x['track']['key'] name = x["track"]["heading"]["title"] + " - " + x["track"][ "heading"]["subtitle"] ts = x['timestamp'] if upto_timestamp and str(ts) <= str(upto_timestamp): # print("Breaking from the loop, not paging") breakout = True break already_exists = check_already_exists(key) if already_exists: info("Song {} already exists, updating timestamp".format( name)) notify("Song {} already downloaded!".format(name), customText=True) update_timestamp(key, ts) if not already_exists[3]: info( "Song {} does not have link. Trying to find downloadable link..." .format(name)) update_song_link(key, name) continue song_keys.add(key) song_names.append(name) save_to_db(song_key=key, song_name=name, song_timestamp=ts) info("Added {} to DB".format(name)) i = 1 if 'token' not in j: break if breakout: break token = j['token'] return song_names
# change the arg to None when continuing # return get_song_list_from_shazam(None) return get_song_list_from_shazam(upto_timestamp=res[0] if res else None) check_old_songs_count = 0 delete_log_file_count = 0 check_old_songs_interval = 30 delete_log_file_interval = 50 poll_interval = 10 while True: try: info("Polling... ") added = check_new() info("{} songs added".format(len(added))) check_old_songs_count += 1 delete_log_file_count += 1 if check_old_songs_count == check_old_songs_interval: info("Checking old songs for new links...") check_old_songs() check_old_songs_count = 0 if delete_log_file_count == delete_log_file_interval: info("Resetting log file...") time.sleep(0.5) delete_log_file_count = 0
if __name__ == '__main__': help(argv[0]) if len(argv) < 2 else None weigths_path = argv[2] if argv[2] else './weights_result' classificator = MalwareClassificator(6, 200, family_labels) try: if len(argv) >= 3: classificator.load_weights(argv[2]) else: classificator.load_weights() except IOError as e: error(str(e)) exit() if not '-q' in argv: debug('model correctly loaded') debug('loading image {}'.format(argv[1])) try: prediction = classificator.predict(argv[1]) except Exception as e: error(str(e)) exit() classificator.model.model().summary() exit() info('prediction for the image: {}'.format(classificator.get_label_from_prediction(prediction)))