def log(self, type, message, level, stdout=False): levels = {} levels['info'] = 20 levels['warning'] = 30 levels['error'] = 40 levels['critical'] = 50 levels['highlight'] = 20 if stdout: IO.put(message, level) if type.lower() == "exception": self.exception_logger.log(levels[level], message) self.transaction_logger.log(levels[level], message)
def authorize(self): self.project.log( "transaction", "Initiating OAUTH2 Protocol with " + self.config['TOKEN_ENDPOINT'], "info", True) key_exists = self.oauth.get(self.key_to_the_kingdom) if not key_exists: self.project.log( "transaction", "No valid {} found...".format(self.key_to_the_kingdom), "warning", True) c_id = self.project.config.get("CLIENT_ID") c_secret = self.project.config.get("CLIENT_SECRET") if not c_id or not c_secret: self.project.log( "transaction", "No CLIENT_ID or CLIENT_SECRET. Asking for user input", "warning", True) IO.put("You must configure your account for OAUTH 2.0") IO.put("Please visit {}".format( self.config["OAUTH_DASHBOARD"])) IO.put("& Create an OAUTH 2 API Application") Common.launch_browser(self.config['OAUTH_DASHBOARD']) client_id = IO.get("{}:".format( self.config["CLIENT_ID_ALIAS"])) client_secret = IO.get("{}:".format( self.config["CLIENT_SECRET_ALIAS"])) self.project.save("CLIENT_ID", client_id) self.project.save("CLIENT_SECRET", client_secret) self.project.log( "transaction", "Received {} and {} from user ({}) ({})".format( self.config['CLIENT_ID_ALIAS'], self.config['CLIENT_SECRET_ALIAS'], client_id, client_secret), "info", True) self.get_access_token(client_id, client_secret) else: self.get_access_token(self.project.config['CLIENT_ID'], self.project.config['CLIENT_SECRET']) else: self.refresh(self.project.config['CLIENT_ID'], self.project.config['CLIENT_SECRET']) self.project.save("OAUTH", self.oauth) self.project.log("transaction", "Authorization completed", "info", True)
def authorize(self): self.project.log("transaction", "Initiating OAUTH2 Protocol with " + self.config['TOKEN_ENDPOINT'], "info", True) key_exists = self.oauth.get(self.key_to_the_kingdom) if not key_exists: self.project.log("transaction", "No valid {} found...".format(self.key_to_the_kingdom), "warning", True) c_id = self.project.config.get("CLIENT_ID") c_secret = self.project.config.get("CLIENT_SECRET") if not c_id or not c_secret: self.project.log("transaction", "No CLIENT_ID or CLIENT_SECRET. Asking for user input", "warning", True) IO.put("You must configure your account for OAUTH 2.0") IO.put("Please visit {}".format(self.config["OAUTH_DASHBOARD"])) IO.put("& Create an OAUTH 2 API Application") Common.launch_browser(self.config['OAUTH_DASHBOARD']) client_id = IO.get("{}:".format(self.config["CLIENT_ID_ALIAS"])) client_secret = IO.get("{}:".format(self.config["CLIENT_SECRET_ALIAS"])) self.project.save("CLIENT_ID", client_id) self.project.save("CLIENT_SECRET", client_secret) self.project.log("transaction", "Received {} and {} from user ({}) ({})".format(self.config['CLIENT_ID_ALIAS'], self.config['CLIENT_SECRET_ALIAS'], client_id, client_secret), "info", True) self.get_access_token(client_id, client_secret) else: self.get_access_token(self.project.config['CLIENT_ID'], self.project.config['CLIENT_SECRET']) else: self.refresh(self.project.config['CLIENT_ID'], self.project.config['CLIENT_SECRET']) self.project.save("OAUTH", self.oauth) self.project.log("transaction", "Authorization completed", "info", True)
def launch_browser(url): IO.put("Attempting to launch {} in a browser.".format(url)) try: webbrowser.open(url) except: IO.put("Please visit the following URL to continue: {}".format(url))
def metadata(self): self.project.log("transaction", "Generating metadata CSV File...", "info", True) if not self.files: self.initialize_items() fname = Common.timely_filename("FileList", ".csv") metadata_file = os.path.join(self.project.working_dir, fname) IO.put("Writing CSV File '{}'".format(metadata_file)) f = open(metadata_file, "w") columns = ( "id,title,fileExtension,fileSize,createdDate,modifiedDate,modifiedByMeDate,md5Checksum," "kind,version,parents,restricted,hidden,trashed,starred,viewed,markedViewedByMeDate,lastViewedByMeDate," "lastModifyingUserName,writersCanShare,sharedWithMeDate,sharingUser,sharingUserEmail,ownerNames{}\n" ) f.write(columns) for i in self.files: row2 = [] # Data normalization row2.append('None' if 'id' not in i else repr(i['id'])) row2.append('None' if 'title' not in i else '"' + i['title'] + '"') row2.append('None' if 'fileExtension' not in i else repr(i['fileExtension'])) row2.append('None' if 'fileSize' not in i else i['fileSize']) row2.append('None' if 'createdDate' not in i else i['createdDate']) row2.append('None' if 'modifiedDate' not in i else i['modifiedDate']) row2.append('None' if 'modifiedByMeDate' not in i else i['modifiedByMeDate']) row2.append('None' if 'md5Checksum' not in i else '"' + i['md5Checksum'] + '"') row2.append('None' if 'kind' not in i else repr(i['kind'])) row2.append('None' if 'version' not in i else i['version']) if 'parents' not in i or len(i['parents']) == 0: row2.append('None') else: parStr = '"' for p in i['parents']: parStr = parStr + str(p['id']) + ',' parStr = parStr[:len(parStr) - 1] parStr = parStr + '"' row2.append(parStr) row2.append('None' if 'labels' not in i else repr(i['labels']['restricted'])) row2.append('None' if 'labels' not in i else repr(i['labels']['hidden'])) row2.append('None' if 'labels' not in i else repr(i['labels']['trashed'])) row2.append('None' if 'labels' not in i else repr(i['labels']['starred'])) row2.append('None' if 'labels' not in i else repr(i['labels']['viewed'])) row2.append('None' if 'markedViewedByMeDate' not in i else i['markedViewedByMeDate']) row2.append('None' if 'lastViewedByMeDate' not in i else i['lastViewedByMeDate']) row2.append('None' if 'lastModifyingUserName' not in i else '"' + i['lastModifyingUserName'] + '"') row2.append('None' if 'writersCanShare' not in i else i['writersCanShare']) row2.append('None' if 'sharedWithMeDate' not in i else i['sharedWithMeDate']) row2.append('None' if 'sharingUser' not in i else '"' + i['sharingUser']['displayName'] + '"') row2.append('None' if 'sharingUser' not in i else '"' + i['sharingUser']['emailAddress'] + '"') if 'ownerNames' not in i or len(i['ownerNames']) == 0: row2.append('None') else: ownStr = '"' for o in i['ownerNames']: ownStr = ownStr + str(o) + ',' ownStr = ownStr[:len(ownStr) - 1] ownStr = ownStr + '"' row2.append(ownStr) rowStr = "" for r in row2: rowStr = rowStr + str(r) + "," rowStr = rowStr[:len(rowStr) - 1] f.write(rowStr + '\n') f.close()
def metadata(self): self.project.log("transaction", "Generating metadata CSV File...", "info", True) if not self.files: self.initialize_items() fname = Common.timely_filename("FileList", ".csv") metadata_file = os.path.join(self.project.working_dir, fname) IO.put("Writing CSV File '{}'".format(metadata_file)) f = open(metadata_file, "w") columns = ("id,title,fileExtension,fileSize,createdDate,modifiedDate,modifiedByMeDate,md5Checksum," "kind,version,parents,restricted,hidden,trashed,starred,viewed,markedViewedByMeDate,lastViewedByMeDate," "lastModifyingUserName,writersCanShare,sharedWithMeDate,sharingUser,sharingUserEmail,ownerNames{}\n") f.write(columns) for i in self.files: row2 = [] # Data normalization row2.append('None' if 'id' not in i else repr(i['id'])) row2.append('None' if 'title' not in i else '"' + i['title'] + '"') row2.append('None' if 'fileExtension' not in i else repr(i['fileExtension'])) row2.append('None' if 'fileSize' not in i else i['fileSize']) row2.append('None' if 'createdDate' not in i else i['createdDate']) row2.append('None' if 'modifiedDate' not in i else i['modifiedDate']) row2.append('None' if 'modifiedByMeDate' not in i else i['modifiedByMeDate']) row2.append('None' if 'md5Checksum' not in i else '"' + i['md5Checksum'] + '"') row2.append('None' if 'kind' not in i else repr(i['kind'])) row2.append('None' if 'version' not in i else i['version']) if 'parents' not in i or len(i['parents']) == 0: row2.append('None') else: parStr = '"' for p in i['parents']: parStr = parStr + str(p['id']) + ',' parStr = parStr[:len(parStr) - 1] parStr = parStr + '"' row2.append(parStr) row2.append('None' if 'labels' not in i else repr(i['labels']['restricted'])) row2.append('None' if 'labels' not in i else repr(i['labels']['hidden'])) row2.append('None' if 'labels' not in i else repr(i['labels']['trashed'])) row2.append('None' if 'labels' not in i else repr(i['labels']['starred'])) row2.append('None' if 'labels' not in i else repr(i['labels']['viewed'])) row2.append('None' if 'markedViewedByMeDate' not in i else i['markedViewedByMeDate']) row2.append('None' if 'lastViewedByMeDate' not in i else i['lastViewedByMeDate']) row2.append('None' if 'lastModifyingUserName' not in i else '"' + i['lastModifyingUserName'] + '"') row2.append('None' if 'writersCanShare' not in i else i['writersCanShare']) row2.append('None' if 'sharedWithMeDate' not in i else i['sharedWithMeDate']) row2.append('None' if 'sharingUser' not in i else '"' + i['sharingUser']['displayName'] + '"') row2.append('None' if 'sharingUser' not in i else '"' + i['sharingUser']['emailAddress'] + '"') if 'ownerNames' not in i or len(i['ownerNames']) == 0: row2.append('None') else: ownStr = '"' for o in i['ownerNames']: ownStr = ownStr + str(o) + ',' ownStr = ownStr[:len(ownStr) - 1] ownStr = ownStr + '"' row2.append(ownStr) rowStr = "" for r in row2: rowStr = rowStr + str(r) + "," rowStr = rowStr[:len(rowStr) - 1] f.write(rowStr + '\n') f.close()
def __init__(self, args): # Meh... working_dir = args.project_dir project_name = args.service threads = args.threads # /Meh... self.args = args self.name = project_name self.threads = threads self.working_dir = os.path.join(working_dir, self.name) self.acquisition_dir = os.path.join(self.working_dir, "acquisition") if os.path.exists(self.working_dir): IO.put("Resuming project in " + self.working_dir, "highlight") else: os.makedirs(self.working_dir, exist_ok=True) IO.put("Initializing project in " + self.working_dir, "highlight") self.project_folders["data"] = os.path.join(self.acquisition_dir, "data") self.project_folders["logs"] = os.path.join(self.working_dir, "logs") self.project_folders["metadata"] = os.path.join(self.acquisition_dir, "metadata") #self.project_folders["trash"] = os.path.join(self.acquisition_dir, "trash") #self.project_folders["trash_metadata"] = os.path.join(self.acquisition_dir, "trash_metadata") self.config_file = os.path.join(self.working_dir, "config.cfg") for f in self.project_folders: IO.put("{} path is {}".format(f, self.project_folders[f])) if not os.path.exists(self.project_folders[f]): IO.put("{} directory not found, creating from scratch.", "warn") os.makedirs(self.project_folders[f], exist_ok=True) IO.put("Config file is " + self.config_file) if not os.path.isfile(self.config_file): IO.put("Config file not found, creating default config file", "warn") with open(self.config_file, 'w') as f: f.write(DefaultConfigs.defaults) self.config = ConfigLoader.ConfigLoader() self.config.from_file(self.config_file) self.transaction_log = os.path.join(self.project_folders["logs"], "transaction.log") self.exception_log = os.path.join(self.project_folders["logs"], "exception.log") self.transaction_logger = logging.getLogger(project_name + "_t") self.exception_logger = logging.getLogger(project_name + "_e") self.transaction_logger.setLevel(20) self.exception_logger.setLevel(20) tfh = FileHandler(self.transaction_log) efh = FileHandler(self.exception_log) fmt = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') fmt.converter = time.gmtime tfh.setFormatter(fmt) efh.setFormatter(fmt) self.transaction_logger.addHandler(tfh) self.exception_logger.addHandler(efh)
def __init__(self, args): # Meh... working_dir = args.project_dir project_name = args.service threads = args.threads # /Meh... self.args = args self.name = project_name self.threads = threads self.working_dir = os.path.join(working_dir, self.name) self.acquisition_dir = os.path.join(self.working_dir, "acquisition") if os.path.exists(self.working_dir): IO.put("Resuming project in " + self.working_dir, "highlight") else: os.makedirs(self.working_dir, exist_ok=True) IO.put("Initializing project in " + self.working_dir, "highlight") self.project_folders["data"] = os.path.join(self.acquisition_dir, "data") self.project_folders["logs"] = os.path.join(self.working_dir, "logs") self.project_folders["metadata"] = os.path.join( self.acquisition_dir, "metadata") #self.project_folders["trash"] = os.path.join(self.acquisition_dir, "trash") #self.project_folders["trash_metadata"] = os.path.join(self.acquisition_dir, "trash_metadata") self.config_file = os.path.join(self.working_dir, "config.cfg") for f in self.project_folders: IO.put("{} path is {}".format(f, self.project_folders[f])) if not os.path.exists(self.project_folders[f]): IO.put("{} directory not found, creating from scratch.", "warn") os.makedirs(self.project_folders[f], exist_ok=True) IO.put("Config file is " + self.config_file) if not os.path.isfile(self.config_file): IO.put("Config file not found, creating default config file", "warn") with open(self.config_file, 'w') as f: f.write(DefaultConfigs.defaults) self.config = ConfigLoader.ConfigLoader() self.config.from_file(self.config_file) self.transaction_log = os.path.join(self.project_folders["logs"], "transaction.log") self.exception_log = os.path.join(self.project_folders["logs"], "exception.log") self.transaction_logger = logging.getLogger(project_name + "_t") self.exception_logger = logging.getLogger(project_name + "_e") self.transaction_logger.setLevel(20) self.exception_logger.setLevel(20) tfh = FileHandler(self.transaction_log) efh = FileHandler(self.exception_log) fmt = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fmt.converter = time.gmtime tfh.setFormatter(fmt) efh.setFormatter(fmt) self.transaction_logger.addHandler(tfh) self.exception_logger.addHandler(efh)