def download_document(filename, email, db): #After ownloading the file to a local server # we will be updating the downloded_on field of #Collaborators table. For version check purpose email_id = get_email_id(email, db).id file_id = get_file_id_by_filename(filename, db) print(email_id, file_id) #this part creates a lock opject with the given file path file_lock_check = LockFile(file_dir + '/' + filename) #using is_locked, we will get to know, it the file is locked or not print("the file status ", file_lock_check.is_locked()) if file_lock_check.is_locked(): if is_owner(email_id, file_id, db) == True: #if the current user is owner we, will release the lock lock.release() print("owner file status ", file_lock_check.is_locked()) else: #otherwise i returns none as the fileis alreay locked print("tje file status ", file_lock_check.is_locked()) return None try: print("downloading from here") update_status = db.query(Collaborators).filter( and_(Collaborators.collaborator_id == email_id, Collaborators.file_id == file_id)) update_status = update_status.update( {'downloaded_on': dt.datetime.now()}) db.commit() print(update_status) return True except: return False
def GetPassword( resourcefile , list ) : count = 0 pwd = [] rb = open_workbook(resourcefile) r_sheet = rb.sheet_by_index(0) from lockfile import LockFile lock = LockFile(resourcefile) lockid = lock.is_locked() print lockid for a in xrange(1, 10): if lockid == False: lock.acquire() print "I have locked Resource File" break else: time.sleep (10) lockid = lock.is_locked() wb = copy(rb) w_sheet = wb.get_sheet(0) keys = [r_sheet.cell(0, col_index).value for col_index in xrange(r_sheet.ncols)] for row_index in xrange(1, r_sheet.nrows): d = {keys[col_index]: r_sheet.cell(row_index, col_index).value for col_index in xrange(r_sheet.ncols)} if ( d['IP'] in list) : count = count + 1 pwd = d['Password'] wb.save(resourcefile) lock.release() if(count == len(list)+1) : break return pwd
def ReadTestCase( resourcefile ) : list = [] row_index = 0 from lockfile import LockFile lock = LockFile(resourcefile) lockid = lock.is_locked() print lockid for a in xrange(1, 2): if lockid == False: lock.acquire() print "I have locked Resource File" break else: time.sleep (10) lockid = lock.is_locked() rb = open_workbook(resourcefile) r_sheet = rb.sheet_by_index(0) wb = copy(rb) w_sheet = wb.get_sheet(0) keys = [r_sheet.cell(0, col_index).value for col_index in xrange(r_sheet.ncols)] j = r_sheet.nrows q = r_sheet.ncols print col_index while row_index < j: d = {keys[col_index]: r_sheet.cell(row_index, col_index).value for col_index in xrange(r_sheet.ncols)} temp = "" if ( d['Execution'] == "yes") : temp = d['TC Name'] print temp list.append(temp) wb.save(resourcefile) row_index = row_index + 1 lock.release() return list
def sync_folder(self): encrypted_folder_lock = LockFile(self.encrypted_folder) if encrypted_folder_lock.is_locked(): self.info("Acquiring the lock of encrypted folder...") with encrypted_folder_lock: plain_folder_lock = LockFile(self.plain_folder) if plain_folder_lock.is_locked(): self.info("Acquiring the lock of plaintext folder...") with plain_folder_lock: self._do_sync_folder()
def run_each_job(conf_dict, uploadurl): if not os.path.exists(CYCLIC_PATH): os.makedirs(CYCLIC_PATH, 02775) # split each cyclic jobs cyclic_jobs_path = get_cyclic_jobs(conf_dict) split_job_params = ' '.join(cyclic_jobs_path) split_cmd = "%s split -j %s -o %s" % (SRC+'/bin/pst', split_job_params, CYCLIC_PATH) split_output = subprocess.check_output(split_cmd, shell=True) split_jobs_path = [split_file.split(' => ')[1] for split_file in split_output.split('\n') if split_file] # run each splited jobs sync_dir = conf_dict.get('sync', {}).get('dir') sync_list = conf_dict.get('sync', {}).get('jobs') wait_timeout = conf_dict.get('sync', {}).get('timeout') if sync_dir and sync_list and 'all' in sync_list: try: lock = LockFile(sync_dir + os.sep + 'all') lock.acquire() for unit_jobfile in split_jobs_path: run_cmd = "%s run -j %s -u %s" % (SRC + '/bin/pst', unit_jobfile, uploadurl) run_shell_cmd(run_cmd) print("Remove: %s" % unit_jobfile) os.remove(unit_jobfile) lock.release() except KeyboardInterrupt: if lock.is_locked(): lock.release() raise else: for unit_jobfile in split_jobs_path: run_cmd = "%s run -j %s -u %s" % (SRC + '/bin/pst', unit_jobfile, uploadurl) # add lock to sync there, e.g. run doker and host test same time testcase_name = common.load_conf(unit_jobfile).get('testcase') if sync_dir and sync_list and testcase_name in sync_list: lock = LockFile(sync_dir + os.sep + testcase_name) try: lock.acquire(timeout=wait_timeout) run_shell_cmd(run_cmd) lock.release() except LockTimeout as e: print e except KeyboardInterrupt: if lock.is_locked(): lock.release() raise else: run_shell_cmd(run_cmd) print("Remove: %s" % unit_jobfile) os.remove(unit_jobfile)
def handle(self, *args, **options): task = options["task"] if not task in ("count", "delete", "process"): raise CommandError( "task must be either 'count', 'delete' or 'process'") if task == "count": count = DownloadRequest.objects.filter(active=True).count() self.stdout.write("%d requests pending" % (count)) if task == "delete": lock_file_path = "/tmp/place.downloads.delete" lock = LockFile(lock_file_path) if lock.is_locked(): return lock.acquire() files = glob("%s/media/zip/*.zip" % BASE_DIR) for file_name in files: date = datetime.datetime.fromtimestamp( os.path.getctime(file_name)) days = (datetime.datetime.now() - date).days if days > 0: os.remove(file_name) lock.release() if task == "process": lock_file_path = "/tmp/place.downloads.process" lock = LockFile(lock_file_path) if lock.is_locked(): return lock.acquire() count = DownloadRequest.objects.filter(active=True).count() for dr in DownloadRequest.objects.filter(active=True): dr.create_zip_file_and_notify() lock.release() if count > 0: self.stdout.write("%d requests processed" % (count))
def share_post(sharing, db: Session): #sharing the file to the user as collaborator #is nothing but adding the user as collaborator #once the user is added as collaborator, the collaborator #can download it, edit and upload it. user_email = sharing.collaborator_id filename = sharing.filename access_level = sharing.access_level try: file_id = get_file_id_by_filename(filename, db) collaborator_id = get_email_id(user_email, db).id file_lock_check = LockFile(file_dir + '/' + filename) if file_lock_check.is_locked(): if is_owner(collaborator_id, file_id, db) == True: lock.release() else: return None shared_status = Collaborators(collaborator_id=collaborator_id, file_id=file_id, access_level=access_level, access_date=dt.datetime.now()) db.add(shared_status) db.commit() db.refresh(shared_status) return True except: return False
def write(self, path): """Writes the PlPlugMapM contents to a file.""" try: from lockfile import LockFile useLockFile = True except: useLockFile = False if useLockFile: # Get locked file lock = LockFile(path) if not lock.is_locked(): # Write plugmap file lock.acquire() ff = open(path, 'w') ff.write(self.data._contents) ff.close() lock.release() else: raise PlPlugMapMFileException( 'path {0} is locked'.format(path)) else: ff = open(path, 'w') ff.write(self.data._contents) ff.close()
def run_function_with_lock(self, function, lock_file, timeout=30, args=[], kwargs={}): self.logger.debug('starting function with lock: %s' % lock_file) lock = LockFile(lock_file) try: while not lock.i_am_locking(): try: lock.acquire(timeout=timeout) except (LockTimeout, NotMyLock) as e: self.logger.debug('breaking lock') lock.break_lock() lock.acquire() self.logger.exception(e) self.logger.debug('lock acquired: starting function') return function(*args, **kwargs) finally: self.logger.debug('function done, releasing lock') if lock.is_locked(): try: lock.release() except NotMyLock: try: os.remove(lock_file) except Exception as e: self.logger.exception(e) self.logger.debug('lock released')
class Repo: """ Class to deal with the metadata surrounding a Git repository """ def __init__(self, parent, repo_url): self.parent = parent self.url = repo_url self.folder_name = os.path.splitext(os.path.basename(repo_url))[0] self.containing_folder = os.path.join(parent.repo_folder, self.folder_name) if not os.path.exists(self.containing_folder): os.makedirs(self.containing_folder) self.path = os.path.join(self.containing_folder, 'repo') self.lockfile_path = os.path.join(self.containing_folder, 'lock') self.lock = LockFile(self.lockfile_path) self.json_path = os.path.join(self.containing_folder, 'metadata.json') self.data = {} if os.path.exists(self.json_path): with open(self.json_path) as json_file: self.data = json.load(json_file) self.__git = None def __enter__(self): """ Update context """ self.lock.acquire(timeout=0) logger.info('Git: Updating %s', self.url) if not os.path.exists(self.path): logger.debug('Cloning %s', self.url) git.Git(self.containing_folder).clone(self.url, self.path) else: try: repo = self.git(is_updater=True) logger.debug('Pulling %s', self.url) repo.git.pull() except Exception as e: logger.debug('Re-Cloning %s because %s', self.url, str(e)) shutil.rmtree(self.path) git.Git(self.containing_folder).clone(self.url, self.path) return self def __exit__(self, type, value, traceback): # Save the updated time self.data['last_updated'] = str(datetime.datetime.utcnow()) self.save() logger.info('Git: Update completed for %s', self.url) self.lock.break_lock() def save(self): with open(self.json_path, 'w') as f: json.dump(self.data, f) def git(self, is_updater=False): if self.lock.is_locked() and (not self.parent.is_updater and not is_updater): raise AlreadyLocked('This repository is being updated, if you can see this message delete {}'.format(self.lockfile_path)) else: if self.__git is None or is_updater: self.__git = git.Repo(self.path) return self.__git
def PythonAllocate ( num , resourcefile , list ) : IPList1 = "" count = 0 row_index = 0 from lockfile import LockFile lock = LockFile(resourcefile) lockid = lock.is_locked() print lockid for a in xrange(1, 2): if lockid == False: lock.acquire() print "I have locked Resource File" break else: time.sleep (10) lockid = lock.is_locked() rb = open_workbook(resourcefile) r_sheet = rb.sheet_by_index(0) wb = copy(rb) w_sheet = wb.get_sheet(0) keys = [r_sheet.cell(0, col_index).value for col_index in xrange(r_sheet.ncols)] j = r_sheet.nrows while row_index < j: d = {keys[col_index]: r_sheet.cell(row_index, col_index).value for col_index in xrange(r_sheet.ncols)} length = len(list) if ( d['STATUS'] == "free") : a = " " w_sheet.write(row_index,col_index-2,"allocated") wb.save(resourcefile) lock.release() count = count + 1 resourcename = d['IP'] string1 = str(list)+ a + str(resourcename) string2 = string1.strip() print string2 list = string2.split( ) length = len(list) print list j = 1 row_index = row_index + 1 print "list",list if ( list == "" ) : lock.release() return list
def _download_rpm(self, nvr, arch): if nvr is None or arch is None: raise ValueError("Invalid option passed to connector") filename = '%s.%s.rpm' % (nvr, arch) file_path = os.path.split(filename) if file_path[0] != '': raise ValueError("Nvr can not contain path elements") if len(arch.split('/')) != 1 or os.path.split(arch)[0] != '': raise ValueError("Arch can not contain path elements") rpm_file_path = os.path.join(self._rpm_cache, filename) if os.path.exists(rpm_file_path): return rpm_file_path lockfile = LockFile(file_path) if lockfile.is_locked(): # block until the lock is released and then assume other # thread was successful lockfile.acquire() lockfile.release() return rpm_file_path # acquire the lock and release when done lockfile.acquire() try: info = self.call('getBuild', {'buildInfo': nvr}) if info is None: return {'error': 'No such build (%s)' % filename} if not os.path.exists(self._rpm_cache): os.mkdir(self._rpm_cache, ) url = '%s/%s/%s/%s/%s/%s' % (self._koji_pkg_url, info['name'], info['version'], info['release'], arch, filename) url_file = grabber.urlopen(url, text=filename) out = os.open(rpm_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0666) try: while 1: buf = url_file.read(4096) if not buf: break os.write(out, buf) except Exception as e: raise e finally: os.close(out) url_file.close() finally: lockfile.release() return rpm_file_path
def _download_rpm(self, nvr, arch): if nvr is None or arch is None: raise ValueError("Invalid option passed to connector") filename = '%s.%s.rpm' % (nvr, arch) file_path = os.path.split(filename) if file_path[0] != '': raise ValueError("Nvr can not contain path elements") if len(arch.split('/')) != 1 or os.path.split(arch)[0] != '': raise ValueError("Arch can not contain path elements") rpm_file_path = os.path.join(self._rpm_cache, filename) if os.path.exists(rpm_file_path): return rpm_file_path lockfile = LockFile(file_path) if lockfile.is_locked(): # block until the lock is released and then assume other # thread was successful lockfile.acquire() lockfile.release() return rpm_file_path # acquire the lock and release when done lockfile.acquire() try: info = self.call('getBuild', {'buildInfo': nvr}) if info is None: return {'error': 'No such build (%s)' % filename} if not os.path.exists(self._rpm_cache): os.mkdir(self._rpm_cache,) url = '%s/%s/%s/%s/%s/%s' % ( self._koji_pkg_url, info['name'], info['version'], info['release'], arch, filename) url_file = grabber.urlopen(url, text=filename) out = os.open( rpm_file_path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0666) try: while 1: buf = url_file.read(4096) if not buf: break os.write(out, buf) except Exception as e: raise e finally: os.close(out) url_file.close() finally: lockfile.release() return rpm_file_path
class State(object): def __init__(self, path=None, lock=False): self.path = path self.lock = lock if not self.path: self.path = join(util.get_home_dir(), "appstate.json") self._state = {} self._prev_state = {} self._lockfile = None def __enter__(self): try: self._lock_state_file() if isfile(self.path): self._state = util.load_json(self.path) except exception.PlatformioException: self._state = {} self._prev_state = deepcopy(self._state) return self._state def __exit__(self, type_, value, traceback): if self._prev_state != self._state: with open(self.path, "w") as fp: if "dev" in __version__: json.dump(self._state, fp, indent=4) else: json.dump(self._state, fp) self._unlock_state_file() def _lock_state_file(self): if not self.lock: return self._lockfile = LockFile(self.path) if self._lockfile.is_locked() and \ (time() - getmtime(self._lockfile.lock_file)) > 10: self._lockfile.break_lock() try: self._lockfile.acquire() except LockFailed: raise exception.PlatformioException( "The directory `{0}` or its parent directory is not owned by " "the current user and PlatformIO can not store configuration " "data. \nPlease check the permissions and owner of that " "directory. Otherwise, please remove manually `{0}` " "directory and PlatformIO will create new from the current " "user.".format(dirname(self.path))) def _unlock_state_file(self): if self._lockfile: self._lockfile.release()
class State(object): def __init__(self, path=None, lock=False): self.path = path self.lock = lock if not self.path: self.path = join(util.get_home_dir(), "appstate.json") self._state = {} self._prev_state = {} self._lockfile = None def __enter__(self): try: self._lock_state_file() if isfile(self.path): self._state = util.load_json(self.path) except ValueError: self._state = {} self._prev_state = deepcopy(self._state) return self._state def __exit__(self, type_, value, traceback): if self._prev_state != self._state: with open(self.path, "w") as fp: if "dev" in __version__: json.dump(self._state, fp, indent=4) else: json.dump(self._state, fp) self._unlock_state_file() def _lock_state_file(self): if not self.lock: return self._lockfile = LockFile(self.path) if self._lockfile.is_locked() and \ (time() - getmtime(self._lockfile.lock_file)) > 10: self._lockfile.break_lock() try: self._lockfile.acquire() except LockFailed: raise exception.PlatformioException( "The directory `{0}` or its parent directory is not owned by " "the current user and PlatformIO can not store configuration " "data. \nPlease check the permissions and owner of that " "directory. Otherwise, please remove manually `{0}` " "directory and PlatformIO will create new from the current " "user.".format(dirname(self.path))) def _unlock_state_file(self): if self._lockfile: self._lockfile.release()
def list_stale_environment_clones(self): """ Lists environment clones left behind on disk and no longer used Stale clones could be left behind because of an error during a previous run. An environment is defined as stale if it fits the name of a managed environment with a suffix, and is not pointed at by any symlinks :return: list(str) """ links = {} candidates = [] stale_clones = [] items = os.listdir(self.environment_dir) for item in items: # Ignore hidden files if item.startswith('.'): continue # Explicitly ignore the master repo name if item == self.master_repo_name: continue # Ignore anything matching the blacklist pattern if self.blacklist.match(item): self.logger.debug( "Ignoring blacklisted environment {0}".format(item)) continue path = os.path.join(self.environment_dir, item) if os.path.islink(path): links[os.readlink(path)] = path elif os.path.isdir(path): candidates.append(path) # Look for candidate environments which aren't the target of any symlinks for candidate in candidates: if candidate not in links: environment_path = self.environment_repo_path( self.identify_environment_name_from_clone_name( self.identify_environment_name_from_path(candidate))) lock = LockFile(environment_path) if lock.is_locked(): # Ignore locked environments, might be in use continue self.logger.debug( "Stale environment detected: {0}".format(candidate)) stale_clones.append(candidate) return stale_clones
class State(object): def __init__(self, path=None, lock=False): self.path = path self.lock = lock if not self.path: self.path = join(util.get_home_dir(), "appstate.json") self._state = {} self._prev_state = {} self._lockfile = None def __enter__(self): try: self._lock_state_file() if isfile(self.path): self._state = util.load_json(self.path) except exception.PlatformioException: self._state = {} self._prev_state = deepcopy(self._state) return self._state def __exit__(self, type_, value, traceback): if self._prev_state != self._state: try: with open(self.path, "w") as fp: if "dev" in __version__: json.dump(self._state, fp, indent=4) else: json.dump(self._state, fp) except IOError: raise exception.HomeDirPermissionsError(util.get_home_dir()) self._unlock_state_file() def _lock_state_file(self): if not self.lock: return self._lockfile = LockFile(self.path) if self._lockfile.is_locked() and \ (time() - getmtime(self._lockfile.lock_file)) > 10: self._lockfile.break_lock() try: self._lockfile.acquire() except LockFailed: raise exception.HomeDirPermissionsError(dirname(self.path)) def _unlock_state_file(self): if self._lockfile: self._lockfile.release()
def lock(self): """ Method used for acquiring a lock using the lockfile module. """ lock = LockFile(self.lockfile) # check if it's locked if lock.is_locked(): # Note that lock.i_am_locking() could be True, so # this apporach is not really efficient from a threading # point of view. However, we must be consistent with # MemcachedCacheQueue's behavior. return False # else we can attempt to acquire a lock # we don't want this to fail silently # so we set timeout=0 lock.acquire(timeout=0) return True
class State(object): def __init__(self, path=None, lock=False): self.path = path self.lock = lock if not self.path: self.path = join(get_home_dir(), "appstate.json") self._state = {} self._prev_state = {} self._lockfile = None def __enter__(self): try: self._lock_state_file() if isfile(self.path): with open(self.path, "r") as fp: self._state = json.load(fp) except ValueError: self._state = {} self._prev_state = deepcopy(self._state) return self._state def __exit__(self, type_, value, traceback): if self._prev_state != self._state: with open(self.path, "w") as fp: if "dev" in __version__: json.dump(self._state, fp, indent=4) else: json.dump(self._state, fp) self._unlock_state_file() def _lock_state_file(self): if not self.lock: return self._lockfile = LockFile(self.path) if (self._lockfile.is_locked() and (time() - getmtime(self._lockfile.lock_file)) > 10): self._lockfile.break_lock() self._lockfile.acquire() def _unlock_state_file(self): if self._lockfile: self._lockfile.release()
class State(object): def __init__(self, path=None, lock=False): self.path = path self.lock = lock if not self.path: self.path = join(get_home_dir(), "appstate.json") self._state = {} self._prev_state = {} self._lockfile = None def __enter__(self): try: self._lock_state_file() if isfile(self.path): with open(self.path, "r") as fp: self._state = json.load(fp) except ValueError: self._state = {} self._prev_state = deepcopy(self._state) return self._state def __exit__(self, type_, value, traceback): if self._prev_state != self._state: with open(self.path, "w") as fp: if "dev" in __version__: json.dump(self._state, fp, indent=4) else: json.dump(self._state, fp) self._unlock_state_file() def _lock_state_file(self): if not self.lock: return self._lockfile = LockFile(self.path) if (self._lockfile.is_locked() and (time() - getmtime(self._lockfile.lock_file)) > 10): self._lockfile.break_lock() self._lockfile.acquire() def _unlock_state_file(self): if self._lockfile: self._lockfile.release()
def handle_selected_files(files): status = {} for file in files: status.update([(file, {'name': file, 'loaded': False, 'errors': []})]) file_path = file_dir + file lock = LockFile(file_path) if lock.is_locked(): status[file]['errors'].append( 'Файл заблокирован (загружается другим процессом).') continue else: lock.acquire() if not isSQLite3(file_path): status[file]['errors'].append('Не корректный формат файла.') lock.release() continue conn = sqlite3.connect(file_path) cursor = conn.cursor() packets_file = exec_query(conn, cursor, query_packets_file) transfer_file = exec_query(conn, cursor, query_transfer_file) if isinstance(transfer_file, str): status[file]['errors'].append(transfer_file) lock.release() continue elif isinstance(packets_file, str): status[file]['errors'].append(packets_file) lock.release() continue pc_name = transfer_file[0][0] mac_addr = transfer_file[0][1] date_db = transfer_file[0][2] uid_id = transfer_file[0][3] if not uid_id: status[file]['errors'].append('Не указан Uid') lock.release() continue if not mac_addr: status[file]['errors'].append('Не указан mac-адрес') lock.release() continue if not date_db: status[file]['errors'].append('Не указана дата выгрузки') lock.release() continue if Transfer.objects.filter(mac_addr=mac_addr, date_db=date_db): status[file]['errors'].append('Данный файл уже загружен -' + mac_addr + ' - ' + date_db) lock.release() continue if not Uid.objects.filter(uid=uid_id): uid_obj = Uid(name='Unknown', uid=uid_id) else: uid_obj = Uid.objects.get(uid=uid_id) transfer_obj = Transfer(pc_name=pc_name, mac_addr=mac_addr, date_db=date_db, uid=uid_obj) uid_obj.save() transfer_obj.save() packets_all = [] for packet in packets_file: packets_all.append( Traffic(datetime=packet[0], src=packet[1], dst=packet[2], pkt_size=packet[3], transfer=transfer_obj)) conn.close() lock.release() # rm = os.remove(file_path) # status[file]['errors'].append(str(rm)) try: os.remove(file_path) except Exception as err: status[file]['errors'].append('Ошибка удаления файла: ' + str(err)) else: Traffic.objects.bulk_create(packets_all) status[file]['loaded'] = True # Traffic.objects.bulk_create(packets_all) # status[file]['loaded'] = True return status
class BatchTrigger(): def __init__(self): self.strExecSummaryHTMLFilePath = "" self.intPassKITCount = 0 self.intFailKITCount = 0 self.lock = None #Function to execute the batch kits for the Build Pipeline def build_execute_kitlist(self): try: strEnvironmentFolderPAth = os.path.abspath( __file__ + "/../../../") + "/02_Manager_Tier/EnviromentFile/" self.strAndroidDeviceJsonFilePath = strEnvironmentFolderPAth + '/android_device_list.json' self.lock = LockFile(self.strAndroidDeviceJsonFilePath) deviceUtils.create_android_device_json() time.sleep(10) deviceUtils.install_app_android_device(strAndroidAppFilePath) utils.setAttribute_KitBatch('batch_execution', 'status', 'YES') oBKitJsonDict = self.get_kit_batch_json() self.current_batch_id = oBKitJsonDict["kit_batch"][ 'current_batch_id'] oKitList = oBKitJsonDict["kit_batch"]['list_of_batches'][ self.current_batch_id]['list_of_kits'] if len(oKitList.keys()) > 0: utils.setAttribute_KitBatch( 'batch_execution', 'current_batch_result_folder', self.current_batch_id + '_' + self.getTimeStamp(True)) #Main Summary result self.Batch_Execution_Summary_Initialize() oKitPriority = {} #List Kits based on priority for current_kit_id in oKitList: oKitPriority[int( oKitList[current_kit_id]['priority'])] = current_kit_id #Trigger the Execution for the kits for oPKey in sorted(oKitPriority.keys()): #self.trigger_parallel_kit_execution(oKitPriority[oPKey]) # Start the Individual Kit parallel execution parallelExecThread = threading.Thread( target=self.trigger_parallel_kit_execution, args=(oKitPriority[oPKey], )) parallelExecThread.daemon = True # This kills the thread when main program exits parallelExecThread.start() parallelExecThread.name = oKitPriority[oPKey] jobs.append(parallelExecThread) time.sleep(50) for oJob in jobs: oJob.join() #Footer for Batch summary report self.Batch_Execution_Summary_Footer() else: print('No kits in the Selected batch') utils.setAttribute_KitBatch('batch_execution', 'status', 'NO') except: print( 'Batch Execution: Exception in build_execute_kitlist Method\n {0}' .format(traceback.format_exc().replace('File', '$~File'))) utils.setAttribute_KitBatch('batch_execution', 'status', 'NO') def trigger_parallel_kit_execution(self, oKitPkey): print("Hello") while True: boolGotDevice, strDevice, strPort, strDeviceID = self.get_device_from_json( ) print('boolGotDevice, strDevice, strPort ', boolGotDevice, strDevice, strPort) if boolGotDevice: break else: time.sleep(30) strResultFolder, User_ID, Test_Suite, Main_Client, Second_Client, strCurrentEnvironment = self.update_gloabalVar_json( self.current_batch_id, oKitPkey, strPort, strDeviceID) self.update_main_result_json(oKitPkey, strResultFolder, User_ID, Test_Suite, Main_Client, Second_Client, strCurrentEnvironment) strHTMLSummaryPath = strResultFolder + "/Hive-" + strCurrentEnvironment + "_Execution_Summary.HTML" #Trigger the test execution for the Kit #utils.setAttribute('common', 'appium_port', strPort) self.trigger_test_run(oKitPkey) #Add Hyper link for the Kit execution '''Main_Client = utils.getAttribute('common', 'mainClient') Second_Client = utils.getAttribute('common', 'secondClient') User_ID = utils.getAttribute('common', 'userName') Test_Suite = utils.getAttribute('common', 'test_suite')''' Kit_Setup = "" oDeviceVersionDict = self.get_device_details() print(oDeviceVersionDict) for oKey in oDeviceVersionDict.keys(): Kit_Setup = Kit_Setup + oDeviceVersionDict[oKey][ 'model'] + ' ==> ' + oDeviceVersionDict[oKey][ 'version'] + '<br>' self.Batch_Execution_Summary_KitAddLink(oKitPkey, strHTMLSummaryPath, Main_Client, Second_Client, User_ID, Kit_Setup, Test_Suite) self.set_device_exec_status(strDevice, 'COMPLETED') def get_device_from_json(self): strPort = "" strDevice = "" boolGotDevice = False strDeviceID = "" try: if not self.lock.is_locked(): self.lock.acquire() print(self.lock.path, 'is locked.') strJson = open(self.strAndroidDeviceJsonFilePath, mode='r') oADLJsonDict = json.loads(strJson.read()) strJson.close() oANDLIST = oADLJsonDict['android_devicelist'] for oDevice in oANDLIST: strDeviceExecStatus = oANDLIST[oDevice]['status'] if not strDeviceExecStatus.upper() == 'IN PROGRESS': strPort = oANDLIST[oDevice]['port'] strDeviceID = oANDLIST[oDevice]['device_id'] strDevice = oDevice boolGotDevice = True oANDLIST[oDevice]['status'] = 'IN PROGRESS' oADLJsonDict['android_devicelist'] = oANDLIST #Write back the JSON to the GlobalVar.JSON oJson = open(self.strAndroidDeviceJsonFilePath, mode='w+') oJson.write( json.dumps(oADLJsonDict, indent=4, sort_keys=True)) oJson.close() break self.lock.release() print(self.lock.path, 'is unlocked.') return boolGotDevice, strDevice, strPort, strDeviceID except: print(traceback.format_exc()) return boolGotDevice, strDevice, strPort, strDeviceID def set_device_exec_status(self, strDevice, strStatus): if not self.lock.is_locked(): self.lock.acquire() print(self.lock.path, 'is locked.') strJson = open(self.strAndroidDeviceJsonFilePath, mode='r') oADLJsonDict = json.loads(strJson.read()) strJson.close() oANDLIST = oADLJsonDict['android_devicelist'] for oDevice in oANDLIST: if oDevice == strDevice: oANDLIST[oDevice]['status'] = strStatus oADLJsonDict['android_devicelist'] = oANDLIST #Write back the JSON to the GlobalVar.JSON oJson = open(self.strAndroidDeviceJsonFilePath, mode='w+') oJson.write( json.dumps(oADLJsonDict, indent=4, sort_keys=True)) oJson.close() self.lock.release() print(self.lock.path, 'is unlocked.') #Updated the GlobalVarJson According to the Selected Kit def update_gloabalVar_json(self, current_batch_id=None, current_kit_id=None, appium_node='4723', strDeviceID=""): strEnvironmentFolderPAth = os.path.abspath( __file__ + "/../../../") + "/02_Manager_Tier/EnviromentFile/" strGlobVarFilePath = strEnvironmentFolderPAth + '/GlobalVar.json' strJson = open(strGlobVarFilePath, mode='r') oGBVJsonDict = json.loads(strJson.read()) strJson.close() oGlobalDict = oGBVJsonDict['globalVariables'] if current_batch_id == None or current_kit_id == None: return if current_batch_id != "" and current_kit_id != "": oJsonDict = self.get_kit_batch_json() oKitDetails = oJsonDict["kit_batch"]['list_of_batches'][ current_batch_id]['list_of_kits'][current_kit_id] oGlobalDict['currentEnvironment'] = oKitDetails[ 'currentEnvironment'] oGlobalDict['apiValidationType'] = oKitDetails['apiValidationType'] oGlobalDict['mainClient'] = oKitDetails['mainClient']['name'] oGlobalDict['secondClient'] = oKitDetails['secondClient']['name'] strAppVer = oKitDetails['currentAppVersion'] oGlobalDict['currentAppVersion'] = strAppVer oGlobalDict['userName'] = oKitDetails['userName'] oGlobalDict['password'] = oKitDetails['password'] oGlobalDict['appium_port'] = appium_node oGlobalDict['appium_udid'] = strDeviceID #Write result folder name oGlobalDict['batch_execution'] = {} oGlobalDict['batch_execution']['status'] = "YES" strResultFolder = current_kit_id + '_' + oGlobalDict[ 'test_suite'] + '_' + self.getTimeStamp(True) oGlobalDict['batch_execution'][ 'result_folder_name'] = strResultFolder print(strResultFolder) #oKitDetails['resultFolderLabel'] = "" # Environment.get() oCurrentEnvDict = oGlobalDict['listOfEnvironments'][ oKitDetails['currentEnvironment']] if 'IOS' in oKitDetails['mainClient']['name'].upper(): oCurrentEnvDict['iOS' + strAppVer][ 'appFileName'] = oKitDetails['mainClient']['appFileName'] oCurrentEnvDict['iOS' + strAppVer]['deviceName'] = oKitDetails[ 'mainClient']['deviceName'] oCurrentEnvDict[ 'iOS' + strAppVer]['udid'] = oKitDetails['mainClient']['udid'] oCurrentEnvDict['iOS' + strAppVer]['platformVersion'] = oKitDetails[ 'mainClient']['platformVersion'] elif 'ANDROID' in oKitDetails['mainClient']['name'].upper(): oCurrentEnvDict['android' + strAppVer][ 'appFileName'] = oKitDetails['mainClient']['appFileName'] oCurrentEnvDict['android' + strAppVer][ 'deviceName'] = oKitDetails['mainClient']['deviceName'] oCurrentEnvDict['android' + strAppVer]['platformVersion'] = oKitDetails[ 'mainClient']['platformVersion'] elif 'WEB' in oKitDetails['mainClient']['name'].upper(): oCurrentEnvDict['web' + strAppVer][ 'browserName'] = oKitDetails['mainClient']['browserName'] oCurrentEnvDict['web' + strAppVer]['loginURL'] = oKitDetails[ 'mainClient']['loginURL'] oGlobalDict['secondClientValidateFlag'] = oKitDetails[ 'secondClientValidateFlag'] if 'IOS' in oKitDetails['secondClient']['name'].upper(): oCurrentEnvDict['iOS' + strAppVer][ 'appFileName'] = oKitDetails['secondClient']['appFileName'] oCurrentEnvDict['iOS' + strAppVer]['deviceName'] = oKitDetails[ 'secondClient']['deviceName'] oCurrentEnvDict[ 'iOS' + strAppVer]['udid'] = oKitDetails['secondClient']['udid'] oCurrentEnvDict['iOS' + strAppVer]['platformVersion'] = oKitDetails[ 'secondClient']['platformVersion'] elif 'ANDROID' in oKitDetails['secondClient']['name'].upper(): oCurrentEnvDict['android' + strAppVer][ 'appFileName'] = oKitDetails['secondClient']['appFileName'] oCurrentEnvDict['android' + strAppVer][ 'deviceName'] = oKitDetails['secondClient']['deviceName'] oCurrentEnvDict['android' + strAppVer]['platformVersion'] = oKitDetails[ 'secondClient']['platformVersion'] elif 'WEB' in oKitDetails['mainClient']['name'].upper(): oCurrentEnvDict['web' + strAppVer][ 'browserName'] = oKitDetails['secondClient']['browserName'] oCurrentEnvDict['web' + strAppVer]['loginURL'] = oKitDetails[ 'secondClient']['loginURL'] oGlobalDict['listOfEnvironments'][ oKitDetails['currentEnvironment']] = oCurrentEnvDict oGBVJsonDict['globalVariables'] = oGlobalDict oGlobalDict['test_suite'] = oKitDetails['test_suite'] #Write back the JSON to the GlobalVar.JSON oJson = open(strGlobVarFilePath, mode='w+') oJson.write(json.dumps(oGBVJsonDict, indent=4, sort_keys=True)) oJson.close() return strResultFolder, oGlobalDict['userName'], oGlobalDict[ 'test_suite'], oGlobalDict['mainClient'], oGlobalDict[ 'secondClient'], oGlobalDict['currentEnvironment'] def update_main_result_json(self, oKitPkey, strResultFolder, User_ID, Test_Suite, Main_Client, Second_Client, strCurrentEnvironment): oMRJson = open(self.MainResultJsonPath, mode='r') oMRJsonDict = json.loads(oMRJson.read()) oMRJson.close() oMRJsonDict["list_of_kits"][oKitPkey] = { "result_folder": strResultFolder, "username": User_ID, "test_suite": Test_Suite, "mainClient": Main_Client, "secondClient": Second_Client, "currentEnvironment": strCurrentEnvironment } #Write back the JSON to the GlobalVar.JSON oMRJson = open(self.MainResultJsonPath, mode='w+') oMRJson.write(json.dumps(oMRJsonDict, indent=4, sort_keys=True)) oMRJson.close() #Create the Main Results Json def create_main_result_json(self): self.MainResultJsonPath = self.BatchResultPath + '/main_result.json' oMRJson = open(self.MainResultJsonPath, mode='w+') oJsonDict = { "main_result_folder": self.BatchResultPath, "current_batch_id": self.current_batch_id, "list_of_kits": {} } oMRJson.write(json.dumps(oJsonDict, indent=4, sort_keys=False)) oMRJson.close() def trigger_test_run(self, strKitID=""): self.set_global_var() '''if not ('ZIGBEE' in utils.getAttribute("common", 'apiValidationType').upper() or 'WEB' in utils.getAttribute("common", 'mainClient').upper()): subprocess.call('killall node', shell=True) subprocess.Popen(striOSAppiumConnectionString, shell=True)''' '''if 'ANDROID' in utils.getAttribute("common", 'mainClient').upper(): subprocess.call('adb kill-server', shell=True) subprocess.call('adb start-server', shell=True) print()''' time.sleep(5) strTestSuite = utils.getAttribute("common", 'test_suite') if strTestSuite == 'BasicSmokeTest_Dual': oProcess = subprocess.Popen("behave --tags=BasicSmokeTest", stdout=subprocess.PIPE, shell=True) elif strTestSuite == 'BasicSmokeTest_Heating': oProcess = subprocess.Popen( "behave --tags=BasicSmokeTest --tags=Heating", stdout=subprocess.PIPE, shell=True) elif strTestSuite == 'BasicSmokeTest_HotWater': oProcess = subprocess.Popen( "behave --tags=BasicSmokeTest --tags=HotWater", stdout=subprocess.PIPE, shell=True) elif strTestSuite == 'ScheduleTest_Dual': oProcess = subprocess.Popen( "behave --tags=ScheduleTest --tags=Verify", stdout=subprocess.PIPE, shell=True) elif strTestSuite == 'ScheduleTest_Heating': oProcess = subprocess.Popen( "behave --tags=ScheduleTest --tags=Verify --tags=Heating", stdout=subprocess.PIPE, shell=True) elif strTestSuite == 'ScheduleTest_HotWater': oProcess = subprocess.Popen( "behave --tags=ScheduleTest --tags=Verify --tags=HotWater", stdout=subprocess.PIPE, shell=True) elif strTestSuite == 'Test_Batch': oProcess = subprocess.Popen("behave --tags=Test_Batch", stdout=subprocess.PIPE, shell=True) print('Test suite Triggered') while True: output = oProcess.stdout.readline() if oProcess.poll() is not None: break if output: print(strKitID, output) print("@@@@@@@@@@@Successfully completed", strKitID, output) def get_kit_batch_json(self): strEnvironmentFolderPAth = os.path.abspath( __file__ + "/../../../") + "/02_Manager_Tier/EnviromentFile/" strGlobVarFilePath = strEnvironmentFolderPAth + '/kit_batch.json' strJson = open(strGlobVarFilePath, mode='r') oJsonDict = json.loads(strJson.read()) strJson.close() return oJsonDict def set_global_var(self): strEnvironmentFolderPAth = os.path.abspath( __file__ + "/../../../") + "/02_Manager_Tier/EnviromentFile" strGlobVarFilePath = strEnvironmentFolderPAth + '/GlobalVar.json' strJson = open(strGlobVarFilePath, mode='r') oJsonDict = json.loads(strJson.read()) strJson.close() utils.oJsonDict = oJsonDict #Gets the Time stamp for creating the folder set or for reporting time stamp based on boolFolderCreate def getTimeStamp(self, boolFolderCreate): if boolFolderCreate: str_format = "%d-%b-%Y_%H-%M-%S" else: str_format = "%d-%b-%Y %H:%M:%S" today = datetime.today() return today.strftime(str_format) #Batch Execution Summary def Batch_Execution_Summary_Initialize(self): strAPI = utils.getAttribute('common', 'apiValidationType') if 'ZIGBEE' in strAPI.upper(): strAPIFolder = 'Device_Test_Automation/' else: strAPIFolder = 'Web-Mobile_Test_Automation' strSystemResultFolderName = '' if 'DARWIN' in platform.system().upper(): if os.path.exists("/volumes/hardware"): strSystemResultFolderName = getpass.getuser( ) + "_" + socket.gethostname().split(".")[0] strTestResultFolder = "/volumes/hardware/" + strAPIFolder + '/Test_Results/' self.ensure_dir(strTestResultFolder + strSystemResultFolderName) elif 'LINUX' in platform.system().upper(): if os.path.exists("/home/pi/hardware"): strSystemResultFolderName = socket.gethostname().split( ".")[0].split("-")[1] strTestResultFolder = "/home/pi/hardware/" + strAPIFolder + '/Test_Results/' self.ensure_dir(strTestResultFolder + strSystemResultFolderName) if not strSystemResultFolderName == "": self.strResultsPath = strTestResultFolder + strSystemResultFolderName + '/' else: self.strResultsPath = os.path.abspath( __file__ + "/../../../") + '/03_Results_Tier/' strResultsPath = self.strResultsPath + utils.getAttribute_KitBatch( 'batch_execution', 'current_batch_result_folder') + '/' if not os.path.exists(strResultsPath): os.makedirs(strResultsPath) self.BatchResultPath = strResultsPath self.strExecSummaryHTMLFilePath = strResultsPath + "HIVE_BATCH_Execution_Summary.HTML" print(self.strExecSummaryHTMLFilePath) self.create_main_result_json() try: strEnvironmentFilePath = os.path.abspath( __file__ + "/../../../") + "/02_Manager_Tier/EnviromentFile" oFileW = open(strEnvironmentFilePath + '/scripts/Temp.txt', 'w') oFileW.write(self.strExecSummaryHTMLFilePath) oFileW.close() strCSSFilePath = strEnvironmentFilePath + "/Style.CSS" oFileReader = open(strCSSFilePath, 'r') strData = oFileReader.read() oFileReader.close() oFileWriter = open(self.strExecSummaryHTMLFilePath, 'w') oFileWriter.write("<!DOCTYPE html>\n") oFileWriter.write("<html>\n") oFileWriter.write("<head>\n") oFileWriter.write(" <meta charset='UTF-8'>\n") oFileWriter.write( " <title>Hive - Automation Execution Results Batch-Summary</title>\n" ) oFileWriter.write(strData + '\n') oFileWriter.write("</head>\n") oFileWriter.write("<body>\n") oFileWriter.write("<table id='header'>\n") oFileWriter.write("<colgroup>\n") oFileWriter.write("<col style='width: 25%' />\n") oFileWriter.write("<col style='width: 25%' />\n") oFileWriter.write("<col style='width: 25%' />\n") oFileWriter.write("<col style='width: 25%' />\n") oFileWriter.write("</colgroup>\n") oFileWriter.write("<thead>\n") oFileWriter.write("<tr class='heading'>\n") oFileWriter.write( "<th colspan='4' style='font-family:Copperplate Gothic Bold; font-size:1.4em;'>\n" ) oFileWriter.write( "Hive - Automation Execution Results Batch-Summary\n") oFileWriter.write("</th>\n") oFileWriter.write("</tr>\n") oFileWriter.write("<tr class='subheading'>\n") oFileWriter.write("<th> Date & Time</th>\n") #oFileWriter.write("<th> : 25-Jul-2014 05:02:20 PM</th>\n") oFileWriter.write("<th> : " + self.getTimeStamp(False) + "</th>\n") self.intExecStartTime = time.monotonic() oFileWriter.write("<th> Batch Name</th>\n") oFileWriter.write("<th> : " + self.current_batch_id + "</th>\n") oFileWriter.write("</tr>\n") oFileWriter.write("</thead>\n") oFileWriter.write("</table>\n") oFileWriter.write("<table id='main'>\n") oFileWriter.write("<colgroup>\n") oFileWriter.write("<col style='width: 10%' />\n") oFileWriter.write("<col style='width: 10%' />\n") oFileWriter.write("<col style='width: 10%' />\n") oFileWriter.write("<col style='width: 15%' />\n") oFileWriter.write("<col style='width: 15%' />\n") oFileWriter.write("<col style='width: 15%' />\n") oFileWriter.write("<col style='width: 15%' />\n") oFileWriter.write("<col style='width: 10%' />\n") oFileWriter.write("</colgroup>\n") oFileWriter.write("<thead>\n") oFileWriter.write("<tr class='heading'>\n") oFileWriter.write("<th>Kit_ID</th>\n") oFileWriter.write("<th>Main_Client</th>\n") oFileWriter.write("<th>Second_Client</th>\n") oFileWriter.write("<th>User_ID</th>\n") oFileWriter.write("<th>Kit_Setup</th>\n") oFileWriter.write("<th>Test_Suite</th>\n") oFileWriter.write("<th>Execution_Time</th>\n") oFileWriter.write("<th>Status</th>\n") oFileWriter.write("</tr>\n") oFileWriter.write("</thead>\n") #Always close files. oFileWriter.close() except: print( 'Reporter Exception in BATCH_HTML_Execution_Summary_Initialize\n {0}' .format(traceback.format_exc().replace('File', '$~File'))) def Batch_Execution_Summary_KitAddLink(self, Kit_ID, strHTMLSummaryPath, Main_Client, Second_Client, User_ID, Kit_Setup, Test_Suite): try: oFileWriter = open(self.strExecSummaryHTMLFilePath, 'a') oFileWriter.write("<tr class='content' >\n") #oFileDet = os.path.split(utils.getAttribute_KitBatch('batch_execution', 'current_kit_result_summary_path')) #strFilePath = utils.getAttribute_KitBatch('batch_execution', 'current_kit_result_summary_path') oFileWriter.write("<td class='justified'><a href='" + './Kit_Results/' + strHTMLSummaryPath + "' target='about_blank'>" + str(Kit_ID) + "</a></td>\n") oFileWriter.write("<td class='justified'>" + Main_Client + "</td>\n") oFileWriter.write("<td class='justified'>" + Second_Client + "</td>\n") oFileWriter.write("<td class='justified'>" + User_ID + "</td>\n") oFileWriter.write("<td class='justified'>" + Kit_Setup + "</td>\n") oFileWriter.write("<td class='justified'>" + Test_Suite + "</td>\n") oFileWriter.write( "<td class='justified'>" + utils.getAttribute_KitBatch( 'batch_execution', 'current_kit_execution_time') + "</td>\n") self.strKitStatus = utils.getAttribute_KitBatch( 'batch_execution', 'current_kit_status') strStatusClass = self.strKitStatus[0:4].lower() oFileWriter.write("<td class='" + strStatusClass + "'>" + self.strKitStatus + "</td>\n") oFileWriter.write("</tr>\n") if (self.strKitStatus == "PASSED"): self.intPassKITCount = self.intPassKITCount + 1 if (self.strKitStatus == "FAILED"): self.intFailKITCount = self.intFailKITCount + 1 #Always close files. oFileWriter.close() except IOError as e: print("I/O error({0}): {1}".format(e.errno, e.strerror)) def Batch_Execution_Summary_Footer(self): try: oFileWriter = open(self.strExecSummaryHTMLFilePath, 'a') oFileWriter.write("</tbody>\n") oFileWriter.write("</table>\n") oFileWriter.write("<table>\n") oFileWriter.write("<script type='text/javascript'>\n") oFileWriter.write("window.onload = function () {\n") oFileWriter.write("CanvasJS.addColorSet('chartshades',\n") oFileWriter.write("[//colorSet Array\n") oFileWriter.write("'lightgreen',\n") oFileWriter.write("'red' \n") oFileWriter.write("]);\n") oFileWriter.write( "var chart = new CanvasJS.Chart('chartContainer',\n") oFileWriter.write("{\n") oFileWriter.write("colorSet: 'chartshades',\n") oFileWriter.write("zoomEnabled: true,\n") oFileWriter.write("title:{\n") oFileWriter.write("fontColor: '#C6FFEC',\n") oFileWriter.write("text: 'Execution Status'\n") oFileWriter.write("},\n") oFileWriter.write("animationEnabled: true,\n") oFileWriter.write("backgroundColor: 'black',\n") oFileWriter.write("legend:{\n") oFileWriter.write("fontColor: '#C6FFEC',\n") oFileWriter.write("verticalAlign: 'bottom',\n") oFileWriter.write("horizontalAlign: 'center'\n") oFileWriter.write("},data: [{ \n") oFileWriter.write("indexLabelFontSize: 20,\n") oFileWriter.write("indexLabelFontFamily: 'Monospace', \n") oFileWriter.write("indexLabelFontColor: '#C6FFEC', \n") oFileWriter.write("indexLabelLineColor: '#C6FFEC', \n") oFileWriter.write("indexLabelPlacement: 'auto',\n") oFileWriter.write("type: 'pie', \n") oFileWriter.write("showInLegend: true,\n") oFileWriter.write( "toolTipContent: '{y} - <strong>#percent%</strong>',\n") oFileWriter.write("dataPoints: [\n") if not self.intPassKITCount == 0: oFileWriter.write( "{ y: " + str(self.intPassKITCount) + ", legendText:'PASS', indexLabel: '{y}' },\n") else: oFileWriter.write("{ y: " + str(self.intPassKITCount) + ", legendText:'PASS'},\n") if not self.intFailKITCount == 0: oFileWriter.write("{ y: " + str(self.intFailKITCount) + ", legendText:'FAIL' , indexLabel: '{y}'}\n") else: oFileWriter.write("{ y: " + str(self.intFailKITCount) + ", legendText:'FAIL'}\n") oFileWriter.write("]}]});chart.render();}\n") oFileWriter.write("</script>\n") oFileWriter.write( "<script type='text/javascript' src='./Kit_Results/Temp/canvasjs.min.js'></script>\n" ) oFileWriter.write( "<tr class='content' ><td><div id='chartContainer' style='height: 300px; width: 100%;'></div></td></tr></table>\n" ) oFileWriter.write("<table id='footer'>\n") oFileWriter.write("<colgroup>\n") oFileWriter.write("<col style='width: 25%' />\n") oFileWriter.write("<col style='width: 25%' />\n") oFileWriter.write("<col style='width: 25%' />\n") oFileWriter.write("<col style='width: 25%' />\n") oFileWriter.write("</colgroup>\n") oFileWriter.write("<tfoot>\n") oFileWriter.write("<tr class='heading'>\n") intExecEndTime = time.monotonic() strDuration = str( timedelta(seconds=intExecEndTime - self.intExecStartTime)) strDuration = self.getDuration(strDuration) oFileWriter.write("<th colspan='4'>Total Duration: " + strDuration + "</th>\n") oFileWriter.write("</tr>\n") oFileWriter.write("<tr class='subheading'>\n") oFileWriter.write("<td class='pass'> Tests passed</td>\n") oFileWriter.write("<td class='pass'> : {}</td>\n".format( self.intPassKITCount)) oFileWriter.write("<td class='fail'> Tests failed</td>\n") oFileWriter.write("<td class='fail'> : {}</td>\n".format( self.intFailKITCount)) oFileWriter.write("</tr>\n") oFileWriter.write("</tfoot>\n") oFileWriter.write("</table>\n") oFileWriter.write("</body>\n") oFileWriter.write("</html>\n") #Always close files. oFileWriter.close() ''' if os.path.exists(self.strCurrentTXTFolder + 'ExecutionInProgress.txt'): os.remove(self.strCurrentTXTFolder + 'ExecutionInProgress.txt')''' except IOError as e: print("I/O error({0}): {1}".format(e.errno, e.strerror)) def get_connectedAndroid_device(self): print() def get_device_details(self): env = utils.getAttribute('common', 'currentEnvironment') username = utils.getAttribute('common', 'userName') password = utils.getAttribute('common', 'password') ALAPI.createCredentials(env, username=username, password=password) self.session = ALAPI.sessionObject() resp = ALAPI.getNodesV6(self.session) oDeviceVersionDict = {} strSLTMacID = "" strSLRMacID = "" for oNode in resp['nodes']: if not 'supportsHotWater' in oNode['attributes']: if 'hardwareVersion' in oNode['attributes']: intHardwareVersion = oNode['attributes'][ 'hardwareVersion']['reportedValue'] intSoftwareVersion = oNode['attributes'][ 'softwareVersion']['reportedValue'] if 'NANO' in intHardwareVersion: oDeviceVersionDict['HUB'] = { "model": intHardwareVersion, "version": intSoftwareVersion } if 'zigBeeNeighbourTable' in oNode['attributes']: for oDevice in oNode['attributes']['zigBeeNeighbourTable'][ 'reportedValue']: if 'relationship' in oDevice: if oDevice['relationship'] == 'CHILD': strSLTMacID = oDevice['neighbourAddress'] elif oDevice['relationship'] == 'NONE': strSLRMacID = oDevice['neighbourAddress'] for oNode in resp['nodes']: if not 'supportsHotWater' in oNode['attributes']: if 'model' in oNode['attributes']: strModel = oNode['attributes']['model']['reportedValue'] intSoftwareVersion = oNode['attributes'][ 'softwareVersion']['reportedValue'] if 'SLT' in strModel: oDeviceVersionDict['Thermostat'] = { "model": strModel, "version": intSoftwareVersion, 'mac_id': strSLTMacID } elif 'SLR' in strModel: oDeviceVersionDict['Boiler Module'] = { "model": strModel, "version": intSoftwareVersion, 'mac_id': strSLRMacID } return oDeviceVersionDict def getDuration(self, strDuration): arrDuration = strDuration.split(':') intHour = int(arrDuration[0]) intMin = int(arrDuration[1]) intSec = int(float(arrDuration[2])) if (intHour > 0): strDuration = str(intHour) + " hour(s), " + str( intMin) + " minute(s), " + str(intSec) + " seconds" if (intHour > 23): intDay = intHour // 24 intHour = intHour % 24 strDuration = str(intDay) + " day(s), " + str( intHour) + " hour(s), " + str( intMin) + " minute(s), " + str(intSec) + " seconds" else: strDuration = str(intMin) + " minute(s), " + str( intSec) + " seconds" return strDuration #Ensures the dirPath exists, if not creates the same def ensure_dir(self, dirpath): if not os.path.exists(dirpath): os.makedirs(dirpath) return dirpath
def is_locked(datadir): """ Return True if this datadir is locked for migrations """ lockfile = LockFile(datadir + '/.migration_lock') return lockfile.is_locked()
# newghaccount['ghsearch'][keyword] = github.githubApiSearchCode( t_tokens, login, keyword ) # employee['ghaccount'][login] = newghaccount # else: # sys.stdout.write( colored('%s\n' % url, 'white') ) # employee['tested'] = 1 ##### # # Display results # sys.stdout.write( colored('[+] %d profiles found.\n' % t_stats['n_ghaccount'], 'green')) functions.displayResults(t_results, t_keywords) ##### # # Save # if lock.is_locked(): lock.release() if f_progress: with open(f_progress, 'w') as json_file: json.dump(t_results, json_file) sys.stdout.write(colored('[+] datas saved: %s\n\n' % f_progress, 'green')) #####
from lockfile import LockFile, LockTimeout import CONSTANTS filename= CONSTANTS.FILE_DIR+"/password.txt" lock = LockFile(filename) #lock.break_lock() print(lock.i_am_locking()) # lock.break_lock() print(lock.is_locked()) # lock.release() lock.acquire() while lock.i_am_locking(): try: print(lock.is_locked()) # wait up to 60 seconds print(lock.i_am_locking()) fb = open(filename, 'r') da = fb.readlines() for d in da[:1000000]: print(d) fb.close() lock.release() except LockTimeout: print("enterd into exception") lock.break_lock() lock.acquire() print(lock.is_locked()) print(lock.i_am_locking()) print("I locked", lock.path) if lock.is_locked(): lock.release()
class ContentCache(object): def __init__(self, cache_dir=None): self.cache_dir = None self._db_path = None self._lockfile = None if not get_setting("enable_cache"): return self.cache_dir = cache_dir or join(util.get_home_dir(), ".cache") self._db_path = join(self.cache_dir, "db.data") def __enter__(self): if not self._db_path or not isfile(self._db_path): return self self.delete() return self def __exit__(self, type_, value, traceback): pass def _lock_dbindex(self): if not self.cache_dir: os.makedirs(self.cache_dir) self._lockfile = LockFile(self.cache_dir) if self._lockfile.is_locked() and \ (time() - getmtime(self._lockfile.lock_file)) > 10: self._lockfile.break_lock() try: self._lockfile.acquire() except LockFailed: return False return True def _unlock_dbindex(self): if self._lockfile: self._lockfile.release() return True def get_cache_path(self, key): assert len(key) > 3 return join(self.cache_dir, key[-2:], key) @staticmethod def key_from_args(*args): h = hashlib.md5() for data in args: h.update(str(data)) return h.hexdigest() def get(self, key): cache_path = self.get_cache_path(key) if not isfile(cache_path): return None with open(cache_path, "rb") as fp: data = fp.read() if data and data[0] in ("{", "["): return json.loads(data) return data def set(self, key, data, valid): cache_path = self.get_cache_path(key) if isfile(cache_path): self.delete(key) if not data: return if not isdir(self.cache_dir): os.makedirs(self.cache_dir) tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} assert valid.endswith(tuple(tdmap.keys())) expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) if not self._lock_dbindex(): return False if not isdir(dirname(cache_path)): os.makedirs(dirname(cache_path)) with open(cache_path, "wb") as fp: if isinstance(data, (dict, list)): json.dump(data, fp) else: fp.write(str(data)) with open(self._db_path, "a") as fp: fp.write("%s=%s\n" % (str(expire_time), cache_path)) return self._unlock_dbindex() def delete(self, keys=None): """ Keys=None, delete expired items """ if not keys: keys = [] if not isinstance(keys, list): keys = [keys] paths_for_delete = [self.get_cache_path(k) for k in keys] found = False newlines = [] with open(self._db_path) as fp: for line in fp.readlines(): if "=" not in line: continue line = line.strip() expire, path = line.split("=") if time() < int(expire) and isfile(path) and \ path not in paths_for_delete: newlines.append(line) continue found = True if isfile(path): try: remove(path) if not listdir(dirname(path)): util.rmtree_(dirname(path)) except OSError: pass if found and self._lock_dbindex(): with open(self._db_path, "w") as fp: fp.write("\n".join(newlines) + "\n") self._unlock_dbindex() return True def clean(self): if not self.cache_dir or not isdir(self.cache_dir): return util.rmtree_(self.cache_dir)
from lockfile import LockFile, LockTimeout import CONSTANTS filename= CONSTANTS.FILE_DIR+"/password.txt" lock = LockFile(filename) print(lock.is_locked())
class ContentCache(object): def __init__(self, cache_dir=None): self.cache_dir = None self._db_path = None self._lockfile = None if not get_setting("enable_cache"): return self.cache_dir = cache_dir or join(util.get_home_dir(), ".cache") self._db_path = join(self.cache_dir, "db.data") def __enter__(self): if not self._db_path or not isfile(self._db_path): return self self.delete() return self def __exit__(self, type_, value, traceback): pass def _lock_dbindex(self): if not self.cache_dir: os.makedirs(self.cache_dir) self._lockfile = LockFile(self.cache_dir) if self._lockfile.is_locked() and \ (time() - getmtime(self._lockfile.lock_file)) > 10: self._lockfile.break_lock() try: self._lockfile.acquire() except LockFailed: return False return True def _unlock_dbindex(self): if self._lockfile: self._lockfile.release() return True def get_cache_path(self, key): assert len(key) > 3 return join(self.cache_dir, key[-2:], key) @staticmethod def key_from_args(*args): h = hashlib.md5() for data in args: h.update(str(data)) return h.hexdigest() def get(self, key): cache_path = self.get_cache_path(key) if not isfile(cache_path): return None with open(cache_path, "rb") as fp: data = fp.read() if data and data[0] in ("{", "["): return json.loads(data) return data def set(self, key, data, valid): cache_path = self.get_cache_path(key) if isfile(cache_path): self.delete(key) if not data: return if not isdir(self.cache_dir): os.makedirs(self.cache_dir) tdmap = {"s": 1, "m": 60, "h": 3600, "d": 86400} assert valid.endswith(tuple(tdmap.keys())) expire_time = int(time() + tdmap[valid[-1]] * int(valid[:-1])) if not self._lock_dbindex(): return False if not isdir(dirname(cache_path)): os.makedirs(dirname(cache_path)) with open(cache_path, "wb") as fp: if isinstance(data, (dict, list)): json.dump(data, fp) else: fp.write(str(data)) with open(self._db_path, "a") as fp: fp.write("%s=%s\n" % (str(expire_time), cache_path)) return self._unlock_dbindex() def delete(self, keys=None): """ Keys=None, delete expired items """ if not keys: keys = [] if not isinstance(keys, list): keys = [keys] paths_for_delete = [self.get_cache_path(k) for k in keys] found = False newlines = [] with open(self._db_path) as fp: for line in fp.readlines(): if "=" not in line: continue line = line.strip() expire, path = line.split("=") if time() < int(expire) and isfile(path) and \ path not in paths_for_delete: newlines.append(line) continue found = True if isfile(path): try: remove(path) if not listdir(dirname(path)): util.rmtree_(dirname(path)) except OSError: pass if found and self._lock_dbindex(): with open(self._db_path, "w") as fp: fp.write("\n".join(newlines) + "\n") self._unlock_dbindex() return True def clean(self): if not self.cache_dir or not isdir(self.cache_dir): return util.rmtree_(self.cache_dir)
def on_post(self, req, resp, id): """ Vraci JSON: { "result": "ok" | "error", "error": String } """ try: user = req.context['user'] # Kontrola opravneni if (not user.is_logged_in()) or (not user.is_org()): req.context['result'] = 'Nedostatecna opravneni' resp.status = falcon.HTTP_400 return # Kontrola existence ulohy task = session.query(model.Task).get(id) if task is None: req.context['result'] = 'Neexistujici uloha' resp.status = falcon.HTTP_404 return # Zverejnene ulohy mohou deployovat pouze admini a garant vlny wave = session.query(model.Wave).get(task.wave) if (datetime.datetime.utcnow() > wave.time_published and not user.is_admin() and user.id != wave.garant): req.context['result'] = ('Po zverejneni ulohy muze deploy ' 'provest pouze administrator nebo ' 'garant vlny.') resp.status = falcon.HTTP_404 return # Kontrola existence gitovske vetve a adresare v databazi if (task.git_branch is None) or (task.git_path is None): req.context['result'] = ('Uloha nema zadanou gitovskou vetev ' 'nebo adresar') resp.status = falcon.HTTP_400 return # Kontrola zamku lock = util.lock.git_locked() if lock: req.context['result'] = ('GIT uzamcen zamkem ' + lock + '\nNekdo momentalne provadi akci s ' 'gitem, opakujte prosim akci za 20 ' 'sekund.') resp.status = falcon.HTTP_409 return # Stav na deploying je potreba nastavit v tomto vlakne task.deploy_status = 'deploying' session.commit() try: deployLock = LockFile(util.admin.taskDeploy.LOCKFILE) deployLock.acquire(60) # Timeout zamku je 1 minuta deployThread = threading.Thread( target=util.admin.taskDeploy.deploy, args=(task.id, deployLock, scoped_session(_session)), kwargs={} ) deployThread.start() finally: if deployLock.is_locked(): deployLock.release() req.context['result'] = {} resp.status = falcon.HTTP_200 except SQLAlchemyError: session.rollback() raise finally: session.close()
def mirror_main(): """Entry point.""" signal.signal(signal.SIGINT, lock_ctrl_c_handler) parser = argparse.ArgumentParser() parser.add_argument('-H', '--host', required=True) parser.add_argument('-P', '--port', type=int, default=22) parser.add_argument('-c', '--netrc-path', default=expanduser('~/.netrc')) parser.add_argument('-r', '--resume', action='store_true', help='Resume incomplete files (experimental)') parser.add_argument('-T', '--move-to', required=True) parser.add_argument('-L', '--label', default='Seeding') parser.add_argument('-d', '--debug', action='store_true') parser.add_argument('-v', '--verbose', action='store_true') parser.add_argument('-s', '--syslog', action='store_true') parser.add_argument('--no-preserve-permissions', action='store_false') parser.add_argument('--no-preserve-times', action='store_false') parser.add_argument('--max-retries', type=int, default=10) parser.add_argument('remote_dir', metavar='REMOTEDIR', nargs=1) parser.add_argument('local_dir', metavar='LOCALDIR', nargs=1) args = parser.parse_args() log = get_logger('xirvik', verbose=args.verbose, debug=args.debug, syslog=args.syslog) if args.debug: logs_to_follow = ( 'requests', ) for name in logs_to_follow: _log = logging.getLogger(name) formatter = logging.Formatter('%(asctime)s - %(name)s - ' '%(levelname)s - %(message)s') channel = logging.StreamHandler(sys.stderr) _log.setLevel(logging.DEBUG) channel.setLevel(logging.DEBUG) channel.setFormatter(formatter) _log.addHandler(channel) local_dir = realpath(args.local_dir[0]) user, _, password = netrc(args.netrc_path).authenticators(args.host) sftp_host = 'sftp://{user:s}@{host:s}'.format( user=user, host=args.host, ) lf_hash = hashlib.sha256(json.dumps( args._get_kwargs()).encode('utf-8')).hexdigest() lf_path = path_join(gettempdir(), 'xirvik-mirror-{}'.format(lf_hash)) log.debug('Acquiring lock at {}.lock'.format(lf_path)) _lock = LockFile(lf_path) if _lock.is_locked(): psax = [x for x in sp.check_output(['ps', 'ax']).decode('utf-8').split('\n') if sys.argv[0] in x] if len(psax) == 1: log.info('Breaking lock') _lock.break_lock() _lock.acquire() log.info('Lock acquired') log.debug('Local directory to sync to: {}'.format(local_dir)) log.debug('Read user and password from netrc file') log.debug('SFTP URI: {}'.format(sftp_host)) client = ruTorrentClient(args.host, user, password, max_retries=args.max_retries) assumed_path_prefix = '/torrents/{}'.format(user) look_for = '{}/{}/'.format(assumed_path_prefix, args.remote_dir[0]) move_to = '{}/{}'.format(assumed_path_prefix, args.move_to) names = {} log.debug('Full completed directory path name: {}'.format(look_for)) log.debug('Moving finished torrents to: {}'.format(move_to)) log.info('Getting current torrent information (ruTorrent)') try: torrents = client.list_torrents() except requests.exceptions.ConnectionError as e: # Assume no Internet connection at this point log.error('Failed to connect: {}'.format(e)) try: _lock.release() except NotLocked: pass cleanup_and_exit(1) for hash, v in torrents.items(): if not v[TORRENT_PATH_INDEX].startswith(look_for): continue bn = basename(v[TORRENT_PATH_INDEX]) names[bn] = (hash, v[TORRENT_PATH_INDEX],) log.info('Completed torrent "{}" found with hash {}'.format(bn, hash,)) sftp_client_args = dict( hostname=args.host, username=user, password=password, port=args.port, ) try: with SFTPClient(**sftp_client_args) as sftp_client: log.info('Verifying contents of {} with previous ' 'response'.format(look_for)) sftp_client.chdir(args.remote_dir[0]) for item in sftp_client.listdir_iter(read_aheads=10): if item.filename not in names: log.error('File or directory "{}" not found in previous ' 'response body'.format(item.filename)) continue log.debug('Found matching torrent "{}" from ls output'.format( item.filename)) if not len(names.items()): log.info('Nothing found to mirror') _lock.release() cleanup_and_exit() mirror(sftp_client, client, destroot=local_dir, keep_modes=not args.no_preserve_permissions, keep_times=not args.no_preserve_times) except Exception as e: if args.debug: _lock.release() cleanup() raise e else: log.error(str(e)) _lock.release() cleanup_and_exit() _all = names.items() exit_status = 0 bad = [] for bn, (hash, fullpath) in _all: # There is a warning that can get raised here by urllib3 if # Content-Disposition header's filename field has any # non-ASCII characters. It is ignorable as the content still gets # downloaded correctly log.info('Verifying "{}"'.format(bn)) r, _ = client.get_torrent(hash) try: verify_torrent_contents(r.content, local_dir) except VerificationError as e: log.error('Could not verify "{}" contents against piece hashes ' 'in torrent file'.format(bn)) exit_status = 1 bad.append(hash) # Move to _seeding directory and set label # Unfortunately, there is no method, via the API, to do this one HTTP # request for bn, (hash, fullpath) in _all: if hash in bad: continue log.info('Moving "{}" to "{}" directory'.format(bn, move_to)) try: client.move_torrent(hash, move_to) except UnexpectedruTorrentError as e: log.error(str(e)) log.info('Setting label to "{}" for downloaded items'.format(args.label)) client.set_label_to_hashes(hashes=[hash for bn, (hash, fullpath) in names.items() if hash not in bad], label=args.label) if exit_status != 0: log.error('Could not verify torrent checksums') _lock.release() cleanup_and_exit(exit_status)
def update_document(destinaton, file, email, db: Session): email_id = get_email_id(email, db) file_id = get_file_id_by_filename(file.filename, db) print(email) #to include version check based on modified time and idownoaded time version_status = version_check(email_id, file_id, db) print("version check is ", version_status) if version_status == False: return None #this part creates a lock object with the given file path file_lock_check = LockFile(destinaton + '/' + file.filename) #checks if the file is locked or not if file_lock_check.is_locked(): if is_owner(email_id, file_id, db) == True: #if the user is owner the lock is released lock.release() else: #returns None if locked and the user is not owner print("file is locked for ", email) return "locked" #locks the file object file_lock_check.acquire() while file_lock_check.i_am_locking(): #lock has been implemented on a record. #No one can do modification through database, until current transaction is completed. #in the file_status, the lock has been implemented on a given file record #then the updated file has been uploaded #and then the modified time is updated and record will be releasaed. #this is row-level locking. #Even though the operations can be done on physical file level #through the database no one can do modifications on it until the lock is released. try: file_status = db.query(Files).filter( Files.id == file_id).with_for_update().first() # #time.sleep(20) #.update({'modified_on': dt.datetime.now()}) file_path = destinaton + "/" + file.filename with open(file_path, "wb+") as file_object: shutil.copyfileobj(file.file, file_object) file_status.modified_on = dt.datetime.now() db.commit() #file_update = update_file_status(file_id, db) update_status = db.query(Collaborators).filter( and_(Collaborators.collaborator_id == email_id.id, Collaborators.file_id == file_id)) update_status = update_status.update( {'downloaded_on': dt.datetime.now()}) db.commit() #releases the lock if anything goes wrong file_lock_check.release() except LockTimeout: print("enterd into exception") #brute method to break the lock file_lock_check.break_lock() file_lock_check.acquire() return False if file_lock_check.is_locked(): file_lock_check.release() return True